Expose cancel state of FileDialog and DirectoryDialog

The FileDialog and DirectoryDialog implementations do currently not
provide any way to identify that the dialog was canceled. The dialogs
return a null string in case no file or folder was selected, but is not
possible to identify whether this was intended (via a cancel action) or
if there was some unexpected error while the dialog was open (such as
too long file paths on Windows).

This change introduces the `openDialog()` method as an alternative to
the existing `open()` method. This new method returns an optional
String, which is only present if a file or directory was successfully
selected. An empty Optional indicates a cancellation by the user and an
actual error is realized as an SWTException. The dialogs evaluate the
operating system's response code for the system's dialog control to
identify the dialogs result state.

bundles/org.eclipse.swt/Eclipse SWT PI/cocoa/org/eclipse/swt/internal/cocoa/OS.java

@@ -179,7 +179,7 @@ public static void beginSheetModalForWindow(NSObject id, NSWindow window, long h
 	 * Custom message that will be sent when setTheme is called for example from Platform UI code.
 	 */
 	public static final long sel_appAppearanceChanged = OS.sel_registerName("appAppearanceChanged");
-	
+
 	/**
 	 * Experimental API for dark theme.
 	 * <p>
@@ -194,7 +194,7 @@ public static void beginSheetModalForWindow(NSObject id, NSWindow window, long h
 	 * On GTK, behavior may be different as the boolean flag doesn't force dark
 	 * theme instead it specify that dark theme is preferred.
 	 * </p>
-	 * 
+	 *
 	 * @param isDarkTheme <code>true</code> for dark theme
 	 */
 	public static void setTheme(boolean isDarkTheme) {
@@ -2216,6 +2216,7 @@ public static Selector getSelector (long value) {
 public static final int NSEventTypeMagnify = 30;
 public static final int NSEventTypeRotate = 18;
 public static final int NSEventTypeSwipe = 31;
+public static final int NSFileHandlingPanelCancelButton = 0;
 public static final int NSFileHandlingPanelOKButton = 1;
 public static final int NSFlagsChanged = 12;
 public static final int NSFocusRingTypeNone = 1;

bundles/org.eclipse.swt/Eclipse SWT PI/win32/org/eclipse/swt/internal/win32/OS.java

@@ -452,6 +452,7 @@ public class OS extends C {
 	public static final int EP_EDITTEXT = 1;
 	public static final int ERROR_FILE_NOT_FOUND = 0x2;
 	public static final int ERROR_NO_MORE_ITEMS = 0x103;
+	public static final int ERROR_CANCELED = 0x4C7;
 	public static final int ESB_DISABLE_BOTH = 0x3;
 	public static final int ESB_ENABLE_BOTH = 0x0;
 	public static final int ES_AUTOHSCROLL = 0x80;

bundles/org.eclipse.swt/Eclipse SWT/cocoa/org/eclipse/swt/widgets/DirectoryDialog.java

@@ -14,6 +14,8 @@
 package org.eclipse.swt.widgets;
 
 
+import java.util.*;
+
 import org.eclipse.swt.*;
 import org.eclipse.swt.internal.*;
 import org.eclipse.swt.internal.cocoa.*;
@@ -139,11 +141,16 @@ void handleResponse (long response) {
 	}
 	Display display = parent != null ? parent.getDisplay() : Display.getCurrent();
 	display.setModalDialog(null);
+	directoryPath = null;
 	if (response  == OS.NSFileHandlingPanelOKButton) {
 		NSString filename = panel.filename();
 		directoryPath = filterPath = filename.getString();
 	}
 	releaseHandles();
+
+	if (response != OS.NSFileHandlingPanelOKButton && response != OS.NSFileHandlingPanelCancelButton) {
+		throw new SWTException(SWT.ERROR_INVALID_RETURN_VALUE);
+	}
 }
 
 /**
@@ -159,6 +166,32 @@ void handleResponse (long response) {
  * </ul>
  */
 public String open () {
+	try {
+		return openDialog().orElse(null);
+	} catch (SWTException e) {
+		if (e.code == SWT.ERROR_INVALID_RETURN_VALUE) {
+			return null;
+		}
+		throw e;
+	}
+}
+
+/**
+ * Makes the dialog visible and brings it to the front of the display.
+ * Equal to {@link DirectoryDialog#open()} but also exposes for state information like user cancellation.
+ *
+ * @return an Optional that either contains the absolute path of the selected directory
+ *         or is empty in case the dialog was canceled
+ *
+ * @exception SWTException <ul>
+ *    <li>ERROR_WIDGET_DISPOSED - if the dialog has been disposed</li>
+ *    <li>ERROR_THREAD_INVALID_ACCESS - if not called from the thread that created the dialog</li>
+ *    <li>ERROR_INVALID_RETURN_VALUE - if the dialog was not cancelled and did not return a valid path</li>
+ * </ul>
+ *
+ * @since 3.125
+ */
+public Optional<String> openDialog () {
 	directoryPath = null;
 	panel = NSOpenPanel.openPanel();
 	if (panel == null) {
@@ -202,7 +235,7 @@ public String open () {
 	}
 
 //	options.optionFlags = OS.kNavSupportPackages | OS.kNavAllowOpenPackages | OS.kNavAllowInvisibleFiles;
-	return directoryPath;
+	return Optional.ofNullable(directoryPath);
 }
 
 void releaseHandles () {

bundles/org.eclipse.swt/Eclipse SWT/cocoa/org/eclipse/swt/widgets/FileDialog.java

@@ -294,6 +294,7 @@ void handleResponse (long response) {
 
 	filterIndex = popup != null ? (int)popup.indexOfSelectedItem() : -1;
 
+	fullPath = null;
 	if (response == OS.NSFileHandlingPanelOKButton) {
 		NSString filename = panel.filename();
 		if ((style & SWT.SAVE) != 0) {
@@ -334,6 +335,10 @@ void handleResponse (long response) {
 		}
 	}
 	releaseHandles();
+
+	if (response != OS.NSFileHandlingPanelOKButton && response != OS.NSFileHandlingPanelCancelButton) {
+		throw new SWTException(SWT.ERROR_INVALID_RETURN_VALUE);
+	}
 }
 
 /**
@@ -349,6 +354,32 @@ void handleResponse (long response) {
  * </ul>
  */
 public String open () {
+	try {
+		return openDialog().orElse(null);
+	} catch (SWTException e) {
+		if (e.code == SWT.ERROR_INVALID_RETURN_VALUE) {
+			return null;
+		}
+		throw e;
+	}
+}
+
+/**
+ * Makes the dialog visible and brings it to the front of the display.
+ * Equal to {@link FileDialog#open()} but also exposes for state information like user cancellation.
+ *
+ * @return an Optional that either contains the absolute path of the first selected file
+ *         or is empty in case the dialog was canceled
+ *
+ * @exception SWTException <ul>
+ *    <li>ERROR_WIDGET_DISPOSED - if the dialog has been disposed</li>
+ *    <li>ERROR_THREAD_INVALID_ACCESS - if not called from the thread that created the dialog</li>
+ *    <li>ERROR_INVALID_RETURN_VALUE - if the dialog was not cancelled and did not return a valid path</li>
+ * </ul>
+ *
+ * @since 3.125
+ */
+public Optional<String> openDialog () {
 	fullPath = null;
 	if ((style & SWT.SAVE) != 0) {
 		NSSavePanel savePanel = NSSavePanel.savePanel();
@@ -443,7 +474,7 @@ public String open () {
 		long response = panel.runModal();
 		handleResponse(response);
 	}
-	return fullPath;
+	return Optional.ofNullable(fullPath);
 }
 
 long panel_shouldEnableURL (long id, long sel, long arg0, long arg1) {

bundles/org.eclipse.swt/Eclipse SWT/gtk/org/eclipse/swt/widgets/DirectoryDialog.java

@@ -15,6 +15,7 @@
 
 
 import java.io.*;
+import java.util.Optional;
 
 import org.eclipse.swt.*;
 import org.eclipse.swt.internal.*;
@@ -122,17 +123,45 @@ public String getMessage () {
  * </ul>
  */
 public String open () {
-		return openNativeChooserDialog();
+	try {
+		return openDialog().orElse(null);
+	} catch (SWTException e) {
+		if (e.code == SWT.ERROR_INVALID_RETURN_VALUE) {
+			return null;
+		}
+		throw e;
+	}
 }
+
+/**
+ * Makes the dialog visible and brings it to the front of the display.
+ * Equal to {@link DirectoryDialog#open()} but also exposes for state information like user cancellation.
+ *
+ * @return an Optional that either contains the absolute path of the selected directory
+ *         or is empty in case the dialog was canceled
+ *
+ * @exception SWTException <ul>
+ *    <li>ERROR_WIDGET_DISPOSED - if the dialog has been disposed</li>
+ *    <li>ERROR_THREAD_INVALID_ACCESS - if not called from the thread that created the dialog</li>
+ *    <li>ERROR_INVALID_RETURN_VALUE - if the dialog was not cancelled and did not return a valid path</li>
+ * </ul>
+ *
+ * @since 3.125
+ */
+public Optional<String> openDialog () {
+	return openNativeChooserDialog();
+}
+
 /**
  * Open the file chooser dialog using the GtkFileChooserNative API (GTK3.20+) for running applications
  * without direct filesystem access (such as Flatpak). API for GtkFileChoosernative does not
  * give access to any GtkWindow or GtkWidget for the dialog, thus this method omits calls that
  * requires such access. These are be handled by the GtkNativeDialog API.
  *
- * @return a string describing the absolute path of the first selected file, or null
+ * @return an Optional that contains a string describing the absolute path of the selected directory
+ *         or is empty if the dialog is canceled
  */
-String openNativeChooserDialog () {
+Optional<String> openNativeChooserDialog () {
 	byte [] titleBytes = Converter.wcsToMbcs (title, true);
 	long shellHandle = parent.topHandle ();
 	Display display = parent != null ? parent.getDisplay (): Display.getCurrent ();
@@ -168,7 +197,7 @@ String openNativeChooserDialog () {
 
 	GTK3setNativeDialogMessage(handle, message);
 
-	String answer = null;
+	String selectedPath = null;
 	display.addIdleProc ();
 	int signalId = 0;
 	long hookId = 0;
@@ -213,14 +242,22 @@ String openNativeChooserDialog () {
 					char [] chars = new char [clength];
 					C.memmove (chars, utf16Ptr, clength * 2);
 					OS.g_free (utf16Ptr);
-					answer = new String (chars);
-					filterPath = answer;
+					selectedPath = new String (chars);
+					filterPath = selectedPath;
 				}
 			}
 		}
 	}
+
+	Optional<String> result = Optional.empty();
+	if (response == GTK.GTK_RESPONSE_ACCEPT) {
+		result = Optional.ofNullable(selectedPath);
+	}
 	display.removeIdleProc ();
-	return answer;
+	if (result.isPresent() || response == GTK.GTK_RESPONSE_CANCEL) {
+		return result;
+	}
+	throw new SWTException(SWT.ERROR_INVALID_RETURN_VALUE);
 }
 
 

bundles/org.eclipse.swt/Eclipse SWT/gtk/org/eclipse/swt/widgets/FileDialog.java

@@ -15,6 +15,7 @@
 
 
 import java.io.*;
+import java.util.Optional;
 
 import org.eclipse.swt.*;
 import org.eclipse.swt.internal.*;
@@ -332,17 +333,45 @@ public boolean getOverwrite () {
  * </ul>
  */
 public String open () {
+	try {
+		return openDialog().orElse(null);
+	} catch (SWTException e) {
+		if (e.code == SWT.ERROR_INVALID_RETURN_VALUE) {
+			return null;
+		}
+		throw e;
+	}
+}
+
+/**
+ * Makes the dialog visible and brings it to the front of the display.
+ * Equal to {@link FileDialog#open()} but also exposes for state information like user cancellation.
+ *
+ * @return an Optional that either contains the absolute path of the first selected file
+ *         or is empty in case the dialog was canceled
+ *
+ * @exception SWTException <ul>
+ *    <li>ERROR_WIDGET_DISPOSED - if the dialog has been disposed</li>
+ *    <li>ERROR_THREAD_INVALID_ACCESS - if not called from the thread that created the dialog</li>
+ *    <li>ERROR_INVALID_RETURN_VALUE - if the dialog was not cancelled and did not return a valid path</li>
+ * </ul>
+ *
+ * @since 3.125
+ */
+public Optional<String> openDialog () {
 	return openNativeChooserDialog();
 }
+
 /**
  * Open the file chooser dialog using the GtkFileChooserNative API (GTK3.20+) for running applications
  * without direct filesystem access (such as Flatpak). API for GtkFileChoosernative does not
  * give access to any GtkWindow or GtkWidget for the dialog, thus this method omits calls that
  * requires such access. These are be handled by the GtkNativeDialog API.
  *
- * @return a string describing the absolute path of the first selected file, or null
+ * @return an Optional that contains a string describing the absolute path of the first selected file
+ *         or is empty if the dialog is canceled 
  */
-String openNativeChooserDialog () {
+Optional<String> openNativeChooserDialog () {
 	byte [] titleBytes = Converter.wcsToMbcs (title, true);
 	int action = (style & SWT.SAVE) != 0 ? GTK.GTK_FILE_CHOOSER_ACTION_SAVE : GTK.GTK_FILE_CHOOSER_ACTION_OPEN;
 	long shellHandle = parent.topHandle();
@@ -356,7 +385,6 @@ String openNativeChooserDialog () {
 	}
 	presetChooserDialog ();
 	display.addIdleProc ();
-	String answer = null;
 	int signalId = 0;
 	long hookId = 0;
 	if ((style & SWT.RIGHT_TO_LEFT) != 0) {
@@ -378,11 +406,16 @@ String openNativeChooserDialog () {
 	if ((style & SWT.RIGHT_TO_LEFT) != 0) {
 		OS.g_signal_remove_emission_hook (signalId, hookId);
 	}
+
+	Optional<String> result = Optional.empty();
 	if (response == GTK.GTK_RESPONSE_ACCEPT) {
-		answer = computeResultChooserDialog ();
+		result = Optional.ofNullable(computeResultChooserDialog ());
 	}
 	display.removeIdleProc ();
-	return answer;
+	if (result.isPresent() || response == GTK.GTK_RESPONSE_CANCEL) {
+		return result;
+	}
+	throw new SWTException(SWT.ERROR_INVALID_RETURN_VALUE);
 }
 
 void presetChooserDialog () {