Expose cancel state of FileDialog and DirectoryDialog

The FileDialog and DirectoryDialog implementations do currently not
provide any way to identify that the dialog was canceled. The dialogs
return a null string in case no file or folder was selected, but is not
possible to identify whether this was intended (via a cancel action) or
if there was some unexpected error while the dialog was open (such as
too long file paths on Windows).

This change introduces the `askSelectFile()`/`askSelectDirectory()`
methods as an alternative to the existing `open()` method. This new
method returns a FileDialogResult object encapsulating the selected file
or folder and the state of the dialog, such as a user cancellation. The
dialogs evaluate the operating system's response code for the system's
dialog control to identify the dialogs result state.

bundles/org.eclipse.swt/Eclipse SWT PI/cocoa/org/eclipse/swt/internal/cocoa/OS.java

@@ -179,7 +179,7 @@ public static void beginSheetModalForWindow(NSObject id, NSWindow window, long h
 	 * Custom message that will be sent when setTheme is called for example from Platform UI code.
 	 */
 	public static final long sel_appAppearanceChanged = OS.sel_registerName("appAppearanceChanged");
-	
+
 	/**
 	 * Experimental API for dark theme.
 	 * <p>
@@ -194,7 +194,7 @@ public static void beginSheetModalForWindow(NSObject id, NSWindow window, long h
 	 * On GTK, behavior may be different as the boolean flag doesn't force dark
 	 * theme instead it specify that dark theme is preferred.
 	 * </p>
-	 * 
+	 *
 	 * @param isDarkTheme <code>true</code> for dark theme
 	 */
 	public static void setTheme(boolean isDarkTheme) {
@@ -2216,6 +2216,7 @@ public static Selector getSelector (long value) {
 public static final int NSEventTypeMagnify = 30;
 public static final int NSEventTypeRotate = 18;
 public static final int NSEventTypeSwipe = 31;
+public static final int NSFileHandlingPanelCancelButton = 0;
 public static final int NSFileHandlingPanelOKButton = 1;
 public static final int NSFlagsChanged = 12;
 public static final int NSFocusRingTypeNone = 1;

bundles/org.eclipse.swt/Eclipse SWT PI/win32/org/eclipse/swt/internal/win32/OS.java

@@ -452,6 +452,7 @@ public class OS extends C {
 	public static final int EP_EDITTEXT = 1;
 	public static final int ERROR_FILE_NOT_FOUND = 0x2;
 	public static final int ERROR_NO_MORE_ITEMS = 0x103;
+	public static final int ERROR_CANCELED = 0x4C7;
 	public static final int ESB_DISABLE_BOTH = 0x3;
 	public static final int ESB_ENABLE_BOTH = 0x0;
 	public static final int ES_AUTOHSCROLL = 0x80;

bundles/org.eclipse.swt/Eclipse SWT/common/org/eclipse/swt/widgets/FileDialogResult.java

@@ -0,0 +1,65 @@
+package org.eclipse.swt.widgets;
+
+import java.util.*;
+
+/**
+ * A representation of the result of file dialogs such as the {@link FileDialog}
+ * or {@link DirectoryDialog}.
+ *
+ * @since 3.125
+ */
+public final class FileDialogResult {
+	public static enum State {
+		/**
+		 * The user has successfully selected a file or folder
+		 */
+		SELECTED,
+		/**
+		 * The user has canceled the dialog
+		 */
+		CANCELED,
+		/**
+		 * An unspecified error occurred while the dialog was open
+		 */
+		UNKNOWN_ERROR
+	}
+
+	private String path;
+
+	private State state;
+
+	private FileDialogResult(String path, State state) {
+		this.path = path;
+		this.state = state;
+	}
+
+	/**
+	 * {@return an optional that contains the string describing the absolute path of
+	 * the selected file or folder or is empty if the dialog was canceled or if
+	 * there was an error}
+	 */
+	public Optional<String> getPath() {
+		return Optional.ofNullable(path);
+	}
+
+	/**
+	 * {@return the result state of the dialog, indicating a successful selection,
+	 * user cancellation and other errors}
+	 */
+	public State getState() {
+		return state;
+	}
+
+	static FileDialogResult createForSuccessfulSelection(String path) {
+		return new FileDialogResult(path, State.SELECTED);
+	}
+
+	static FileDialogResult createForUserCancellation() {
+		return new FileDialogResult(null, State.CANCELED);
+	}
+
+	static FileDialogResult createForUnkownError() {
+		return new FileDialogResult(null, State.UNKNOWN_ERROR);
+	}
+
+}

bundles/org.eclipse.swt/Eclipse SWT/win32/org/eclipse/swt/widgets/DirectoryDialog.java

@@ -123,9 +123,28 @@ public String getMessage () {
  * </ul>
  */
 public String open() {
+	return askSelectDirectory().getPath().orElse(null);
+}
+
+/**
+ * Makes the dialog visible and brings it to the front of the display.
+ * Equal to {@link DirectoryDialog#open()} but also exposes for state information like user cancellation.
+ *
+ * @return a result object with the absolute path of the selected directory,
+ *         or a state describing that the dialog was cancelled or an error occurred
+ *
+ * @exception SWTException <ul>
+ *    <li>ERROR_WIDGET_DISPOSED - if the dialog has been disposed</li>
+ *    <li>ERROR_THREAD_INVALID_ACCESS - if not called from the thread that created the dialog</li>
+ * </ul>
+ *
+ * @since 3.125
+ */
+public FileDialogResult askSelectDirectory() {
 	this.directoryPath = null;
 
 	long [] ppv = new long [1];
+	int hr = COM.E_FAIL;
 	if (COM.CoCreateInstance(COM.CLSID_FileOpenDialog, 0, COM.CLSCTX_INPROC_SERVER, COM.IID_IFileOpenDialog, ppv) == OS.S_OK) {
 		IFileDialog fileDialog = new IFileDialog(ppv[0]);
 
@@ -162,7 +181,8 @@ public String open() {
 		Display display = parent.getDisplay();
 		long hwndOwner = parent.handle;
 		display.externalEventLoop = true;
-		if (fileDialog.Show(hwndOwner) == OS.S_OK) {
+		hr = fileDialog.Show(hwndOwner);
+		if (hr == OS.S_OK) {
 			if (fileDialog.GetResult(ppv) == OS.S_OK) {
 				IShellItem psi = new IShellItem(ppv[0]);
 				if (psi.GetDisplayName(OS.SIGDN_FILESYSPATH, ppv) == OS.S_OK) {
@@ -182,7 +202,13 @@ public String open() {
 		fileDialog.Release();
 	}
 
-	return directoryPath;
+	if (hr == COM.S_OK) {
+		return FileDialogResult.createForSuccessfulSelection(directoryPath);
+	} else if (hr == OS.HRESULT_FROM_WIN32(OS.ERROR_CANCELED)) {
+		return FileDialogResult.createForUserCancellation();
+	} else {
+		return FileDialogResult.createForUnkownError();
+	}
 }
 
 /**

bundles/org.eclipse.swt/Eclipse SWT/win32/org/eclipse/swt/widgets/FileDialog.java

@@ -216,6 +216,24 @@ static Path getItemPath (IShellItem psi) {
  * </ul>
  */
 public String open () {
+	return askSelectFile().getPath().orElse(null);
+}
+
+/**
+ * Makes the dialog visible and brings it to the front of the display.
+ * Equal to {@link FileDialog#open()} but also exposes for state information like user cancellation.
+ *
+ * @return a result object with the absolute path of the first selected file,
+ *         or a state describing that the dialog was cancelled or an error occurred
+ *
+ * @exception SWTException <ul>
+ *    <li>ERROR_WIDGET_DISPOSED - if the dialog has been disposed</li>
+ *    <li>ERROR_THREAD_INVALID_ACCESS - if not called from the thread that created the dialog</li>
+ * </ul>
+ *
+ * @since 3.125
+ */
+public FileDialogResult askSelectFile() {
 	/* Create Common Item Dialog */
 	long[] ppv = new long[1];
 	int hr;
@@ -385,8 +403,13 @@ public String open () {
 
 	fileDialog.Release();
 
-	/* Answer the full path or null */
-	return fullPath;
+	if (hr == COM.S_OK) {
+		return FileDialogResult.createForSuccessfulSelection(fullPath);
+	} else if (hr == OS.HRESULT_FROM_WIN32(OS.ERROR_CANCELED)) {
+		return FileDialogResult.createForUserCancellation();
+	} else {
+		return FileDialogResult.createForUnkownError();
+	}
 }
 
 /**