diff --git a/org.eclipse.jdt.junit/src/org/eclipse/jdt/internal/junit/ui/JUnitMessages.java b/org.eclipse.jdt.junit/src/org/eclipse/jdt/internal/junit/ui/JUnitMessages.java
index 87212316228..f3b13b54b20 100644
--- a/org.eclipse.jdt.junit/src/org/eclipse/jdt/internal/junit/ui/JUnitMessages.java
+++ b/org.eclipse.jdt.junit/src/org/eclipse/jdt/internal/junit/ui/JUnitMessages.java
@@ -118,6 +118,8 @@ public final class JUnitMessages extends NLS {
 
 	public static String JUnitLaunchConfigurationTab_error_notJavaProject;
 
+	public static String JUnitLaunchConfigurationTab_error_operation_canceled;
+
 	public static String JUnitLaunchConfigurationTab_error_projectnotdefined;
 
 	public static String JUnitLaunchConfigurationTab_error_projectnotexists;
@@ -372,4 +374,6 @@ private JUnitMessages() {
 	public static String TestRunnerViewPart_JUnitPasteAction_label;
 
 	public static String TestRunnerViewPart_layout_menu;
+
+	public static String TestSearchEngine_search_message_progress_monitor;
 }
diff --git a/org.eclipse.jdt.junit/src/org/eclipse/jdt/internal/junit/ui/JUnitMessages.properties b/org.eclipse.jdt.junit/src/org/eclipse/jdt/internal/junit/ui/JUnitMessages.properties
index f3d34405358..aa63f8ba3e4 100644
--- a/org.eclipse.jdt.junit/src/org/eclipse/jdt/internal/junit/ui/JUnitMessages.properties
+++ b/org.eclipse.jdt.junit/src/org/eclipse/jdt/internal/junit/ui/JUnitMessages.properties
@@ -169,6 +169,7 @@ JUnitLaunchConfigurationTab_folderdialog_message=Choose a Project, Source Folder
 JUnitLaunchConfigurationTab_error_projectnotdefined=Project not specified
 JUnitLaunchConfigurationTab_error_projectnotexists=Project does not exist
 JUnitLaunchConfigurationTab_error_notJavaProject=Specified project is not a Java project
+JUnitLaunchConfigurationTab_error_operation_canceled=(Operation canceled by the user)
 JUnitLaunchConfigurationTab_error_testnotdefined=Test not specified
 JUnitLaunchConfigurationTab_error_testcasenotonpath=Cannot find class 'junit.framework.TestCase' on project build path.
 JUnitLaunchConfigurationTab_addtag_label=Con&figure...
@@ -276,3 +277,5 @@ JUnitViewEditorLauncher_dialog_title=Import Test Run
 JUnitViewEditorLauncher_error_occurred=An error occurred while opening a test run file.
 ClasspathVariableMarkerResolutionGenerator_use_JUnit3=Use the JUnit 3 library
 ClasspathVariableMarkerResolutionGenerator_use_JUnit3_desc=Changes the classpath variable entry to use the JUnit 3 library
+
+TestSearchEngine_search_message_progress_monitor=Searching for test methods in ''{0}''
diff --git a/org.eclipse.jdt.junit/src/org/eclipse/jdt/internal/junit/util/TestSearchEngine.java b/org.eclipse.jdt.junit/src/org/eclipse/jdt/internal/junit/util/TestSearchEngine.java
index c41c85adcfc..60ec8300d4a 100644
--- a/org.eclipse.jdt.junit/src/org/eclipse/jdt/internal/junit/util/TestSearchEngine.java
+++ b/org.eclipse.jdt.junit/src/org/eclipse/jdt/internal/junit/util/TestSearchEngine.java
@@ -18,14 +18,27 @@
 import java.util.Set;
 
 import org.eclipse.core.runtime.CoreException;
+import org.eclipse.core.runtime.IProgressMonitor;
+import org.eclipse.core.runtime.SubMonitor;
 
 import org.eclipse.jface.operation.IRunnableContext;
 import org.eclipse.jface.operation.IRunnableWithProgress;
 
+import org.eclipse.jdt.core.IAnnotation;
 import org.eclipse.jdt.core.IJavaElement;
+import org.eclipse.jdt.core.IJavaProject;
+import org.eclipse.jdt.core.IMethod;
 import org.eclipse.jdt.core.IType;
+import org.eclipse.jdt.core.JavaModelException;
+import org.eclipse.jdt.core.Signature;
+import org.eclipse.jdt.core.dom.Modifier;
 
+import org.eclipse.jdt.internal.junit.JUnitCorePlugin;
+import org.eclipse.jdt.internal.junit.Messages;
 import org.eclipse.jdt.internal.junit.launcher.ITestKind;
+import org.eclipse.jdt.internal.junit.launcher.TestKind;
+import org.eclipse.jdt.internal.junit.launcher.TestKindRegistry;
+import org.eclipse.jdt.internal.junit.ui.JUnitMessages;
 
 
 /**
@@ -48,4 +61,124 @@ public static Set<IType> findTests(IRunnableContext context, final IJavaElement
 		return result;
 	}
 
+	public static Set<String> findTestMethods(IRunnableContext context, final IJavaProject javaProject, IType type, TestKind testKind) throws InvocationTargetException, InterruptedException {
+		final Set<String> result= new HashSet<>();
+
+		IRunnableWithProgress runnable= progressMonitor -> {
+			try {
+				String message= Messages.format(JUnitMessages.TestSearchEngine_search_message_progress_monitor, type.getElementName());
+				SubMonitor subMonitor= SubMonitor.convert(progressMonitor, message, 1);
+
+				collectMethodNames(type, javaProject, testKind.getId(), result, subMonitor.split(1));
+			} catch (CoreException e) {
+				throw new InvocationTargetException(e);
+			}
+		};
+		context.run(true, true, runnable);
+		return result;
+	}
+
+	private static void collectMethodNames(IType type, IJavaProject javaProject, String testKindId, Set<String> methodNames, IProgressMonitor monitor) throws JavaModelException {
+		if (type == null) {
+			return;
+		}
+
+		SubMonitor subMonitor= SubMonitor.convert(monitor, 3);
+
+		collectDeclaredMethodNames(type, javaProject, testKindId, methodNames);
+		subMonitor.split(1);
+
+		String superclassName= type.getSuperclassName();
+		IType superType= getResolvedType(superclassName, type, javaProject);
+		collectMethodNames(superType, javaProject, testKindId, methodNames, subMonitor.split(1));
+
+		String[] superInterfaceNames= type.getSuperInterfaceNames();
+		subMonitor.setWorkRemaining(superInterfaceNames.length);
+		for (String interfaceName : superInterfaceNames) {
+			superType= getResolvedType(interfaceName, type, javaProject);
+			collectMethodNames(superType, javaProject, testKindId, methodNames, subMonitor.split(1));
+		}
+	}
+
+	private static IType getResolvedType(String typeName, IType type, IJavaProject javaProject) throws JavaModelException {
+		IType resolvedType= null;
+		if (typeName != null) {
+			int pos= typeName.indexOf('<');
+			if (pos != -1) {
+				typeName= typeName.substring(0, pos);
+			}
+			String[][] resolvedTypeNames= type.resolveType(typeName);
+			if (resolvedTypeNames != null && resolvedTypeNames.length > 0) {
+				String[] resolvedTypeName= resolvedTypeNames[0];
+				resolvedType= javaProject.findType(resolvedTypeName[0], resolvedTypeName[1]); // secondary types not found by this API
+			}
+		}
+		return resolvedType;
+	}
+
+	private static void collectDeclaredMethodNames(IType type, IJavaProject javaProject, String testKindId, Set<String> methodNames) throws JavaModelException {
+		IMethod[] methods= type.getMethods();
+		for (IMethod method : methods) {
+			String methodName= method.getElementName();
+			int flags= method.getFlags();
+			// Only include public, non-static, no-arg methods that return void and start with "test":
+			if (Modifier.isPublic(flags) && !Modifier.isStatic(flags) &&
+					method.getNumberOfParameters() == 0 && Signature.SIG_VOID.equals(method.getReturnType()) &&
+					methodName.startsWith("test")) { //$NON-NLS-1$
+				methodNames.add(methodName);
+			}
+			boolean isJUnit3= TestKindRegistry.JUNIT3_TEST_KIND_ID.equals(testKindId);
+			boolean isJUnit5= TestKindRegistry.JUNIT5_TEST_KIND_ID.equals(testKindId);
+			if (!isJUnit3 && !Modifier.isPrivate(flags) && !Modifier.isStatic(flags)) {
+				IAnnotation annotation= method.getAnnotation("Test"); //$NON-NLS-1$
+				if (annotation.exists()) {
+					methodNames.add(methodName + JUnitStubUtility.getParameterTypes(method, false));
+				} else if (isJUnit5) {
+					boolean hasAnyTestAnnotation= method.getAnnotation("TestFactory").exists() //$NON-NLS-1$
+							|| method.getAnnotation("Testable").exists() //$NON-NLS-1$
+							|| method.getAnnotation("TestTemplate").exists() //$NON-NLS-1$
+							|| method.getAnnotation("ParameterizedTest").exists() //$NON-NLS-1$
+							|| method.getAnnotation("RepeatedTest").exists(); //$NON-NLS-1$
+					if (hasAnyTestAnnotation || isAnnotatedWithTestable(method, type, javaProject)) {
+						methodNames.add(methodName + JUnitStubUtility.getParameterTypes(method, false));
+					}
+				}
+			}
+		}
+	}
+
+	// See JUnit5TestFinder.Annotation#annotates also.
+	private static boolean isAnnotatedWithTestable(IMethod method, IType declaringType, IJavaProject javaProject) throws JavaModelException {
+		for (IAnnotation annotation : method.getAnnotations()) {
+			IType annotationType= getResolvedType(annotation.getElementName(), declaringType, javaProject);
+			if (annotationType != null) {
+				if (matchesTestable(annotationType)) {
+					return true;
+				}
+				Set<IType> hierarchy= new HashSet<>();
+				if (matchesTestableInAnnotationHierarchy(annotationType, javaProject, hierarchy)) {
+					return true;
+				}
+			}
+		}
+		return false;
+	}
+
+	private static boolean matchesTestable(IType annotationType) {
+		return annotationType != null && JUnitCorePlugin.JUNIT5_TESTABLE_ANNOTATION_NAME.equals(annotationType.getFullyQualifiedName());
+	}
+
+	private static boolean matchesTestableInAnnotationHierarchy(IType annotationType, IJavaProject javaProject, Set<IType> hierarchy) throws JavaModelException {
+		if (annotationType != null) {
+			for (IAnnotation annotation : annotationType.getAnnotations()) {
+				IType annType= getResolvedType(annotation.getElementName(), annotationType, javaProject);
+				if (annType != null && hierarchy.add(annType)) {
+					if (matchesTestable(annType) || matchesTestableInAnnotationHierarchy(annType, javaProject, hierarchy)) {
+						return true;
+					}
+				}
+			}
+		}
+		return false;
+	}
 }
diff --git a/org.eclipse.jdt.junit/src/org/eclipse/jdt/junit/launcher/JUnitLaunchConfigurationTab.java b/org.eclipse.jdt.junit/src/org/eclipse/jdt/junit/launcher/JUnitLaunchConfigurationTab.java
index 9e744e20093..1d9a72fc2b4 100644
--- a/org.eclipse.jdt.junit/src/org/eclipse/jdt/junit/launcher/JUnitLaunchConfigurationTab.java
+++ b/org.eclipse.jdt.junit/src/org/eclipse/jdt/junit/launcher/JUnitLaunchConfigurationTab.java
@@ -73,19 +73,15 @@
 
 import org.eclipse.debug.ui.AbstractLaunchConfigurationTab;
 
-import org.eclipse.jdt.core.IAnnotation;
 import org.eclipse.jdt.core.IJavaElement;
 import org.eclipse.jdt.core.IJavaModel;
 import org.eclipse.jdt.core.IJavaProject;
-import org.eclipse.jdt.core.IMethod;
 import org.eclipse.jdt.core.IPackageFragment;
 import org.eclipse.jdt.core.IPackageFragmentRoot;
 import org.eclipse.jdt.core.ISourceReference;
 import org.eclipse.jdt.core.IType;
 import org.eclipse.jdt.core.JavaCore;
 import org.eclipse.jdt.core.JavaModelException;
-import org.eclipse.jdt.core.Signature;
-import org.eclipse.jdt.core.dom.Modifier;
 import org.eclipse.jdt.core.search.IJavaSearchScope;
 import org.eclipse.jdt.core.search.SearchEngine;
 
@@ -186,9 +182,7 @@ public class JUnitLaunchConfigurationTab extends AbstractLaunchConfigurationTab
 
 	private boolean fIsValid= true;
 
-	private Set<String> fMethodsCache;
-
-	private String fMethodsCacheKey;
+	private TestMethodsCache fTestMethodsCache= new TestMethodsCache();
 
 	/**
 	 * Creates a JUnit launch configuration tab.
@@ -271,7 +265,10 @@ public String getText(Object element) {
 		fTestLoaderViewer.setInput(items);
 		fTestLoaderViewer.addSelectionChangedListener(event -> {
 			setEnableTagsGroup(event);
-			validatePage();
+			try (var __= fTestMethodsCache.runNestedCancelable()) {
+				calculateMethodsCache();
+				validatePage();
+			}
 			updateLaunchConfigurationDialog();
 		});
 	}
@@ -320,7 +317,10 @@ public void widgetSelected(SelectionEvent e) {
 		fProjText= new Text(comp, SWT.SINGLE | SWT.BORDER);
 		fProjText.setLayoutData(new GridData(GridData.FILL_HORIZONTAL));
 		fProjText.addModifyListener(evt -> {
-			validatePage();
+			try (var __= fTestMethodsCache.runNestedCancelable()) {
+				calculateMethodsCache();
+				validatePage();
+			}
 			updateLaunchConfigurationDialog();
 			fSearchButton.setEnabled(fTestRadioButton.getSelection() && fProjText.getText().length() > 0);
 		});
@@ -345,8 +345,10 @@ public void widgetSelected(SelectionEvent evt) {
 		fTestText= new Text(comp, SWT.SINGLE | SWT.BORDER);
 		fTestText.setLayoutData(new GridData(GridData.FILL_HORIZONTAL));
 		fTestText.addModifyListener(evt -> {
-			fTestMethodSearchButton.setEnabled(fTestText.getText().length() > 0);
-			validatePage();
+			try (var __= fTestMethodsCache.runNestedCancelable()) {
+				calculateMethodsCache();
+				validatePage();
+			}
 			updateLaunchConfigurationDialog();
 		});
 
@@ -380,7 +382,6 @@ public void widgetSelected(SelectionEvent evt) {
 
 
 		fTestMethodSearchButton= new Button(comp, SWT.PUSH);
-		fTestMethodSearchButton.setEnabled(fTestText.getText().length() > 0);
 		fTestMethodSearchButton.setText(JUnitMessages.JUnitLaunchConfigurationTab_label_search_method);
 		fTestMethodSearchButton.addSelectionListener(new SelectionAdapter() {
 			@Override
@@ -468,23 +469,27 @@ private static Image createImage(String path) {
 
 	@Override
 	public void initializeFrom(ILaunchConfiguration config) {
-		fLaunchConfiguration= config;
+		try (var __= fTestMethodsCache.runNestedCancelable()) {
+			fLaunchConfiguration= config;
 
-		updateProjectFromConfig(config);
-		String containerHandle= ""; //$NON-NLS-1$
-		try {
-			containerHandle= config.getAttribute(JUnitLaunchConfigurationConstants.ATTR_TEST_CONTAINER, ""); //$NON-NLS-1$
-		} catch (CoreException ce) {
-		}
+			updateProjectFromConfig(config);
+			String containerHandle= ""; //$NON-NLS-1$
+			try {
+				containerHandle= config.getAttribute(JUnitLaunchConfigurationConstants.ATTR_TEST_CONTAINER, ""); //$NON-NLS-1$
+			} catch (CoreException ce) {
+			}
 
-		if (containerHandle.length() > 0)
-			updateTestContainerFromConfig(config);
-		else
-			updateTestTypeFromConfig(config);
-		updateKeepRunning(config);
-		updateTestLoaderFromConfig(config);
+			if (containerHandle.length() > 0) {
+				updateTestContainerFromConfig(config);
+			} else {
+				updateTestTypeFromConfig(config);
+			}
+			updateKeepRunning(config);
+			updateTestLoaderFromConfig(config);
 
-		validatePage();
+			calculateMethodsCache();
+			validatePage();
+		}
 	}
 
 
@@ -498,7 +503,9 @@ private void updateTestLoaderFromConfig(ILaunchConfiguration config) {
 				testKind= TestKindRegistry.getDefault().getKind(TestKindRegistry.JUNIT3_TEST_KIND_ID);
 			}
 		}
-		fTestLoaderViewer.setSelection(new StructuredSelection(testKind));
+		try (var __= fTestMethodsCache.runNestedCancelable()) {
+			fTestLoaderViewer.setSelection(new StructuredSelection(testKind));
+		}
 	}
 
 	private TestKind getSelectedTestKind() {
@@ -521,7 +528,9 @@ private void updateProjectFromConfig(ILaunchConfiguration config) {
 			projectName= config.getAttribute(IJavaLaunchConfigurationConstants.ATTR_PROJECT_NAME, ""); //$NON-NLS-1$
 		} catch (CoreException ce) {
 		}
-		fProjText.setText(projectName);
+		try (var __= fTestMethodsCache.runNestedCancelable()) {
+			fProjText.setText(projectName);
+		}
 	}
 
 	private void updateTestTypeFromConfig(ILaunchConfiguration config) {
@@ -536,9 +545,12 @@ private void updateTestTypeFromConfig(ILaunchConfiguration config) {
 		setEnableSingleTestGroup(true);
 		setEnableContainerTestGroup(false);
 		fTestContainerRadioButton.setSelection(false);
-		fTestText.setText(testTypeName);
-		fContainerText.setText(""); //$NON-NLS-1$
-		fTestMethodText.setText(fOriginalTestMethodName);
+
+		try (var __= fTestMethodsCache.runNestedCancelable()) {
+			fTestText.setText(testTypeName);
+			fContainerText.setText(""); //$NON-NLS-1$
+			fTestMethodText.setText(fOriginalTestMethodName);
+		}
 	}
 
 	private void updateTestContainerFromConfig(ILaunchConfiguration config) {
@@ -559,7 +571,10 @@ private void updateTestContainerFromConfig(ILaunchConfiguration config) {
 		fTestRadioButton.setSelection(false);
 		if (fContainerElement != null)
 			fContainerText.setText(getPresentationName(fContainerElement));
-		fTestText.setText(""); //$NON-NLS-1$
+
+		try (var __= fTestMethodsCache.runNestedCancelable()) {
+			fTestText.setText(""); //$NON-NLS-1$
+		}
 	}
 
 	@Override
@@ -678,9 +693,11 @@ public ITypeInfoFilterExtension getFilterExtension() {
 		IType type= (IType) results[0];
 
 		if (type != null) {
-			fTestText.setText(type.getFullyQualifiedName('.'));
-			javaProject= type.getJavaProject();
-			fProjText.setText(javaProject.getElementName());
+			try (var __= fTestMethodsCache.runNestedCancelable()) {
+				fTestText.setText(type.getFullyQualifiedName('.'));
+				javaProject= type.getJavaProject();
+				fProjText.setText(javaProject.getElementName());
+			}
 		}
 	}
 
@@ -695,142 +712,109 @@ private void handleProjectButtonSelected() {
 			return;
 		}
 
-		String projectName= project.getElementName();
-		fProjText.setText(projectName);
+		try (var __= fTestMethodsCache.runNestedCancelable()) {
+			String projectName= project.getElementName();
+			fProjText.setText(projectName);
+		}
 	}
 
 	private void handleTestMethodSearchButtonSelected() {
-		try {
+		try (var __= fTestMethodsCache.runNestedCancelable()) {
 			IJavaProject javaProject= getJavaProject();
 			IType testType= javaProject.findType(fTestText.getText());
 			Set<String> methodNames= getMethodsForType(javaProject, testType, getSelectedTestKind());
+
+			// I can't put this logic inside getMethodsForType because that would cause
+			// a bug, making it necessary to cancel the search twice when first opening
+			// a JUnit configuration
+			if (methodNames.isEmpty()) {
+				calculateMethodsCache();
+				methodNames= getMethodsForType(javaProject, testType, getSelectedTestKind());
+			}
+
+			if (fTestMethodsCache.isCanceled()) {
+				return;
+			}
+
 			String methodName= chooseMethodName(methodNames);
 
 			if (methodName != null) {
 				fTestMethodText.setText(methodName);
-				validatePage();
-				updateLaunchConfigurationDialog();
 			}
+			validatePage();
+			updateLaunchConfigurationDialog();
 		} catch (JavaModelException e) {
 			JUnitPlugin.log(e.getStatus());
 		}
 	}
 
-	private Set<String> getMethodsForType(IJavaProject javaProject, IType type, TestKind testKind) throws JavaModelException {
+	private Set<String> getMethodsForType(IJavaProject javaProject, IType type, TestKind testKind) {
 		if (javaProject == null || type == null || testKind == null)
 			return Collections.emptySet();
 
 		String testKindId= testKind.getId();
-		String methodsCacheKey= javaProject.getElementName() + '\n' + type.getFullyQualifiedName() + '\n' + testKindId;
-		if (methodsCacheKey.equals(fMethodsCacheKey))
-			return fMethodsCache;
-
-		Set<String> methodNames= new HashSet<>();
-		fMethodsCache= methodNames;
-		fMethodsCacheKey= methodsCacheKey;
-
-		collectMethodNames(type, javaProject, testKindId, methodNames);
+		String methodsCacheKey= getMethodsCacheKey(javaProject, type, testKindId);
+		if (fTestMethodsCache.containsKey(methodsCacheKey)) {
+			return fTestMethodsCache.get(methodsCacheKey);
+		}
 
-		return methodNames;
+		return Collections.emptySet();
 	}
 
-	private void collectMethodNames(IType type, IJavaProject javaProject, String testKindId, Set<String> methodNames) throws JavaModelException {
-		if (type == null) {
+	private void calculateMethodsCache() {
+		fTestMethodText.setEnabled(false);
+
+		if (fTestMethodsCache.isCanceled()) {
 			return;
 		}
-		collectDeclaredMethodNames(type, javaProject, testKindId, methodNames);
 
-		String superclassName= type.getSuperclassName();
-		IType superType= getResolvedType(superclassName, type, javaProject);
-		collectMethodNames(superType, javaProject, testKindId, methodNames);
-
-		String[] superInterfaceNames= type.getSuperInterfaceNames();
-		for (String interfaceName : superInterfaceNames) {
-			superType= getResolvedType(interfaceName, type, javaProject);
-			collectMethodNames(superType, javaProject, testKindId, methodNames);
-		}
-	}
+		try {
+			IJavaProject javaProject= getJavaProject();
 
-	private IType getResolvedType(String typeName, IType type, IJavaProject javaProject) throws JavaModelException {
-		IType resolvedType= null;
-		if (typeName != null) {
-			int pos= typeName.indexOf('<');
-			if (pos != -1) {
-				typeName= typeName.substring(0, pos);
-			}
-			String[][] resolvedTypeNames= type.resolveType(typeName);
-			if (resolvedTypeNames != null && resolvedTypeNames.length > 0) {
-				String[] resolvedTypeName= resolvedTypeNames[0];
-				resolvedType= javaProject.findType(resolvedTypeName[0], resolvedTypeName[1]); // secondary types not found by this API
+			if (javaProject == null) {
+				// can't find methods if the project
+				return;
 			}
-		}
-		return resolvedType;
-	}
 
-	private void collectDeclaredMethodNames(IType type, IJavaProject javaProject, String testKindId, Set<String> methodNames) throws JavaModelException {
-		IMethod[] methods= type.getMethods();
-		for (IMethod method : methods) {
-			String methodName= method.getElementName();
-			int flags= method.getFlags();
-			// Only include public, non-static, no-arg methods that return void and start with "test":
-			if (Modifier.isPublic(flags) && !Modifier.isStatic(flags) &&
-					method.getNumberOfParameters() == 0 && Signature.SIG_VOID.equals(method.getReturnType()) &&
-					methodName.startsWith("test")) { //$NON-NLS-1$
-				methodNames.add(methodName);
-			}
-			boolean isJUnit3= TestKindRegistry.JUNIT3_TEST_KIND_ID.equals(testKindId);
-			boolean isJUnit5= TestKindRegistry.JUNIT5_TEST_KIND_ID.equals(testKindId);
-			if (!isJUnit3 && !Modifier.isPrivate(flags) && !Modifier.isStatic(flags)) {
-				IAnnotation annotation= method.getAnnotation("Test"); //$NON-NLS-1$
-				if (annotation.exists()) {
-					methodNames.add(methodName + JUnitStubUtility.getParameterTypes(method, false));
-				} else if (isJUnit5) {
-					boolean hasAnyTestAnnotation= method.getAnnotation("TestFactory").exists() //$NON-NLS-1$
-							|| method.getAnnotation("Testable").exists() //$NON-NLS-1$
-							|| method.getAnnotation("TestTemplate").exists() //$NON-NLS-1$
-							|| method.getAnnotation("ParameterizedTest").exists() //$NON-NLS-1$
-							|| method.getAnnotation("RepeatedTest").exists(); //$NON-NLS-1$
-					if (hasAnyTestAnnotation || isAnnotatedWithTestable(method, type, javaProject)) {
-						methodNames.add(methodName + JUnitStubUtility.getParameterTypes(method, false));
-					}
-				}
+			IType testClass= javaProject.findType(fTestText.getText());
+
+			if (testClass == null) {
+				// can't find methods if the class doesn't exist
+				return;
 			}
-		}
-	}
 
-	// See JUnit5TestFinder.Annotation#annotates also.
-	private boolean isAnnotatedWithTestable(IMethod method, IType declaringType, IJavaProject javaProject) throws JavaModelException {
-		for (IAnnotation annotation : method.getAnnotations()) {
-			IType annotationType= getResolvedType(annotation.getElementName(), declaringType, javaProject);
-			if (annotationType != null) {
-				if (matchesTestable(annotationType)) {
-					return true;
-				}
-				Set<IType> hierarchy= new HashSet<>();
-				if (matchesTestableInAnnotationHierarchy(annotationType, javaProject, hierarchy)) {
-					return true;
-				}
+			TestKind testKind= getSelectedTestKind();
+
+			if (testKind == null) {
+				// no need to search for methods if the type (JUnit3/4/5) is not set
+				return;
 			}
-		}
-		return false;
-	}
 
-	private boolean matchesTestable(IType annotationType) {
-		return annotationType != null && JUnitCorePlugin.JUNIT5_TESTABLE_ANNOTATION_NAME.equals(annotationType.getFullyQualifiedName());
-	}
+			String methodsCacheKey= getMethodsCacheKey(javaProject, testClass, testKind.getId());
 
-	private boolean matchesTestableInAnnotationHierarchy(IType annotationType, IJavaProject javaProject, Set<IType> hierarchy) throws JavaModelException {
-		if (annotationType != null) {
-			for (IAnnotation annotation : annotationType.getAnnotations()) {
-				IType annType= getResolvedType(annotation.getElementName(), annotationType, javaProject);
-				if (annType != null && hierarchy.add(annType)) {
-					if (matchesTestable(annType) || matchesTestableInAnnotationHierarchy(annType, javaProject, hierarchy)) {
-						return true;
-					}
-				}
+			if (fTestMethodsCache.containsKey(methodsCacheKey)) {
+				// no need to recalculate since the source code can't change while the dialog is open.
+				fTestMethodText.setEnabled(true);
+				return;
 			}
+
+			fTestMethodsCache.put(methodsCacheKey, //
+					TestSearchEngine.findTestMethods(getLaunchConfigurationDialog(), javaProject, testClass, testKind));
+
+			// calculation successful, reactivate the UI
+			fTestMethodText.setEnabled(true);
+		} catch (InvocationTargetException | JavaModelException e) {
+			JUnitPlugin.log(e);
+		} catch (InterruptedException e) {
+			// the user probably canceled the operation. Sadly there is no way to know it for sure since ModalContext::run
+			// doesn't throw the original OperationCanceledException, it throws a new InterruptedException.
+			JUnitPlugin.log(e);
+			fTestMethodsCache.setCanceled(true);
 		}
-		return false;
+	}
+
+	private String getMethodsCacheKey(IJavaProject javaProject, IType type, String testKindId) {
+		return javaProject.getElementName() + '\n' + type.getFullyQualifiedName() + '\n' + testKindId;
 	}
 
 	private String chooseMethodName(Set<String> methodNames) {
@@ -949,7 +933,11 @@ private void testModeChanged() {
 	@Override
 	protected void setErrorMessage(String errorMessage) {
 		fIsValid= errorMessage == null;
-		super.setErrorMessage(errorMessage);
+		if (fTestMethodsCache.isCanceled()) {
+			super.setErrorMessage(JUnitMessages.JUnitLaunchConfigurationTab_error_operation_canceled + " " + errorMessage); //$NON-NLS-1$
+		} else {
+			super.setErrorMessage(errorMessage);
+		}
 	}
 
 	private void validatePage() {
@@ -1078,8 +1066,6 @@ private void setEnableSingleTestGroup(boolean enabled) {
 		boolean projectTextHasContents= fProjText.getText().length() > 0;
 		fSearchButton.setEnabled(enabled && projectTextHasContents);
 		fTestMethodLabel.setEnabled(enabled);
-		fTestMethodText.setEnabled(enabled);
-		fTestMethodSearchButton.setEnabled(enabled && projectTextHasContents && fTestText.getText().length() > 0);
 	}
 
 	@Override
diff --git a/org.eclipse.jdt.junit/src/org/eclipse/jdt/junit/launcher/TestMethodsCache.java b/org.eclipse.jdt.junit/src/org/eclipse/jdt/junit/launcher/TestMethodsCache.java
new file mode 100644
index 00000000000..754b8e1fcd5
--- /dev/null
+++ b/org.eclipse.jdt.junit/src/org/eclipse/jdt/junit/launcher/TestMethodsCache.java
@@ -0,0 +1,87 @@
+/*******************************************************************************
+ * Copyright (c) 2023 Vector Informatik GmbH and others.
+ *
+ * This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License 2.0
+ * which accompanies this distribution, and is available at
+ * https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ * Contributors:
+ *     Vector Informatik GmbH - initial API and implementation
+ *******************************************************************************/
+package org.eclipse.jdt.junit.launcher;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * This class has the necessary logic to calculate the cache of all test methods that belong to a
+ * JUnit configuration.
+ */
+class TestMethodsCache {
+	/**
+	 * An <code>AutoCloseable</code> that can contain nested instances and can run a
+	 * <code>Runnable</code> upon closing the <i>outer</i> instance. <br/>
+	 * <br/>
+	 * Example:
+	 *
+	 * <pre>
+	 *
+	 * try (var outer= new NestedAutoCloseable(() -> System.out.println("Bye bye outer"))) {
+	 * 	try (var inner= new NestedAutoCloseable(() -> System.out.println("Bye bye inner"))) {
+	 * 		// ...
+	 * 	} // doesn't print anything
+	 * } // prints "Bye bye outer"
+	 * </pre>
+	 */
+	private static class NestedAutoCloseable implements AutoCloseable {
+		private static int fgDepth;
+
+		NestedAutoCloseable(Runnable onCreateOuterBlock) {
+			if (fgDepth == 0) {
+				onCreateOuterBlock.run();
+			}
+			fgDepth++;
+		}
+
+		@Override
+		public final void close() {
+			fgDepth--;
+		}
+	}
+
+	private boolean fCanceled;
+
+	private final Map<String, Set<String>> fCacheMap= new HashMap<>();
+
+	void put(String key, Set<String> value) {
+		fCacheMap.put(key, value);
+	}
+
+	Set<String> get(String key) {
+		return fCacheMap.get(key);
+	}
+
+	boolean containsKey(String key) {
+		return fCacheMap.containsKey(key);
+	}
+
+	boolean isCanceled() {
+		return fCanceled;
+	}
+
+	void setCanceled(boolean canceled) {
+		fCanceled= canceled;
+	}
+
+	/**
+	 * @return an <code>AutoCloseable</code> that guarantees that searching for test methods needs
+	 *         to be canceled only once even in nested calls.
+	 */
+	NestedAutoCloseable runNestedCancelable() {
+		return new NestedAutoCloseable(() -> fCanceled= false);
+	}
+}