Skip to content

Commit

Permalink
[TP-Editor] Completely parallelize P2 metadata fetching
Browse files Browse the repository at this point in the history
  • Loading branch information
@HannesWell
HannesWell committed Oct 19, 2024
1 parent d141f1d commit b92ac06
Show file tree
Hide file tree
Showing 6 changed files with 70 additions and 103 deletions.
30 changes: 9 additions & 21 deletions ...c/org/eclipse/pde/internal/genericeditor/target/extension/command/UpdateUnitVersions.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@
import org.eclipse.core.commands.AbstractHandler;
import org.eclipse.core.commands.ExecutionEvent;
import org.eclipse.core.commands.ExecutionException;
import org.eclipse.core.runtime.jobs.Job;
import org.eclipse.equinox.p2.metadata.IVersionedId;
import org.eclipse.jface.dialogs.IPageChangeProvider;
import org.eclipse.jface.dialogs.MessageDialog;
Expand All @@ -33,7 +32,6 @@
import org.eclipse.pde.internal.genericeditor.target.extension.model.RepositoryCache;
import org.eclipse.pde.internal.genericeditor.target.extension.model.UnitNode;
import org.eclipse.pde.internal.genericeditor.target.extension.model.xml.Parser;
import org.eclipse.pde.internal.genericeditor.target.extension.p2.UpdateJob;
import org.eclipse.swt.widgets.Display;
import org.eclipse.ui.IEditorInput;
import org.eclipse.ui.IEditorPart;
Expand Down Expand Up @@ -69,20 +67,19 @@ public Object execute(ExecutionEvent event) throws ExecutionException {

int offsetChange = 0;
String documentText = document.get();
for (Node n1 : locationsNode.get(0).getChildNodesByTag(ITargetConstants.LOCATION_TAG)) {
LocationNode locationNode = (LocationNode) n1;

List<LocationNode> locationNodes = locationsNode.get(0).getChildNodesByTag(ITargetConstants.LOCATION_TAG)
.stream().map(LocationNode.class::cast).toList();

// Fetch all repos at once to fetch pending metadata in parallel
locationNodes.stream().map(LocationNode::getRepositoryLocations).flatMap(List::stream)
.forEach(RepositoryCache::prefetchP2MetadataOfRepository);

for (LocationNode locationNode : locationNodes) {
List<String> repositoryLocations = locationNode.getRepositoryLocations();
if (repositoryLocations.isEmpty()) {
continue;
}
if (!repositoryLocations.stream().allMatch(RepositoryCache::isUpToDate)) {
try {
updateCache(locationNode);
} catch (InterruptedException e) {
e.printStackTrace();
continue;
}
}
Map<String, List<IVersionedId>> repositoryUnits = RepositoryCache
.fetchP2UnitsFromRepos(repositoryLocations);
for (Node n2 : locationNode.getChildNodesByTag(ITargetConstants.UNIT_TAG)) {
Expand Down Expand Up @@ -129,15 +126,6 @@ public Object execute(ExecutionEvent event) throws ExecutionException {
});
}

private void updateCache(LocationNode locationNode) throws InterruptedException {
Job job = new UpdateJob(locationNode);
job.setUser(true);
job.schedule();
while (job.getResult() == null) {
Thread.sleep(50);
}
}

private IDocument getDocument() {
IEditorPart editor = PlatformUI.getWorkbench().getActiveWorkbenchWindow().getActivePage().getActiveEditor();
IDocumentProvider provider = null;
Expand Down
69 changes: 48 additions & 21 deletions ...on/src/org/eclipse/pde/internal/genericeditor/target/extension/model/RepositoryCache.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,17 +14,22 @@
*******************************************************************************/
package org.eclipse.pde.internal.genericeditor.target.extension.model;

import java.util.ArrayList;
import java.util.Collection;
import java.util.Comparator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import org.eclipse.core.runtime.jobs.Job;
import org.eclipse.equinox.p2.metadata.IVersionedId;
import org.eclipse.osgi.util.NLS;
import org.eclipse.pde.internal.genericeditor.target.extension.p2.Messages;
import org.eclipse.pde.internal.genericeditor.target.extension.p2.P2Fetcher;

/**
Expand All @@ -41,7 +46,7 @@ private RepositoryCache() {
//avoid instantiation
}

private static final Map<String, Map<String, List<IVersionedId>>> CACHE = new ConcurrentHashMap<>();
private static final Map<String, CompletableFuture<Map<String, List<IVersionedId>>>> CACHE = new ConcurrentHashMap<>();

/**
* Fetches information and caches it.
Expand All @@ -57,17 +62,50 @@ private RepositoryCache() {
*/
public static Map<String, List<IVersionedId>> fetchP2UnitsFromRepos(List<String> repositories) {
if (repositories.size() == 1) {
return fetchP2UnitsFromRepo(repositories.get(0));
try {
return fetchP2DataOfRepo(repositories.get(0)).get();
} catch (InterruptedException | ExecutionException e) {
return Map.of();
}
}
List<Map<String, List<IVersionedId>>> units = new ArrayList<>(repositories.size());
for (String repository : repositories) {
units.add(fetchP2UnitsFromRepo(repository));
}
return toSortedMap(units.stream().map(Map::values).flatMap(Collection::stream).flatMap(List::stream));
List<Future<Map<String, List<IVersionedId>>>> repos = repositories.stream()
.map(RepositoryCache::fetchP2DataOfRepo).toList();
// Fetch all repos at once to await pending metadata in parallel
return toSortedMap(repos.stream().<Map<String, List<IVersionedId>>>map(f -> {
try {
return f.get();
} catch (InterruptedException | ExecutionException e) {
return Map.of();
}
}).map(Map::values).flatMap(Collection::stream).flatMap(List::stream));
}

public static void prefetchP2MetadataOfRepository(String repository) {
fetchP2DataOfRepo(repository);
}

private static Map<String, List<IVersionedId>> fetchP2UnitsFromRepo(String repository) {
return CACHE.computeIfAbsent(repository, r -> toSortedMap(P2Fetcher.fetchAvailableUnits(r)));
private static Future<Map<String, List<IVersionedId>>> fetchP2DataOfRepo(String repository) {
return CACHE.compute(repository, (r, f) -> {
if (f != null && (!f.isDone() || !f.isCompletedExceptionally() && !f.isCancelled())) {
return f;
}
CompletableFuture<Map<String, List<IVersionedId>>> future = new CompletableFuture<>();
// Fetching P2 repository information is a costly operation
// time-wise. Thus it is done in a job.
Job job = Job.create(NLS.bind(Messages.UpdateJob_P2DataFetch, r), m -> {
try {
Map<String, List<IVersionedId>> units = toSortedMap(P2Fetcher.fetchAvailableUnits(r, m));
future.complete(units);
} catch (Throwable e) {
future.completeExceptionally(e);
CACHE.remove(repository);
throw e;
}
});
job.setUser(true);
job.schedule();
return future;
});
}

private static final Comparator<IVersionedId> BY_ID_FIRST_THEN_DESCENDING_VERSION = Comparator
Expand Down Expand Up @@ -125,15 +163,4 @@ public static List<IVersionedId> getUnitsBySearchTerm(String repo, String search
return allUnits.values().stream().flatMap(List::stream) //
.filter(unit -> unit.getId().contains(searchTerm)).toList();
}

/**
* Classic cache up-to-date check.
*
* @param repo
* repository URL
* @return whether the cache is up to date for this repo
*/
public static boolean isUpToDate(String repo) {
return CACHE.get(repo) != null;
}
}
1 change: 0 additions & 1 deletion ...or.extension/src/org/eclipse/pde/internal/genericeditor/target/extension/p2/Messages.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
public class Messages extends NLS {
private static final String BUNDLE_NAME = "org.eclipse.pde.internal.genericeditor.target.extension.p2.messages"; //$NON-NLS-1$
public static String UpdateJob_P2DataFetch;
public static String UpdateJob_ErrorMessage;
static {
// initialize resource bundle
NLS.initializeMessages(BUNDLE_NAME, Messages.class);
Expand Down
24 changes: 11 additions & 13 deletions ...r.extension/src/org/eclipse/pde/internal/genericeditor/target/extension/p2/P2Fetcher.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,12 @@
import java.net.URISyntaxException;
import java.util.stream.Stream;

import org.eclipse.core.runtime.ILog;
import org.eclipse.core.runtime.CoreException;
import org.eclipse.core.runtime.IProgressMonitor;
import org.eclipse.core.runtime.Status;
import org.eclipse.core.runtime.SubMonitor;
import org.eclipse.equinox.p2.core.IProvisioningAgent;
import org.eclipse.equinox.p2.core.IProvisioningAgentProvider;
import org.eclipse.equinox.p2.core.ProvisionException;
import org.eclipse.equinox.p2.metadata.IInstallableUnit;
import org.eclipse.equinox.p2.metadata.IVersionedId;
import org.eclipse.equinox.p2.metadata.VersionedId;
Expand All @@ -46,32 +48,28 @@ public class P2Fetcher {
* URL string of a p2 repository
* @return List of available installable unit models. See {@link UnitNode}
*/
public static Stream<IVersionedId> fetchAvailableUnits(String repositoryLocation) {
public static Stream<IVersionedId> fetchAvailableUnits(String repositoryLocation, IProgressMonitor monitor)
throws CoreException {
SubMonitor subMonitor = SubMonitor.convert(monitor, 31);
try {
URI uri = new URI(repositoryLocation);

BundleContext context = FrameworkUtil.getBundle(P2Fetcher.class).getBundleContext();
ServiceReference<IProvisioningAgentProvider> sr = context
.getServiceReference(IProvisioningAgentProvider.class);
IProvisioningAgentProvider agentProvider = context.getService(sr);
IProvisioningAgent agent = null;
IProvisioningAgent agent;
try {
agent = agentProvider.createAgent(null);
} catch (ProvisionException e) {
ILog.get().error("Failed to create provisioning-agent", e);
} finally {
context.ungetService(sr);
}
IMetadataRepositoryManager manager = agent.getService(IMetadataRepositoryManager.class);
IMetadataRepository repository = manager.loadRepository(uri, null);
IQueryResult<IInstallableUnit> result = repository.query(QueryUtil.ALL_UNITS, null);

IMetadataRepository repository = manager.loadRepository(uri, subMonitor.split(30));
IQueryResult<IInstallableUnit> result = repository.query(QueryUtil.ALL_UNITS, subMonitor.split(1));
return result.stream().map(iu -> new VersionedId(iu.getId(), iu.getVersion()));
} catch (URISyntaxException e) {
return Stream.empty();
} catch (Exception e) {
ILog.get().error("Failed to fetch metadata of repository: " + repositoryLocation, e);
return Stream.empty();
throw new CoreException(Status.error("Invalid repository URI: " + repositoryLocation, e));
}
}

Expand Down
44 changes: 0 additions & 44 deletions ...r.extension/src/org/eclipse/pde/internal/genericeditor/target/extension/p2/UpdateJob.java
View file

This file was deleted.

5 changes: 2 additions & 3 deletions ...ension/src/org/eclipse/pde/internal/genericeditor/target/extension/p2/messages.properties
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
###############################################################################
# Copyright (c) 2016 Red Hat Inc. and others
# Copyright (c) 2016, 2024 Red Hat Inc. and others
#
# This program and the accompanying materials
# are made available under the terms of the Eclipse Public License 2.0
Expand All @@ -11,5 +11,4 @@
# Contributors:
# Sopot Cela (Red Hat Inc.) - initial implementation
###############################################################################
UpdateJob_P2DataFetch=Fetching p2 metadata from repository
UpdateJob_ErrorMessage=Issue fetching data from repository. Please check URL or see log for even more details.
UpdateJob_P2DataFetch=Fetching p2 metadata from repository: {0}

0 comments on commit b92ac06

Please sign in to comment.