Skip to content

Commit

Permalink
Use original URL to decide if a IU is provided by reference
Browse files Browse the repository at this point in the history 
Currently the location of the repository is used, but this can lead to
false results if the location is actually is mirrored. In that case a
reference is removed even though it contributes to the solution and is
afterwards missing.

This now uses a Map of the original URI used to load the repository to
check for items that can be removed. Beside that a repository does not
only contributes its own IUs but also those that are enclosed in their
references as well.
  • Loading branch information
@laeubi
laeubi committed May 17, 2024
1 parent cde7b90 commit f53ebeb
Show file tree
Hide file tree
Showing 2 changed files with 82 additions and 34 deletions.
8 changes: 4 additions & 4 deletions tycho-core/src/main/java/org/eclipse/tycho/p2tools/MirrorApplicationServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ public void mirrorStandalone(RepositoryReferences sources, DestinationRepository
Collection<IUDescription> seedIUs, MirrorOptions mirrorOptions, BuildDirectory tempDirectory)
throws FacadeException {
agent.getService(IArtifactRepositoryManager.class); //force init of framework if not already done!
final TychoMirrorApplication mirrorApp = createMirrorApplication(sources, destination, agent);
final TychoMirrorApplication mirrorApp = createMirrorApplication(sources, destination, agent, logger);
mirrorApp.setSlicingOptions(createSlicingOptions(mirrorOptions));
mirrorApp.setIgnoreErrors(mirrorOptions.isIgnoreErrors());
try {
Expand Down Expand Up @@ -176,7 +176,7 @@ public void mirrorReactor(RepositoryReferences sources, DestinationRepositoryDes
boolean includeAllSource, boolean includeRequiredBundles, boolean includeRequiredFeatures,
boolean filterProvided, boolean addOnlyProvidingRepoReferences, Map<String, String> filterProperties)
throws FacadeException {
final TychoMirrorApplication mirrorApp = createMirrorApplication(sources, destination, agent);
final TychoMirrorApplication mirrorApp = createMirrorApplication(sources, destination, agent, logger);

// mirror scope: seed units...
try {
Expand Down Expand Up @@ -250,8 +250,8 @@ public void recreateArtifactRepository(DestinationRepositoryDescriptor destinati
}

private static TychoMirrorApplication createMirrorApplication(RepositoryReferences sources,
DestinationRepositoryDescriptor destination, IProvisioningAgent agent) {
final TychoMirrorApplication mirrorApp = new TychoMirrorApplication(agent, destination);
DestinationRepositoryDescriptor destination, IProvisioningAgent agent, Logger logger) {
final TychoMirrorApplication mirrorApp = new TychoMirrorApplication(agent, destination, logger);
mirrorApp.setRaw(false);

List<RepositoryDescriptor> sourceDescriptors = createSourceDescriptors(sources);
Expand Down
108 changes: 78 additions & 30 deletions tycho-core/src/main/java/org/eclipse/tycho/p2tools/TychoMirrorApplication.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,9 @@

import static java.util.stream.Collectors.groupingBy;
import static java.util.stream.Collectors.mapping;
import static java.util.stream.Collectors.toList;

import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Comparator;
import java.util.HashMap;
Expand All @@ -32,6 +30,7 @@
import java.util.stream.Collectors;
import java.util.stream.Stream;

import org.codehaus.plexus.logging.Logger;
import org.eclipse.core.runtime.IProgressMonitor;
import org.eclipse.equinox.internal.p2.metadata.IRequiredCapability;
import org.eclipse.equinox.internal.p2.metadata.InstallableUnit;
Expand Down Expand Up @@ -62,12 +61,9 @@
import org.eclipse.tycho.p2tools.copiedfromp2.PermissiveSlicer;
import org.eclipse.tycho.p2tools.copiedfromp2.RepositoryDescriptor;
import org.eclipse.tycho.p2tools.copiedfromp2.Slicer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class TychoMirrorApplication extends org.eclipse.tycho.p2tools.copiedfromp2.MirrorApplication {

private static final Logger LOGGER = LoggerFactory.getLogger(TychoMirrorApplication.class);
private static final String SOURCE_SUFFIX = ".source";
private static final String FEATURE_GROUP = ".feature.group";
private final DestinationRepositoryDescriptor destination;
Expand All @@ -77,10 +73,13 @@ public class TychoMirrorApplication extends org.eclipse.tycho.p2tools.copiedfrom
private boolean filterProvided;
private boolean addOnlyProvidingRepoReferences;
private TargetPlatform targetPlatform;
private Logger logger;

public TychoMirrorApplication(IProvisioningAgent agent, DestinationRepositoryDescriptor destination) {
public TychoMirrorApplication(IProvisioningAgent agent, DestinationRepositoryDescriptor destination,
Logger logger) {
super(agent);
this.destination = destination;
this.logger = logger;
this.removeAddedRepositories = false;
}

Expand Down Expand Up @@ -267,8 +266,8 @@ protected void finalizeRepositories() {
if (repository != null) {
Collection<IRepositoryReference> references = repository.getReferences();
if (!references.isEmpty()) {
LOGGER.info("Adding references to the following repositories:");
references.stream().map(r -> r.getLocation()).distinct().forEach(loc -> LOGGER.info(" {}", loc));
logger.info("Adding references to the following repositories:");
references.stream().map(r -> r.getLocation()).distinct().forEach(loc -> logger.info(" " + loc));
}
}
super.finalizeRepositories();
Expand All @@ -289,11 +288,12 @@ protected Set<IInstallableUnit> collectUnits(IQueryable<IInstallableUnit> slice,
throws ProvisionException {
Set<IInstallableUnit> units = super.collectUnits(slice, monitor);
if (isFilterProvidedItems()) {
Map<String, List<Version>> fullRepositoryContent = units.stream()
.collect(groupingBy(IInstallableUnit::getId, mapping(IInstallableUnit::getVersion, toList())));
Map<String, Set<Version>> fullRepositoryContent = units.stream().collect(
groupingBy(IInstallableUnit::getId, mapping(IInstallableUnit::getVersion, Collectors.toSet())));

List<IRepository<IInstallableUnit>> metadataRepositories = removeProvidedItems(units,
getMetadataRepositoryManager(), IRepository.TYPE_METADATA, monitor);
IMetadataRepositoryManager manager = getMetadataRepositoryManager();
Map<URI, IRepository<IInstallableUnit>> metadataRepositories = removeProvidedItems(units, manager,
IRepository.TYPE_METADATA, monitor);

if (addOnlyProvidingRepoReferences) {
Set<URI> removableReferences = destination.getFilterableRepositoryReferences().stream()
Expand All @@ -302,7 +302,8 @@ protected Set<IInstallableUnit> collectUnits(IQueryable<IInstallableUnit> slice,
.forEach(removableReferences::remove); // keep reference if explicitly added to the repository
if (!removableReferences.isEmpty()) {
// Assume that for all units that correspond to artifacts the metadata either has a co-located artifact repository or a references to to one that contains it.
removeNotProvidingReferences(fullRepositoryContent, metadataRepositories, removableReferences);
removeNotProvidingReferences(fullRepositoryContent, metadataRepositories, removableReferences,
manager);
}
}
}
Expand All @@ -313,17 +314,18 @@ private boolean isFilterProvidedItems() {
return filterProvided && !destinationMetadataRepository.getReferences().isEmpty();
}

private <T> List<IRepository<T>> removeProvidedItems(Collection<T> allElements, IRepositoryManager<T> repoManager,
int repositoryType, IProgressMonitor monitor) throws ProvisionException {
List<IRepository<T>> referencedRepositories = new ArrayList<>();
private <T> Map<URI, IRepository<T>> removeProvidedItems(Collection<T> allElements,
IRepositoryManager<T> repoManager, int repositoryType, IProgressMonitor monitor) throws ProvisionException {
Map<URI, IRepository<T>> referencedRepositories = new HashMap<>();
for (IRepositoryReference reference : destinationMetadataRepository.getReferences()) {
if (reference.getType() != repositoryType) {
continue;
}
try {
URI location = reference.getLocation();
IRepository<T> repository = repoManager.loadRepository(location, monitor);
referencedRepositories.add(repository);
//We need to retain the location in the map as the repo manager might rewrite it to a mirrored location
referencedRepositories.put(location, repository);
} catch (IllegalArgumentException e) {
if (e.getCause() instanceof URISyntaxException uriException) {
throw new ProvisionException("Can't parse referenced URI!", uriException);
Expand All @@ -332,35 +334,81 @@ private <T> List<IRepository<T>> removeProvidedItems(Collection<T> allElements,
}
}
}
allElements.removeIf(e -> referencedRepositories.stream().anyMatch(repo -> repo.contains(e)));
allElements.removeIf(e -> referencedRepositories.values().stream().anyMatch(repo -> repo.contains(e)));
return referencedRepositories;
}

private void removeNotProvidingReferences(Map<String, List<Version>> fullRepositoryContent,
List<IRepository<IInstallableUnit>> metadataRepositories, Set<URI> removableReferenceURIs) {
private void removeNotProvidingReferences(Map<String, Set<Version>> fullRepositoryContent,
Map<URI, IRepository<IInstallableUnit>> metadataRepositories, Set<URI> removableReferenceURIs,
IMetadataRepositoryManager manager) {
Map<URI, Set<IInstallableUnit>> usedRepositoryItems = new HashMap<>();
for (IRepository<IInstallableUnit> repo : metadataRepositories) {
Set<IInstallableUnit> usedRepoContent = repo.query(QueryUtil.ALL_UNITS, null).stream()
.filter(a -> fullRepositoryContent.getOrDefault(a.getId(), List.of()).contains(a.getVersion()))
Map<URI, Set<IInstallableUnit>> providedItems = new HashMap<>();
for (Entry<URI, IRepository<IInstallableUnit>> repo : metadataRepositories.entrySet()) {
Set<IInstallableUnit> content = getRepositoryContent(repo.getKey(), repo.getValue(), new HashSet<>(),
manager).collect(Collectors.toSet());
Set<IInstallableUnit> usedRepoContent = content.stream()
.filter(a -> fullRepositoryContent.getOrDefault(a.getId(), Set.of()).contains(a.getVersion()))
.collect(Collectors.toSet());
usedRepositoryItems.put(repo.getLocation(), usedRepoContent);
usedRepositoryItems.put(repo.getKey(), usedRepoContent);
providedItems.put(repo.getKey(), content);
}
// Remove filterable references that contribute nothing or whose relevant content is also provided by another repo
usedRepositoryItems.entrySet().removeIf(repo -> {
if (!removableReferenceURIs.contains(repo.getKey())) {
return false;
if (removableReferenceURIs.contains(repo.getKey())) {
Set<IInstallableUnit> usedContent = repo.getValue();
if (usedContent.isEmpty()) {
logger.info(
"Remove reference " + repo.getKey() + " because no units are contained in the repository.");
return true;
}
for (Entry<URI, Set<IInstallableUnit>> entry : usedRepositoryItems.entrySet()) {
if (entry == repo) {
continue;
}
Set<IInstallableUnit> other = providedItems.getOrDefault(entry.getKey(), Set.of());
if (!other.isEmpty() && other.containsAll(usedContent)) {
logger.info("Remove reference " + repo.getKey()
+ " because all units are also contained in reference " + entry.getKey() + " already.");
return true;
}
}
}
Set<IInstallableUnit> usedContent = repo.getValue();
return usedContent.isEmpty()
|| usedRepositoryItems.entrySet().stream().filter(e -> e != repo).map(Entry::getValue)
.anyMatch(other -> other.size() >= usedContent.size() && other.containsAll(usedContent));
return false;
});
IMetadataRepository repository = getDestinationMetadataRepository();
List<IRepositoryReference> discardedReferences = repository.getReferences().stream()
.filter(rr -> !usedRepositoryItems.keySet().contains(rr.getLocation())).toList();
repository.removeReferences(discardedReferences);
}

private Stream<IInstallableUnit> getRepositoryContent(URI uri, IRepository<IInstallableUnit> repo, Set<URI> visited,
IMetadataRepositoryManager manager) {
if (visited.add(uri)) {
Stream<IInstallableUnit> stream = repo.query(QueryUtil.ALL_UNITS, null).stream();
if (repo instanceof IMetadataRepository meta) {
Collection<IRepositoryReference> references = meta.getReferences();
for (IRepositoryReference reference : references) {
if (reference.getType() == IRepository.TYPE_METADATA && isEnabled(reference)) {
try {
URI referenceLocation = reference.getLocation();
IMetadataRepository referenceRepository = manager.loadRepository(referenceLocation, null);
stream = Stream.concat(stream,
getRepositoryContent(referenceLocation, referenceRepository, visited, manager));
} catch (ProvisionException e) {
//can't use then...
}
}
}
}
return stream;
}
return Stream.empty();
}

private boolean isEnabled(IRepositoryReference reference) {
return (reference.getOptions() & IRepository.ENABLED) != 0;
}

public void setIncludeSources(boolean includeAllSource, TargetPlatform targetPlatform) {
this.includeAllSource = includeAllSource;
this.targetPlatform = targetPlatform;
Expand Down

0 comments on commit f53ebeb

Please sign in to comment.