Skip to content

Commit

Permalink
implement GH repo processing. Fixes #3, related to #1
Browse files Browse the repository at this point in the history
  • Loading branch information
andrewazores committed Aug 13, 2024
1 parent e1872e8 commit 9b4be74
Show file tree
Hide file tree
Showing 6 changed files with 226 additions and 44 deletions.
3 changes: 3 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@ status. If the dependency is not available it will also list available versions.
Given a GitHub Pull Request URL, the tool will attempt to use the [`gh`](https://github.com/cli/cli) tool to get the Pull Request title. If this meets the expected Dependabot
title format, the tool will extract the GAV from the title and act as if that GAV were specified directly.

Given a GitHub repository URL, the tool will attempt to use `gh` to get the repository's `pom.xml` from its default branch. Then it will use `mvn` to resolve all of the
project dependencies, and report on this list of GAVs.

## Building

`./mvnw clean package`
Expand Down
137 changes: 137 additions & 0 deletions src/main/java/com/github/andrewazores/GitHubRepositoryIntegration.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,137 @@
/*
* Copyright Andrew Azores.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.github.andrewazores;

import java.io.BufferedInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URL;
import java.nio.file.Files;
import java.util.List;
import java.util.regex.Pattern;

import io.quarkus.logging.Log;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import org.eclipse.microprofile.config.inject.ConfigProperty;

@ApplicationScoped
class GitHubRepositoryIntegration implements SourceIntegration {
private static final Pattern GH_REPO_PATTERN =
Pattern.compile(
"^https?://(?:www.)?github.com/(?<owner>[\\w.-]+)/(?<repo>[\\w.-]+)/?$",
Pattern.MULTILINE | Pattern.CASE_INSENSITIVE);
private static final Pattern DEP_PATTERN =
Pattern.compile(
"^[\\s]*(?<group>[a-z0-9._-]+):(?<artifact>[a-z0-9._-]+):(?<packaging>[a-z0-9._-]+):(?<version>[a-z0-9._-]+).*",
Pattern.MULTILINE | Pattern.CASE_INSENSITIVE);

@Inject CliSupport cli;

@ConfigProperty(name = "maven-gav-checker.integration.github-repository.transitive-deps")
boolean enableTransitiveDeps;

@Override
public boolean test(URL url) {
Log.debugv("Testing {0} on {1}", GH_REPO_PATTERN.pattern(), url.toString());
var m = GH_REPO_PATTERN.matcher(url.toString());
return m.matches();
}

@Override
public List<GroupArtifactVersion> apply(URL url) throws IOException, InterruptedException {
Log.debugv("Processing GitHub repository: {0}", url);
var m = GH_REPO_PATTERN.matcher(url.toString());
if (!m.matches()) throw new IllegalStateException();
var owner = m.group("owner");
var repo = m.group("repo");
var repoId = String.format("%s/%s", owner, repo);
var checkoutRef = getDefaultBranchRef(repoId);
var pomUrl = getPomUrl(repoId, checkoutRef);
var workDir = Files.createTempDirectory(getClass().getSimpleName());
var pom = workDir.resolve("pom.xml");
var depsFile = workDir.resolve("deps.txt");

try (BufferedInputStream in = new BufferedInputStream(pomUrl.openStream());
FileOutputStream fileOutputStream = new FileOutputStream(pom.toFile())) {
var dataBuffer = new byte[8 * 1024];
int bytesRead;
while ((bytesRead = in.read(dataBuffer, 0, 1024)) != -1) {
fileOutputStream.write(dataBuffer, 0, bytesRead);
}

Log.debug(Files.readString(pom));

var proc =
cli.script(
"mvn",
"-B",
"-q",
"-Dsilent",
"-DincludeScope=compile",
String.format("-DexcludeTransitive=%b", !enableTransitiveDeps),
"-DincludeParents",
"-Dmdep.outputScope=false",
"-DoutputFile=" + depsFile.toAbsolutePath().toString(),
"-f",
pom.toAbsolutePath().toString(),
"dependency:list");
proc.assertOk();
return Files.readAllLines(depsFile).stream()
.peek(l -> Log.tracev("dependency: {0}", l))
.filter(s -> DEP_PATTERN.matcher(s).matches())
.map(
s -> {
var m2 = DEP_PATTERN.matcher(s);
if (!m2.matches()) throw new IllegalStateException();
return new GroupArtifactVersion(
m2.group("group"),
m2.group("artifact"),
m2.group("version"));
})
.toList();
} finally {
Files.deleteIfExists(pom);
Files.deleteIfExists(depsFile);
}
}

private String getDefaultBranchRef(String repo) throws IOException, InterruptedException {
var proc =
cli.script(
"gh",
"repo",
"view",
"--json",
"defaultBranchRef",
"--jq",
".defaultBranchRef.name",
repo);
proc.assertOk();
return proc.out().get(0);
}

private URL getPomUrl(String repo, String checkoutRef) {
try {
return new URL(
String.format(
"https://raw.githubusercontent.com/%s/%s/pom.xml", repo, checkoutRef));
} catch (Exception e) {
e.printStackTrace();
throw new RuntimeException(e);
}
}
}
27 changes: 20 additions & 7 deletions src/main/java/com/github/andrewazores/Main.java
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import javax.net.ssl.X509TrustManager;

import io.quarkus.arc.All;
import io.quarkus.logging.Log;
import jakarta.inject.Inject;
import org.eclipse.microprofile.config.inject.ConfigProperty;
import picocli.CommandLine;
Expand Down Expand Up @@ -67,9 +68,9 @@ public class Main implements Callable<Integer> {
description =
"The Maven repository root URL to search, ex."
+ " https://repo.maven.apache.org/maven2/ . If the configuration property"
+ " gav-checker.maven-repository.url (or the environment variable"
+ " GAV_CHECKER_MAVEN_REPOSITORY_URL) is set, that will take precedence"
+ " over this option.",
+ " maven-gav-checker.maven-repository.url (or the environment variable"
+ " MAVEN_GAV_CHECKER_MAVEN_REPOSITORY_URL) is set, that will take"
+ " precedence over this option.",
defaultValue = "https://repo.maven.apache.org/maven2/")
private String repoRoot;

Expand All @@ -88,9 +89,10 @@ public class Main implements Callable<Integer> {
names = {"-k", "--insecure"},
description =
"Disable TLS validation on the remote Maven repository. This can also be set"
+ " with the configuration property"
+ " gav-checker.maven-repository.skip-tls-validation (or the environment"
+ " variable GAV_CHECKER_MAVEN_REPOSITORY_SKIP_TLS_VALIDATION).",
+ " with the configuration property"
+ " maven-gav-checker.maven-repository.skip-tls-validation (or the"
+ " environment variable"
+ " MAVEN_GAV_CHECKER_MAVEN_REPOSITORY_SKIP_TLS_VALIDATION).",
defaultValue = "false")
private boolean insecure;

Expand Down Expand Up @@ -119,11 +121,21 @@ public Integer call() throws Exception {
Collection<GroupArtifactVersion> gavs = new CopyOnWriteArrayList<>();
try {
var url = new URL(gav);
boolean matched = false;
for (var integration : sourceIntegrations) {
if (integration.test(url)) {
boolean applies = integration.test(url);
matched |= applies;
Log.debugv(
"integration {0} applies to {1} -> {2}",
integration.getClass().getName(), url, applies);
if (applies) {
gavs.addAll(integration.apply(url));
Log.trace(gavs.toString());
}
}
if (!matched) {
throw new RuntimeException("No matching integrations found for provided URL");
}
} catch (IOException | InterruptedException mue) {
var matcher = GAV_PATTERN.matcher(gav);
if (!matcher.matches()) {
Expand All @@ -135,6 +147,7 @@ public Integer call() throws Exception {
gavs.add(new GroupArtifactVersion(groupId, artifactId, version));
}
}
Log.tracev("Processing GAVs: {0}", gavs);
return processor.execute(gavs, repoRoot, count).get();
}

Expand Down
42 changes: 7 additions & 35 deletions src/main/java/com/github/andrewazores/MavenVersioning.java
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@
import javax.xml.parsers.ParserConfigurationException;

import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;

public record MavenVersioning(String latest, String release, List<String> versions) {
Expand All @@ -47,49 +46,22 @@ public static MavenVersioning from(String uri)

var root = xmlDoc.getDocumentElement();

var versioning = getChild(root, "versioning").orElseThrow();
var latest = getChild(versioning, "latest").map(Node::getTextContent).orElse("N/A");
var release = getChild(versioning, "release").map(Node::getTextContent).orElse("N/A");
var versions = getChild(versioning, "versions").orElseThrow();
var versioning = XmlParser.getChild(root, "versioning").orElseThrow();
var latest =
XmlParser.getChild(versioning, "latest").map(Node::getTextContent).orElse("N/A");
var release =
XmlParser.getChild(versioning, "release").map(Node::getTextContent).orElse("N/A");
var versions = XmlParser.getChild(versioning, "versions").orElseThrow();
var versionList =
new ArrayList<>(
getChildren(versions, "version").stream()
XmlParser.getChildren(versions, "version").stream()
.map(Node::getTextContent)
.toList());
Collections.reverse(versionList);

return new MavenVersioning(latest, release, versionList);
}

private static Optional<Node> getChild(Node parent, String childName) {
NodeList list = parent.getChildNodes();
int idx = 0;
while (idx < list.getLength()) {
var node = list.item(idx);
var name = node.getNodeName();
if (childName.equals(name)) {
return Optional.of(node);
}
idx++;
}
return Optional.empty();
}

private static List<Node> getChildren(Node parent, String childName) {
List<Node> out = new ArrayList<>();
NodeList list = parent.getChildNodes();
int idx = 0;
while (idx < list.getLength()) {
var node = list.item(idx);
var name = node.getNodeName();
if (childName.equals(name)) {
out.add(node);
}
idx++;
}
return out;
}

private static boolean versionCompare(String request, String found) {
return found.startsWith(String.format("%s-", request))
|| found.startsWith(String.format("%s.", request));
Expand Down
55 changes: 55 additions & 0 deletions src/main/java/com/github/andrewazores/XmlParser.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
/*
* Copyright Andrew Azores.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.github.andrewazores;

import java.util.ArrayList;
import java.util.List;
import java.util.Optional;

import org.w3c.dom.Node;
import org.w3c.dom.NodeList;

class XmlParser {

static Optional<Node> getChild(Node parent, String childName) {
NodeList list = parent.getChildNodes();
int idx = 0;
while (idx < list.getLength()) {
var node = list.item(idx);
var name = node.getNodeName();
if (childName.equals(name)) {
return Optional.of(node);
}
idx++;
}
return Optional.empty();
}

static List<Node> getChildren(Node parent, String childName) {
List<Node> out = new ArrayList<>();
NodeList list = parent.getChildNodes();
int idx = 0;
while (idx < list.getLength()) {
var node = list.item(idx);
var name = node.getNodeName();
if (childName.equals(name)) {
out.add(node);
}
idx++;
}
return out;
}
}
6 changes: 4 additions & 2 deletions src/main/resources/application.properties
Original file line number Diff line number Diff line change
Expand Up @@ -12,5 +12,7 @@ quarkus.native.enable-https-url-handler=true
quarkus.native.container-build=true
quarkus.native.container-runtime=podman-rootless

gav-checker.maven-repository.url=
gav-checker.maven-repository.skip-tls-validation=
maven-gav-checker.maven-repository.url=
maven-gav-checker.maven-repository.skip-tls-validation=

maven-gav-checker.integration.github-repository.transitive-deps=false

0 comments on commit 9b4be74

Please sign in to comment.