diff --git a/.gitignore b/.gitignore index 0138782..5785898 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,3 @@ buildNumber.properties # IntelliJ .idea/ *.iml - diff --git a/pom.xml b/pom.xml index 92af0df..30de8a5 100644 --- a/pom.xml +++ b/pom.xml @@ -35,6 +35,8 @@ 5.22.0 2.5.0 1.20.3 + 0.12.3 + 1.79 5.11.3 3.26.3 @@ -168,6 +170,29 @@ testcontainers ${testcontainers.version} + + io.jsonwebtoken + jjwt-api + ${jjwt.version} + + + io.jsonwebtoken + jjwt-impl + ${jjwt.version} + runtime + + + io.jsonwebtoken + jjwt-jackson + ${jjwt.version} + runtime + + + org.bouncycastle + bcprov-jdk18on + ${bouncycastle.version} + + diff --git a/src/main/kotlin/com/github/lernejo/korekto/toolkit/GradingJob.kt b/src/main/kotlin/com/github/lernejo/korekto/toolkit/GradingJob.kt index 170c3e4..7a06771 100644 --- a/src/main/kotlin/com/github/lernejo/korekto/toolkit/GradingJob.kt +++ b/src/main/kotlin/com/github/lernejo/korekto/toolkit/GradingJob.kt @@ -68,6 +68,7 @@ class GradingJob( val failedSlugs = mutableListOf() val jobDurations = mutableListOf() for ((jobIndex, userSlug) in userSlugs.withIndex()) { + System.setProperty("github_user", userSlug) val gradingConfiguration = GradingConfiguration( repoUrlBuilder(userSlug), "", diff --git a/src/main/kotlin/com/github/lernejo/korekto/toolkit/launcher/GradingJobLauncher.kt b/src/main/kotlin/com/github/lernejo/korekto/toolkit/launcher/GradingJobLauncher.kt index 6f2fbc7..9f0a489 100644 --- a/src/main/kotlin/com/github/lernejo/korekto/toolkit/launcher/GradingJobLauncher.kt +++ b/src/main/kotlin/com/github/lernejo/korekto/toolkit/launcher/GradingJobLauncher.kt @@ -68,6 +68,7 @@ class GradingJobLauncher : Callable { 0 } slug.isPresent -> { + System.setProperty("github_user", slug.get()) val repoUrl = grader.slugToRepoUrl(slug.get()) val configuration = GradingConfiguration(repoUrl, "", "") val branch = System.getProperty("git.branch") diff --git a/src/main/kotlin/com/github/lernejo/korekto/toolkit/thirdparty/git/ExerciseCloner.kt b/src/main/kotlin/com/github/lernejo/korekto/toolkit/thirdparty/git/ExerciseCloner.kt index 4f8bce7..4264cb8 100644 --- a/src/main/kotlin/com/github/lernejo/korekto/toolkit/thirdparty/git/ExerciseCloner.kt +++ b/src/main/kotlin/com/github/lernejo/korekto/toolkit/thirdparty/git/ExerciseCloner.kt @@ -38,7 +38,7 @@ class ExerciseCloner(private val workspace: Path) { if (potentialRepo.isPresent) { if (!forcePull) { try { - forcePull(potentialRepo.get(), uri) + forcePull(potentialRepo.get()) } catch (e: RuntimeException) { throw RuntimeException("Could not pull -f repository: $path ($uri), ${e.message}", e) } diff --git a/src/main/kotlin/com/github/lernejo/korekto/toolkit/thirdparty/git/GitRepository.kt b/src/main/kotlin/com/github/lernejo/korekto/toolkit/thirdparty/git/GitRepository.kt index 6a44b60..4a2036a 100644 --- a/src/main/kotlin/com/github/lernejo/korekto/toolkit/thirdparty/git/GitRepository.kt +++ b/src/main/kotlin/com/github/lernejo/korekto/toolkit/thirdparty/git/GitRepository.kt @@ -1,6 +1,7 @@ package com.github.lernejo.korekto.toolkit.thirdparty.git import com.github.lernejo.korekto.toolkit.WarningException +import com.github.lernejo.korekto.toolkit.thirdparty.github.GitHubAuthenticationHolder import org.eclipse.jgit.api.Git import org.eclipse.jgit.api.ResetCommand import org.eclipse.jgit.api.errors.GitAPIException @@ -8,7 +9,6 @@ import org.eclipse.jgit.api.errors.InvalidRemoteException import org.eclipse.jgit.api.errors.JGitInternalException import org.eclipse.jgit.api.errors.NoHeadException import org.eclipse.jgit.storage.file.FileRepositoryBuilder -import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider import org.slf4j.LoggerFactory import java.io.IOException import java.io.UncheckedIOException @@ -20,7 +20,6 @@ object GitRepository { private val LOGGER = LoggerFactory.getLogger(GitRepository::class.java) private val URI_WITH_CRED_PATTERN = Pattern.compile("(?https?://)(?[^@]+)@(?.+)") - private val URI_WITHOUT_CRED_PATTERN = Pattern.compile("(?https?://)(?.+)") data class Creds(val uriWithoutCred: String, val username: String?, val password: String?) @@ -51,42 +50,22 @@ object GitRepository { } } - private fun insertToken(uri: String, token: String): String { - val uriCredMatcher = URI_WITH_CRED_PATTERN.matcher(uri) - return if (uriCredMatcher.matches()) { - uri - } else { - val uriMatcher = URI_WITHOUT_CRED_PATTERN.matcher(uri) - if (uriMatcher.matches()) { - uriMatcher.group("protocol") + "x-access-token:" + token + "@" + uriMatcher.group("hostAndMore") - } else { - uri - } - } - } - @JvmStatic fun clone(uri: String, path: Path): Git { - val creds = extractCredParts(uri) - return try { val cloneCommand = Git.cloneRepository() .setURI(uri) .setDirectory(path.toFile()) - val token = System.getProperty("github_token") - if (creds.username != null) { - cloneCommand.setCredentialsProvider(UsernamePasswordCredentialsProvider(creds.username, creds.password)) - } else if (token != null) { - cloneCommand - .setURI(insertToken(uri, token)) - .setCredentialsProvider(UsernamePasswordCredentialsProvider(token, "")) - } + cloneCommand + .setURI(uri) + GitHubAuthenticationHolder.auth.configure(cloneCommand) + val git = cloneCommand .setCloneAllBranches(true) .call() LOGGER.debug("Cloning in: " + git.repository.directory) git - } catch(e: InvalidRemoteException) { + } catch (e: InvalidRemoteException) { throw WarningException("Unable to clone in ${path.toAbsolutePath()}: Missing or inaccessible repository", e) } catch (e: GitAPIException) { throw buildWarningException(path, e) @@ -122,38 +101,12 @@ object GitRepository { } } - internal fun extractCreds(uri: String): Creds { - val token = System.getProperty("github_token") - return if (token != null) { - Creds(uri, token, "") - } else { - extractCredParts(uri) - } - } - @JvmStatic - fun forcePull(git: Git, uri: String) { - val creds = extractCreds(uri) - - try { - try { - forcePull(git, creds, "origin/master") - } catch (e: JGitInternalException) { - forcePull(git, creds, "origin/main") - } - } catch (e: GitAPIException) { - throw RuntimeException(e) - } - } - - fun forcePull(git: Git, creds: Creds, ref: String) { + fun forcePull(git: Git) { val fetchCommand = git.fetch() .setForceUpdate(true) - if (creds.username != null) { - fetchCommand.setCredentialsProvider(UsernamePasswordCredentialsProvider(creds.username, creds.password)) - } - fetchCommand - .call() + GitHubAuthenticationHolder.auth.configure(fetchCommand) + fetchCommand.call() git.reset().setMode(ResetCommand.ResetType.HARD).call() git.clean().setCleanDirectories(true).setForce(true).call() git.pull().call() diff --git a/src/main/kotlin/com/github/lernejo/korekto/toolkit/thirdparty/github/GitHubAuthenticationHolder.kt b/src/main/kotlin/com/github/lernejo/korekto/toolkit/thirdparty/github/GitHubAuthenticationHolder.kt new file mode 100644 index 0000000..e07f712 --- /dev/null +++ b/src/main/kotlin/com/github/lernejo/korekto/toolkit/thirdparty/github/GitHubAuthenticationHolder.kt @@ -0,0 +1,167 @@ +package com.github.lernejo.korekto.toolkit.thirdparty.github + +import io.jsonwebtoken.JwtBuilder +import io.jsonwebtoken.Jwts +import io.jsonwebtoken.security.SecureDigestAlgorithm +import org.bouncycastle.jce.provider.BouncyCastleProvider +import org.eclipse.jgit.api.GitCommand +import org.eclipse.jgit.api.TransportCommand +import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider +import org.kohsuke.github.GitHubBuilder +import org.testcontainers.shaded.org.bouncycastle.openssl.PEMKeyPair +import org.testcontainers.shaded.org.bouncycastle.openssl.PEMParser +import org.testcontainers.shaded.org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter +import java.io.File +import java.io.FileReader +import java.net.HttpURLConnection +import java.security.PrivateKey +import java.security.PublicKey +import java.security.Security +import java.time.Instant +import java.util.* + +object GitHubAuthenticationHolder { + val auth: GitHubAuthentication by lazy { + val token = System.getProperty("github_token") + val appId = System.getProperty("github_app_id") + val appPk = System.getProperty("github_app_pk") + if (token != null) { + TokenGitHubAuthentication(token) + } else if (appId != null && appPk != null) { + AppGitHubAuthentication(appId, appPk) + } else { + NoopTokenGitHubAuthentication() + } + } +} + +interface GitHubAuthentication { + val type: String + fun > configure(command: TransportCommand) + fun configure(conn: HttpURLConnection) + fun configure(builder: GitHubBuilder) +} + +class NoopTokenGitHubAuthentication : GitHubAuthentication { + override val type = "noop" + + override fun > configure(command: TransportCommand) { + } + + override fun configure(conn: HttpURLConnection) { + } + + override fun configure(builder: GitHubBuilder) { + } +} + +data class TokenGitHubAuthentication(val token: String) : GitHubAuthentication { + override val type = "token" + + override fun > configure(command: TransportCommand) { + command + .setCredentialsProvider(UsernamePasswordCredentialsProvider(token, "")) + } + + override fun configure(conn: HttpURLConnection) { + conn.setRequestProperty("Authorization", "token $token") + } + + override fun configure(builder: GitHubBuilder) { + builder.withOAuthToken(token) + } +} + +class AppGitHubAuthentication(private val appId: String, appPk: String) : GitHubAuthentication { + override val type = "app-$appId (installation: " + getToken().installationId + ")" + + companion object { + init { + Security.removeProvider("BC") //remove old/legacy Android-provided BC provider + Security.addProvider(BouncyCastleProvider()) // add 'real'/correct BC provider + } + } + + private val privateKey = readPrivateKey(appPk) + private var jwt: Jwt? = null + private val tokensByUser: MutableMap = mutableMapOf() + + override fun > configure(command: TransportCommand) { + command.setCredentialsProvider(UsernamePasswordCredentialsProvider("x-access-token", getToken().token)) + } + + override fun configure(conn: HttpURLConnection) { + val token = getToken().token + conn.setRequestProperty("Authorization", "Bearer $token") + } + + override fun configure(builder: GitHubBuilder) { + builder.withAppInstallationToken(getToken().token) + } + + private fun getToken(): InstallationToken { + val user = System.getProperty("github_user") ?: throw IllegalStateException("Missing github_user env prop") + val token = tokensByUser[user] + if (token == null || token.isExpired()) { + tokensByUser[user] = refreshToken(user) + } + return tokensByUser[user]!! + } + + private fun refreshToken(user: String): InstallationToken { + val jwt = getJwt() + val gitHubApp = GitHubBuilder().withJwtToken(jwt).build() + val installation = gitHubApp.app.getInstallationByUser(user) + + val tokenResponse = installation.createToken().create() + + return InstallationToken(tokenResponse.token, installation.id, tokenResponse.expiresAt.toInstant()) + } + + private fun getJwt(): String { + if (jwt == null || jwt!!.isExpired()) { + jwt = createJWT(appId, 590000, privateKey) + } + return jwt!!.token + } +} + +class Jwt(val token: String, private val start: Long, private val duration: Long) { + fun isExpired() = (start + duration - 10_000) > System.currentTimeMillis() +} + +class InstallationToken(val token: String, val installationId: Long, private val expiresAt: Instant) { + fun isExpired() = expiresAt.isAfter(Instant.now().minusSeconds(60L)) +} + +fun readPrivateKey(filename: String): PrivateKey { + val pemParser = PEMParser(FileReader(File(filename))) + val o: PEMKeyPair = pemParser.readObject() as PEMKeyPair + val converter = JcaPEMKeyConverter().setProvider("BC") + val kp = converter.getKeyPair(o) + return kp.private +} + +fun createJWT(githubAppId: String, ttlMillis: Long, privateKey: PrivateKey): Jwt { + //The JWT signature algorithm we will be using to sign the token + val signatureAlgorithm: SecureDigestAlgorithm = Jwts.SIG.RS256 + + val nowMillis = System.currentTimeMillis() + val now = Date(nowMillis) + + //Let's set the JWT Claims + val builder: JwtBuilder = Jwts.builder() + .issuedAt(now) + .issuer(githubAppId) + .signWith(privateKey, signatureAlgorithm) + + //if it has been specified, let's add the expiration + if (ttlMillis > 0) { + val expMillis = nowMillis + ttlMillis + val exp = Date(expMillis) + builder.expiration(exp) + } + + //Builds the JWT and serializes it to a compact, URL-safe string + return Jwt(builder.compact(), nowMillis, ttlMillis) +} diff --git a/src/main/kotlin/com/github/lernejo/korekto/toolkit/thirdparty/github/GitHubNature.kt b/src/main/kotlin/com/github/lernejo/korekto/toolkit/thirdparty/github/GitHubNature.kt index 14679e5..716c649 100644 --- a/src/main/kotlin/com/github/lernejo/korekto/toolkit/thirdparty/github/GitHubNature.kt +++ b/src/main/kotlin/com/github/lernejo/korekto/toolkit/thirdparty/github/GitHubNature.kt @@ -1,20 +1,16 @@ package com.github.lernejo.korekto.toolkit.thirdparty.github -import com.github.lernejo.korekto.toolkit.Exercise -import com.github.lernejo.korekto.toolkit.Nature -import com.github.lernejo.korekto.toolkit.NatureContext -import com.github.lernejo.korekto.toolkit.NatureFactory -import com.github.lernejo.korekto.toolkit.objectMapper; +import com.github.lernejo.korekto.toolkit.* import okhttp3.OkHttpClient import org.eclipse.jgit.api.Git import org.kohsuke.github.GHRepository import org.kohsuke.github.GitHub import org.kohsuke.github.GitHubBuilder -import org.kohsuke.github.extras.okhttp3.OkHttpConnector +import org.kohsuke.github.extras.okhttp3.OkHttpGitHubConnector import org.slf4j.LoggerFactory import java.io.IOException import java.net.HttpURLConnection -import java.net.URL +import java.net.URI import java.nio.charset.StandardCharsets import java.util.* import java.util.concurrent.TimeUnit @@ -25,17 +21,14 @@ internal object GitHubClientHolder { val client: GitHub by lazy { val builder = GitHubBuilder() .withConnector( - OkHttpConnector( + OkHttpGitHubConnector( OkHttpClient.Builder() .readTimeout(2L, TimeUnit.SECONDS) .build() ) ) - val token = System.getProperty("github_token") - if (token != null) { - builder.withOAuthToken(token) - } - logger.debug("[gh-client] Creating the GitHub client" + if (token != null) " (using token)" else " (public)") + GitHubAuthenticationHolder.auth.configure(builder) + logger.debug("[gh-client] Creating the GitHub client (type: " + GitHubAuthenticationHolder.auth.type + ")") builder.build() } } @@ -45,12 +38,9 @@ class GitHubNature(val context: GitHubContext) : Nature { fun listActionRuns(): List { val requestURL = "https://api.github.com/repos/${context.exerciseName}/actions/runs" - val url = URL(requestURL) + val url = URI(requestURL).toURL() val conn: HttpURLConnection = url.openConnection() as HttpURLConnection - val token = System.getProperty("github_token") - if (token != null) { - conn.setRequestProperty("Authorization", "token $token") - } + GitHubAuthenticationHolder.auth.configure(conn) return conn.inputStream.use { `is` -> Scanner(`is`, StandardCharsets.UTF_8).use { scanner -> @@ -61,7 +51,10 @@ class GitHubNature(val context: GitHubContext) : Nature { } } +@Suppress("PropertyName") data class ActionRunsResponse(val workflow_runs: List) + +@Suppress("PropertyName") data class WorkflowRun( val name: String, val head_branch: String, @@ -69,14 +62,17 @@ data class WorkflowRun( val conclusion: WorkflowRunConclusion? ) +@Suppress("EnumEntryName", "unused") enum class WorkflowRunStatus { queued, in_progress, completed } +@Suppress("EnumEntryName", "unused") enum class WorkflowRunConclusion { action_required, cancelled, failure, neutral, success, skipped, stale, startup_failure, timed_out } +@Suppress("MemberVisibilityCanBePrivate") class GitHubContext(val gitHub: GitHub, val exerciseName: String) : NatureContext { val repository: GHRepository by lazy { logger.debug("[gh-client] Loading repository $exerciseName") diff --git a/src/test/kotlin/com/github/lernejo/korekto/toolkit/thirdparty/git/ExerciseClonerTest.kt b/src/test/kotlin/com/github/lernejo/korekto/toolkit/thirdparty/git/ExerciseClonerTest.kt index 7c0a67d..bf482c7 100644 --- a/src/test/kotlin/com/github/lernejo/korekto/toolkit/thirdparty/git/ExerciseClonerTest.kt +++ b/src/test/kotlin/com/github/lernejo/korekto/toolkit/thirdparty/git/ExerciseClonerTest.kt @@ -8,8 +8,9 @@ internal class ExerciseClonerTest { @Test @EnabledIfSystemProperty(named = "github_token", matches = ".+") internal fun sample_clone() { - var ex = ExerciseCloner(Paths.get("target/repositories")).gitClone( - "https://github.com/lernejo/git_training" + System.setProperty("github_user", "ledoyen") + val ex = ExerciseCloner(Paths.get("target/repositories")).gitClone( + "https://github.com/ledoyen/spring-todo-list" ) println(ex) }