diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b7dda07..7873aed 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,5 +19,12 @@ jobs: steps: - uses: actions/checkout@v2 + - name: Set up JDK 11 # it auto caches https://github.com/actions/setup-java#caching-packages-dependencies + uses: actions/setup-java@v3.11.0 + with: + java-version: '11' + distribution: 'temurin' + cache: 'sbt' + - name: Run tests - run: SBT_VERSION="${{ matrix.sbt_version }}" make test + run: sbt scalafmtAll scripted diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index aba3ec0..e709c3b 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -2,3 +2,51 @@ name: Publish on: workflow_dispatch: + +# can maybe test via https://github.com/nektos/act +# act -s DOCKER_USERNAME=abc -s DOCKER_PASSWORD=xyz --container-architecture linux/arm64 -W .github/workflows/release.yml release +jobs: + release: + runs-on: ubuntu-20.04 + steps: + - name: Dump GitHub context + env: + GITHUB_CONTEXT: ${{ toJson(github) }} + run: echo "$GITHUB_CONTEXT" + + - name: Configure git user to push + run: | + git config --global user.email "sbt@tubi.tv" + git config --global user.name "sbt" + + - name: Clone and checkout to current branch + uses: actions/checkout@v3.5.2 + with: + fetch-depth: 0 + + - name: Set up JDK 11 # it auto caches https://github.com/actions/setup-java#caching-packages-dependencies + uses: actions/setup-java@v3.11.0 + with: + java-version: '11' + distribution: 'temurin' + cache: 'sbt' + + - name: Publish + run: ./scripts/github-actions/publish.sh + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ARTIFACTORY_USERNAME: ${{ secrets.ARTIFACTORY_USERNAME }} + ARTIFACTORY_PASSWORD: ${{ secrets.ARTIFACTORY_PASSWORD }} + +# - name: notify build status +# if: always() +# uses: slackapi/slack-github-action@v1.23.0 +# with: +# # Slack channel id, channel name, or user id to post message. +# # See also: https://api.slack.com/methods/chat.postMessage#channels +# # You can pass in multiple channels to post to by providing a comma-delimited list of channel IDs. +# channel-id: ${{ github.event.repository.name }}-cicd +# # For posting a simple plain text message, no md just for link shortening +# slack-message: "${{ steps.build_info.outputs.tubi-project-name }}-v${{ steps.build_info.outputs.tubi-project-version }} release ${{ job.status }}: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" +# env: +# SLACK_BOT_TOKEN: ${{ secrets.BUILD_NOTIFY_SLACK_APP_TOKEN }} diff --git a/.gitignore b/.gitignore index 93f7430..34a77d8 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ /project/project/ /project/target/ /target/ +/.idea/ +/.bsp/ diff --git a/.scalafmt.conf b/.scalafmt.conf new file mode 100644 index 0000000..3f81bd6 --- /dev/null +++ b/.scalafmt.conf @@ -0,0 +1,11 @@ +version = "3.4.3" +runner.dialect = scala213 +preset = IntelliJ +maxColumn = 120 +docstrings.style = SpaceAsterisk +docstrings.wrap = "no" +newlines.beforeCurlyLambdaParams = multiline +rewrite.rules = [Imports] +rewrite.imports.sort = scalastyle +rewrite.imports.groups = [["sbt\\..*"], ["java\\..*", "javax\\..*"], ["scala\\..*"]] +rewrite.trailingCommas.style = keep diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 3d36705..0000000 --- a/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM debian:buster-slim - -RUN apt-get update -RUN apt-get -y install locales-all - -ENV LANG ja_JP.UTF-8 -ENV LANGUAGE ja_JP:ja -ENV LC_ALL ja_JP.UTF-8 - -RUN apt-get update && \ - apt-get install -y build-essential \ - openjdk-11-jdk \ - curl -RUN apt-get install -y postgresql - -CMD "/bin/bash" diff --git a/Makefile b/Makefile deleted file mode 100644 index d523f7c..0000000 --- a/Makefile +++ /dev/null @@ -1,20 +0,0 @@ -.PHONY: test up down - -test: down up - if [ "$(SBT_VERSION)" == "" ]; then \ - echo "SBT_VERSION is not set"; \ - exit 1 ; \ - fi - docker-compose exec -T scala ./sbt ^^$(SBT_VERSION) test:compile clean - docker-compose exec -T scala ./sbt ^^$(SBT_VERSION) scripted - -up: - docker-compose build - docker-compose up -d - -down: - docker-compose down - -sbt: - curl -Ls https://git.io/sbt > sbt - chmod +x ./sbt diff --git a/README.md b/README.md index a11c205..28c1a1f 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![CI](https://github.com/tototoshi/sbt-slick-codegen/actions/workflows/ci.yml/badge.svg)](https://github.com/tototoshi/sbt-slick-codegen/actions/workflows/ci.yml) -slick-codegen compile hook for sbt +slick-codegen plugin, forked from [sbt-slick-codegen](https://github.com/tototoshi/sbt-slick-codegen), which aims to generate code specially for Postgres ## Install diff --git a/build.sbt b/build.sbt index 5cf5277..d15bdfe 100644 --- a/build.sbt +++ b/build.sbt @@ -1,69 +1,28 @@ -import scalariform.formatter.preferences._ -import scala.collection.JavaConverters._ import java.lang.management.ManagementFactory -enablePlugins(SbtPlugin) +import scala.collection.JavaConverters.* +//import scalariform.formatter.preferences.* -scalariformPreferences := scalariformPreferences.value - .setPreference(AlignSingleLineCaseStatements, true) - .setPreference(DoubleIndentConstructorArguments, true) - .setPreference(DanglingCloseParenthesis, Preserve) +enablePlugins(SbtPlugin) sbtPlugin := true - name := """sbt-slick-codegen""" - -organization := "com.github.tototoshi" - -version := "2.0.0" - -crossSbtVersions := Seq("1.8.0") - -val slickVersion = SettingKey[String]("slickVersion") - -slickVersion := "3.3.3" +organization := "com.tubitv" libraryDependencies ++= Seq( - "com.typesafe.slick" %% "slick" % slickVersion.value, - "com.typesafe.slick" %% "slick-codegen" % slickVersion.value + "com.typesafe.slick" %% "slick" % Versions.slick, + "com.typesafe.slick" %% "slick-codegen" % Versions.slick, + "org.postgresql" % "postgresql" % Versions.postgresql, + "com.github.docker-java" % "docker-java" % Versions.dockerJava, ) +addSbtPlugin("io.github.davidmweber" % "flyway-sbt" % Versions.flywaySbt) -publishMavenStyle := true - -publishTo := { - val nexus = "https://oss.sonatype.org/" - if (version.value.trim.endsWith("SNAPSHOT")) Some("snapshots" at nexus + "content/repositories/snapshots") - else Some("releases" at nexus + "service/local/staging/deploy/maven2") -} - +publishTo := Some(if (isSnapshot.value) Repo.Jfrog.Tubins.sbtDev else Repo.Jfrog.Tubins.sbtRelease) +ThisBuild / versionScheme := Some("early-semver") Test / publishArtifact := false -pomExtra := - https://github.com/tototoshi/sbt-slick-codegen - - - Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0.html - repo - - - - git@github.com:tototoshi/sbt-slick-codegen - scm:git:git@github.com:tototoshi/sbt-slick-codegen.git - - - - tototoshi - Toshiyuki Takahashi - https://tototoshi.github.io - - - scriptedBufferLog := false -scriptedLaunchOpts ++= ManagementFactory.getRuntimeMXBean.getInputArguments.asScala.toList.filter(a => - Seq("-Xmx", "-Xms", "-XX", "-Dsbt.log.noformat").exists(a.startsWith) -) -scriptedLaunchOpts ++= Seq( - "-Dplugin.version=" + version.value, - "-Dslick.version=" + slickVersion.value +scriptedLaunchOpts ++= ManagementFactory.getRuntimeMXBean.getInputArguments.asScala.toList.filter( + a => Seq("-Xmx", "-Xms", "-XX", "-Dsbt.log.noformat").exists(a.startsWith) ) +scriptedLaunchOpts ++= Seq("-Dplugin.version=" + version.value, "-Dslick.version=" + Versions.slick) diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 07a5650..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,17 +0,0 @@ -version: '3' -services: - scala: - build: . - stdin_open: true - working_dir: $PWD - volumes: - - $PWD:$PWD - postgres: - environment: - - POSTGRES_DB=example - - POSTGRES_USER=test - - POSTGRES_PASSWORD=test - - POSTGRES_HOST_AUTH_METHOD=trust - image: "postgres:latest" - ports: - - "5432:5432" diff --git a/project/Repo.scala b/project/Repo.scala new file mode 100644 index 0000000..5938304 --- /dev/null +++ b/project/Repo.scala @@ -0,0 +1,22 @@ +import sbt.* + +object Repo { + + object Jfrog { + private val domain = "tubins.jfrog.io" + private val jFrogRoot = s"https://$domain" + + object Tubins { + private val pathPrefix = "tubins" + + lazy val sbtDev: MavenRepository = "sbt-dev" at s"$jFrogRoot/$pathPrefix/sbt-dev" + + lazy val sbtRelease: MavenRepository = "sbt-release" at s"$jFrogRoot/$pathPrefix/sbt-release" + + lazy val jvmSnapshot: MavenRepository = "jvm-snapshot" at s"$jFrogRoot/$pathPrefix/jvm-snapshots" + + lazy val jvm: MavenRepository = "jvm-release" at s"$jFrogRoot/$pathPrefix/jvm" + } + } + +} diff --git a/project/Versions.scala b/project/Versions.scala new file mode 100644 index 0000000..ac69567 --- /dev/null +++ b/project/Versions.scala @@ -0,0 +1,6 @@ +object Versions { + val slick = "3.4.0" + val postgresql = "42.6.0" + val dockerJava = "3.3.4" + val flywaySbt = "7.4.0" +} \ No newline at end of file diff --git a/project/artifactory.sbt b/project/artifactory.sbt new file mode 100644 index 0000000..c3af7a4 --- /dev/null +++ b/project/artifactory.sbt @@ -0,0 +1,23 @@ +ThisBuild / credentials ++= { + val logger = streams.value.log + if (sys.env.contains("ARTIFACTORY_USERNAME")) { + logger.info("spotted credential in env, will add to credentials") + Some( + Credentials( + "Artifactory Realm", + "tubins.jfrog.io", + sys.env.getOrElse("ARTIFACTORY_USERNAME", ""), + sys.env.getOrElse("ARTIFACTORY_PASSWORD", "") + ) + ) + } else { + Credentials.loadCredentials(Path.userHome / ".artifactory" / "credentials") match { + case Right(credentials: DirectCredentials) => + logger.info(s"Using credentials found in the home directory for host ${credentials.host}") + Some(credentials) + case Left(err: String) => + logger.warn(s"Could not find artifactory credentials in home directory: $err") + None + } + } +} \ No newline at end of file diff --git a/project/plugins.sbt b/project/plugins.sbt index b6cb072..820d270 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,3 +1,3 @@ -addSbtPlugin("org.scalariform" % "sbt-scalariform" % "1.8.3") -addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "3.9.18") -addSbtPlugin("com.github.sbt" % "sbt-pgp" % "2.2.1") +addSbtPlugin("com.github.sbt" % "sbt-dynver" % "5.0.1") +addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.5.2") + diff --git a/src/main/scala/com/github/tototoshi/sbt/slick/CodegenPlugin.scala b/src/main/scala/com/github/tototoshi/sbt/slick/CodegenPlugin.scala index 1796156..409b7de 100644 --- a/src/main/scala/com/github/tototoshi/sbt/slick/CodegenPlugin.scala +++ b/src/main/scala/com/github/tototoshi/sbt/slick/CodegenPlugin.scala @@ -1,29 +1,38 @@ package com.github.tototoshi.sbt.slick -import sbt._ -import Keys._ -import slick.codegen.SourceCodeGenerator -import slick.jdbc.JdbcProfile -import slick.{ model => m } +import sbt.* +import scala.collection.mutable.ListBuffer import scala.concurrent.Await import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.duration.Duration -import scala.collection.mutable.ListBuffer +import scala.io.Source +import scala.util.Using -object CodegenPlugin extends sbt.AutoPlugin { +import Keys.* +import com.tubitv.{CodeGenConfig, CodeGenPostgresProfile, CustomizeSourceCodeGenerator, PostgresContainer} +import slick.{model => m} +import slick.codegen.SourceCodeGenerator +import slick.jdbc.JdbcProfile +import slick.jdbc.meta.MTable - object autoImport { - lazy val slickCodegen: TaskKey[Seq[File]] = taskKey[Seq[File]]("Command to run codegen") +object CodegenPlugin extends sbt.AutoPlugin with PluginDBSupport { + private val createdConfigs_ = + settingKey[Set[Configuration]]("The configurations for the config, other than default").withRank(KeyRanks.Invisible) + private val generators_ = settingKey[() => SourceCodeGenerator]( + "The setting to create the generator, to avoid the boilerplate code" + ).withRank(KeyRanks.Invisible) - lazy val slickCodegenDatabaseUrl: SettingKey[String] = - settingKey[String]("URL of database used by codegen") + private val generateCode_ = taskKey[Seq[File]]("Generate the code, without starting db").withRank(KeyRanks.Invisible) + private val verifyCode_ = taskKey[Unit]("Verify the generated code, without starting db").withRank(KeyRanks.Invisible) - lazy val slickCodegenDatabaseUser: SettingKey[String] = - settingKey[String]("User of database used by codegen") + object autoImport { + lazy val slickCodegen: TaskKey[Seq[File]] = taskKey[Seq[File]]("Command to run codegen") + lazy val slickCodegenAll: TaskKey[Unit] = taskKey[Unit]("Command to run all the codegen") + lazy val slickCodegenVerifyAll: TaskKey[Unit] = taskKey("Verify any of the generated code is out of date") - lazy val slickCodegenDatabasePassword: SettingKey[String] = - settingKey[String]("Password of database used by codegen") + lazy val slickCodegenGeneratorConfig: SettingKey[CodeGenConfig] = + settingKey("The configuration for the code generator") lazy val slickCodegenDriver: SettingKey[JdbcProfile] = settingKey[JdbcProfile]("Slick driver used by codegen") @@ -57,142 +66,155 @@ object CodegenPlugin extends sbt.AutoPlugin { "Tables that should be included. If this list is not nil, only the included tables minus excluded will be taken." ) - lazy val defaultSourceCodeGenerator: m.Model => SourceCodeGenerator = (model: m.Model) => - new SourceCodeGenerator(model) - - @deprecated("use enablePlugins(CodegenPlugin)", "") - lazy val slickCodegenSettings: Seq[Setting[_]] = projectSettings + /** Define a new configuration scope for slick code gen, + * for the cases where there are multiple codegen in a project + * @param configName the name of this config + * @param setting customized settings for the code gen + * @return + */ + def codeGen(configName: String)(setting: Setting[_]*): Seq[Setting[_]] = { + val theConfig = Configuration.of(configName.take(1).toUpperCase() + configName.drop(1), configName) + Seq(createdConfigs_ += theConfig) ++ inConfig(theConfig)(defaultConfigs ++ setting) + } } import autoImport._ - private def gen( - generator: m.Model => SourceCodeGenerator, - driver: JdbcProfile, - jdbcDriver: String, - url: String, - user: String, - password: String, - outputDir: String, - pkg: String, - fileName: String, - outputToMultipleFiles: Boolean, - container: String, - excluded: Seq[String], - included: Seq[String], - s: TaskStreams - ): Seq[File] = { - - val database = driver.api.Database.forURL(url = url, driver = jdbcDriver, user = user, password = password) - - try { - database.source.createConnection().close() - } catch { - case e: Throwable => - throw new RuntimeException("Failed to run slick-codegen: " + e.getMessage, e) - } - - s.log.info(s"Generate source code with slick-codegen: url=${url}, user=${user}") - - val tables = driver.defaultTables - .map(ts => ts.filter(t => included.isEmpty || (included contains t.name.name))) - .map(ts => ts.filterNot(t => excluded contains t.name.name)) + private def profile(p: JdbcProfile): String = { + val driverClassName = p.getClass.getName + // if it's a singleton object, then just reference it directly + if (driverClassName.endsWith("$")) driverClassName.stripSuffix("$") + // if it's an instance of a regular class, we don't know constructor args; try the no-arguments constructor and hope for the best + else s"new $driverClassName()" + } - val driverClassName = driver.getClass.getName - val profile = { - // if it's a singleton object, then just reference it directly - if (driverClassName.endsWith("$")) driverClassName.stripSuffix("$") - // if it's an instance of a regular class, we don't know constructor args; try the no-arguments constructor and hope for the best - else s"new $driverClassName()" - } + private def generate: Def.Initialize[Task[Seq[File]]] = Def.task { + val p = profile(slickCodegenDriver.value) + val outDir = { + val folder = slickCodegenOutputDir.value - val dbio = for { - m <- driver.createModel(Some(tables)) - } yield { - val sourceGen = generator(m) - if (outputToMultipleFiles) { - sourceGen.writeToMultipleFiles( - profile = profile, - folder = outputDir, - pkg = pkg, - container = container - ) + if (folder.exists()) { + require(folder.isDirectory, s"file :[$folder] is not a directory") } else { - sourceGen.writeToFile( - profile = profile, - folder = outputDir, - pkg = pkg, - container = container, - fileName = fileName - ) + folder.mkdir() } + folder.getPath } - Await.result(database.run(dbio), Duration.Inf) + val pkg = slickCodegenOutputPackage.value + val fileName = slickCodegenOutputFile.value + val container = slickCodegenOutputContainer.value + + val s = streams.value + val outputToMultipleFiles = slickCodegenOutputToMultipleFiles.value + val sourceGen = generators_.value() if (outputToMultipleFiles) { - val outDir = file(outputDir) - s.log.info(s"Source code files have been generated in ${outDir.getAbsolutePath}") - listScalaFileRecursively(outDir) + sourceGen.writeToMultipleFiles(profile = p, folder = outDir, pkg = pkg, container = container) + val outDirFile = file(outDir) + s.log.info(s"Source code files have been generated in ${outDirFile.getAbsolutePath}") + listScalaFileRecursively(outDirFile) } else { - val generatedFile = outputDir + "/" + pkg.replaceAllLiterally(".", "/") + "/" + fileName - s.log.info(s"Source code has generated in ${generatedFile}") - Seq(file(generatedFile)) + sourceGen.writeToFile(profile = p, folder = outDir, pkg = pkg, container = container, fileName = fileName) + val generatedFile = file(outDir + "/" + pkg.replaceAllLiterally(".", "/") + "/" + fileName) + s.log.info(s"Source code has generated in ${generatedFile.getAbsolutePath}") + Seq(generatedFile) } + } - override lazy val projectSettings: Seq[Setting[_]] = Seq( - slickCodegenDriver := slick.jdbc.PostgresProfile, + private def verify: Def.Initialize[Task[Unit]] = Def.task { + val p = profile(slickCodegenDriver.value) + val pkg = slickCodegenOutputPackage.value + val container = slickCodegenOutputContainer.value + + val theConf = configuration.?.value + val s = streams.value + + val sourceGen = generators_.value() + val theCode = sourceGen.packageCode(profile = p, pkg = pkg, container = container, sourceGen.parentType) + val file = slickCodegenOutputDir.value + "/" + pkg.replace(".", "/") + "/" + slickCodegenOutputFile.value + val generated = Using(Source.fromFile(file))(_.mkString).get + if (theCode.trim != generated.trim) { + throw new Exception( + s"Schema file: $file is out of date, please re-generate it by [ ${theConf.map(c => c.id + " / ").getOrElse("")}slickCodegen ]" + ) + } + s.log.info(s"Verify schema $file success") + + } + + private def defaultConfigs = Seq( + generators_ := { + val generator = slickCodegenCodeGenerator.value + val driver = slickCodegenDriver.value + val url = postgresDbUrl.value + val jdbcDriver = slickCodegenJdbcDriver.value + val excluded = slickCodegenExcludedTables.value + val included = slickCodegenIncludedTables.value + val database = driver.api.Database.forURL(url = url, driver = jdbcDriver, user = dbUser, password = dbPass) + + () => { + try { + database.source.createConnection().close() + } catch { + case e: Throwable => + throw new RuntimeException("Failed to run slick-codegen: " + e.getMessage, e) + } + + val tables = MTable + .getTables(None, None, None, Some(Seq("TABLE", "VIEW", "MATERIALIZED VIEW"))) + .map(ts => ts.filter(t => included.isEmpty || (included contains t.name.name))) + .map(ts => ts.filterNot(t => excluded contains t.name.name)) + + val dbio = for { + m <- driver.createModel(Some(tables)) + } yield generator(m) + + Await.result(database.run(dbio), Duration.Inf) + } + }, + slickCodegenGeneratorConfig := CodeGenConfig(), + slickCodegenDriver := new CodeGenPostgresProfile(slickCodegenGeneratorConfig.value), slickCodegenJdbcDriver := "org.postgresql.Driver", - slickCodegenDatabaseUrl := "Database url is not set", - slickCodegenDatabaseUser := "Database user is not set", - slickCodegenDatabasePassword := "Database password is not set", slickCodegenOutputPackage := "com.example", - slickCodegenOutputFile := "Tables.scala", + slickCodegenOutputFile := s"${slickCodegenOutputContainer.value}.scala", slickCodegenOutputToMultipleFiles := false, - slickCodegenOutputDir := (Compile / sourceManaged).value, + slickCodegenOutputDir := (Compile / Keys.scalaSource).value, slickCodegenOutputContainer := "Tables", slickCodegenExcludedTables := Seq(), slickCodegenIncludedTables := Seq(), - slickCodegenCodeGenerator := defaultSourceCodeGenerator, - slickCodegen := { - val outDir = { - val folder = slickCodegenOutputDir.value - if (folder.exists()) { - require(folder.isDirectory, s"file :[$folder] is not a directory") - } else { - folder.mkdir() - } - folder.getPath - } - val outPkg = (slickCodegenOutputPackage).value - val outFile = (slickCodegenOutputFile).value - val outputToMultipleFiles = slickCodegenOutputToMultipleFiles.value - gen( - (slickCodegenCodeGenerator).value, - (slickCodegenDriver).value, - (slickCodegenJdbcDriver).value, - (slickCodegenDatabaseUrl).value, - (slickCodegenDatabaseUser).value, - (slickCodegenDatabasePassword).value, - outDir, - outPkg, - outFile, - outputToMultipleFiles, - slickCodegenOutputContainer.value, - slickCodegenExcludedTables.value, - slickCodegenIncludedTables.value, - streams.value - ) - } + slickCodegenCodeGenerator := { (m) => new CustomizeSourceCodeGenerator(m, slickCodegenGeneratorConfig.value) }, + generateCode_ := generate.value, + verifyCode_ := verify.value, + slickCodegen := withDb(generate).value ) + override lazy val projectSettings: Seq[Setting[_]] = + dbSettings ++ defaultConfigs ++ + Seq( + createdConfigs_ := Set.empty, + slickCodegenAll := withDb( + Def.taskDyn( + Def + .sequential(generateCode_ +: createdConfigs_.value.toList.map(c => c / generateCode_)) + ) + ).value, + slickCodegenVerifyAll := withDb( + Def.taskDyn( + Def + .sequential(verifyCode_ +: createdConfigs_.value.toList.map(c => c / verifyCode_)) + ) + ).value + ) + private def listScalaFileRecursively(dir: File): Seq[File] = { val buf = new ListBuffer[File]() def addFiles(d: File): Unit = { - d.listFiles().foreach { f => - if (f.isDirectory) { addFiles(f) } - else if (f.getName.endsWith(".scala")) { buf += f } + d.listFiles().foreach { + f => + if (f.isDirectory) { addFiles(f) } + else if (f.getName.endsWith(".scala")) { buf += f } } } addFiles(dir) diff --git a/src/main/scala/com/github/tototoshi/sbt/slick/PluginDBSupport.scala b/src/main/scala/com/github/tototoshi/sbt/slick/PluginDBSupport.scala new file mode 100644 index 0000000..24db30f --- /dev/null +++ b/src/main/scala/com/github/tototoshi/sbt/slick/PluginDBSupport.scala @@ -0,0 +1,63 @@ +package com.github.tototoshi.sbt.slick + +import sbt.* + +import _root_.io.github.davidmweber.FlywayPlugin +import com.tubitv.PostgresContainer + +trait PluginDBSupport { + + protected final val dbUser = "postgres" + protected final val dbPass = "password" + + protected val postgresDbUrl = settingKey[String]("The database urlt") + protected val stopDb = taskKey[Unit]("Start the postgres docker container and run flyway migrate") + protected val startDb = taskKey[Unit]("Stop and remove the postgres container") + + lazy val postgresContainerPort = settingKey[Int]("The port of postgres port") + lazy val postgresVersion = settingKey[String]("The postgres version") + + protected def dbSettings: Seq[sbt.Setting[_]] = { + import FlywayPlugin.autoImport._ + FlywayPlugin.projectSettings ++ + Seq( + flywayDefaults / Keys.logLevel := Level.Warn, + postgresDbUrl := s"jdbc:postgresql://127.0.0.1:${postgresContainerPort.value}/postgres", + postgresContainerPort := 15432, + postgresVersion := "13.7", + flywayUrl := postgresDbUrl.value, + flywayUser := dbUser, + flywayPassword := dbPass, + flywayLocations := Seq(s"filesystem:${(Compile / Keys.resourceDirectory).value.getAbsoluteFile}/db/migration"), + startDb := Def + .sequential( + Def.task { + PostgresContainer.start( + exportPort = postgresContainerPort.value, + password = dbPass, + postgresVersion = postgresVersion.value, + logger = Keys.streams.value.log + ) + }, + FlywayPlugin.autoImport.flywayMigrate + ) + .value, + stopDb := { + PostgresContainer.stop(Keys.streams.value.log) + } + ) + } + + /** Run the task with db ready , will start the postgres docker, and run flyway migrate before the task + * and also, it will make sure stop and remove the container after the task + * @param task the task to be executed + * @tparam A + * @return + */ + protected def withDb[A](task: Def.Initialize[Task[A]]): Def.Initialize[Task[A]] = Def.taskDyn { + task + .dependsOn(startDb) + .doFinally(stopDb.taskValue) + } + +} diff --git a/src/main/scala/com/tubitv/CodeGenConfig.scala b/src/main/scala/com/tubitv/CodeGenConfig.scala new file mode 100644 index 0000000..c35319e --- /dev/null +++ b/src/main/scala/com/tubitv/CodeGenConfig.scala @@ -0,0 +1,10 @@ +package com.tubitv + +case class CodeGenConfig( + byNameMapper: PartialFunction[(String, String), String] = PartialFunction.empty, + byTypeMapper: PartialFunction[String, String] = PartialFunction.empty, + ignoredColumns: ((String, String)) => Boolean = _ => false, + profile: String = "slick.jdbc.PostgresProfile", + // some extra imports need for the generated code + extraImports: Seq[String] = Seq.empty, +) diff --git a/src/main/scala/com/tubitv/CodeGenPostgresProfile.scala b/src/main/scala/com/tubitv/CodeGenPostgresProfile.scala new file mode 100644 index 0000000..bd43e2b --- /dev/null +++ b/src/main/scala/com/tubitv/CodeGenPostgresProfile.scala @@ -0,0 +1,57 @@ +package com.tubitv + +import java.sql.Types.{CHAR, LONGNVARCHAR, LONGVARCHAR, NCHAR, NVARCHAR, VARCHAR} + +import scala.concurrent.ExecutionContext + +import slick.jdbc.PostgresProfile +import slick.jdbc.meta._ + +class CodeGenPostgresProfile(config: CodeGenConfig) extends PostgresProfile { + + override def createModelBuilder(tables: Seq[MTable], ignoreInvalidDefaults: Boolean)(implicit + ec: ExecutionContext + ): slick.jdbc.JdbcModelBuilder = new ModelBuilder(tables, ignoreInvalidDefaults)(ec) { + + val typeMapper: PartialFunction[String, String] = config.byTypeMapper orElse { + case "name" | "text" | "varchar" => "String" + case "int4" | "serial" => "Int" + case "int2" | "smallserial" => "Short" + case "int8" | "bigserial" | "oid" => "Long" + case "bool" | "bit" => "Boolean" + } + + val arraryDetector: PartialFunction[String, String] = { + case a if a.startsWith("_") => a.substring(1) + } + + override def createColumnBuilder(tableBuilder: TableBuilder, meta: MColumn): ColumnBuilder = + new ColumnBuilder(tableBuilder, meta) { + override def tpe = + config.byNameMapper + .lift(meta.table.name -> meta.name) + .orElse(typeMapper.lift(meta.typeName)) + .orElse(arraryDetector.andThen(typeMapper).andThen(a => s"List[$a]").lift(meta.typeName)) + .getOrElse({ + val rt = super.tpe + if (rt == "String" && !isJdbcStringType(meta.sqlType)) { + logger.warn( + s"Column [${meta.name}] in table [${meta.table.name}] with type [${meta.typeName}] " + + s"does not have a custom mapping, so defaults map as [String], consider defining a custom type mapper" + ) + } + rt + }) + } + + override def readColumns(t: MTable): api.DBIO[Vector[MColumn]] = super.readColumns(t).map { + v => + v.filterNot(m => config.ignoredColumns(m.table.name, m.name)) + } + + private def isJdbcStringType(sqlType: Int) = sqlType match { + case CHAR | VARCHAR | LONGVARCHAR | NCHAR | NVARCHAR | LONGNVARCHAR => true + case _ => false + } + } +} diff --git a/src/main/scala/com/tubitv/CustomizeSourceCodeGenerator.scala b/src/main/scala/com/tubitv/CustomizeSourceCodeGenerator.scala new file mode 100644 index 0000000..d362b53 --- /dev/null +++ b/src/main/scala/com/tubitv/CustomizeSourceCodeGenerator.scala @@ -0,0 +1,28 @@ +package com.tubitv + +import slick.{model => m} +import slick.codegen.SourceCodeGenerator + +class CustomizeSourceCodeGenerator(model: m.Model, config: CodeGenConfig) extends SourceCodeGenerator(model) { + override def packageCode(profile: String, pkg: String, container: String, parentType: Option[String]): String = { + s""" + |package $pkg + | + |// format: off + |// AUTO-GENERATED Slick data model + |// scalastyle:off + |/** + | * Stand-alone Slick data model for immediate use + | * Please do not touch this file manually, use slick-codegen + | */ + |object $container extends { + | ${indent(config.extraImports.map(i => s"import ${i}").mkString("\n"))} + | + | val profile = ${config.profile} + | import profile.api._ + | + | ${indent(code)} + |}""".stripMargin.trim() + + } +} diff --git a/src/main/scala/com/tubitv/PostgresContainer.scala b/src/main/scala/com/tubitv/PostgresContainer.scala new file mode 100644 index 0000000..397336a --- /dev/null +++ b/src/main/scala/com/tubitv/PostgresContainer.scala @@ -0,0 +1,89 @@ +package com.tubitv + +import sbt.Logger + +import java.sql.DriverManager +import java.util.concurrent.atomic.AtomicReference + +import scala.concurrent.{Await, Promise} +import scala.concurrent.duration.DurationInt +import scala.util.{Try, Using} + +import com.github.dockerjava.api.async.ResultCallback +import com.github.dockerjava.api.exception.NotFoundException +import com.github.dockerjava.api.model.{PortBinding, PullResponseItem} +import com.github.dockerjava.core.DockerClientBuilder + +object PostgresContainer { + + private val runningDb = new AtomicReference[Option[String]](None) + + def start(exportPort: Int, password: String, postgresVersion: String, logger: Logger): Unit = { + runningDb.get() match { + case None => + val image = s"postgres:$postgresVersion" + val dockerClient = DockerClientBuilder.getInstance.build + Try(dockerClient.inspectImageCmd(image).exec()).recover { + case _: NotFoundException => + logger.info(s"Pulling image ${image} ...") + val done = Promise[Unit] + dockerClient + .pullImageCmd(image) + .exec(new ResultCallback.Adapter[PullResponseItem] { + override def onComplete(): Unit = done.success(()) + }) + + Await.result(done.future, 5.minutes) + logger.info(s"Image $image pull done") + } + + val container = dockerClient + .createContainerCmd(image) + .withEnv(s"POSTGRES_PASSWORD=$password") + + container.getHostConfig + .withPortBindings(PortBinding.parse(s"$exportPort:5432")) + + val containerId = container.exec().getId + if (runningDb.compareAndSet(None, Some(containerId))) { + logger.info(s"Starting docker container:${containerId.substring(0, 12)} [postgres:$postgresVersion]") + + dockerClient.startContainerCmd(containerId).exec() + + var isReady = false + val deadLine = 3.minutes.fromNow + while (!isReady) { + Class.forName("org.postgresql.Driver") + val url = s"jdbc:postgresql://127.0.0.1:${exportPort}/postgres" + Using(DriverManager.getConnection(url, "postgres", password))(_ => ()).toOption match { + case Some(_) => + isReady = true + case _ => + if (deadLine.isOverdue()) { + throw new Exception("Postgres container is not ready after 3 minutes") + } + Thread.sleep(1000) + } + } + logger.info("Docker container postgres started") + } + case _ => + } + } + + def stop(logger: Logger): Unit = { + runningDb.get() match { + case Some(id) => + val dockerClient = DockerClientBuilder.getInstance.build + try { + dockerClient.stopContainerCmd(id).exec() + } catch { + case _: Throwable => + } + dockerClient.removeContainerCmd(id).exec() + runningDb.set(None) + logger.info(s"Docker container [postgres] is stopped") + case _ => + } + } +} diff --git a/src/sbt-test/test/basic/build.sbt b/src/sbt-test/test/basic/build.sbt index 2057d74..ab3552e 100644 --- a/src/sbt-test/test/basic/build.sbt +++ b/src/sbt-test/test/basic/build.sbt @@ -1,17 +1,14 @@ + crossScalaVersions := Seq("2.12.15", "2.13.8") Global / onChangedBuildSource := ReloadOnSourceChanges -libraryDependencies += "com.typesafe.slick" %% "slick" % System.getProperty("slick.version") - enablePlugins(CodegenPlugin) -Compile / sourceGenerators += slickCodegen - -slickCodegenDatabaseUrl := "jdbc:postgresql://postgres/example" - -slickCodegenDatabaseUser := "test" - -slickCodegenDatabasePassword := "test" +slickCodegenOutputContainer := "Table" +slickCodegenOutputPackage := "com.demo" +//) -slickCodegenOutputToMultipleFiles := true +codeGen("etl")( + slickCodegenOutputContainer := "Etl", +) diff --git a/src/sbt-test/test/basic/project/plugins.sbt b/src/sbt-test/test/basic/project/plugins.sbt index 1a21b61..1e5f998 100644 --- a/src/sbt-test/test/basic/project/plugins.sbt +++ b/src/sbt-test/test/basic/project/plugins.sbt @@ -1,5 +1,5 @@ sys.props.get("plugin.version") match { - case Some(x) => addSbtPlugin("com.github.tototoshi" % "sbt-slick-codegen" % x) + case Some(x) => addSbtPlugin("com.tubitv" % "sbt-slick-codegen" % x) case _ => sys.error("""|The system property 'plugin.version' is not defined. |Specify this property using the scriptedLaunchOpts -D.""".stripMargin) } diff --git a/src/sbt-test/test/basic/src/main/resources/db/migration/V001__create-demo-table.sql b/src/sbt-test/test/basic/src/main/resources/db/migration/V001__create-demo-table.sql new file mode 100644 index 0000000..e69de29 diff --git a/src/sbt-test/test/basic/test b/src/sbt-test/test/basic/test index 0287d84..730ff77 100644 --- a/src/sbt-test/test/basic/test +++ b/src/sbt-test/test/basic/test @@ -1,5 +1,4 @@ -$ exec psql -c 'create table if not exists users (id bigint primary key, name varchar(256));' -U test -h postgres example -> + compile +> + slickCodegenAll -$ exists target/scala-2.13/src_managed/main/com/example/Tables.scala -$ exists target/scala-2.13/src_managed/main/com/example/UsersTable.scala +$ exists src/main/scala/com/demo/Table.scala +$ exists src/main/scala/com/example/Etl.scala