diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 200165e..ef1f1f4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,10 +16,21 @@ jobs: matrix: java-version: [8, 11] runs-on: ubuntu-latest + defaults: + run: + shell: bash -el {0} steps: - uses: actions/checkout@v2 with: fetch-depth: 0 + - name: Install Conda environment with Micromamba + uses: mamba-org/provision-with-micromamba@main + with: + cache-downloads: true + channels: "conda-forge,defaults" + environment-file: false + environment-name: conda-env-builder-test + extra-specs: "conda-lock=1.4.0" - uses: actions/setup-java@v1 with: java-version: ${{ matrix.java-version }} @@ -31,7 +42,7 @@ jobs: git config --add user.email "mill-ci@localhost" ./mill --jobs 2 clean ./mill --jobs 2 --disable-ticker -s _.compile - ./mill --jobs 2 --disable-ticker -s tools.test + ./mill --jobs 2 --disable-ticker -s tools.test.testOnly -- -l ExcludeGithubActions ./mill --jobs 2 --disable-ticker -s tools.scoverage.xmlReport ./mill --jobs 2 --disable-ticker -s tools.scoverage.htmlReport bash <(curl -s https://codecov.io/bash) -c -F tools -f '!*.txt' diff --git a/README.md b/README.md index 3782cc0..19b94a0 100644 --- a/README.md +++ b/README.md @@ -50,14 +50,12 @@ Build and maintain multiple custom conda environments all in once place. Install with [`conda`](https://conda.io/projects/conda/en/latest/index.html): `conda install --channel conda-forge conda-env-builder`. - ## Goals - -* Specify multiple environments in one place +* Specify multiple conda environments in one place * Reduce duplication with cross-environment defaults and environment inheritance -* Install `pip` packages into your conda environment, as well as custom commands -* Produce easy scripts to build your environments +* Produce easy scripts to build your environments (using `conda env create` or `conda-lock install`) +* Install `pip` packages into your conda environment, as well as custom commands (e.g. `git clone ... && make install`) ## Overview @@ -90,7 +88,8 @@ Below we highlight a few tools that you may find useful. * `Assemble`: builds per-environment conda environment and custom command build scripts. * Builds `.yaml` for your conda+pip environment specification YAML. * Builds `.build-conda.sh` to build your conda environment. - * Builds `.build-local.sh` to execute any custom commands after creating the conda envirnment. + * Builds `.build-local.sh` to execute any custom commands after creating the conda environment. + * Builds [conda-lock](https://github.com/conda/conda-lock) environment YAML to `/.` is specified * `Solve`: updates the configuration with a full list of packages and versions for the environment. * For each environment, builds it (`conda env create`), exports it (`conda env export`), and update the specification * `Tabulate`: writes the specification in a tabular format. diff --git a/tools/src/com/github/condaincubator/condaenvbuilder/CondaEnvironmentBuilderDef.scala b/tools/src/com/github/condaincubator/condaenvbuilder/CondaEnvironmentBuilderDef.scala index 84e08a8..09d7209 100644 --- a/tools/src/com/github/condaincubator/condaenvbuilder/CondaEnvironmentBuilderDef.scala +++ b/tools/src/com/github/condaincubator/condaenvbuilder/CondaEnvironmentBuilderDef.scala @@ -1,9 +1,9 @@ package com.github.condaincubator.condaenvbuilder -import java.nio.file.Path - import com.fulcrumgenomics.commons.CommonsDef +import java.nio.file.Path + /** * Object that is designed to be imported with `import CondaEnvironmentBuilderDef._` in any/all classes * much like the way that scala.PreDef is imported in all files automatically. diff --git a/tools/src/com/github/condaincubator/condaenvbuilder/cmdline/CondaEnvironmentBuilderMain.scala b/tools/src/com/github/condaincubator/condaenvbuilder/cmdline/CondaEnvironmentBuilderMain.scala index 343382a..15fbb20 100644 --- a/tools/src/com/github/condaincubator/condaenvbuilder/cmdline/CondaEnvironmentBuilderMain.scala +++ b/tools/src/com/github/condaincubator/condaenvbuilder/cmdline/CondaEnvironmentBuilderMain.scala @@ -39,7 +39,7 @@ class CondaEnvironmentBuilderCommonArgs Logger.level = this.logLevel CondaEnvironmentBuilderTool.UseMamba = mamba - CondaEnvironmentBuilderTool.FileExtension = extension + CondaEnvironmentBuilderTool.YamlFileExtension = extension } class CondaEnvironmentBuilderMain extends LazyLogging { diff --git a/tools/src/com/github/condaincubator/condaenvbuilder/cmdline/CondaEnvironmentBuilderTool.scala b/tools/src/com/github/condaincubator/condaenvbuilder/cmdline/CondaEnvironmentBuilderTool.scala index 327ac72..9a78d7b 100644 --- a/tools/src/com/github/condaincubator/condaenvbuilder/cmdline/CondaEnvironmentBuilderTool.scala +++ b/tools/src/com/github/condaincubator/condaenvbuilder/cmdline/CondaEnvironmentBuilderTool.scala @@ -2,14 +2,18 @@ package com.github.condaincubator.condaenvbuilder.cmdline import com.fulcrumgenomics.commons.util.LazyLogging import com.fulcrumgenomics.sopt.cmdline.ValidationException -import CondaEnvironmentBuilderMain.FailureException +import com.github.condaincubator.condaenvbuilder.cmdline.CondaEnvironmentBuilderMain.FailureException object CondaEnvironmentBuilderTool { /** True to use `mamba` instead of `conda`, false otherwise. */ var UseMamba: Boolean = false + /** True to use `micromamba` instead of `conda` or `mamba`, false otherwise. Needed for testing in micromamba + * environments*/ + var UseMicromamba: Boolean = false + /** The file extension to use for YAML files. */ - var FileExtension: String = "yml" + var YamlFileExtension: String = "yml" } @@ -33,6 +37,10 @@ trait CondaEnvironmentBuilderTool extends LazyLogging { def validate(test: Boolean, message: => String): Unit = if (!test) throw new ValidationException(message) /** Returns the conda executable to use. */ - protected def condaExecutable: String = if (CondaEnvironmentBuilderTool.UseMamba) "mamba" else "conda" + protected def condaExecutable: String = { + if (CondaEnvironmentBuilderTool.UseMicromamba) "micromamba" + else if (CondaEnvironmentBuilderTool.UseMamba) "mamba" + else "conda" + } } diff --git a/tools/src/com/github/condaincubator/condaenvbuilder/io/BuildWriter.scala b/tools/src/com/github/condaincubator/condaenvbuilder/io/BuildWriter.scala index 5c2fc64..4558c10 100644 --- a/tools/src/com/github/condaincubator/condaenvbuilder/io/BuildWriter.scala +++ b/tools/src/com/github/condaincubator/condaenvbuilder/io/BuildWriter.scala @@ -5,76 +5,52 @@ import com.fulcrumgenomics.commons.io.Io import com.fulcrumgenomics.commons.util.{LazyLogging, Logger} import com.github.condaincubator.condaenvbuilder.CondaEnvironmentBuilderDef.PathToYaml import com.github.condaincubator.condaenvbuilder.api.CodeStep.Command +import com.github.condaincubator.condaenvbuilder.api.CondaStep.Platform import com.github.condaincubator.condaenvbuilder.api.{CodeStep, CondaStep, Environment, PipStep} import com.github.condaincubator.condaenvbuilder.cmdline.CondaEnvironmentBuilderTool import java.io.PrintWriter import java.nio.file.Paths -/** Companion to [[BuildWriter]]. */ -object BuildWriter { +trait BuildWriterConstants { /** Returns the path to the environment's conda YAML. */ - private def toEnvironmentYaml(environment: Environment, output: DirPath): PathToYaml = { - output.resolve(f"${environment.name}.${CondaEnvironmentBuilderTool.FileExtension}") + protected def toEnvironmentYaml(environment: Environment, output: DirPath): PathToYaml = { + output.resolve(f"${environment.name}.${CondaEnvironmentBuilderTool.YamlFileExtension}") } + /** Returns the path to the environment's conda LOCK file. */ + protected def toEnvironmentLockYaml(environment: Environment, platform: Platform, output: DirPath): PathToYaml = { + output.resolve(f"${environment.name}.${platform}.conda-lock.${CondaEnvironmentBuilderTool.YamlFileExtension}") + } + + /** Returns the path to the environment's conda build script. */ - private def toCondaBuildScript(environment: Environment, output: DirPath): FilePath = { + protected def toCondaBuildScript(environment: Environment, output: DirPath): FilePath = { output.resolve(f"${environment.name}.build-conda.sh") } /** Returns the path to the environment's custom code build script. */ - private def toCodeBuildScript(environment: Environment, output: DirPath): FilePath = { + protected def toCodeBuildScript(environment: Environment, output: DirPath): FilePath = { output.resolve(f"${environment.name}.build-local.sh") } - - /** Builds a new [[BuildWriter]] for the given environment. - * - * @param environment the environment for which build files should be created. - * @param output the output directory where build files should be created. - * @param environmentYaml the path to use for the environment's conda YAML, otherwise `/.yml`. - * @param condaBuildScript the path to use for the environment's conda build script, - * otherwise `/.build-conda.sh`. - * @param codeBuildScript the path to use for the environment's custom code build script, - * otherwise `/.build-local.sh`. - * @param condaEnvironmentDirectory the directory in which conda environments should be stored when created. - * @return - */ - def apply(environment: Environment, - output: DirPath, - environmentYaml: Option[PathToYaml] = None, - condaBuildScript: Option[FilePath] = None, - codeBuildScript: Option[FilePath] = None, - condaEnvironmentDirectory: Option[DirPath] = None): BuildWriter = { - BuildWriter( - environment = environment, - environmentYaml = environmentYaml.getOrElse(toEnvironmentYaml(environment, output)), - condaBuildScript = condaBuildScript.getOrElse(toCondaBuildScript(environment, output)), - codeBuildScript = codeBuildScript.getOrElse(toCodeBuildScript(environment, output)), - condaEnvironmentDirectory = condaEnvironmentDirectory - ) - } } -/** Writer that is used to create the build scripts for the conda environments. - * - * The conda build script should be executed first, then the custom code build script. The conda environment - * specification is stored in the given environment YAML path. - * - * @param environment the environment for which build files should be created. - * @param environmentYaml the path to use for the environment's conda YAML. - * @param condaBuildScript the path to use for the environment's conda build script - * @param codeBuildScript the path to use for the environment's custom code build script - * @param condaEnvironmentDirectory the directory in which conda environments should be stored when created. - */ -case class BuildWriter(environment: Environment, - environmentYaml: PathToYaml, - condaBuildScript: FilePath, - codeBuildScript: FilePath, - condaEnvironmentDirectory: Option[DirPath]) extends LazyLogging { - - private lazy val condaExecutable: String = if (CondaEnvironmentBuilderTool.UseMamba) "mamba" else "conda" + +trait BuildWriter extends LazyLogging { + def environment: Environment + + /** the path to use for the environment's conda YAML */ + def environmentYaml: PathToYaml + + /** The path to use for the environment's conda build script */ + def condaBuildScript: FilePath + + /** The path to use for the environment's custom code build script */ + def codeBuildScript: FilePath + + /** The directory in which conda environments should be stored when created */ + def condaEnvironmentDirectory: Option[DirPath] /** Returns all the output files that will be written by this writer */ def allOutputs: Iterable[FilePath] = Seq(environmentYaml, condaBuildScript, codeBuildScript) @@ -91,20 +67,20 @@ case class BuildWriter(environment: Environment, /** Writes the conda environment file. */ def writeEnvironmentYaml(logger: Logger = this.logger): Unit = { - logger.info(s"Writing the environment YAML for ${environment.name} to: $environmentYaml") + logger.info(s"Writing the conda environment YAML for ${environment.name} to: $environmentYaml") val condaStep: Option[CondaStep] = environment.steps.collect { case step: CondaStep => step } match { - case Seq() => None + case Seq() => None case Seq(step) => Some(step) - case steps => throw new IllegalArgumentException( + case steps => throw new IllegalArgumentException( s"Expected a single conda step, found ${steps.length} conda steps. Did you forget to compile?" ) } val pipStep: Option[PipStep] = environment.steps.collect { case step: PipStep => step } match { - case Seq() => None + case Seq() => None case Seq(step) => Some(step) - case steps => throw new IllegalArgumentException( + case steps => throw new IllegalArgumentException( s"Expected a single pip step, found ${steps.length} pip steps. Did you forget to compile?" ) } @@ -139,8 +115,11 @@ case class BuildWriter(environment: Environment, writer.close() } + /** Write the conda build command. */ + protected def writeCondaBuildCommand(writer: PrintWriter): Unit + /** Writes the conda build script. */ - def writeCondaBuildScript(logger: Logger = this.logger): Unit = { + private def writeCondaBuildScript(logger: Logger = this.logger): Unit = { logger.info(s"Writing conda build script for ${environment.name} to: $condaBuildScript") val writer = new PrintWriter(Io.toWriter(condaBuildScript)) writer.println("#/bin/bash\n") @@ -149,24 +128,19 @@ case class BuildWriter(environment: Environment, writer.println("# Move to the scripts directory") writer.println("pushd $(dirname $0)\n") writer.println("# Build the conda environment") - writer.write(f"$condaExecutable env create --force --verbose --quiet") - condaEnvironmentDirectory match { - case Some(pre) => writer.write(f" --prefix ${pre.toAbsolutePath}/${environment.name}") - case None => writer.write(f" --name ${environment.name}") - } - writer.println(f" --file ${environmentYaml.toFile.getName}\n") + this.writeCondaBuildCommand(writer=writer) writer.println("popd\n") writer.close() } /** Writes the custom code build script. */ - def writeCodeBuildScript(logger: Logger = this.logger): Unit = { + protected def writeCodeBuildScript(logger: Logger = this.logger): Unit = { logger.info(s"Writing custom code build script for ${environment.name} to: $codeBuildScript") val codeStep: Option[CodeStep] = environment.steps.collect { case step: CodeStep => step } match { - case Seq() => None + case Seq() => None case Seq(step) => Some(step) - case steps => throw new IllegalArgumentException( + case steps => throw new IllegalArgumentException( s"Expected a single code step, found ${steps.length} code steps. Did you forget to compile?" ) } @@ -178,11 +152,11 @@ case class BuildWriter(environment: Environment, val buildPath = codeStep.map(_.path).getOrElse(Paths.get(".")) writer.println(f"""repo_root=$${1:-"$buildPath"}\n""") codeStep match { - case None => writer.println("# No custom commands") + case None => writer.println("# No custom commands") case Some(step) => writer.println(f"# Activate conda environment: ${environment.name}") - writer.println("set +eu") // because of unbound variables - writer.println("PS1=dummy\n") // for sourcing + writer.println("set +eu") // because of unbound variables + writer.println("PS1=dummy\n") // for sourcing writer.println(f". $$(conda info --base | tail -n 1)/etc/profile.d/conda.sh") // tail to ignore mamba header writer.println(f"conda activate ${environment.name}") writer.println() diff --git a/tools/src/com/github/condaincubator/condaenvbuilder/io/CondaBuildWriter.scala b/tools/src/com/github/condaincubator/condaenvbuilder/io/CondaBuildWriter.scala new file mode 100644 index 0000000..f55e69c --- /dev/null +++ b/tools/src/com/github/condaincubator/condaenvbuilder/io/CondaBuildWriter.scala @@ -0,0 +1,73 @@ +package com.github.condaincubator.condaenvbuilder.io + +import com.fulcrumgenomics.commons.CommonsDef.{DirPath, FilePath} +import com.fulcrumgenomics.commons.io.Io +import com.fulcrumgenomics.commons.util.Logger +import com.github.condaincubator.condaenvbuilder.CondaEnvironmentBuilderDef.PathToYaml +import com.github.condaincubator.condaenvbuilder.api.Environment +import com.github.condaincubator.condaenvbuilder.cmdline.CondaEnvironmentBuilderTool + +import java.io.PrintWriter + + +/** Companion to [[CondaBuildWriter]]. */ +object CondaBuildWriter extends BuildWriterConstants { + + /** Builds a new [[CondaBuildWriter]] for the given environment. + * + * @param environment the environment for which build files should be created. + * @param output the output directory where build files should be created. + * @param environmentYaml the path to use for the environment's conda YAML, otherwise `/.yml`. + * @param condaBuildScript the path to use for the environment's conda build script, + * otherwise `/.build-conda.sh`. + * @param codeBuildScript the path to use for the environment's custom code build script, + * otherwise `/.build-local.sh`. + * @param condaEnvironmentDirectory the directory in which conda environments should be stored when created. + * @return + */ + def apply(environment: Environment, + output: DirPath, + environmentYaml: Option[PathToYaml] = None, + condaBuildScript: Option[FilePath] = None, + codeBuildScript: Option[FilePath] = None, + condaEnvironmentDirectory: Option[DirPath] = None): CondaBuildWriter = { + CondaBuildWriter( + environment = environment, + environmentYaml = environmentYaml.getOrElse(toEnvironmentYaml(environment, output)), + condaBuildScript = condaBuildScript.getOrElse(toCondaBuildScript(environment, output)), + codeBuildScript = codeBuildScript.getOrElse(toCodeBuildScript(environment, output)), + condaEnvironmentDirectory = condaEnvironmentDirectory + ) + } +} + + +/** Writer that is used to create the build scripts for the conda environments. + * + * The conda build script should be executed first, then the custom code build script. The conda environment + * specification is stored in the given environment YAML path. + * + * @param environment the environment for which build files should be created. + * @param environmentYaml the path to use for the environment's conda YAML. + * @param condaBuildScript the path to use for the environment's conda build script + * @param codeBuildScript the path to use for the environment's custom code build script + * @param condaEnvironmentDirectory the directory in which conda environments should be stored when created. + */ +case class CondaBuildWriter(environment: Environment, + environmentYaml: PathToYaml, + condaBuildScript: FilePath, + codeBuildScript: FilePath, + condaEnvironmentDirectory: Option[DirPath]) extends BuildWriter { + + private lazy val condaExecutable: String = if (CondaEnvironmentBuilderTool.UseMamba) "mamba" else "conda" + + /** Writes the conda build command. */ + protected def writeCondaBuildCommand(writer: PrintWriter): Unit = { + writer.write(f"$condaExecutable env create --force --verbose --quiet") + condaEnvironmentDirectory match { + case Some(pre) => writer.write(f" --prefix ${pre.toAbsolutePath}/${environment.name}") + case None => writer.write(f" --name ${environment.name}") + } + writer.println(f" --file ${environmentYaml.toFile.getName}\n") + } +} diff --git a/tools/src/com/github/condaincubator/condaenvbuilder/io/CondaLockInstallWriter.scala b/tools/src/com/github/condaincubator/condaenvbuilder/io/CondaLockInstallWriter.scala new file mode 100644 index 0000000..b2dee03 --- /dev/null +++ b/tools/src/com/github/condaincubator/condaenvbuilder/io/CondaLockInstallWriter.scala @@ -0,0 +1,108 @@ +package com.github.condaincubator.condaenvbuilder.io + +import com.fulcrumgenomics.commons.CommonsDef.{DirPath, FilePath} +import com.fulcrumgenomics.commons.util.Logger +import com.github.condaincubator.condaenvbuilder.CondaEnvironmentBuilderDef.PathToYaml +import com.github.condaincubator.condaenvbuilder.api.CondaStep.Platform +import com.github.condaincubator.condaenvbuilder.api.Environment +import com.github.condaincubator.condaenvbuilder.cmdline.CondaEnvironmentBuilderTool +import com.github.condaincubator.condaenvbuilder.util.Process + +import java.io.PrintWriter + +/** Companion to [[CondaLockInstallWriter]]. */ +object CondaLockInstallWriter extends BuildWriterConstants { + + /** Builds a new [[CondaLockInstallWriter]] for the given environment. + * + * @param environment the environment for which build files should be created. + * @param output the output directory where build files should be created. + * @param platform the platform on which to install + * @param environmentYaml the path to use for the environment's conda YAML, otherwise `/.yml`. + * @param environmentLockYaml the path to use for the environment's conda YAML, otherwise `/..conda-lock.yml`. + * @param condaBuildScript the path to use for the environment's conda build script, + * otherwise `/.build-conda.sh`. + * @param codeBuildScript the path to use for the environment's custom code build script, + * otherwise `/.build-local.sh`. + * @param condaEnvironmentDirectory the directory in which conda environments should be stored when created. + * @return + */ + def apply(environment: Environment, + output: DirPath, + platform: Platform, + environmentYaml: Option[PathToYaml] = None, + environmentLockYaml: Option[PathToYaml] = None, + condaBuildScript: Option[FilePath] = None, + codeBuildScript: Option[FilePath] = None, + condaEnvironmentDirectory: Option[DirPath] = None): CondaLockInstallWriter = { + new CondaLockInstallWriter( + environment = environment, + platform = platform, + environmentYaml = environmentYaml.getOrElse(toEnvironmentYaml(environment, output)), + environmentLockYaml = environmentLockYaml.getOrElse(toEnvironmentLockYaml(environment, platform, output)), + condaBuildScript = condaBuildScript.getOrElse(toCondaBuildScript(environment, output)), + codeBuildScript = codeBuildScript.getOrElse(toCodeBuildScript(environment, output)), + condaEnvironmentDirectory = condaEnvironmentDirectory + ) + } +} + +/** Writer that is used to create the build scripts for the conda environments. + * + * The conda build script should be executed first, then the custom code build script. The conda environment + * specification is stored in the given environment YAML path. + * + * @param environment the environment for which build files should be created. + * @param platform the platform on which to install + * @param environmentYaml the path to use for the environment's conda YAML. + * @param environmentLockYaml the path to use for the environment's conda LOCK file. + * @param condaBuildScript the path to use for the environment's conda build script + * @param codeBuildScript the path to use for the environment's custom code build script + * @param condaEnvironmentDirectory the directory in which conda environments should be stored when created. + */ +case class CondaLockInstallWriter(environment: Environment, + environmentYaml: PathToYaml, + platform: Platform, + environmentLockYaml: PathToYaml, + condaBuildScript: FilePath, + codeBuildScript: FilePath, + condaEnvironmentDirectory: Option[DirPath]) extends BuildWriter { + + private lazy val condaLockExecutable: String = "conda-lock" + + /** Returns all the output files that will be written by this writer */ + override def allOutputs: Iterable[FilePath] = super.allOutputs ++ Iterator(environmentLockYaml) + + /** Writes the conda environment file and all the build scripts. */ + override def write(logger: Logger = this.logger): Unit = { + super.write(logger=logger) + + // Write the lock file + this.writeEnvironmentLock(logger=this.logger) + } + + /** Writes the conda build command. */ + protected def writeCondaBuildCommand(writer: PrintWriter): Unit = { + writer.write(f"$condaLockExecutable install") + if (CondaEnvironmentBuilderTool.UseMamba) writer.write(" --mamba") + condaEnvironmentDirectory match { + case Some(pre) => writer.write(f" --prefix ${pre.toAbsolutePath}/${environment.name}") + case None => writer.write(f" --name ${environment.name}") + } + writer.println(f" ${environmentLockYaml.toFile.getName}\n") + } + + private def writeEnvironmentLock(logger: Logger = this.logger): Unit = { + // Export the environment + logger.info(s"Writing the conda-lock environment YAML for ${environment.name} to: $environmentLockYaml") + val condaLockOptionalArgs: String = if (CondaEnvironmentBuilderTool.UseMamba) "--mamba" else "" + Process.run( + logger = logger, + f"$condaLockExecutable lock $condaLockOptionalArgs" + + f" --platform $platform" + + f" --file $environmentYaml" + + f" --kind lock" + + f" --lockfile $environmentLockYaml" + ) + } +} diff --git a/tools/src/com/github/condaincubator/condaenvbuilder/tools/Assemble.scala b/tools/src/com/github/condaincubator/condaenvbuilder/tools/Assemble.scala index e23aecf..8f0f236 100644 --- a/tools/src/com/github/condaincubator/condaenvbuilder/tools/Assemble.scala +++ b/tools/src/com/github/condaincubator/condaenvbuilder/tools/Assemble.scala @@ -4,9 +4,10 @@ import com.fulcrumgenomics.commons.CommonsDef.DirPath import com.fulcrumgenomics.commons.io.Io import com.fulcrumgenomics.sopt.{arg, clp} import com.github.condaincubator.condaenvbuilder.CondaEnvironmentBuilderDef._ +import com.github.condaincubator.condaenvbuilder.api.CondaStep.Platform import com.github.condaincubator.condaenvbuilder.api.{Environment, Spec} import com.github.condaincubator.condaenvbuilder.cmdline.{ClpGroups, CondaEnvironmentBuilderTool} -import com.github.condaincubator.condaenvbuilder.io.{BuildWriter, SpecParser} +import com.github.condaincubator.condaenvbuilder.io.{CondaBuildWriter, CondaLockInstallWriter, SpecParser} import java.nio.file.Files @@ -19,6 +20,7 @@ import java.nio.file.Files |1. the conda environment YAML to `/.yml` |2. the conda environment build script to `/.build-conda.sh` |3. the custom code build script to `/.build-local.sh` + |4. the conda-lock environment YAML to `/.` is specified | |The directory in which conda environment(s) are created can be specified with the `--prefix` option. |""", @@ -30,7 +32,8 @@ class Assemble @arg(flag='f', doc="Overwrite existing files.") val overwrite: Boolean = false, @arg(flag='n', doc="Assemble environments with the given name(s).", minElements=0) val names: Set[String] = Set.empty, @arg(flag='g', doc="Assemble environments with the given group(s).", minElements=0) val groups: Set[String] = Set.empty, - @arg(doc="Compile the YAML configuration file before assembling.") compile: Boolean = true + @arg(doc="Compile the YAML configuration file before assembling.") compile: Boolean = true, + @arg(doc="Lock and the environment for the given platform, and use conda-lock to install the conda environment in the build script") condaLock: Option[Platform] = None ) extends CondaEnvironmentBuilderTool { Io.assertReadable(config) @@ -50,7 +53,11 @@ class Assemble logger.info(f"Building ${environments.length}%,d out of ${spec.specs.length}%,d environments.") environments.zipWithIndex.map { case (environment, index) => - val writer = BuildWriter(environment=environment, output=output, condaEnvironmentDirectory=prefix) + val writer = condaLock match { + case None => CondaBuildWriter(environment=environment, output=output, condaEnvironmentDirectory=prefix) + case Some(platform) => CondaLockInstallWriter(environment=environment, platform=platform, output=output, condaEnvironmentDirectory=prefix) + } + if (!overwrite) { logger.info(f"Checking environment (${index+1}/${environments.length}): ${environment.name}") writer.allOutputs.foreach { path => diff --git a/tools/src/com/github/condaincubator/condaenvbuilder/tools/Solve.scala b/tools/src/com/github/condaincubator/condaenvbuilder/tools/Solve.scala index ed719b1..ae45318 100644 --- a/tools/src/com/github/condaincubator/condaenvbuilder/tools/Solve.scala +++ b/tools/src/com/github/condaincubator/condaenvbuilder/tools/Solve.scala @@ -3,17 +3,17 @@ package com.github.condaincubator.condaenvbuilder.tools import cats.syntax.either._ import com.fulcrumgenomics.commons.CommonsDef.{DirPath, SafelyClosable} import com.fulcrumgenomics.commons.io.Io -import com.fulcrumgenomics.commons.util.Logger import com.fulcrumgenomics.sopt.{arg, clp} import com.github.condaincubator.condaenvbuilder.CondaEnvironmentBuilderDef._ import com.github.condaincubator.condaenvbuilder.api.CondaStep.{Channel, Platform} import com.github.condaincubator.condaenvbuilder.api._ import com.github.condaincubator.condaenvbuilder.cmdline.{ClpGroups, CondaEnvironmentBuilderTool} -import com.github.condaincubator.condaenvbuilder.io.{BuildWriter, SpecParser, SpecWriter} +import com.github.condaincubator.condaenvbuilder.io.{CondaBuildWriter, SpecParser, SpecWriter} +import com.github.condaincubator.condaenvbuilder.util.Process import io.circe.Decoder.Result import io.circe.{Decoder, DecodingFailure, HCursor, yaml} -import scala.sys.process.{ProcessBuilder, ProcessLogger, _} +import scala.sys.process._ @clp(description = """ @@ -38,7 +38,6 @@ class Solve @arg(doc="Remove build specification from dependencies (i.e. use `conda env export --no-builds`)") noBuilds: Boolean = false, private[tools] val dryRun: Boolean = false // for testing ) extends CondaEnvironmentBuilderTool { - import Solve._ Io.assertReadable(config) Io.assertCanWriteFile(output) @@ -72,8 +71,8 @@ class Solve // Set up where files and conda environments will get written val condaEnvironmentPrefix: DirPath = condaEnvironmentDir.resolve(environment.name) - val environmentYaml: PathToYaml = Io.makeTempFile("config.", f".${environment.name}.${CondaEnvironmentBuilderTool.FileExtension}") - val writer = BuildWriter( + val environmentYaml: PathToYaml = Io.makeTempFile("config.", f".${environment.name}.${CondaEnvironmentBuilderTool.YamlFileExtension}") + val writer = CondaBuildWriter( environment = environment, output = assemblyOutputDir, environmentYaml = Some(environmentYaml), @@ -86,26 +85,26 @@ class Solve val exportedYaml: PathToYaml = if (dryRun) writer.environmentYaml else { // Build the environment logger.info(s"Building a temporary conda environment for ${environment.name} to: $condaEnvironmentPrefix") - run( + Process.run( logger=logger, - f"$condaExecutable env create --verbose --quiet --prefix $condaEnvironmentPrefix --file $environmentYaml" + f"$condaExecutable env create --verbose --verbose --verbose --quiet --prefix $condaEnvironmentPrefix --file $environmentYaml" ) // Export the environment logger.info(s"Exporting the conda environment for ${environment.name}") - val exportedYaml: PathToYaml = Io.makeTempFile("config.", f".${environment.name}.${CondaEnvironmentBuilderTool.FileExtension}") + val exportedYaml: PathToYaml = Io.makeTempFile("config.", f".${environment.name}.${CondaEnvironmentBuilderTool.YamlFileExtension}") val condaEnvExportArgs: String = if (noBuilds) "--no-builds" else "" - run( + Process.run( logger=logger, f"$condaExecutable env export --prefix $condaEnvironmentPrefix $condaEnvExportArgs" #| """egrep -v "^prefix"""" - #| f"""sed "s/name: null/name: ${environment.name}/"""" + #| f"""sed "s/^name: .*/name: ${environment.name}/"""" // the name may contain an absolute path, so change it! #> exportedYaml.toFile ) // Remove the temporary environment logger.info(s"Removing the temporary the conda environment for ${environment.name}") - run(logger=logger, s"rm -rv $condaEnvironmentPrefix") + Process.run(logger=logger, s"rm -rv $condaEnvironmentPrefix") exportedYaml } @@ -162,39 +161,6 @@ class Solve } } -object Solve { - /** The exit code of any interrupted execution of a task. */ - private val InterruptedExitCode = 255 - - private def run(logger: Logger, processBuilder: ProcessBuilder): Unit = { - val processOutput: StringBuilder = new StringBuilder() - val (process: Option[scala.sys.process.Process], exitCode: Int, throwable: Option[Throwable]) = { - try { - //logger.info(s"Executing: $command") - val _process = processBuilder.run(ProcessLogger(fn=(line: String) => processOutput.append(line + "\n"))) - (Some(_process), _process.exitValue(), None) - } catch { - case e: InterruptedException => (None, InterruptedExitCode, Some(e)) - case t: Throwable => (None, 1, Some(t)) - } - } - - // destroy the process regardless - process.foreach(p => p.destroy()) - - // throw the exception if something happened - throwable.foreach { thr => - logger.error(processOutput) - throw thr - } - - if (exitCode != 0) { - logger.error(processOutput) - throw new IllegalStateException(s"Command exited with exit code '$exitCode': $processBuilder") - } - } -} - private case class CondaEnvironment (name: String, platforms: Seq[Platform], diff --git a/tools/src/com/github/condaincubator/condaenvbuilder/util/Process.scala b/tools/src/com/github/condaincubator/condaenvbuilder/util/Process.scala new file mode 100644 index 0000000..ccd5dba --- /dev/null +++ b/tools/src/com/github/condaincubator/condaenvbuilder/util/Process.scala @@ -0,0 +1,39 @@ +package com.github.condaincubator.condaenvbuilder.util + +import com.fulcrumgenomics.commons.util.Logger + +import scala.sys.process.{ProcessBuilder, ProcessLogger} + +object Process { + /** The exit code of any interrupted execution of a task. */ + private val InterruptedExitCode = 255 + + /** Runs the given processes(es). */ + def run(logger: Logger, processBuilder: ProcessBuilder): Unit = { + logger.info(s"running ${processBuilder.toString}") + val processOutput: StringBuilder = new StringBuilder() + val (process: Option[scala.sys.process.Process], exitCode: Int, throwable: Option[Throwable]) = { + try { + val _process = processBuilder.run(ProcessLogger(fn = (line: String) => processOutput.append(line + "\n"))) + (Some(_process), _process.exitValue(), None) + } catch { + case e: InterruptedException => (None, InterruptedExitCode, Some(e)) + case t: Throwable => (None, 1, Some(t)) + } + } + + // destroy the process regardless + process.foreach(p => p.destroy()) + + // throw the exception if something happened + throwable.foreach { thr => + logger.error(processOutput) + throw thr + } + + if (exitCode != 0) { + logger.error(processOutput) + throw new IllegalStateException(s"Command exited with exit code '$exitCode': $processBuilder") + } + } +} diff --git a/tools/test/src/com/github/condaincubator/condaenvbuilder/tools/ToolsTest.scala b/tools/test/src/com/github/condaincubator/condaenvbuilder/tools/ToolsTest.scala index 3f66c86..2873991 100644 --- a/tools/test/src/com/github/condaincubator/condaenvbuilder/tools/ToolsTest.scala +++ b/tools/test/src/com/github/condaincubator/condaenvbuilder/tools/ToolsTest.scala @@ -1,10 +1,15 @@ package com.github.condaincubator.condaenvbuilder.tools -import java.nio.file.Files import com.fulcrumgenomics.commons.io.Io +import com.github.condaincubator.condaenvbuilder.api.{CodeStep, CondaStep, PipStep, Step} import com.github.condaincubator.condaenvbuilder.cmdline.CondaEnvironmentBuilderTool +import com.github.condaincubator.condaenvbuilder.io.SpecParser import com.github.condaincubator.condaenvbuilder.testing.UnitSpec +import java.nio.file.Files +import scala.reflect.ClassTag + + class ToolsTest extends UnitSpec { // YAML input to the Compile tool @@ -432,8 +437,8 @@ class ToolsTest extends UnitSpec { } "Compile" should "compile a YAML configuration file" in { - val specPath = makeTempFile("in.", CondaEnvironmentBuilderTool.FileExtension) - val compiledPath = makeTempFile("out.", CondaEnvironmentBuilderTool.FileExtension) + val specPath = makeTempFile("in.", "." + CondaEnvironmentBuilderTool.YamlFileExtension) + val compiledPath = makeTempFile("out.", "." + CondaEnvironmentBuilderTool.YamlFileExtension) Io.writeLines(path=specPath, lines=Seq(specString)) @@ -443,24 +448,49 @@ class ToolsTest extends UnitSpec { Io.readLines(path=compiledPath).mkString("\n") shouldBe compiledString } - Seq(true, false).foreach { compile => - val compileString = if (compile) "a pre-compiled" else "and compile a" - "Assemble" should s"assemble $compileString YAML configuration file" in { - val compiledPath = makeTempFile("compiled.", CondaEnvironmentBuilderTool.FileExtension) + private case class AssembleArgs(compile: Boolean = false, condaLock: Option[String] = None) { + def testCaseName: String = { + val builder = new StringBuilder() + builder.append("assemble") + if (compile) builder.append(" and compile a") + else builder.append(" a pre-compiled") + builder.append(" YAML configuration file") + if (condaLock.nonEmpty) builder.append(" and produce a conda-lock file") + builder.toString + } + } + + private val assembleTestCases = Seq( + AssembleArgs(compile=true, condaLock=None), + AssembleArgs(compile=true, condaLock=Some("linux-64")), + AssembleArgs(compile=false, condaLock=None), + AssembleArgs(compile=false, condaLock=Some("linux-64")), + ) + + assembleTestCases.foreach { assembleArgs => + "Assemble" should assembleArgs.testCaseName in { + val compiledPath = makeTempFile("compiled.", "." + CondaEnvironmentBuilderTool.YamlFileExtension) val outputDir = Files.createTempDirectory("output") - val lines = if (compile) Seq(compiledString) else Seq(specString) + val lines = if (assembleArgs.compile) Seq(compiledString) else Seq(specString) Io.writeLines(path=compiledPath, lines=lines) - val assemble = new Assemble(config=compiledPath, output=outputDir, compile=compile) + // NB: since conda-lock can be slow, only build the bwa environment when producing conda-lock output. + val names: Set[String] = if (assembleArgs.condaLock.isEmpty) Set.empty else Set("bwa") + + CondaEnvironmentBuilderTool.UseMamba = assembleArgs.condaLock.isDefined + CondaEnvironmentBuilderTool.UseMicromamba = assembleArgs.condaLock.isDefined + val assemble = new Assemble(config=compiledPath, output=outputDir, compile=assembleArgs.compile, condaLock=assembleArgs.condaLock, names=names) assemble.execute() + CondaEnvironmentBuilderTool.UseMamba = false + CondaEnvironmentBuilderTool.UseMicromamba = false - val bwaYamlPath = outputDir.resolve(s"bwa.${CondaEnvironmentBuilderTool.FileExtension}") + val bwaYamlPath = outputDir.resolve(s"bwa.${CondaEnvironmentBuilderTool.YamlFileExtension}") val bwaCondaPath = outputDir.resolve("bwa.build-conda.sh") val bwaCodePath = outputDir.resolve("bwa.build-local.sh") Io.readLines(bwaYamlPath).mkString("\n") shouldBe { - """name: bwa + """name: bwa |platforms: | - linux-32 |channels: @@ -470,21 +500,6 @@ class ToolsTest extends UnitSpec { | - samtools=1.9 | - bwa=0.7.17""".stripMargin } - Io.readLines(bwaCondaPath).mkString("\n") shouldBe { - """#/bin/bash - | - |# Conda build file for environment: bwa - |set -xeuo pipefail - | - |# Move to the scripts directory - |pushd $(dirname $0) - | - |# Build the conda environment - |conda env create --force --verbose --quiet --name bwa --file bwa.yml - | - |popd - |""".stripMargin - } Io.readLines(bwaCodePath).mkString("\n") shouldBe { """#/bin/bash |# Custom code build file for environment: bwa @@ -494,36 +509,72 @@ class ToolsTest extends UnitSpec { | |# No custom commands""".stripMargin } - - val condaEnvBuilderCodePath = outputDir.resolve("conda-env-builder.build-local.sh") - Io.readLines(condaEnvBuilderCodePath).mkString("\n") shouldBe { - """#/bin/bash - |# Custom code build file for environment: conda-env-builder - |set -xeuo pipefail - | - |repo_root=${1:-"."} - | - |# Activate conda environment: conda-env-builder - |set +eu - |PS1=dummy - | - |. $(conda info --base | tail -n 1)/etc/profile.d/conda.sh - |conda activate conda-env-builder - | - |set -eu - |pushd ${repo_root} - |python setup.py develop - |popd - | - |""".stripMargin + assembleArgs.condaLock match { + case None => + Io.readLines(bwaCondaPath).mkString("\n") shouldBe { + """#/bin/bash + | + |# Conda build file for environment: bwa + |set -xeuo pipefail + | + |# Move to the scripts directory + |pushd $(dirname $0) + | + |# Build the conda environment + |conda env create --force --verbose --quiet --name bwa --file bwa.yml + | + |popd + |""".stripMargin + } + // only when we do not use conda-lock will we have this environment built + val condaEnvBuilderCodePath = outputDir.resolve("conda-env-builder.build-local.sh") + Io.readLines(condaEnvBuilderCodePath).mkString("\n") shouldBe { + """#/bin/bash + |# Custom code build file for environment: conda-env-builder + |set -xeuo pipefail + | + |repo_root=${1:-"."} + | + |# Activate conda environment: conda-env-builder + |set +eu + |PS1=dummy + | + |. $(conda info --base | tail -n 1)/etc/profile.d/conda.sh + |conda activate conda-env-builder + | + |set -eu + |pushd ${repo_root} + |python setup.py develop + |popd + | + |""".stripMargin + } + case Some(platform) => + Io.readLines(bwaCondaPath).mkString("\n") shouldBe { + f"""#/bin/bash + | + |# Conda build file for environment: bwa + |set -xeuo pipefail + | + |# Move to the scripts directory + |pushd $$(dirname $$0) + | + |# Build the conda environment + |conda-lock install --mamba --name bwa bwa.$platform.conda-lock.yml + | + |popd + |""".stripMargin + } + val bwaCondaLockYaml = outputDir.resolve(f"bwa.$platform.conda-lock.${CondaEnvironmentBuilderTool.YamlFileExtension}") + Io.assertReadable(bwaCondaLockYaml) + // TODO: check that the packages in the environment YAML are found in the lock file } - } } - "Solve" should s"solve a compiled YAML file" in { - val compiledPath = makeTempFile("compiled.", CondaEnvironmentBuilderTool.FileExtension) - val solvedPath = makeTempFile("output.", CondaEnvironmentBuilderTool.FileExtension) + "Solve" should "solve a compiled YAML file (dry-run)" in { + val compiledPath = makeTempFile("compiled.", "." + CondaEnvironmentBuilderTool.YamlFileExtension) + val solvedPath = makeTempFile("output.", "." + CondaEnvironmentBuilderTool.YamlFileExtension) Io.writeLines(path=compiledPath, lines=Seq(compiledString)) @@ -533,9 +584,9 @@ class ToolsTest extends UnitSpec { Io.readLines(path=solvedPath).mkString("\n") shouldBe compiledReformatted // since we skipped the internal solving step } - "Solve" should s"solve a compiled YAML file for a given group" in { - val compiledPath = makeTempFile("compiled.", CondaEnvironmentBuilderTool.FileExtension) - val solvedPath = makeTempFile("output.", CondaEnvironmentBuilderTool.FileExtension) + it should "solve a compiled YAML file for a given group (dry-run)" in { + val compiledPath = makeTempFile("compiled.", "." + CondaEnvironmentBuilderTool.YamlFileExtension) + val solvedPath = makeTempFile("output.", "." + CondaEnvironmentBuilderTool.YamlFileExtension) Io.writeLines(path=compiledPath, lines=Seq(compiledString)) @@ -545,9 +596,88 @@ class ToolsTest extends UnitSpec { Io.readLines(path=solvedPath).mkString("\n") shouldBe compiledReformattedAlignmentOnly // since we skipped the internal solving step } + private def checkSteps[T<:Step](solvedSteps: Seq[Step], + compiledSteps: Seq[Step], + checkFunc: (T, T) => Unit) + (implicit tag: ClassTag[T]): Unit = { + val solvedTSteps = solvedSteps.collect { case step: T => step } + val compiledTSteps = compiledSteps.collect { case step: T => step } + solvedTSteps.length shouldBe compiledTSteps.length + solvedTSteps.length should be <= 1 + (solvedTSteps.headOption, compiledTSteps.headOption) match { + case (Some(solvedConda), Some(compiledConda)) => checkFunc(solvedConda, compiledConda) + case _ => () + } + } + + it should s"solve a compiled YAML file for a given environment (with mamba)" taggedAs org.scalatest.Tag("ExcludeGithubActions") in { + val compiledPath = makeTempFile("compiled.", "." + CondaEnvironmentBuilderTool.YamlFileExtension) + val solvedPath = makeTempFile("output.", "." + CondaEnvironmentBuilderTool.YamlFileExtension) + + Io.writeLines(path = compiledPath, lines = Seq(compiledString)) + + //CondaEnvironmentBuilderTool.UseMamba = true + val solve = new Solve(config = compiledPath, output = solvedPath, names = Set("bwa"), dryRun = false) + solve.execute() + //CondaEnvironmentBuilderTool.UseMamba = false + + val solvedSpec = SpecParser(solvedPath) + solvedSpec.defaults.size shouldBe 0 // defaults should no longer be present + solvedSpec.specs.foreach(_.inherits.length shouldBe 0) // inheritance should no longer be present + + val compiledEnvironments = SpecParser(compiledPath).specs.map(_.environment).sortBy(_.name) + val solvedEnvironments = solvedSpec.specs.map(_.environment).sortBy(_.name) + + solvedEnvironments.length shouldBe compiledEnvironments.length + solvedEnvironments.zip(compiledEnvironments).foreach { case (solvedEnvironment, compiledEnvironment) => + solvedEnvironment.name shouldBe compiledEnvironment.name + solvedEnvironment.group shouldBe compiledEnvironment.group + solvedEnvironment.steps.length shouldBe compiledEnvironment.steps.length + + // Check the conda steps + checkSteps[CondaStep]( + solvedSteps = solvedEnvironment.steps, + compiledSteps = compiledEnvironment.steps, + checkFunc = (solvedConda, compiledConda) => { + solvedConda.channels should contain theSameElementsInOrderAs compiledConda.channels + solvedConda.platforms should contain theSameElementsInOrderAs compiledConda.platforms + compiledConda.requirements.foreach { compiledRequirement => + solvedConda.requirements.exists { + _.name == compiledRequirement.name + } + } + } + ) + + // Check the pip steps + checkSteps[PipStep]( + solvedSteps = solvedEnvironment.steps, + compiledSteps = compiledEnvironment.steps, + checkFunc = (solvedPip, compiledPip) => { + solvedPip.args should contain theSameElementsInOrderAs compiledPip.args + compiledPip.requirements.foreach { compiledRequirement => + solvedPip.requirements.exists { + _.name == compiledRequirement.name + } + } + } + ) + + // Check the code steps + checkSteps[CodeStep]( + solvedSteps = solvedEnvironment.steps, + compiledSteps = compiledEnvironment.steps, + checkFunc = (solvedCode, compiledCode) => { + solvedCode.path shouldBe compiledCode.path + solvedCode.commands should contain theSameElementsInOrderAs compiledCode.commands + } + ) + } + } + "Tabulate" should "tabulate the input YAML file" in { - val specPath = makeTempFile("in.", CondaEnvironmentBuilderTool.FileExtension) - val tabulatedPath = makeTempFile("out.", CondaEnvironmentBuilderTool.FileExtension) + val specPath = makeTempFile("in.", "." + CondaEnvironmentBuilderTool.YamlFileExtension) + val tabulatedPath = makeTempFile("out.", "." + CondaEnvironmentBuilderTool.YamlFileExtension) Io.writeLines(path=specPath, lines=Seq(specString)) diff --git a/tools/test/src/com/github/condaincubator/condaenvbuilder/util/ProcessTest.scala b/tools/test/src/com/github/condaincubator/condaenvbuilder/util/ProcessTest.scala new file mode 100644 index 0000000..c915c87 --- /dev/null +++ b/tools/test/src/com/github/condaincubator/condaenvbuilder/util/ProcessTest.scala @@ -0,0 +1,27 @@ +package com.github.condaincubator.condaenvbuilder.util +import com.fulcrumgenomics.commons.util.LazyLogging +import com.github.condaincubator.condaenvbuilder.testing.UnitSpec + +class ProcessTest extends UnitSpec with LazyLogging { + "Process.run" should "run a simple command" in { + Process.run(logger=logger, processBuilder=f"echo 'Hello World'") + } + + it should "fail if the command cannot be parsed" in { + an[Exception] should be thrownBy { + Process.run(logger = logger, processBuilder = f"echo 'Hello World") // unmatched quote + } + } + + it should "fail if the command cannot be found" in { + an[java.io.IOException] should be thrownBy { + Process.run(logger = logger, processBuilder = f"foo-bar-123") + } + } + + it should "fail if the command exits non-zero" in { + an[IllegalStateException] should be thrownBy { + Process.run(logger = logger, processBuilder = f"conda foo") + } + } +} \ No newline at end of file