diff --git a/.polystat.conf b/.polystat.conf index f523d49..3d8fc61 100644 --- a/.polystat.conf +++ b/.polystat.conf @@ -4,5 +4,4 @@ polystat { outputTo = . tempDir = tmp outputFormats = [sarif] - # excludeRules = [e, c, dialect] } diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b4b80a..9370cf9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ -## Polystat v0.1.3 +## Polystat v0.1.4 -In this release the `odin` dependency was updated to 0.4.0. +In this release the `py2eo` project was integrated into `polystat-cli`. You can now run: +``` +polystat py --in python_files +``` -The CD pipeline was updated to allow releasing a specified version. +...to analyze a directory with a bunch of python files. For more options and explanations, run: +``` +polystat --help +``` +or +``` +polystat list --config +``` +if you want to use the config file. diff --git a/build.sbt b/build.sbt index 9da8a72..f7ad0fd 100644 --- a/build.sbt +++ b/build.sbt @@ -26,6 +26,7 @@ libraryDependencies ++= Seq( "io.circe" %% "circe-core" % "0.14.1", "org.scalameta" %% "munit" % "1.0.0-M3" % Test, "org.slf4j" % "slf4j-nop" % "1.7.36", + "org.polystat.py2eo" % "transpiler" % "0.0.10", ) assembly / assemblyJarName := "polystat.jar" diff --git a/sandbox_python/test.py b/sandbox_python/test.py new file mode 100644 index 0000000..b4f5a22 --- /dev/null +++ b/sandbox_python/test.py @@ -0,0 +1,3 @@ +def conditionalCheck2(): + a = 4 + b = 2 diff --git a/src/main/scala/org/polystat/EO.scala b/src/main/scala/org/polystat/EO.scala new file mode 100644 index 0000000..702df92 --- /dev/null +++ b/src/main/scala/org/polystat/EO.scala @@ -0,0 +1,44 @@ +package org.polystat + +import cats.effect.IO +import PolystatConfig.* +import InputUtils.* +import org.polystat.odin.analysis.ASTAnalyzer +import org.polystat.odin.analysis.EOOdinAnalyzer +import org.polystat.odin.parser.EoParser.sourceCodeEoParser +import cats.syntax.traverse.* +import cats.syntax.foldable.* + +object EO: + + def runAnalyzers( + analyzers: List[ASTAnalyzer[IO]] + )(code: String): IO[List[EOOdinAnalyzer.OdinAnalysisResult]] = + analyzers.traverse(a => + EOOdinAnalyzer + .analyzeSourceCode(a)(code)(cats.Monad[IO], sourceCodeEoParser[IO]()) + ) + + def analyze(cfg: ProcessedConfig): IO[Unit] = + val inputFiles = readCodeFromInput(".eo", cfg.input) + inputFiles + .evalMap { case (codePath, code) => + for + _ <- IO.println(s"Analyzing $codePath...") + analyzed <- runAnalyzers(cfg.filteredAnalyzers)(code) + _ <- cfg.output match + case Output.ToConsole => IO.println(analyzed) + case Output.ToDirectory(out) => + cfg.fmts.traverse_ { case OutputFormat.Sarif => + val outPath = + out / "sarif" / codePath.replaceExt(".sarif.json") + val sarifJson = SarifOutput(analyzed).json.toString + IO.println(s"Writing results to $outPath") *> + writeOutputTo(outPath)(sarifJson) + } + yield () + } + .compile + .drain + end analyze +end EO diff --git a/src/main/scala/org/polystat/InputUtils.scala b/src/main/scala/org/polystat/InputUtils.scala index bb1fd7f..b1f25f8 100644 --- a/src/main/scala/org/polystat/InputUtils.scala +++ b/src/main/scala/org/polystat/InputUtils.scala @@ -9,7 +9,7 @@ import fs2.text.utf8 import java.io.FileNotFoundException -import PolystatConfig.Input +import PolystatConfig.{Input, PolystatUsage} object InputUtils: extension (path: Path) @@ -73,4 +73,22 @@ object InputUtils: readCodeFromDir(ext = ext, dir = path) case Input.FromStdin => readCodeFromStdin.map(code => (Path("stdin" + ext), "\n" + code + "\n")) + + def readConfigFromFile(path: Path): IO[PolystatUsage.Analyze] = + HoconConfig(path).config.load + + def writeOutputTo(path: Path)(output: String): IO[Unit] = + for + _ <- path.parent + .map(Files[IO].createDirectories) + .getOrElse(IO.unit) + _ <- Stream + .emits(output.getBytes) + .through(Files[IO].writeAll(path)) + .compile + .drain + yield () + end for + end writeOutputTo + end InputUtils diff --git a/src/main/scala/org/polystat/J2EO.scala b/src/main/scala/org/polystat/Java.scala similarity index 66% rename from src/main/scala/org/polystat/J2EO.scala rename to src/main/scala/org/polystat/Java.scala index e72b603..49c63d7 100644 --- a/src/main/scala/org/polystat/J2EO.scala +++ b/src/main/scala/org/polystat/Java.scala @@ -13,8 +13,10 @@ import org.http4s.client.middleware.FollowRedirect import fs2.io.file.{Path, Files} import org.http4s.Uri import sys.process.* +import PolystatConfig.* +import InputUtils.* -object J2EO: +object Java: private val DEFAULT_J2EO_PATH = Path("j2eo.jar") private val J2EO_URL = @@ -53,7 +55,11 @@ object J2EO: } end downloadJ2EO - def run(j2eo: Option[Path], inputDir: Path, outputDir: Path): IO[Unit] = + private def runJ2EO( + j2eo: Option[Path], + inputDir: Path, + outputDir: Path, + ): IO[Unit] = val command = s"java -jar ${j2eo.getOrElse(DEFAULT_J2EO_PATH)} -o $outputDir $inputDir" for @@ -69,6 +75,28 @@ object J2EO: ) yield () end for - end run + end runJ2EO -end J2EO + def analyze(j2eo: Option[Path], cfg: ProcessedConfig): IO[Unit] = + for + tmp <- cfg.tempDir + _ <- cfg.input match // writing EO files to tempDir + case Input.FromStdin => + for + code <- readCodeFromStdin.compile.string + stdinTmp <- Files[IO].createTempDirectory.map(path => + path / "stdin.eo" + ) + _ <- writeOutputTo(stdinTmp)(code) + _ <- runJ2EO(j2eo, inputDir = stdinTmp, outputDir = tmp) + yield () + case Input.FromFile(path) => + runJ2EO(j2eo, inputDir = path, outputDir = tmp) + case Input.FromDirectory(path) => + runJ2EO(j2eo, inputDir = path, outputDir = tmp) + _ <- EO.analyze( + cfg.copy(input = Input.FromDirectory(tmp)) + ) + yield () + +end Java diff --git a/src/main/scala/org/polystat/Main.scala b/src/main/scala/org/polystat/Main.scala index 548d53a..0a64c35 100644 --- a/src/main/scala/org/polystat/Main.scala +++ b/src/main/scala/org/polystat/Main.scala @@ -14,10 +14,12 @@ import org.polystat.odin.analysis.ASTAnalyzer import org.polystat.odin.analysis.EOOdinAnalyzer import org.polystat.odin.analysis.EOOdinAnalyzer.OdinAnalysisResult import org.polystat.odin.parser.EoParser.sourceCodeEoParser +import org.polystat.py2eo.transpiler.Transpile import PolystatConfig.* import IncludeExclude.* import InputUtils.* +import org.polystat.py2eo.parser.PythonLexer object Main extends IOApp: override def run(args: List[String]): IO[ExitCode] = for exitCode <- CommandIOApp.run( @@ -49,66 +51,14 @@ object Main extends IOApp: } case None => analyzers.map(_._2) - def analyze( - analyzers: List[ASTAnalyzer[IO]] - )(code: String): IO[List[EOOdinAnalyzer.OdinAnalysisResult]] = - analyzers.traverse(a => - EOOdinAnalyzer - .analyzeSourceCode(a)(code)(cats.Monad[IO], sourceCodeEoParser[IO]()) - ) - - def listAnalyzers: IO[Unit] = analyzers.traverse_ { case (name, _) => - IO.println(name) - } - - def readConfigFromFile(path: Path): IO[PolystatUsage.Analyze] = - HoconConfig(path).config.load - - def writeOutputTo(path: Path)(output: String): IO[Unit] = - for - _ <- path.parent - .map(Files[IO].createDirectories) - .getOrElse(IO.unit) - _ <- Stream - .emits(output.getBytes) - .through(Files[IO].writeAll(path)) - .compile - .drain - yield () - end for - end writeOutputTo - - def analyzeEO( - inputFiles: Stream[IO, (Path, String)], - outputFormats: List[OutputFormat], - out: Output, - filteredAnalyzers: List[ASTAnalyzer[IO]], - ): IO[Unit] = - inputFiles - .evalMap { case (codePath, code) => - for - _ <- IO.println(s"Analyzing $codePath...") - analyzed <- analyze(filteredAnalyzers)(code) - _ <- out match - case Output.ToConsole => IO.println(analyzed) - case Output.ToDirectory(out) => - outputFormats.traverse_ { case OutputFormat.Sarif => - val outPath = - out / "sarif" / codePath.replaceExt(".sarif.json") - val sarifJson = SarifOutput(analyzed).json.toString - IO.println(s"Writing results to $outPath") *> - writeOutputTo(outPath)(sarifJson) - } - yield () - } - .compile - .drain - def execute(usage: PolystatUsage): IO[Unit] = usage match case PolystatUsage.List(cfg) => - if (cfg) then IO.println(HoconConfig.keys.explanation) - else listAnalyzers + if cfg then IO.println(HoconConfig.keys.explanation) + else + analyzers.traverse_ { case (name, _) => + IO.println(name) + } case PolystatUsage.Misc(version, config) => if (version) then IO.println(BuildInfo.version) else @@ -118,55 +68,26 @@ object Main extends IOApp: lang, AnalyzerConfig(inex, input, tmp, fmts, out), ) => - val filteredAnalyzers = filterAnalyzers(inex) - val tempDir: IO[Path] = tmp match - case Some(path) => IO.pure(path) - case None => Files[IO].createTempDirectory - val inputExt: String = lang match - case SupportedLanguage.EO => ".eo" - case SupportedLanguage.Java(_) => ".java" - case SupportedLanguage.Python => ".py" - + val processedConfig = ProcessedConfig( + filteredAnalyzers = filterAnalyzers(inex), + tempDir = tmp match + case Some(path) => + (IO.println(s"Cleaning ${path.absolute}...") *> + Files[IO].deleteRecursively(path) *> + Files[IO].createDirectory(path)) + .as(path) + case None => Files[IO].createTempDirectory + , + output = out, + input = input, + fmts = fmts, + ) val analysisResults: IO[Unit] = lang match - case SupportedLanguage.EO => - val inputFiles = readCodeFromInput(ext = inputExt, input = input) - analyzeEO( - inputFiles = inputFiles, - outputFormats = fmts, - out = out, - filteredAnalyzers = filteredAnalyzers, - ) + case SupportedLanguage.EO => EO.analyze(processedConfig) case SupportedLanguage.Java(j2eo) => - for - tmp <- tempDir - _ <- input match // writing EO files to tempDir - case Input.FromStdin => - for - code <- readCodeFromStdin.compile.string - stdinTmp <- Files[IO].createTempDirectory.map(path => - path / "stdin.eo" - ) - _ <- writeOutputTo(stdinTmp)(code) - _ <- J2EO.run(j2eo, inputDir = stdinTmp, outputDir = tmp) - yield () - case Input.FromFile(path) => - J2EO.run(j2eo, inputDir = path, outputDir = tmp) - case Input.FromDirectory(path) => - J2EO.run(j2eo, inputDir = path, outputDir = tmp) - inputFiles = readCodeFromInput( - ".eo", - Input.FromDirectory(tmp), - ) - _ <- analyzeEO( - inputFiles = inputFiles, - outputFormats = fmts, - out = out, - filteredAnalyzers = filteredAnalyzers, - ) - yield () - case SupportedLanguage.Python => - IO.println("Analyzing Python is not implemented yet!") + Java.analyze(j2eo, processedConfig) + case SupportedLanguage.Python => Python.analyze(processedConfig) analysisResults end execute diff --git a/src/main/scala/org/polystat/PolystatConfig.scala b/src/main/scala/org/polystat/PolystatConfig.scala index 15a23bb..a70f32a 100644 --- a/src/main/scala/org/polystat/PolystatConfig.scala +++ b/src/main/scala/org/polystat/PolystatConfig.scala @@ -6,6 +6,7 @@ import cats.syntax.apply.* import com.monovore.decline.Opts import fs2.Stream import fs2.io.file.Path +import org.polystat.odin.analysis.ASTAnalyzer object PolystatConfig: @@ -17,6 +18,14 @@ object PolystatConfig: output: Output, ) + case class ProcessedConfig( + filteredAnalyzers: List[ASTAnalyzer[IO]], + tempDir: IO[Path], + input: Input, + fmts: List[OutputFormat], + output: Output, + ) + enum SupportedLanguage: case EO, Python case Java(j2eo: Option[Path]) diff --git a/src/main/scala/org/polystat/Python.scala b/src/main/scala/org/polystat/Python.scala new file mode 100644 index 0000000..d4cc1cc --- /dev/null +++ b/src/main/scala/org/polystat/Python.scala @@ -0,0 +1,29 @@ +package org.polystat + +import org.polystat.py2eo.transpiler.Transpile +import fs2.io.file.{Files, Path} +import cats.effect.{IO, IOApp} +import org.polystat.PolystatConfig.* +import org.polystat.InputUtils.* + +object Python: + + def analyze(cfg: ProcessedConfig): IO[Unit] = + for + tmp <- cfg.tempDir + _ <- readCodeFromInput(".py", cfg.input) + .evalMap { case (path, code) => + for + maybeCode <- IO.blocking(Transpile(path.toString, code)) + _ <- maybeCode match + case Some(code) => + writeOutputTo(tmp / path.replaceExt(".eo"))(code) + case None => IO.println(s"Couldn't analyze $path...") + yield () + } + .compile + .drain + _ <- EO.analyze(cfg.copy(input = Input.FromDirectory(tmp))) + yield () + +end Python