Merge pull request #12 from nikololiahim/py2eo

added integration with py2eo

.polystat.conf

@@ -4,5 +4,4 @@ polystat {
     outputTo = .
     tempDir = tmp
     outputFormats = [sarif]
-    # excludeRules = [e, c, dialect]
 }

CHANGELOG.md

@@ -1,5 +1,16 @@
-## Polystat v0.1.3
+## Polystat v0.1.4
 
-In this release the `odin` dependency was updated to 0.4.0. 
+In this release the `py2eo` project was integrated into `polystat-cli`. You can now run:
+```
+polystat py --in python_files
+```
 
-The CD pipeline was updated to allow releasing a specified version. 
+...to analyze a directory with a bunch of python files. For more options and explanations, run:
+```
+polystat --help
+```
+or
+```
+polystat list --config
+```
+if you want to use the config file. 

build.sbt

@@ -26,6 +26,7 @@ libraryDependencies ++= Seq(
   "io.circe" %% "circe-core" % "0.14.1",
   "org.scalameta" %% "munit" % "1.0.0-M3" % Test,
   "org.slf4j" % "slf4j-nop" % "1.7.36",
+  "org.polystat.py2eo" % "transpiler" % "0.0.10",
 )
 
 assembly / assemblyJarName := "polystat.jar"

sandbox_python/test.py

@@ -0,0 +1,3 @@
+def conditionalCheck2():
+    a = 4
+    b = 2

src/main/scala/org/polystat/EO.scala

@@ -0,0 +1,44 @@
+package org.polystat
+
+import cats.effect.IO
+import PolystatConfig.*
+import InputUtils.*
+import org.polystat.odin.analysis.ASTAnalyzer
+import org.polystat.odin.analysis.EOOdinAnalyzer
+import org.polystat.odin.parser.EoParser.sourceCodeEoParser
+import cats.syntax.traverse.*
+import cats.syntax.foldable.*
+
+object EO:
+
+  def runAnalyzers(
+      analyzers: List[ASTAnalyzer[IO]]
+  )(code: String): IO[List[EOOdinAnalyzer.OdinAnalysisResult]] =
+    analyzers.traverse(a =>
+      EOOdinAnalyzer
+        .analyzeSourceCode(a)(code)(cats.Monad[IO], sourceCodeEoParser[IO]())
+    )
+
+  def analyze(cfg: ProcessedConfig): IO[Unit] =
+    val inputFiles = readCodeFromInput(".eo", cfg.input)
+    inputFiles
+      .evalMap { case (codePath, code) =>
+        for
+          _ <- IO.println(s"Analyzing $codePath...")
+          analyzed <- runAnalyzers(cfg.filteredAnalyzers)(code)
+          _ <- cfg.output match
+            case Output.ToConsole => IO.println(analyzed)
+            case Output.ToDirectory(out) =>
+              cfg.fmts.traverse_ { case OutputFormat.Sarif =>
+                val outPath =
+                  out / "sarif" / codePath.replaceExt(".sarif.json")
+                val sarifJson = SarifOutput(analyzed).json.toString
+                IO.println(s"Writing results to $outPath") *>
+                  writeOutputTo(outPath)(sarifJson)
+              }
+        yield ()
+      }
+      .compile
+      .drain
+  end analyze
+end EO

src/main/scala/org/polystat/InputUtils.scala

@@ -9,7 +9,7 @@ import fs2.text.utf8
 
 import java.io.FileNotFoundException
 
-import PolystatConfig.Input
+import PolystatConfig.{Input, PolystatUsage}
 
 object InputUtils:
   extension (path: Path)
@@ -73,4 +73,22 @@ object InputUtils:
         readCodeFromDir(ext = ext, dir = path)
       case Input.FromStdin =>
         readCodeFromStdin.map(code => (Path("stdin" + ext), "\n" + code + "\n"))
+
+  def readConfigFromFile(path: Path): IO[PolystatUsage.Analyze] =
+    HoconConfig(path).config.load
+
+  def writeOutputTo(path: Path)(output: String): IO[Unit] =
+    for
+      _ <- path.parent
+        .map(Files[IO].createDirectories)
+        .getOrElse(IO.unit)
+      _ <- Stream
+        .emits(output.getBytes)
+        .through(Files[IO].writeAll(path))
+        .compile
+        .drain
+    yield ()
+    end for
+  end writeOutputTo
+
 end InputUtils

src/main/scala/org/polystat/J2EO.scala → src/main/scala/org/polystat/Java.scala

@@ -13,8 +13,10 @@ import org.http4s.client.middleware.FollowRedirect
 import fs2.io.file.{Path, Files}
 import org.http4s.Uri
 import sys.process.*
+import PolystatConfig.*
+import InputUtils.*
 
-object J2EO:
+object Java:
 
   private val DEFAULT_J2EO_PATH = Path("j2eo.jar")
   private val J2EO_URL =
@@ -53,7 +55,11 @@ object J2EO:
       }
   end downloadJ2EO
 
-  def run(j2eo: Option[Path], inputDir: Path, outputDir: Path): IO[Unit] =
+  private def runJ2EO(
+      j2eo: Option[Path],
+      inputDir: Path,
+      outputDir: Path,
+  ): IO[Unit] =
     val command =
       s"java -jar ${j2eo.getOrElse(DEFAULT_J2EO_PATH)} -o $outputDir $inputDir"
     for
@@ -69,6 +75,28 @@ object J2EO:
         )
     yield ()
     end for
-  end run
+  end runJ2EO
 
-end J2EO
+  def analyze(j2eo: Option[Path], cfg: ProcessedConfig): IO[Unit] =
+    for
+      tmp <- cfg.tempDir
+      _ <- cfg.input match // writing EO files to tempDir
+        case Input.FromStdin =>
+          for
+            code <- readCodeFromStdin.compile.string
+            stdinTmp <- Files[IO].createTempDirectory.map(path =>
+              path / "stdin.eo"
+            )
+            _ <- writeOutputTo(stdinTmp)(code)
+            _ <- runJ2EO(j2eo, inputDir = stdinTmp, outputDir = tmp)
+          yield ()
+        case Input.FromFile(path) =>
+          runJ2EO(j2eo, inputDir = path, outputDir = tmp)
+        case Input.FromDirectory(path) =>
+          runJ2EO(j2eo, inputDir = path, outputDir = tmp)
+      _ <- EO.analyze(
+        cfg.copy(input = Input.FromDirectory(tmp))
+      )
+    yield ()
+
+end Java

src/main/scala/org/polystat/Main.scala

@@ -14,10 +14,12 @@ import org.polystat.odin.analysis.ASTAnalyzer
 import org.polystat.odin.analysis.EOOdinAnalyzer
 import org.polystat.odin.analysis.EOOdinAnalyzer.OdinAnalysisResult
 import org.polystat.odin.parser.EoParser.sourceCodeEoParser
+import org.polystat.py2eo.transpiler.Transpile
 
 import PolystatConfig.*
 import IncludeExclude.*
 import InputUtils.*
+import org.polystat.py2eo.parser.PythonLexer
 object Main extends IOApp:
   override def run(args: List[String]): IO[ExitCode] =
     for exitCode <- CommandIOApp.run(
@@ -49,66 +51,14 @@ object Main extends IOApp:
         }
       case None => analyzers.map(_._2)
 
-  def analyze(
-      analyzers: List[ASTAnalyzer[IO]]
-  )(code: String): IO[List[EOOdinAnalyzer.OdinAnalysisResult]] =
-    analyzers.traverse(a =>
-      EOOdinAnalyzer
-        .analyzeSourceCode(a)(code)(cats.Monad[IO], sourceCodeEoParser[IO]())
-    )
-
-  def listAnalyzers: IO[Unit] = analyzers.traverse_ { case (name, _) =>
-    IO.println(name)
-  }
-
-  def readConfigFromFile(path: Path): IO[PolystatUsage.Analyze] =
-    HoconConfig(path).config.load
-
-  def writeOutputTo(path: Path)(output: String): IO[Unit] =
-    for
-      _ <- path.parent
-        .map(Files[IO].createDirectories)
-        .getOrElse(IO.unit)
-      _ <- Stream
-        .emits(output.getBytes)
-        .through(Files[IO].writeAll(path))
-        .compile
-        .drain
-    yield ()
-    end for
-  end writeOutputTo
-
-  def analyzeEO(
-      inputFiles: Stream[IO, (Path, String)],
-      outputFormats: List[OutputFormat],
-      out: Output,
-      filteredAnalyzers: List[ASTAnalyzer[IO]],
-  ): IO[Unit] =
-    inputFiles
-      .evalMap { case (codePath, code) =>
-        for
-          _ <- IO.println(s"Analyzing $codePath...")
-          analyzed <- analyze(filteredAnalyzers)(code)
-          _ <- out match
-            case Output.ToConsole => IO.println(analyzed)
-            case Output.ToDirectory(out) =>
-              outputFormats.traverse_ { case OutputFormat.Sarif =>
-                val outPath =
-                  out / "sarif" / codePath.replaceExt(".sarif.json")
-                val sarifJson = SarifOutput(analyzed).json.toString
-                IO.println(s"Writing results to $outPath") *>
-                  writeOutputTo(outPath)(sarifJson)
-              }
-        yield ()
-      }
-      .compile
-      .drain
-
   def execute(usage: PolystatUsage): IO[Unit] =
     usage match
       case PolystatUsage.List(cfg) =>
-        if (cfg) then IO.println(HoconConfig.keys.explanation)
-        else listAnalyzers
+        if cfg then IO.println(HoconConfig.keys.explanation)
+        else
+          analyzers.traverse_ { case (name, _) =>
+            IO.println(name)
+          }
       case PolystatUsage.Misc(version, config) =>
         if (version) then IO.println(BuildInfo.version)
         else
@@ -118,55 +68,26 @@ object Main extends IOApp:
             lang,
             AnalyzerConfig(inex, input, tmp, fmts, out),
           ) =>
-        val filteredAnalyzers = filterAnalyzers(inex)
-        val tempDir: IO[Path] = tmp match
-          case Some(path) => IO.pure(path)
-          case None       => Files[IO].createTempDirectory
-        val inputExt: String = lang match
-          case SupportedLanguage.EO      => ".eo"
-          case SupportedLanguage.Java(_) => ".java"
-          case SupportedLanguage.Python  => ".py"
-
+        val processedConfig = ProcessedConfig(
+          filteredAnalyzers = filterAnalyzers(inex),
+          tempDir = tmp match
+            case Some(path) =>
+              (IO.println(s"Cleaning ${path.absolute}...") *>
+                Files[IO].deleteRecursively(path) *>
+                Files[IO].createDirectory(path))
+                .as(path)
+            case None => Files[IO].createTempDirectory
+          ,
+          output = out,
+          input = input,
+          fmts = fmts,
+        )
         val analysisResults: IO[Unit] =
           lang match
-            case SupportedLanguage.EO =>
-              val inputFiles = readCodeFromInput(ext = inputExt, input = input)
-              analyzeEO(
-                inputFiles = inputFiles,
-                outputFormats = fmts,
-                out = out,
-                filteredAnalyzers = filteredAnalyzers,
-              )
+            case SupportedLanguage.EO => EO.analyze(processedConfig)
             case SupportedLanguage.Java(j2eo) =>
-              for
-                tmp <- tempDir
-                _ <- input match // writing EO files to tempDir
-                  case Input.FromStdin =>
-                    for
-                      code <- readCodeFromStdin.compile.string
-                      stdinTmp <- Files[IO].createTempDirectory.map(path =>
-                        path / "stdin.eo"
-                      )
-                      _ <- writeOutputTo(stdinTmp)(code)
-                      _ <- J2EO.run(j2eo, inputDir = stdinTmp, outputDir = tmp)
-                    yield ()
-                  case Input.FromFile(path) =>
-                    J2EO.run(j2eo, inputDir = path, outputDir = tmp)
-                  case Input.FromDirectory(path) =>
-                    J2EO.run(j2eo, inputDir = path, outputDir = tmp)
-                inputFiles = readCodeFromInput(
-                  ".eo",
-                  Input.FromDirectory(tmp),
-                )
-                _ <- analyzeEO(
-                  inputFiles = inputFiles,
-                  outputFormats = fmts,
-                  out = out,
-                  filteredAnalyzers = filteredAnalyzers,
-                )
-              yield ()
-            case SupportedLanguage.Python =>
-              IO.println("Analyzing Python is not implemented yet!")
+              Java.analyze(j2eo, processedConfig)
+            case SupportedLanguage.Python => Python.analyze(processedConfig)
         analysisResults
   end execute
 

src/main/scala/org/polystat/PolystatConfig.scala

@@ -6,6 +6,7 @@ import cats.syntax.apply.*
 import com.monovore.decline.Opts
 import fs2.Stream
 import fs2.io.file.Path
+import org.polystat.odin.analysis.ASTAnalyzer
 
 object PolystatConfig:
 
@@ -17,6 +18,14 @@ object PolystatConfig:
       output: Output,
   )
 
+  case class ProcessedConfig(
+      filteredAnalyzers: List[ASTAnalyzer[IO]],
+      tempDir: IO[Path],
+      input: Input,
+      fmts: List[OutputFormat],
+      output: Output,
+  )
+
   enum SupportedLanguage:
     case EO, Python
     case Java(j2eo: Option[Path])

src/main/scala/org/polystat/Python.scala

@@ -0,0 +1,29 @@
+package org.polystat
+
+import org.polystat.py2eo.transpiler.Transpile
+import fs2.io.file.{Files, Path}
+import cats.effect.{IO, IOApp}
+import org.polystat.PolystatConfig.*
+import org.polystat.InputUtils.*
+
+object Python:
+
+  def analyze(cfg: ProcessedConfig): IO[Unit] =
+    for
+      tmp <- cfg.tempDir
+      _ <- readCodeFromInput(".py", cfg.input)
+        .evalMap { case (path, code) =>
+          for
+            maybeCode <- IO.blocking(Transpile(path.toString, code))
+            _ <- maybeCode match
+              case Some(code) =>
+                writeOutputTo(tmp / path.replaceExt(".eo"))(code)
+              case None => IO.println(s"Couldn't analyze $path...")
+          yield ()
+        }
+        .compile
+        .drain
+      _ <- EO.analyze(cfg.copy(input = Input.FromDirectory(tmp)))
+    yield ()
+
+end Python