squashed: flatgraph port WIP

backup of individual commits:
michael/odb2-transition/dropin-replacement-wip-backup*
michael/flatgraph.backup*

back to prefix with _ for technical accessors

fixup

upgrade cpg

test with heapspike fix

depends on joernio/flatgraph#213, not merged yet

latest cpg

latest cpg with heapspikefix #2

minor work on reachingDef

upgrade cpg

build.sbt

@@ -1,8 +1,9 @@
 name                     := "joern"
-ThisBuild / organization := "io.joern"
+// TODO change org back to io.joern, only changed temporarily so I can debug something on jenkins
+ThisBuild / organization := "com.michaelpollmeier"
 ThisBuild / scalaVersion := "3.4.2"
 
-val cpgVersion = "1.6.18"
+val cpgVersion = "1.6.16+21-b420d745"
 
 lazy val joerncli          = Projects.joerncli
 lazy val querydb           = Projects.querydb
@@ -73,7 +74,8 @@ Global / onChangedBuildSource := ReloadOnSourceChanges
 
 // publishing info for sonatype / maven central
 ThisBuild / publishTo  := sonatypePublishToBundle.value
-sonatypeCredentialHost := "s01.oss.sonatype.org"
+// TODO change back
+// sonatypeCredentialHost := "s01.oss.sonatype.org"
 ThisBuild / scmInfo    := Some(ScmInfo(url("https://github.com/joernio/joern"), "scm:git@github.com:joernio/joern.git"))
 ThisBuild / homepage   := Some(url("https://joern.io/"))
 ThisBuild / licenses   := List("Apache-2.0" -> url("http://www.apache.org/licenses/LICENSE-2.0"))

console/build.sbt

@@ -10,7 +10,7 @@ dependsOn(
 )
 
 libraryDependencies ++= Seq(
-  "io.shiftleft"         %% "codepropertygraph"    % Versions.cpg,
+  "com.michaelpollmeier"         %% "codepropertygraph"    % Versions.cpg,
   "com.michaelpollmeier" %% "scala-repl-pp-server" % Versions.scalaReplPP,
   "com.github.scopt"     %% "scopt"                % Versions.scopt,
   "org.typelevel"        %% "cats-effect"          % Versions.catsEffect,

console/src/main/scala/io/joern/console/Console.scala

@@ -13,7 +13,7 @@ import io.shiftleft.semanticcpg.language.*
 import io.shiftleft.semanticcpg.language.dotextension.ImageViewer
 import io.shiftleft.semanticcpg.layers.{LayerCreator, LayerCreatorContext}
 import io.shiftleft.codepropertygraph.generated.help.Doc
-import overflowdb.traversal.help.Table.AvailableWidthProvider
+import flatgraph.help.Table.AvailableWidthProvider
 
 import scala.sys.process.Process
 import scala.util.control.NoStackTrace
@@ -349,10 +349,14 @@ class Console[T <: Project](loader: WorkspaceLoader[T], baseDir: File = File.cur
 
     val cpgDestinationPath = cpgDestinationPathOpt.get
 
-    if (CpgLoader.isLegacyCpg(cpgFile)) {
-      report("You have provided a legacy proto CPG. Attempting conversion.")
+    val isProtoFormat      = CpgLoader.isProtoFormat(cpgFile.path)
+    val isOverflowDbFormat = CpgLoader.isOverflowDbFormat(cpgFile.path)
+    if (isProtoFormat || isOverflowDbFormat) {
+      if (isProtoFormat) report("You have provided a legacy proto CPG. Attempting conversion.")
+      else if (isOverflowDbFormat) report("You have provided a legacy overflowdb CPG. Attempting conversion.")
       try {
-        CpgConverter.convertProtoCpgToOverflowDb(cpgFile.path.toString, cpgDestinationPath.toString)
+        val cpg = CpgLoader.load(cpgFile.path, cpgDestinationPath)
+        cpg.close()
       } catch {
         case exc: Exception =>
           report("Error converting legacy CPG: " + exc.getMessage)

console/src/main/scala/io/joern/console/CpgConverter.scala

@@ -1,14 +1,18 @@
 package io.joern.console
 
-import io.shiftleft.codepropertygraph.cpgloading.{CpgLoader, CpgLoaderConfig}
-import overflowdb.Config
+import io.shiftleft.codepropertygraph.cpgloading.{CpgLoader, ProtoCpgLoader}
+
+import java.nio.file.Paths
 
 object CpgConverter {
 
-  def convertProtoCpgToOverflowDb(srcFilename: String, dstFilename: String): Unit = {
-    val odbConfig = Config.withDefaults.withStorageLocation(dstFilename)
-    val config    = CpgLoaderConfig.withDefaults.doNotCreateIndexesOnLoad.withOverflowConfig(odbConfig)
-    CpgLoader.load(srcFilename, config).close
+  def convertProtoCpgToFlatgraph(srcFilename: String, dstFilename: String): Unit = {
+    val cpg = ProtoCpgLoader.loadFromProtoZip(srcFilename, Option(Paths.get(dstFilename)))
+    cpg.close()
   }
 
+  @deprecated("method got renamed to `convertProtoCpgToFlatgraph, please use that instead", "joern v3")
+  def convertProtoCpgToOverflowDb(srcFilename: String, dstFilename: String): Unit =
+    convertProtoCpgToFlatgraph(srcFilename, dstFilename)
+
 }

console/src/main/scala/io/joern/console/Help.scala

@@ -1,8 +1,8 @@
 package io.joern.console
 
-import overflowdb.traversal.help.DocFinder.*
-import overflowdb.traversal.help.Table.AvailableWidthProvider
-import overflowdb.traversal.help.{DocFinder, Table}
+import flatgraph.help.DocFinder.*
+import flatgraph.help.Table.AvailableWidthProvider
+import flatgraph.help.{DocFinder, Table}
 
 object Help {
 

console/src/main/scala/io/joern/console/Run.scala

@@ -75,7 +75,7 @@ object Run {
 
     val toStringCode =
       s"""
-         |  import overflowdb.traversal.help.Table
+         |  import flatgraph.help.Table
          |  override def toString() : String = {
          |    val columnNames = List("name", "description")
          |    val rows =

console/src/main/scala/io/joern/console/cpgcreation/CpgGeneratorFactory.scala

@@ -2,10 +2,9 @@ package io.joern.console.cpgcreation
 
 import better.files.Dsl.*
 import better.files.File
-import io.shiftleft.codepropertygraph.cpgloading.{CpgLoader, CpgLoaderConfig}
+import io.shiftleft.codepropertygraph.cpgloading.CpgLoader
 import io.shiftleft.codepropertygraph.generated.Languages
-import io.joern.console.ConsoleConfig
-import overflowdb.Config
+import io.joern.console.{ConsoleConfig, CpgConverter}
 
 import java.nio.file.Path
 import scala.util.Try
@@ -60,12 +59,12 @@ class CpgGeneratorFactory(config: ConsoleConfig) {
       generator.generate(inputPath, outputPath).map(File(_))
     outputFileOpt.map { outFile =>
       val parentPath = outFile.parent.path.toAbsolutePath
-      if (isZipFile(outFile)) {
+      if (CpgLoader.isProtoFormat(outFile.path)) {
         report("Creating database from bin.zip")
         val srcFilename = outFile.path.toAbsolutePath.toString
         val dstFilename = parentPath.resolve("cpg.bin").toAbsolutePath.toString
         // MemoryHelper.hintForInsufficientMemory(srcFilename).map(report)
-        convertProtoCpgToOverflowDb(srcFilename, dstFilename)
+        convertProtoCpgToFlatgraph(srcFilename, dstFilename)
       } else {
         report("moving cpg.bin.zip to cpg.bin because it is already a database file")
         val srcPath = parentPath.resolve("cpg.bin.zip")
@@ -77,18 +76,13 @@ class CpgGeneratorFactory(config: ConsoleConfig) {
     }
   }
 
-  def convertProtoCpgToOverflowDb(srcFilename: String, dstFilename: String): Unit = {
-    val odbConfig = Config.withDefaults.withStorageLocation(dstFilename)
-    val config    = CpgLoaderConfig.withDefaults.doNotCreateIndexesOnLoad.withOverflowConfig(odbConfig)
-    CpgLoader.load(srcFilename, config).close
-    File(srcFilename).delete()
-  }
+  @deprecated("method got renamed to `convertProtoCpgToFlatgraph, please use that instead", "joern v3")
+  def convertProtoCpgToOverflowDb(srcFilename: String, dstFilename: String): Unit =
+    convertProtoCpgToFlatgraph(srcFilename, dstFilename)
 
-  def isZipFile(file: File): Boolean = {
-    val bytes = file.bytes
-    Try {
-      bytes.next() == 'P' && bytes.next() == 'K'
-    }.getOrElse(false)
+  def convertProtoCpgToFlatgraph(srcFilename: String, dstFilename: String): Unit = {
+    CpgConverter.convertProtoCpgToFlatgraph(srcFilename, dstFilename)
+    File(srcFilename).delete()
   }
 
   private def report(str: String): Unit = System.err.println(str)

console/src/main/scala/io/joern/console/cpgcreation/ImportCode.scala

@@ -5,8 +5,8 @@ import io.joern.console.workspacehandling.Project
 import io.joern.console.{ConsoleException, FrontendConfig, Reporting}
 import io.shiftleft.codepropertygraph.generated.Cpg
 import io.shiftleft.codepropertygraph.generated.Languages
-import overflowdb.traversal.help.Table
-import overflowdb.traversal.help.Table.AvailableWidthProvider
+import flatgraph.help.Table
+import flatgraph.help.Table.AvailableWidthProvider
 
 import java.nio.file.Path
 import scala.util.{Failure, Success, Try}

console/src/main/scala/io/joern/console/package.scala

@@ -1,6 +1,6 @@
 package io.joern
 
-import overflowdb.traversal.help.Table.AvailableWidthProvider
+import flatgraph.help.Table.AvailableWidthProvider
 import replpp.Operators.*
 import replpp.Colors
 

console/src/main/scala/io/joern/console/scan/package.scala

@@ -11,11 +11,6 @@ package object scan {
 
   private val logger: Logger = LoggerFactory.getLogger(this.getClass)
 
-  implicit class ScannerStarters(val cpg: Cpg) extends AnyVal {
-    def finding: Iterator[Finding] =
-      overflowdb.traversal.InitialTraversal.from[Finding](cpg.graph, NodeTypes.FINDING)
-  }
-
   implicit class QueryWrapper(q: Query) {
 
     /** Obtain list of findings by running query on CPG

console/src/main/scala/io/joern/console/workspacehandling/Workspace.scala

@@ -1,8 +1,8 @@
 package io.joern.console.workspacehandling
 
 import io.joern.console.defaultAvailableWidthProvider
-import overflowdb.traversal.help.Table
-import overflowdb.traversal.help.Table.AvailableWidthProvider
+import flatgraph.help.Table
+import flatgraph.help.Table.AvailableWidthProvider
 
 import scala.collection.mutable.ListBuffer
 

console/src/main/scala/io/joern/console/workspacehandling/WorkspaceLoader.scala

@@ -2,7 +2,7 @@ package io.joern.console.workspacehandling
 
 import better.files.Dsl.mkdirs
 import better.files.File
-import overflowdb.traversal.help.Table.AvailableWidthProvider
+import flatgraph.help.Table.AvailableWidthProvider
 
 import java.nio.file.Path
 import scala.collection.mutable.ListBuffer

console/src/main/scala/io/joern/console/workspacehandling/WorkspaceManager.scala

@@ -6,13 +6,12 @@ import io.joern.console
 import io.joern.console.defaultAvailableWidthProvider
 import io.joern.console.Reporting
 import io.shiftleft.codepropertygraph.generated.Cpg
-import io.shiftleft.codepropertygraph.cpgloading.{CpgLoader, CpgLoaderConfig}
+import io.shiftleft.codepropertygraph.cpgloading.CpgLoader
 import org.json4s.DefaultFormats
-import org.json4s.native.Serialization.{write => jsonWrite}
-import overflowdb.Config
+import org.json4s.native.Serialization.write as jsonWrite
 
 import java.net.URLEncoder
-import java.nio.file.Path
+import java.nio.file.{Path, Paths}
 import scala.collection.mutable.ListBuffer
 import scala.util.{Failure, Success, Try}
 
@@ -305,17 +304,12 @@ class WorkspaceManager[ProjectType <: Project](path: String, loader: WorkspaceLo
 
   private def loadCpgRaw(cpgFilename: String): Option[Cpg] = {
     Try {
-      val odbConfig = Config.withDefaults.withStorageLocation(cpgFilename)
-      val config =
-        CpgLoaderConfig.withDefaults.doNotCreateIndexesOnLoad.withOverflowConfig(odbConfig)
-      val newCpg = CpgLoader.loadFromOverflowDb(config)
-      CpgLoader.createIndexes(newCpg)
-      newCpg
+      CpgLoader.load(cpgFilename)
     } match {
       case Success(v) => Some(v)
       case Failure(ex) =>
         System.err.println("Error loading CPG")
-        System.err.println(ex)
+        ex.printStackTrace()
         None
     }
   }

console/src/test/scala/io/joern/console/ConsoleTests.scala

@@ -428,7 +428,7 @@ class ConsoleTests extends AnyWordSpec with Matchers {
   "cpg" should {
     "provide .help command" in ConsoleFixture() { (console, codeDir) =>
       // part of Predefined.shared, which makes the below work in the repl without separate import
-      import io.shiftleft.codepropertygraph.Cpg.docSearchPackages
+      import io.shiftleft.semanticcpg.language.docSearchPackages
       import io.joern.console.testing.availableWidthProvider
 
       console.importCode(codeDir.toString)

console/src/test/scala/io/joern/console/testing/package.scala

@@ -3,7 +3,7 @@ package io.joern.console
 import better.files.Dsl.*
 import better.files.*
 import io.joern.console.workspacehandling.Project
-import overflowdb.traversal.help.Table.{AvailableWidthProvider, ConstantWidth}
+import flatgraph.help.Table.{AvailableWidthProvider, ConstantWidth}
 
 import scala.util.Try
 

dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/dotgenerator/DdgGenerator.scala

@@ -2,14 +2,12 @@ package io.joern.dataflowengineoss.dotgenerator
 
 import io.joern.dataflowengineoss.DefaultSemantics
 import io.shiftleft.codepropertygraph.generated.nodes.*
-import io.shiftleft.codepropertygraph.generated.{EdgeTypes, Properties}
+import io.shiftleft.codepropertygraph.generated.EdgeTypes
 import io.joern.dataflowengineoss.language.*
 import io.joern.dataflowengineoss.semanticsloader.Semantics
 import io.shiftleft.semanticcpg.dotgenerator.DotSerializer.{Edge, Graph}
 import io.shiftleft.semanticcpg.language.*
 import io.shiftleft.semanticcpg.utils.MemberAccess.isGenericMemberAccessName
-import overflowdb.Node
-import overflowdb.traversal.jIteratortoTraversal
 
 import scala.collection.mutable
 
@@ -59,7 +57,7 @@ class DdgGenerator {
     }
   }
 
-  private def shouldBeDisplayed(v: Node): Boolean = !(
+  private def shouldBeDisplayed(v: StoredNode): Boolean = !(
     v.isInstanceOf[ControlStructure] ||
       v.isInstanceOf[JumpTarget]
   )
@@ -91,7 +89,16 @@ class DdgGenerator {
     val allInEdges = v
       .inE(EdgeTypes.REACHING_DEF)
       .map(x =>
-        Edge(x.outNode.asInstanceOf[StoredNode], v, srcVisible = true, x.property(Properties.Variable), edgeType)
+        // note: this looks strange, but let me explain...
+        // in overflowdb, edges were allowed multiple properties and this used to be `x.property(Properties.VARIABLE)`
+        // in flatgraph an edge may have zero or one properties and they're not named...
+        // in this case we know that we're dealing with ReachingDef edges which has the `variable` property
+        val variablePropertyMaybe = x.property match {
+          case null                     => null
+          case variableProperty: String => variableProperty
+          case _                        => null
+        }
+        Edge(x.src.asInstanceOf[StoredNode], v, srcVisible = true, variablePropertyMaybe, edgeType)
       )
 
     v match {

dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/language/Path.scala

@@ -3,8 +3,8 @@ package io.joern.dataflowengineoss.language
 import io.shiftleft.codepropertygraph.generated.nodes.{AstNode, CfgNode, Member, MethodParameterIn}
 import io.shiftleft.semanticcpg
 import io.shiftleft.semanticcpg.language.*
-import overflowdb.traversal.help.Table
-import overflowdb.traversal.help.Table.AvailableWidthProvider
+import flatgraph.help.Table
+import flatgraph.help.Table.AvailableWidthProvider
 
 case class Path(elements: List[AstNode]) {
   def resultPairs(): List[(String, Option[Int])] = {

dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/passes/reachingdef/DdgGenerator.scala

@@ -4,7 +4,7 @@ import io.joern.dataflowengineoss.{globalFromLiteral, identifierToFirstUsages}
 import io.joern.dataflowengineoss.queryengine.AccessPathUsage.toTrackedBaseAndAccessPathSimple
 import io.joern.dataflowengineoss.semanticsloader.Semantics
 import io.shiftleft.codepropertygraph.generated.nodes.*
-import io.shiftleft.codepropertygraph.generated.{EdgeTypes, Operators, PropertyNames}
+import io.shiftleft.codepropertygraph.generated.{EdgeTypes, Operators}
 import io.shiftleft.semanticcpg.accesspath.MatchResult
 import io.shiftleft.semanticcpg.language.*
 import io.shiftleft.codepropertygraph.generated.DiffGraphBuilder
@@ -132,7 +132,7 @@ class DdgGenerator(semantics: Semantics) {
       // There is always an edge from the method input parameter
       // to the corresponding method output parameter as modifications
       // of the input parameter only affect a copy.
-      paramOut.paramIn.foreach { paramIn =>
+      paramOut.start.paramIn.foreach { paramIn =>
         addEdge(paramIn, paramOut, paramIn.name)
       }
       usageAnalyzer.usedIncomingDefs(paramOut).foreach { case (_, inElements) =>
@@ -224,7 +224,7 @@ class DdgGenerator(semantics: Semantics) {
 
     (fromNode, toNode) match {
       case (parentNode: CfgNode, childNode: CfgNode) if EdgeValidator.isValidEdge(childNode, parentNode) =>
-        dstGraph.addEdge(fromNode, toNode, EdgeTypes.REACHING_DEF, PropertyNames.VARIABLE, variable)
+        dstGraph.addEdge(fromNode, toNode, EdgeTypes.REACHING_DEF, variable)
       case _ =>
 
     }