From cd98eee54585259ba25013aff8278520afecb873 Mon Sep 17 00:00:00 2001 From: Michael Pollmeier Date: Thu, 14 Sep 2023 14:57:15 +0200 Subject: [PATCH] squashed: flatgraph port WIP backup of individual commits: michael/odb2-transition/dropin-replacement-wip-backup* michael/flatgraph.backup* back to prefix with _ for technical accessors fixup upgrade cpg test with heapspike fix depends on https://github.com/joernio/flatgraph/pull/213, not merged yet latest cpg latest cpg with heapspikefix #2 minor work on reachingDef upgrade cpg --- build.sbt | 8 +- console/build.sbt | 2 +- .../main/scala/io/joern/console/Console.scala | 12 +- .../scala/io/joern/console/CpgConverter.scala | 16 +- .../main/scala/io/joern/console/Help.scala | 6 +- .../src/main/scala/io/joern/console/Run.scala | 2 +- .../cpgcreation/CpgGeneratorFactory.scala | 26 +- .../console/cpgcreation/ImportCode.scala | 4 +- .../main/scala/io/joern/console/package.scala | 2 +- .../scala/io/joern/console/scan/package.scala | 5 - .../console/workspacehandling/Workspace.scala | 4 +- .../workspacehandling/WorkspaceLoader.scala | 2 +- .../workspacehandling/WorkspaceManager.scala | 16 +- .../scala/io/joern/console/ConsoleTests.scala | 2 +- .../io/joern/console/testing/package.scala | 2 +- .../cpgloading/TestProtoCpg.scala | 45 --- .../dotgenerator/DdgGenerator.scala | 17 +- .../dataflowengineoss/language/Path.scala | 4 +- .../passes/reachingdef/DdgGenerator.scala | 6 +- .../queryengine/AccessPathUsage.scala | 2 +- .../queryengine/Engine.scala | 15 +- .../queryengine/SourcesToStartingPoints.scala | 5 +- .../queryengine/TaskCreator.scala | 2 +- .../slicing/DataFlowSlicing.scala | 15 +- .../slicing/UsageSlicing.scala | 15 +- .../dataflowengineoss/slicing/package.scala | 17 +- .../queryengine/AccessPathUsageTests.scala | 44 +-- joern-cli/build.sbt | 2 +- .../astcreation/AstForFunctionsCreator.scala | 2 +- .../c2cpg/astcreation/AstNodeBuilder.scala | 5 +- .../joern/c2cpg/dataflow/DataFlowTests.scala | 5 +- .../c2cpg/macros/MacroHandlingTests.scala | 2 +- .../c2cpg/passes/MetaDataPassTests.scala | 6 +- .../passes/ast/AstCreationPassTests.scala | 7 +- .../io/joern/c2cpg/passes/ast/CallTests.scala | 2 +- .../querying/NodeTypeStarterQueryTests.scala | 4 +- joern-cli/frontends/csharpsrc2cpg/build.sbt | 4 +- .../astcreation/AstCreator.scala | 2 +- .../astcreation/AstCreatorHelper.scala | 8 +- .../astcreation/AstForStatementsCreator.scala | 2 +- .../astcreation/AstSummaryVisitor.scala | 10 +- .../datastructures/CSharpProgramSummary.scala | 2 +- joern-cli/frontends/ghidra2cpg/build.sbt | 4 +- .../ghidra2cpg/passes/FunctionPass.scala | 2 +- .../joern/ghidra2cpg/passes/PCodePass.scala | 3 +- .../passes/arm/ArmFunctionPass.scala | 3 +- .../ghidra2cpg/passes/mips/LoHiPass.scala | 5 +- .../passes/mips/MipsFunctionPass.scala | 3 +- .../passes/mips/MipsReturnEdgesPass.scala | 5 +- .../passes/x86/ReturnEdgesPass.scala | 6 +- .../fixtures/DataFlowBinToCpgSuite.scala | 5 +- .../test/testbinaries/coverage/testscript.sc | 2 +- joern-cli/frontends/gosrc2cpg/build.sbt | 4 +- .../astcreation/AstCreatorHelper.scala | 2 +- .../astcreation/AstForFunctionsCreator.scala | 1 + .../AstForMethodCallExpressionCreator.scala | 2 +- .../AstForPackageConstructorCreator.scala | 2 + .../astcreation/CommonCacheBuilder.scala | 4 +- .../passes/ast/DownloadDependencyTest.scala | 18 +- .../go2cpg/passes/ast/MethodCallTests.scala | 3 +- joern-cli/frontends/javasrc2cpg/build.sbt | 2 +- .../javasrc2cpg/astcreation/AstCreator.scala | 8 +- .../declarations/AstForMethodsCreator.scala | 4 +- .../AstForCallExpressionsCreator.scala | 4 +- .../expressions/AstForLambdasCreator.scala | 3 +- .../AstForVarDeclAndAssignsCreator.scala | 6 +- .../statements/AstForForLoopsCreator.scala | 2 +- .../passes/TypeInferencePass.scala | 16 +- .../javasrc2cpg/scope/JavaScopeElement.scala | 2 +- .../io/joern/javasrc2cpg/util/Util.scala | 5 +- .../passes/ConfigFileCreationPassTests.scala | 7 +- .../javasrc2cpg/querying/CallTests.scala | 6 +- .../querying/ConstructorInvocationTests.scala | 32 +- .../querying/ControlStructureTests.scala | 1 - .../javasrc2cpg/querying/LambdaTests.scala | 2 +- joern-cli/frontends/jimple2cpg/build.sbt | 2 +- .../AstForDeclarationsCreator.scala | 10 +- .../statements/AstForStatementsCreator.scala | 10 +- .../jimple2cpg/querying/MetaDataTests.scala | 4 +- .../unpacking/JarUnpackingTests.scala | 9 +- joern-cli/frontends/jssrc2cpg/build.sbt | 4 +- .../astcreation/AstCreatorHelper.scala | 4 +- .../astcreation/AstForTypesCreator.scala | 10 +- .../astcreation/AstNodeBuilder.scala | 5 +- .../passes/JavaScriptTypeNodePass.scala | 5 +- .../jssrc2cpg/dataflow/DataflowTests.scala | 2 +- .../passes/JsMetaDataPassTests.scala | 4 +- .../ast/MixedAstCreationPassTests.scala | 4 +- .../ast/SimpleAstCreationPassTests.scala | 2 +- .../ast/TsClassesAstCreationPassTests.scala | 4 +- joern-cli/frontends/kotlin2cpg/build.sbt | 2 +- .../io/joern/kotlin2cpg/ast/AstCreator.scala | 3 +- .../ast/AstForDeclarationsCreator.scala | 2 +- .../ast/AstForFunctionsCreator.scala | 4 +- .../ast/AstForStatementsCreator.scala | 4 +- .../querying/ComplexExpressionsTests.scala | 1 - .../kotlin2cpg/querying/LambdaTests.scala | 2 +- .../kotlin2cpg/querying/MethodTests.scala | 2 +- joern-cli/frontends/php2cpg/build.sbt | 2 +- .../php2cpg/astcreation/AstCreator.scala | 17 +- .../io/joern/php2cpg/passes/AnyTypePass.scala | 2 +- .../php2cpg/passes/AstParentInfoPass.scala | 5 +- .../joern/php2cpg/passes/ClosureRefPass.scala | 2 +- .../php2cpg/passes/LocalCreationPass.scala | 4 +- .../passes/PhpTypeRecoveryPassTests.scala | 2 +- .../php2cpg/querying/OperatorTests.scala | 2 +- joern-cli/frontends/pysrc2cpg/build.sbt | 2 +- .../scala/io/joern/pysrc2cpg/Py2Cpg.scala | 4 +- .../io/joern/pysrc2cpg/PythonAstVisitor.scala | 1 + .../io/joern/pysrc2cpg/cpg/CallCpgTests.scala | 1 - .../pysrc2cpg/cpg/FunctionDefCpgTests.scala | 6 +- .../cpg/VariableReferencingCpgTests.scala | 8 +- .../pysrc2cpg/dataflow/DataFlowTests.scala | 4 +- .../passes/TypeRecoveryPassTests.scala | 11 +- joern-cli/frontends/rubysrc2cpg/build.sbt | 4 +- .../rubysrc2cpg/astcreation/AstCreator.scala | 3 +- .../AstForExpressionsCreator.scala | 2 +- .../astcreation/AstForStatementsCreator.scala | 2 +- .../astcreation/AstForTypesCreator.scala | 3 +- .../astcreation/AstSummaryVisitor.scala | 7 +- .../deprecated/astcreation/AstCreator.scala | 1 + .../deprecated/astcreation/RubyScope.scala | 1 + .../rubysrc2cpg/passes/AstCreationPass.scala | 4 +- .../RubyTypeRecoveryPassGenerator.scala | 2 +- .../dataflow/SingleAssignmentTests.scala | 14 +- .../deprecated/passes/MetaDataPassTests.scala | 6 +- .../deprecated/passes/ast/CallCpgTests.scala | 4 +- .../passes/ast/MethodTwoTests.scala | 4 +- .../rubysrc2cpg/querying/DoBlockTests.scala | 2 +- joern-cli/frontends/swiftsrc2cpg/build.sbt | 2 +- .../swiftsrc2cpg/astcreation/AstCreator.scala | 8 +- .../astcreation/AstCreatorHelper.scala | 5 +- .../passes/SwiftTypeNodePass.scala | 5 +- .../swiftsrc2cpg/dataflow/DataFlowTests.scala | 3 +- .../src/main/scala/io/joern/x2cpg/Ast.scala | 9 +- .../scala/io/joern/x2cpg/AstCreatorBase.scala | 3 +- .../src/main/scala/io/joern/x2cpg/X2Cpg.scala | 18 +- .../jssrc2cpg/JavaScriptTypeRecovery.scala | 13 +- .../php2cpg/PhpTypeRecovery.scala | 5 +- .../DynamicTypeHintFullNamePass.scala | 1 + .../swiftsrc2cpg/SwiftTypeRecovery.scala | 8 +- .../scala/io/joern/x2cpg/layers/Base.scala | 1 - .../x2cpg/passes/base/ContainsEdgePass.scala | 3 +- .../x2cpg/passes/base/FileCreationPass.scala | 10 +- .../x2cpg/passes/base/TypeEvalPass.scala | 17 +- .../joern/x2cpg/passes/base/TypeRefPass.scala | 17 +- .../passes/callgraph/DynamicCallLinker.scala | 29 +- .../passes/callgraph/MethodRefLinker.scala | 9 +- .../passes/callgraph/NaiveCallLinker.scala | 1 - .../cfgdominator/CfgDominatorFrontier.scala | 2 +- .../x2cpg/passes/frontend/TypeNodePass.scala | 26 +- .../frontend/XInheritanceFullNamePass.scala | 2 +- .../passes/frontend/XTypeHintCallLinker.scala | 5 +- .../x2cpg/passes/frontend/XTypeRecovery.scala | 39 +- .../typerelations/AliasLinkerPass.scala | 1 + .../typerelations/FieldAccessLinkerPass.scala | 2 +- .../typerelations/TypeHierarchyPass.scala | 3 +- .../joern/x2cpg/utils/AstPropertiesUtil.scala | 2 +- .../scala/io/joern/x2cpg/utils/KeyPool.scala | 80 +++++ .../io/joern/x2cpg/utils/LinkingUtil.scala | 27 +- .../test/scala/io/joern/x2cpg/AstTests.scala | 6 +- .../scala/io/joern/x2cpg/X2CpgTests.scala | 10 +- .../passes/CfgDominatorFrontierTests.scala | 96 +++-- .../x2cpg/passes/CfgDominatorPassTests.scala | 85 +++-- .../x2cpg/passes/ContainsEdgePassTest.scala | 62 ++-- .../passes/MethodDecoratorPassTests.scala | 34 +- .../x2cpg/passes/NamespaceCreatorTests.scala | 14 +- .../testfixtures/EmptyGraphFixture.scala | 13 +- .../io/joern/x2cpg/testfixtures/TestCpg.scala | 4 +- .../io/joern/x2cpg/utils/KeyPoolTests.scala | 74 ++++ .../io/joern/joerncli/CpgBasedTool.scala | 13 +- .../io/joern/joerncli/DefaultOverlays.scala | 2 +- .../scala/io/joern/joerncli/JoernExport.scala | 36 +- .../scala/io/joern/joerncli/JoernFlow.scala | 2 +- .../scala/io/joern/joerncli/JoernSlice.scala | 2 +- .../io/joern/joerncli/JoernVectors.scala | 11 +- .../joern/joerncli/console/Predefined.scala | 23 +- .../src/universal/schema-extender/build.sbt | 12 +- .../schema/src/main/scala/CpgExtCodegen.scala | 13 +- .../src/universal/schema-extender/test.sh | 2 +- macros/build.sbt | 5 +- .../scanners/c/HeapBasedOverflowTests.scala | 1 + .../io/joern/scanners/c/MetricsTests.scala | 1 + .../io/joern/suites/JavaQueryTestSuite.scala | 1 + .../joern/suites/KotlinQueryTestSuite.scala | 1 + semanticcpg/build.sbt | 16 +- .../io/shiftleft/semanticcpg/Overlays.scala | 1 + .../semanticcpg/accesspath/TrackedBase.scala | 1 + .../dotgenerator/CdgGenerator.scala | 2 +- .../dotgenerator/CfgGenerator.scala | 5 +- .../dotgenerator/DotSerializer.scala | 8 +- .../language/AccessPathHandling.scala | 8 +- .../language/LocationCreator.scala | 4 +- .../language/NodeExtensionFinder.scala | 25 +- .../semanticcpg/language/NodeSteps.scala | 30 +- .../language/NodeTypeStarters.scala | 338 +++--------------- .../shiftleft/semanticcpg/language/Show.scala | 13 +- .../semanticcpg/language/Steps.scala | 7 +- .../android/ConfigFileTraversal.scala | 5 +- .../callgraphextension/MethodTraversal.scala | 4 +- .../language/dotextension/AstNodeDot.scala | 2 +- .../language/dotextension/CfgNodeDot.scala | 2 +- .../ResolvedImportAsTagTraversal.scala | 7 +- .../ModuleVariableAsNodeTraversal.scala | 10 +- .../ModuleVariableTraversal.scala | 6 +- .../modulevariable/NodeTypeStarters.scala | 2 +- .../language/modulevariable/OpNodes.scala | 3 +- .../ModuleVariableAsNodeMethods.scala | 2 +- .../nodemethods/ModuleVariableMethods.scala | 8 +- .../language/nodemethods/AstNodeMethods.scala | 7 +- .../language/nodemethods/CallMethods.scala | 24 +- .../language/nodemethods/CfgNodeMethods.scala | 6 +- .../nodemethods/IdentifierMethods.scala | 2 +- .../language/nodemethods/LiteralMethods.scala | 2 +- .../language/nodemethods/LocalMethods.scala | 4 +- .../language/nodemethods/MethodMethods.scala | 6 +- .../MethodParameterInMethods.scala | 2 +- .../MethodParameterOutMethods.scala | 2 +- .../nodemethods/MethodRefMethods.scala | 2 +- .../nodemethods/MethodReturnMethods.scala | 3 +- .../language/nodemethods/NodeMethods.scala | 5 +- .../ArrayAccessTraversal.scala | 4 +- .../AssignmentTraversal.scala | 5 +- .../FieldAccessTraversal.scala | 4 +- .../operatorextension/Implicits.scala | 7 +- .../operatorextension/NodeTypeStarters.scala | 5 +- .../OpAstNodeTraversal.scala | 2 + .../operatorextension/TargetTraversal.scala | 2 + .../nodemethods/ArrayAccessMethods.scala | 2 +- .../semanticcpg/language/package.scala | 65 ++-- .../types/expressions/CallTraversal.scala | 13 +- .../ControlStructureTraversal.scala | 8 +- .../expressions/IdentifierTraversal.scala | 2 +- .../generalizations/AstNodeTraversal.scala | 35 +- .../generalizations/CfgNodeTraversal.scala | 6 +- .../DeclarationTraversal.scala | 23 +- .../generalizations/ExpressionTraversal.scala | 4 +- .../propertyaccessors/ModifierAccessors.scala | 4 +- .../types/structure/AnnotationTraversal.scala | 16 +- .../types/structure/DependencyTraversal.scala | 8 +- .../types/structure/ImportTraversal.scala | 2 +- .../types/structure/LocalTraversal.scala | 4 +- .../types/structure/MemberTraversal.scala | 5 +- .../MethodParameterOutTraversal.scala | 18 +- .../structure/MethodParameterTraversal.scala | 23 +- .../structure/MethodReturnTraversal.scala | 10 +- .../types/structure/MethodTraversal.scala | 8 +- .../structure/NamespaceBlockTraversal.scala | 7 +- .../types/structure/NamespaceTraversal.scala | 4 +- .../types/structure/TypeDeclTraversal.scala | 5 +- .../types/structure/TypeTraversal.scala | 16 +- .../io/shiftleft/semanticcpg/package.scala | 2 +- .../semanticcpg/testing/DummyNode.scala | 47 +-- .../semanticcpg/testing/MockCpg.scala | 229 ++++++++++++ .../semanticcpg/testing/package.scala | 233 ------------ .../language/NewNodeStepsTests.scala | 32 +- .../{ => language}/OverlaysTests.scala | 0 .../semanticcpg/language/StepsTest.scala | 69 ++-- .../accesspath/AccessPathTests.scala | 0 .../OperatorExtensionTests.scala | 1 - .../utils/CountStatementsTests.scala | 0 261 files changed, 1508 insertions(+), 1639 deletions(-) delete mode 100644 console/src/test/scala/io/shiftleft/codepropertygraph/cpgloading/TestProtoCpg.scala create mode 100644 joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/KeyPool.scala create mode 100644 joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/utils/KeyPoolTests.scala create mode 100644 semanticcpg/src/main/scala/io/shiftleft/semanticcpg/testing/MockCpg.scala delete mode 100644 semanticcpg/src/main/scala/io/shiftleft/semanticcpg/testing/package.scala rename semanticcpg/src/test/scala/io/shiftleft/semanticcpg/{ => language}/OverlaysTests.scala (100%) rename semanticcpg/src/test/scala/io/shiftleft/semanticcpg/{ => language}/accesspath/AccessPathTests.scala (100%) rename semanticcpg/src/test/scala/io/shiftleft/semanticcpg/{ => language}/utils/CountStatementsTests.scala (100%) diff --git a/build.sbt b/build.sbt index 13c4a0844990..7013fb2c4e6d 100644 --- a/build.sbt +++ b/build.sbt @@ -1,8 +1,9 @@ name := "joern" -ThisBuild / organization := "io.joern" +// TODO change org back to io.joern, only changed temporarily so I can debug something on jenkins +ThisBuild / organization := "com.michaelpollmeier" ThisBuild / scalaVersion := "3.4.2" -val cpgVersion = "1.6.18" +val cpgVersion = "1.6.16+21-b420d745" lazy val joerncli = Projects.joerncli lazy val querydb = Projects.querydb @@ -73,7 +74,8 @@ Global / onChangedBuildSource := ReloadOnSourceChanges // publishing info for sonatype / maven central ThisBuild / publishTo := sonatypePublishToBundle.value -sonatypeCredentialHost := "s01.oss.sonatype.org" +// TODO change back +// sonatypeCredentialHost := "s01.oss.sonatype.org" ThisBuild / scmInfo := Some(ScmInfo(url("https://github.com/joernio/joern"), "scm:git@github.com:joernio/joern.git")) ThisBuild / homepage := Some(url("https://joern.io/")) ThisBuild / licenses := List("Apache-2.0" -> url("http://www.apache.org/licenses/LICENSE-2.0")) diff --git a/console/build.sbt b/console/build.sbt index b5bdf0047f19..18194e799110 100644 --- a/console/build.sbt +++ b/console/build.sbt @@ -10,7 +10,7 @@ dependsOn( ) libraryDependencies ++= Seq( - "io.shiftleft" %% "codepropertygraph" % Versions.cpg, + "com.michaelpollmeier" %% "codepropertygraph" % Versions.cpg, "com.michaelpollmeier" %% "scala-repl-pp-server" % Versions.scalaReplPP, "com.github.scopt" %% "scopt" % Versions.scopt, "org.typelevel" %% "cats-effect" % Versions.catsEffect, diff --git a/console/src/main/scala/io/joern/console/Console.scala b/console/src/main/scala/io/joern/console/Console.scala index 1fe5e188f77d..63236e5de1e2 100644 --- a/console/src/main/scala/io/joern/console/Console.scala +++ b/console/src/main/scala/io/joern/console/Console.scala @@ -13,7 +13,7 @@ import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.dotextension.ImageViewer import io.shiftleft.semanticcpg.layers.{LayerCreator, LayerCreatorContext} import io.shiftleft.codepropertygraph.generated.help.Doc -import overflowdb.traversal.help.Table.AvailableWidthProvider +import flatgraph.help.Table.AvailableWidthProvider import scala.sys.process.Process import scala.util.control.NoStackTrace @@ -349,10 +349,14 @@ class Console[T <: Project](loader: WorkspaceLoader[T], baseDir: File = File.cur val cpgDestinationPath = cpgDestinationPathOpt.get - if (CpgLoader.isLegacyCpg(cpgFile)) { - report("You have provided a legacy proto CPG. Attempting conversion.") + val isProtoFormat = CpgLoader.isProtoFormat(cpgFile.path) + val isOverflowDbFormat = CpgLoader.isOverflowDbFormat(cpgFile.path) + if (isProtoFormat || isOverflowDbFormat) { + if (isProtoFormat) report("You have provided a legacy proto CPG. Attempting conversion.") + else if (isOverflowDbFormat) report("You have provided a legacy overflowdb CPG. Attempting conversion.") try { - CpgConverter.convertProtoCpgToOverflowDb(cpgFile.path.toString, cpgDestinationPath.toString) + val cpg = CpgLoader.load(cpgFile.path, cpgDestinationPath) + cpg.close() } catch { case exc: Exception => report("Error converting legacy CPG: " + exc.getMessage) diff --git a/console/src/main/scala/io/joern/console/CpgConverter.scala b/console/src/main/scala/io/joern/console/CpgConverter.scala index 3019e65ab816..7ce22c6bb670 100644 --- a/console/src/main/scala/io/joern/console/CpgConverter.scala +++ b/console/src/main/scala/io/joern/console/CpgConverter.scala @@ -1,14 +1,18 @@ package io.joern.console -import io.shiftleft.codepropertygraph.cpgloading.{CpgLoader, CpgLoaderConfig} -import overflowdb.Config +import io.shiftleft.codepropertygraph.cpgloading.{CpgLoader, ProtoCpgLoader} + +import java.nio.file.Paths object CpgConverter { - def convertProtoCpgToOverflowDb(srcFilename: String, dstFilename: String): Unit = { - val odbConfig = Config.withDefaults.withStorageLocation(dstFilename) - val config = CpgLoaderConfig.withDefaults.doNotCreateIndexesOnLoad.withOverflowConfig(odbConfig) - CpgLoader.load(srcFilename, config).close + def convertProtoCpgToFlatgraph(srcFilename: String, dstFilename: String): Unit = { + val cpg = ProtoCpgLoader.loadFromProtoZip(srcFilename, Option(Paths.get(dstFilename))) + cpg.close() } + @deprecated("method got renamed to `convertProtoCpgToFlatgraph, please use that instead", "joern v3") + def convertProtoCpgToOverflowDb(srcFilename: String, dstFilename: String): Unit = + convertProtoCpgToFlatgraph(srcFilename, dstFilename) + } diff --git a/console/src/main/scala/io/joern/console/Help.scala b/console/src/main/scala/io/joern/console/Help.scala index a1b3e017709b..b45739f88f42 100644 --- a/console/src/main/scala/io/joern/console/Help.scala +++ b/console/src/main/scala/io/joern/console/Help.scala @@ -1,8 +1,8 @@ package io.joern.console -import overflowdb.traversal.help.DocFinder.* -import overflowdb.traversal.help.Table.AvailableWidthProvider -import overflowdb.traversal.help.{DocFinder, Table} +import flatgraph.help.DocFinder.* +import flatgraph.help.Table.AvailableWidthProvider +import flatgraph.help.{DocFinder, Table} object Help { diff --git a/console/src/main/scala/io/joern/console/Run.scala b/console/src/main/scala/io/joern/console/Run.scala index 8724056ca080..7ca248d2b670 100644 --- a/console/src/main/scala/io/joern/console/Run.scala +++ b/console/src/main/scala/io/joern/console/Run.scala @@ -75,7 +75,7 @@ object Run { val toStringCode = s""" - | import overflowdb.traversal.help.Table + | import flatgraph.help.Table | override def toString() : String = { | val columnNames = List("name", "description") | val rows = diff --git a/console/src/main/scala/io/joern/console/cpgcreation/CpgGeneratorFactory.scala b/console/src/main/scala/io/joern/console/cpgcreation/CpgGeneratorFactory.scala index fc406b336e01..37362dfad732 100644 --- a/console/src/main/scala/io/joern/console/cpgcreation/CpgGeneratorFactory.scala +++ b/console/src/main/scala/io/joern/console/cpgcreation/CpgGeneratorFactory.scala @@ -2,10 +2,9 @@ package io.joern.console.cpgcreation import better.files.Dsl.* import better.files.File -import io.shiftleft.codepropertygraph.cpgloading.{CpgLoader, CpgLoaderConfig} +import io.shiftleft.codepropertygraph.cpgloading.CpgLoader import io.shiftleft.codepropertygraph.generated.Languages -import io.joern.console.ConsoleConfig -import overflowdb.Config +import io.joern.console.{ConsoleConfig, CpgConverter} import java.nio.file.Path import scala.util.Try @@ -60,12 +59,12 @@ class CpgGeneratorFactory(config: ConsoleConfig) { generator.generate(inputPath, outputPath).map(File(_)) outputFileOpt.map { outFile => val parentPath = outFile.parent.path.toAbsolutePath - if (isZipFile(outFile)) { + if (CpgLoader.isProtoFormat(outFile.path)) { report("Creating database from bin.zip") val srcFilename = outFile.path.toAbsolutePath.toString val dstFilename = parentPath.resolve("cpg.bin").toAbsolutePath.toString // MemoryHelper.hintForInsufficientMemory(srcFilename).map(report) - convertProtoCpgToOverflowDb(srcFilename, dstFilename) + convertProtoCpgToFlatgraph(srcFilename, dstFilename) } else { report("moving cpg.bin.zip to cpg.bin because it is already a database file") val srcPath = parentPath.resolve("cpg.bin.zip") @@ -77,18 +76,13 @@ class CpgGeneratorFactory(config: ConsoleConfig) { } } - def convertProtoCpgToOverflowDb(srcFilename: String, dstFilename: String): Unit = { - val odbConfig = Config.withDefaults.withStorageLocation(dstFilename) - val config = CpgLoaderConfig.withDefaults.doNotCreateIndexesOnLoad.withOverflowConfig(odbConfig) - CpgLoader.load(srcFilename, config).close - File(srcFilename).delete() - } + @deprecated("method got renamed to `convertProtoCpgToFlatgraph, please use that instead", "joern v3") + def convertProtoCpgToOverflowDb(srcFilename: String, dstFilename: String): Unit = + convertProtoCpgToFlatgraph(srcFilename, dstFilename) - def isZipFile(file: File): Boolean = { - val bytes = file.bytes - Try { - bytes.next() == 'P' && bytes.next() == 'K' - }.getOrElse(false) + def convertProtoCpgToFlatgraph(srcFilename: String, dstFilename: String): Unit = { + CpgConverter.convertProtoCpgToFlatgraph(srcFilename, dstFilename) + File(srcFilename).delete() } private def report(str: String): Unit = System.err.println(str) diff --git a/console/src/main/scala/io/joern/console/cpgcreation/ImportCode.scala b/console/src/main/scala/io/joern/console/cpgcreation/ImportCode.scala index 1f683d49a741..a84aad1712f5 100644 --- a/console/src/main/scala/io/joern/console/cpgcreation/ImportCode.scala +++ b/console/src/main/scala/io/joern/console/cpgcreation/ImportCode.scala @@ -5,8 +5,8 @@ import io.joern.console.workspacehandling.Project import io.joern.console.{ConsoleException, FrontendConfig, Reporting} import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.Languages -import overflowdb.traversal.help.Table -import overflowdb.traversal.help.Table.AvailableWidthProvider +import flatgraph.help.Table +import flatgraph.help.Table.AvailableWidthProvider import java.nio.file.Path import scala.util.{Failure, Success, Try} diff --git a/console/src/main/scala/io/joern/console/package.scala b/console/src/main/scala/io/joern/console/package.scala index a29435afba29..958f26202bde 100644 --- a/console/src/main/scala/io/joern/console/package.scala +++ b/console/src/main/scala/io/joern/console/package.scala @@ -1,6 +1,6 @@ package io.joern -import overflowdb.traversal.help.Table.AvailableWidthProvider +import flatgraph.help.Table.AvailableWidthProvider import replpp.Operators.* import replpp.Colors diff --git a/console/src/main/scala/io/joern/console/scan/package.scala b/console/src/main/scala/io/joern/console/scan/package.scala index 41e01d2e32e4..31230076bb8d 100644 --- a/console/src/main/scala/io/joern/console/scan/package.scala +++ b/console/src/main/scala/io/joern/console/scan/package.scala @@ -11,11 +11,6 @@ package object scan { private val logger: Logger = LoggerFactory.getLogger(this.getClass) - implicit class ScannerStarters(val cpg: Cpg) extends AnyVal { - def finding: Iterator[Finding] = - overflowdb.traversal.InitialTraversal.from[Finding](cpg.graph, NodeTypes.FINDING) - } - implicit class QueryWrapper(q: Query) { /** Obtain list of findings by running query on CPG diff --git a/console/src/main/scala/io/joern/console/workspacehandling/Workspace.scala b/console/src/main/scala/io/joern/console/workspacehandling/Workspace.scala index 0d3a068d4bc2..82caaff7897b 100644 --- a/console/src/main/scala/io/joern/console/workspacehandling/Workspace.scala +++ b/console/src/main/scala/io/joern/console/workspacehandling/Workspace.scala @@ -1,8 +1,8 @@ package io.joern.console.workspacehandling import io.joern.console.defaultAvailableWidthProvider -import overflowdb.traversal.help.Table -import overflowdb.traversal.help.Table.AvailableWidthProvider +import flatgraph.help.Table +import flatgraph.help.Table.AvailableWidthProvider import scala.collection.mutable.ListBuffer diff --git a/console/src/main/scala/io/joern/console/workspacehandling/WorkspaceLoader.scala b/console/src/main/scala/io/joern/console/workspacehandling/WorkspaceLoader.scala index c0297fb43551..4d7d943db5e1 100644 --- a/console/src/main/scala/io/joern/console/workspacehandling/WorkspaceLoader.scala +++ b/console/src/main/scala/io/joern/console/workspacehandling/WorkspaceLoader.scala @@ -2,7 +2,7 @@ package io.joern.console.workspacehandling import better.files.Dsl.mkdirs import better.files.File -import overflowdb.traversal.help.Table.AvailableWidthProvider +import flatgraph.help.Table.AvailableWidthProvider import java.nio.file.Path import scala.collection.mutable.ListBuffer diff --git a/console/src/main/scala/io/joern/console/workspacehandling/WorkspaceManager.scala b/console/src/main/scala/io/joern/console/workspacehandling/WorkspaceManager.scala index 189d617c84a0..3beb2345060a 100644 --- a/console/src/main/scala/io/joern/console/workspacehandling/WorkspaceManager.scala +++ b/console/src/main/scala/io/joern/console/workspacehandling/WorkspaceManager.scala @@ -6,13 +6,12 @@ import io.joern.console import io.joern.console.defaultAvailableWidthProvider import io.joern.console.Reporting import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.codepropertygraph.cpgloading.{CpgLoader, CpgLoaderConfig} +import io.shiftleft.codepropertygraph.cpgloading.CpgLoader import org.json4s.DefaultFormats -import org.json4s.native.Serialization.{write => jsonWrite} -import overflowdb.Config +import org.json4s.native.Serialization.write as jsonWrite import java.net.URLEncoder -import java.nio.file.Path +import java.nio.file.{Path, Paths} import scala.collection.mutable.ListBuffer import scala.util.{Failure, Success, Try} @@ -305,17 +304,12 @@ class WorkspaceManager[ProjectType <: Project](path: String, loader: WorkspaceLo private def loadCpgRaw(cpgFilename: String): Option[Cpg] = { Try { - val odbConfig = Config.withDefaults.withStorageLocation(cpgFilename) - val config = - CpgLoaderConfig.withDefaults.doNotCreateIndexesOnLoad.withOverflowConfig(odbConfig) - val newCpg = CpgLoader.loadFromOverflowDb(config) - CpgLoader.createIndexes(newCpg) - newCpg + CpgLoader.load(cpgFilename) } match { case Success(v) => Some(v) case Failure(ex) => System.err.println("Error loading CPG") - System.err.println(ex) + ex.printStackTrace() None } } diff --git a/console/src/test/scala/io/joern/console/ConsoleTests.scala b/console/src/test/scala/io/joern/console/ConsoleTests.scala index f88cf0ad0996..728a83d55331 100644 --- a/console/src/test/scala/io/joern/console/ConsoleTests.scala +++ b/console/src/test/scala/io/joern/console/ConsoleTests.scala @@ -428,7 +428,7 @@ class ConsoleTests extends AnyWordSpec with Matchers { "cpg" should { "provide .help command" in ConsoleFixture() { (console, codeDir) => // part of Predefined.shared, which makes the below work in the repl without separate import - import io.shiftleft.codepropertygraph.Cpg.docSearchPackages + import io.shiftleft.semanticcpg.language.docSearchPackages import io.joern.console.testing.availableWidthProvider console.importCode(codeDir.toString) diff --git a/console/src/test/scala/io/joern/console/testing/package.scala b/console/src/test/scala/io/joern/console/testing/package.scala index 897bb9e9633c..1b2053d82a7c 100644 --- a/console/src/test/scala/io/joern/console/testing/package.scala +++ b/console/src/test/scala/io/joern/console/testing/package.scala @@ -3,7 +3,7 @@ package io.joern.console import better.files.Dsl.* import better.files.* import io.joern.console.workspacehandling.Project -import overflowdb.traversal.help.Table.{AvailableWidthProvider, ConstantWidth} +import flatgraph.help.Table.{AvailableWidthProvider, ConstantWidth} import scala.util.Try diff --git a/console/src/test/scala/io/shiftleft/codepropertygraph/cpgloading/TestProtoCpg.scala b/console/src/test/scala/io/shiftleft/codepropertygraph/cpgloading/TestProtoCpg.scala deleted file mode 100644 index 7649f012eccd..000000000000 --- a/console/src/test/scala/io/shiftleft/codepropertygraph/cpgloading/TestProtoCpg.scala +++ /dev/null @@ -1,45 +0,0 @@ -package io.shiftleft.codepropertygraph.cpgloading - -import better.files.File -import io.shiftleft.proto.cpg.Cpg -import io.shiftleft.proto.cpg.Cpg.CpgStruct - -import java.io.FileOutputStream - -object TestProtoCpg { - - def createTestProtoCpg: File = { - val outDir = better.files.File.newTemporaryDirectory("cpgloadertests") - val outStream = new FileOutputStream((outDir / "1.proto").pathAsString) - CpgStruct - .newBuilder() - .addNode( - CpgStruct.Node - .newBuilder() - .setKey(1) - .setType(CpgStruct.Node.NodeType.valueOf("METHOD")) - .addProperty( - CpgStruct.Node.Property - .newBuilder() - .setName(Cpg.NodePropertyName.valueOf("FULL_NAME")) - .setValue( - Cpg.PropertyValue - .newBuilder() - .setStringValue("foo") - .build() - ) - .build - ) - .build() - ) - .build() - .writeTo(outStream) - outStream.close() - - val zipFile = better.files.File.newTemporaryFile("cpgloadertests", ".bin.zip") - outDir.zipTo(zipFile) - outDir.delete() - zipFile - } - -} diff --git a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/dotgenerator/DdgGenerator.scala b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/dotgenerator/DdgGenerator.scala index cfc548dc8431..ca007968914d 100644 --- a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/dotgenerator/DdgGenerator.scala +++ b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/dotgenerator/DdgGenerator.scala @@ -2,14 +2,12 @@ package io.joern.dataflowengineoss.dotgenerator import io.joern.dataflowengineoss.DefaultSemantics import io.shiftleft.codepropertygraph.generated.nodes.* -import io.shiftleft.codepropertygraph.generated.{EdgeTypes, Properties} +import io.shiftleft.codepropertygraph.generated.EdgeTypes import io.joern.dataflowengineoss.language.* import io.joern.dataflowengineoss.semanticsloader.Semantics import io.shiftleft.semanticcpg.dotgenerator.DotSerializer.{Edge, Graph} import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.utils.MemberAccess.isGenericMemberAccessName -import overflowdb.Node -import overflowdb.traversal.jIteratortoTraversal import scala.collection.mutable @@ -59,7 +57,7 @@ class DdgGenerator { } } - private def shouldBeDisplayed(v: Node): Boolean = !( + private def shouldBeDisplayed(v: StoredNode): Boolean = !( v.isInstanceOf[ControlStructure] || v.isInstanceOf[JumpTarget] ) @@ -91,7 +89,16 @@ class DdgGenerator { val allInEdges = v .inE(EdgeTypes.REACHING_DEF) .map(x => - Edge(x.outNode.asInstanceOf[StoredNode], v, srcVisible = true, x.property(Properties.Variable), edgeType) + // note: this looks strange, but let me explain... + // in overflowdb, edges were allowed multiple properties and this used to be `x.property(Properties.VARIABLE)` + // in flatgraph an edge may have zero or one properties and they're not named... + // in this case we know that we're dealing with ReachingDef edges which has the `variable` property + val variablePropertyMaybe = x.property match { + case null => null + case variableProperty: String => variableProperty + case _ => null + } + Edge(x.src.asInstanceOf[StoredNode], v, srcVisible = true, variablePropertyMaybe, edgeType) ) v match { diff --git a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/language/Path.scala b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/language/Path.scala index 241a3d95750d..981b054ec657 100644 --- a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/language/Path.scala +++ b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/language/Path.scala @@ -3,8 +3,8 @@ package io.joern.dataflowengineoss.language import io.shiftleft.codepropertygraph.generated.nodes.{AstNode, CfgNode, Member, MethodParameterIn} import io.shiftleft.semanticcpg import io.shiftleft.semanticcpg.language.* -import overflowdb.traversal.help.Table -import overflowdb.traversal.help.Table.AvailableWidthProvider +import flatgraph.help.Table +import flatgraph.help.Table.AvailableWidthProvider case class Path(elements: List[AstNode]) { def resultPairs(): List[(String, Option[Int])] = { diff --git a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/passes/reachingdef/DdgGenerator.scala b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/passes/reachingdef/DdgGenerator.scala index de3b6a8a99a1..7693f28757ff 100644 --- a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/passes/reachingdef/DdgGenerator.scala +++ b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/passes/reachingdef/DdgGenerator.scala @@ -4,7 +4,7 @@ import io.joern.dataflowengineoss.{globalFromLiteral, identifierToFirstUsages} import io.joern.dataflowengineoss.queryengine.AccessPathUsage.toTrackedBaseAndAccessPathSimple import io.joern.dataflowengineoss.semanticsloader.Semantics import io.shiftleft.codepropertygraph.generated.nodes.* -import io.shiftleft.codepropertygraph.generated.{EdgeTypes, Operators, PropertyNames} +import io.shiftleft.codepropertygraph.generated.{EdgeTypes, Operators} import io.shiftleft.semanticcpg.accesspath.MatchResult import io.shiftleft.semanticcpg.language.* import io.shiftleft.codepropertygraph.generated.DiffGraphBuilder @@ -132,7 +132,7 @@ class DdgGenerator(semantics: Semantics) { // There is always an edge from the method input parameter // to the corresponding method output parameter as modifications // of the input parameter only affect a copy. - paramOut.paramIn.foreach { paramIn => + paramOut.start.paramIn.foreach { paramIn => addEdge(paramIn, paramOut, paramIn.name) } usageAnalyzer.usedIncomingDefs(paramOut).foreach { case (_, inElements) => @@ -224,7 +224,7 @@ class DdgGenerator(semantics: Semantics) { (fromNode, toNode) match { case (parentNode: CfgNode, childNode: CfgNode) if EdgeValidator.isValidEdge(childNode, parentNode) => - dstGraph.addEdge(fromNode, toNode, EdgeTypes.REACHING_DEF, PropertyNames.VARIABLE, variable) + dstGraph.addEdge(fromNode, toNode, EdgeTypes.REACHING_DEF, variable) case _ => } diff --git a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/queryengine/AccessPathUsage.scala b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/queryengine/AccessPathUsage.scala index f4b2f0cf6b1f..534b8bb565b8 100644 --- a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/queryengine/AccessPathUsage.scala +++ b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/queryengine/AccessPathUsage.scala @@ -2,7 +2,7 @@ package io.joern.dataflowengineoss.queryengine import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.semanticcpg.accesspath.* -import io.shiftleft.semanticcpg.language.{AccessPathHandling, toCallMethods} +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.utils.MemberAccess import org.slf4j.LoggerFactory diff --git a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/queryengine/Engine.scala b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/queryengine/Engine.scala index 1c0c279a9758..91f63efd25a3 100644 --- a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/queryengine/Engine.scala +++ b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/queryengine/Engine.scala @@ -1,14 +1,14 @@ package io.joern.dataflowengineoss.queryengine +import flatgraph.Edge import io.joern.dataflowengineoss.DefaultSemantics import io.joern.dataflowengineoss.language.* import io.joern.dataflowengineoss.passes.reachingdef.EdgeValidator import io.joern.dataflowengineoss.semanticsloader.{FlowSemantic, Semantics} import io.shiftleft.codepropertygraph.generated.nodes.* -import io.shiftleft.codepropertygraph.generated.{EdgeTypes, Properties} +import io.shiftleft.codepropertygraph.generated.EdgeTypes import io.shiftleft.semanticcpg.language.* import org.slf4j.{Logger, LoggerFactory} -import overflowdb.Edge import java.util.concurrent.* import scala.collection.mutable @@ -205,9 +205,11 @@ object Engine { private def elemForEdge(e: Edge, callSiteStack: List[Call] = List())(implicit semantics: Semantics ): Option[PathElement] = { - val curNode = e.inNode().asInstanceOf[CfgNode] - val parNode = e.outNode().asInstanceOf[CfgNode] - val outLabel = Some(e.property(Properties.Variable)).getOrElse("") + val curNode = e.dst.asInstanceOf[CfgNode] + val parNode = e.src.asInstanceOf[CfgNode] + // note: flatgraph only allows at most one property per edge, and since we know :tm: that this is a ReachingDef edge it must be the Variable property... + val variablePropertyMaybe = Option(e.property).map(_.asInstanceOf[String]) + val outLabel = variablePropertyMaybe.getOrElse("") if (!EdgeValidator.isValidEdge(curNode, parNode)) { return None @@ -254,9 +256,8 @@ object Engine { private def ddgInE(node: CfgNode, path: Vector[PathElement], callSiteStack: List[Call] = List()): Vector[Edge] = { node .inE(EdgeTypes.REACHING_DEF) - .asScala .filter { e => - e.outNode() match { + e.src match { case srcNode: CfgNode => !srcNode.isInstanceOf[Method] && !path .map(x => x.node) diff --git a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/queryengine/SourcesToStartingPoints.scala b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/queryengine/SourcesToStartingPoints.scala index cac222fbd476..c686271ea0f6 100644 --- a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/queryengine/SourcesToStartingPoints.scala +++ b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/queryengine/SourcesToStartingPoints.scala @@ -32,7 +32,7 @@ object SourcesToStartingPoints { .map(src => { // We need to get Cpg wrapper from graph. Hence we are taking head element from source iterator. // This will also ensure if the source list is empty then these tasks are invoked. - val cpg = Cpg(src.graph()) + val cpg = Cpg(src.graph) val (startingPoints, methodTasks) = calculateStartingPoints(sources, executorService) val startingPointFromUsageInOtherClasses = calculateStatingPointsWithUsageInOtherClasses(methodTasks, cpg, executorService) @@ -189,7 +189,8 @@ abstract class BaseSourceToStartingPoints extends Callable[Unit] { protected def sourceToStartingPoints(src: StoredNode): (List[CfgNode], List[UsageInput]) = { src match { case methodReturn: MethodReturn => - (methodReturn.method.callIn.l, Nil) + // n.b. there's a generated `callIn` step that we really want to use, but it's shadowed by `MethodTraversal.callIn` + (methodReturn.method._callIn.cast[Call].l, Nil) case lit: Literal => val usageInput = targetsToClassIdentifierPair(literalToInitializedMembers(lit), src) val uses = usages(usageInput) diff --git a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/queryengine/TaskCreator.scala b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/queryengine/TaskCreator.scala index f10f105d2409..5c8713e505c8 100644 --- a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/queryengine/TaskCreator.scala +++ b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/queryengine/TaskCreator.scala @@ -100,7 +100,7 @@ class TaskCreator(context: EngineContext) { */ private def paramToMethodRefCallReceivers(param: MethodParameterIn): List[Expression] = - new Cpg(param.graph()).methodRef.methodFullNameExact(param.method.fullName).inCall.argument(0).l + new Cpg(param.graph).methodRef.methodFullNameExact(param.method.fullName).inCall.argument(0).l /** Create new tasks from all results that end in an output argument, including return arguments. In this case, we * want to traverse to corresponding method output parameters and method return nodes respectively. diff --git a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/slicing/DataFlowSlicing.scala b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/slicing/DataFlowSlicing.scala index bfcc4a4ad67b..2675ffe8ab09 100644 --- a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/slicing/DataFlowSlicing.scala +++ b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/slicing/DataFlowSlicing.scala @@ -2,8 +2,7 @@ package io.joern.dataflowengineoss.slicing import io.joern.dataflowengineoss.language.* import io.joern.x2cpg.utils.ConcurrentTaskUtil -import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.codepropertygraph.generated.PropertyNames +import io.shiftleft.codepropertygraph.generated.{Cpg, Properties} import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.semanticcpg.language.* import org.slf4j.LoggerFactory @@ -47,10 +46,9 @@ object DataFlowSlicing { val sliceNodes = sinks.iterator.repeat(_.ddgIn)(_.maxDepth(config.sliceDepth).emit).dedup.l val sliceNodesIdSet = sliceNodes.id.toSet // Lazily set up the rest if the filters are satisfied - lazy val sliceEdges = sliceNodes - .flatMap(_.outE) - .filter(x => sliceNodesIdSet.contains(x.inNode().id())) - .map { e => SliceEdge(e.outNode().id(), e.inNode().id(), e.label()) } + lazy val sliceEdges = sliceNodes.outE + .filter(x => sliceNodesIdSet.contains(x.dst.id())) + .map { e => SliceEdge(e.src.id(), e.dst.id(), e.label) } .toSet lazy val slice = Option(DataFlowSlice(sliceNodes.map(cfgNodeToSliceNode).toSet, sliceEdges)) @@ -81,10 +79,7 @@ object DataFlowSlicing { case n: MethodRef => sliceNode.copy(name = n.methodFullName, code = n.code) case n: TypeRef => sliceNode.copy(name = n.typeFullName, code = n.code) case n => - sliceNode.copy( - name = n.property(PropertyNames.NAME, ""), - typeFullName = n.property(PropertyNames.TYPE_FULL_NAME, "") - ) + sliceNode.copy(name = n.property(Properties.Name), typeFullName = n.property(Properties.TypeFullName)) } } diff --git a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/slicing/UsageSlicing.scala b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/slicing/UsageSlicing.scala index 2636af71d20a..a95974c2e57f 100644 --- a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/slicing/UsageSlicing.scala +++ b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/slicing/UsageSlicing.scala @@ -3,7 +3,7 @@ package io.joern.dataflowengineoss.slicing import io.joern.x2cpg.utils.ConcurrentTaskUtil import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.* -import io.shiftleft.codepropertygraph.generated.{Operators, PropertyNames} +import io.shiftleft.codepropertygraph.generated.{Operators, Properties} import io.shiftleft.semanticcpg.language.* import org.slf4j.LoggerFactory @@ -182,15 +182,12 @@ object UsageSlicing { .getOrElse(Iterator.empty) else baseCall.argument) .collect { case n: Expression if n.argumentIndex > 0 => n } - .flatMap { - case _: MethodRef => Option("LAMBDA") + .map { + case _: MethodRef => "LAMBDA" case x => - Option( - x.property( - PropertyNames.TYPE_FULL_NAME, - x.property(PropertyNames.DYNAMIC_TYPE_HINT_FULL_NAME, Seq("ANY")).headOption - ) - ) + x.propertyOption(Properties.TypeFullName) + .orElse(x.property(Properties.DynamicTypeHintFullName).headOption) + .getOrElse("ANY") } .collect { case x: String => x } .toList diff --git a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/slicing/package.scala b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/slicing/package.scala index e1b705041298..18510bc4c191 100644 --- a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/slicing/package.scala +++ b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/slicing/package.scala @@ -1,11 +1,10 @@ package io.joern.dataflowengineoss import better.files.File -import io.shiftleft.codepropertygraph.generated.PropertyNames +import io.shiftleft.codepropertygraph.generated.{Properties, PropertyNames} import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.semanticcpg.language.* import org.slf4j.LoggerFactory -import overflowdb.PropertyKey import upickle.default.* import java.util.concurrent.{ExecutorService, Executors} @@ -332,13 +331,15 @@ package object slicing { * extracted. */ def fromNode(node: StoredNode, typeMap: Map[String, String] = Map.empty[String, String]): DefComponent = { - val nodeType = (node.property(PropertyNames.TYPE_FULL_NAME, "ANY") +: node.property( - PropertyNames.DYNAMIC_TYPE_HINT_FULL_NAME, - Seq.empty[String] - )).filterNot(_.matches("(ANY|UNKNOWN)")).headOption.getOrElse("ANY") + val typeFullNameProperty = node.propertyOption(Properties.TypeFullName).getOrElse("ANY") + val dynamicTypeHintFullNamesProperty = node.property(Properties.DynamicTypeHintFullName) + val nodeType = (typeFullNameProperty +: dynamicTypeHintFullNamesProperty) + .filterNot(_.matches("(ANY|UNKNOWN)")) + .headOption + .getOrElse("ANY") val typeFullName = typeMap.getOrElse(nodeType, nodeType) - val lineNumber = Option(node.property(new PropertyKey[Integer](PropertyNames.LINE_NUMBER))).map(_.toInt) - val columnNumber = Option(node.property(new PropertyKey[Integer](PropertyNames.COLUMN_NUMBER))).map(_.toInt) + val lineNumber = node.propertyOption[Int](PropertyNames.LINE_NUMBER) + val columnNumber = node.propertyOption[Int](PropertyNames.COLUMN_NUMBER) node match { case x: MethodParameterIn => ParamDef(x.name, typeFullName, x.index, lineNumber, columnNumber) case x: Call if x.code.startsWith("new ") => diff --git a/dataflowengineoss/src/test/scala/io/joern/dataflowengineoss/queryengine/AccessPathUsageTests.scala b/dataflowengineoss/src/test/scala/io/joern/dataflowengineoss/queryengine/AccessPathUsageTests.scala index 668c37ce2fb2..9749294a9984 100644 --- a/dataflowengineoss/src/test/scala/io/joern/dataflowengineoss/queryengine/AccessPathUsageTests.scala +++ b/dataflowengineoss/src/test/scala/io/joern/dataflowengineoss/queryengine/AccessPathUsageTests.scala @@ -1,13 +1,14 @@ package io.joern.dataflowengineoss.queryengine -import io.shiftleft.OverflowDbTestInstance +import flatgraph.{GNode, Graph} +import flatgraph.misc.TestUtils.* +import io.shiftleft.codepropertygraph.generated.PropertyNames import io.shiftleft.codepropertygraph.generated.nodes.* -import io.shiftleft.codepropertygraph.generated.{EdgeTypes, NodeTypes, Operators, Properties} +import io.shiftleft.codepropertygraph.generated.{Cpg, EdgeTypes, NodeTypes, Operators} import io.joern.dataflowengineoss.queryengine.AccessPathUsage.toTrackedBaseAndAccessPathSimple import io.shiftleft.semanticcpg.accesspath.* import org.scalatest.matchers.should.Matchers.* import org.scalatest.wordspec.AnyWordSpec -import overflowdb.* class AccessPathUsageTests extends AnyWordSpec { @@ -22,35 +23,28 @@ class AccessPathUsageTests extends AnyWordSpec { private val VS = VariablePointerShift private val S = PointerShift - private val g = OverflowDbTestInstance.create + private val g = Cpg.empty.graph - private def genCALL(graph: Graph, op: String, args: Node*): Call = { - val ret = graph + NodeTypes.CALL // (NodeTypes.CALL, Properties.NAME -> op) - ret.setProperty(Properties.Name, op) + private def genCALL(graph: Graph, op: String, args: GNode*): Call = { + val diffGraphBuilder = Cpg.newDiffGraphBuilder + val newCall = NewCall().name(op) + diffGraphBuilder.addNode(newCall) args.reverse.zipWithIndex.foreach { case (arg, idx) => - ret --- EdgeTypes.ARGUMENT --> arg - arg.setProperty(Properties.ArgumentIndex, idx + 1) + diffGraphBuilder.setNodeProperty(arg, PropertyNames.ARGUMENT_INDEX, idx + 1) + diffGraphBuilder.addEdge(newCall, arg, EdgeTypes.ARGUMENT) } - ret.asInstanceOf[Call] + diffGraphBuilder.apply(graph) + newCall.storedRef.get } - private def genLit(graph: Graph, payload: String): Literal = { - val ret = graph + NodeTypes.LITERAL - ret.setProperty(Properties.Code, payload) - ret.asInstanceOf[Literal] - } + private def genLit(graph: Graph, payload: String): Literal = + graph.addNode(NewLiteral().code(payload)) - private def genID(graph: Graph, payload: String): Identifier = { - val ret = graph + NodeTypes.IDENTIFIER - ret.setProperty(Properties.Name, payload) - ret.asInstanceOf[Identifier] - } + private def genID(graph: Graph, payload: String): Identifier = + graph.addNode(NewIdentifier().name(payload)) - private def genFID(graph: Graph, payload: String): FieldIdentifier = { - val ret = graph + NodeTypes.FIELD_IDENTIFIER - ret.setProperty(Properties.CanonicalName, payload) - ret.asInstanceOf[FieldIdentifier] - } + private def genFID(graph: Graph, payload: String): FieldIdentifier = + graph.addNode(NewFieldIdentifier().canonicalName(payload)) private def toTrackedAccessPath(node: StoredNode): AccessPath = toTrackedBaseAndAccessPathSimple(node)._2 diff --git a/joern-cli/build.sbt b/joern-cli/build.sbt index 88b17ef355bd..dec7c5d4d1ef 100644 --- a/joern-cli/build.sbt +++ b/joern-cli/build.sbt @@ -3,7 +3,7 @@ name := "joern-cli" dependsOn(Projects.console, Projects.console % "test->test", Projects.dataflowengineoss, Projects.x2cpg) libraryDependencies ++= Seq( - "io.shiftleft" %% "codepropertygraph" % Versions.cpg, + "com.michaelpollmeier" %% "codepropertygraph" % Versions.cpg, "com.lihaoyi" %% "requests" % Versions.requests, "com.lihaoyi" %% "upickle" % Versions.upickle, "com.github.scopt" %% "scopt" % Versions.scopt, diff --git a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstForFunctionsCreator.scala b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstForFunctionsCreator.scala index f2879dff7463..3174464b89de 100644 --- a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstForFunctionsCreator.scala +++ b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstForFunctionsCreator.scala @@ -37,7 +37,7 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { th val parentNode: NewTypeDecl = methodAstParentStack.collectFirst { case t: NewTypeDecl => t }.getOrElse { val astParentType = methodAstParentStack.head.label - val astParentFullName = methodAstParentStack.head.properties("FULL_NAME").toString + val astParentFullName = methodAstParentStack.head.propertiesMap.get("FULL_NAME").toString val typeDeclNode_ = typeDeclNode( node, normalizedName, diff --git a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstNodeBuilder.scala b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstNodeBuilder.scala index 90e44534fe4a..5d4f8821e82c 100644 --- a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstNodeBuilder.scala +++ b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstNodeBuilder.scala @@ -1,8 +1,9 @@ package io.joern.c2cpg.astcreation +import io.joern.x2cpg.utils.NodeBuilders.{newMethodReturnNode => newMethodReturnNode_} import io.shiftleft.codepropertygraph.generated.nodes.* -import org.eclipse.cdt.core.dom.ast.IASTLabelStatement -import org.eclipse.cdt.core.dom.ast.IASTNode +import org.eclipse.cdt.core.dom.ast.{IASTLabelStatement, IASTNode} +import org.eclipse.cdt.core.dom.ast.IASTPreprocessorIncludeStatement import org.eclipse.cdt.internal.core.model.ASTStringUtil trait AstNodeBuilder { this: AstCreator => diff --git a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/dataflow/DataFlowTests.scala b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/dataflow/DataFlowTests.scala index a050ffde48cd..c78a5cd3fe8a 100644 --- a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/dataflow/DataFlowTests.scala +++ b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/dataflow/DataFlowTests.scala @@ -6,8 +6,7 @@ import io.joern.dataflowengineoss.queryengine.{EngineConfig, EngineContext} import io.shiftleft.codepropertygraph.generated.EdgeTypes import io.shiftleft.codepropertygraph.generated.nodes.{CfgNode, Identifier, Literal} import io.shiftleft.semanticcpg.language.* -import overflowdb.traversal.help.Table.AvailableWidthProvider -import overflowdb.traversal.toNodeTraversal +import flatgraph.help.Table.AvailableWidthProvider class DataFlowTests extends DataFlowCodeToCpgSuite { @@ -1082,7 +1081,7 @@ class DataFlowTests extends DataFlowCodeToCpgSuite { cpg .call("bar") .outE(EdgeTypes.REACHING_DEF) - .count(_.inNode() == cpg.ret.head) shouldBe 1 + .count(_.dst == cpg.ret.head) shouldBe 1 } } diff --git a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/macros/MacroHandlingTests.scala b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/macros/MacroHandlingTests.scala index d83e4447fd98..08e2e3c81434 100644 --- a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/macros/MacroHandlingTests.scala +++ b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/macros/MacroHandlingTests.scala @@ -231,7 +231,7 @@ class MacroHandlingTests extends C2CpgSuite { """.stripMargin) "should not result in malformed CFGs when expanding a nested macro with block" in { - cpg.all.collectAll[Block].l.count(b => b.cfgOut.size > 1) shouldBe 0 + cpg.all.collectAll[Block].l.count(b => b._cfgOut.size > 1) shouldBe 0 } } diff --git a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/MetaDataPassTests.scala b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/MetaDataPassTests.scala index cfb64ade809f..9996ac6b6995 100644 --- a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/MetaDataPassTests.scala +++ b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/MetaDataPassTests.scala @@ -8,8 +8,6 @@ import io.joern.x2cpg.passes.frontend.MetaDataPass import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec -import scala.jdk.CollectionConverters.* - class MetaDataPassTests extends AnyWordSpec with Matchers { "MetaDataPass" should { @@ -17,11 +15,11 @@ class MetaDataPassTests extends AnyWordSpec with Matchers { new MetaDataPass(cpg, Languages.C, "").createAndApply() "create exactly two nodes" in { - cpg.graph.V.asScala.size shouldBe 2 + cpg.graph.allNodes.size shouldBe 2 } "create no edges" in { - cpg.graph.E.asScala.size shouldBe 0 + cpg.graph.allNodes.outE.size shouldBe 0 } "create a metadata node with correct language" in { diff --git a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/AstCreationPassTests.scala b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/AstCreationPassTests.scala index 0511dc0c43e7..3f807e31a929 100644 --- a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/AstCreationPassTests.scala +++ b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/AstCreationPassTests.scala @@ -12,7 +12,6 @@ import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.operatorextension.OpNodes import io.shiftleft.semanticcpg.language.types.structure.NamespaceTraversal -import overflowdb.traversal.toNodeTraversal class AstCreationPassTests extends AstC2CpgSuite { @@ -1373,11 +1372,9 @@ class AstCreationPassTests extends AstC2CpgSuite { .name("d") .ast .isReturn - .outE(EdgeTypes.ARGUMENT) + .out(EdgeTypes.ARGUMENT) .head - .inNode() - .get - .asInstanceOf[CallDb] + .asInstanceOf[Call] .code shouldBe "x * 2" } diff --git a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/CallTests.scala b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/CallTests.scala index 2d1e164e23c2..d16ff45eb0e0 100644 --- a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/CallTests.scala +++ b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/CallTests.scala @@ -112,7 +112,7 @@ class CallTests extends C2CpgSuite { "have the correct callIn" in { val List(m) = cpg.method.nameNot("").where(_.ast.isReturn.code(".*nullptr.*")).l val List(c) = cpg.call.codeExact("b->GetObj()").l - c.callee.head shouldBe m + c.callee.l should contain(m) val List(callIn) = m.callIn.l callIn.code shouldBe "b->GetObj()" } diff --git a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/querying/NodeTypeStarterQueryTests.scala b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/querying/NodeTypeStarterQueryTests.scala index 85a019e14423..5a1c504eba8b 100644 --- a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/querying/NodeTypeStarterQueryTests.scala +++ b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/querying/NodeTypeStarterQueryTests.scala @@ -117,8 +117,8 @@ class NodeTypeStarterQueryTests extends C2CpgSuite { val method1 = cpg.method.name("main").head val method2 = cpg.method.name("libfunc").head - cpg.id(method1.id).l shouldBe Seq(method1) - cpg.ids(method1.id, method2.id).l shouldBe Seq(method1, method2) + cpg.all.id(method1.id).l shouldBe Seq(method1) + cpg.all.id(method1.id, method2.id).l shouldBe Seq(method1, method2) } } diff --git a/joern-cli/frontends/csharpsrc2cpg/build.sbt b/joern-cli/frontends/csharpsrc2cpg/build.sbt index a112ff51dcac..0dcdd792e1d6 100644 --- a/joern-cli/frontends/csharpsrc2cpg/build.sbt +++ b/joern-cli/frontends/csharpsrc2cpg/build.sbt @@ -20,7 +20,9 @@ lazy val astGenVersion = settingKey[String]("dotnetastgen version") astGenVersion := appProperties.value.getString("csharpsrc2cpg.dotnetastgen_version") libraryDependencies ++= Seq( - "io.shiftleft" %% "codepropertygraph" % Versions.cpg, + // TODO back to io.shiftleft! + //"io.shiftleft" %% "codepropertygraph" % Versions.cpg, + "com.michaelpollmeier" %% "codepropertygraph" % Versions.cpg, "org.scalatest" %% "scalatest" % Versions.scalatest % Test ) diff --git a/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/astcreation/AstCreator.scala b/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/astcreation/AstCreator.scala index 7a1608fab132..ab5c0e77c16b 100644 --- a/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/astcreation/AstCreator.scala +++ b/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/astcreation/AstCreator.scala @@ -1,5 +1,6 @@ package io.joern.csharpsrc2cpg.astcreation +import flatgraph.DiffGraphBuilder import io.joern.csharpsrc2cpg.{CSharpDefines, Constants} import io.joern.csharpsrc2cpg.datastructures.{CSharpProgramSummary, CSharpScope} import io.joern.csharpsrc2cpg.parser.DotNetJsonAst.* @@ -8,7 +9,6 @@ import io.joern.x2cpg.astgen.{AstGenNodeBuilder, ParserResult} import io.joern.x2cpg.{Ast, AstCreatorBase, ValidationMode} import io.shiftleft.codepropertygraph.generated.NodeTypes import io.shiftleft.codepropertygraph.generated.nodes.{NewFile, NewTypeDecl} -import io.shiftleft.passes.IntervalKeyPool import org.slf4j.{Logger, LoggerFactory} import io.shiftleft.codepropertygraph.generated.DiffGraphBuilder import ujson.Value diff --git a/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/astcreation/AstCreatorHelper.scala b/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/astcreation/AstCreatorHelper.scala index a070f8c8b539..c865f43f2624 100644 --- a/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/astcreation/AstCreatorHelper.scala +++ b/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/astcreation/AstCreatorHelper.scala @@ -3,14 +3,15 @@ package io.joern.csharpsrc2cpg.astcreation import io.joern.csharpsrc2cpg.parser.DotNetJsonAst.* import io.joern.csharpsrc2cpg.parser.{DotNetJsonAst, DotNetNodeInfo, ParserKeys} import io.joern.csharpsrc2cpg.{CSharpDefines, Constants, astcreation} +import io.joern.x2cpg.utils.IntervalKeyPool import io.joern.x2cpg.{Ast, Defines, ValidationMode} import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators, PropertyNames} -import io.shiftleft.passes.IntervalKeyPool import ujson.Value import scala.annotation.tailrec import scala.util.{Failure, Success, Try} + trait AstCreatorHelper(implicit withSchemaValidation: ValidationMode) { this: AstCreator => private val anonymousTypeKeyPool = new IntervalKeyPool(first = 0, last = Long.MaxValue) @@ -62,8 +63,7 @@ trait AstCreatorHelper(implicit withSchemaValidation: ValidationMode) { this: As protected def getTypeFullNameFromAstNode(ast: Ast): String = { ast.root - .flatMap(_.properties.get(PropertyNames.TYPE_FULL_NAME)) - .map(_.toString) + .map(_.propertiesMap.getOrDefault(PropertyNames.TYPE_FULL_NAME, Defines.Any).toString) .getOrElse(Defines.Any) } @@ -83,7 +83,7 @@ trait AstCreatorHelper(implicit withSchemaValidation: ValidationMode) { this: As case x: NewMethodParameterIn => identifierNode(dotNetNode.orNull, x.name, x.code, x.typeFullName, x.dynamicTypeHintFullName) case x => - logger.warn(s"Unhandled declaration type '${x.label()}' for ${x.name}") + logger.warn(s"Unhandled declaration type '${x.label}' for ${x.name}") identifierNode(dotNetNode.orNull, x.name, x.name, Defines.Any) } diff --git a/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/astcreation/AstForStatementsCreator.scala b/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/astcreation/AstForStatementsCreator.scala index cf2de044a993..50e704a18e96 100644 --- a/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/astcreation/AstForStatementsCreator.scala +++ b/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/astcreation/AstForStatementsCreator.scala @@ -286,7 +286,7 @@ trait AstForStatementsCreator(implicit withSchemaValidation: ValidationMode) { t val declNode = createDotNetNodeInfo(usingStmt.json(ParserKeys.Declaration)) val declAst = astForNode(declNode) - val finallyAst = declAst.flatMap(_.nodes).collectFirst { case x: NewIdentifier => x.copy }.map { id => + val finallyAst = declAst.flatMap(_.nodes).collectFirst { case x: NewIdentifier => x.copy() }.map { id => val callCode = s"${id.name}.Dispose()" id.code(callCode) val disposeCall = callNode( diff --git a/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/astcreation/AstSummaryVisitor.scala b/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/astcreation/AstSummaryVisitor.scala index 9b1ba7c73646..551e3a71b4bd 100644 --- a/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/astcreation/AstSummaryVisitor.scala +++ b/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/astcreation/AstSummaryVisitor.scala @@ -1,5 +1,6 @@ package io.joern.csharpsrc2cpg.astcreation +import flatgraph.DiffGraphApplier import io.joern.csharpsrc2cpg.Constants import io.joern.csharpsrc2cpg.datastructures.{ CSharpField, @@ -12,9 +13,8 @@ import io.joern.csharpsrc2cpg.datastructures.{ import io.joern.csharpsrc2cpg.parser.ParserKeys import io.joern.x2cpg.{Ast, ValidationMode} import io.shiftleft.codepropertygraph.generated.nodes.* -import io.shiftleft.codepropertygraph.generated.{Cpg, DiffGraphBuilder, EdgeTypes} +import io.shiftleft.codepropertygraph.generated.{Cpg, EdgeTypes} import io.shiftleft.semanticcpg.language.* -import overflowdb.{BatchedUpdate, Config} import scala.collection.mutable import scala.util.Using @@ -29,11 +29,11 @@ trait AstSummaryVisitor(implicit withSchemaValidation: ValidationMode) { this: A this.parseLevel = AstParseLevel.SIGNATURES val fileNode = NewFile().name(relativeFileName) val compilationUnit = createDotNetNodeInfo(parserResult.json(ParserKeys.AstRoot)) - Using.resource(Cpg.withConfig(Config.withoutOverflow())) { cpg => + Using.resource(Cpg.empty) { cpg => // Build and store compilation unit AST val ast = Ast(fileNode).withChildren(astForCompilationUnit(compilationUnit)) Ast.storeInDiffGraph(ast, diffGraph) - BatchedUpdate.applyDiff(cpg.graph, diffGraph) + DiffGraphApplier.applyDiff(cpg.graph, diffGraph) // Simulate AST Linker for global namespace val globalNode = NewNamespaceBlock().fullName(Constants.Global).name(Constants.Global) @@ -41,7 +41,7 @@ trait AstSummaryVisitor(implicit withSchemaValidation: ValidationMode) { this: A cpg.typeDecl .where(_.astParentFullNameExact(Constants.Global)) .foreach(globalDiffGraph.addEdge(globalNode, _, EdgeTypes.AST)) - BatchedUpdate.applyDiff(cpg.graph, globalDiffGraph) + DiffGraphApplier.applyDiff(cpg.graph, globalDiffGraph) // Summarize findings summarize(cpg) diff --git a/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/datastructures/CSharpProgramSummary.scala b/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/datastructures/CSharpProgramSummary.scala index 6ff1c0715021..ca6598bea083 100644 --- a/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/datastructures/CSharpProgramSummary.scala +++ b/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/datastructures/CSharpProgramSummary.scala @@ -25,7 +25,7 @@ type NamespaceToTypeMap = mutable.Map[String, mutable.Set[CSharpType]] * @see * [[CSharpProgramSummary.jsonToInitialMapping]] for generating initial mappings. */ -case class CSharpProgramSummary(val namespaceToType: NamespaceToTypeMap, val imports: Set[String]) +case class CSharpProgramSummary(namespaceToType: NamespaceToTypeMap, imports: Set[String]) extends ProgramSummary[CSharpType, CSharpMethod, CSharpField] { def findGlobalTypes: Set[CSharpType] = namespaceToType.getOrElse(Constants.Global, Set.empty).toSet diff --git a/joern-cli/frontends/ghidra2cpg/build.sbt b/joern-cli/frontends/ghidra2cpg/build.sbt index 2a8dba0adc58..799f7755271c 100644 --- a/joern-cli/frontends/ghidra2cpg/build.sbt +++ b/joern-cli/frontends/ghidra2cpg/build.sbt @@ -6,8 +6,8 @@ libraryDependencies ++= Seq( "io.joern" % "ghidra" % Versions.ghidra, "com.github.scopt" %% "scopt" % Versions.scopt, "commons-io" % "commons-io" % Versions.commonsIo, - "io.shiftleft" %% "codepropertygraph" % Versions.cpg, - "io.shiftleft" %% "codepropertygraph-protos" % Versions.cpg, + "com.michaelpollmeier" %% "codepropertygraph" % Versions.cpg, + "com.michaelpollmeier" %% "codepropertygraph-protos" % Versions.cpg, "org.scalatest" %% "scalatest" % Versions.scalatest % Test ) diff --git a/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/FunctionPass.scala b/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/FunctionPass.scala index 44fc9ee0e3a4..c32880c96990 100644 --- a/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/FunctionPass.scala +++ b/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/FunctionPass.scala @@ -60,7 +60,7 @@ abstract class FunctionPass( override def generateParts(): Array[Function] = functions.toArray - implicit def intToIntegerOption(intOption: Option[Int]): Option[Integer] = intOption.map(intValue => { + implicit def intToIntegerOption(intOption: Option[Int]): Option[Int] = intOption.map(intValue => { val integerValue = intValue integerValue }) diff --git a/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/PCodePass.scala b/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/PCodePass.scala index 5fd653dfd44e..de8fca302cdf 100644 --- a/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/PCodePass.scala +++ b/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/PCodePass.scala @@ -5,9 +5,8 @@ import ghidra.program.util.DefinedDataIterator import io.joern.ghidra2cpg.* import io.joern.ghidra2cpg.utils.Utils.* import io.joern.ghidra2cpg.utils.{Decompiler, PCodeMapper} -import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.{NewBlock, NewMethod} -import io.shiftleft.codepropertygraph.generated.{EdgeTypes, nodes} +import io.shiftleft.codepropertygraph.generated.{Cpg, EdgeTypes, nodes} import io.shiftleft.passes.ForkJoinParallelCpgPass import scala.jdk.CollectionConverters.* diff --git a/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/arm/ArmFunctionPass.scala b/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/arm/ArmFunctionPass.scala index c0ebcf7d26df..60963b18126e 100644 --- a/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/arm/ArmFunctionPass.scala +++ b/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/arm/ArmFunctionPass.scala @@ -5,9 +5,8 @@ import io.joern.ghidra2cpg.utils.Decompiler import io.joern.ghidra2cpg.passes.FunctionPass import io.joern.ghidra2cpg.processors.ArmProcessor import io.joern.ghidra2cpg.utils.Utils.{checkIfExternal, createMethodNode, createReturnNode} -import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.NewBlock -import io.shiftleft.codepropertygraph.generated.{EdgeTypes, nodes} +import io.shiftleft.codepropertygraph.generated.{Cpg, EdgeTypes, nodes} class ArmFunctionPass( currentProgram: Program, diff --git a/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/mips/LoHiPass.scala b/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/mips/LoHiPass.scala index 36e43c08854b..70f7bbe485dc 100644 --- a/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/mips/LoHiPass.scala +++ b/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/mips/LoHiPass.scala @@ -23,6 +23,9 @@ class LoHiPass(cpg: Cpg) extends ForkJoinParallelCpgPass[(Call, Call)](cpg) { }.toArray override def runOnPart(diffGraph: DiffGraphBuilder, pair: (Call, Call)): Unit = { - diffGraph.addEdge(pair._1, pair._2, EdgeTypes.REACHING_DEF, PropertyNames.VARIABLE, pair._1.code) + // in flatgraph an edge may have zero or one properties and they're not named... + // in this case we know that we're dealing with ReachingDef edges which has the `variable` property + val variableProperty = pair._1.code + diffGraph.addEdge(pair._1, pair._2, EdgeTypes.REACHING_DEF, variableProperty) } } diff --git a/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/mips/MipsFunctionPass.scala b/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/mips/MipsFunctionPass.scala index 71222e896100..84a8ee611c4f 100644 --- a/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/mips/MipsFunctionPass.scala +++ b/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/mips/MipsFunctionPass.scala @@ -10,9 +10,8 @@ import io.joern.ghidra2cpg.processors.MipsProcessor import io.joern.ghidra2cpg.utils.Utils.* import io.joern.ghidra2cpg.Types import io.joern.ghidra2cpg.utils.Decompiler -import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.{CfgNodeNew, NewBlock} -import io.shiftleft.codepropertygraph.generated.{EdgeTypes, nodes} +import io.shiftleft.codepropertygraph.generated.{Cpg, EdgeTypes, nodes} import org.slf4j.LoggerFactory import scala.jdk.CollectionConverters.* diff --git a/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/mips/MipsReturnEdgesPass.scala b/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/mips/MipsReturnEdgesPass.scala index 7637edca44f7..468d6a3e82f5 100644 --- a/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/mips/MipsReturnEdgesPass.scala +++ b/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/mips/MipsReturnEdgesPass.scala @@ -17,7 +17,10 @@ class MipsReturnEdgesPass(cpg: Cpg) extends CpgPass(cpg) { // the first .cfgNext is skipping a _nop instruction after the call val to = from.cfgNext.cfgNext.isCall.argument.code("v(0|1)").headOption if (to.nonEmpty) { - diffGraph.addEdge(from, to.get, EdgeTypes.REACHING_DEF, PropertyNames.VARIABLE, from.code) + // in flatgraph an edge may have zero or one properties and they're not named... + // in this case we know that we're dealing with ReachingDef edges which has the `variable` property + val variableProperty = from.code + diffGraph.addEdge(from, to.get, EdgeTypes.REACHING_DEF, variableProperty) } } } diff --git a/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/x86/ReturnEdgesPass.scala b/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/x86/ReturnEdgesPass.scala index 42dfc354d787..0669e45b7a16 100644 --- a/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/x86/ReturnEdgesPass.scala +++ b/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/x86/ReturnEdgesPass.scala @@ -15,7 +15,11 @@ class ReturnEdgesPass(cpg: Cpg) extends CpgPass(cpg) { cpg.call.nameNot(".*").foreach { from => // We expect RAX/EAX as return val to = from.cfgNext.isCall.argument.code("(R|E)AX").headOption - if (to.nonEmpty) diffGraph.addEdge(from, to.get, EdgeTypes.REACHING_DEF, PropertyNames.VARIABLE, from.code) + + // in flatgraph an edge may have zero or one properties and they're not named... + // in this case we know that we're dealing with ReachingDef edges which has the `variable` property + val variableProperty = from.code + if (to.nonEmpty) diffGraph.addEdge(from, to.get, EdgeTypes.REACHING_DEF, variableProperty) } } diff --git a/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/fixtures/DataFlowBinToCpgSuite.scala b/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/fixtures/DataFlowBinToCpgSuite.scala index f8b6abfbdf82..a586f3206acd 100644 --- a/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/fixtures/DataFlowBinToCpgSuite.scala +++ b/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/fixtures/DataFlowBinToCpgSuite.scala @@ -9,12 +9,13 @@ import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.dotextension.ImageViewer import io.shiftleft.semanticcpg.layers.* +import scala.compiletime.uninitialized import scala.sys.process.Process import scala.util.Try class DataFlowBinToCpgSuite extends GhidraBinToCpgSuite { - implicit var context: EngineContext = scala.compiletime.uninitialized + implicit var context: EngineContext = uninitialized override def beforeAll(): Unit = { super.beforeAll() @@ -33,7 +34,7 @@ class DataFlowBinToCpgSuite extends GhidraBinToCpgSuite { new OssDataFlow(options).run(context) } - protected implicit def int2IntegerOption(x: Int): Option[Integer] = + protected implicit def int2IntegerOption(x: Int): Option[Int] = Some(x) protected def getMemberOfType(cpg: Cpg, typeName: String, memberName: String): Iterator[Member] = diff --git a/joern-cli/frontends/ghidra2cpg/src/test/testbinaries/coverage/testscript.sc b/joern-cli/frontends/ghidra2cpg/src/test/testbinaries/coverage/testscript.sc index 81cab22add9e..2cd441ae3888 100644 --- a/joern-cli/frontends/ghidra2cpg/src/test/testbinaries/coverage/testscript.sc +++ b/joern-cli/frontends/ghidra2cpg/src/test/testbinaries/coverage/testscript.sc @@ -2,7 +2,7 @@ import io.shiftleft.codepropertygraph.generated.Cpg import io.joern.dataflowengineoss.language._ import io.shiftleft.semanticcpg.language._ import io.shiftleft.semanticcpg.language.operatorextension.OpNodes.Assignment -import overflowdb.traversal._ +import flatgraph.traversal._ @main def main(testBinary: String) = { importCode.ghidra(testBinary) diff --git a/joern-cli/frontends/gosrc2cpg/build.sbt b/joern-cli/frontends/gosrc2cpg/build.sbt index b91c175eaa13..646c8c2de03b 100644 --- a/joern-cli/frontends/gosrc2cpg/build.sbt +++ b/joern-cli/frontends/gosrc2cpg/build.sbt @@ -9,7 +9,9 @@ name := "gosrc2cpg" dependsOn(Projects.dataflowengineoss % "compile->compile;test->test", Projects.x2cpg % "compile->compile;test->test") libraryDependencies ++= Seq( - "io.shiftleft" %% "codepropertygraph" % Versions.cpg, + // TODO back to io.shiftleft! + //"io.shiftleft" %% "codepropertygraph" % Versions.cpg, + "com.michaelpollmeier" %% "codepropertygraph" % Versions.cpg, "org.scalatest" %% "scalatest" % Versions.scalatest % Test, "com.lihaoyi" %% "os-lib" % Versions.osLib ) diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstCreatorHelper.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstCreatorHelper.scala index 93f690fd64f9..cb7689d0f55a 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstCreatorHelper.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstCreatorHelper.scala @@ -83,7 +83,7 @@ trait AstCreatorHelper { this: AstCreator => protected def getTypeFullNameFromAstNode(ast: Seq[Ast]): String = { ast.headOption .flatMap(_.root) - .map(_.properties.get(PropertyNames.TYPE_FULL_NAME).get.toString) + .map(_.propertiesMap.get(PropertyNames.TYPE_FULL_NAME).toString) .getOrElse(Defines.anyTypeName) } diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForFunctionsCreator.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForFunctionsCreator.scala index 4cce137d4030..c83139f26cad 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForFunctionsCreator.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForFunctionsCreator.scala @@ -6,6 +6,7 @@ import io.joern.x2cpg.datastructures.Stack.* import io.joern.x2cpg.utils.NodeBuilders import io.joern.x2cpg.{Ast, ValidationMode} import io.shiftleft.codepropertygraph.generated.{EvaluationStrategies, NodeTypes} +import io.shiftleft.codepropertygraph.generated.nodes.* import ujson.Value import scala.collection.mutable diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForMethodCallExpressionCreator.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForMethodCallExpressionCreator.scala index 1882d37cc897..06bf3b2da8a9 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForMethodCallExpressionCreator.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForMethodCallExpressionCreator.scala @@ -153,7 +153,7 @@ trait AstForMethodCallExpressionCreator(implicit withSchemaValidation: Validatio val receiverTypeFullName = receiverAst.headOption .flatMap(_.root) - .map(_.properties.get(PropertyNames.TYPE_FULL_NAME).get.toString) + .map(_.propertiesMap.get(PropertyNames.TYPE_FULL_NAME).toString) .getOrElse(Defines.anyTypeName) .stripPrefix("*") val callMethodFullName = s"$receiverTypeFullName.$methodName" diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForPackageConstructorCreator.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForPackageConstructorCreator.scala index 31b8d03a1732..0a29cd71b48e 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForPackageConstructorCreator.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForPackageConstructorCreator.scala @@ -1,11 +1,13 @@ package io.joern.gosrc2cpg.astcreation +import flatgraph.DiffGraphBuilder import io.joern.gosrc2cpg.datastructures.PackageMemberAst import io.joern.gosrc2cpg.parser.ParserAst.Unknown import io.joern.gosrc2cpg.parser.ParserNodeInfo import io.joern.x2cpg.astgen.AstGenNodeBuilder import io.joern.x2cpg.{Ast, AstCreatorBase, ValidationMode, Defines as XDefines} import io.shiftleft.codepropertygraph.generated.NodeTypes +import flatgraph.DiffGraphBuilder import org.apache.commons.lang3.StringUtils import io.shiftleft.codepropertygraph.generated.DiffGraphBuilder import ujson.Value diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/CommonCacheBuilder.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/CommonCacheBuilder.scala index 4ae269223322..6df9165623ba 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/CommonCacheBuilder.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/CommonCacheBuilder.scala @@ -4,7 +4,9 @@ import io.joern.gosrc2cpg.datastructures.MethodCacheMetaData import io.joern.gosrc2cpg.parser.ParserAst.* import io.joern.gosrc2cpg.parser.{ParserKeys, ParserNodeInfo} import io.joern.x2cpg.{Ast, ValidationMode} -import ujson.Value +import io.shiftleft.codepropertygraph.generated.Cpg +import io.shiftleft.codepropertygraph.generated.nodes.NewNamespaceBlock +import ujson.{Arr, Obj, Value} import scala.util.Try diff --git a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/DownloadDependencyTest.scala b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/DownloadDependencyTest.scala index 4453314d54ba..4cc107c8ad32 100644 --- a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/DownloadDependencyTest.scala +++ b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/DownloadDependencyTest.scala @@ -225,9 +225,9 @@ class DownloadDependencyTest extends GoCodeToCpgSuite { "not create any entry in method full name to return type map" in { // This should only contain the `main` method return type mapping as main source code is not invoking any of the dependency method. goGlobal.nameSpaceMetaDataMap.size() shouldBe 1 - val Array(metadata) = goGlobal.nameSpaceMetaDataMap.values().iterator().toArray + val Array(metadata) = goGlobal.nameSpaceMetaDataMap.values().iterator().asScala.toArray metadata.methodMetaMap.size() shouldBe 1 - val List(mainfullname) = metadata.methodMetaMap.keys().asIterator().toList + val List(mainfullname) = metadata.methodMetaMap.keys().asIterator().asScala.toList mainfullname shouldBe "main" val Array(returnType) = metadata.methodMetaMap.values().toArray returnType shouldBe MethodCacheMetaData(Defines.voidTypeName, "main.main()") @@ -236,7 +236,7 @@ class DownloadDependencyTest extends GoCodeToCpgSuite { "not create any entry in struct member to type map" in { // This should be empty as neither main code has defined any struct type nor we are accessing the third party struct type. goGlobal.nameSpaceMetaDataMap.size() shouldBe 1 - val Array(metadata) = goGlobal.nameSpaceMetaDataMap.values().iterator().toArray + val Array(metadata) = goGlobal.nameSpaceMetaDataMap.values().iterator().asScala.toArray metadata.structTypeMembers.size() shouldBe 0 } } @@ -298,9 +298,9 @@ class DownloadDependencyTest extends GoCodeToCpgSuite { "not create any entry in method full name to return type map" ignore { // This should only contain the `main` method return type mapping as main source code is not invoking any of the dependency method. goGlobal.nameSpaceMetaDataMap.size() shouldBe 1 - val Array(metadata) = goGlobal.nameSpaceMetaDataMap.values().iterator().toArray + val Array(metadata) = goGlobal.nameSpaceMetaDataMap.values().iterator().asScala.toArray metadata.methodMetaMap.size() shouldBe 1 - val List(mainfullname) = metadata.methodMetaMap.keys().asIterator().toList + val List(mainfullname) = metadata.methodMetaMap.keys().asIterator().asScala.toList mainfullname shouldBe "main" val Array(returnType) = metadata.methodMetaMap.values().toArray returnType shouldBe MethodCacheMetaData(Defines.voidTypeName, "main.main()") @@ -310,7 +310,7 @@ class DownloadDependencyTest extends GoCodeToCpgSuite { "not create any entry in struct member to type map" ignore { // This should be empty as neither main code has defined any struct type nor we are accessing the third party struct type. goGlobal.nameSpaceMetaDataMap.size() shouldBe 1 - val Array(metadata) = goGlobal.nameSpaceMetaDataMap.values().iterator().toArray + val Array(metadata) = goGlobal.nameSpaceMetaDataMap.values().iterator().asScala.toArray metadata.structTypeMembers.size() shouldBe 0 } } @@ -397,9 +397,9 @@ class DownloadDependencyTest extends GoCodeToCpgSuite { // TODO: While doing the implementation we need update this test // Lambda expression return types are also getting recorded under this map goGlobal.nameSpaceMetaDataMap.size() shouldBe 1 - val Array(metadata) = goGlobal.nameSpaceMetaDataMap.values().iterator().toArray + val Array(metadata) = goGlobal.nameSpaceMetaDataMap.values().iterator().asScala.toArray metadata.methodMetaMap.size() shouldBe 1 - val List(mainfullname) = metadata.methodMetaMap.keys().asIterator().toList + val List(mainfullname) = metadata.methodMetaMap.keys().asIterator().asScala.toList mainfullname shouldBe "main" val Array(returnType) = metadata.methodMetaMap.values().toArray returnType shouldBe MethodCacheMetaData(Defines.voidTypeName, "main.main()") @@ -412,7 +412,7 @@ class DownloadDependencyTest extends GoCodeToCpgSuite { // 2. Struct Type is being passed as parameter or returned as value of method that is being used. // 3. A method of Struct Type being used. goGlobal.nameSpaceMetaDataMap.size() shouldBe 1 - val Array(metadata) = goGlobal.nameSpaceMetaDataMap.values().iterator().toArray + val Array(metadata) = goGlobal.nameSpaceMetaDataMap.values().iterator().asScala.toArray metadata.structTypeMembers.size() shouldBe 0 } } diff --git a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/MethodCallTests.scala b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/MethodCallTests.scala index ceb574e867ed..56972e8865a6 100644 --- a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/MethodCallTests.scala +++ b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/MethodCallTests.scala @@ -7,9 +7,8 @@ import io.shiftleft.codepropertygraph.generated.edges.Ref import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators, nodes} import io.shiftleft.semanticcpg.language.* -import overflowdb.traversal.{jIteratortoTraversal, toNodeTraversal} - import java.io.File + class MethodCallTests extends GoCodeToCpgSuite(withOssDataflow = true) { "Simple method call use case" should { diff --git a/joern-cli/frontends/javasrc2cpg/build.sbt b/joern-cli/frontends/javasrc2cpg/build.sbt index 130b79c3c0cc..3e6e28bdfedb 100644 --- a/joern-cli/frontends/javasrc2cpg/build.sbt +++ b/joern-cli/frontends/javasrc2cpg/build.sbt @@ -3,7 +3,7 @@ name := "javasrc2cpg" dependsOn(Projects.dataflowengineoss % "compile->compile;test->test", Projects.x2cpg % "compile->compile;test->test") libraryDependencies ++= Seq( - "io.shiftleft" %% "codepropertygraph" % Versions.cpg, + "com.michaelpollmeier" %% "codepropertygraph" % Versions.cpg, "com.github.javaparser" % "javaparser-symbol-solver-core" % Versions.javaParser, "org.gradle" % "gradle-tooling-api" % Versions.gradleTooling, "org.scalatest" %% "scalatest" % Versions.scalatest % Test, diff --git a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/AstCreator.scala b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/AstCreator.scala index 047f5490b2ea..c7b5e5a46c59 100644 --- a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/AstCreator.scala +++ b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/AstCreator.scala @@ -135,10 +135,10 @@ class AstCreator( .removeOption(new DefaultConfigurationOption(ConfigOption.PRINT_COMMENTS)) .removeOption(new DefaultConfigurationOption(ConfigOption.PRINT_JAVADOC)) - protected def line(node: Node): Option[Int] = node.getBegin.map(x => x.line).toScala - protected def column(node: Node): Option[Int] = node.getBegin.map(x => x.column).toScala - protected def lineEnd(node: Node): Option[Int] = node.getEnd.map(x => x.line).toScala - protected def columnEnd(node: Node): Option[Int] = node.getEnd.map(x => x.column).toScala + protected def line(node: Node): Option[Int] = node.getBegin.map(_.line).toScala + protected def column(node: Node): Option[Int] = node.getBegin.map(_.column).toScala + protected def lineEnd(node: Node): Option[Int] = node.getEnd.map(_.line).toScala + protected def columnEnd(node: Node): Option[Int] = node.getEnd.map(_.column).toScala protected def code(node: Node): String = node.toString(codePrinterOptions) private val lineOffsetTable = OffsetUtils.getLineOffsetTable(fileContent) diff --git a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/declarations/AstForMethodsCreator.scala b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/declarations/AstForMethodsCreator.scala index 9352e254b8f8..1780318be784 100644 --- a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/declarations/AstForMethodsCreator.scala +++ b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/declarations/AstForMethodsCreator.scala @@ -464,8 +464,8 @@ private[declarations] trait AstForMethodsCreator { this: AstCreator => } private def constructorReturnNode(constructorDeclaration: ConstructorDeclaration): NewMethodReturn = { - val line = constructorDeclaration.getEnd.map(x => x.line).toScala - val column = constructorDeclaration.getEnd.map(x => x.column).toScala + val line = constructorDeclaration.getEnd.map(_.line).toScala + val column = constructorDeclaration.getEnd.map(_.column).toScala newMethodReturnNode(TypeConstants.Void, None, line, column) } diff --git a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/expressions/AstForCallExpressionsCreator.scala b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/expressions/AstForCallExpressionsCreator.scala index 88687b18fa29..64a4e399bf65 100644 --- a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/expressions/AstForCallExpressionsCreator.scala +++ b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/expressions/AstForCallExpressionsCreator.scala @@ -150,7 +150,7 @@ trait AstForCallExpressionsCreator { this: AstCreator => // then use the types of those to fix the local type. val assignTarget = identifierNode(expr, tmpName, tmpName, TypeConstants.Any) val allocAndInitAst = - inlinedAstsForObjectCreationExpr(expr, Ast(assignTarget.copy), expectedType, resetAssignmentTargetType = true) + inlinedAstsForObjectCreationExpr(expr, Ast(assignTarget.copy()), expectedType, resetAssignmentTargetType = true) assignTarget.typeFullName(allocAndInitAst.allocAst.rootType.getOrElse(TypeConstants.Any)) val tmpLocal = localNode(expr, tmpName, tmpName, assignTarget.typeFullName) @@ -171,7 +171,7 @@ trait AstForCallExpressionsCreator { this: AstCreator => val allocAssignTargetAst = Ast(allocAssignTargetNode).withRefEdge(allocAssignTargetNode, tmpLocal) val allocAssignAst = callAst(allocAssignCall, allocAssignTargetAst :: allocAndInitAst.allocAst :: Nil) - val returnedIdentifier = assignTarget.copy + val returnedIdentifier = assignTarget.copy() val returnedIdentifierAst = Ast(returnedIdentifier).withRefEdge(returnedIdentifier, tmpLocal) Ast(blockNode(expr).typeFullName(returnedIdentifier.typeFullName)) diff --git a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/expressions/AstForLambdasCreator.scala b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/expressions/AstForLambdasCreator.scala index f38c1d958779..98e756c4a52d 100644 --- a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/expressions/AstForLambdasCreator.scala +++ b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/expressions/AstForLambdasCreator.scala @@ -17,13 +17,12 @@ import io.joern.javasrc2cpg.util.BindingTable.createBindingTable import io.joern.javasrc2cpg.util.Util.{composeMethodFullName, composeMethodLikeSignature, composeUnresolvedSignature} import io.joern.javasrc2cpg.util.{BindingTable, BindingTableAdapterForLambdas, LambdaBindingInfo, NameConstants} import io.joern.x2cpg.utils.AstPropertiesUtil.* -import io.joern.x2cpg.utils.NodeBuilders +import io.joern.x2cpg.utils.{IntervalKeyPool, NodeBuilders} import io.joern.x2cpg.utils.NodeBuilders.* import io.joern.x2cpg.{Ast, Defines} import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.codepropertygraph.generated.nodes.MethodParameterIn.PropertyDefaults as ParameterDefaults import io.shiftleft.codepropertygraph.generated.{EdgeTypes, EvaluationStrategies, ModifierTypes} -import io.shiftleft.passes.IntervalKeyPool import org.slf4j.LoggerFactory import scala.jdk.CollectionConverters.* diff --git a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/expressions/AstForVarDeclAndAssignsCreator.scala b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/expressions/AstForVarDeclAndAssignsCreator.scala index 174eabe7617a..00c1081dbe7d 100644 --- a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/expressions/AstForVarDeclAndAssignsCreator.scala +++ b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/expressions/AstForVarDeclAndAssignsCreator.scala @@ -71,7 +71,7 @@ trait AstForVarDeclAndAssignsCreator { this: AstCreator => private def copyAstForVarDeclInit(targetAst: Ast): Ast = { targetAst.root match { - case Some(identifier: NewIdentifier) => Ast(identifier.copy) + case Some(identifier: NewIdentifier) => Ast(identifier.copy()) case Some(fieldAccess: NewCall) if fieldAccess.name == Operators.fieldAccess => val maybeIdentifier = targetAst.nodes.collectFirst { case node if node.isInstanceOf[NewIdentifier] => node } @@ -79,8 +79,8 @@ trait AstForVarDeclAndAssignsCreator { this: AstCreator => (maybeIdentifier, maybeField) match { case (Some(identifier), Some(fieldIdentifier)) => - val args = List(identifier, fieldIdentifier).map(node => Ast(node.copy)) - callAst(fieldAccess.copy, args) + val args = List(identifier, fieldIdentifier).map(node => Ast(node.copy())) + callAst(fieldAccess.copy(), args) case _ => logger.warn(s"Attempting to copy field access without required children: ${fieldAccess.code}") diff --git a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/statements/AstForForLoopsCreator.scala b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/statements/AstForForLoopsCreator.scala index 0370d6ccc52e..5938bff74273 100644 --- a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/statements/AstForForLoopsCreator.scala +++ b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/statements/AstForForLoopsCreator.scala @@ -6,6 +6,7 @@ import io.joern.javasrc2cpg.astcreation.{AstCreator, ExpectedType} import io.joern.javasrc2cpg.scope.NodeTypeInfo import io.joern.javasrc2cpg.typesolvers.TypeInfoCalculator.TypeConstants import io.joern.x2cpg.Ast +import io.joern.x2cpg.utils.IntervalKeyPool import io.joern.x2cpg.utils.NodeBuilders.{newCallNode, newFieldIdentifierNode, newIdentifierNode, newOperatorCallNode} import io.shiftleft.codepropertygraph.generated.nodes.Call.PropertyDefaults import io.shiftleft.codepropertygraph.generated.nodes.{ @@ -18,7 +19,6 @@ import io.shiftleft.codepropertygraph.generated.nodes.{ NewNode } import io.shiftleft.codepropertygraph.generated.{ControlStructureTypes, DispatchTypes, Operators} -import io.shiftleft.passes.IntervalKeyPool import org.slf4j.LoggerFactory import scala.jdk.CollectionConverters.* diff --git a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/passes/TypeInferencePass.scala b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/passes/TypeInferencePass.scala index 3ebc038b928f..a92e51fe2c42 100644 --- a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/passes/TypeInferencePass.scala +++ b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/passes/TypeInferencePass.scala @@ -3,8 +3,7 @@ package io.joern.javasrc2cpg.passes import com.github.javaparser.symbolsolver.cache.GuavaCache import com.google.common.cache.CacheBuilder import io.joern.x2cpg.Defines -import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.codepropertygraph.generated.ModifierTypes +import io.shiftleft.codepropertygraph.generated.{Cpg, ModifierTypes, Properties} import io.shiftleft.codepropertygraph.generated.nodes.{Call, Method} import io.shiftleft.passes.ForkJoinParallelCpgPass import io.shiftleft.semanticcpg.language.* @@ -55,11 +54,8 @@ class TypeInferencePass(cpg: Cpg) extends ForkJoinParallelCpgPass[Call](cpg) { val callArgs = if (skipCallThis) call.argument.toList.tail else call.argument.toList val hasDifferingArg = method.parameter.zip(callArgs).exists { case (parameter, argument) => - val maybeArgumentType = Option(argument.property(PropertyNames.TypeFullName)) - .map(_.toString()) - .getOrElse(TypeConstants.Any) - - val argMatches = maybeArgumentType == TypeConstants.Any || maybeArgumentType == parameter.typeFullName + val maybeArgumentType = argument.propertyOption(Properties.TypeFullName).getOrElse(TypeConstants.Any) + val argMatches = maybeArgumentType == TypeConstants.Any || maybeArgumentType == parameter.typeFullName !argMatches } @@ -80,10 +76,8 @@ class TypeInferencePass(cpg: Cpg) extends ForkJoinParallelCpgPass[Call](cpg) { } private def getReplacementMethod(call: Call): Option[Method] = { - val argTypes = - call.argument.flatMap(arg => Option(arg.property(PropertyNames.TypeFullName)).map(_.toString)).mkString(":") - val callKey = - s"${call.methodFullName}:$argTypes" + val argTypes = call.argument.property(Properties.TypeFullName).mkString(":") + val callKey = s"${call.methodFullName}:$argTypes" cache.get(callKey).toScala.getOrElse { val callNameParts = getNameParts(call.name, call.methodFullName) resolvedMethodIndex.get(call.name).flatMap { candidateMethods => diff --git a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/scope/JavaScopeElement.scala b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/scope/JavaScopeElement.scala index 04f5df7ea105..005cf188c7f3 100644 --- a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/scope/JavaScopeElement.scala +++ b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/scope/JavaScopeElement.scala @@ -11,7 +11,7 @@ import io.shiftleft.codepropertygraph.generated.nodes.NewMethodParameterIn import io.shiftleft.codepropertygraph.generated.nodes.NewLocal import io.shiftleft.codepropertygraph.generated.nodes.NewMember import io.joern.javasrc2cpg.util.{BindingTable, BindingTableEntry, NameConstants} -import io.shiftleft.passes.IntervalKeyPool +import io.joern.x2cpg.utils.IntervalKeyPool import io.joern.x2cpg.Ast trait JavaScopeElement { diff --git a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/util/Util.scala b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/util/Util.scala index cb725733a243..9d844758da35 100644 --- a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/util/Util.scala +++ b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/util/Util.scala @@ -2,10 +2,7 @@ package io.joern.javasrc2cpg.util import com.github.javaparser.resolution.declarations.ResolvedReferenceTypeDeclaration import com.github.javaparser.resolution.types.ResolvedReferenceType -import io.joern.javasrc2cpg.typesolvers.TypeInfoCalculator.TypeConstants -import io.joern.x2cpg.{Ast, Defines} -import io.shiftleft.codepropertygraph.generated.{DispatchTypes, PropertyNames} -import io.shiftleft.codepropertygraph.generated.nodes.{NewCall, NewFieldIdentifier, NewMember} +import io.joern.x2cpg.Defines import org.slf4j.LoggerFactory import scala.collection.mutable diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/passes/ConfigFileCreationPassTests.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/passes/ConfigFileCreationPassTests.scala index 1c67e821d798..d48a1a873f6e 100644 --- a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/passes/ConfigFileCreationPassTests.scala +++ b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/passes/ConfigFileCreationPassTests.scala @@ -1,14 +1,13 @@ package io.joern.javasrc2cpg.passes import better.files.File +import flatgraph.misc.TestUtils.* import io.joern.javasrc2cpg.testfixtures.JavaSrcCode2CpgFixture import io.joern.x2cpg.passes.frontend.JavaConfigFileCreationPass import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.NewMetaData import io.shiftleft.semanticcpg.language.* import io.shiftleft.utils.ProjectRoot -import overflowdb.BatchedUpdate -import io.shiftleft.codepropertygraph.generated.DiffGraphBuilder import java.nio.file.Paths @@ -18,8 +17,8 @@ class ConfigFileCreationPassTests extends JavaSrcCode2CpgFixture { ProjectRoot.relativise("joern-cli/frontends/javasrc2cpg/src/test/resources/config_tests") "it should find the correct config files" in { - val cpg = new Cpg() - BatchedUpdate.applyDiff(cpg.graph, Cpg.newDiffGraphBuilder.addNode(NewMetaData().root(testConfigDir)).build()) + val cpg = Cpg.from(_.addNode(NewMetaData().root(testConfigDir))) + val foundFiles = new JavaConfigFileCreationPass(cpg).generateParts().map(_.canonicalPath) val absoluteConfigDir = File(testConfigDir).canonicalPath foundFiles should contain theSameElementsAs Array( diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/CallTests.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/CallTests.scala index 86b4d600d1cd..9e62224b211f 100644 --- a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/CallTests.scala +++ b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/CallTests.scala @@ -7,8 +7,6 @@ import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators, nodes import io.shiftleft.codepropertygraph.generated.nodes.{Call, FieldIdentifier, Identifier, Literal, MethodParameterIn} import io.shiftleft.semanticcpg.language.NoResolve import io.shiftleft.semanticcpg.language.* -import overflowdb.traversal.jIteratortoTraversal -import overflowdb.traversal.toNodeTraversal class NewCallTests extends JavaSrcCode2CpgFixture { "calls to imported methods" when { @@ -284,7 +282,7 @@ class NewCallTests extends JavaSrcCode2CpgFixture { cpg.method.name("test").call.name("foo").argument(0).outE.collectAll[Ref].l match { case List(ref) => - ref.inNode match { + ref.dst match { case param: MethodParameterIn => param.name shouldBe "this" param.index shouldBe 0 @@ -309,7 +307,7 @@ class NewCallTests extends JavaSrcCode2CpgFixture { cpg.method.name("test").call.name("foo").argument(0).outE.collectAll[Ref].l match { case List(ref) => - ref.inNode match { + ref.dst match { case param: MethodParameterIn => param.name shouldBe "this" param.index shouldBe 0 diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/ConstructorInvocationTests.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/ConstructorInvocationTests.scala index 2c3c5a20b2ab..c6a05511822d 100644 --- a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/ConstructorInvocationTests.scala +++ b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/ConstructorInvocationTests.scala @@ -262,20 +262,20 @@ class ConstructorInvocationTests extends JavaSrcCode2CpgFixture { case List(method) => val List(_: Local, assign: Call, init: Call) = method.astChildren.isBlock.astChildren.l: @unchecked - assign.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.toString + assign.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.name() assign.name shouldBe Operators.assignment val alloc = assign.argument(2).asInstanceOf[Call] alloc.name shouldBe ".alloc" alloc.code shouldBe "new Bar(4, 2)" - alloc.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.toString + alloc.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.name() alloc.methodFullName shouldBe ".alloc" alloc.typeFullName shouldBe "Bar" alloc.argument.size shouldBe 0 init.name shouldBe io.joern.x2cpg.Defines.ConstructorMethodName init.methodFullName shouldBe s"Bar.${io.joern.x2cpg.Defines.ConstructorMethodName}:void(int,int)" - init.callOut.head.fullName shouldBe s"Bar.${io.joern.x2cpg.Defines.ConstructorMethodName}:void(int,int)" - init.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.toString + init._methodViaCallOut.head.fullName shouldBe s"Bar.${io.joern.x2cpg.Defines.ConstructorMethodName}:void(int,int)" + init.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.name() init.typeFullName shouldBe "void" init.signature shouldBe "void(int,int)" init.code shouldBe "new Bar(4, 2)" @@ -303,20 +303,20 @@ class ConstructorInvocationTests extends JavaSrcCode2CpgFixture { case List(method) => val List(assign: Call, init: Call) = method.astChildren.isBlock.astChildren.l: @unchecked - assign.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.toString + assign.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.name() assign.name shouldBe Operators.assignment val alloc = assign.argument(2).asInstanceOf[Call] alloc.name shouldBe ".alloc" alloc.code shouldBe "new Bar(4, 2)" - alloc.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.toString + alloc.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.name() alloc.methodFullName shouldBe ".alloc" alloc.typeFullName shouldBe "Bar" alloc.argument.size shouldBe 0 init.name shouldBe io.joern.x2cpg.Defines.ConstructorMethodName init.methodFullName shouldBe s"Bar.${io.joern.x2cpg.Defines.ConstructorMethodName}:void(int,int)" - init.callOut.head.fullName shouldBe s"Bar.${io.joern.x2cpg.Defines.ConstructorMethodName}:void(int,int)" - init.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.toString + init._methodViaCallOut.head.fullName shouldBe s"Bar.${io.joern.x2cpg.Defines.ConstructorMethodName}:void(int,int)" + init.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.name() init.typeFullName shouldBe "void" init.signature shouldBe "void(int,int)" init.code shouldBe "new Bar(4, 2)" @@ -362,16 +362,16 @@ class ConstructorInvocationTests extends JavaSrcCode2CpgFixture { alloc.order shouldBe 2 alloc.argumentIndex shouldBe 2 alloc.code shouldBe "new Bar(42)" - alloc.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.toString + alloc.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.name() alloc.typeFullName shouldBe "Bar" alloc.argument.size shouldBe 0 init.name shouldBe io.joern.x2cpg.Defines.ConstructorMethodName init.methodFullName shouldBe s"Bar.${io.joern.x2cpg.Defines.ConstructorMethodName}:void(int)" - init.callOut.head.fullName shouldBe s"Bar.${io.joern.x2cpg.Defines.ConstructorMethodName}:void(int)" + init._methodViaCallOut.head.fullName shouldBe s"Bar.${io.joern.x2cpg.Defines.ConstructorMethodName}:void(int)" init.signature shouldBe "void(int)" init.code shouldBe "new Bar(42)" - init.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.toString + init.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.name() init.argument.size shouldBe 2 val List(obj: Identifier, initArg1: Literal) = init.argument.l: @unchecked @@ -411,16 +411,16 @@ class ConstructorInvocationTests extends JavaSrcCode2CpgFixture { alloc.order shouldBe 2 alloc.argumentIndex shouldBe 2 alloc.code shouldBe "new Bar(42)" - alloc.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.toString + alloc.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.name() alloc.typeFullName shouldBe "Bar" alloc.argument.size shouldBe 0 init.name shouldBe io.joern.x2cpg.Defines.ConstructorMethodName init.methodFullName shouldBe s"Bar.${io.joern.x2cpg.Defines.ConstructorMethodName}:void(int)" - init.callOut.head.fullName shouldBe s"Bar.${io.joern.x2cpg.Defines.ConstructorMethodName}:void(int)" + init._methodViaCallOut.head.fullName shouldBe s"Bar.${io.joern.x2cpg.Defines.ConstructorMethodName}:void(int)" init.signature shouldBe "void(int)" init.code shouldBe "new Bar(42)" - init.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.toString + init.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.name() init.argument.size shouldBe 2 val List(obj: Identifier, initArg1: Literal) = init.argument.l: @unchecked @@ -447,7 +447,7 @@ class ConstructorInvocationTests extends JavaSrcCode2CpgFixture { val List(init: Call) = method.astChildren.isBlock.astChildren.l: @unchecked init.name shouldBe io.joern.x2cpg.Defines.ConstructorMethodName init.methodFullName shouldBe s"Bar.${io.joern.x2cpg.Defines.ConstructorMethodName}:void(int)" - init.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.toString + init.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.name() init.typeFullName shouldBe "void" init.signature shouldBe "void(int)" @@ -475,7 +475,7 @@ class ConstructorInvocationTests extends JavaSrcCode2CpgFixture { val List(init: Call) = method.astChildren.isBlock.astChildren.l: @unchecked init.name shouldBe io.joern.x2cpg.Defines.ConstructorMethodName init.methodFullName shouldBe s"Foo.${io.joern.x2cpg.Defines.ConstructorMethodName}:void(int)" - init.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.toString + init.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.name() init.typeFullName shouldBe "void" init.signature shouldBe "void(int)" diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/ControlStructureTests.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/ControlStructureTests.scala index 6ed2dd311e14..c26cd0355f5b 100644 --- a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/ControlStructureTests.scala +++ b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/ControlStructureTests.scala @@ -14,7 +14,6 @@ import io.shiftleft.codepropertygraph.generated.nodes.{ Return } import io.shiftleft.semanticcpg.language.* -import overflowdb.traversal.toNodeTraversal import scala.jdk.CollectionConverters.* diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/LambdaTests.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/LambdaTests.scala index aecb56aa7fd2..88fd953d2fc8 100644 --- a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/LambdaTests.scala +++ b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/LambdaTests.scala @@ -708,7 +708,7 @@ class LambdaTests extends JavaSrcCode2CpgFixture { case myValue :: Nil => myValue.closureOriginalName.head shouldBe "myValue" myValue._localViaRefOut.get.name shouldBe "myValue" - myValue._captureIn.collectFirst { case x: MethodRef => + myValue.captureIn.collectFirst { case x: MethodRef => x.methodFullName }.head shouldBe "Foo.0:(1)" diff --git a/joern-cli/frontends/jimple2cpg/build.sbt b/joern-cli/frontends/jimple2cpg/build.sbt index 2ea5dbe57b93..ffcb15524ca6 100644 --- a/joern-cli/frontends/jimple2cpg/build.sbt +++ b/joern-cli/frontends/jimple2cpg/build.sbt @@ -3,7 +3,7 @@ name := "jimple2cpg" dependsOn(Projects.dataflowengineoss % "compile->compile;test->test", Projects.x2cpg % "compile->compile;test->test") libraryDependencies ++= Seq( - "io.shiftleft" %% "codepropertygraph" % Versions.cpg, + "com.michaelpollmeier" %% "codepropertygraph" % Versions.cpg, "org.soot-oss" % "soot" % Versions.soot, "org.typelevel" %% "cats-core" % Versions.catsCore, "org.scalatest" %% "scalatest" % Versions.scalatest % Test, diff --git a/joern-cli/frontends/jimple2cpg/src/main/scala/io/joern/jimple2cpg/astcreation/declarations/AstForDeclarationsCreator.scala b/joern-cli/frontends/jimple2cpg/src/main/scala/io/joern/jimple2cpg/astcreation/declarations/AstForDeclarationsCreator.scala index a2af7244d5f3..8e9cd988b683 100644 --- a/joern-cli/frontends/jimple2cpg/src/main/scala/io/joern/jimple2cpg/astcreation/declarations/AstForDeclarationsCreator.scala +++ b/joern-cli/frontends/jimple2cpg/src/main/scala/io/joern/jimple2cpg/astcreation/declarations/AstForDeclarationsCreator.scala @@ -10,7 +10,8 @@ import soot.tagkit.* import scala.collection.mutable import scala.collection.mutable.ListBuffer -import scala.jdk.CollectionConverters.CollectionHasAsScala +import scala.jdk.CollectionConverters.* + trait AstForDeclarationsCreator(implicit withSchemaValidation: ValidationMode) extends AstForTypeDeclsCreator with AstForMethodsCreator { this: AstCreator => @@ -56,7 +57,8 @@ trait AstForDeclarationsCreator(implicit withSchemaValidation: ValidationMode) val typeFullName = registerType(annotationTag.getType.parseAsJavaType) val name = typeFullName.split('.').last val elementNodes = annotationTag.getElems.asScala.map(astForAnnotationElement(_, host)).toSeq - val code = s"@$name(${elementNodes.flatMap(_.root).flatMap(_.properties.get(PropertyNames.CODE)).mkString(", ")})" + val code = + s"@$name(${elementNodes.flatMap(_.root).flatMap(_.propertiesMap.asScala.get(PropertyNames.CODE)).mkString(", ")})" val annotation = annotationNode(host, code, name, typeFullName) annotationAst(annotation, elementNodes) @@ -94,7 +96,7 @@ trait AstForDeclarationsCreator(implicit withSchemaValidation: ValidationMode) astChildren.append(annoElement match { case x: AnnotationAnnotationElem => val rhsAst = astsForAnnotations(x.getValue, parent) - codeBuilder.append(s"${rhsAst.root.flatMap(_.properties.get(PropertyNames.CODE)).mkString(", ")}") + codeBuilder.append(s"${rhsAst.root.flatMap(_.propertiesMap.asScala.get(PropertyNames.CODE)).mkString(", ")}") rhsAst case x: AnnotationArrayElem => val (rhsAst, code) = astForAnnotationArrayElement(x, parent) @@ -125,7 +127,7 @@ trait AstForDeclarationsCreator(implicit withSchemaValidation: ValidationMode) private def astForAnnotationArrayElement(x: AnnotationArrayElem, parent: AbstractHost): (Ast, String) = { val elems = x.getValues.asScala.map(astForAnnotationElement(_, parent)).toSeq - val code = s"{${elems.flatMap(_.root).flatMap(_.properties.get(PropertyNames.CODE)).mkString(", ")}}" + val code = s"{${elems.flatMap(_.root).flatMap(_.propertiesMap.asScala.get(PropertyNames.CODE)).mkString(", ")}}" val array = NewArrayInitializer().code(code).lineNumber(line(parent)).columnNumber(column(parent)) setArgumentIndices(elems) (Ast(array).withChildren(elems), code) diff --git a/joern-cli/frontends/jimple2cpg/src/main/scala/io/joern/jimple2cpg/astcreation/statements/AstForStatementsCreator.scala b/joern-cli/frontends/jimple2cpg/src/main/scala/io/joern/jimple2cpg/astcreation/statements/AstForStatementsCreator.scala index cbdad1fe6bc6..aa28167786b7 100644 --- a/joern-cli/frontends/jimple2cpg/src/main/scala/io/joern/jimple2cpg/astcreation/statements/AstForStatementsCreator.scala +++ b/joern-cli/frontends/jimple2cpg/src/main/scala/io/joern/jimple2cpg/astcreation/statements/AstForStatementsCreator.scala @@ -10,7 +10,7 @@ import soot.tagkit.Host import soot.{Unit, Value} import scala.collection.mutable -import scala.jdk.CollectionConverters.CollectionHasAsScala +import scala.jdk.CollectionConverters.* import scala.collection.mutable.ArrayBuffer trait AstForStatementsCreator(implicit withSchemaValidation: ValidationMode) { this: AstCreator => @@ -65,12 +65,12 @@ trait AstForStatementsCreator(implicit withSchemaValidation: ValidationMode) { t case x: FieldRef => Seq(astForFieldRef(x, assignStmt)) case x => astsForValue(x, assignStmt) } - val lhsCode = identifier.flatMap(_.root).flatMap(_.properties.get(PropertyNames.CODE)).mkString + val lhsCode = identifier.flatMap(_.root).flatMap(_.propertiesMap.asScala.get(PropertyNames.CODE)).mkString val initAsts = astsForValue(initializer, assignStmt) val rhsCode = initAsts .flatMap(_.root) - .map(_.properties.getOrElse(PropertyNames.CODE, "")) + .map(_.propertiesMap.asScala.getOrElse(PropertyNames.CODE, "")) .mkString(", ") val assignment = operatorNode( @@ -142,7 +142,7 @@ trait AstForStatementsCreator(implicit withSchemaValidation: ValidationMode) { t private def astsForMonitorStmt(monitorStmt: MonitorStmt): Seq[Ast] = { val opAst = astsForValue(monitorStmt.getOp, monitorStmt) - val typeString = opAst.flatMap(_.root).map(_.properties(PropertyNames.CODE)).mkString + val typeString = opAst.flatMap(_.root).map(_.propertiesMap.get(PropertyNames.CODE)).mkString val code = monitorStmt match { case _: EnterMonitorStmt => s"entermonitor $typeString" case _: ExitMonitorStmt => s"exitmonitor $typeString" @@ -181,7 +181,7 @@ trait AstForStatementsCreator(implicit withSchemaValidation: ValidationMode) { t private def astsForReturnStmt(returnStmt: ReturnStmt): Seq[Ast] = { val astChildren = astsForValue(returnStmt.getOp, returnStmt) val returnNode = NewReturn() - .code(s"return ${astChildren.flatMap(_.root).map(_.properties(PropertyNames.CODE)).mkString(" ")};") + .code(s"return ${astChildren.flatMap(_.root).map(_.propertiesMap.get(PropertyNames.CODE)).mkString(" ")};") .lineNumber(line(returnStmt)) .columnNumber(column(returnStmt)) diff --git a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/MetaDataTests.scala b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/MetaDataTests.scala index 5054b51cb5ae..1830f176ce88 100644 --- a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/MetaDataTests.scala +++ b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/MetaDataTests.scala @@ -19,8 +19,8 @@ class MetaDataTests extends JimpleCode2CpgFixture { "should not have any incoming or outgoing edges" in { cpg.metaData.size shouldBe 1 - cpg.metaData.in().l shouldBe List() - cpg.metaData.out().l shouldBe List() + cpg.metaData.in.l shouldBe List() + cpg.metaData.out.l shouldBe List() } } diff --git a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/unpacking/JarUnpackingTests.scala b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/unpacking/JarUnpackingTests.scala index b36deb4b9189..5f641af9a7b7 100644 --- a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/unpacking/JarUnpackingTests.scala +++ b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/unpacking/JarUnpackingTests.scala @@ -12,14 +12,15 @@ import org.scalatest.matchers.should.Matchers.* import org.scalatest.wordspec.AnyWordSpec import java.nio.file.{Files, Path, Paths} +import scala.compiletime.uninitialized import scala.util.{Failure, Success, Try} class JarUnpackingTests extends AnyWordSpec with Matchers with BeforeAndAfterAll { - var recurseCpgs: Map[String, Cpg] = scala.compiletime.uninitialized - var noRecurseCpgs: Map[String, Cpg] = scala.compiletime.uninitialized - var depthsCpgs: Map[String, Cpg] = scala.compiletime.uninitialized - var slippyCpg: Cpg = scala.compiletime.uninitialized + var recurseCpgs: Map[String, Cpg] = uninitialized + var noRecurseCpgs: Map[String, Cpg] = uninitialized + var depthsCpgs: Map[String, Cpg] = uninitialized + var slippyCpg: Cpg = uninitialized override protected def beforeAll(): Unit = { super.beforeAll() diff --git a/joern-cli/frontends/jssrc2cpg/build.sbt b/joern-cli/frontends/jssrc2cpg/build.sbt index f44a65b66d99..73450a20a695 100644 --- a/joern-cli/frontends/jssrc2cpg/build.sbt +++ b/joern-cli/frontends/jssrc2cpg/build.sbt @@ -18,7 +18,9 @@ lazy val astGenVersion = settingKey[String]("astgen version") astGenVersion := appProperties.value.getString("jssrc2cpg.astgen_version") libraryDependencies ++= Seq( - "io.shiftleft" %% "codepropertygraph" % Versions.cpg, + // TODO back to io.shiftleft + // "io.shiftleft" %% "codepropertygraph" % Versions.cpg, + "com.michaelpollmeier" %% "codepropertygraph" % Versions.cpg, "org.scalatest" %% "scalatest" % Versions.scalatest % Test ) diff --git a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstCreatorHelper.scala b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstCreatorHelper.scala index e4c16a4c1892..4575953dda5c 100644 --- a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstCreatorHelper.scala +++ b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstCreatorHelper.scala @@ -4,15 +4,15 @@ import io.joern.jssrc2cpg.datastructures.* import io.joern.jssrc2cpg.parser.BabelAst.* import io.joern.jssrc2cpg.parser.BabelNodeInfo import io.joern.x2cpg.frontendspecific.jssrc2cpg.Defines +import io.joern.x2cpg.utils.IntervalKeyPool import io.joern.x2cpg.utils.NodeBuilders.{newClosureBindingNode, newLocalNode} import io.joern.x2cpg.{Ast, ValidationMode} import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.codepropertygraph.generated.{EdgeTypes, EvaluationStrategies} import io.shiftleft.codepropertygraph.generated.nodes.File.PropertyDefaults -import io.shiftleft.passes.IntervalKeyPool import ujson.Value -import scala.collection.{mutable, SortedMap} +import scala.collection.{SortedMap, mutable} import scala.util.{Success, Try} trait AstCreatorHelper(implicit withSchemaValidation: ValidationMode) { this: AstCreator => diff --git a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstForTypesCreator.scala b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstForTypesCreator.scala index ccb700cf3f0d..572d7d3f9353 100644 --- a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstForTypesCreator.scala +++ b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstForTypesCreator.scala @@ -8,7 +8,7 @@ import io.joern.x2cpg.datastructures.Stack.* import io.joern.x2cpg.frontendspecific.jssrc2cpg.Defines import io.joern.x2cpg.utils.NodeBuilders.newBindingNode import io.shiftleft.codepropertygraph.generated.nodes.* -import io.shiftleft.codepropertygraph.generated.{DispatchTypes, EdgeTypes, ModifierTypes, Operators} +import io.shiftleft.codepropertygraph.generated.{DispatchTypes, EdgeTypes, ModifierTypes, Operators, PropertyNames} import ujson.Value import scala.util.Try @@ -30,7 +30,7 @@ trait AstForTypesCreator(implicit withSchemaValidation: ValidationMode) { this: } else nameTpe val astParentType = methodAstParentStack.head.label - val astParentFullName = methodAstParentStack.head.properties("FULL_NAME").toString + val astParentFullName = methodAstParentStack.head.propertiesMap.get(PropertyNames.FULL_NAME).toString val aliasTypeDeclNode = typeDeclNode(alias, aliasName, aliasFullName, parserResult.filename, alias.code, astParentType, astParentFullName) @@ -237,7 +237,7 @@ trait AstForTypesCreator(implicit withSchemaValidation: ValidationMode) { this: registerType(typeFullName) val astParentType = methodAstParentStack.head.label - val astParentFullName = methodAstParentStack.head.properties("FULL_NAME").toString + val astParentFullName = methodAstParentStack.head.propertiesMap.get(PropertyNames.FULL_NAME).toString val typeDeclNode_ = typeDeclNode( tsEnum, @@ -319,7 +319,7 @@ trait AstForTypesCreator(implicit withSchemaValidation: ValidationMode) { this: registerType(typeFullName) val astParentType = methodAstParentStack.head.label - val astParentFullName = methodAstParentStack.head.properties("FULL_NAME").toString + val astParentFullName = methodAstParentStack.head.propertiesMap.get(PropertyNames.FULL_NAME).toString val superClass = Try(createBabelNodeInfo(clazz.json("superClass")).code).toOption.toSeq val implements = Try(clazz.json("implements").arr.map(createBabelNodeInfo(_).code)).toOption.toSeq.flatten @@ -474,7 +474,7 @@ trait AstForTypesCreator(implicit withSchemaValidation: ValidationMode) { this: registerType(typeFullName) val astParentType = methodAstParentStack.head.label - val astParentFullName = methodAstParentStack.head.properties("FULL_NAME").toString + val astParentFullName = methodAstParentStack.head.propertiesMap.get(PropertyNames.FULL_NAME).toString val extendz = Try(tsInterface.json("extends").arr.map(createBabelNodeInfo(_).code)).toOption.toSeq.flatten diff --git a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstNodeBuilder.scala b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstNodeBuilder.scala index 2ef960276b6a..3218d6544c64 100644 --- a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstNodeBuilder.scala +++ b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstNodeBuilder.scala @@ -6,8 +6,7 @@ import io.joern.x2cpg.{Ast, ValidationMode} import io.joern.x2cpg.frontendspecific.jssrc2cpg.Defines import io.joern.x2cpg.utils.NodeBuilders.newMethodReturnNode import io.shiftleft.codepropertygraph.generated.nodes.* -import io.shiftleft.codepropertygraph.generated.DispatchTypes -import io.shiftleft.codepropertygraph.generated.Operators +import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators, PropertyNames} trait AstNodeBuilder(implicit withSchemaValidation: ValidationMode) { this: AstCreator => protected def createMethodReturnNode(func: BabelNodeInfo): NewMethodReturn = { @@ -251,7 +250,7 @@ trait AstNodeBuilder(implicit withSchemaValidation: ValidationMode) { this: AstC registerType(methodFullName) val astParentType = parentNode.label - val astParentFullName = parentNode.properties("FULL_NAME").toString + val astParentFullName = parentNode.propertiesMap.get(PropertyNames.FULL_NAME).toString val functionTypeDeclNode = typeDeclNode( node, diff --git a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/passes/JavaScriptTypeNodePass.scala b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/passes/JavaScriptTypeNodePass.scala index b7eb8ea08bea..032b34655a4f 100644 --- a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/passes/JavaScriptTypeNodePass.scala +++ b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/passes/JavaScriptTypeNodePass.scala @@ -4,14 +4,13 @@ import io.joern.x2cpg.frontendspecific.jssrc2cpg.Defines import io.joern.x2cpg.passes.frontend.TypeNodePass import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.semanticcpg.language.* -import io.shiftleft.passes.KeyPool import scala.collection.mutable object JavaScriptTypeNodePass { - def withRegisteredTypes(registeredTypes: List[String], cpg: Cpg, keyPool: Option[KeyPool] = None): TypeNodePass = { - new TypeNodePass(registeredTypes, cpg, keyPool, getTypesFromCpg = false) { + def withRegisteredTypes(registeredTypes: List[String], cpg: Cpg): TypeNodePass = { + new TypeNodePass(registeredTypes, cpg, getTypesFromCpg = false) { override def fullToShortName(typeName: String): String = { typeName match { diff --git a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/dataflow/DataflowTests.scala b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/dataflow/DataflowTests.scala index 0de84f2d723a..6696b70e20de 100644 --- a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/dataflow/DataflowTests.scala +++ b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/dataflow/DataflowTests.scala @@ -461,7 +461,7 @@ class DataflowTests extends DataFlowCodeToCpgSuite { cpg.call .code("bar.*") .outE(EdgeTypes.REACHING_DEF) - .count(_.inNode() == cpg.ret.head) shouldBe 1 + .count(_.dst == cpg.ret.head) shouldBe 1 } "Flow from outer params to inner params" in { diff --git a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/JsMetaDataPassTests.scala b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/JsMetaDataPassTests.scala index 399723cdc93c..bba23114b188 100644 --- a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/JsMetaDataPassTests.scala +++ b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/JsMetaDataPassTests.scala @@ -18,11 +18,11 @@ class JsMetaDataPassTests extends AnyWordSpec with Matchers with Inside { new JavaScriptMetaDataPass(cpg, "somehash", "").createAndApply() "create exactly 1 node" in { - cpg.graph.V.asScala.size shouldBe 1 + cpg.graph.allNodes.size shouldBe 1 } "create no edges" in { - cpg.graph.E.asScala.size shouldBe 0 + cpg.graph.allNodes.outE.size shouldBe 0 } "create a metadata node with correct language" in { diff --git a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/ast/MixedAstCreationPassTests.scala b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/ast/MixedAstCreationPassTests.scala index c582fa0879f3..7653f019c727 100644 --- a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/ast/MixedAstCreationPassTests.scala +++ b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/ast/MixedAstCreationPassTests.scala @@ -4,7 +4,7 @@ import io.joern.jssrc2cpg.testfixtures.AstJsSrc2CpgSuite import io.shiftleft.codepropertygraph.generated.DispatchTypes import io.shiftleft.codepropertygraph.generated.EvaluationStrategies import io.shiftleft.codepropertygraph.generated.Operators -import io.shiftleft.codepropertygraph.generated.nodes.MethodParameterIn +import io.shiftleft.codepropertygraph.generated.nodes.{ClosureBinding, MethodParameterIn} import io.shiftleft.semanticcpg.language.* class MixedAstCreationPassTests extends AstJsSrc2CpgSuite { @@ -285,7 +285,7 @@ class MixedAstCreationPassTests extends AstJsSrc2CpgSuite { val List(fooLocalY) = fooBlock.astChildren.isLocal.nameExact("y").l val List(barRef) = fooBlock.astChildren.isCall.astChildren.isMethodRef.l - val List(closureBindForY, closureBindForX) = barRef.captureOut.l + val List(closureBindForY, closureBindForX) = barRef.captureOut.cast[ClosureBinding].l closureBindForX.closureOriginalName shouldBe Option("x") closureBindForX.closureBindingId shouldBe Option("Test0.js::program:foo:bar:x") diff --git a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/ast/SimpleAstCreationPassTests.scala b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/ast/SimpleAstCreationPassTests.scala index f1bab7a456f8..cd85aac864df 100644 --- a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/ast/SimpleAstCreationPassTests.scala +++ b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/ast/SimpleAstCreationPassTests.scala @@ -859,7 +859,7 @@ class SimpleAstCreationPassTests extends AstJsSrc2CpgSuite { val List(typeDecl) = cpg.typeDecl.nameExact("method").l typeDecl.fullName should endWith("Test0.js::program:method") - val List(binding) = typeDecl.bindsOut.l + val List(binding) = typeDecl.bindsOut.cast[Binding].l binding.name shouldBe "" binding.signature shouldBe "" diff --git a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/ast/TsClassesAstCreationPassTests.scala b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/ast/TsClassesAstCreationPassTests.scala index fed85c602569..907a9ac24b56 100644 --- a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/ast/TsClassesAstCreationPassTests.scala +++ b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/ast/TsClassesAstCreationPassTests.scala @@ -339,7 +339,7 @@ class TsClassesAstCreationPassTests extends AstJsSrc2CpgSuite(".ts") { val List(credentialsParam) = cpg.parameter.nameExact("credentials").l credentialsParam.typeFullName shouldBe "Test0.ts::program:Test:run:0" // should not produce dangling nodes that are meant to be inside procedures - cpg.all.collectAll[CfgNode].whereNot(_._astIn).size shouldBe 0 + cpg.all.collectAll[CfgNode].whereNot(_.astParent).size shouldBe 0 cpg.identifier.count(_.refsTo.size > 1) shouldBe 0 cpg.identifier.whereNot(_.refsTo).size shouldBe 0 // should not produce assignment calls directly under typedecls @@ -359,7 +359,7 @@ class TsClassesAstCreationPassTests extends AstJsSrc2CpgSuite(".ts") { val List(credentialsParam) = cpg.parameter.nameExact("param1_0").l credentialsParam.typeFullName shouldBe "Test0.ts::program:apiCall:0" // should not produce dangling nodes that are meant to be inside procedures - cpg.all.collectAll[CfgNode].whereNot(_._astIn).size shouldBe 0 + cpg.all.collectAll[CfgNode].whereNot(_.astParent).size shouldBe 0 cpg.identifier.count(_.refsTo.size > 1) shouldBe 0 cpg.identifier.whereNot(_.refsTo).size shouldBe 0 // should not produce assignment calls directly under typedecls diff --git a/joern-cli/frontends/kotlin2cpg/build.sbt b/joern-cli/frontends/kotlin2cpg/build.sbt index 83032b7c6024..7384c6ed7b22 100644 --- a/joern-cli/frontends/kotlin2cpg/build.sbt +++ b/joern-cli/frontends/kotlin2cpg/build.sbt @@ -12,7 +12,7 @@ libraryDependencies ++= Seq( "com.lihaoyi" %% "requests" % Versions.requests, "com.lihaoyi" %% "ujson" % Versions.upickle, "com.squareup.tools.build" % "maven-archeologist" % Versions.mavenArcheologist, - "io.shiftleft" %% "codepropertygraph" % Versions.cpg, + "com.michaelpollmeier" %% "codepropertygraph" % Versions.cpg, "org.gradle" % "gradle-tooling-api" % Versions.gradleTooling, "org.jetbrains.kotlin" % "kotlin-stdlib-jdk8" % kotlinVersion, "org.jetbrains.kotlin" % "kotlin-stdlib" % kotlinVersion, diff --git a/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/ast/AstCreator.scala b/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/ast/AstCreator.scala index 9b03f2aa8180..9d7b9af63d36 100644 --- a/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/ast/AstCreator.scala +++ b/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/ast/AstCreator.scala @@ -1,5 +1,6 @@ package io.joern.kotlin2cpg.ast +import flatgraph.DiffGraphBuilder import io.joern.kotlin2cpg.Constants import io.joern.kotlin2cpg.KtFileWithMeta import io.joern.kotlin2cpg.datastructures.Scope @@ -13,11 +14,11 @@ import io.joern.x2cpg.Defines import io.joern.x2cpg.ValidationMode import io.joern.x2cpg.datastructures.Global import io.joern.x2cpg.datastructures.Stack.* +import io.joern.x2cpg.utils.IntervalKeyPool import io.joern.x2cpg.utils.NodeBuilders import io.joern.x2cpg.utils.NodeBuilders.newMethodReturnNode import io.shiftleft.codepropertygraph.generated.* import io.shiftleft.codepropertygraph.generated.nodes.* -import io.shiftleft.passes.IntervalKeyPool import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.types.structure.NamespaceTraversal import org.jetbrains.kotlin.com.intellij.psi.PsiElement diff --git a/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/ast/AstForDeclarationsCreator.scala b/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/ast/AstForDeclarationsCreator.scala index b9474e5b8415..ea2e9c9b32d9 100644 --- a/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/ast/AstForDeclarationsCreator.scala +++ b/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/ast/AstForDeclarationsCreator.scala @@ -425,7 +425,7 @@ trait AstForDeclarationsCreator(implicit withSchemaValidation: ValidationMode) { implicit typeInfoProvider: TypeInfoProvider ): Seq[Ast] = { ctors.map { ctor => - val primaryCtorCallAst = List(Ast(primaryCtorCall.copy)) + val primaryCtorCallAst = List(Ast(primaryCtorCall.copy())) val constructorParams = ctor.getValueParameters.asScala.toList val defaultSignature = typeInfoProvider.anySignature(constructorParams) val defaultFullName = s"$classFullName.${TypeConstants.initPrefix}:$defaultSignature" diff --git a/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/ast/AstForFunctionsCreator.scala b/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/ast/AstForFunctionsCreator.scala index 45ceb2aa6850..b7c60d2947cb 100644 --- a/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/ast/AstForFunctionsCreator.scala +++ b/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/ast/AstForFunctionsCreator.scala @@ -40,8 +40,8 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { filename: String ): Ast = { val astParentType = parentNode.label - val astParentName = parentNode.properties(TypeDecl.Properties.Name.name).toString - val astParentFullName = parentNode.properties(TypeDecl.Properties.FullName.name).toString + val astParentName = parentNode.propertiesMap.get(TypeDecl.PropertyNames.Name).toString + val astParentFullName = parentNode.propertiesMap.get(TypeDecl.PropertyNames.FullName).toString val functionTypeDeclNode = typeDeclNode( node, diff --git a/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/ast/AstForStatementsCreator.scala b/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/ast/AstForStatementsCreator.scala index 65ce497f60a9..f083d75d63af 100644 --- a/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/ast/AstForStatementsCreator.scala +++ b/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/ast/AstForStatementsCreator.scala @@ -62,7 +62,7 @@ trait AstForStatementsCreator(implicit withSchemaValidation: ValidationMode) { // private def astForForWithDestructuringLHS(expr: KtForExpression)(implicit typeInfoProvider: TypeInfoProvider): Ast = { val loopRangeText = expr.getLoopRange.getText - val iteratorName = s"${Constants.iteratorPrefix}${iteratorKeyPool.next()}" + val iteratorName = s"${Constants.iteratorPrefix}${iteratorKeyPool.next}" val localForIterator = localNode(expr, iteratorName, iteratorName, TypeConstants.any) val iteratorAssignmentLhs = newIdentifierNode(iteratorName, TypeConstants.any) val iteratorLocalAst = Ast(localForIterator).withRefEdge(iteratorAssignmentLhs, localForIterator) @@ -182,7 +182,7 @@ trait AstForStatementsCreator(implicit withSchemaValidation: ValidationMode) { // private def astForForWithSimpleVarLHS(expr: KtForExpression)(implicit typeInfoProvider: TypeInfoProvider): Ast = { val loopRangeText = expr.getLoopRange.getText - val iteratorName = s"${Constants.iteratorPrefix}${iteratorKeyPool.next()}" + val iteratorName = s"${Constants.iteratorPrefix}${iteratorKeyPool.next}" val iteratorLocal = localNode(expr, iteratorName, iteratorName, TypeConstants.any) val iteratorAssignmentLhs = newIdentifierNode(iteratorName, TypeConstants.any) val iteratorLocalAst = Ast(iteratorLocal).withRefEdge(iteratorAssignmentLhs, iteratorLocal) diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ComplexExpressionsTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ComplexExpressionsTests.scala index ff8abf0fe9af..b469e6f84ca1 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ComplexExpressionsTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ComplexExpressionsTests.scala @@ -4,7 +4,6 @@ import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture import io.shiftleft.codepropertygraph.generated.Operators import io.shiftleft.codepropertygraph.generated.edges.Argument import io.shiftleft.semanticcpg.language.* -import overflowdb.traversal.jIteratortoTraversal class ComplexExpressionsTests extends KotlinCode2CpgFixture(withOssDataflow = false) { "CPG for code with _and_/_or_ operator and try-catch as one of the arguments" should { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/LambdaTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/LambdaTests.scala index 6de2838c017e..2b87334b7e77 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/LambdaTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/LambdaTests.scala @@ -17,7 +17,7 @@ import io.shiftleft.codepropertygraph.generated.nodes.MethodRef import io.shiftleft.codepropertygraph.generated.nodes.Return import io.shiftleft.codepropertygraph.generated.nodes.TypeDecl import io.shiftleft.semanticcpg.language.* -import overflowdb.traversal.jIteratortoTraversal +import io.joern.x2cpg.Defines class LambdaTests extends KotlinCode2CpgFixture(withOssDataflow = false, withDefaultJars = true) { "CPG for code with a simple lambda which captures a method parameter" should { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/MethodTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/MethodTests.scala index 73eff0a85c5b..f6c472a5b0a9 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/MethodTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/MethodTests.scala @@ -174,7 +174,7 @@ class MethodTests extends KotlinCode2CpgFixture(withOssDataflow = false) { |""".stripMargin) "pass the lambda to a `sortedWith` call which is then under the method `sorted`" in { - inside(cpg.methodRef(".*.*").inCall.l) { + inside(cpg.methodRefWithName(".*.*").inCall.l) { case sortedWith :: Nil => sortedWith.name shouldBe "sortedWith" sortedWith.method.name shouldBe "sorted" diff --git a/joern-cli/frontends/php2cpg/build.sbt b/joern-cli/frontends/php2cpg/build.sbt index 203ef91a60e1..37259fd05e91 100644 --- a/joern-cli/frontends/php2cpg/build.sbt +++ b/joern-cli/frontends/php2cpg/build.sbt @@ -15,7 +15,7 @@ dependsOn(Projects.dataflowengineoss % "compile->compile;test->test", Projects.x libraryDependencies ++= Seq( "com.lihaoyi" %% "upickle" % Versions.upickle, "com.lihaoyi" %% "ujson" % Versions.upickle, - "io.shiftleft" %% "codepropertygraph" % Versions.cpg, + "com.michaelpollmeier" %% "codepropertygraph" % Versions.cpg, "com.github.sh4869" %% "semver-parser-scala" % Versions.semverParser, "org.scalatest" %% "scalatest" % Versions.scalatest % Test ) diff --git a/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/astcreation/AstCreator.scala b/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/astcreation/AstCreator.scala index 278433b62ad5..ac8be55f138a 100644 --- a/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/astcreation/AstCreator.scala +++ b/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/astcreation/AstCreator.scala @@ -1,5 +1,6 @@ package io.joern.php2cpg.astcreation +import flatgraph.DiffGraphBuilder import io.joern.php2cpg.astcreation.AstCreator.{NameConstants, TypeConstants, operatorSymbols} import io.joern.php2cpg.datastructures.ArrayIndexTracker import io.joern.php2cpg.parser.Domain.* @@ -8,12 +9,13 @@ import io.joern.php2cpg.utils.Scope import io.joern.x2cpg.Ast.storeInDiffGraph import io.joern.x2cpg.Defines.{StaticInitMethodName, UnresolvedNamespace, UnresolvedSignature} import io.joern.x2cpg.utils.AstPropertiesUtil.RootProperties +import io.joern.x2cpg.utils.IntervalKeyPool import io.joern.x2cpg.utils.NodeBuilders.* import io.joern.x2cpg.{Ast, AstCreatorBase, AstNodeBuilder, ValidationMode} import io.shiftleft.codepropertygraph.generated.* import io.shiftleft.codepropertygraph.generated.nodes.* -import io.shiftleft.passes.IntervalKeyPool import io.shiftleft.semanticcpg.language.types.structure.NamespaceTraversal +import scala.jdk.CollectionConverters.* import org.slf4j.LoggerFactory import java.nio.charset.StandardCharsets @@ -584,7 +586,7 @@ class AstCreator(filename: String, phpAst: PhpFile, fileContent: Option[String], val iteratorAssignAst = simpleAssignAst(Ast(iterIdentifier), iterValue, line(stmt)) // - Assigned item assign - val itemInitAst = getItemAssignAstForForeach(stmt, assignItemTargetAst, iterIdentifier.copy) + val itemInitAst = getItemAssignAstForForeach(stmt, assignItemTargetAst, iterIdentifier.copy()) // Condition ast val isNullName = PhpOperators.isNull @@ -597,7 +599,7 @@ class AstCreator(filename: String, phpAst: PhpFile, fileContent: Option[String], val conditionAst = callAst(notIsNull, isNullAst :: Nil) // Update asts - val nextIterIdent = Ast(iterIdentifier.copy) + val nextIterIdent = Ast(iterIdentifier.copy()) val nextSignature = "void()" val nextCallCode = s"${nextIterIdent.rootCodeOrEmpty}->next()" val nextCallNode = callNode( @@ -1411,8 +1413,9 @@ class AstCreator(filename: String, phpAst: PhpFile, fileContent: Option[String], case PhpVariable(PhpNameExpr(name, _), _) => val typeFullName = scope .lookupVariable(name) - .flatMap(_.properties.get(PropertyNames.TYPE_FULL_NAME).map(_.toString)) + .map(node => Option(node.propertiesMap.get(PropertyNames.TYPE_FULL_NAME))) .getOrElse(TypeConstants.Any) + .toString val byRefPrefix = if (closureUse.byRef) "&" else "" Some(localNode(closureExpr, name, s"$byRefPrefix$$$name", typeFullName)) @@ -1525,11 +1528,11 @@ class AstCreator(filename: String, phpAst: PhpFile, fileContent: Option[String], Some(initSignature), Some(TypeConstants.Any) ) - val initReceiver = Ast(tmpIdentifier.copy) + val initReceiver = Ast(tmpIdentifier.copy()) val initCallAst = callAst(initCallNode, initArgs, base = Option(initReceiver)) // Return identifier - val returnIdentifierAst = Ast(tmpIdentifier.copy) + val returnIdentifierAst = Ast(tmpIdentifier.copy()) Ast(blockNode(expr, "", TypeConstants.Any)) .withChild(allocAssignAst) @@ -1676,7 +1679,7 @@ class AstCreator(filename: String, phpAst: PhpFile, fileContent: Option[String], case nameExpr: PhpNameExpr => scope .lookupVariable(nameExpr.name) - .flatMap(_.properties.get(PropertyNames.TYPE_FULL_NAME).map(_.toString)) + .flatMap(_.propertiesMap.asScala.get(PropertyNames.TYPE_FULL_NAME).map(_.toString)) .getOrElse(nameExpr.name) case expr => diff --git a/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/passes/AnyTypePass.scala b/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/passes/AnyTypePass.scala index 601228aa26cf..c65478137eb0 100644 --- a/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/passes/AnyTypePass.scala +++ b/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/passes/AnyTypePass.scala @@ -13,7 +13,7 @@ import io.shiftleft.semanticcpg.language.* class AnyTypePass(cpg: Cpg) extends ForkJoinParallelCpgPass[AstNode](cpg) { override def generateParts(): Array[AstNode] = { - cpg.has(PropertyNames.TYPE_FULL_NAME, PropertyDefaults.TypeFullName).collectAll[AstNode].toArray + cpg.graph.nodesWithProperty(PropertyNames.TYPE_FULL_NAME, PropertyDefaults.TypeFullName).collectAll[AstNode].toArray } override def runOnPart(diffGraph: DiffGraphBuilder, node: AstNode): Unit = { diff --git a/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/passes/AstParentInfoPass.scala b/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/passes/AstParentInfoPass.scala index 602474493d7c..701b7523d033 100644 --- a/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/passes/AstParentInfoPass.scala +++ b/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/passes/AstParentInfoPass.scala @@ -1,7 +1,6 @@ package io.joern.php2cpg.passes -import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.codepropertygraph.generated.PropertyNames +import io.shiftleft.codepropertygraph.generated.{Cpg, Properties, PropertyNames} import io.shiftleft.codepropertygraph.generated.nodes.{AstNode, NamespaceBlock, Method, TypeDecl} import io.shiftleft.passes.ForkJoinParallelCpgPass import io.shiftleft.semanticcpg.language.* @@ -15,7 +14,7 @@ class AstParentInfoPass(cpg: Cpg) extends ForkJoinParallelCpgPass[AstNode](cpg) override def runOnPart(diffGraph: DiffGraphBuilder, node: AstNode): Unit = { findParent(node).foreach { parentNode => val astParentType = parentNode.label - val astParentFullName = parentNode.property(PropertyNames.FULL_NAME) + val astParentFullName = parentNode.property(Properties.FullName) diffGraph.setNodeProperty(node, PropertyNames.AST_PARENT_TYPE, astParentType) diffGraph.setNodeProperty(node, PropertyNames.AST_PARENT_FULL_NAME, astParentFullName) diff --git a/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/passes/ClosureRefPass.scala b/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/passes/ClosureRefPass.scala index 62f6ecdbfd2c..295e5d06f0a5 100644 --- a/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/passes/ClosureRefPass.scala +++ b/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/passes/ClosureRefPass.scala @@ -22,7 +22,7 @@ class ClosureRefPass(cpg: Cpg) extends ForkJoinParallelCpgPass[ClosureBinding](c * that is the scope in which the closure would have originally been created. */ override def runOnPart(diffGraph: DiffGraphBuilder, closureBinding: ClosureBinding): Unit = { - closureBinding.captureIn.collectAll[MethodRef].toList match { + closureBinding._captureIn.collectAll[MethodRef].toList match { case Nil => logger.error(s"No MethodRef corresponding to closureBinding ${closureBinding.closureBindingId}") diff --git a/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/passes/LocalCreationPass.scala b/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/passes/LocalCreationPass.scala index 2ae92127f214..a0e8475978dd 100644 --- a/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/passes/LocalCreationPass.scala +++ b/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/passes/LocalCreationPass.scala @@ -1,7 +1,7 @@ package io.joern.php2cpg.passes -import io.shiftleft.passes.ForkJoinParallelCpgPass import io.shiftleft.codepropertygraph.generated.Cpg +import io.shiftleft.passes.ForkJoinParallelCpgPass import io.shiftleft.codepropertygraph.generated.EdgeTypes import io.shiftleft.codepropertygraph.generated.nodes.{ AstNode, @@ -96,7 +96,7 @@ abstract class LocalCreationPass[ScopeType <: AstNode](cpg: Cpg) ): Unit = { val identifierMap = getIdentifiersInScope(bodyNode) - .filter(_.refOut.isEmpty) + .filter(_._refOut.isEmpty) .filterNot(excludeIdentifierFn) .groupBy(_.name) diff --git a/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/passes/PhpTypeRecoveryPassTests.scala b/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/passes/PhpTypeRecoveryPassTests.scala index d5d19ab7c5bd..0384627d1110 100644 --- a/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/passes/PhpTypeRecoveryPassTests.scala +++ b/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/passes/PhpTypeRecoveryPassTests.scala @@ -525,7 +525,7 @@ class PhpTypeRecoveryPassTests extends PhpCode2CpgFixture() { "propagate this QueryBuilder type to the identifier assigned to the inherited call for the wrapped `createQueryBuilder`" in { cpg.method .nameExact("findSomething") - ._containsOut + .containsOut .collectAll[Identifier] .nameExact("queryBuilder") .typeFullName diff --git a/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/OperatorTests.scala b/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/OperatorTests.scala index 8eee50343241..73891f94bb9a 100644 --- a/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/OperatorTests.scala +++ b/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/OperatorTests.scala @@ -4,9 +4,9 @@ import io.joern.php2cpg.astcreation.AstCreator.TypeConstants import io.joern.php2cpg.parser.Domain.{PhpDomainTypeConstants, PhpOperators} import io.joern.php2cpg.testfixtures.PhpCode2CpgFixture import io.joern.x2cpg.Defines +import io.joern.x2cpg.utils.IntervalKeyPool import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators} import io.shiftleft.codepropertygraph.generated.nodes.{Block, Call, Identifier, Literal, TypeRef} -import io.shiftleft.passes.IntervalKeyPool import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.types.structure.NamespaceTraversal diff --git a/joern-cli/frontends/pysrc2cpg/build.sbt b/joern-cli/frontends/pysrc2cpg/build.sbt index 8e42aabbf429..0abd6a8ebfc2 100644 --- a/joern-cli/frontends/pysrc2cpg/build.sbt +++ b/joern-cli/frontends/pysrc2cpg/build.sbt @@ -3,7 +3,7 @@ name := "pysrc2cpg" dependsOn(Projects.dataflowengineoss % "compile->compile;test->test", Projects.x2cpg % "compile->compile;test->test") libraryDependencies ++= Seq( - "io.shiftleft" %% "codepropertygraph" % Versions.cpg, + "com.michaelpollmeier" %% "codepropertygraph" % Versions.cpg, "org.scala-lang.modules" %% "scala-parallel-collections" % Versions.scalaParallel, "org.scalatest" %% "scalatest" % Versions.scalatest % Test ) diff --git a/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/Py2Cpg.scala b/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/Py2Cpg.scala index 6171292d3347..6b2ddff1f43c 100644 --- a/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/Py2Cpg.scala +++ b/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/Py2Cpg.scala @@ -4,7 +4,7 @@ import io.joern.x2cpg.ValidationMode import io.joern.x2cpg.frontendspecific.pysrc2cpg.Constants import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.Languages -import overflowdb.BatchedUpdate +import flatgraph.DiffGraphApplier import io.shiftleft.codepropertygraph.generated.DiffGraphBuilder object Py2Cpg { @@ -45,7 +45,7 @@ class Py2Cpg( val anyTypeDecl = nodeBuilder.typeDeclNode(Constants.ANY, Constants.ANY, "N/A", Nil, LineAndColumn(1, 1, 1, 1, 1, 1)) edgeBuilder.astEdge(anyTypeDecl, globalNamespaceBlock, 0) - BatchedUpdate.applyDiff(outputCpg.graph, diffGraph) + DiffGraphApplier.applyDiff(outputCpg.graph, diffGraph) new CodeToCpg(outputCpg, inputProviders, schemaValidationMode, enableFileContent).createAndApply() new ConfigFileCreationPass(outputCpg, requirementsTxt).createAndApply() new DependenciesFromRequirementsTxtPass(outputCpg).createAndApply() diff --git a/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/PythonAstVisitor.scala b/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/PythonAstVisitor.scala index fe5053bf0b29..a881248fe608 100644 --- a/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/PythonAstVisitor.scala +++ b/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/PythonAstVisitor.scala @@ -8,6 +8,7 @@ import io.joern.x2cpg.frontendspecific.pysrc2cpg.Constants import io.joern.x2cpg.{AstCreatorBase, ValidationMode} import io.shiftleft.codepropertygraph.generated.* import io.shiftleft.codepropertygraph.generated.nodes.{NewCall, NewIdentifier, NewNode, NewTypeDecl} +import flatgraph.DiffGraphBuilder import org.slf4j.LoggerFactory import io.shiftleft.codepropertygraph.generated.DiffGraphBuilder diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/CallCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/CallCpgTests.scala index f1dfcad4f710..503becf26f54 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/CallCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/CallCpgTests.scala @@ -3,7 +3,6 @@ package io.joern.pysrc2cpg.cpg import io.joern.pysrc2cpg.PySrc2CpgFixture import io.shiftleft.codepropertygraph.generated.DispatchTypes import io.shiftleft.semanticcpg.language.* -import overflowdb.traversal.NodeOps import java.io.File diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/FunctionDefCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/FunctionDefCpgTests.scala index 8c526ec859bb..c818c5c53431 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/FunctionDefCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/FunctionDefCpgTests.scala @@ -55,12 +55,12 @@ class FunctionDefCpgTests extends AnyFreeSpec with Matchers { } "test function method ref" in { - cpg.methodRef("func").referencedMethod.fullName.head shouldBe + cpg.methodRefWithName("func").referencedMethod.fullName.head shouldBe "test.py:.func" } "test assignment of method ref to local variable" in { - val assignNode = cpg.methodRef("func").astParent.isCall.head + val assignNode = cpg.methodRefWithName("func").astParent.isCall.head assignNode.code shouldBe "func = def func(...)" } @@ -132,7 +132,7 @@ class FunctionDefCpgTests extends AnyFreeSpec with Matchers { |""".stripMargin) "test decorator wrapping of method reference" in { - val (staticMethod: Call) :: Nil = cpg.methodRef("func").astParent.l: @unchecked + val (staticMethod: Call) :: Nil = cpg.methodRefWithName("func").astParent.l: @unchecked staticMethod.code shouldBe "staticmethod(def func(...))" staticMethod.name shouldBe "staticmethod" val (abc: Call) :: Nil = staticMethod.start.astParent.l: @unchecked diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/VariableReferencingCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/VariableReferencingCpgTests.scala index 987baa0a4ee9..bcd4dc12a3c7 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/VariableReferencingCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/VariableReferencingCpgTests.scala @@ -149,7 +149,7 @@ class VariableReferencingCpgTests extends AnyFreeSpec with Matchers { } "test method reference closure binding" in { - val methodRefNode = cpg.methodRef("f").head + val methodRefNode = cpg.methodRefWithName("f").head val closureBinding = methodRefNode._closureBindingViaCaptureOut.next() closureBinding.closureBindingId shouldBe Some("test.py:.f:x") closureBinding.evaluationStrategy shouldBe EvaluationStrategies.BY_REFERENCE @@ -185,7 +185,7 @@ class VariableReferencingCpgTests extends AnyFreeSpec with Matchers { } "test method reference closure binding" in { - val methodRefNode = cpg.methodRef("f").head + val methodRefNode = cpg.methodRefWithName("f").head val closureBinding = methodRefNode._closureBindingViaCaptureOut.next() closureBinding.closureBindingId shouldBe Some("test.py:.f:x") closureBinding.evaluationStrategy shouldBe EvaluationStrategies.BY_REFERENCE @@ -223,7 +223,7 @@ class VariableReferencingCpgTests extends AnyFreeSpec with Matchers { } "test method reference closure binding of f in g" in { - val methodRefNode = cpg.methodRef("f").head + val methodRefNode = cpg.methodRefWithName("f").head val closureBinding = methodRefNode._closureBindingViaCaptureOut.next() closureBinding.closureBindingId shouldBe Some("test.py:.g.f:x") closureBinding.evaluationStrategy shouldBe EvaluationStrategies.BY_REFERENCE @@ -240,7 +240,7 @@ class VariableReferencingCpgTests extends AnyFreeSpec with Matchers { } "test method reference closure binding of g in module" in { - val methodRefNode = cpg.methodRef("g").head + val methodRefNode = cpg.methodRefWithName("g").head val closureBinding = methodRefNode._closureBindingViaCaptureOut.next() closureBinding.closureBindingId shouldBe Some("test.py:.g:x") closureBinding.evaluationStrategy shouldBe EvaluationStrategies.BY_REFERENCE diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/dataflow/DataFlowTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/dataflow/DataFlowTests.scala index 034b5183c0b7..6a7bf52adfff 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/dataflow/DataFlowTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/dataflow/DataFlowTests.scala @@ -969,8 +969,8 @@ class RegexDefinedFlowsDataFlowTests |print(Foo.func()) |""".stripMargin) "be found" in { - val src = cpg.call.code("Foo.func").l - val snk = cpg.call("print").l + val src = cpg.call.code("Foo.func") + val snk = cpg.call("print") snk.argument.reachableByFlows(src).size shouldBe 1 } } diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/passes/TypeRecoveryPassTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/passes/TypeRecoveryPassTests.scala index 8ae43b3361ba..c074f654d6fd 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/passes/TypeRecoveryPassTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/passes/TypeRecoveryPassTests.scala @@ -1051,7 +1051,7 @@ class TypeRecoveryPassTests extends PySrc2CpgFixture(withOssDataflow = false) { "assert the method properties in RedisDB, especially quoted type hints" in { val Some(redisDB) = cpg.typeDecl.nameExact("RedisDB").method.nameExact("").headOption: @unchecked - val List(instanceM, getRedisM, setM) = redisDB.astOut.isMethod.nameExact("instance", "get_redis", "set").l + val List(instanceM, getRedisM, setM) = redisDB.astChildren.isMethod.nameExact("instance", "get_redis", "set").l instanceM.methodReturn.typeFullName shouldBe Seq("db", "redis.py:.RedisDB").mkString(File.separator) getRedisM.methodReturn.typeFullName shouldBe "aioredis.py:.Redis" @@ -1324,12 +1324,9 @@ class TypeRecoveryPassTests extends PySrc2CpgFixture(withOssDataflow = false) { val variables = cpg.moduleVariables .where(_.typeFullName(".*FastAPI.*")) .l - val appIncludeRouterCalls = - variables.invokingCalls - .nameExact("include_router") - .l - val includedRouters = appIncludeRouterCalls.argument.argumentIndexGte(1).moduleVariables.l - val definitionsOfRouters = includedRouters.definitions.whereNot(_.source.isCall.nameExact("import")).l + val appIncludeRouterCalls = variables.invokingCalls.nameExact("include_router") + val includedRouters = appIncludeRouterCalls.argument.argumentIndexGte(1).moduleVariables.l + val definitionsOfRouters = includedRouters.definitions.whereNot(_.source.isCall.nameExact("import")).l val List(adminRouter, normalRouter, itemsRouter) = definitionsOfRouters.map(x => (x.code, x.method.fullName)).sortBy(_._1).l: @unchecked diff --git a/joern-cli/frontends/rubysrc2cpg/build.sbt b/joern-cli/frontends/rubysrc2cpg/build.sbt index 9f2372e82bb6..aef98ef7e181 100644 --- a/joern-cli/frontends/rubysrc2cpg/build.sbt +++ b/joern-cli/frontends/rubysrc2cpg/build.sbt @@ -15,7 +15,7 @@ lazy val joernTypeStubsVersion = settingKey[String]("joern_type_stub version") joernTypeStubsVersion := appProperties.value.getString("rubysrc2cpg.joern_type_stubs_version") libraryDependencies ++= Seq( - "io.shiftleft" %% "codepropertygraph" % Versions.cpg, + "com.michaelpollmeier" %% "codepropertygraph" % Versions.cpg, "org.apache.commons" % "commons-compress" % Versions.commonsCompress, // For unpacking Gems with `--download-dependencies` "org.scalatest" %% "scalatest" % Versions.scalatest % Test, "org.antlr" % "antlr4-runtime" % Versions.antlr @@ -65,4 +65,4 @@ joernTypeStubsDlTask := { Compile / compile := ((Compile / compile) dependsOn joernTypeStubsDlTask).value Universal / packageName := name.value -Universal / topLevelDirectory := None \ No newline at end of file +Universal / topLevelDirectory := None diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala index d52317dfb0f1..d6407f42eb32 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala @@ -1,5 +1,6 @@ package io.joern.rubysrc2cpg.astcreation +import flatgraph.DiffGraphBuilder import io.joern.rubysrc2cpg.astcreation.RubyIntermediateAst.* import io.joern.rubysrc2cpg.datastructures.{BlockScope, NamespaceScope, RubyProgramSummary, RubyScope, RubyStubbedType} import io.joern.rubysrc2cpg.parser.{RubyNodeCreator, RubyParser} @@ -43,7 +44,7 @@ class AstCreator( .map(_.stripPrefix(java.io.File.separator)) .getOrElse(fileName) - private def internalLineAndColNum: Option[Integer] = Option(1) + private def internalLineAndColNum: Option[Int] = Option(1) /** The relative file name, in a unix path delimited format. */ diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForExpressionsCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForExpressionsCreator.scala index 027e0fc3f877..1091e143f054 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForExpressionsCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForExpressionsCreator.scala @@ -161,7 +161,7 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { decl.dynamicTypeHintFullName.headOption case _ => astForExpression(baseNode).nodes - .flatMap(_.properties.get(PropertyNames.TYPE_FULL_NAME).map(_.toString)) + .flatMap(node => Option(node.propertiesMap.get(PropertyNames.TYPE_FULL_NAME)).map(_.toString)) .filterNot(_ == XDefines.Any) .headOption } diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForStatementsCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForStatementsCreator.scala index 68504e6cd561..ea611e67e034 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForStatementsCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForStatementsCreator.scala @@ -223,7 +223,7 @@ trait AstForStatementsCreator(implicit withSchemaValidation: ValidationMode) { t // Set span contents methodAstsWithRefs.flatMap(_.nodes).foreach { - case m: NewMethodRef => DummyNode(m.copy)(block.span.spanStart(m.code)) + case m: NewMethodRef => DummyNode(m.copy())(block.span.spanStart(m.code)) case _ => } diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForTypesCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForTypesCreator.scala index 583855ed0472..12b9b9c60328 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForTypesCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForTypesCreator.scala @@ -70,7 +70,8 @@ trait AstForTypesCreator(implicit withSchemaValidation: ValidationMode) { this: To signify the singleton type, we add the tag. */ - val singletonTypeDecl = typeDecl.copy + val singletonTypeDecl = typeDecl + .copy() .name(s"$className") .fullName(s"$classFullName") .inheritsFromTypeFullName(inheritsFrom.map(x => s"$x")) diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstSummaryVisitor.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstSummaryVisitor.scala index a1aceea6f561..fbbbe156af99 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstSummaryVisitor.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstSummaryVisitor.scala @@ -1,6 +1,6 @@ package io.joern.rubysrc2cpg.astcreation -import better.files.File +import flatgraph.DiffGraphApplier import io.joern.rubysrc2cpg.astcreation.RubyIntermediateAst.StatementList import io.joern.rubysrc2cpg.datastructures.{RubyField, RubyMethod, RubyProgramSummary, RubyStubbedType, RubyType} import io.joern.rubysrc2cpg.parser.RubyNodeCreator @@ -12,7 +12,6 @@ import io.shiftleft.codepropertygraph.cpgloading.CpgLoader import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.{Local, Member, Method, TypeDecl} import io.shiftleft.semanticcpg.language.* -import overflowdb.{BatchedUpdate, Config} import java.io.File as JavaFile import java.util.regex.Matcher @@ -28,8 +27,8 @@ trait AstSummaryVisitor(implicit withSchemaValidation: ValidationMode) { this: A val rootNode = new RubyNodeCreator().visit(programCtx).asInstanceOf[StatementList] val ast = astForRubyFile(rootNode) Ast.storeInDiffGraph(ast, diffGraph) - BatchedUpdate.applyDiff(cpg.graph, diffGraph) - CpgLoader.createIndexes(cpg) + DiffGraphApplier.applyDiff(cpg.graph, diffGraph) + // Link basic AST elements AstLinkerPass(cpg).createAndApply() // Summarize findings diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/astcreation/AstCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/astcreation/AstCreator.scala index f987a89f3ae9..e77ab5beb937 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/astcreation/AstCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/astcreation/AstCreator.scala @@ -1,5 +1,6 @@ package io.joern.rubysrc2cpg.deprecated.astcreation +import flatgraph.DiffGraphBuilder import io.joern.rubysrc2cpg.deprecated.parser.DeprecatedRubyParser import io.joern.rubysrc2cpg.deprecated.parser.DeprecatedRubyParser.* import io.joern.rubysrc2cpg.deprecated.passes.Defines diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/astcreation/RubyScope.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/astcreation/RubyScope.scala index 1788679262a1..e62639097f96 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/astcreation/RubyScope.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/astcreation/RubyScope.scala @@ -1,5 +1,6 @@ package io.joern.rubysrc2cpg.deprecated.astcreation +import flatgraph.DiffGraphBuilder import io.joern.x2cpg.datastructures.Scope import io.shiftleft.codepropertygraph.generated.{DiffGraphBuilder, EdgeTypes} import io.shiftleft.codepropertygraph.generated.nodes.{DeclarationNew, NewIdentifier, NewLocal, NewNode} diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/AstCreationPass.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/AstCreationPass.scala index 234a844522df..8e43529dc25e 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/AstCreationPass.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/AstCreationPass.scala @@ -1,5 +1,6 @@ package io.joern.rubysrc2cpg.passes +import flatgraph.DiffGraphApplier import io.joern.rubysrc2cpg.astcreation.AstCreator import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.NodeTypes @@ -7,7 +8,6 @@ import io.shiftleft.codepropertygraph.generated.nodes.NewTypeDecl import io.shiftleft.passes.ForkJoinParallelCpgPass import io.shiftleft.semanticcpg.language.types.structure.NamespaceTraversal import org.slf4j.LoggerFactory -import overflowdb.BatchedUpdate class AstCreationPass(cpg: Cpg, astCreators: List[AstCreator]) extends ForkJoinParallelCpgPass[AstCreator](cpg) { @@ -32,7 +32,7 @@ class AstCreationPass(cpg: Cpg, astCreators: List[AstCreator]) extends ForkJoinP .astParentFullName(NamespaceTraversal.globalNamespaceName) .isExternal(true) diffGraph.addNode(emptyType).addNode(anyType) - BatchedUpdate.applyDiff(cpg.graph, diffGraph) + DiffGraphApplier.applyDiff(cpg.graph, diffGraph) } override def runOnPart(diffGraph: DiffGraphBuilder, astCreator: AstCreator): Unit = { diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/RubyTypeRecoveryPassGenerator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/RubyTypeRecoveryPassGenerator.scala index 696eb5882074..ad0d2131cea5 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/RubyTypeRecoveryPassGenerator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/RubyTypeRecoveryPassGenerator.scala @@ -6,7 +6,7 @@ import io.joern.x2cpg.passes.frontend.XTypeRecovery.AllNodeTypesFromNodeExt import io.shiftleft.codepropertygraph.generated.{Cpg, Operators, PropertyNames} import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.semanticcpg.language.operatorextension.OpNodes.FieldAccess -import io.shiftleft.semanticcpg.language.{types, *} +import io.shiftleft.semanticcpg.language.* import io.shiftleft.codepropertygraph.generated.DiffGraphBuilder class RubyTypeRecoveryPassGenerator(cpg: Cpg, config: XTypeRecoveryConfig = XTypeRecoveryConfig()) diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/dataflow/SingleAssignmentTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/dataflow/SingleAssignmentTests.scala index 7f20321567c8..c9ace03bcd28 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/dataflow/SingleAssignmentTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/dataflow/SingleAssignmentTests.scala @@ -15,13 +15,13 @@ class SingleAssignmentTests extends RubyCode2CpgFixture(withPostProcessing = tru |""".stripMargin) val source = cpg.literal.l val sink = cpg.method.name("puts").callIn.argument.l - val flows = sink.reachableByFlows(source).map(flowToResultPairs).distinct.sortBy(_.length).l - val List(flow1, flow2, flow3, flow4, flow5) = flows - flow1 shouldBe List(("y = 1", 2), ("puts y", 3)) - flow2 shouldBe List(("y = 1", 2), ("x = y = 1", 2), ("puts x", 4)) - flow3 shouldBe List(("y = 1", 2), ("puts y", 3), ("puts x", 4)) - flow4 shouldBe List(("y = 1", 2), ("x = y = 1", 2), ("z = x = y = 1", 2), ("puts z", 5)) - flow5 shouldBe List(("y = 1", 2), ("x = y = 1", 2), ("puts x", 4), ("puts z", 5)) + val flows = sink.reachableByFlows(source).map(flowToResultPairs).distinct.l + flows.size shouldBe 5 + flows should contain(List(("y = 1", 2), ("puts y", 3))) + flows should contain(List(("y = 1", 2), ("x = y = 1", 2), ("puts x", 4))) + flows should contain(List(("y = 1", 2), ("puts y", 3), ("puts x", 4))) + flows should contain(List(("y = 1", 2), ("x = y = 1", 2), ("z = x = y = 1", 2), ("puts z", 5))) + flows should contain(List(("y = 1", 2), ("x = y = 1", 2), ("puts x", 4), ("puts z", 5))) } "flow through expressions" in { diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/MetaDataPassTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/MetaDataPassTests.scala index eeeb970917c1..56f28ffdd521 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/MetaDataPassTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/MetaDataPassTests.scala @@ -13,8 +13,10 @@ class MetaDataPassTests extends AnyWordSpec with Matchers { "create a metadata node with correct language" in { File.usingTemporaryDirectory("rubysrc2cpgTest") { dir => - val config = Config().withInputPath(dir.pathAsString).withOutputPath(dir.pathAsString) - val cpg = new RubySrc2Cpg().createCpg(config).get + val config = Config() + .withInputPath(dir.createChild("dummyinputfile").pathAsString) + .withOutputPath(dir.createChild("dummyoutputfile").pathAsString) + val cpg = new RubySrc2Cpg().createCpg(config).get cpg.metaData.language.l shouldBe List(Languages.RUBYSRC) } } diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/CallCpgTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/CallCpgTests.scala index b2c499a6cd93..1b698ae9798c 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/CallCpgTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/CallCpgTests.scala @@ -29,7 +29,7 @@ class CallCpgTests extends RubyCode2CpgFixture(withPostProcessing = true, useDep "test astChildren" taggedAs SameInNewFrontend in { val callNode = cpg.call.name("foo").head - val children = callNode.astChildren + val children = callNode.astChildren.l children.size shouldBe 2 val firstChild = children.head @@ -62,7 +62,7 @@ class CallCpgTests extends RubyCode2CpgFixture(withPostProcessing = true, useDep "test astChildren" in { val callNode = cpg.call.name("foo").head - val children = callNode.astChildren + val children = callNode.astChildren.l children.size shouldBe 3 val firstChild = children.head diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/MethodTwoTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/MethodTwoTests.scala index 4037cebde09f..e10824107855 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/MethodTwoTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/MethodTwoTests.scala @@ -57,8 +57,8 @@ class MethodTwoTests extends RubyCode2CpgFixture(useDeprecatedFrontend = true) { // TODO: Need to be fixed "test function method ref" ignore { - cpg.methodRef("foo").referencedMethod.fullName.l should not be empty - cpg.methodRef("foo").referencedMethod.fullName.head shouldBe + cpg.methodRefWithName("foo").referencedMethod.fullName.l should not be empty + cpg.methodRefWithName("foo").referencedMethod.fullName.head shouldBe "Test0.rb::program:foo" } diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/DoBlockTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/DoBlockTests.scala index 7089d11825b3..0d7e1ec0ae06 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/DoBlockTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/DoBlockTests.scala @@ -232,7 +232,7 @@ class DoBlockTests extends RubyCode2CpgFixture { case None => fail("Expected closure binding refer to the captured local") } - inside(myValue._captureIn.l) { + inside(myValue.captureIn.l) { case (x: MethodRef) :: Nil => x.methodFullName shouldBe "Test0.rb:::program:0" case xs => fail(s"Expected single method ref binding but got [${xs.mkString(",")}]") } diff --git a/joern-cli/frontends/swiftsrc2cpg/build.sbt b/joern-cli/frontends/swiftsrc2cpg/build.sbt index e9c4c39045b9..c8fe586483a4 100644 --- a/joern-cli/frontends/swiftsrc2cpg/build.sbt +++ b/joern-cli/frontends/swiftsrc2cpg/build.sbt @@ -17,7 +17,7 @@ lazy val astGenVersion = settingKey[String]("astgen version") astGenVersion := appProperties.value.getString("swiftsrc2cpg.astgen_version") libraryDependencies ++= Seq( - "io.shiftleft" %% "codepropertygraph" % Versions.cpg, + "com.michaelpollmeier" %% "codepropertygraph" % Versions.cpg, "com.lihaoyi" %% "upickle" % Versions.upickle, "org.scalatest" %% "scalatest" % Versions.scalatest % Test ) diff --git a/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/astcreation/AstCreator.scala b/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/astcreation/AstCreator.scala index 8b7045bc0353..af843e9d2568 100644 --- a/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/astcreation/AstCreator.scala +++ b/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/astcreation/AstCreator.scala @@ -121,10 +121,10 @@ class AstCreator(val config: Config, val global: Global, val parserResult: Parse case null => notHandledYet(node) } - override protected def line(node: SwiftNode): Option[Int] = node.startLine.map(Integer.valueOf) - override protected def column(node: SwiftNode): Option[Int] = node.startColumn.map(Integer.valueOf) - override protected def lineEnd(node: SwiftNode): Option[Int] = node.endLine.map(Integer.valueOf) - override protected def columnEnd(node: SwiftNode): Option[Int] = node.endColumn.map(Integer.valueOf) + override protected def line(node: SwiftNode): Option[Int] = node.startLine + override protected def column(node: SwiftNode): Option[Int] = node.startColumn + override protected def lineEnd(node: SwiftNode): Option[Int] = node.endLine + override protected def columnEnd(node: SwiftNode): Option[Int] = node.endColumn private val lineOffsetTable = OffsetUtils.getLineOffsetTable(Option(parserResult.fileContent)) diff --git a/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/astcreation/AstCreatorHelper.scala b/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/astcreation/AstCreatorHelper.scala index 20b47f96f7bb..f9fe22915aef 100644 --- a/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/astcreation/AstCreatorHelper.scala +++ b/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/astcreation/AstCreatorHelper.scala @@ -10,6 +10,7 @@ import io.joern.swiftsrc2cpg.parser.SwiftNodeSyntax.GuardStmtSyntax import io.joern.swiftsrc2cpg.parser.SwiftNodeSyntax.InitializerDeclSyntax import io.joern.swiftsrc2cpg.parser.SwiftNodeSyntax.SwiftNode import io.joern.x2cpg.frontendspecific.swiftsrc2cpg.Defines +import io.joern.x2cpg.utils.IntervalKeyPool import io.joern.x2cpg.{Ast, ValidationMode} import io.joern.x2cpg.utils.NodeBuilders.{newClosureBindingNode, newLocalNode} import io.shiftleft.codepropertygraph.generated.nodes.NewNode @@ -17,8 +18,6 @@ import io.shiftleft.codepropertygraph.generated.{EdgeTypes, EvaluationStrategies import io.shiftleft.codepropertygraph.generated.nodes.NewNamespaceBlock import io.shiftleft.codepropertygraph.generated.nodes.NewTypeDecl import io.shiftleft.codepropertygraph.generated.ControlStructureTypes -import io.shiftleft.codepropertygraph.generated.PropertyNames -import io.shiftleft.passes.IntervalKeyPool import scala.collection.mutable @@ -95,7 +94,7 @@ trait AstCreatorHelper(implicit withSchemaValidation: ValidationMode) { this: As protected def astParentInfo(): (String, String) = { val astParentType = methodAstParentStack.head.label - val astParentFullName = methodAstParentStack.head.properties(PropertyNames.FULL_NAME).toString + val astParentFullName = methodAstParentStack.head.propertiesMap.get("FULL_NAME").toString (astParentType, astParentFullName) } diff --git a/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/passes/SwiftTypeNodePass.scala b/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/passes/SwiftTypeNodePass.scala index 75a5d0348985..1e78013d2b25 100644 --- a/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/passes/SwiftTypeNodePass.scala +++ b/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/passes/SwiftTypeNodePass.scala @@ -3,15 +3,14 @@ package io.joern.swiftsrc2cpg.passes import io.shiftleft.codepropertygraph.generated.Cpg import io.joern.x2cpg.passes.frontend.TypeNodePass import io.shiftleft.semanticcpg.language.* -import io.shiftleft.passes.KeyPool import io.shiftleft.semanticcpg.language.types.structure.NamespaceTraversal import scala.collection.mutable object SwiftTypeNodePass { - def withRegisteredTypes(registeredTypes: List[String], cpg: Cpg, keyPool: Option[KeyPool] = None): TypeNodePass = { - new TypeNodePass(registeredTypes, cpg, keyPool, getTypesFromCpg = false) { + def withRegisteredTypes(registeredTypes: List[String], cpg: Cpg): TypeNodePass = { + new TypeNodePass(registeredTypes, cpg, getTypesFromCpg = false) { override def fullToShortName(typeName: String): String = { typeName match { diff --git a/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/dataflow/DataFlowTests.scala b/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/dataflow/DataFlowTests.scala index e7de93beec41..15ac7441b461 100644 --- a/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/dataflow/DataFlowTests.scala +++ b/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/dataflow/DataFlowTests.scala @@ -8,7 +8,6 @@ import io.shiftleft.codepropertygraph.generated.EdgeTypes import io.shiftleft.codepropertygraph.generated.nodes.Identifier import io.shiftleft.codepropertygraph.generated.nodes.Literal import io.shiftleft.semanticcpg.language.* -import overflowdb.traversal.toNodeTraversal class DataFlowTests extends DataFlowCodeToCpgSuite { @@ -871,7 +870,7 @@ class DataFlowTests extends DataFlowCodeToCpgSuite { cpg .call("bar") .outE(EdgeTypes.REACHING_DEF) - .count(_.inNode() == cpg.ret.head) shouldBe 1 + .count(_.dst == cpg.ret.head) shouldBe 1 } } diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/Ast.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/Ast.scala index f60c6b2f931b..da02b01315aa 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/Ast.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/Ast.scala @@ -5,7 +5,7 @@ import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.codepropertygraph.generated.nodes.AstNode.PropertyDefaults import org.slf4j.LoggerFactory import io.shiftleft.codepropertygraph.generated.DiffGraphBuilder -import overflowdb.SchemaViolationException +import flatgraph.SchemaViolationException case class AstEdge(src: NewNode, dst: NewNode) @@ -58,7 +58,7 @@ object Ast { !(src.isValidOutNeighbor(edge, dst) && dst.isValidInNeighbor(edge, src)) ) { throw new SchemaViolationException( - s"Malformed AST detected: (${src.label()}) -[$edge]-> (${dst.label()}) violates the schema." + s"Malformed AST detected: (${src.label}) -[$edge]-> (${dst.label}) violates the schema." ) } @@ -222,9 +222,10 @@ case class Ast( * copy. */ def subTreeCopy(node: AstNodeNew, argIndex: Int = -1, replacementNode: Option[AstNodeNew] = None): Ast = { - val newNode = replacementNode match + val newNode = replacementNode match { case Some(n) => n - case None => node.copy + case None => node.copy() + } if (argIndex != -1) { // newNode.order = argIndex newNode match { diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/AstCreatorBase.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/AstCreatorBase.scala index 1614e3633b97..f907c052d841 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/AstCreatorBase.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/AstCreatorBase.scala @@ -1,11 +1,12 @@ package io.joern.x2cpg +import flatgraph.DiffGraphBuilder import io.joern.x2cpg.passes.frontend.MetaDataPass +import io.joern.x2cpg.utils.IntervalKeyPool import io.joern.x2cpg.utils.NodeBuilders.newMethodReturnNode import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.codepropertygraph.generated.{ControlStructureTypes, ModifierTypes} -import io.shiftleft.passes.IntervalKeyPool import io.shiftleft.semanticcpg.language.types.structure.NamespaceTraversal import io.shiftleft.codepropertygraph.generated.DiffGraphBuilder diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/X2Cpg.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/X2Cpg.scala index 3aa87ba2ef31..73ec53c17190 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/X2Cpg.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/X2Cpg.scala @@ -6,7 +6,6 @@ import io.joern.x2cpg.layers.{Base, CallGraph, ControlFlow, TypeRelations} import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.semanticcpg.layers.{LayerCreator, LayerCreatorContext} import org.slf4j.LoggerFactory -import overflowdb.Config import scopt.OParser import java.io.PrintWriter @@ -178,7 +177,7 @@ trait X2CpgFrontend[T <: X2CpgConfig[?]] { withErrorsToConsole(config) { _ => createCpg(config) match { case Success(cpg) => - cpg.close() + cpg.close() // persists to disk Success(cpg) case Failure(exception) => Failure(exception) @@ -300,19 +299,16 @@ object X2Cpg { /** Create an empty CPG, backed by the file at `optionalOutputPath` or in-memory if `optionalOutputPath` is empty. */ def newEmptyCpg(optionalOutputPath: Option[String] = None): Cpg = { - val odbConfig = optionalOutputPath - .map { outputPath => - val outFile = File(outputPath) + optionalOutputPath match { + case Some(outputPath) => + lazy val outFile = File(outputPath) if (outputPath != "" && outFile.exists) { logger.info("Output file exists, removing: " + outputPath) outFile.delete() } - Config.withDefaults.withStorageLocation(outputPath) - } - .getOrElse { - Config.withDefaults() - } - Cpg.withConfig(odbConfig) + Cpg.withStorage(outFile.path) + case None => Cpg.empty + } } /** Apply function `applyPasses` to a newly created CPG. The CPG is wrapped in a `Try` and returned. On failure, the diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/jssrc2cpg/JavaScriptTypeRecovery.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/jssrc2cpg/JavaScriptTypeRecovery.scala index 4872f080a718..2d9c8072f1f2 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/jssrc2cpg/JavaScriptTypeRecovery.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/jssrc2cpg/JavaScriptTypeRecovery.scala @@ -5,7 +5,7 @@ import io.joern.x2cpg.Defines.ConstructorMethodName import io.joern.x2cpg.passes.frontend.* import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.* -import io.shiftleft.codepropertygraph.generated.{Operators, PropertyNames} +import io.shiftleft.codepropertygraph.generated.{Operators, Properties, PropertyNames} import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.operatorextension.OpNodes.FieldAccess import io.shiftleft.codepropertygraph.generated.DiffGraphBuilder @@ -48,9 +48,9 @@ private class RecoverForJavaScriptFile(cpg: Cpg, cu: File, builder: DiffGraphBui override protected def prepopulateSymbolTableEntry(x: AstNode): Unit = x match { case x @ (_: Identifier | _: Local | _: MethodParameterIn) - if x.property(PropertyNames.TYPE_FULL_NAME, Defines.Any) != Defines.Any => - val typeFullName = x.property(PropertyNames.TYPE_FULL_NAME, Defines.Any) - val typeHints = symbolTable.get(LocalVar(x.property(PropertyNames.TYPE_FULL_NAME, Defines.Any))) - typeFullName + if x.propertyOption(Properties.TypeFullName).getOrElse(Defines.Any) != Defines.Any => + val typeFullName = x.propertyOption(Properties.TypeFullName).getOrElse(Defines.Any) + val typeHints = symbolTable.get(LocalVar(typeFullName)) - typeFullName lazy val cpgTypeFullName = cpg.typeDecl.nameExact(typeFullName).fullName.toSet val resolvedTypeHints = if (typeHints.nonEmpty) symbolTable.put(x, typeHints) @@ -59,9 +59,8 @@ private class RecoverForJavaScriptFile(cpg: Cpg, cu: File, builder: DiffGraphBui if (!resolvedTypeHints.contains(typeFullName) && resolvedTypeHints.sizeIs == 1) builder.setNodeProperty(x, PropertyNames.TYPE_FULL_NAME, resolvedTypeHints.head) - case x @ (_: Identifier | _: Local | _: MethodParameterIn) - if x.property(PropertyNames.POSSIBLE_TYPES, Seq.empty[String]).nonEmpty => - val possibleTypes = x.property(PropertyNames.POSSIBLE_TYPES, Seq.empty[String]) + case x @ (_: Identifier | _: Local | _: MethodParameterIn) if x.property(Properties.PossibleTypes).nonEmpty => + val possibleTypes = x.property(Properties.PossibleTypes) if (possibleTypes.sizeIs == 1 && !possibleTypes.contains("ANY")) { val typeFullName = possibleTypes.head val typeHints = symbolTable.get(LocalVar(typeFullName)) - typeFullName diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/php2cpg/PhpTypeRecovery.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/php2cpg/PhpTypeRecovery.scala index 39fff713b025..6e00f48b0f0e 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/php2cpg/PhpTypeRecovery.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/php2cpg/PhpTypeRecovery.scala @@ -1,5 +1,6 @@ package io.joern.x2cpg.frontendspecific.php2cpg +import flatgraph.DiffGraphBuilder import io.joern.x2cpg.Defines import io.joern.x2cpg.passes.frontend.* import io.joern.x2cpg.passes.frontend.XTypeRecovery.AllNodeTypesFromNodeExt @@ -150,7 +151,7 @@ private class RecoverForPhpFile(cpg: Cpg, cu: NamespaceBlock, builder: DiffGraph symbolTable.append(head, callees) case _ => Set.empty } - val returnTypes = extractTypes(ret.argumentOut.l) + val returnTypes = extractTypes(ret.argumentOut.cast[CfgNode].l) existingTypes.addAll(returnTypes) /* Check whether method return is already known, and if so, remove dummy value */ @@ -221,7 +222,7 @@ private class RecoverForPhpFile(cpg: Cpg, cu: NamespaceBlock, builder: DiffGraph .getOrElse(XTypeRecovery.DummyIndexAccess) else x.name - val collectionVar = Option(c.argumentOut.l match { + val collectionVar = Option(c.argumentOut.cast[CfgNode].l match { case List(i: Identifier, idx: Literal) => CollectionVar(i.name, idx.code) case List(i: Identifier, idx: Identifier) => CollectionVar(i.name, idx.code) case List(c: Call, idx: Call) => CollectionVar(callName(c), callName(idx)) diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/pysrc2cpg/DynamicTypeHintFullNamePass.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/pysrc2cpg/DynamicTypeHintFullNamePass.scala index 2e70aea7e38f..6f297f52e5d4 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/pysrc2cpg/DynamicTypeHintFullNamePass.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/pysrc2cpg/DynamicTypeHintFullNamePass.scala @@ -1,5 +1,6 @@ package io.joern.x2cpg.frontendspecific.pysrc2cpg +import flatgraph.DiffGraphBuilder import io.joern.x2cpg.passes.frontend.ImportStringHandling import io.shiftleft.codepropertygraph.generated.{Cpg, PropertyNames} import io.shiftleft.codepropertygraph.generated.nodes.{CfgNode, MethodParameterIn, MethodReturn, StoredNode} diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/swiftsrc2cpg/SwiftTypeRecovery.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/swiftsrc2cpg/SwiftTypeRecovery.scala index 04bd681c0ca7..50f1f2bc8fcc 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/swiftsrc2cpg/SwiftTypeRecovery.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/swiftsrc2cpg/SwiftTypeRecovery.scala @@ -5,7 +5,7 @@ import io.joern.x2cpg.Defines.ConstructorMethodName import io.joern.x2cpg.passes.frontend.* import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.* -import io.shiftleft.codepropertygraph.generated.{Operators, PropertyNames} +import io.shiftleft.codepropertygraph.generated.{Operators, Properties, PropertyNames} import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.operatorextension.OpNodes.FieldAccess import io.shiftleft.codepropertygraph.generated.DiffGraphBuilder @@ -47,9 +47,9 @@ private class RecoverForSwiftFile(cpg: Cpg, cu: File, builder: DiffGraphBuilder, override protected def prepopulateSymbolTableEntry(x: AstNode): Unit = x match { case x @ (_: Identifier | _: Local | _: MethodParameterIn) - if x.property(PropertyNames.TYPE_FULL_NAME, Defines.Any) != Defines.Any => - val typeFullName = x.property(PropertyNames.TYPE_FULL_NAME, Defines.Any) - val typeHints = symbolTable.get(LocalVar(x.property(PropertyNames.TYPE_FULL_NAME, Defines.Any))) - typeFullName + if x.propertyOption(Properties.TypeFullName).getOrElse(Defines.Any) != Defines.Any => + val typeFullName = x.propertyOption(Properties.TypeFullName).getOrElse(Defines.Any) + val typeHints = symbolTable.get(LocalVar(typeFullName)) - typeFullName lazy val cpgTypeFullName = cpg.typeDecl.nameExact(typeFullName).fullName.toSet val resolvedTypeHints = if (typeHints.nonEmpty) symbolTable.put(x, typeHints) diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/layers/Base.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/layers/Base.scala index 8e59eb053b69..f53e6f2b348e 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/layers/Base.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/layers/Base.scala @@ -32,7 +32,6 @@ class Base extends LayerCreator { override def create(context: LayerCreatorContext): Unit = { val cpg = context.cpg - cpg.graph.indexManager.createNodePropertyIndex(PropertyNames.FULL_NAME) Base.passes(cpg).zipWithIndex.foreach { case (pass, index) => runPass(pass, context, index) } diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/ContainsEdgePass.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/ContainsEdgePass.scala index e3d5145b7d8f..53d16ff44297 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/ContainsEdgePass.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/ContainsEdgePass.scala @@ -4,6 +4,7 @@ import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.codepropertygraph.generated.{EdgeTypes, NodeTypes} import io.shiftleft.passes.ForkJoinParallelCpgPass +import io.shiftleft.semanticcpg.language.* import scala.collection.mutable import scala.jdk.CollectionConverters.* @@ -15,7 +16,7 @@ class ContainsEdgePass(cpg: Cpg) extends ForkJoinParallelCpgPass[AstNode](cpg) { import ContainsEdgePass._ override def generateParts(): Array[AstNode] = - cpg.graph.nodes(sourceTypes*).asScala.map(_.asInstanceOf[AstNode]).toArray + cpg.graph.nodes(sourceTypes*).cast[AstNode].toArray override def runOnPart(dstGraph: DiffGraphBuilder, source: AstNode): Unit = { // AST is assumed to be a tree. If it contains cycles, then this will give a nice endless loop with OOM diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/FileCreationPass.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/FileCreationPass.scala index a045ef5971e3..b5bef4d0c258 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/FileCreationPass.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/FileCreationPass.scala @@ -1,9 +1,8 @@ package io.joern.x2cpg.passes.base import io.joern.x2cpg.utils.LinkingUtil -import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.codepropertygraph.generated.nodes.{NewFile, StoredNode} -import io.shiftleft.codepropertygraph.generated.{EdgeTypes, NodeTypes, PropertyNames} +import io.shiftleft.codepropertygraph.generated.nodes.{File, NewFile, StoredNode} +import io.shiftleft.codepropertygraph.generated.{Cpg, EdgeTypes, NodeTypes, PropertyNames} import io.shiftleft.passes.CpgPass import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.types.structure.FileTraversal @@ -25,7 +24,7 @@ class FileCreationPass(cpg: Cpg) extends CpgPass(cpg) with LinkingUtil { } def createFileIfDoesNotExist(srcNode: StoredNode, destFullName: String): Unit = { - if (destFullName != srcNode.propertyDefaultValue(PropertyNames.FILENAME)) { + if (destFullName != File.PropertyDefaults.Name) { val dstFullName = if (destFullName == "") { FileTraversal.UNKNOWN } else { destFullName } val newFile = newFileNameToNode.getOrElseUpdate( @@ -42,7 +41,7 @@ class FileCreationPass(cpg: Cpg) extends CpgPass(cpg) with LinkingUtil { // Create SOURCE_FILE edges from nodes of various types to FILE linkToSingle( cpg, - srcNodes = cpg.graph.nodes(srcLabels*).toList, + srcNodes = cpg.graph.nodes(srcLabels*).cast[StoredNode].toList, srcLabels = srcLabels, dstNodeLabel = NodeTypes.FILE, edgeType = EdgeTypes.SOURCE_FILE, @@ -50,6 +49,7 @@ class FileCreationPass(cpg: Cpg) extends CpgPass(cpg) with LinkingUtil { originalFileNameToNode.get(x) }, dstFullNameKey = PropertyNames.FILENAME, + dstDefaultPropertyValue = File.PropertyDefaults.Name, dstGraph, Some(createFileIfDoesNotExist) ) diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/TypeEvalPass.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/TypeEvalPass.scala index 00547bf2142d..b51a934c0e98 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/TypeEvalPass.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/TypeEvalPass.scala @@ -1,14 +1,12 @@ package io.joern.x2cpg.passes.base import io.joern.x2cpg.utils.LinkingUtil -import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.codepropertygraph.generated.{EdgeTypes, NodeTypes, PropertyNames} +import io.shiftleft.codepropertygraph.generated.{Cpg, EdgeTypes, NodeTypes, PropertyNames} +import io.shiftleft.codepropertygraph.generated.nodes.{Local, StoredNode} import io.shiftleft.passes.ForkJoinParallelCpgPass -import overflowdb.Node -import overflowdb.traversal.* - -class TypeEvalPass(cpg: Cpg) extends ForkJoinParallelCpgPass[List[Node]](cpg) with LinkingUtil { +import io.shiftleft.semanticcpg.language.* +class TypeEvalPass(cpg: Cpg) extends ForkJoinParallelCpgPass[List[StoredNode]](cpg) with LinkingUtil { private val srcLabels = List( NodeTypes.METHOD_PARAMETER_IN, NodeTypes.METHOD_PARAMETER_OUT, @@ -24,11 +22,11 @@ class TypeEvalPass(cpg: Cpg) extends ForkJoinParallelCpgPass[List[Node]](cpg) wi NodeTypes.UNKNOWN ) - def generateParts(): Array[List[Node]] = { - cpg.graph.nodes(srcLabels*).toList.grouped(MAX_BATCH_SIZE).toArray + def generateParts(): Array[List[StoredNode]] = { + cpg.graph.nodes(srcLabels*).cast[StoredNode].toList.grouped(MAX_BATCH_SIZE).toArray } - def runOnPart(builder: DiffGraphBuilder, part: List[overflowdb.Node]): Unit = { + def runOnPart(builder: DiffGraphBuilder, part: List[StoredNode]): Unit = { linkToSingle( cpg = cpg, srcNodes = part, @@ -37,6 +35,7 @@ class TypeEvalPass(cpg: Cpg) extends ForkJoinParallelCpgPass[List[Node]](cpg) wi edgeType = EdgeTypes.EVAL_TYPE, dstNodeMap = typeFullNameToNode(cpg, _), dstFullNameKey = PropertyNames.TYPE_FULL_NAME, + dstDefaultPropertyValue = Local.PropertyDefaults.TypeFullName, dstGraph = builder, None ) diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/TypeRefPass.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/TypeRefPass.scala index 65dbae189c28..d851b16e201d 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/TypeRefPass.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/TypeRefPass.scala @@ -1,21 +1,19 @@ package io.joern.x2cpg.passes.base import io.joern.x2cpg.utils.LinkingUtil -import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.codepropertygraph.generated.{EdgeTypes, NodeTypes, PropertyNames} +import io.shiftleft.codepropertygraph.generated.{Cpg, EdgeTypes, NodeTypes, PropertyNames} +import io.shiftleft.codepropertygraph.generated.nodes.{Type, StoredNode} import io.shiftleft.passes.ForkJoinParallelCpgPass -import overflowdb.Node -import overflowdb.traversal.* - -class TypeRefPass(cpg: Cpg) extends ForkJoinParallelCpgPass[List[Node]](cpg) with LinkingUtil { +import io.shiftleft.semanticcpg.language.* +class TypeRefPass(cpg: Cpg) extends ForkJoinParallelCpgPass[List[StoredNode]](cpg) with LinkingUtil { private val srcLabels = List(NodeTypes.TYPE) - def generateParts(): Array[List[Node]] = { - cpg.graph.nodes(srcLabels*).toList.grouped(MAX_BATCH_SIZE).toArray + def generateParts(): Array[List[StoredNode]] = { + cpg.graph.nodes(srcLabels*).cast[StoredNode].toList.grouped(MAX_BATCH_SIZE).toArray } - def runOnPart(builder: DiffGraphBuilder, part: List[overflowdb.Node]): Unit = { + def runOnPart(builder: DiffGraphBuilder, part: List[StoredNode]): Unit = { linkToSingle( cpg = cpg, srcNodes = part, @@ -24,6 +22,7 @@ class TypeRefPass(cpg: Cpg) extends ForkJoinParallelCpgPass[List[Node]](cpg) wit edgeType = EdgeTypes.REF, dstNodeMap = typeDeclFullNameToNode(cpg, _), dstFullNameKey = PropertyNames.TYPE_DECL_FULL_NAME, + dstDefaultPropertyValue = Type.PropertyDefaults.TypeDeclFullName, dstGraph = builder, None ) diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/callgraph/DynamicCallLinker.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/callgraph/DynamicCallLinker.scala index dc631ba8d5ed..ed3496e7e94c 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/callgraph/DynamicCallLinker.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/callgraph/DynamicCallLinker.scala @@ -2,12 +2,11 @@ package io.joern.x2cpg.passes.callgraph import io.joern.x2cpg.Defines.DynamicCallUnknownFullName import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.codepropertygraph.generated.nodes.{Call, Method, TypeDecl} +import io.shiftleft.codepropertygraph.generated.nodes.{Call, Method, StoredNode, Type, TypeDecl} import io.shiftleft.codepropertygraph.generated.{DispatchTypes, EdgeTypes, PropertyNames} import io.shiftleft.passes.CpgPass import io.shiftleft.semanticcpg.language.* import org.slf4j.{Logger, LoggerFactory} -import overflowdb.{NodeDb, NodeRef} import scala.collection.mutable import scala.jdk.CollectionConverters.* @@ -56,10 +55,10 @@ class DynamicCallLinker(cpg: Cpg) extends CpgPass(cpg) { initMaps() // ValidM maps class C and method name N to the set of // func ptrs implementing N for C and its subclasses - for ( - typeDecl <- cpg.typeDecl; + for { + typeDecl <- cpg.typeDecl method <- typeDecl._methodViaAstOut - ) { + } { val methodName = method.fullName val candidates = allSubclasses(typeDecl.fullName).flatMap { staticLookup(_, method) } validM.put(methodName, candidates) @@ -114,8 +113,8 @@ class DynamicCallLinker(cpg: Cpg) extends CpgPass(cpg) { if (visitedNodes.contains(cur)) return visitedNodes visitedNodes.addOne(cur) - (if (inSuperDirection) cpg.typeDecl.fullNameExact(cur.fullName).flatMap(_.inheritsFromOut.referencedTypeDecl) - else cpg.typ.fullNameExact(cur.fullName).flatMap(_.inheritsFromIn)) + (if (inSuperDirection) cpg.typeDecl.fullNameExact(cur.fullName)._typeViaInheritsFromOut.referencedTypeDecl + else cpg.typ.fullNameExact(cur.fullName).inheritsFromIn) .collectAll[TypeDecl] .to(mutable.LinkedHashSet) match { case classesToEval if classesToEval.isEmpty => visitedNodes @@ -174,16 +173,8 @@ class DynamicCallLinker(cpg: Cpg) extends CpgPass(cpg) { validM.get(call.methodFullName) match { case Some(tgts) => - val callsOut = call.callOut.fullName.toSetImmutable - val tgtMs = tgts - .flatMap(destMethod => - if (cpg.graph.indexManager.isIndexed(PropertyNames.FULL_NAME)) { - methodFullNameToNode(destMethod) - } else { - cpg.method.fullNameExact(destMethod).headOption - } - ) - .toSet + val callsOut = call._callOut.cast[Method].fullName.toSetImmutable + val tgtMs = tgts.flatMap(destMethod => methodFullNameToNode(destMethod)).toSet // Non-overridden methods linked as external stubs should be excluded if they are detected val (externalMs, internalMs) = tgtMs.partition(_.isExternal) (if (externalMs.nonEmpty && internalMs.nonEmpty) internalMs else tgtMs) @@ -209,8 +200,8 @@ class DynamicCallLinker(cpg: Cpg) extends CpgPass(cpg) { } } - private def nodesWithFullName(x: String): Iterable[NodeRef[? <: NodeDb]] = - cpg.graph.indexManager.lookup(PropertyNames.FULL_NAME, x).asScala + private def nodesWithFullName(x: String): Iterator[StoredNode] = + cpg.graph.nodesWithProperty(PropertyNames.FULL_NAME, x).cast[StoredNode] private def methodFullNameToNode(x: String): Option[Method] = nodesWithFullName(x).collectFirst { case x: Method => x } diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/callgraph/MethodRefLinker.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/callgraph/MethodRefLinker.scala index 86174f9a872d..e0411f8dead4 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/callgraph/MethodRefLinker.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/callgraph/MethodRefLinker.scala @@ -1,28 +1,27 @@ package io.joern.x2cpg.passes.callgraph import io.joern.x2cpg.utils.LinkingUtil -import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.* +import io.shiftleft.codepropertygraph.generated.nodes.Method import io.shiftleft.passes.CpgPass -import overflowdb.traversal.* +import io.shiftleft.semanticcpg.language.* /** This pass has MethodStubCreator and TypeDeclStubCreator as prerequisite for language frontends which do not provide * method stubs and type decl stubs. */ class MethodRefLinker(cpg: Cpg) extends CpgPass(cpg) with LinkingUtil { - private val srcLabels = List(NodeTypes.METHOD_REF) - override def run(dstGraph: DiffGraphBuilder): Unit = { // Create REF edges from METHOD_REFs to METHOD linkToSingle( cpg, - srcNodes = cpg.graph.nodes(srcLabels*).toList, + srcNodes = cpg.methodRef.l, srcLabels = List(NodeTypes.METHOD_REF), dstNodeLabel = NodeTypes.METHOD, edgeType = EdgeTypes.REF, dstNodeMap = methodFullNameToNode(cpg, _), dstFullNameKey = PropertyNames.METHOD_FULL_NAME, + dstDefaultPropertyValue = Method.PropertyDefaults.FullName, dstGraph, None ) diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/callgraph/NaiveCallLinker.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/callgraph/NaiveCallLinker.scala index 9dc8aade4d15..f9f1332af728 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/callgraph/NaiveCallLinker.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/callgraph/NaiveCallLinker.scala @@ -4,7 +4,6 @@ import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.{EdgeTypes, PropertyNames} import io.shiftleft.passes.CpgPass import io.shiftleft.semanticcpg.language.* -import overflowdb.traversal.jIteratortoTraversal /** Link remaining unlinked calls to methods only by their name (not full name) * @param cpg diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/controlflow/cfgdominator/CfgDominatorFrontier.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/controlflow/cfgdominator/CfgDominatorFrontier.scala index e0823e835ee7..b05e62aee1be 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/controlflow/cfgdominator/CfgDominatorFrontier.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/controlflow/cfgdominator/CfgDominatorFrontier.scala @@ -17,7 +17,7 @@ class CfgDominatorFrontier[NodeType](cfgAdapter: CfgAdapter[NodeType], domTreeAd private def withIDom(x: NodeType, preds: Seq[NodeType]) = doms(x).map(i => (x, preds, i)) - def calculate(cfgNodes: Seq[NodeType]): mutable.Map[NodeType, mutable.Set[NodeType]] = { + def calculate(cfgNodes: Iterator[NodeType]): mutable.Map[NodeType, mutable.Set[NodeType]] = { val domFrontier = mutable.Map.empty[NodeType, mutable.Set[NodeType]] for { diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/frontend/TypeNodePass.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/frontend/TypeNodePass.scala index 2c705f8d6075..55ca4d19d12e 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/frontend/TypeNodePass.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/frontend/TypeNodePass.scala @@ -1,26 +1,21 @@ package io.joern.x2cpg.passes.frontend import io.joern.x2cpg.passes.frontend.TypeNodePass.fullToShortName -import io.shiftleft.codepropertygraph.generated.Cpg +import io.shiftleft.codepropertygraph.generated.{Cpg, Properties} import io.shiftleft.codepropertygraph.generated.nodes.NewType -import io.shiftleft.passes.{KeyPool, CpgPass} +import io.shiftleft.passes.CpgPass import io.shiftleft.semanticcpg.language.* -import io.shiftleft.codepropertygraph.generated.PropertyNames +import io.shiftleft.semanticcpg.language.types.structure.NamespaceTraversal import scala.collection.mutable -import io.shiftleft.semanticcpg.language.types.structure.NamespaceTraversal /** Creates a `TYPE` node for each type in `usedTypes` as well as all inheritsFrom type names in the CPG * * Alternatively, set `getTypesFromCpg = true`. If this is set, the `registeredTypes` argument will be ignored. * Instead, type nodes will be created for every unique `TYPE_FULL_NAME` value in the CPG. */ -class TypeNodePass protected ( - registeredTypes: List[String], - cpg: Cpg, - keyPool: Option[KeyPool], - getTypesFromCpg: Boolean -) extends CpgPass(cpg, "types", keyPool) { +class TypeNodePass protected (registeredTypes: List[String], cpg: Cpg, getTypesFromCpg: Boolean) + extends CpgPass(cpg, "types") { protected def typeDeclTypes: mutable.Set[String] = { val typeDeclTypes = mutable.Set[String]() @@ -33,9 +28,8 @@ class TypeNodePass protected ( protected def typeFullNamesFromCpg: Set[String] = { cpg.all - .map(_.property(PropertyNames.TYPE_FULL_NAME)) + .map(_.property(Properties.TypeFullName)) .filter(_ != null) - .map(_.toString) .toSet } @@ -65,12 +59,12 @@ class TypeNodePass protected ( } object TypeNodePass { - def withTypesFromCpg(cpg: Cpg, keyPool: Option[KeyPool] = None): TypeNodePass = { - new TypeNodePass(Nil, cpg, keyPool, getTypesFromCpg = true) + def withTypesFromCpg(cpg: Cpg): TypeNodePass = { + new TypeNodePass(Nil, cpg, getTypesFromCpg = true) } - def withRegisteredTypes(registeredTypes: List[String], cpg: Cpg, keyPool: Option[KeyPool] = None): TypeNodePass = { - new TypeNodePass(registeredTypes, cpg, keyPool, getTypesFromCpg = false) + def withRegisteredTypes(registeredTypes: List[String], cpg: Cpg): TypeNodePass = { + new TypeNodePass(registeredTypes, cpg, getTypesFromCpg = false) } def fullToShortName(typeName: String): String = { diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/frontend/XInheritanceFullNamePass.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/frontend/XInheritanceFullNamePass.scala index c49e62798293..7a3adaf38617 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/frontend/XInheritanceFullNamePass.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/frontend/XInheritanceFullNamePass.scala @@ -41,7 +41,7 @@ abstract class XInheritanceFullNamePass(cpg: Cpg) extends ForkJoinParallelCpgPas inheritedTypes == Seq("ANY") || inheritedTypes == Seq("object") || inheritedTypes.isEmpty private def extractTypeDeclFromNode(node: AstNode): Option[String] = node match { - case x: Call if x.isCallForImportOut.nonEmpty => + case x: Call if x._isCallForImportOut.nonEmpty => x.isCallForImportOut.importedEntity.map { case imp if relativePathPattern.matcher(imp).matches() => imp.split(pathSep).toList match { diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/frontend/XTypeHintCallLinker.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/frontend/XTypeHintCallLinker.scala index 7447a091f588..92733810ca93 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/frontend/XTypeHintCallLinker.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/frontend/XTypeHintCallLinker.scala @@ -4,9 +4,8 @@ import io.joern.x2cpg.passes.base.MethodStubCreator import io.joern.x2cpg.passes.frontend.XTypeRecovery.isDummyType import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.* -import io.shiftleft.codepropertygraph.generated.{EdgeTypes, NodeTypes, PropertyNames} +import io.shiftleft.codepropertygraph.generated.{DispatchTypes, EdgeTypes, NodeTypes, PropertyNames} import io.shiftleft.passes.CpgPass -import io.shiftleft.proto.cpg.Cpg.DispatchTypes import io.shiftleft.semanticcpg.language.* import java.util.regex.Pattern @@ -151,7 +150,7 @@ abstract class XTypeHintCallLinker(cpg: Cpg) extends CpgPass(cpg) { name, fullName, "", - DispatchTypes.DYNAMIC_DISPATCH.name(), + DispatchTypes.DYNAMIC_DISPATCH, argSize, builder, isExternal, diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/frontend/XTypeRecovery.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/frontend/XTypeRecovery.scala index 3d9033149b0a..7e08d6cf114c 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/frontend/XTypeRecovery.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/frontend/XTypeRecovery.scala @@ -1,7 +1,15 @@ package io.joern.x2cpg.passes.frontend import io.joern.x2cpg.{Defines, X2CpgConfig} -import io.shiftleft.codepropertygraph.generated.{Cpg, DispatchTypes, EdgeTypes, NodeTypes, Operators, PropertyNames} +import io.shiftleft.codepropertygraph.generated.{ + Cpg, + DispatchTypes, + EdgeTypes, + NodeTypes, + Operators, + Properties, + PropertyNames +} import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.passes.{CpgPass, CpgPassBase, ForkJoinParallelCpgPass} import io.shiftleft.semanticcpg.language.* @@ -264,9 +272,9 @@ object XTypeRecovery { // the symbol table then perhaps this would work out better implicit class AllNodeTypesFromNodeExt(x: StoredNode) { def allTypes: Iterator[String] = - (x.property(PropertyNames.TYPE_FULL_NAME, "ANY") +: - (x.property(PropertyNames.DYNAMIC_TYPE_HINT_FULL_NAME, Seq.empty) - ++ x.property(PropertyNames.POSSIBLE_TYPES, Seq.empty))).iterator + (x.propertyOption(Properties.TypeFullName).getOrElse("ANY") +: + (x.property(Properties.DynamicTypeHintFullName) ++ + x.property(Properties.PossibleTypes))).iterator def getKnownTypes: Set[String] = { x.allTypes.toSet.filterNot(XTypeRecovery.unknownTypePattern.matches) @@ -435,7 +443,8 @@ abstract class RecoverForXCompilationUnit[CompilationUnitType <: AstNode]( * @param a * assignment call pointer. */ - protected def visitAssignments(a: Assignment): Set[String] = visitAssignmentArguments(a.argumentOut.l) + protected def visitAssignments(a: Assignment): Set[String] = + visitAssignmentArguments(a.argumentOut.cast[CfgNode].l) protected def visitAssignmentArguments(args: List[AstNode]): Set[String] = args match { case List(i: Identifier, b: Block) => visitIdentifierAssignedToBlock(i, b) @@ -809,7 +818,8 @@ abstract class RecoverForXCompilationUnit[CompilationUnitType <: AstNode]( case ::(_: TypeRef, ::(f: FieldIdentifier, _)) => f.canonicalName case xs => - logger.warn(s"Unhandled field structure ${xs.map(x => (x.label, x.code)).mkString(",")} @ ${debugLocation(fa)}") + val debugInfo = xs.collect { case x: CfgNode => (x.label(), x.code) }.mkString(",") + logger.warn(s"Unhandled field structure $debugInfo @ ${debugLocation(fa)}") wrapName("") } } @@ -817,7 +827,7 @@ abstract class RecoverForXCompilationUnit[CompilationUnitType <: AstNode]( protected def visitCallAssignedToLiteral(c: Call, l: Literal): Set[String] = { if (c.name.equals(Operators.indexAccess)) { // For now, we will just handle this on a very basic level - c.argumentOut.l match { + c.argumentOut.cast[CfgNode].l match { case List(_: Identifier, _: Literal) => indexAccessToCollectionVar(c).map(cv => symbolTable.append(cv, getLiteralType(l))).getOrElse(Set.empty) case List(_: Identifier, idx: Identifier) if symbolTable.contains(idx) => @@ -860,7 +870,7 @@ abstract class RecoverForXCompilationUnit[CompilationUnitType <: AstNode]( .getOrElse(XTypeRecovery.DummyIndexAccess) else x.name - Option(c.argumentOut.l match { + Option(c.argumentOut.cast[CfgNode].l match { case List(i: Identifier, idx: Literal) => CollectionVar(i.name, idx.code) case List(i: Identifier, idx: Identifier) => CollectionVar(i.name, idx.code) case List(c: Call, idx: Call) => CollectionVar(callName(c), callName(idx)) @@ -969,7 +979,7 @@ abstract class RecoverForXCompilationUnit[CompilationUnitType <: AstNode]( callPaths.map(c => s"$c$pathSep${XTypeRecovery.DummyReturnType}") else returnValues - case ::(head: Call, Nil) if head.argumentOut.headOption.exists(symbolTable.contains) => + case ::(head: Call, Nil) if head.argumentOut.cast[CfgNode].headOption.exists(symbolTable.contains) => symbolTable .get(head.argumentOut.head) .map(t => Seq(t, head.name, XTypeRecovery.DummyReturnType).mkString(pathSep)) @@ -979,7 +989,7 @@ abstract class RecoverForXCompilationUnit[CompilationUnitType <: AstNode]( extractTypes(head.argument.l) case _ => Set.empty } - val returnTypes = extractTypes(ret.argumentOut.l) + val returnTypes = extractTypes(ret.argumentOut.cast[CfgNode].l) existingTypes.addAll(returnTypes) builder.setNodeProperty(ret.method.methodReturn, PropertyNames.DYNAMIC_TYPE_HINT_FULL_NAME, existingTypes) } @@ -1230,7 +1240,8 @@ abstract class RecoverForXCompilationUnit[CompilationUnitType <: AstNode]( lazy val existingTypes = storedNode.getKnownTypes val hasUnknownTypeFullName = storedNode - .property(PropertyNames.TYPE_FULL_NAME, Defines.Any) + .propertyOption(PropertyNames.TYPE_FULL_NAME) + .getOrElse(Defines.Any) .matches(XTypeRecovery.unknownTypePattern.pattern.pattern()) if (types.nonEmpty && (hasUnknownTypeFullName || types.toSet != existingTypes)) { @@ -1269,10 +1280,12 @@ abstract class RecoverForXCompilationUnit[CompilationUnitType <: AstNode]( */ protected def storeDefaultTypeInfo(n: StoredNode, types: Seq[String]): Unit = val hasUnknownType = - n.property(PropertyNames.TYPE_FULL_NAME, Defines.Any).matches(XTypeRecovery.unknownTypePattern.pattern.pattern()) + n.propertyOption(PropertyNames.TYPE_FULL_NAME) + .getOrElse(Defines.Any) + .matches(XTypeRecovery.unknownTypePattern.pattern.pattern()) if (types.toSet != n.getKnownTypes || (hasUnknownType && types.nonEmpty)) { - setTypes(n, (n.property(PropertyNames.DYNAMIC_TYPE_HINT_FULL_NAME, Seq.empty) ++ types).distinct) + setTypes(n, (n.propertyOption(PropertyNames.DYNAMIC_TYPE_HINT_FULL_NAME).getOrElse(Seq.empty) ++ types).distinct) } /** If there is only 1 type hint then this is set to the `typeFullName` property and `dynamicTypeHintFullName` is diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/typerelations/AliasLinkerPass.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/typerelations/AliasLinkerPass.scala index 3714a9c3b6c1..6d9509df1d56 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/typerelations/AliasLinkerPass.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/typerelations/AliasLinkerPass.scala @@ -4,6 +4,7 @@ import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.TypeDecl import io.shiftleft.codepropertygraph.generated.{EdgeTypes, NodeTypes, PropertyNames} import io.shiftleft.passes.CpgPass +import io.shiftleft.semanticcpg.language.* import io.joern.x2cpg.utils.LinkingUtil class AliasLinkerPass(cpg: Cpg) extends CpgPass(cpg) with LinkingUtil { diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/typerelations/FieldAccessLinkerPass.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/typerelations/FieldAccessLinkerPass.scala index e60479b1be34..f88245a68d81 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/typerelations/FieldAccessLinkerPass.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/typerelations/FieldAccessLinkerPass.scala @@ -68,7 +68,7 @@ class FieldAccessLinkerPass(cpg: Cpg) extends CpgPass(cpg) with LinkingUtil { dstGraph: DiffGraphBuilder ): Unit = { val dereference = Dereference(cpg) - cpg.graph.nodes(srcLabels*).asScala.cast[SRC_NODE_TYPE].filterNot(_.outE(edgeType).hasNext).foreach { srcNode => + cpg.graph.nodes(srcLabels*).cast[SRC_NODE_TYPE].filterNot(_.outE(edgeType).hasNext).foreach { srcNode => if (!srcNode.outE(edgeType).hasNext) { getDstFullNames(srcNode).foreach { dstFullName => val dereferenceDstFullName = dereference.dereferenceTypeFullName(dstFullName) diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/typerelations/TypeHierarchyPass.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/typerelations/TypeHierarchyPass.scala index 5f96ba2e76dc..d1a3af47136d 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/typerelations/TypeHierarchyPass.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/typerelations/TypeHierarchyPass.scala @@ -1,10 +1,11 @@ package io.joern.x2cpg.passes.typerelations import io.shiftleft.codepropertygraph.generated.Cpg +import io.joern.x2cpg.utils.LinkingUtil import io.shiftleft.codepropertygraph.generated.nodes.TypeDecl import io.shiftleft.codepropertygraph.generated.{EdgeTypes, NodeTypes, PropertyNames} import io.shiftleft.passes.CpgPass -import io.joern.x2cpg.utils.LinkingUtil +import io.shiftleft.semanticcpg.language.* /** Create INHERITS_FROM edges from `TYPE_DECL` nodes to `TYPE` nodes. */ diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/AstPropertiesUtil.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/AstPropertiesUtil.scala index 2257502407fc..53f1f695d9b3 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/AstPropertiesUtil.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/AstPropertiesUtil.scala @@ -8,7 +8,7 @@ object AstPropertiesUtil { implicit class RootProperties(val ast: Ast) extends AnyVal { private def rootProperty(propertyName: String): Option[String] = { - ast.root.flatMap(_.properties.get(propertyName).map(_.toString)) + ast.root.flatMap(node => Option(node.propertiesMap.get(propertyName)).map(_.toString)) } def rootType: Option[String] = rootProperty(PropertyNames.TYPE_FULL_NAME) diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/KeyPool.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/KeyPool.scala new file mode 100644 index 000000000000..0faa1f1fa216 --- /dev/null +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/KeyPool.scala @@ -0,0 +1,80 @@ +package io.joern.x2cpg.utils + +import java.util.concurrent.atomic.{AtomicInteger, AtomicLong} + +/** A pool of long integers. Using the method `next`, the pool provides the next id in a thread-safe manner. */ +trait KeyPool { + def next: Long +} + +/** A key pool that returns the integers of the interval [first, last] in a thread-safe manner. + */ +class IntervalKeyPool(val first: Long, val last: Long) extends KeyPool { + + /** Get next number in interval or raise if number is larger than `last` + */ + def next: Long = { + if (!valid) { + throw new IllegalStateException("Call to `next` on invalidated IntervalKeyPool.") + } + val n = cur.incrementAndGet() + if (n > last) { + throw new RuntimeException("Pool exhausted") + } else { + n + } + } + + /** Split key pool into `numberOfPartitions` partitions of mostly equal size. Invalidates the current pool to ensure + * that the user does not continue to use both the original pool and pools derived from it via `split`. + */ + def split(numberOfPartitions: Int): Iterator[IntervalKeyPool] = { + valid = false + if (numberOfPartitions == 0) { + Iterator() + } else { + val curFirst = cur.get() + val k = (last - curFirst) / numberOfPartitions + (1 to numberOfPartitions).map { i => + val poolFirst = curFirst + (i - 1) * k + new IntervalKeyPool(poolFirst, poolFirst + k - 1) + }.iterator + } + } + + private val cur: AtomicLong = new AtomicLong(first - 1) + private var valid: Boolean = true +} + +/** A key pool that returns elements of `seq` in order in a thread-safe manner. + */ +class SequenceKeyPool(seq: Seq[Long]) extends KeyPool { + + val seqLen: Int = seq.size + var cur = new AtomicInteger(-1) + + override def next: Long = { + val i = cur.incrementAndGet() + if (i >= seqLen) { + throw new RuntimeException("Pool exhausted") + } else { + seq(i) + } + } +} + +object KeyPoolCreator { + + /** Divide the keyspace into n intervals and return a list of corresponding key pools. + */ + def obtain(n: Long, minValue: Long = 0, maxValue: Long = Long.MaxValue): List[IntervalKeyPool] = { + val nIntervals = Math.max(n, 1) + val intervalLen: Long = (maxValue - minValue) / nIntervals + List.range(0L, nIntervals).map { i => + val first = i * intervalLen + minValue + val last = first + intervalLen - 1 + new IntervalKeyPool(first, last) + } + } + +} diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/LinkingUtil.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/LinkingUtil.scala index 72c962aab116..0080a29f118c 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/LinkingUtil.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/LinkingUtil.scala @@ -1,15 +1,13 @@ package io.joern.x2cpg.utils import io.joern.x2cpg.passes.frontend.Dereference -import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.* -import io.shiftleft.codepropertygraph.generated.{Properties, PropertyNames} +import io.shiftleft.codepropertygraph.generated.{Cpg, Properties, PropertyNames} +import io.shiftleft.codepropertygraph.generated.nodes.NamespaceBlock +import io.shiftleft.codepropertygraph.generated.nodes.Type +import io.shiftleft.semanticcpg.language.* import org.slf4j.{Logger, LoggerFactory} -import overflowdb.traversal.* -import overflowdb.traversal.ChainedImplicitsTemp.* -import overflowdb.{Node, NodeDb, NodeRef, PropertyKey} -import scala.collection.mutable import scala.jdk.CollectionConverters.* trait LinkingUtil { @@ -32,21 +30,21 @@ trait LinkingUtil { def namespaceBlockFullNameToNode(cpg: Cpg, x: String): Option[NamespaceBlock] = nodesWithFullName(cpg, x).collectFirst { case x: NamespaceBlock => x } - def nodesWithFullName(cpg: Cpg, x: String): mutable.Seq[NodeRef[? <: NodeDb]] = - cpg.graph.indexManager.lookup(PropertyNames.FULL_NAME, x).asScala + def nodesWithFullName(cpg: Cpg, x: String): Iterator[StoredNode] = + cpg.graph.nodesWithProperty(propertyName = PropertyNames.FULL_NAME, value = x).cast[StoredNode] /** For all nodes `n` with a label in `srcLabels`, determine the value of `n.\$dstFullNameKey`, use that to lookup the * destination node in `dstNodeMap`, and create an edge of type `edgeType` between `n` and the destination node. */ - protected def linkToSingle( cpg: Cpg, - srcNodes: List[Node], + srcNodes: List[StoredNode], srcLabels: List[String], dstNodeLabel: String, edgeType: String, dstNodeMap: String => Option[StoredNode], dstFullNameKey: String, + dstDefaultPropertyValue: Any, dstGraph: DiffGraphBuilder, dstNotExistsHandler: Option[(StoredNode, String) => Unit] ): Unit = { @@ -56,14 +54,13 @@ trait LinkingUtil { // If the source node does not have any outgoing edges of this type // This check is just required for backward compatibility if (srcNode.outE(edgeType).isEmpty) { - val key = new PropertyKey[String](dstFullNameKey) srcNode - .propertyOption(key) + .propertyOption[String](dstFullNameKey) .filter { dstFullName => val dereferenceDstFullName = dereference.dereferenceTypeFullName(dstFullName) - srcNode.propertyDefaultValue(dstFullNameKey) != dereferenceDstFullName + dstDefaultPropertyValue != dereferenceDstFullName } - .ifPresent { dstFullName => + .map { dstFullName => // for `UNKNOWN` this is not always set, so we're using an Option here val srcStoredNode = srcNode.asInstanceOf[StoredNode] val dereferenceDstFullName = dereference.dereferenceTypeFullName(dstFullName) @@ -111,7 +108,7 @@ trait LinkingUtil { ): Unit = { var loggedDeprecationWarning = false val dereference = Dereference(cpg) - cpg.graph.nodes(srcLabels*).asScala.cast[SRC_NODE_TYPE].foreach { srcNode => + cpg.graph.nodes(srcLabels*).cast[SRC_NODE_TYPE].foreach { srcNode => if (!srcNode.outE(edgeType).hasNext) { getDstFullNames(srcNode).foreach { dstFullName => val dereferenceDstFullName = dereference.dereferenceTypeFullName(dstFullName) diff --git a/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/AstTests.scala b/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/AstTests.scala index 86fbb6c1860c..0b66e6112a2a 100644 --- a/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/AstTests.scala +++ b/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/AstTests.scala @@ -1,9 +1,9 @@ package io.joern.x2cpg -import io.shiftleft.codepropertygraph.generated.nodes.{AstNodeNew, NewCall, NewClosureBinding, NewIdentifier} +import flatgraph.SchemaViolationException +import io.shiftleft.codepropertygraph.generated.nodes.{AstNodeNew, Call, NewCall, NewClosureBinding, NewIdentifier} import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec -import overflowdb.SchemaViolationException class AstTests extends AnyWordSpec with Matchers { @@ -35,7 +35,7 @@ class AstTests extends AnyWordSpec with Matchers { copied.root match { case Some(root: NewCall) => root should not be Some(moo) - root.properties("NAME") shouldBe "moo" + root.propertiesMap.get(Call.PropertyNames.Name) shouldBe "moo" root.argumentIndex shouldBe 123 case _ => fail() } diff --git a/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/X2CpgTests.scala b/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/X2CpgTests.scala index 18ea217c43ef..7c9acf560a51 100644 --- a/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/X2CpgTests.scala +++ b/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/X2CpgTests.scala @@ -12,8 +12,7 @@ class X2CpgTests extends AnyWordSpec with Matchers { "create an empty in-memory CPG when no output path is given" in { val cpg = X2Cpg.newEmptyCpg(None) - cpg.graph.V.hasNext shouldBe false - cpg.graph.E.hasNext shouldBe false + cpg.graph.allNodes.hasNext shouldBe false cpg.close() } @@ -22,9 +21,9 @@ class X2CpgTests extends AnyWordSpec with Matchers { file.delete() file.exists shouldBe false val cpg = X2Cpg.newEmptyCpg(Some(file.path.toString)) + cpg.close() file.exists shouldBe true Files.size(file.path) should not be 0 - cpg.close() } "overwrite existing file to create empty CPG" in { @@ -32,11 +31,10 @@ class X2CpgTests extends AnyWordSpec with Matchers { file.exists shouldBe true Files.size(file.path) shouldBe 0 val cpg = X2Cpg.newEmptyCpg(Some(file.path.toString)) - cpg.graph.V.hasNext shouldBe false - cpg.graph.E.hasNext shouldBe false + cpg.graph.allNodes.hasNext shouldBe false + cpg.close() file.exists shouldBe true Files.size(file.path) should not be 0 - cpg.close() } } } diff --git a/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/passes/CfgDominatorFrontierTests.scala b/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/passes/CfgDominatorFrontierTests.scala index 07c13d7a31b6..eadfe3f51682 100644 --- a/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/passes/CfgDominatorFrontierTests.scala +++ b/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/passes/CfgDominatorFrontierTests.scala @@ -1,48 +1,62 @@ package io.joern.x2cpg.passes -import io.shiftleft.OverflowDbTestInstance +import flatgraph.misc.TestUtils.* import io.joern.x2cpg.passes.controlflow.cfgdominator.{CfgAdapter, CfgDominator, CfgDominatorFrontier, DomTreeAdapter} +import io.shiftleft.codepropertygraph.generated.{Cpg, EdgeTypes} +import io.shiftleft.codepropertygraph.generated.nodes.{NewUnknown, StoredNode} +import io.shiftleft.semanticcpg.language.* import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec -import overflowdb.* - import scala.jdk.CollectionConverters.* class CfgDominatorFrontierTests extends AnyWordSpec with Matchers { - private class TestCfgAdapter extends CfgAdapter[Node] { - override def successors(node: Node): IterableOnce[Node] = - node.out("CFG").asScala + private class TestCfgAdapter extends CfgAdapter[StoredNode] { + override def successors(node: StoredNode): Iterator[StoredNode] = + node.out("CFG").cast[StoredNode] - override def predecessors(node: Node): IterableOnce[Node] = - node.in("CFG").asScala + override def predecessors(node: StoredNode): Iterator[StoredNode] = + node.in("CFG").cast[StoredNode] } - private class TestDomTreeAdapter(immediateDominators: scala.collection.Map[Node, Node]) extends DomTreeAdapter[Node] { - override def immediateDominator(cfgNode: Node): Option[Node] = { + private class TestDomTreeAdapter(immediateDominators: scala.collection.Map[StoredNode, StoredNode]) + extends DomTreeAdapter[StoredNode] { + override def immediateDominator(cfgNode: StoredNode): Option[StoredNode] = { immediateDominators.get(cfgNode) } } "Cfg dominance frontier test" in { - val graph = OverflowDbTestInstance.create - - val v0 = graph + "UNKNOWN" - val v1 = graph + "UNKNOWN" - val v2 = graph + "UNKNOWN" - val v3 = graph + "UNKNOWN" - val v4 = graph + "UNKNOWN" - val v5 = graph + "UNKNOWN" - val v6 = graph + "UNKNOWN" - - v0 --- "CFG" --> v1 - v1 --- "CFG" --> v2 - v2 --- "CFG" --> v3 - v2 --- "CFG" --> v5 - v3 --- "CFG" --> v4 - v4 --- "CFG" --> v2 - v4 --- "CFG" --> v5 - v5 --- "CFG" --> v6 + val cpg = Cpg.empty + val graph = cpg.graph + + val v0 = graph.addNode(NewUnknown()) + val v1 = graph.addNode(NewUnknown()) + val v2 = graph.addNode(NewUnknown()) + val v3 = graph.addNode(NewUnknown()) + val v4 = graph.addNode(NewUnknown()) + val v5 = graph.addNode(NewUnknown()) + val v6 = graph.addNode(NewUnknown()) + + // TODO MP get arrow syntax back +// v0 --- "CFG" --> v1 +// v1 --- "CFG" --> v2 +// v2 --- "CFG" --> v3 +// v2 --- "CFG" --> v5 +// v3 --- "CFG" --> v4 +// v4 --- "CFG" --> v2 +// v4 --- "CFG" --> v5 +// v5 --- "CFG" --> v6 + graph.applyDiff { diffGraphBuilder => + diffGraphBuilder.addEdge(v0, v1, EdgeTypes.CFG) + diffGraphBuilder.addEdge(v1, v2, EdgeTypes.CFG) + diffGraphBuilder.addEdge(v2, v3, EdgeTypes.CFG) + diffGraphBuilder.addEdge(v2, v5, EdgeTypes.CFG) + diffGraphBuilder.addEdge(v3, v4, EdgeTypes.CFG) + diffGraphBuilder.addEdge(v4, v2, EdgeTypes.CFG) + diffGraphBuilder.addEdge(v4, v5, EdgeTypes.CFG) + diffGraphBuilder.addEdge(v5, v6, EdgeTypes.CFG) + } val cfgAdapter = new TestCfgAdapter val cfgDominatorCalculator = new CfgDominator(cfgAdapter) @@ -50,7 +64,7 @@ class CfgDominatorFrontierTests extends AnyWordSpec with Matchers { val domTreeAdapter = new TestDomTreeAdapter(immediateDominators) val cfgDominatorFrontier = new CfgDominatorFrontier(cfgAdapter, domTreeAdapter) - val dominanceFrontier = cfgDominatorFrontier.calculate(graph.nodes.asScala.toList) + val dominanceFrontier = cfgDominatorFrontier.calculate(cpg.all) dominanceFrontier.get(v0) shouldBe None dominanceFrontier.get(v1) shouldBe None @@ -62,14 +76,20 @@ class CfgDominatorFrontierTests extends AnyWordSpec with Matchers { } "Cfg domiance frontier with dead code test" in { - val graph = OverflowDbTestInstance.create - - val v0 = graph + "UNKNOWN" - val v1 = graph + "UNKNOWN" // This node simulates dead code as it is not reachable from the entry v0. - val v2 = graph + "UNKNOWN" - - v0 --- "CFG" --> v2 - v1 --- "CFG" --> v2 + val cpg = Cpg.empty + val graph = cpg.graph + + val v0 = graph.addNode(NewUnknown()) + val v1 = graph.addNode(NewUnknown()) // This node simulates dead code as it is not reachable from the entry v0. + val v2 = graph.addNode(NewUnknown()) + + // TODO MP get arrow syntax back +// v0 --- "CFG" --> v2 +// v1 --- "CFG" --> v2 + graph.applyDiff { diffGraphBuilder => + diffGraphBuilder.addEdge(v0, v2, EdgeTypes.CFG) + diffGraphBuilder.addEdge(v1, v2, EdgeTypes.CFG) + } val cfgAdapter = new TestCfgAdapter val cfgDominatorCalculator = new CfgDominator(cfgAdapter) @@ -77,7 +97,7 @@ class CfgDominatorFrontierTests extends AnyWordSpec with Matchers { val domTreeAdapter = new TestDomTreeAdapter(immediateDominators) val cfgDominatorFrontier = new CfgDominatorFrontier(cfgAdapter, domTreeAdapter) - val dominanceFrontier = cfgDominatorFrontier.calculate(graph.nodes.asScala.toList) + val dominanceFrontier = cfgDominatorFrontier.calculate(cpg.all) dominanceFrontier.get(v0) shouldBe None dominanceFrontier.apply(v1) shouldBe Set(v2) diff --git a/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/passes/CfgDominatorPassTests.scala b/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/passes/CfgDominatorPassTests.scala index 59f92d169999..1b5b615676ea 100644 --- a/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/passes/CfgDominatorPassTests.scala +++ b/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/passes/CfgDominatorPassTests.scala @@ -1,80 +1,93 @@ package io.joern.x2cpg.passes -import io.shiftleft.OverflowDbTestInstance -import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.codepropertygraph.generated.{EdgeTypes, NodeTypes} +import io.shiftleft.codepropertygraph.generated.{Cpg, EdgeTypes, NodeTypes} +import flatgraph.misc.TestUtils.* import io.joern.x2cpg.passes.controlflow.cfgdominator.CfgDominatorPass +import io.shiftleft.codepropertygraph.generated.{Cpg, EdgeTypes, NodeTypes} +import io.shiftleft.codepropertygraph.generated.nodes.{NewMethod, NewMethodReturn, NewUnknown} +import io.shiftleft.semanticcpg.language.* import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec -import overflowdb.* import scala.jdk.CollectionConverters.* class CfgDominatorPassTests extends AnyWordSpec with Matchers { "Have correct DOMINATE/POST_DOMINATE edges after CfgDominatorPass run." in { - val graph = OverflowDbTestInstance.create - val cpg = new Cpg(graph) + val cpg = Cpg.empty + val graph = cpg.graph - val v0 = graph + NodeTypes.METHOD - val v1 = graph + NodeTypes.UNKNOWN - val v2 = graph + NodeTypes.UNKNOWN - val v3 = graph + NodeTypes.UNKNOWN - val v4 = graph + NodeTypes.UNKNOWN - val v5 = graph + NodeTypes.UNKNOWN - val v6 = graph + NodeTypes.METHOD_RETURN + val v0 = graph.addNode(NewMethod()) + val v1 = graph.addNode(NewUnknown()) + val v2 = graph.addNode(NewUnknown()) + val v3 = graph.addNode(NewUnknown()) + val v4 = graph.addNode(NewUnknown()) + val v5 = graph.addNode(NewUnknown()) + val v6 = graph.addNode(NewMethodReturn()) - v0 --- EdgeTypes.AST --> v6 + // TODO MP get arrow syntax back +// v0 --- EdgeTypes.AST --> v6 +// +// v0 --- EdgeTypes.CFG --> v1 +// v1 --- EdgeTypes.CFG --> v2 +// v2 --- EdgeTypes.CFG --> v3 +// v2 --- EdgeTypes.CFG --> v5 +// v3 --- EdgeTypes.CFG --> v4 +// v4 --- EdgeTypes.CFG --> v2 +// v4 --- EdgeTypes.CFG --> v5 +// v5 --- EdgeTypes.CFG --> v6 + graph.applyDiff { diffGraphBuilder => + diffGraphBuilder.addEdge(v0, v6, EdgeTypes.AST) - v0 --- EdgeTypes.CFG --> v1 - v1 --- EdgeTypes.CFG --> v2 - v2 --- EdgeTypes.CFG --> v3 - v2 --- EdgeTypes.CFG --> v5 - v3 --- EdgeTypes.CFG --> v4 - v4 --- EdgeTypes.CFG --> v2 - v4 --- EdgeTypes.CFG --> v5 - v5 --- EdgeTypes.CFG --> v6 + diffGraphBuilder.addEdge(v0, v1, EdgeTypes.CFG) + diffGraphBuilder.addEdge(v1, v2, EdgeTypes.CFG) + diffGraphBuilder.addEdge(v2, v3, EdgeTypes.CFG) + diffGraphBuilder.addEdge(v2, v5, EdgeTypes.CFG) + diffGraphBuilder.addEdge(v3, v4, EdgeTypes.CFG) + diffGraphBuilder.addEdge(v4, v5, EdgeTypes.CFG) + diffGraphBuilder.addEdge(v5, v6, EdgeTypes.CFG) + } val dominatorTreePass = new CfgDominatorPass(cpg) dominatorTreePass.createAndApply() - val v0Dominates = v0.out(EdgeTypes.DOMINATE).asScala.toList + val v0Dominates = v0.out(EdgeTypes.DOMINATE).l v0Dominates.size shouldBe 1 v0Dominates.toSet shouldBe Set(v1) - val v1Dominates = v1.out(EdgeTypes.DOMINATE).asScala.toList + val v1Dominates = v1.out(EdgeTypes.DOMINATE).l v1Dominates.size shouldBe 1 v1Dominates.toSet shouldBe Set(v2) - val v2Dominates = v2.out(EdgeTypes.DOMINATE).asScala.toList + val v2Dominates = v2.out(EdgeTypes.DOMINATE).l v2Dominates.size shouldBe 2 v2Dominates.toSet shouldBe Set(v3, v5) - val v3Dominates = v3.out(EdgeTypes.DOMINATE).asScala.toList + val v3Dominates = v3.out(EdgeTypes.DOMINATE).l v3Dominates.size shouldBe 1 v3Dominates.toSet shouldBe Set(v4) - val v4Dominates = v4.out(EdgeTypes.DOMINATE).asScala.toList + val v4Dominates = v4.out(EdgeTypes.DOMINATE).l v4Dominates.size shouldBe 0 - val v5Dominates = v5.out(EdgeTypes.DOMINATE).asScala.toList + val v5Dominates = v5.out(EdgeTypes.DOMINATE).l v5Dominates.size shouldBe 1 v5Dominates.toSet shouldBe Set(v6) - val v6Dominates = v6.out(EdgeTypes.DOMINATE).asScala.toList + val v6Dominates = v6.out(EdgeTypes.DOMINATE).l v6Dominates.size shouldBe 0 - val v6PostDominates = v6.out(EdgeTypes.POST_DOMINATE).asScala.toList + val v6PostDominates = v6.out(EdgeTypes.POST_DOMINATE).l v6PostDominates.size shouldBe 1 v6PostDominates.toSet shouldBe Set(v5) - val v5PostDominates = v5.out(EdgeTypes.POST_DOMINATE).asScala.toList + val v5PostDominates = v5.out(EdgeTypes.POST_DOMINATE).l v5PostDominates.size shouldBe 2 v5PostDominates.toSet shouldBe Set(v2, v4) - val v4PostDominates = v4.out(EdgeTypes.POST_DOMINATE).asScala.toList + val v4PostDominates = v4.out(EdgeTypes.POST_DOMINATE).l v4PostDominates.size shouldBe 1 v4PostDominates.toSet shouldBe Set(v3) - val v3PostDominates = v3.out(EdgeTypes.POST_DOMINATE).asScala.toList + val v3PostDominates = v3.out(EdgeTypes.POST_DOMINATE).l v3PostDominates.size shouldBe 0 - val v2PostDominates = v2.out(EdgeTypes.POST_DOMINATE).asScala.toList + val v2PostDominates = v2.out(EdgeTypes.POST_DOMINATE).l v2PostDominates.size shouldBe 1 v2PostDominates.toSet shouldBe Set(v1) - val v1PostDominates = v1.out(EdgeTypes.POST_DOMINATE).asScala.toList + val v1PostDominates = v1.out(EdgeTypes.POST_DOMINATE).l v1PostDominates.size shouldBe 1 v1PostDominates.toSet shouldBe Set(v0) - val v0PostDominates = v0.out(EdgeTypes.POST_DOMINATE).asScala.toList + val v0PostDominates = v0.out(EdgeTypes.POST_DOMINATE).l v0PostDominates.size shouldBe 0 } } diff --git a/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/passes/ContainsEdgePassTest.scala b/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/passes/ContainsEdgePassTest.scala index a88b49de7876..ba049a318622 100644 --- a/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/passes/ContainsEdgePassTest.scala +++ b/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/passes/ContainsEdgePassTest.scala @@ -1,14 +1,13 @@ package io.joern.x2cpg.passes -import io.shiftleft.OverflowDbTestInstance -import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.codepropertygraph.generated.{EdgeTypes, NodeTypes} +import io.shiftleft.codepropertygraph.generated.{Cpg, EdgeTypes, NodeTypes} +import flatgraph.misc.TestUtils.* import io.joern.x2cpg.passes.base.ContainsEdgePass +import io.shiftleft.codepropertygraph.generated.nodes.{NewCall, NewFile, NewMethod, NewTypeDecl} +import io.shiftleft.codepropertygraph.generated.{Cpg, EdgeTypes} +import io.shiftleft.semanticcpg.language.* import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec -import overflowdb.* - -import scala.jdk.CollectionConverters.* class ContainsEdgePassTest extends AnyWordSpec with Matchers { @@ -16,26 +15,26 @@ class ContainsEdgePassTest extends AnyWordSpec with Matchers { "Files " can { "contain Methods" in Fixture { fixture => - fixture.methodVertex.in(EdgeTypes.CONTAINS).asScala.toList shouldBe List(fixture.fileVertex) + fixture.methodVertex.in(EdgeTypes.CONTAINS).l shouldBe List(fixture.fileVertex) } "contain Classes" in Fixture { fixture => - fixture.typeDeclVertex.in(EdgeTypes.CONTAINS).asScala.toList shouldBe List(fixture.fileVertex) + fixture.typeDeclVertex.in(EdgeTypes.CONTAINS).l shouldBe List(fixture.fileVertex) } } "Classes " can { "contain Methods" in Fixture { fixture => - fixture.typeMethodVertex.in(EdgeTypes.CONTAINS).asScala.toList shouldBe List(fixture.typeDeclVertex) + fixture.typeMethodVertex.in(EdgeTypes.CONTAINS).l shouldBe List(fixture.typeDeclVertex) } } "Methods " can { "contain Methods" in Fixture { fixture => - fixture.innerMethodVertex.in(EdgeTypes.CONTAINS).asScala.toList shouldBe List(fixture.methodVertex) + fixture.innerMethodVertex.in(EdgeTypes.CONTAINS).l shouldBe List(fixture.methodVertex) } "contain expressions" in Fixture { fixture => - fixture.expressionVertex.in(EdgeTypes.CONTAINS).asScala.toList shouldBe List(fixture.methodVertex) - fixture.innerExpressionVertex.in(EdgeTypes.CONTAINS).asScala.toList shouldBe List(fixture.innerMethodVertex) + fixture.expressionVertex.in(EdgeTypes.CONTAINS).l shouldBe List(fixture.methodVertex) + fixture.innerExpressionVertex.in(EdgeTypes.CONTAINS).l shouldBe List(fixture.innerMethodVertex) } } @@ -43,23 +42,34 @@ class ContainsEdgePassTest extends AnyWordSpec with Matchers { object ContainsEdgePassTest { private class Fixture { - private val graph = OverflowDbTestInstance.create + private val cpg = Cpg.empty + private val graph = cpg.graph - val fileVertex = graph + NodeTypes.FILE - val typeDeclVertex = graph + NodeTypes.TYPE_DECL - val typeMethodVertex = graph + NodeTypes.METHOD - val methodVertex = graph + NodeTypes.METHOD - val innerMethodVertex = graph + NodeTypes.METHOD - val expressionVertex = graph + NodeTypes.CALL - val innerExpressionVertex = graph + NodeTypes.CALL + val fileVertex = graph.addNode(NewFile()) + val typeDeclVertex = graph.addNode(NewTypeDecl()) + val typeMethodVertex = graph.addNode(NewMethod()) + val methodVertex = graph.addNode(NewMethod()) + val innerMethodVertex = graph.addNode(NewMethod()) + val expressionVertex = graph.addNode(NewCall()) + val innerExpressionVertex = graph.addNode(NewCall()) - fileVertex --- EdgeTypes.AST --> typeDeclVertex - typeDeclVertex --- EdgeTypes.AST --> typeMethodVertex + // TODO MP get arrow syntax back +// fileVertex --- EdgeTypes.AST --> typeDeclVertex +// typeDeclVertex --- EdgeTypes.AST --> typeMethodVertex +// +// fileVertex --- EdgeTypes.AST --> methodVertex +// methodVertex --- EdgeTypes.AST --> innerMethodVertex +// methodVertex --- EdgeTypes.AST --> expressionVertex +// innerMethodVertex --- EdgeTypes.AST --> innerExpressionVertex + graph.applyDiff { diffGraphBuilder => + diffGraphBuilder.addEdge(fileVertex, typeDeclVertex, EdgeTypes.AST) + diffGraphBuilder.addEdge(typeDeclVertex, typeMethodVertex, EdgeTypes.AST) - fileVertex --- EdgeTypes.AST --> methodVertex - methodVertex --- EdgeTypes.AST --> innerMethodVertex - methodVertex --- EdgeTypes.AST --> expressionVertex - innerMethodVertex --- EdgeTypes.AST --> innerExpressionVertex + diffGraphBuilder.addEdge(fileVertex, methodVertex, EdgeTypes.AST) + diffGraphBuilder.addEdge(methodVertex, innerMethodVertex, EdgeTypes.AST) + diffGraphBuilder.addEdge(methodVertex, expressionVertex, EdgeTypes.AST) + diffGraphBuilder.addEdge(innerMethodVertex, innerExpressionVertex, EdgeTypes.AST) + } val containsEdgeCalculator = new ContainsEdgePass(new Cpg(graph)) containsEdgeCalculator.createAndApply() diff --git a/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/passes/MethodDecoratorPassTests.scala b/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/passes/MethodDecoratorPassTests.scala index 1864a5952254..6977bd1456cb 100644 --- a/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/passes/MethodDecoratorPassTests.scala +++ b/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/passes/MethodDecoratorPassTests.scala @@ -1,30 +1,32 @@ package io.joern.x2cpg.passes -import io.shiftleft.codepropertygraph.generated.Cpg +import flatgraph.misc.TestUtils.* import io.shiftleft.codepropertygraph.generated.* -import io.shiftleft.codepropertygraph.generated.nodes.MethodParameterIn +import io.shiftleft.codepropertygraph.generated.nodes.* +import io.shiftleft.semanticcpg.language.* import io.joern.x2cpg.passes.base.MethodDecoratorPass import io.joern.x2cpg.testfixtures.EmptyGraphFixture import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec -import overflowdb.* class MethodDecoratorPassTests extends AnyWordSpec with Matchers { "MethodDecoratorTest" in EmptyGraphFixture { graph => - val method = graph + NodeTypes.METHOD - val parameterIn = graph - .+( - NodeTypes.METHOD_PARAMETER_IN, - Properties.Code -> "p1", - Properties.Order -> 1, - Properties.Name -> "p1", - Properties.EvaluationStrategy -> EvaluationStrategies.BY_REFERENCE, - Properties.TypeFullName -> "some.Type", - Properties.LineNumber -> 10 - ) - .asInstanceOf[MethodParameterIn] + val method = graph.addNode(NewMethod()) + val parameterIn = graph.addNode( + NewMethodParameterIn() + .code("p1") + .order(1) + .name("p1") + .evaluationStrategy(EvaluationStrategies.BY_REFERENCE) + .typeFullName("some.Type") + .lineNumber(10) + ) - method --- EdgeTypes.AST --> parameterIn + // TODO MP get arrow syntax back +// method --- EdgeTypes.AST --> parameterIn + graph.applyDiff { diffGraphBuilder => + diffGraphBuilder.addEdge(method, parameterIn, EdgeTypes.AST) + } val methodDecorator = new MethodDecoratorPass(new Cpg(graph)) methodDecorator.createAndApply() diff --git a/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/passes/NamespaceCreatorTests.scala b/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/passes/NamespaceCreatorTests.scala index d4e7d0a39a77..2f22f3c62b2e 100644 --- a/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/passes/NamespaceCreatorTests.scala +++ b/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/passes/NamespaceCreatorTests.scala @@ -1,22 +1,22 @@ package io.joern.x2cpg.passes -import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.codepropertygraph.generated.{NodeTypes, Properties} +import flatgraph.misc.TestUtils.addNode +import io.shiftleft.codepropertygraph.generated.{Cpg, NodeTypes} import io.shiftleft.semanticcpg.language.* import io.joern.x2cpg.passes.base.NamespaceCreator import io.joern.x2cpg.testfixtures.EmptyGraphFixture +import io.shiftleft.codepropertygraph.generated.nodes.NewNamespaceBlock import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec -import overflowdb.* class NamespaceCreatorTests extends AnyWordSpec with Matchers { "NamespaceCreateor test " in EmptyGraphFixture { graph => val cpg = new Cpg(graph) - val block1 = graph + (NodeTypes.NAMESPACE_BLOCK, Properties.Name -> "namespace1") - val block2 = graph + (NodeTypes.NAMESPACE_BLOCK, Properties.Name -> "namespace1") - val block3 = graph + (NodeTypes.NAMESPACE_BLOCK, Properties.Name -> "namespace2") + val block1 = graph.addNode(NewNamespaceBlock().name("namespace1")) + val block2 = graph.addNode(NewNamespaceBlock().name("namespace1")) + val block3 = graph.addNode(NewNamespaceBlock().name("namespace2")) - val namespaceCreator = new NamespaceCreator(new Cpg(graph)) + val namespaceCreator = new NamespaceCreator(cpg) namespaceCreator.createAndApply() val namespaces = cpg.namespace.l diff --git a/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/testfixtures/EmptyGraphFixture.scala b/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/testfixtures/EmptyGraphFixture.scala index 4a378c095580..36c6dcb2c481 100644 --- a/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/testfixtures/EmptyGraphFixture.scala +++ b/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/testfixtures/EmptyGraphFixture.scala @@ -1,12 +1,11 @@ package io.joern.x2cpg.testfixtures -import io.shiftleft.OverflowDbTestInstance -import overflowdb.Graph +import flatgraph.Graph +import io.shiftleft.codepropertygraph.generated.Cpg + +import scala.util.Using object EmptyGraphFixture { - def apply[T](fun: Graph => T): T = { - val graph = OverflowDbTestInstance.create - try fun(graph) - finally { graph.close() } - } + def apply[T](fun: Graph => T): T = + Using.resource(Cpg.empty.graph)(fun) } diff --git a/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/testfixtures/TestCpg.scala b/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/testfixtures/TestCpg.scala index b70cddfb4c56..7dac8f7b7bee 100644 --- a/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/testfixtures/TestCpg.scala +++ b/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/testfixtures/TestCpg.scala @@ -1,9 +1,9 @@ package io.joern.x2cpg.testfixtures +import flatgraph.Graph import io.joern.x2cpg.X2CpgConfig import io.joern.x2cpg.utils.TestCodeWriter import io.shiftleft.codepropertygraph.generated.Cpg -import overflowdb.Graph import java.nio.file.{Files, Path} import java.util.Comparator @@ -11,7 +11,7 @@ import java.util.Comparator // Lazily populated test CPG which is created upon first access to the underlying graph. // The trait LanguageFrontend is mixed in and not property/field of this class in order // to allow the configuration of language frontend specific properties on the CPG object. -abstract class TestCpg extends Cpg() with LanguageFrontend with TestCodeWriter { +abstract class TestCpg extends Cpg(Cpg.empty.graph) with LanguageFrontend with TestCodeWriter { private var _graph = Option.empty[Graph] protected var _withPostProcessing = false diff --git a/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/utils/KeyPoolTests.scala b/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/utils/KeyPoolTests.scala new file mode 100644 index 000000000000..4815c4827332 --- /dev/null +++ b/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/utils/KeyPoolTests.scala @@ -0,0 +1,74 @@ +package io.joern.x2cpg.utils + +import org.scalatest.matchers.should.Matchers +import org.scalatest.wordspec.AnyWordSpec + +class KeyPoolTests extends AnyWordSpec with Matchers { + + "IntervalKeyPool" should { + "return [first, ..., last] and then raise" in { + val keyPool = new IntervalKeyPool(10, 19) + List.range(0, 10).map(_ => keyPool.next) shouldBe List.range(10, 20) + assertThrows[RuntimeException] { keyPool.next } + assertThrows[RuntimeException] { keyPool.next } + } + + "allow splitting into multiple pools" in { + val keyPool = new IntervalKeyPool(1, 1000) + val pools = keyPool.split(11).toList + assertThrows[IllegalStateException] { keyPool.next } + pools.size shouldBe 11 + // Pools should all have the same size + pools + .map { x => + (x.last - x.first) + } + .distinct + .size shouldBe 1 + // Pools should be pairwise disjoint + val keySets = pools.map { x => + (x.first to x.last).toSet + } + keySets.combinations(2).foreach { + case List(x: Set[Long], y: Set[Long]) => + x.intersect(y).isEmpty shouldBe true + case _ => + fail() + } + } + + "return empty iterator when asked to create 0 partitions" in { + val keyPool = new IntervalKeyPool(1, 1000) + keyPool.split(0).hasNext shouldBe false + } + + } + + "SequenceKeyPool" should { + "return elements of sequence one by one and then raise" in { + val seq = List[Long](1, 2, 3) + val keyPool = new SequenceKeyPool(seq) + List.range(0, 3).map(_ => keyPool.next) shouldBe seq + assertThrows[RuntimeException] { keyPool.next } + assertThrows[RuntimeException] { keyPool.next } + } + } + + "KeyPoolCreator" should { + "split into n pools and honor minimum value" in { + val minValue = 10 + val pools = KeyPoolCreator.obtain(3, minValue) + pools.size shouldBe 3 + pools match { + case List(pool1, pool2, pool3) => + pool1.first shouldBe minValue + pool1.last should be < pool2.first + pool2.last should be < pool3.first + pool3.last shouldBe Long.MaxValue - 1 + case _ => fail() + } + } + + } + +} diff --git a/joern-cli/src/main/scala/io/joern/joerncli/CpgBasedTool.scala b/joern-cli/src/main/scala/io/joern/joerncli/CpgBasedTool.scala index ba82da57fed9..274920f8a7d4 100644 --- a/joern-cli/src/main/scala/io/joern/joerncli/CpgBasedTool.scala +++ b/joern-cli/src/main/scala/io/joern/joerncli/CpgBasedTool.scala @@ -4,22 +4,23 @@ import better.files.File import io.joern.dataflowengineoss.layers.dataflows.{OssDataFlow, OssDataFlowOptions} import io.joern.dataflowengineoss.semanticsloader.Semantics import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.codepropertygraph.cpgloading.CpgLoaderConfig import io.shiftleft.semanticcpg.layers.LayerCreatorContext import io.shiftleft.semanticcpg.language.* +import io.shiftleft.codepropertygraph.cpgloading.CpgLoader object CpgBasedTool { + def loadFromFile(filename: String): Cpg = + CpgLoader.load(filename) + /** Load code property graph from overflowDB * * @param filename * name of the file that stores the CPG */ - def loadFromOdb(filename: String): Cpg = { - val odbConfig = overflowdb.Config.withDefaults().withStorageLocation(filename) - val config = CpgLoaderConfig().withOverflowConfig(odbConfig).doNotCreateIndexesOnLoad - io.shiftleft.codepropertygraph.cpgloading.CpgLoader.loadFromOverflowDb(config) - } + @deprecated("use `loadFromFile` instead", "joern v3") + def loadFromOdb(filename: String): Cpg = + loadFromFile(filename) /** Add the data flow layer to the CPG if it does not exist yet. */ diff --git a/joern-cli/src/main/scala/io/joern/joerncli/DefaultOverlays.scala b/joern-cli/src/main/scala/io/joern/joerncli/DefaultOverlays.scala index 920bb131f49b..3adc182a4eb3 100644 --- a/joern-cli/src/main/scala/io/joern/joerncli/DefaultOverlays.scala +++ b/joern-cli/src/main/scala/io/joern/joerncli/DefaultOverlays.scala @@ -16,7 +16,7 @@ object DefaultOverlays { * the filename of the cpg */ def create(storeFilename: String, maxNumberOfDefinitions: Int = defaultMaxNumberOfDefinitions): Cpg = { - val cpg = CpgBasedTool.loadFromOdb(storeFilename) + val cpg = CpgBasedTool.loadFromFile(storeFilename) applyDefaultOverlays(cpg) val context = new LayerCreatorContext(cpg) val options = new OssDataFlowOptions(maxNumberOfDefinitions) diff --git a/joern-cli/src/main/scala/io/joern/joerncli/JoernExport.scala b/joern-cli/src/main/scala/io/joern/joerncli/JoernExport.scala index 6b0b78612602..241893ff5a9a 100644 --- a/joern-cli/src/main/scala/io/joern/joerncli/JoernExport.scala +++ b/joern-cli/src/main/scala/io/joern/joerncli/JoernExport.scala @@ -2,6 +2,12 @@ package io.joern.joerncli import better.files.Dsl.* import better.files.File +import flatgraph.{Accessors, Edge, GNode} +import flatgraph.formats.ExportResult +import flatgraph.formats.dot.DotExporter +import flatgraph.formats.graphml.GraphMLExporter +import flatgraph.formats.graphson.GraphSONExporter +import flatgraph.formats.neo4jcsv.Neo4jCsvExporter import io.joern.dataflowengineoss.DefaultSemantics import io.joern.dataflowengineoss.layers.dataflows.* import io.joern.dataflowengineoss.semanticsloader.Semantics @@ -9,14 +15,8 @@ import io.joern.joerncli.CpgBasedTool.exitIfInvalid import io.joern.x2cpg.layers.* import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.NodeTypes -import io.shiftleft.semanticcpg.language.{toAstNodeMethods, toNodeTypeStarters} +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.layers.* -import overflowdb.formats.ExportResult -import overflowdb.formats.dot.DotExporter -import overflowdb.formats.graphml.GraphMLExporter -import overflowdb.formats.graphson.GraphSONExporter -import overflowdb.formats.neo4jcsv.Neo4jCsvExporter -import overflowdb.{Edge, Node} import java.nio.file.{Path, Paths} import scala.collection.mutable @@ -64,7 +64,7 @@ object JoernExport { exitIfInvalid(outDir, config.cpgFileName) mkdir(File(outDir)) - Using.resource(CpgBasedTool.loadFromOdb(config.cpgFileName)) { cpg => + Using.resource(CpgBasedTool.loadFromFile(config.cpgFileName)) { cpg => exportCpg(cpg, config.repr, config.format, Paths.get(outDir).toAbsolutePath) } } @@ -105,15 +105,15 @@ object JoernExport { format match { case Format.Dot if representation == Representation.All || representation == Representation.Cpg => - exportWithOdbFormat(cpg, representation, outDir, DotExporter) + exportWithFlatgraphFormat(cpg, representation, outDir, DotExporter) case Format.Dot => exportDot(representation, outDir, context) case Format.Neo4jCsv => - exportWithOdbFormat(cpg, representation, outDir, Neo4jCsvExporter) + exportWithFlatgraphFormat(cpg, representation, outDir, Neo4jCsvExporter) case Format.Graphml => - exportWithOdbFormat(cpg, representation, outDir, GraphMLExporter) + exportWithFlatgraphFormat(cpg, representation, outDir, GraphMLExporter) case Format.Graphson => - exportWithOdbFormat(cpg, representation, outDir, GraphSONExporter) + exportWithFlatgraphFormat(cpg, representation, outDir, GraphSONExporter) case other => throw new NotImplementedError(s"repr=$representation not yet supported for format=$format") } @@ -133,11 +133,11 @@ object JoernExport { } } - private def exportWithOdbFormat( + private def exportWithFlatgraphFormat( cpg: Cpg, repr: Representation.Value, outDir: Path, - exporter: overflowdb.formats.Exporter + exporter: flatgraph.formats.Exporter ): Unit = { val ExportResult(nodeCount, edgeCount, _, additionalInfo) = repr match { case Representation.All => @@ -154,7 +154,7 @@ object JoernExport { windowsFilenameDeduplicationHelper ) val outFileName = outDir.resolve(relativeFilename) - exporter.runExport(nodes, subGraph.edges, outFileName) + exporter.runExport(cpg.graph.schema, nodes, subGraph.edges, outFileName) } .reduce(plus) } else { @@ -220,12 +220,12 @@ object JoernExport { private def emptyExportResult = ExportResult(0, 0, Seq.empty, Option("Empty CPG")) - case class MethodSubGraph(methodName: String, methodFilename: String, nodes: Set[Node]) { + case class MethodSubGraph(methodName: String, methodFilename: String, nodes: Set[GNode]) { def edges: Set[Edge] = { for { node <- nodes - edge <- node.bothE.asScala - if nodes.contains(edge.inNode) && nodes.contains(edge.outNode) + edge <- Accessors.getEdgesOut(node) + if nodes.contains(edge.dst) } yield edge } } diff --git a/joern-cli/src/main/scala/io/joern/joerncli/JoernFlow.scala b/joern-cli/src/main/scala/io/joern/joerncli/JoernFlow.scala index 07cf60a4ef28..211bcd1ec0b4 100644 --- a/joern-cli/src/main/scala/io/joern/joerncli/JoernFlow.scala +++ b/joern-cli/src/main/scala/io/joern/joerncli/JoernFlow.scala @@ -28,7 +28,7 @@ object JoernFlow { } debugOut("Loading graph... ") - val cpg = CpgBasedTool.loadFromOdb(config.cpgFileName) + val cpg = CpgBasedTool.loadFromFile(config.cpgFileName) debugOut("[DONE]\n") implicit val resolver: ICallResolver = NoResolve diff --git a/joern-cli/src/main/scala/io/joern/joerncli/JoernSlice.scala b/joern-cli/src/main/scala/io/joern/joerncli/JoernSlice.scala index c16f09eb428a..453d370eed5b 100644 --- a/joern-cli/src/main/scala/io/joern/joerncli/JoernSlice.scala +++ b/joern-cli/src/main/scala/io/joern/joerncli/JoernSlice.scala @@ -122,7 +122,7 @@ object JoernSlice { } else { config.inputPath.pathAsString } - Using.resource(CpgBasedTool.loadFromOdb(inputCpgPath)) { cpg => + Using.resource(CpgBasedTool.loadFromFile(inputCpgPath)) { cpg => checkAndApplyOverlays(cpg) // Slice the CPG (config match { diff --git a/joern-cli/src/main/scala/io/joern/joerncli/JoernVectors.scala b/joern-cli/src/main/scala/io/joern/joerncli/JoernVectors.scala index 57ba6673bcc0..d32bceb28f33 100644 --- a/joern-cli/src/main/scala/io/joern/joerncli/JoernVectors.scala +++ b/joern-cli/src/main/scala/io/joern/joerncli/JoernVectors.scala @@ -15,11 +15,10 @@ import scala.util.hashing.MurmurHash3 class BagOfPropertiesForNodes extends EmbeddingGenerator[AstNode, (String, String)] { override def structureToString(pair: (String, String)): String = pair._1 + ":" + pair._2 - override def extractObjects(cpg: Cpg): Iterator[AstNode] = cpg.graph.V.collect { case x: AstNode => x } + override def extractObjects(cpg: Cpg): Iterator[AstNode] = cpg.astNode override def enumerateSubStructures(obj: AstNode): List[(String, String)] = { val relevantFieldTypes = Set(PropertyNames.NAME, PropertyNames.FULL_NAME, PropertyNames.CODE) - val relevantFields = obj - .propertiesMap() + val relevantFields = obj.propertiesMap .entrySet() .asScala .toList @@ -136,7 +135,7 @@ object JoernVectors { def main(args: Array[String]) = { parseConfig(args).foreach { config => exitIfInvalid(config.outDir, config.cpgFileName) - Using.resource(CpgBasedTool.loadFromOdb(config.cpgFileName)) { cpg => + Using.resource(CpgBasedTool.loadFromFile(config.cpgFileName)) { cpg => val generator = new BagOfPropertiesForNodes() val embedding = generator.embed(cpg) println("{") @@ -150,8 +149,8 @@ object JoernVectors { traversalToJson(embedding.vectors, generator.vectorToString) println(",\"edges\":") traversalToJson( - cpg.graph.edges().map { x => - Map("src" -> x.outNode().id(), "dst" -> x.inNode().id(), "label" -> x.label()) + cpg.graph.allEdges.map { edge => + Map("src" -> edge.src.id, "dst" -> edge.dst.id, "label" -> edge.label) }, generator.defaultToString ) diff --git a/joern-cli/src/main/scala/io/joern/joerncli/console/Predefined.scala b/joern-cli/src/main/scala/io/joern/joerncli/console/Predefined.scala index f3107c8ff907..1a263145f0e3 100644 --- a/joern-cli/src/main/scala/io/joern/joerncli/console/Predefined.scala +++ b/joern-cli/src/main/scala/io/joern/joerncli/console/Predefined.scala @@ -6,26 +6,21 @@ object Predefined { val shared: Seq[String] = Seq( - "import _root_.io.joern.console._", - "import _root_.io.joern.joerncli.console.JoernConsole._", - "import _root_.io.shiftleft.codepropertygraph.Cpg.docSearchPackages", - "import _root_.io.shiftleft.codepropertygraph.generated.Cpg", - "import _root_.io.shiftleft.codepropertygraph.cpgloading._", - "import _root_.io.shiftleft.codepropertygraph.generated._", - "import _root_.io.shiftleft.codepropertygraph.generated.nodes._", - "import _root_.io.shiftleft.codepropertygraph.generated.edges._", - "import _root_.io.joern.dataflowengineoss.language._", - "import _root_.io.shiftleft.semanticcpg.language._", - "import overflowdb._", - "import overflowdb.traversal.{`package` => _, help => _, _}", - "import scala.jdk.CollectionConverters._", + "import _root_.io.joern.console.*", + "import _root_.io.joern.joerncli.console.JoernConsole.*", + "import _root_.io.shiftleft.codepropertygraph.cpgloading.*", + "import _root_.io.shiftleft.codepropertygraph.generated.*", + "import _root_.io.shiftleft.codepropertygraph.generated.nodes.*", + "import _root_.io.joern.dataflowengineoss.language.*", + "import _root_.io.shiftleft.semanticcpg.language.*", + "import scala.jdk.CollectionConverters.*", "implicit val resolver: ICallResolver = NoResolve", "implicit val finder: NodeExtensionFinder = DefaultNodeExtensionFinder" ) val forInteractiveShell: Seq[String] = { shared ++ - Seq("import _root_.io.joern.joerncli.console.Joern._") ++ + Seq("import _root_.io.joern.joerncli.console.Joern.*") ++ Run.codeForRunCommand().linesIterator ++ Help.codeForHelpCommand(classOf[io.joern.joerncli.console.JoernConsole]).linesIterator ++ Seq("ossDataFlowOptions = opts.ossdataflow") diff --git a/joern-cli/src/universal/schema-extender/build.sbt b/joern-cli/src/universal/schema-extender/build.sbt index 7632c73b0f74..4ce6bc842612 100644 --- a/joern-cli/src/universal/schema-extender/build.sbt +++ b/joern-cli/src/universal/schema-extender/build.sbt @@ -1,10 +1,10 @@ name := "schema-extender" -ThisBuild / scalaVersion := "3.4.1" +ThisBuild / scalaVersion := "3.4.2" val cpgVersion = IO.read(file("cpg-version")) -val generateDomainClasses = taskKey[Seq[File]]("generate overflowdb domain classes for our schema") +val generateDomainClasses = taskKey[Seq[File]]("generate domain classes for our schema") val joernInstallPath = settingKey[String]("path to joern installation, e.g. `/home/username/bin/joern/joern-cli` or `../../joern/joern-cli`") @@ -26,16 +26,16 @@ replaceDomainClassesInJoern := { } ThisBuild / libraryDependencies ++= Seq( - "io.shiftleft" %% "codepropertygraph-schema" % cpgVersion, - "io.shiftleft" %% "codepropertygraph-domain-classes" % cpgVersion + "com.michaelpollmeier" %% "codepropertygraph-schema" % cpgVersion, + "com.michaelpollmeier" %% "codepropertygraph-domain-classes" % cpgVersion ) lazy val schema = project .in(file("schema")) .settings(generateDomainClasses := { - val outputRoot = target.value / "odb-codegen" + val outputRoot = target.value / "fg-codegen" FileUtils.deleteRecursively(outputRoot) - val invoked = (Compile / runMain).toTask(s" CpgExtCodegen schema/target/odb-codegen").value + val invoked = (Compile / runMain).toTask(s" CpgExtCodegen schema/target/fg-codegen").value FileUtils.listFilesRecursively(outputRoot) }) diff --git a/joern-cli/src/universal/schema-extender/schema/src/main/scala/CpgExtCodegen.scala b/joern-cli/src/universal/schema-extender/schema/src/main/scala/CpgExtCodegen.scala index 2b8cba9f6fd2..5bdd11cdac44 100644 --- a/joern-cli/src/universal/schema-extender/schema/src/main/scala/CpgExtCodegen.scala +++ b/joern-cli/src/universal/schema-extender/schema/src/main/scala/CpgExtCodegen.scala @@ -1,14 +1,13 @@ import io.shiftleft.codepropertygraph.schema.* -import overflowdb.codegen.CodeGen -import overflowdb.schema.SchemaBuilder -import overflowdb.schema.Property.ValueType - -import java.io.File +import flatgraph.codegen.DomainClassesGenerator +import flatgraph.schema.SchemaBuilder +import flatgraph.schema.Property.ValueType +import java.nio.file.Paths object CpgExtCodegen { def main(args: Array[String]): Unit = { val outputDir = args.headOption - .map(new File(_)) + .map(Paths.get(_)) .getOrElse(throw new AssertionError("please pass outputDir as first parameter")) val builder = new SchemaBuilder(domainShortName = "Cpg", basePackage = "io.shiftleft.codepropertygraph.generated") @@ -24,6 +23,6 @@ object CpgExtCodegen { cpgSchema.fs.file.addProperties(exampleProperty) // END extensions for this build - new CodeGen(builder.build).run(outputDir) + new DomainClassesGenerator(builder.build).run(outputDir) } } diff --git a/joern-cli/src/universal/schema-extender/test.sh b/joern-cli/src/universal/schema-extender/test.sh index 309ab80b710a..4c3fba8ac3b6 100755 --- a/joern-cli/src/universal/schema-extender/test.sh +++ b/joern-cli/src/universal/schema-extender/test.sh @@ -9,6 +9,6 @@ set -x #verbose on # we should now be able to use our new `EXAMPLE_NODE` node mkdir -p scripts echo 'assert(nodes.ExampleNode.Label == "EXAMPLE_NODE") -assert(nodes.ExampleNode.PropertyNames.all.contains("EXAMPLE_PROPERTY"))' > scripts/SchemaExtenderTest.sc +assert(nodes.ExampleNode.PropertyNames.ExampleProperty == "EXAMPLE_PROPERTY")' > scripts/SchemaExtenderTest.sc ./joern --script scripts/SchemaExtenderTest.sc diff --git a/macros/build.sbt b/macros/build.sbt index d54e400c5e88..c09dc4f8f720 100644 --- a/macros/build.sbt +++ b/macros/build.sbt @@ -3,8 +3,9 @@ name := "macros" dependsOn(Projects.semanticcpg % Test) libraryDependencies ++= Seq( - "io.shiftleft" %% "codepropertygraph" % Versions.cpg, - "org.scalatest" %% "scalatest" % Versions.scalatest % Test + "com.michaelpollmeier" %% "codepropertygraph" % Versions.cpg, + "net.oneandone.reflections8" % "reflections8" % "0.11.7", + "org.scalatest" %% "scalatest" % Versions.scalatest % Test ) enablePlugins(JavaAppPackaging) diff --git a/querydb/src/test/scala/io/joern/scanners/c/HeapBasedOverflowTests.scala b/querydb/src/test/scala/io/joern/scanners/c/HeapBasedOverflowTests.scala index b28669fd07f0..e3bedcba6b1e 100644 --- a/querydb/src/test/scala/io/joern/scanners/c/HeapBasedOverflowTests.scala +++ b/querydb/src/test/scala/io/joern/scanners/c/HeapBasedOverflowTests.scala @@ -3,6 +3,7 @@ package io.joern.scanners.c import io.joern.suites.CQueryTestSuite import io.shiftleft.codepropertygraph.generated.nodes import io.joern.console.scan.* +import io.shiftleft.semanticcpg.language.* class HeapBasedOverflowTests extends CQueryTestSuite(HeapBasedOverflow) { diff --git a/querydb/src/test/scala/io/joern/scanners/c/MetricsTests.scala b/querydb/src/test/scala/io/joern/scanners/c/MetricsTests.scala index 1815c24ba2e0..a741cea32590 100644 --- a/querydb/src/test/scala/io/joern/scanners/c/MetricsTests.scala +++ b/querydb/src/test/scala/io/joern/scanners/c/MetricsTests.scala @@ -3,6 +3,7 @@ package io.joern.scanners.c import io.joern.suites.CQueryTestSuite import io.shiftleft.codepropertygraph.generated.nodes import io.joern.console.scan.* +import io.shiftleft.semanticcpg.language.* class MetricsTests extends CQueryTestSuite(Metrics) { diff --git a/querydb/src/test/scala/io/joern/suites/JavaQueryTestSuite.scala b/querydb/src/test/scala/io/joern/suites/JavaQueryTestSuite.scala index 141a3969ade1..00711ed73b9b 100644 --- a/querydb/src/test/scala/io/joern/suites/JavaQueryTestSuite.scala +++ b/querydb/src/test/scala/io/joern/suites/JavaQueryTestSuite.scala @@ -7,6 +7,7 @@ import io.joern.util.QueryUtil import io.joern.x2cpg.testfixtures.TestCpg import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.{Call, Literal, Method, StoredNode} +import io.shiftleft.semanticcpg.language.* class JavaQueryTestSuite[QB <: QueryBundle](val queryBundle: QB) extends JavaSrcCode2CpgFixture(withOssDataflow = true) { diff --git a/querydb/src/test/scala/io/joern/suites/KotlinQueryTestSuite.scala b/querydb/src/test/scala/io/joern/suites/KotlinQueryTestSuite.scala index 1c2419156035..469c1517b919 100644 --- a/querydb/src/test/scala/io/joern/suites/KotlinQueryTestSuite.scala +++ b/querydb/src/test/scala/io/joern/suites/KotlinQueryTestSuite.scala @@ -6,6 +6,7 @@ import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture import io.joern.x2cpg.testfixtures.TestCpg import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.{Call, Method} +import io.shiftleft.semanticcpg.language.* import io.joern.console.scan.* import io.shiftleft.utils.ProjectRoot diff --git a/semanticcpg/build.sbt b/semanticcpg/build.sbt index ea2590fc0a0d..7e4a6ee34d07 100644 --- a/semanticcpg/build.sbt +++ b/semanticcpg/build.sbt @@ -1,11 +1,15 @@ name := "semanticcpg" libraryDependencies ++= Seq( - "io.shiftleft" %% "codepropertygraph" % Versions.cpg, - "com.michaelpollmeier" %% "scala-repl-pp" % Versions.scalaReplPP, - "org.json4s" %% "json4s-native" % Versions.json4s, - "org.apache.commons" % "commons-text" % Versions.commonsText, - "org.scalatest" %% "scalatest" % Versions.scalatest % Test + "com.michaelpollmeier" %% "codepropertygraph" % Versions.cpg, + "com.michaelpollmeier" %% "scala-repl-pp" % Versions.scalaReplPP, + "org.json4s" %% "json4s-native" % Versions.json4s, + "org.scala-lang.modules" %% "scala-xml" % "2.2.0", + "org.apache.commons" % "commons-text" % Versions.commonsText, + "org.scalatest" %% "scalatest" % Versions.scalatest % Test ) -Compile / doc / scalacOptions ++= Seq("-doc-title", "semanticcpg apidocs", "-doc-version", version.value) +Compile/compile/scalacOptions ++= Seq( + // TODO remove + "-Wconf:any:silent", // silence warnings for now +) diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/Overlays.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/Overlays.scala index 7a62e226f138..c45bb02d3e43 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/Overlays.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/Overlays.scala @@ -4,6 +4,7 @@ import io.shiftleft.codepropertygraph.generated.{Cpg, DiffGraphBuilder} import io.shiftleft.codepropertygraph.generated.Properties import io.shiftleft.passes.CpgPass import io.shiftleft.semanticcpg.language.* +import flatgraph.DiffGraphBuilder object Overlays { diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/accesspath/TrackedBase.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/accesspath/TrackedBase.scala index 9172fb77b960..2f6ff9fd4c27 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/accesspath/TrackedBase.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/accesspath/TrackedBase.scala @@ -1,6 +1,7 @@ package io.shiftleft.semanticcpg.accesspath import io.shiftleft.codepropertygraph.generated.nodes.* +import io.shiftleft.semanticcpg.language.* trait TrackedBase case class TrackedNamedVariable(name: String) extends TrackedBase diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/dotgenerator/CdgGenerator.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/dotgenerator/CdgGenerator.scala index 9223d7d98218..8f396a6af39c 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/dotgenerator/CdgGenerator.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/dotgenerator/CdgGenerator.scala @@ -11,6 +11,6 @@ class CdgGenerator extends CfgGenerator { override val edgeType: String = EdgeTypes.CDG override def expand(v: StoredNode): Iterator[Edge] = - v._cdgOut.map(node => Edge(v, node, edgeType = edgeType)) + v._cdgOut.iterator.map(node => Edge(v, node, edgeType = edgeType)) } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/dotgenerator/CfgGenerator.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/dotgenerator/CfgGenerator.scala index f7bb2f1abea8..067ce1302792 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/dotgenerator/CfgGenerator.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/dotgenerator/CfgGenerator.scala @@ -4,7 +4,6 @@ import io.shiftleft.codepropertygraph.generated.EdgeTypes import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.semanticcpg.dotgenerator.DotSerializer.{Edge, Graph} import io.shiftleft.semanticcpg.language.* -import overflowdb.Node class CfgGenerator { @@ -44,12 +43,12 @@ class CfgGenerator { protected def expand(v: StoredNode): Iterator[Edge] = v._cfgOut.map(node => Edge(v, node, edgeType = edgeType)) - private def isConditionInControlStructure(v: Node): Boolean = v match { + private def isConditionInControlStructure(v: StoredNode): Boolean = v match { case id: Identifier => id.astParent.isControlStructure case _ => false } - private def cfgNodeShouldBeDisplayed(v: Node): Boolean = + private def cfgNodeShouldBeDisplayed(v: StoredNode): Boolean = isConditionInControlStructure(v) || !(v.isInstanceOf[Literal] || v.isInstanceOf[Identifier] || diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/dotgenerator/DotSerializer.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/dotgenerator/DotSerializer.scala index 6ee27691b66f..a7429d8e86b2 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/dotgenerator/DotSerializer.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/dotgenerator/DotSerializer.scala @@ -1,5 +1,6 @@ package io.shiftleft.semanticcpg.dotgenerator +import flatgraph.Accessors import io.shiftleft.codepropertygraph.generated.PropertyNames import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.semanticcpg.language.* @@ -70,7 +71,10 @@ object DotSerializer { } private def stringRepr(vertex: StoredNode): String = { - val maybeLineNo: Optional[AnyRef] = vertex.propertyOption(PropertyNames.LINE_NUMBER) + // TODO MP after the initial flatgraph migration (where we want to maintain semantics as far as + // possible) this might become `vertex.property(Properties.LineNumber)` which derives to `Option[Int]` + val lineNoMaybe = vertex.propertyOption[Int](PropertyNames.LINE_NUMBER) + StringEscapeUtils.escapeHtml4(vertex match { case call: Call => (call.name, limit(call.code)).toString case contrl: ControlStructure => (contrl.label, contrl.controlStructureType, contrl.code).toString @@ -87,7 +91,7 @@ object DotSerializer { case typeDecl: TypeDecl => (typeDecl.label, typeDecl.name).toString() case member: Member => (member.label, member.name).toString() case _ => "" - }) + (if (maybeLineNo.isPresent) s"${maybeLineNo.get()}" else "") + }) + lineNoMaybe.map(lineNo => s"$lineNo").getOrElse("") } private def toCfgNode(node: StoredNode): CfgNode = { diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/AccessPathHandling.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/AccessPathHandling.scala index 131c904f1bfc..6292879b1b0d 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/AccessPathHandling.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/AccessPathHandling.scala @@ -1,8 +1,9 @@ package io.shiftleft.semanticcpg.language -import io.shiftleft.codepropertygraph.generated.{Operators, Properties, PropertyNames} +import io.shiftleft.codepropertygraph.generated.{Operators, Properties} import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.semanticcpg.accesspath.* +import io.shiftleft.semanticcpg.language.* import org.slf4j.LoggerFactory import scala.jdk.CollectionConverters.IteratorHasAsScala @@ -42,8 +43,9 @@ object AccessPathHandling { .collect { case node: Literal => ConstantAccess(node.code) case node: Identifier => ConstantAccess(node.name) - case other if other.propertyOption(PropertyNames.NAME).isPresent => - logger.warn(s"unexpected/deprecated node encountered: $other with properties: ${other.propertiesMap()}") + case other if other.propertyOption(Properties.Name).isDefined => + val properties = other.propertiesMap + logger.warn(s"unexpected/deprecated node encountered: $other with properties: $properties") ConstantAccess(other.property(Properties.Name)) } .getOrElse(VariableAccess) :: tail diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/LocationCreator.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/LocationCreator.scala index b233ad2e0abb..008e87131dbb 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/LocationCreator.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/LocationCreator.scala @@ -1,8 +1,8 @@ package io.shiftleft.semanticcpg.language import io.shiftleft.codepropertygraph.generated.nodes.* +import io.shiftleft.semanticcpg.language.* import org.slf4j.{Logger, LoggerFactory} -import overflowdb.traversal.* import scala.annotation.tailrec @@ -64,7 +64,7 @@ object LocationCreator { @tailrec private def findVertex(node: StoredNode, instanceCheck: StoredNode => Boolean): Option[StoredNode] = - node._astIn.nextOption() match { + node._astIn.iterator.nextOption() match { case Some(head) if instanceCheck(head) => Some(head) case Some(head) => findVertex(head, instanceCheck) case None => None diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/NodeExtensionFinder.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/NodeExtensionFinder.scala index 6c7c4024c26b..f0712ce1f87d 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/NodeExtensionFinder.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/NodeExtensionFinder.scala @@ -1,29 +1,8 @@ package io.shiftleft.semanticcpg.language -import io.shiftleft.codepropertygraph.generated.nodes.{ - Call, - Identifier, - Literal, - Local, - Method, - MethodParameterIn, - MethodParameterOut, - MethodRef, - MethodReturn, - StoredNode -} +import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.semanticcpg.NodeExtension -import io.shiftleft.semanticcpg.language.nodemethods.{ - CallMethods, - IdentifierMethods, - LiteralMethods, - LocalMethods, - MethodMethods, - MethodParameterInMethods, - MethodParameterOutMethods, - MethodRefMethods, - MethodReturnMethods -} +import io.shiftleft.semanticcpg.language.nodemethods.* trait NodeExtensionFinder { def apply(n: StoredNode): Option[NodeExtension] diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/NodeSteps.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/NodeSteps.scala index 000884a91267..337714e94255 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/NodeSteps.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/NodeSteps.scala @@ -4,15 +4,13 @@ import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.codepropertygraph.generated.{EdgeTypes, NodeTypes} import io.shiftleft.semanticcpg.codedumper.CodeDumper -import overflowdb.Node -import overflowdb.traversal.* -import io.shiftleft.codepropertygraph.generated.help.Doc +import io.shiftleft.codepropertygraph.generated.help.{Doc, Traversal} /** Steps for all node types * * This is the base class for all steps defined on */ -@help.Traversal(elementType = classOf[StoredNode]) +@Traversal(elementType = classOf[StoredNode]) class NodeSteps[NodeType <: StoredNode](val traversal: Iterator[NodeType]) extends AnyVal { @Doc( @@ -23,15 +21,16 @@ class NodeSteps[NodeType <: StoredNode](val traversal: Iterator[NodeType]) exten |the file node that represents that source file. |""" ) - def file: Iterator[File] = - traversal - .choose(_.label) { - case NodeTypes.NAMESPACE => _.in(EdgeTypes.REF).out(EdgeTypes.SOURCE_FILE) - case NodeTypes.COMMENT => _.in(EdgeTypes.AST).hasLabel(NodeTypes.FILE) - case _ => - _.repeat(_.coalesce(_.out(EdgeTypes.SOURCE_FILE), _.in(EdgeTypes.AST)))(_.until(_.hasLabel(NodeTypes.FILE))) - } - .cast[File] + def file: Iterator[File] = { + traversal.flatMap { + case namespace: Namespace => + namespace.refIn.sourceFileOut + case comment: Comment => + comment.astIn + case node => + Iterator(node).repeat(_.coalesce(_._sourceFileOut, _._astIn))(_.until(_.hasLabel(File.Label))).cast[File] + } + } @Doc( info = "Location, including filename and line number", @@ -84,11 +83,6 @@ class NodeSteps[NodeType <: StoredNode](val traversal: Iterator[NodeType]) exten }.l } - /* follow the incoming edges of the given type as long as possible */ - protected def walkIn(edgeType: String): Iterator[Node] = - traversal - .repeat(_.in(edgeType))(_.until(_.in(edgeType).countTrav.filter(_ == 0))) - @Doc( info = "Tag node with `tagName`", longInfo = """ diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/NodeTypeStarters.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/NodeTypeStarters.scala index 89f6dab3532d..4f45f3f3bea3 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/NodeTypeStarters.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/NodeTypeStarters.scala @@ -3,316 +3,90 @@ package io.shiftleft.semanticcpg.language import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.codepropertygraph.generated.{NodeTypes, Properties} -import overflowdb.* -import overflowdb.traversal.help -import io.shiftleft.codepropertygraph.generated.help.Doc -import overflowdb.traversal.{InitialTraversal, TraversalSource} +import io.shiftleft.codepropertygraph.generated.help.{Doc, TraversalSource} +import io.shiftleft.semanticcpg.language.* -import scala.jdk.CollectionConverters.IteratorHasAsScala +/** Starting point for a new traversal, e.g. + * - `cpg.method`, `cpg.call` etc. - these are generated by the flatgraph codegenerator and automatically inherited + * - `cpg.method.name` + */ +@TraversalSource +class NodeTypeStarters(cpg: Cpg) { -@help.TraversalSource -class NodeTypeStarters(cpg: Cpg) extends TraversalSource(cpg.graph) { - - /** Traverse to all nodes. - */ - @Doc(info = "All nodes of the graph") - override def all: Traversal[StoredNode] = - cpg.graph.nodes.asScala.cast[StoredNode] - - /** Traverse to all annotations - */ - def annotation: Traversal[Annotation] = - InitialTraversal.from[Annotation](cpg.graph, NodeTypes.ANNOTATION) - - /** Traverse to all arguments passed to methods - */ + /** Traverse to all arguments passed to methods */ @Doc(info = "All arguments (actual parameters)") - def argument: Traversal[Expression] = - call.argument + def argument: Iterator[Expression] = + cpg.call.argument - /** Shorthand for `cpg.argument.code(code)` - */ - def argument(code: String): Traversal[Expression] = - argument.code(code) + /** Shorthand for `cpg.argument.code(code)` */ + def argument(code: String): Iterator[Expression] = + cpg.argument.code(code) @Doc(info = "All breaks (`ControlStructure` nodes)") - def break: Traversal[ControlStructure] = - controlStructure.isBreak - - /** Traverse to all call sites - */ - @Doc(info = "All call sites") - def call: Traversal[Call] = - InitialTraversal.from[Call](cpg.graph, NodeTypes.CALL) - - /** Shorthand for `cpg.call.name(name)` - */ - def call(name: String): Traversal[Call] = - call.name(name) - - /** Traverse to all comments in source-based CPGs. - */ - @Doc(info = "All comments in source-based CPGs") - def comment: Traversal[Comment] = - InitialTraversal.from[Comment](cpg.graph, NodeTypes.COMMENT) - - /** Shorthand for `cpg.comment.code(code)` - */ - def comment(code: String): Traversal[Comment] = - comment.has(Properties.Code -> code) - - /** Traverse to all config files - */ - @Doc(info = "All config files") - def configFile: Traversal[ConfigFile] = - InitialTraversal.from[ConfigFile](cpg.graph, NodeTypes.CONFIG_FILE) - - /** Shorthand for `cpg.configFile.name(name)` - */ - def configFile(name: String): Traversal[ConfigFile] = - configFile.name(name) - - /** Traverse to all dependencies - */ - @Doc(info = "All dependencies") - def dependency: Traversal[Dependency] = - InitialTraversal.from[Dependency](cpg.graph, NodeTypes.DEPENDENCY) - - /** Shorthand for `cpg.dependency.name(name)` - */ - def dependency(name: String): Traversal[Dependency] = - dependency.name(name) - - @Doc(info = "All control structures (source-based frontends)") - def controlStructure: Traversal[ControlStructure] = - InitialTraversal.from[ControlStructure](cpg.graph, NodeTypes.CONTROL_STRUCTURE) + def break: Iterator[ControlStructure] = + cpg.controlStructure.isBreak @Doc(info = "All continues (`ControlStructure` nodes)") - def continue: Traversal[ControlStructure] = - controlStructure.isContinue + def continue: Iterator[ControlStructure] = + cpg.controlStructure.isContinue @Doc(info = "All do blocks (`ControlStructure` nodes)") - def doBlock: Traversal[ControlStructure] = - controlStructure.isDo + def doBlock: Iterator[ControlStructure] = + cpg.controlStructure.isDo @Doc(info = "All else blocks (`ControlStructure` nodes)") - def elseBlock: Traversal[ControlStructure] = - controlStructure.isElse + def elseBlock: Iterator[ControlStructure] = + cpg.controlStructure.isElse @Doc(info = "All throws (`ControlStructure` nodes)") - def throws: Traversal[ControlStructure] = - controlStructure.isThrow - - /** Traverse to all source files - */ - @Doc(info = "All source files") - def file: Traversal[File] = - InitialTraversal.from[File](cpg.graph, NodeTypes.FILE) - - /** Shorthand for `cpg.file.name(name)` - */ - def file(name: String): Traversal[File] = - file.name(name) + def throws: Iterator[ControlStructure] = + cpg.controlStructure.isThrow @Doc(info = "All for blocks (`ControlStructure` nodes)") - def forBlock: Traversal[ControlStructure] = - controlStructure.isFor + def forBlock: Iterator[ControlStructure] = + cpg.controlStructure.isFor @Doc(info = "All gotos (`ControlStructure` nodes)") - def goto: Traversal[ControlStructure] = - controlStructure.isGoto - - /** Traverse to all identifiers, e.g., occurrences of local variables or class members in method bodies. - */ - @Doc(info = "All identifier usages") - def identifier: Traversal[Identifier] = - InitialTraversal.from[Identifier](cpg.graph, NodeTypes.IDENTIFIER) - - /** Shorthand for `cpg.identifier.name(name)` - */ - def identifier(name: String): Traversal[Identifier] = - identifier.name(name) + def goto: Iterator[ControlStructure] = + cpg.controlStructure.isGoto @Doc(info = "All if blocks (`ControlStructure` nodes)") - def ifBlock: Traversal[ControlStructure] = - controlStructure.isIf - - /** Traverse to all jump targets - */ - @Doc(info = "All jump targets, i.e., labels") - def jumpTarget: Traversal[JumpTarget] = - InitialTraversal.from[JumpTarget](cpg.graph, NodeTypes.JUMP_TARGET) - - /** Traverse to all local variable declarations - */ - @Doc(info = "All local variables") - def local: Traversal[Local] = - InitialTraversal.from[Local](cpg.graph, NodeTypes.LOCAL) - - /** Shorthand for `cpg.local.name` - */ - def local(name: String): Traversal[Local] = - local.name(name) - - /** Traverse to all literals (constant strings and numbers provided directly in the code). - */ - @Doc(info = "All literals, e.g., numbers or strings") - def literal: Traversal[Literal] = - InitialTraversal.from[Literal](cpg.graph, NodeTypes.LITERAL) - - /** Shorthand for `cpg.literal.code(code)` - */ - def literal(code: String): Traversal[Literal] = - literal.code(code) - - /** Traverse to all methods - */ - @Doc(info = "All methods") - def method: Traversal[Method] = - InitialTraversal.from[Method](cpg.graph, NodeTypes.METHOD) - - /** Shorthand for `cpg.method.name(name)` - */ - @Doc(info = "All methods with a name that matches the given pattern") - def method(namePattern: String): Traversal[Method] = - method.name(namePattern) - - /** Traverse to all formal return parameters - */ - @Doc(info = "All formal return parameters") - def methodReturn: Traversal[MethodReturn] = - InitialTraversal.from[MethodReturn](cpg.graph, NodeTypes.METHOD_RETURN) - - /** Traverse to all class members - */ - @Doc(info = "All members of complex types (e.g., classes/structures)") - def member: Traversal[Member] = - InitialTraversal.from[Member](cpg.graph, NodeTypes.MEMBER) - - /** Shorthand for `cpg.member.name(name)` - */ - def member(name: String): Traversal[Member] = - member.name(name) - - /** Traverse to all meta data entries - */ - @Doc(info = "Meta data blocks for graph") - def metaData: Traversal[MetaData] = - InitialTraversal.from[MetaData](cpg.graph, NodeTypes.META_DATA) - - /** Traverse to all method references - */ - @Doc(info = "All method references") - def methodRef: Traversal[MethodRef] = - InitialTraversal.from[MethodRef](cpg.graph, NodeTypes.METHOD_REF) - - /** Shorthand for `cpg.methodRef.filter(_.referencedMethod.name(name))` - */ - def methodRef(name: String): Traversal[MethodRef] = - methodRef.where(_.referencedMethod.name(name)) - - /** Traverse to all namespaces, e.g., packages in Java. - */ - @Doc(info = "All namespaces") - def namespace: Traversal[Namespace] = - InitialTraversal.from[Namespace](cpg.graph, NodeTypes.NAMESPACE) - - /** Shorthand for `cpg.namespace.name(name)` - */ - def namespace(name: String): Traversal[Namespace] = - namespace.name(name) - - /** Traverse to all namespace blocks, e.g., packages in Java. - */ - def namespaceBlock: Traversal[NamespaceBlock] = - InitialTraversal.from[NamespaceBlock](cpg.graph, NodeTypes.NAMESPACE_BLOCK) - - /** Shorthand for `cpg.namespaceBlock.name(name)` - */ - def namespaceBlock(name: String): Traversal[NamespaceBlock] = - namespaceBlock.name(name) - - /** Traverse to all input parameters - */ + def ifBlock: Iterator[ControlStructure] = + cpg.controlStructure.isIf + + /** Shorthand for `cpg.methodRef.where(_.referencedMethod.name(name))` + * + * Note re API design: this step was supposed to be called `methodRef(name: String)`, but due to limitations in + * Scala's implicit resolution (and the setup of our implicit steps) we have to disambiguate it from `.methodRef` by + * name. + * + * More precisely: Scala's implicit resolution reports 'ambiguous implicits' if two methods with the same name but + * different parameters are defined in two different (implicitly reachable) classes. The `.methodRef` step is defined + * in `generated.CpgNodeStarter`. This step (filter by name) doesn't get generated by the codegen because it's more + * complex than the other 'filter by primary key' starter steps. + */ + def methodRefWithName(name: String): Iterator[MethodRef] = + cpg.methodRef.where(_.referencedMethod.name(name)) + + /** Traverse to all input parameters */ @Doc(info = "All parameters") - def parameter: Traversal[MethodParameterIn] = - InitialTraversal.from[MethodParameterIn](cpg.graph, NodeTypes.METHOD_PARAMETER_IN) + def parameter: Iterator[MethodParameterIn] = + cpg.methodParameterIn - /** Shorthand for `cpg.parameter.name(name)` - */ - def parameter(name: String): Traversal[MethodParameterIn] = + /** Shorthand for `cpg.parameter.name(name)` */ + def parameter(name: String): Iterator[MethodParameterIn] = parameter.name(name) - /** Traverse to all return expressions - */ - @Doc(info = "All actual return parameters") - def ret: Traversal[Return] = - InitialTraversal.from[Return](cpg.graph, NodeTypes.RETURN) - - /** Shorthand for `returns.code(code)` - */ - def ret(code: String): Traversal[Return] = - ret.code(code) - - @Doc(info = "All imports") - def imports: Traversal[Import] = - InitialTraversal.from[Import](cpg.graph, NodeTypes.IMPORT) - @Doc(info = "All switch blocks (`ControlStructure` nodes)") - def switchBlock: Traversal[ControlStructure] = - controlStructure.isSwitch + def switchBlock: Iterator[ControlStructure] = + cpg.controlStructure.isSwitch @Doc(info = "All try blocks (`ControlStructure` nodes)") - def tryBlock: Traversal[ControlStructure] = - controlStructure.isTry - - /** Traverse to all types, e.g., Set - */ - @Doc(info = "All used types") - def typ: Traversal[Type] = - InitialTraversal.from[Type](cpg.graph, NodeTypes.TYPE) - - /** Shorthand for `cpg.typ.name(name)` - */ - @Doc(info = "All used types with given name") - def typ(name: String): Traversal[Type] = - typ.name(name) - - /** Traverse to all declarations, e.g., Set - */ - @Doc(info = "All declarations of types") - def typeDecl: Traversal[TypeDecl] = - InitialTraversal.from[TypeDecl](cpg.graph, NodeTypes.TYPE_DECL) - - /** Shorthand for cpg.typeDecl.name(name) - */ - def typeDecl(name: String): Traversal[TypeDecl] = - typeDecl.name(name) - - /** Traverse to all tags - */ - @Doc(info = "All tags") - def tag: Traversal[Tag] = - InitialTraversal.from[Tag](cpg.graph, NodeTypes.TAG) - - @Doc(info = "All tags with given name") - def tag(name: String): Traversal[Tag] = - tag.name(name) - - /** Traverse to all template DOM nodes - */ - @Doc(info = "All template DOM nodes") - def templateDom: Traversal[TemplateDom] = - InitialTraversal.from[TemplateDom](cpg.graph, NodeTypes.TEMPLATE_DOM) - - /** Traverse to all type references - */ - @Doc(info = "All type references") - def typeRef: Traversal[TypeRef] = - InitialTraversal.from[TypeRef](cpg.graph, NodeTypes.TYPE_REF) + def tryBlock: Iterator[ControlStructure] = + cpg.controlStructure.isTry @Doc(info = "All while blocks (`ControlStructure` nodes)") - def whileBlock: Traversal[ControlStructure] = - controlStructure.isWhile + def whileBlock: Iterator[ControlStructure] = + cpg.controlStructure.isWhile } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/Show.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/Show.scala index fcca10131992..ff700b1cdbb1 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/Show.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/Show.scala @@ -1,7 +1,6 @@ package io.shiftleft.semanticcpg.language -import io.shiftleft.codepropertygraph.generated.nodes.NewNode -import overflowdb.Node +import io.shiftleft.codepropertygraph.generated.nodes.{AbstractNode, NewNode, StoredNode} import scala.jdk.CollectionConverters.* @@ -18,20 +17,20 @@ object Show { override def apply(a: Any): String = a match { case node: NewNode => val label = node.label - val properties = propsToString(node.properties.toList) + val properties = propsToString(node.propertiesMap.asScala.toMap) s"($label): $properties" - case node: Node => + case node: StoredNode => val label = node.label val id = node.id().toString - val properties = propsToString(node.propertiesMap.asScala.toList) + val properties = propsToString(node.propertiesMap.asScala.toMap) s"($label,$id): $properties" case other => other.toString } - private def propsToString(keyValues: List[(String, Any)]): String = { - keyValues + private def propsToString(properties: Map[String, Any]): String = { + properties .filter(_._2.toString.nonEmpty) .sortBy(_._1) .map { case (key, value) => s"$key: $value" } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/Steps.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/Steps.scala index 7046591a68f6..3db74bbff3aa 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/Steps.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/Steps.scala @@ -1,9 +1,9 @@ package io.shiftleft.semanticcpg.language -import io.shiftleft.codepropertygraph.generated.nodes.AbstractNode +import io.shiftleft.codepropertygraph.generated.nodes.{AbstractNode, StoredNode} import org.json4s.native.Serialization.{write, writePretty} import org.json4s.{CustomSerializer, Extraction, Formats} -import io.shiftleft.codepropertygraph.generated.help.Doc +import io.shiftleft.codepropertygraph.generated.help.{Doc, Traversal} import replpp.Colors import replpp.Operators.* @@ -14,6 +14,7 @@ import scala.jdk.CollectionConverters.* /** Base class for our DSL These are the base steps available in all steps of the query language. There are no * constraints on the element types, unlike e.g. [[NodeSteps]] */ +@Traversal(elementType = classOf[AnyRef]) class Steps[A](val traversal: Iterator[A]) extends AnyVal { /** Execute the traversal and convert it to a mutable buffer @@ -82,7 +83,7 @@ object Steps { private lazy val nodeSerializer = new CustomSerializer[AbstractNode](implicit format => ( { case _ => ??? }, - { case node: (AbstractNode & Product) => + { case node: AbstractNode => val elementMap = (0 until node.productArity).map { i => val label = node.productElementName(i) val element = node.productElement(i) diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/android/ConfigFileTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/android/ConfigFileTraversal.scala index 936cf1439df9..c2a8d6e5efc9 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/android/ConfigFileTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/android/ConfigFileTraversal.scala @@ -1,9 +1,10 @@ package io.shiftleft.semanticcpg.language.android import io.joern.semanticcpg.utils.SecureXmlParsing -import io.shiftleft.codepropertygraph.generated.nodes +import io.shiftleft.codepropertygraph.generated.nodes.ConfigFile +import io.shiftleft.semanticcpg.language.* -class ConfigFileTraversal(val traversal: Iterator[nodes.ConfigFile]) extends AnyVal { +class ConfigFileTraversal(val traversal: Iterator[ConfigFile]) extends AnyVal { def usesCleartextTraffic = traversal .filter(_.name.endsWith(Constants.androidManifestXml)) diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/callgraphextension/MethodTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/callgraphextension/MethodTraversal.scala index 825c7fe793c3..d1ebbaf3d99c 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/callgraphextension/MethodTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/callgraphextension/MethodTraversal.scala @@ -1,9 +1,11 @@ package io.shiftleft.semanticcpg.language.callgraphextension -import io.shiftleft.codepropertygraph.generated.nodes.{Call, Method} +import flatgraph.help.{Doc, Traversal} +import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.semanticcpg.language.* import io.shiftleft.codepropertygraph.generated.help.Doc +@Traversal(elementType = classOf[Method]) class MethodTraversal(val traversal: Iterator[Method]) extends AnyVal { /** Intended for internal use! Traverse to direct and transitive callers of the method. diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/dotextension/AstNodeDot.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/dotextension/AstNodeDot.scala index aa10a624c79f..80f3622f27ae 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/dotextension/AstNodeDot.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/dotextension/AstNodeDot.scala @@ -2,7 +2,7 @@ package io.shiftleft.semanticcpg.language.dotextension import io.shiftleft.codepropertygraph.generated.nodes.AstNode import io.shiftleft.semanticcpg.dotgenerator.DotAstGenerator -import overflowdb.traversal.* +import io.shiftleft.semanticcpg.language.* class AstNodeDot[NodeType <: AstNode](val traversal: Iterator[NodeType]) extends AnyVal { diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/dotextension/CfgNodeDot.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/dotextension/CfgNodeDot.scala index 107b19037495..84d1bdf4647d 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/dotextension/CfgNodeDot.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/dotextension/CfgNodeDot.scala @@ -2,7 +2,7 @@ package io.shiftleft.semanticcpg.language.dotextension import io.shiftleft.codepropertygraph.generated.nodes.Method import io.shiftleft.semanticcpg.dotgenerator.{DotCdgGenerator, DotCfgGenerator} -import overflowdb.traversal.* +import io.shiftleft.semanticcpg.language.* class CfgNodeDot(val traversal: Iterator[Method]) extends AnyVal { diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/importresolver/ResolvedImportAsTagTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/importresolver/ResolvedImportAsTagTraversal.scala index 7bc957d4d70a..31619993991b 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/importresolver/ResolvedImportAsTagTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/importresolver/ResolvedImportAsTagTraversal.scala @@ -1,5 +1,6 @@ package io.shiftleft.semanticcpg.language.importresolver +import flatgraph.help.{Doc, Traversal} import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.{AstNode, Declaration, Member, Tag} import io.shiftleft.semanticcpg.language.* @@ -7,12 +8,10 @@ import io.shiftleft.codepropertygraph.generated.help.Doc class ResolvedImportAsTagExt(node: Tag) extends AnyVal { - @Doc(info = "Parses this tag as an EvaluatedImport class") def _toEvaluatedImport: Option[EvaluatedImport] = EvaluatedImport.tagToEvaluatedImport(node) - @Doc(info = "If this tag represents a resolved import, will attempt to find the CPG entities this refers to") def resolvedEntity: Iterator[AstNode] = { - val cpg = Cpg(node.graph()) + val cpg = Cpg(node.graph) node._toEvaluatedImport.iterator .collectAll[ResolvedImport] .flatMap { @@ -25,9 +24,9 @@ class ResolvedImportAsTagExt(node: Tag) extends AnyVal { } .iterator } - } +@Traversal(elementType = classOf[Tag]) class ResolvedImportAsTagTraversal(steps: Iterator[Tag]) extends AnyVal { @Doc(info = "Parses these tags as EvaluatedImport classes") diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/modulevariable/ModuleVariableAsNodeTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/modulevariable/ModuleVariableAsNodeTraversal.scala index cc4406a338a8..f6029f18e662 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/modulevariable/ModuleVariableAsNodeTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/modulevariable/ModuleVariableAsNodeTraversal.scala @@ -1,5 +1,6 @@ package io.shiftleft.semanticcpg.language.modulevariable +import flatgraph.help.{Doc, Traversal} import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.codepropertygraph.generated.{Cpg, Operators} import io.shiftleft.semanticcpg.language.* @@ -7,6 +8,7 @@ import io.shiftleft.semanticcpg.language.modulevariable.OpNodes import io.shiftleft.semanticcpg.language.operatorextension.OpNodes.FieldAccess import io.shiftleft.codepropertygraph.generated.help.Doc +@Traversal(elementType = classOf[Local]) class ModuleVariableAsLocalTraversal(traversal: Iterator[Local]) extends AnyVal { @Doc(info = "Locals representing module variables") @@ -16,6 +18,7 @@ class ModuleVariableAsLocalTraversal(traversal: Iterator[Local]) extends AnyVal } +@Traversal(elementType = classOf[Identifier]) class ModuleVariableAsIdentifierTraversal(traversal: Iterator[Identifier]) extends AnyVal { @Doc(info = "Identifiers representing module variables") @@ -25,12 +28,13 @@ class ModuleVariableAsIdentifierTraversal(traversal: Iterator[Identifier]) exten } +@Traversal(elementType = classOf[FieldIdentifier]) class ModuleVariableAsFieldIdentifierTraversal(traversal: Iterator[FieldIdentifier]) extends AnyVal { @Doc(info = "Field identifiers representing module variables") def moduleVariables: Iterator[OpNodes.ModuleVariable] = { traversal.flatMap { fieldIdentifier => - Cpg(fieldIdentifier.graph()).method + Cpg(fieldIdentifier.graph).method .fullNameExact(fieldIdentifier.inFieldAccess.argument(1).isIdentifier.typeFullName.toSeq*) .isModule .local @@ -40,13 +44,14 @@ class ModuleVariableAsFieldIdentifierTraversal(traversal: Iterator[FieldIdentifi } +@Traversal(elementType = classOf[Member]) class ModuleVariableAsMemberTraversal(traversal: Iterator[Member]) extends AnyVal { @Doc(info = "Members representing module variables") def moduleVariables: Iterator[OpNodes.ModuleVariable] = { val members = traversal.toList lazy val memberNames = members.name.toSeq - members.headOption.map(m => Cpg(m.graph())) match + members.headOption.map(m => Cpg(m.graph)) match case Some(cpg) => cpg.method .fullNameExact(members.typeDecl.fullName.toSeq*) @@ -57,6 +62,7 @@ class ModuleVariableAsMemberTraversal(traversal: Iterator[Member]) extends AnyVa } +@Traversal(elementType = classOf[Expression]) class ModuleVariableAsExpressionTraversal(traversal: Iterator[Expression]) extends AnyVal { @Doc(info = "Expression nodes representing module variables") diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/modulevariable/ModuleVariableTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/modulevariable/ModuleVariableTraversal.scala index d8ee902b5252..fe9d773eb5c4 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/modulevariable/ModuleVariableTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/modulevariable/ModuleVariableTraversal.scala @@ -1,11 +1,13 @@ package io.shiftleft.semanticcpg.language.modulevariable +import flatgraph.help.{Doc, Traversal} import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.operatorextension.OpNodes.Assignment import io.shiftleft.codepropertygraph.generated.help.Doc +@Traversal(elementType = classOf[Local]) class ModuleVariableTraversal(traversal: Iterator[OpNodes.ModuleVariable]) extends AnyVal { @Doc(info = "All assignments where the module variables in this traversal are the target across the program") @@ -32,7 +34,7 @@ class ModuleVariableTraversal(traversal: Iterator[OpNodes.ModuleVariable]) exten ) def references: Iterator[Identifier | FieldIdentifier] = { val variables = traversal.toList - variables.headOption.map(node => Cpg(node.graph())) match + variables.headOption.map(node => Cpg(node.graph)) match case Some(cpg) => val modules = cpg.method.isModule.l val variableNames = variables.name.toSet @@ -78,7 +80,7 @@ class ModuleVariableTraversal(traversal: Iterator[OpNodes.ModuleVariable]) exten val variables = traversal.toList lazy val moduleNames = variables.method.isModule.fullName.dedup.toSeq lazy val variableNames = variables.name.toSeq - variables.headOption.map(node => Cpg(node.graph())) match + variables.headOption.map(node => Cpg(node.graph)) match case Some(cpg) => cpg.typeDecl.fullNameExact(moduleNames*).member.nameExact(variableNames*) case None => Iterator.empty } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/modulevariable/NodeTypeStarters.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/modulevariable/NodeTypeStarters.scala index b3462fd1807e..e48dfd73f941 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/modulevariable/NodeTypeStarters.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/modulevariable/NodeTypeStarters.scala @@ -1,7 +1,7 @@ package io.shiftleft.semanticcpg.language.modulevariable +import flatgraph.help.{Doc, TraversalSource} import io.shiftleft.codepropertygraph.generated.Cpg -import overflowdb.traversal.help.{Doc, TraversalSource} import io.shiftleft.semanticcpg.language.* @TraversalSource diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/modulevariable/OpNodes.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/modulevariable/OpNodes.scala index dd442178f8af..2c43a8bf567c 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/modulevariable/OpNodes.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/modulevariable/OpNodes.scala @@ -1,6 +1,6 @@ package io.shiftleft.semanticcpg.language.modulevariable -import io.shiftleft.codepropertygraph.generated.nodes.{Block, Local, Member, StaticType} +import io.shiftleft.codepropertygraph.generated.nodes.{Local, StaticType} trait ModuleVariableT object OpNodes { @@ -8,7 +8,6 @@ object OpNodes { /** Represents a module-level global variable. This kind of node behaves like both a local variable and a field access * and is common in languages such as Python/JavaScript. */ - type ModuleVariable = Local & StaticType[ModuleVariableT] } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/modulevariable/nodemethods/ModuleVariableAsNodeMethods.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/modulevariable/nodemethods/ModuleVariableAsNodeMethods.scala index a9559ae38e4e..f70927b3dea2 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/modulevariable/nodemethods/ModuleVariableAsNodeMethods.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/modulevariable/nodemethods/ModuleVariableAsNodeMethods.scala @@ -6,7 +6,7 @@ import io.shiftleft.codepropertygraph.generated.help.Doc class ModuleVariableAsLocalMethods(node: Local) extends AnyVal { - @Doc(info = "If this local is declared on the module-defining method level") + /** If this local is declared on the module-defining method level */ def isModuleVariable: Boolean = node.method.isModule.nonEmpty } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/modulevariable/nodemethods/ModuleVariableMethods.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/modulevariable/nodemethods/ModuleVariableMethods.scala index 87e0b62b8bee..5a13de925796 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/modulevariable/nodemethods/ModuleVariableMethods.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/modulevariable/nodemethods/ModuleVariableMethods.scala @@ -10,15 +10,15 @@ import io.shiftleft.codepropertygraph.generated.help.Doc class ModuleVariableMethods(node: OpNodes.ModuleVariable) extends AnyVal { - @Doc(info = "References of this module variable across the codebase, as either identifiers or field identifiers") + /** References of this module variable across the codebase, as either identifiers or field identifiers */ def references: Iterator[Identifier | FieldIdentifier] = node.start.references - @Doc(info = "The module members being referenced in the respective module type declaration") + /** The module members being referenced in the respective module type declaration */ def referencingMembers: Iterator[Member] = { - Cpg(node.graph()).typeDecl.fullNameExact(node.method.fullName.toSeq*).member.nameExact(node.name) + Cpg(node.graph).typeDecl.fullNameExact(node.method.fullName.toSeq*).member.nameExact(node.name) } - @Doc(info = "Returns the assignments where the module variable is the target (LHS)") + /** Returns the assignments where the module variable is the target (LHS) */ def definitions: Iterator[OpExtNodes.Assignment] = node.start.definitions } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/AstNodeMethods.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/AstNodeMethods.scala index 14ad5766be64..c23ff1a463d5 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/AstNodeMethods.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/AstNodeMethods.scala @@ -56,7 +56,7 @@ class AstNodeMethods(val node: AstNode) extends AnyVal with NodeExtension { val additionalDepth = if (p(node)) { 1 } else { 0 } - val childDepths = node.astChildren.map(_.depth(p)).l + val childDepths = astChildren.map(_.depth(p)).l additionalDepth + (if (childDepths.isEmpty) { 0 } else { @@ -70,7 +70,7 @@ class AstNodeMethods(val node: AstNode) extends AnyVal with NodeExtension { /** Direct children of node in the AST. Siblings are ordered by their `order` fields */ def astChildren: Iterator[AstNode] = - node._astOut.cast[AstNode].sortBy(_.order).iterator + node._astOut.cast[AstNode].toSeq.sortBy(_.order).iterator /** Siblings of this node in the AST, ordered by their `order` fields */ @@ -108,8 +108,7 @@ class AstNodeMethods(val node: AstNode) extends AnyVal with NodeExtension { case member: Member => member case node: MethodParameterIn => node.method - case node: MethodParameterOut => - node.method.methodReturn + case node: MethodParameterOut => node.method.methodReturn case node: Call if MemberAccess.isGenericMemberAccessName(node.name) => parentExpansion(node) diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/CallMethods.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/CallMethods.scala index b124dcbd0c2e..635732ed7248 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/CallMethods.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/CallMethods.scala @@ -6,20 +6,27 @@ import io.shiftleft.semanticcpg.NodeExtension import io.shiftleft.semanticcpg.language.* class CallMethods(val node: Call) extends AnyVal with NodeExtension with HasLocation { + + def isStatic: Boolean = + node.dispatchType == "STATIC_DISPATCH" + + def isDynamic: Boolean = + node.dispatchType == "DYNAMIC_DISPATCH" + def receiver: Iterator[Expression] = - node.receiverOut + node.receiverOut.collectAll[Expression] def arguments(index: Int): Iterator[Expression] = - node._argumentOut - .collect { - case expr: Expression if expr.argumentIndex == index => expr - } + node.argumentOut.collect { + case expr: Expression if expr.argumentIndex == index => expr + } + // TODO define as named step in the schema def argument: Iterator[Expression] = - node._argumentOut.collectAll[Expression] + node.argumentOut.collectAll[Expression] def argument(index: Int): Expression = - arguments(index).head + arguments(index).next def argumentOption(index: Int): Option[Expression] = node._argumentOut.collectFirst { @@ -32,7 +39,6 @@ class CallMethods(val node: Call) extends AnyVal with NodeExtension with HasLoca node.astChildren.isBlock.maxByOption(_.order).iterator.expressionDown } - override def location: NewLocation = { + override def location: NewLocation = LocationCreator(node, node.code, node.label, node.lineNumber, node.method) - } } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/CfgNodeMethods.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/CfgNodeMethods.scala index 08ee7d72a7e6..fa14297ad531 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/CfgNodeMethods.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/CfgNodeMethods.scala @@ -11,9 +11,8 @@ class CfgNodeMethods(val node: CfgNode) extends AnyVal with NodeExtension { /** Successors in the CFG */ - def cfgNext: Iterator[CfgNode] = { + def cfgNext: Iterator[CfgNode] = Iterator.single(node).cfgNext - } /** Maps each node in the traversal to a traversal returning its n successors. */ @@ -31,9 +30,8 @@ class CfgNodeMethods(val node: CfgNode) extends AnyVal with NodeExtension { /** Predecessors in the CFG */ - def cfgPrev: Iterator[CfgNode] = { + def cfgPrev: Iterator[CfgNode] = Iterator.single(node).cfgPrev - } /** Recursively determine all nodes on which this CFG node is control-dependent. */ diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/IdentifierMethods.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/IdentifierMethods.scala index e04ff421b06a..134f42ae3397 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/IdentifierMethods.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/IdentifierMethods.scala @@ -2,7 +2,7 @@ package io.shiftleft.semanticcpg.language.nodemethods import io.shiftleft.codepropertygraph.generated.nodes.{Declaration, Identifier, NewLocation} import io.shiftleft.semanticcpg.NodeExtension -import io.shiftleft.semanticcpg.language.{HasLocation, LocationCreator, *} +import io.shiftleft.semanticcpg.language.* class IdentifierMethods(val identifier: Identifier) extends AnyVal with NodeExtension with HasLocation { override def location: NewLocation = { diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/LiteralMethods.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/LiteralMethods.scala index 3bbc9892b5bc..8962c206ff4a 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/LiteralMethods.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/LiteralMethods.scala @@ -2,7 +2,7 @@ package io.shiftleft.semanticcpg.language.nodemethods import io.shiftleft.codepropertygraph.generated.nodes.{Literal, NewLocation} import io.shiftleft.semanticcpg.NodeExtension -import io.shiftleft.semanticcpg.language.{HasLocation, LocationCreator, _} +import io.shiftleft.semanticcpg.language.* class LiteralMethods(val literal: Literal) extends AnyVal with NodeExtension with HasLocation { override def location: NewLocation = { diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/LocalMethods.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/LocalMethods.scala index 9bfa5eac78bc..5da4534d1bbe 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/LocalMethods.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/LocalMethods.scala @@ -1,12 +1,12 @@ package io.shiftleft.semanticcpg.language.nodemethods import io.shiftleft.codepropertygraph.generated.nodes.{Local, Method, NewLocation} -import io.shiftleft.semanticcpg.NodeExtension import io.shiftleft.semanticcpg.language.* +import io.shiftleft.semanticcpg.NodeExtension class LocalMethods(val local: Local) extends AnyVal with NodeExtension with HasLocation { override def location: NewLocation = { - LocationCreator(local, local.name, local.label, local.lineNumber, local.method.head) + LocationCreator(local, local.name, local.label, local.lineNumber, method.head) } /** The method hosting this local variable diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/MethodMethods.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/MethodMethods.scala index 52a2034ae28f..f5d1b1a02d69 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/MethodMethods.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/MethodMethods.scala @@ -1,8 +1,8 @@ package io.shiftleft.semanticcpg.language.nodemethods import io.shiftleft.codepropertygraph.generated.nodes.* -import io.shiftleft.semanticcpg.NodeExtension import io.shiftleft.semanticcpg.language.* +import io.shiftleft.semanticcpg.NodeExtension class MethodMethods(val method: Method) extends AnyVal with NodeExtension with HasLocation { @@ -37,14 +37,14 @@ class MethodMethods(val method: Method) extends AnyVal with NodeExtension with H /** List of CFG nodes in reverse post order */ def reversePostOrder: Iterator[CfgNode] = { - def expand(x: CfgNode) = { x.cfgNext.iterator } + def expand(x: CfgNode) = x.cfgNext.iterator NodeOrdering.reverseNodeList(NodeOrdering.postOrderNumbering(method, expand).toList).iterator } /** List of CFG nodes in post order */ def postOrder: Iterator[CfgNode] = { - def expand(x: CfgNode) = { x.cfgNext.iterator } + def expand(x: CfgNode) = x.cfgNext.iterator NodeOrdering.nodeList(NodeOrdering.postOrderNumbering(method, expand).toList).iterator } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/MethodParameterInMethods.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/MethodParameterInMethods.scala index 1f2a0c5420cc..1b39a89145c3 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/MethodParameterInMethods.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/MethodParameterInMethods.scala @@ -2,7 +2,7 @@ package io.shiftleft.semanticcpg.language.nodemethods import io.shiftleft.codepropertygraph.generated.nodes.{MethodParameterIn, NewLocation} import io.shiftleft.semanticcpg.NodeExtension -import io.shiftleft.semanticcpg.language.{HasLocation, LocationCreator} +import io.shiftleft.semanticcpg.language.* class MethodParameterInMethods(val paramIn: MethodParameterIn) extends AnyVal with NodeExtension with HasLocation { override def location: NewLocation = { diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/MethodParameterOutMethods.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/MethodParameterOutMethods.scala index 851c5949abbb..29471342f6ca 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/MethodParameterOutMethods.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/MethodParameterOutMethods.scala @@ -2,7 +2,7 @@ package io.shiftleft.semanticcpg.language.nodemethods import io.shiftleft.codepropertygraph.generated.nodes.{MethodParameterOut, NewLocation} import io.shiftleft.semanticcpg.NodeExtension -import io.shiftleft.semanticcpg.language.{HasLocation, LocationCreator} +import io.shiftleft.semanticcpg.language.* class MethodParameterOutMethods(val paramOut: MethodParameterOut) extends AnyVal with NodeExtension with HasLocation { override def location: NewLocation = { diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/MethodRefMethods.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/MethodRefMethods.scala index fc971fff51c9..03ea62a6dd3a 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/MethodRefMethods.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/MethodRefMethods.scala @@ -2,7 +2,7 @@ package io.shiftleft.semanticcpg.language.nodemethods import io.shiftleft.codepropertygraph.generated.nodes.{MethodRef, NewLocation} import io.shiftleft.semanticcpg.NodeExtension -import io.shiftleft.semanticcpg.language.{HasLocation, LocationCreator} +import io.shiftleft.semanticcpg.language.* class MethodRefMethods(val methodRef: MethodRef) extends AnyVal with NodeExtension with HasLocation { override def location: NewLocation = { diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/MethodReturnMethods.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/MethodReturnMethods.scala index 4ea458735b3b..edbe6ff6c9cf 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/MethodReturnMethods.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/MethodReturnMethods.scala @@ -19,5 +19,6 @@ class MethodReturnMethods(val node: MethodReturn) extends AnyVal with NodeExtens callsites.collectAll[Call] } - def typ: Iterator[Type] = node.evalTypeOut + // TODO define in schema as named step + def typ: Iterator[Type] = node._evalTypeOut.collectAll[Type] } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/NodeMethods.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/NodeMethods.scala index 1dc2a76f29fb..25325e1faccb 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/NodeMethods.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/NodeMethods.scala @@ -1,11 +1,10 @@ package io.shiftleft.semanticcpg.language.nodemethods -import io.shiftleft.codepropertygraph.generated.nodes.{NewLocation, StoredNode} +import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.semanticcpg.NodeExtension import io.shiftleft.semanticcpg.language.* -import overflowdb.NodeOrDetachedNode -class NodeMethods(val node: NodeOrDetachedNode) extends AnyVal with NodeExtension { +class NodeMethods(val node: AbstractNode) extends AnyVal with NodeExtension { def location(implicit finder: NodeExtensionFinder): NewLocation = node match { diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/ArrayAccessTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/ArrayAccessTraversal.scala index 15e6578550b6..618edf48142c 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/ArrayAccessTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/ArrayAccessTraversal.scala @@ -1,9 +1,11 @@ package io.shiftleft.semanticcpg.language.operatorextension -import io.shiftleft.codepropertygraph.generated.nodes.{Expression, Identifier} +import flatgraph.help.{Doc, Traversal} +import io.shiftleft.codepropertygraph.generated.nodes.{Call, Expression, Identifier} import io.shiftleft.semanticcpg.language.* import io.shiftleft.codepropertygraph.generated.help.Doc +@Traversal(elementType = classOf[Call]) class ArrayAccessTraversal(val traversal: Iterator[OpNodes.ArrayAccess]) extends AnyVal { @Doc(info = "The expression representing the array") diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/AssignmentTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/AssignmentTraversal.scala index 3e7265e979e7..ea687378e856 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/AssignmentTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/AssignmentTraversal.scala @@ -1,11 +1,12 @@ package io.shiftleft.semanticcpg.language.operatorextension +import flatgraph.help.{Doc, Traversal} import io.shiftleft.codepropertygraph.generated.nodes +import io.shiftleft.codepropertygraph.generated.nodes.{Call, Expression} import io.shiftleft.semanticcpg.language.* -import overflowdb.traversal.help import io.shiftleft.codepropertygraph.generated.help.Doc -@help.Traversal(elementType = classOf[nodes.Call]) +@Traversal(elementType = classOf[Call]) class AssignmentTraversal(val traversal: Iterator[OpNodes.Assignment]) extends AnyVal { @Doc(info = "Left-hand sides of assignments") diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/FieldAccessTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/FieldAccessTraversal.scala index 31bc44d8019a..7881262b88c2 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/FieldAccessTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/FieldAccessTraversal.scala @@ -1,9 +1,11 @@ package io.shiftleft.semanticcpg.language.operatorextension -import io.shiftleft.codepropertygraph.generated.nodes.{FieldIdentifier, Member, TypeDecl} +import flatgraph.help.{Doc, Traversal} +import io.shiftleft.codepropertygraph.generated.nodes.{Call, FieldIdentifier, Member, TypeDecl} import io.shiftleft.semanticcpg.language.* import io.shiftleft.codepropertygraph.generated.help.Doc +@Traversal(elementType = classOf[Call]) class FieldAccessTraversal(val traversal: Iterator[OpNodes.FieldAccess]) extends AnyVal { @Doc(info = "Attempts to resolve the type declaration for this field access") diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/Implicits.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/Implicits.scala index 45db146b49ac..5f63f72efef3 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/Implicits.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/Implicits.scala @@ -5,19 +5,24 @@ import io.shiftleft.codepropertygraph.generated.nodes.{AstNode, Expression} import io.shiftleft.semanticcpg.language.operatorextension.nodemethods.* trait Implicits { + implicit def toNodeTypeStartersOperatorExtension(cpg: Cpg): NodeTypeStarters = new NodeTypeStarters(cpg) implicit def toArrayAccessExt(arrayAccess: OpNodes.ArrayAccess): ArrayAccessMethods = new ArrayAccessMethods(arrayAccess) + implicit def toArrayAccessTrav(steps: Iterator[OpNodes.ArrayAccess]): ArrayAccessTraversal = new ArrayAccessTraversal(steps) implicit def toFieldAccessExt(fieldAccess: OpNodes.FieldAccess): FieldAccessMethods = new FieldAccessMethods(fieldAccess) + implicit def toFieldAccessTrav(steps: Iterator[OpNodes.FieldAccess]): FieldAccessTraversal = new FieldAccessTraversal(steps) - implicit def toAssignmentExt(assignment: OpNodes.Assignment): AssignmentMethods = new AssignmentMethods(assignment) + implicit def toAssignmentExt(assignment: OpNodes.Assignment): AssignmentMethods = + new AssignmentMethods(assignment) + implicit def toAssignmentTrav(steps: Iterator[OpNodes.Assignment]): AssignmentTraversal = new AssignmentTraversal(steps) diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/NodeTypeStarters.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/NodeTypeStarters.scala index 5ddf05c29036..805fdb5cb23f 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/NodeTypeStarters.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/NodeTypeStarters.scala @@ -1,11 +1,10 @@ package io.shiftleft.semanticcpg.language.operatorextension +import flatgraph.help.{Doc, TraversalSource} import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.semanticcpg.language.* -import overflowdb.traversal.help.{Doc, TraversalSource} -/** Steps that allow traversing from `cpg` to operators. - */ +/** Steps that allow traversing from `cpg` to operators. */ @TraversalSource class NodeTypeStarters(cpg: Cpg) { diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/OpAstNodeTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/OpAstNodeTraversal.scala index 604317f19089..83fde64aa528 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/OpAstNodeTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/OpAstNodeTraversal.scala @@ -1,9 +1,11 @@ package io.shiftleft.semanticcpg.language.operatorextension +import flatgraph.help.{Doc, Traversal} import io.shiftleft.codepropertygraph.generated.nodes.AstNode import io.shiftleft.semanticcpg.language.* import io.shiftleft.codepropertygraph.generated.help.Doc +@Traversal(elementType = classOf[AstNode]) class OpAstNodeTraversal[A <: AstNode](val traversal: Iterator[A]) extends AnyVal { @Doc(info = "Any assignments that this node is a part of (traverse up)") diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/TargetTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/TargetTraversal.scala index 1c7004083a5b..1e7b89251632 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/TargetTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/TargetTraversal.scala @@ -1,9 +1,11 @@ package io.shiftleft.semanticcpg.language.operatorextension +import flatgraph.help.{Doc, Traversal} import io.shiftleft.codepropertygraph.generated.nodes.Expression import io.shiftleft.semanticcpg.language.* import io.shiftleft.codepropertygraph.generated.help.Doc +@Traversal(elementType = classOf[Expression]) class TargetTraversal(val traversal: Iterator[Expression]) extends AnyVal { @Doc( diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/nodemethods/ArrayAccessMethods.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/nodemethods/ArrayAccessMethods.scala index 8675591639b8..578c429f54ed 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/nodemethods/ArrayAccessMethods.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/nodemethods/ArrayAccessMethods.scala @@ -22,7 +22,7 @@ class ArrayAccessMethods(val arrayAccess: OpNodes.ArrayAccess) extends AnyVal { } def simpleName: Iterator[String] = { - arrayAccess.array match { + array match { case id: Identifier => Iterator.single(id.name) case _ => Iterator.empty } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/package.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/package.scala index 1917298f0a5f..4b81d85e26fd 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/package.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/package.scala @@ -1,8 +1,9 @@ package io.shiftleft.semanticcpg +import flatgraph.help.DocSearchPackages +import io.shiftleft.codepropertygraph.generated import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.* -import io.shiftleft.codepropertygraph.generated.traversal.NodeTraversalImplicits import io.shiftleft.semanticcpg.language.bindingextension.{ MethodTraversal as BindingMethodTraversal, TypeDeclTraversal as BindingTypeDeclTraversal @@ -10,16 +11,11 @@ import io.shiftleft.semanticcpg.language.bindingextension.{ import io.shiftleft.semanticcpg.language.callgraphextension.{CallTraversal, MethodTraversal} import io.shiftleft.semanticcpg.language.dotextension.{AstNodeDot, CfgNodeDot, InterproceduralNodeDot} import io.shiftleft.semanticcpg.language.nodemethods.* -import io.shiftleft.semanticcpg.language.types.expressions.generalizations.{ - AstNodeTraversal, - CfgNodeTraversal, - DeclarationTraversal, - ExpressionTraversal -} +import io.shiftleft.semanticcpg.language.types.expressions.generalizations.* import io.shiftleft.semanticcpg.language.types.expressions.{CallTraversal as OriginalCall, *} import io.shiftleft.semanticcpg.language.types.propertyaccessors.* import io.shiftleft.semanticcpg.language.types.structure.{MethodTraversal as OriginalMethod, *} -import overflowdb.NodeOrDetachedNode +import io.shiftleft.semanticcpg.language.types.structure.* /** Language for traversing the code property graph * @@ -27,20 +23,19 @@ import overflowdb.NodeOrDetachedNode * `steps` package, e.g. `Steps` */ package object language - extends operatorextension.Implicits + extends generated.language + with operatorextension.Implicits with modulevariable.Implicits with importresolver.Implicits - with LowPrioImplicits - with NodeTraversalImplicits { + with LowPrioImplicits { // Implicit conversions from generated node types. We use these to add methods // to generated node types. + implicit def cfgNodeToAstNode(node: CfgNode): AstNodeMethods = new AstNodeMethods(node) + implicit def toExtendedNode(node: AbstractNode): NodeMethods = new NodeMethods(node) + implicit def toExtendedStoredNode(node: StoredNode): StoredNodeMethods = new StoredNodeMethods(node) + implicit def toAstNodeMethods(node: AstNode): AstNodeMethods = new AstNodeMethods(node) + implicit def toExpressionMethods(node: Expression): ExpressionMethods = new ExpressionMethods(node) - implicit def cfgNodeToAsNode(node: CfgNode): AstNodeMethods = new AstNodeMethods(node) - implicit def toExtendedNode(node: NodeOrDetachedNode): NodeMethods = new NodeMethods(node) - implicit def toExtendedStoredNode(node: StoredNode): StoredNodeMethods = new StoredNodeMethods(node) - implicit def toAstNodeMethods(node: AstNode): AstNodeMethods = new AstNodeMethods(node) - implicit def toCfgNodeMethods(node: CfgNode): CfgNodeMethods = new CfgNodeMethods(node) - implicit def toExpressionMethods(node: Expression): ExpressionMethods = new ExpressionMethods(node) implicit def toMethodMethods(node: Method): MethodMethods = new MethodMethods(node) implicit def toMethodReturnMethods(node: MethodReturn): MethodReturnMethods = new MethodReturnMethods(node) implicit def toCallMethods(node: Call): CallMethods = new CallMethods(node) @@ -68,8 +63,7 @@ package object language implicit def iterOnceToTypeDeclTrav[A <: TypeDecl](a: IterableOnce[A]): TypeDeclTraversal = new TypeDeclTraversal(a.iterator) - implicit def iterOnceToOriginalCallTrav[A <: Call](a: IterableOnce[A]): OriginalCall = - new OriginalCall(a.iterator) + implicit def iterOnceToOriginalCallTrav(traversal: IterableOnce[Call]): OriginalCall = new OriginalCall(traversal) implicit def singleToControlStructureTrav[A <: ControlStructure](a: A): ControlStructureTraversal = new ControlStructureTraversal(Iterator.single(a)) @@ -110,8 +104,6 @@ package object language implicit def iterOnceToMethodParameterInTrav[A <: MethodParameterIn](a: IterableOnce[A]): MethodParameterTraversal = new MethodParameterTraversal(a.iterator) - implicit def singleToMethodParameterOutTrav[A <: MethodParameterOut](a: A): MethodParameterOutTraversal = - new MethodParameterOutTraversal(Iterator.single(a)) implicit def iterOnceToMethodParameterOutTrav[A <: MethodParameterOut]( a: IterableOnce[A] ): MethodParameterOutTraversal = @@ -152,7 +144,7 @@ package object language new CallTraversal(a.iterator) // / Call graph extension - // Binding extensions +// // Binding extensions implicit def singleToBindingMethodTrav[A <: Method](a: A): BindingMethodTraversal = new BindingMethodTraversal(Iterator.single(a)) implicit def iterOnceToBindingMethodTrav[A <: Method](a: IterableOnce[A]): BindingMethodTraversal = @@ -163,11 +155,6 @@ package object language implicit def iterOnceToBindingTypeDeclTrav[A <: TypeDecl](a: IterableOnce[A]): BindingTypeDeclTraversal = new BindingTypeDeclTraversal(a.iterator) - implicit def singleToAstNodeDot[A <: AstNode](a: A): AstNodeDot[A] = - new AstNodeDot(Iterator.single(a)) - implicit def iterOnceToAstNodeDot[A <: AstNode](a: IterableOnce[A]): AstNodeDot[A] = - new AstNodeDot(a.iterator) - implicit def singleToCfgNodeDot[A <: Method](a: A): CfgNodeDot = new CfgNodeDot(Iterator.single(a)) implicit def iterOnceToCfgNodeDot[A <: Method](a: IterableOnce[A]): CfgNodeDot = @@ -268,11 +255,29 @@ package object language implicit def toExpression[A <: Expression](a: IterableOnce[A]): ExpressionTraversal[A] = new ExpressionTraversal[A](a.iterator) + + object NonStandardImplicits { + + // note: this causes problems because MethodParameterOut has an `index` property and the `MethodParameterOutTraversal` defines an `index` step... + implicit def singleToMethodParameterOutTrav[A <: MethodParameterOut](a: A): MethodParameterOutTraversal = + new MethodParameterOutTraversal(Iterator.single(a)) + + } } -trait LowPrioImplicits extends overflowdb.traversal.Implicits { - implicit def singleToCfgNodeTraversal[A <: CfgNode](a: A): CfgNodeTraversal[A] = - new CfgNodeTraversal[A](Iterator.single(a)) +trait LowPrioImplicits { + implicit val docSearchPackages: DocSearchPackages = + Cpg.defaultDocSearchPackage + .withAdditionalPackage("io.joern") + .withAdditionalPackage("io.shiftleft") + + implicit def singleToAstNodeDot[A <: AstNode](a: A): AstNodeDot[A] = + new AstNodeDot(Iterator.single(a)) + implicit def iterOnceToAstNodeDot[A <: AstNode](a: IterableOnce[A]): AstNodeDot[A] = + new AstNodeDot(a.iterator) + + implicit def toCfgNodeMethods(node: CfgNode): CfgNodeMethods = new CfgNodeMethods(node) + implicit def iterOnceToCfgNodeTraversal[A <: CfgNode](a: IterableOnce[A]): CfgNodeTraversal[A] = new CfgNodeTraversal[A](a.iterator) diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/CallTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/CallTraversal.scala index 33e47f74f375..5fc9724a0c34 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/CallTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/CallTraversal.scala @@ -5,19 +5,16 @@ import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.operatorextension.OpNodes.Assignment import io.shiftleft.semanticcpg.language.operatorextension.allAssignmentTypes -/** A call site - */ +/** A call site. */ class CallTraversal(val traversal: Iterator[Call]) extends AnyVal { - /** Only statically dispatched calls - */ + /** Only statically dispatched calls */ def isStatic: Iterator[Call] = - traversal.dispatchType("STATIC_DISPATCH") + traversal.filter(_.isStatic) - /** Only dynamically dispatched calls - */ + /** Only dynamically dispatched calls */ def isDynamic: Iterator[Call] = - traversal.dispatchType("DYNAMIC_DISPATCH") + traversal.filter(_.isDynamic) /** Only assignment calls */ diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/ControlStructureTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/ControlStructureTraversal.scala index e72e5add6675..18d2f2b8cbd1 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/ControlStructureTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/ControlStructureTraversal.scala @@ -1,7 +1,8 @@ package io.shiftleft.semanticcpg.language.types.expressions +import flatgraph.help.{Doc, Traversal} import io.shiftleft.codepropertygraph.generated.nodes.{AstNode, ControlStructure, Expression} -import io.shiftleft.codepropertygraph.generated.{ControlStructureTypes, Properties} +import io.shiftleft.codepropertygraph.generated.ControlStructureTypes import io.shiftleft.semanticcpg.language.* import io.shiftleft.codepropertygraph.generated.help.Doc @@ -10,6 +11,7 @@ object ControlStructureTraversal { val thirdChildIndex = 3 } +@Traversal(elementType = classOf[ControlStructure]) class ControlStructureTraversal(val traversal: Iterator[ControlStructure]) extends AnyVal { import ControlStructureTraversal.* @@ -23,11 +25,11 @@ class ControlStructureTraversal(val traversal: Iterator[ControlStructure]) exten @Doc(info = "Sub tree taken when condition evaluates to true") def whenTrue: Iterator[AstNode] = - traversal.out.has(Properties.Order, secondChildIndex: Int).cast[AstNode] + traversal.out.collectAll[AstNode].order(secondChildIndex) @Doc(info = "Sub tree taken when condition evaluates to false") def whenFalse: Iterator[AstNode] = - traversal.out.has(Properties.Order, thirdChildIndex).cast[AstNode] + traversal.out.collectAll[AstNode].order(thirdChildIndex) @Doc(info = "Only `Try` control structures") def isTry: Iterator[ControlStructure] = diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/IdentifierTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/IdentifierTraversal.scala index 8b11e5635c21..b9c6ea9f7e69 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/IdentifierTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/IdentifierTraversal.scala @@ -1,7 +1,7 @@ package io.shiftleft.semanticcpg.language.types.expressions import io.shiftleft.codepropertygraph.generated.nodes.{Declaration, Identifier} -import io.shiftleft.semanticcpg.language.toTraversalSugarExt +import io.shiftleft.semanticcpg.language.* /** An identifier, e.g., an instance of a local variable, or a temporary variable */ diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/AstNodeTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/AstNodeTraversal.scala index 5b70766effa4..b939e1ae3576 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/AstNodeTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/AstNodeTraversal.scala @@ -1,20 +1,18 @@ package io.shiftleft.semanticcpg.language.types.expressions.generalizations +import flatgraph.help.{Doc, Traversal} import io.shiftleft.codepropertygraph.generated.nodes.* -import io.shiftleft.codepropertygraph.generated.{EdgeTypes, NodeTypes} import io.shiftleft.semanticcpg.language.* -import overflowdb.traversal.help import io.shiftleft.codepropertygraph.generated.help.Doc -@help.Traversal(elementType = classOf[AstNode]) +@Traversal(elementType = classOf[AstNode]) class AstNodeTraversal[A <: AstNode](val traversal: Iterator[A]) extends AnyVal { /** Nodes of the AST rooted in this node, including the node itself. */ @Doc(info = "All nodes of the abstract syntax tree") - def ast: Iterator[AstNode] = { - traversal.repeat(_.out(EdgeTypes.AST))(_.emit).cast[AstNode] - } + def ast: Iterator[AstNode] = + traversal.repeat(_._astOut)(_.emit).cast[AstNode] /** All nodes of the abstract syntax tree rooted in this node, which match `predicate`. Equivalent of `match` in the * original CPG paper. @@ -38,7 +36,7 @@ class AstNodeTraversal[A <: AstNode](val traversal: Iterator[A]) extends AnyVal /** Nodes of the AST rooted in this node, minus the node itself */ def astMinusRoot: Iterator[AstNode] = - traversal.repeat(_.out(EdgeTypes.AST))(_.emitAllButFirst).cast[AstNode] + traversal.repeat(_._astOut)(_.emitAllButFirst).cast[AstNode] /** Direct children of node in the AST. Siblings are ordered by their `order` fields */ @@ -48,7 +46,7 @@ class AstNodeTraversal[A <: AstNode](val traversal: Iterator[A]) extends AnyVal /** Parent AST node */ def astParent: Iterator[AstNode] = - traversal.in(EdgeTypes.AST).cast[AstNode] + traversal._astIn.cast[AstNode] /** Siblings of this node in the AST, ordered by their `order` fields */ @@ -58,7 +56,7 @@ class AstNodeTraversal[A <: AstNode](val traversal: Iterator[A]) extends AnyVal /** Traverses up the AST and returns the first block node. */ def parentBlock: Iterator[Block] = - traversal.repeat(_.in(EdgeTypes.AST))(_.emit.until(_.hasLabel(NodeTypes.BLOCK))).collectAll[Block] + traversal.repeat(_._astIn)(_.emit.until(_.hasLabel(Block.Label))).collectAll[Block] /** Nodes of the AST obtained by expanding AST edges backwards until the method root is reached */ @@ -72,24 +70,26 @@ class AstNodeTraversal[A <: AstNode](val traversal: Iterator[A]) extends AnyVal /** Nodes of the AST obtained by expanding AST edges backwards until `root` or the method root is reached */ - def inAst(root: AstNode): Iterator[AstNode] = + def inAst(root: AstNode): Iterator[AstNode] = { traversal - .repeat(_.in(EdgeTypes.AST))( + .repeat(_._astIn)( _.emit - .until(_.or(_.hasLabel(NodeTypes.METHOD), _.filter(n => root != null && root == n))) + .until(_.or(_.hasLabel(Method.Label), _.filter(n => root != null && root == n))) ) .cast[AstNode] + } /** Nodes of the AST obtained by expanding AST edges backwards until `root` or the method root is reached, minus this * node */ - def inAstMinusLeaf(root: AstNode): Iterator[AstNode] = + def inAstMinusLeaf(root: AstNode): Iterator[AstNode] = { traversal - .repeat(_.in(EdgeTypes.AST))( + .repeat(_._astIn)( _.emitAllButFirst - .until(_.or(_.hasLabel(NodeTypes.METHOD), _.filter(n => root != null && root == n))) + .until(_.or(_.hasLabel(Method.Label), _.filter(n => root != null && root == n))) ) .cast[AstNode] + } /** Traverse only to those AST nodes that are also control flow graph nodes */ @@ -208,10 +208,11 @@ class AstNodeTraversal[A <: AstNode](val traversal: Iterator[A]) extends AnyVal def isTypeDecl: Iterator[TypeDecl] = traversal.collectAll[TypeDecl] - def walkAstUntilReaching(labels: List[String]): Iterator[StoredNode] = + def walkAstUntilReaching(labels: List[String]): Iterator[StoredNode] = { traversal - .repeat(_.out(EdgeTypes.AST))(_.emitAllButFirst.until(_.hasLabel(labels*))) + .repeat(_._astOut)(_.emitAllButFirst.until(_.hasLabel(labels*))) .dedup .cast[StoredNode] + } } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/CfgNodeTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/CfgNodeTraversal.scala index 0114f2ebed7b..a8ac6c9ed901 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/CfgNodeTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/CfgNodeTraversal.scala @@ -1,11 +1,12 @@ package io.shiftleft.semanticcpg.language.types.expressions.generalizations +import flatgraph.help.{Doc, Traversal} import io.shiftleft.codepropertygraph.generated.nodes.* +import io.shiftleft.codepropertygraph.generated.neighboraccessors.Lang.* import io.shiftleft.semanticcpg.language.* -import overflowdb.traversal.help import io.shiftleft.codepropertygraph.generated.help.Doc -@help.Traversal(elementType = classOf[CfgNode]) +@Traversal(elementType = classOf[CfgNode]) class CfgNodeTraversal[A <: CfgNode](val traversal: Iterator[A]) extends AnyVal { /** Textual representation of CFG node @@ -21,7 +22,6 @@ class CfgNodeTraversal[A <: CfgNode](val traversal: Iterator[A]) extends AnyVal /** Traverse to next expression in CFG. */ - @Doc(info = "Nodes directly reachable via outgoing CFG edges") def cfgNext: Iterator[CfgNode] = traversal._cfgOut diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/DeclarationTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/DeclarationTraversal.scala index f1075dad19c3..a93b04f72358 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/DeclarationTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/DeclarationTraversal.scala @@ -1,28 +1,27 @@ package io.shiftleft.semanticcpg.language.types.expressions.generalizations +import flatgraph.help.{Doc, Traversal} import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.semanticcpg.language.* -import overflowdb.traversal.help -/** A declaration, such as a local or parameter. - */ -@help.Traversal(elementType = classOf[Declaration]) +/** A declaration, such as a local or parameter. */ +@Traversal(elementType = classOf[Declaration]) class DeclarationTraversal[NodeType <: Declaration](val traversal: Iterator[NodeType]) extends AnyVal { - /** The closure binding node referenced by this declaration - */ + /** The closure binding node referenced by this declaration */ + @Doc(info = "The closure binding node referenced by this declaration") def closureBinding: Iterator[ClosureBinding] = traversal.flatMap(_._refIn).collectAll[ClosureBinding] - /** Methods that capture this declaration - */ + /** Methods that capture this declaration */ + @Doc(info = "Methods that capture this declaration") def capturedByMethodRef: Iterator[MethodRef] = closureBinding.flatMap(_._captureIn).collectAll[MethodRef] - /** Types that capture this declaration - */ + /** Types that capture this declaration */ + @Doc(info = "Types that capture this declaration") def capturedByTypeRef: Iterator[TypeRef] = closureBinding.flatMap(_._captureIn).collectAll[TypeRef] - /** The parent method. - */ + /** The parent method. */ + @Doc(info = "The parent method.") def method: Iterator[Method] = traversal.flatMap { case x: Local => x.method case x: MethodParameterIn => x.method diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/ExpressionTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/ExpressionTraversal.scala index 741f69202624..c3e22ff0dfc7 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/ExpressionTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/ExpressionTraversal.scala @@ -59,8 +59,8 @@ class ExpressionTraversal[NodeType <: Expression](val traversal: Iterator[NodeTy */ def method: Iterator[Method] = traversal._containsIn - .flatMap { - case x: Method => x.start + .map { + case x: Method => x case x: TypeDecl => x.astParent } .collectAll[Method] diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/propertyaccessors/ModifierAccessors.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/propertyaccessors/ModifierAccessors.scala index 679b220aafaa..76b7d1672f61 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/propertyaccessors/ModifierAccessors.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/propertyaccessors/ModifierAccessors.scala @@ -1,10 +1,8 @@ package io.shiftleft.semanticcpg.language.types.propertyaccessors import io.shiftleft.codepropertygraph.generated.ModifierTypes -import io.shiftleft.codepropertygraph.generated.nodes.{AstNode, Modifier} -import io.shiftleft.codepropertygraph.generated.traversal.toModifierTraversalExtGen +import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.semanticcpg.language.* -import overflowdb.* class ModifierAccessors[A <: AstNode](val traversal: Iterator[A]) extends AnyVal { diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/AnnotationTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/AnnotationTraversal.scala index a44ec38216ba..104ebab2ea2d 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/AnnotationTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/AnnotationTraversal.scala @@ -1,34 +1,34 @@ package io.shiftleft.semanticcpg.language.types.structure -import io.shiftleft.codepropertygraph.generated.nodes -import overflowdb.traversal.* +import io.shiftleft.codepropertygraph.generated.nodes.* +import io.shiftleft.semanticcpg.language.* /** An (Java-) annotation, e.g., @Test. */ -class AnnotationTraversal(val traversal: Iterator[nodes.Annotation]) extends AnyVal { +class AnnotationTraversal(val traversal: Iterator[Annotation]) extends AnyVal { /** Traverse to parameter assignments */ - def parameterAssign: Iterator[nodes.AnnotationParameterAssign] = + def parameterAssign: Iterator[AnnotationParameterAssign] = traversal.flatMap(_._annotationParameterAssignViaAstOut) /** Traverse to methods annotated with this annotation. */ - def method: Iterator[nodes.Method] = + def method: Iterator[Method] = traversal.flatMap(_._methodViaAstIn) /** Traverse to type declarations annotated by this annotation */ - def typeDecl: Iterator[nodes.TypeDecl] = + def typeDecl: Iterator[TypeDecl] = traversal.flatMap(_._typeDeclViaAstIn) /** Traverse to member annotated by this annotation */ - def member: Iterator[nodes.Member] = + def member: Iterator[Member] = traversal.flatMap(_._memberViaAstIn) /** Traverse to parameter annotated by this annotation */ - def parameter: Iterator[nodes.MethodParameterIn] = + def parameter: Iterator[MethodParameterIn] = traversal.flatMap(_._methodParameterInViaAstIn) } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/DependencyTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/DependencyTraversal.scala index 36309ff44325..93dbb4f4ce87 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/DependencyTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/DependencyTraversal.scala @@ -1,9 +1,9 @@ package io.shiftleft.semanticcpg.language.types.structure -import io.shiftleft.codepropertygraph.generated.nodes.Import -import io.shiftleft.codepropertygraph.generated.{EdgeTypes, nodes} +import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.semanticcpg.language.* -class DependencyTraversal(val traversal: Iterator[nodes.Dependency]) extends AnyVal { - def imports: Iterator[Import] = traversal.in(EdgeTypes.IMPORTS).cast[Import] +class DependencyTraversal(val traversal: Iterator[Dependency]) extends AnyVal { + def imports: Iterator[Import] = + traversal.importsIn } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/ImportTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/ImportTraversal.scala index e5131658039a..e46ff2b324e6 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/ImportTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/ImportTraversal.scala @@ -1,6 +1,6 @@ package io.shiftleft.semanticcpg.language.types.structure -import io.shiftleft.codepropertygraph.generated.nodes.{Call, Import, NamespaceBlock} +import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.semanticcpg.language.* class ImportTraversal(val traversal: Iterator[Import]) extends AnyVal { diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/LocalTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/LocalTraversal.scala index 26ff08d2c77c..5319bfc24365 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/LocalTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/LocalTraversal.scala @@ -13,8 +13,8 @@ class LocalTraversal(val traversal: Iterator[Local]) extends AnyVal { def method: Iterator[Method] = { // TODO The following line of code is here for backwards compatibility. // Use the lower commented out line once not required anymore. - traversal.repeat(_.in(EdgeTypes.AST))(_.until(_.hasLabel(NodeTypes.METHOD))).cast[Method] - // definingBlock.method + traversal.repeat(_._astIn)(_.until(_.hasLabel(Method.Label))).cast[Method] +// definingBlock.method } } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/MemberTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/MemberTraversal.scala index 628777f11f2c..5f45484923ea 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/MemberTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/MemberTraversal.scala @@ -1,7 +1,6 @@ package io.shiftleft.semanticcpg.language.types.structure -import io.shiftleft.codepropertygraph.generated.* -import io.shiftleft.codepropertygraph.generated.nodes.{Call, Member} +import io.shiftleft.codepropertygraph.generated.nodes.{Annotation, Call, Member} import io.shiftleft.semanticcpg.language.* /** A member variable of a class/type. @@ -10,7 +9,7 @@ class MemberTraversal(val traversal: Iterator[Member]) extends AnyVal { /** Traverse to annotations of member */ - def annotation: Iterator[nodes.Annotation] = + def annotation: Iterator[Annotation] = traversal.flatMap(_._annotationViaAstOut) /** Places where diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/MethodParameterOutTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/MethodParameterOutTraversal.scala index abb9778a1fd6..71caa46b8745 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/MethodParameterOutTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/MethodParameterOutTraversal.scala @@ -3,18 +3,19 @@ package io.shiftleft.semanticcpg.language.types.structure import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.semanticcpg.language.* -import scala.jdk.CollectionConverters.* - class MethodParameterOutTraversal(val traversal: Iterator[MethodParameterOut]) extends AnyVal { - def paramIn: Iterator[MethodParameterIn] = traversal.flatMap(_.parameterLinkIn.headOption) + def paramIn: Iterator[MethodParameterIn] = { + // TODO define a named step in schema + traversal.flatMap(_.parameterLinkIn.collectAll[MethodParameterIn]) + } /* method parameter indexes are based, i.e. first parameter has index (that's how java2cpg generates it) */ def index(num: Int): Iterator[MethodParameterOut] = traversal.filter { _.index == num } - /* get all parameters from (and including) - * method parameter indexes are based, i.e. first parameter has index (that's how java2cpg generates it) */ + /* get all parameters from (and including) + * method parameter indexes are based, i.e. first parameter has index (that's how java2cpg generates it) */ def indexFrom(num: Int): Iterator[MethodParameterOut] = traversal.filter(_.index >= num) @@ -27,9 +28,10 @@ class MethodParameterOutTraversal(val traversal: Iterator[MethodParameterOut]) e for { paramOut <- traversal method = paramOut.method - call <- method.callIn - arg <- call.argumentOut.collectAll[Expression] - if paramOut.parameterLinkIn.index.headOption.contains(arg.argumentIndex) + call <- method._callIn + arg <- call._argumentOut.collectAll[Expression] + // TODO define 'parameterLinkIn' as named step in schema + if paramOut.parameterLinkIn.collectAll[MethodParameterIn].index.headOption.contains(arg.argumentIndex) } yield arg } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/MethodParameterTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/MethodParameterTraversal.scala index 2f2af5ea8302..958cd670d47a 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/MethodParameterTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/MethodParameterTraversal.scala @@ -1,33 +1,32 @@ package io.shiftleft.semanticcpg.language.types.structure +import flatgraph.help.{Doc, Traversal} import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.semanticcpg.language.* -import overflowdb.traversal.help import scala.jdk.CollectionConverters.* -/** Formal method input parameter - */ -@help.Traversal(elementType = classOf[MethodParameterIn]) +/** Formal method input parameter */ +@Traversal(elementType = classOf[MethodParameterIn]) class MethodParameterTraversal(val traversal: Iterator[MethodParameterIn]) extends AnyVal { - /** Traverse to parameter annotations - */ + /** Traverse to parameter annotations */ + @Doc(info = "Traverse to parameter annotations") def annotation: Iterator[Annotation] = traversal.flatMap(_._annotationViaAstOut) - /** Traverse to all parameters with index greater or equal than `num` - */ + /** Traverse to all parameters with index greater or equal than `num` */ + @Doc(info = "Traverse to all parameters with index greater or equal than `num`") def indexFrom(num: Int): Iterator[MethodParameterIn] = traversal.filter(_.index >= num) - /** Traverse to all parameters with index smaller or equal than `num` - */ + /** Traverse to all parameters with index smaller or equal than `num` */ + @Doc(info = "Traverse to all parameters with index smaller or equal than `num`") def indexTo(num: Int): Iterator[MethodParameterIn] = traversal.filter(_.index <= num) - /** Traverse to arguments (actual parameters) associated with this formal parameter - */ + /** Traverse to arguments (actual parameters) associated with this formal parameter */ + @Doc(info = "Traverse to arguments (actual parameters) associated with this formal parameter") def argument(implicit callResolver: ICallResolver): Iterator[Expression] = for { paramIn <- traversal diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/MethodReturnTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/MethodReturnTraversal.scala index 99a29873ef4d..e7e59d8a7ed1 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/MethodReturnTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/MethodReturnTraversal.scala @@ -1,16 +1,16 @@ package io.shiftleft.semanticcpg.language.types.structure +import flatgraph.help.{Doc, Traversal} import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.semanticcpg.language.* -import overflowdb.traversal.help import io.shiftleft.codepropertygraph.generated.help.Doc -@help.Traversal(elementType = classOf[MethodReturn]) +@Traversal(elementType = classOf[MethodReturn]) class MethodReturnTraversal(val traversal: Iterator[MethodReturn]) extends AnyVal { @Doc(info = "traverse to parent method") def method: Iterator[Method] = - traversal.flatMap(_._methodViaAstIn) + traversal._methodViaAstIn def returnUser(implicit callResolver: ICallResolver): Iterator[Call] = traversal.flatMap(_.returnUser) @@ -19,11 +19,11 @@ class MethodReturnTraversal(val traversal: Iterator[MethodReturn]) extends AnyVa */ @Doc(info = "traverse to last expressions in CFG (can be multiple)") def cfgLast: Iterator[CfgNode] = - traversal.flatMap(_.cfgIn) + traversal.cfgIn /** Traverse to return type */ @Doc(info = "traverse to return type") def typ: Iterator[Type] = - traversal.flatMap(_.evalTypeOut) + traversal.evalTypeOut } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/MethodTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/MethodTraversal.scala index 4037bc4f40c8..6c66571b6b8e 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/MethodTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/MethodTraversal.scala @@ -1,15 +1,14 @@ package io.shiftleft.semanticcpg.language.types.structure +import flatgraph.help.{Doc, Traversal} import io.shiftleft.codepropertygraph.generated.* import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.semanticcpg.language.* -import overflowdb.* -import overflowdb.traversal.help import io.shiftleft.codepropertygraph.generated.help.Doc /** A method, function, or procedure */ -@help.Traversal(elementType = classOf[Method]) +@Traversal(elementType = classOf[Method]) class MethodTraversal(val traversal: Iterator[Method]) extends AnyVal { /** Traverse to annotations of method @@ -164,7 +163,8 @@ class MethodTraversal(val traversal: Iterator[Method]) extends AnyVal { // some language frontends don't have a TYPE_DECL for a METHOD case Some(namespaceBlock: NamespaceBlock) => namespaceBlock.start // other language frontends always embed their method in a TYPE_DECL - case _ => m.definingTypeDecl.namespaceBlock + case _ => + m.definingTypeDecl.iterator.namespaceBlock } } } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/NamespaceBlockTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/NamespaceBlockTraversal.scala index 7f95dee3285a..c6aaecf5b1a3 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/NamespaceBlockTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/NamespaceBlockTraversal.scala @@ -5,16 +5,17 @@ import io.shiftleft.semanticcpg.language.* class NamespaceBlockTraversal(val traversal: Iterator[NamespaceBlock]) extends AnyVal { - /** Namespaces for namespace blocks. + /** Namespaces for namespace blocks. TODO define a name in the schema */ def namespace: Iterator[Namespace] = - traversal.flatMap(_.refOut) + traversal.flatMap(_._namespaceViaRefOut) - /** The type declarations defined in this namespace + /** The type declarations defined in this namespace TODO define a name in the schema */ def typeDecl: Iterator[TypeDecl] = traversal.flatMap(_._typeDeclViaAstOut) + // TODO define a name in the schema def method: Iterator[Method] = traversal.flatMap(_._methodViaAstOut) } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/NamespaceTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/NamespaceTraversal.scala index c636047e2dff..3509c5f6a058 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/NamespaceTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/NamespaceTraversal.scala @@ -10,12 +10,12 @@ class NamespaceTraversal(val traversal: Iterator[Namespace]) extends AnyVal { /** The type declarations defined in this namespace */ def typeDecl: Iterator[TypeDecl] = - traversal.flatMap(_.refIn).flatMap(_._typeDeclViaAstOut) + traversal.refIn.astOut.collectAll[TypeDecl] /** Methods defined in this namespace */ def method: Iterator[Method] = - traversal.flatMap(_.refIn).flatMap(_._methodViaAstOut) + traversal.refIn.astOut.collectAll[Method] /** External namespaces - any namespaces which contain one or more external type. */ diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/TypeDeclTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/TypeDeclTraversal.scala index 0df69620c7a0..eec2e73c9044 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/TypeDeclTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/TypeDeclTraversal.scala @@ -1,6 +1,5 @@ package io.shiftleft.semanticcpg.language.types.structure -import io.shiftleft.codepropertygraph.generated.nodes import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.semanticcpg.language.* @@ -11,13 +10,13 @@ class TypeDeclTraversal(val traversal: Iterator[TypeDecl]) extends AnyVal { /** Annotations of the type declaration */ - def annotation: Iterator[nodes.Annotation] = + def annotation: Iterator[Annotation] = traversal.flatMap(_._annotationViaAstOut) /** Types referencing to this type declaration. */ def referencingType: Iterator[Type] = - traversal.flatMap(_.refIn) + traversal.refIn /** Namespace in which this type declaration is defined */ diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/TypeTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/TypeTraversal.scala index ea2bf4e0e536..402ec87b14bc 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/TypeTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/TypeTraversal.scala @@ -1,6 +1,5 @@ package io.shiftleft.semanticcpg.language.types.structure -import io.shiftleft.codepropertygraph.generated.nodes import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.semanticcpg.language.* @@ -8,7 +7,7 @@ class TypeTraversal(val traversal: Iterator[Type]) extends AnyVal { /** Annotations of the corresponding type declaration. */ - def annotation: Iterator[nodes.Annotation] = + def annotation: Iterator[Annotation] = traversal.referencedTypeDecl.annotation /** Namespaces in which the corresponding type declaration is defined. @@ -59,7 +58,7 @@ class TypeTraversal(val traversal: Iterator[Type]) extends AnyVal { /** Type declarations which derive from this type. */ def derivedTypeDecl: Iterator[TypeDecl] = - traversal.flatMap(_.inheritsFromIn) + traversal.inheritsFromIn /** Direct alias types. */ @@ -72,23 +71,24 @@ class TypeTraversal(val traversal: Iterator[Type]) extends AnyVal { traversal.repeat(_.aliasType)(_.emitAllButFirst) def localOfType: Iterator[Local] = - traversal.flatMap(_._localViaEvalTypeIn) + traversal._localViaEvalTypeIn def memberOfType: Iterator[Member] = - traversal.flatMap(_.evalTypeIn).collectAll[Member] + traversal.evalTypeIn.collectAll[Member] @deprecated("Please use `parameterOfType`") def parameter: Iterator[MethodParameterIn] = parameterOfType def parameterOfType: Iterator[MethodParameterIn] = - traversal.flatMap(_.evalTypeIn).collectAll[MethodParameterIn] + traversal.evalTypeIn.collectAll[MethodParameterIn] def methodReturnOfType: Iterator[MethodReturn] = - traversal.flatMap(_.evalTypeIn).collectAll[MethodReturn] + traversal.evalTypeIn.collectAll[MethodReturn] def expressionOfType: Iterator[Expression] = expression + // TODO define in schema def expression: Iterator[Expression] = - traversal.flatMap(_.evalTypeIn).collectAll[Expression] + traversal.evalTypeIn.collectAll[Expression] } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/package.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/package.scala index 9b0f25175138..a427ab3d9678 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/package.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/package.scala @@ -1,6 +1,6 @@ package io.shiftleft -import overflowdb.traversal.help.Table.AvailableWidthProvider +import flatgraph.help.Table.AvailableWidthProvider /** Domain specific language for querying code property graphs * diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/testing/DummyNode.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/testing/DummyNode.scala index 3c5a80dabddc..b523d8893fd3 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/testing/DummyNode.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/testing/DummyNode.scala @@ -1,52 +1,7 @@ package io.shiftleft.semanticcpg.testing import io.shiftleft.codepropertygraph.generated.nodes.StoredNode -import overflowdb.{Edge, Node, Property, PropertyKey} -import java.util - -/** mixin trait for test nodes */ trait DummyNodeImpl extends StoredNode { - // Members declared in overflowdb.Element - def graph(): overflowdb.Graph = ??? - def property[A](x$1: overflowdb.PropertyKey[A]): A = ??? - def property(x$1: String): Object = ??? - def propertyKeys(): java.util.Set[String] = ??? - def propertiesMap(): java.util.Map[String, Object] = ??? - def propertyOption(x$1: String): java.util.Optional[Object] = ??? - def propertyOption[A](x$1: overflowdb.PropertyKey[A]): java.util.Optional[A] = ??? - override def addEdgeImpl(label: String, inNode: Node, keyValues: Any*): Edge = ??? - override def addEdgeImpl(label: String, inNode: Node, keyValues: util.Map[String, AnyRef]): Edge = ??? - override def addEdgeSilentImpl(label: String, inNode: Node, keyValues: Any*): Unit = ??? - override def addEdgeSilentImpl(label: String, inNode: Node, keyValues: util.Map[String, AnyRef]): Unit = ??? - override def setPropertyImpl(key: String, value: Any): Unit = ??? - override def setPropertyImpl[A](key: PropertyKey[A], value: A): Unit = ??? - override def setPropertyImpl(property: Property[?]): Unit = ??? - override def removePropertyImpl(key: String): Unit = ??? - override def removeImpl(): Unit = ??? - - // Members declared in scala.Equals - def canEqual(that: Any): Boolean = ??? - - def both(x$1: String*): java.util.Iterator[overflowdb.Node] = ??? - def both(): java.util.Iterator[overflowdb.Node] = ??? - def bothE(x$1: String*): java.util.Iterator[overflowdb.Edge] = ??? - def bothE(): java.util.Iterator[overflowdb.Edge] = ??? - def id(): Long = ??? - def in(x$1: String*): java.util.Iterator[overflowdb.Node] = ??? - def in(): java.util.Iterator[overflowdb.Node] = ??? - def inE(x$1: String*): java.util.Iterator[overflowdb.Edge] = ??? - def inE(): java.util.Iterator[overflowdb.Edge] = ??? - def out(x$1: String*): java.util.Iterator[overflowdb.Node] = ??? - def out(): java.util.Iterator[overflowdb.Node] = ??? - def outE(x$1: String*): java.util.Iterator[overflowdb.Edge] = ??? - def outE(): java.util.Iterator[overflowdb.Edge] = ??? - - // Members declared in scala.Product - def productArity: Int = ??? - def productElement(n: Int): Any = ??? - - // Members declared in io.shiftleft.codepropertygraph.generated.nodes.StoredNode - def productElementLabel(n: Int): String = ??? - def valueMap: java.util.Map[String, AnyRef] = ??? + def propertiesMap: java.util.Map[String, Any] = ??? } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/testing/MockCpg.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/testing/MockCpg.scala new file mode 100644 index 000000000000..4b9fd5416639 --- /dev/null +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/testing/MockCpg.scala @@ -0,0 +1,229 @@ +package io.shiftleft.semanticcpg.testing + +import io.shiftleft.codepropertygraph.Cpg +import io.shiftleft.codepropertygraph.generated.nodes.* +import io.shiftleft.codepropertygraph.generated.{EdgeTypes, Languages, ModifierTypes} +import io.shiftleft.passes.CpgPass +import io.shiftleft.semanticcpg.language.* +import flatgraph.DiffGraphBuilder + +object MockCpg { + + def apply(): MockCpg = new MockCpg + + def apply(f: (DiffGraphBuilder, Cpg) => Unit): MockCpg = new MockCpg().withCustom(f) +} + +case class MockCpg(cpg: Cpg = Cpg.emptyCpg) { + + def withMetaData(language: String = Languages.C): MockCpg = withMetaData(language, Nil) + + def withMetaData(language: String, overlays: List[String]): MockCpg = { + withCustom { (diffGraph, _) => + diffGraph.addNode(NewMetaData().language(language).overlays(overlays)) + } + } + + def withFile(filename: String, content: Option[String] = None): MockCpg = + withCustom { (graph, _) => + val newFile = NewFile().name(filename) + content.foreach(newFile.content(_)) + graph.addNode(newFile) + } + + def withNamespace(name: String, inFile: Option[String] = None): MockCpg = + withCustom { (graph, _) => + { + val namespaceBlock = NewNamespaceBlock().name(name) + val namespace = NewNamespace().name(name) + graph.addNode(namespaceBlock) + graph.addNode(namespace) + graph.addEdge(namespaceBlock, namespace, EdgeTypes.REF) + if (inFile.isDefined) { + val fileNode = cpg.file(inFile.get).head + graph.addEdge(namespaceBlock, fileNode, EdgeTypes.SOURCE_FILE) + } + } + } + + def withTypeDecl( + name: String, + isExternal: Boolean = false, + inNamespace: Option[String] = None, + inFile: Option[String] = None, + offset: Option[Int] = None, + offsetEnd: Option[Int] = None + ): MockCpg = + withCustom { (graph, _) => + { + val typeNode = NewType().name(name) + val typeDeclNode = NewTypeDecl() + .name(name) + .fullName(name) + .isExternal(isExternal) + + offset.foreach(typeDeclNode.offset(_)) + offsetEnd.foreach(typeDeclNode.offsetEnd(_)) + + val member = NewMember().name("amember") + val modifier = NewModifier().modifierType(ModifierTypes.STATIC) + + graph.addNode(typeDeclNode) + graph.addNode(typeNode) + graph.addNode(member) + graph.addNode(modifier) + graph.addEdge(typeNode, typeDeclNode, EdgeTypes.REF) + graph.addEdge(typeDeclNode, member, EdgeTypes.AST) + graph.addEdge(member, modifier, EdgeTypes.AST) + + if (inNamespace.isDefined) { + val namespaceBlock = cpg.namespaceBlock(inNamespace.get).head + graph.addEdge(namespaceBlock, typeDeclNode, EdgeTypes.AST) + } + if (inFile.isDefined) { + val fileNode = cpg.file(inFile.get).head + graph.addEdge(typeDeclNode, fileNode, EdgeTypes.SOURCE_FILE) + } + } + } + + def withMethod( + name: String, + external: Boolean = false, + inTypeDecl: Option[String] = None, + fileName: String = "", + offset: Option[Int] = None, + offsetEnd: Option[Int] = None + ): MockCpg = + withCustom { (graph, _) => + val retParam = NewMethodReturn().typeFullName("int").order(10) + val param = NewMethodParameterIn().order(1).index(1).name("param1") + val paramType = NewType().name("paramtype") + val paramOut = NewMethodParameterOut().name("param1").order(1) + val method = + NewMethod().isExternal(external).name(name).fullName(name).signature("asignature").filename(fileName) + offset.foreach(method.offset(_)) + offsetEnd.foreach(method.offsetEnd(_)) + val block = NewBlock().typeFullName("int") + val modifier = NewModifier().modifierType("modifiertype") + + graph.addNode(method) + graph.addNode(retParam) + graph.addNode(param) + graph.addNode(paramType) + graph.addNode(paramOut) + graph.addNode(block) + graph.addNode(modifier) + graph.addEdge(method, retParam, EdgeTypes.AST) + graph.addEdge(method, param, EdgeTypes.AST) + graph.addEdge(param, paramOut, EdgeTypes.PARAMETER_LINK) + graph.addEdge(method, block, EdgeTypes.AST) + graph.addEdge(param, paramType, EdgeTypes.EVAL_TYPE) + graph.addEdge(paramOut, paramType, EdgeTypes.EVAL_TYPE) + graph.addEdge(method, modifier, EdgeTypes.AST) + + if (inTypeDecl.isDefined) { + val typeDeclNode = cpg.typeDecl(inTypeDecl.get).head + graph.addEdge(typeDeclNode, method, EdgeTypes.AST) + } + + if (fileName != "") { + val file = cpg.file + .nameExact(fileName) + .headOption + .getOrElse(throw new RuntimeException(s"file with name='$fileName' not found")) + graph.addEdge(method, file, EdgeTypes.SOURCE_FILE) + } + } + + def withTagsOnMethod( + methodName: String, + methodTags: List[(String, String)] = List(), + paramTags: List[(String, String)] = List() + ): MockCpg = + withCustom { (graph, cpg) => + implicit val diffGraph: DiffGraphBuilder = graph + methodTags.foreach { case (k, v) => + cpg.method(methodName).newTagNodePair(k, v).store()(diffGraph) + } + paramTags.foreach { case (k, v) => + cpg.method(methodName).parameter.newTagNodePair(k, v).store()(diffGraph) + } + } + + def withCallInMethod(methodName: String, callName: String, code: Option[String] = None): MockCpg = + withCustom { (graph, cpg) => + val methodNode = cpg.method(methodName).head + val blockNode = methodNode.block + val callNode = NewCall().name(callName).code(code.getOrElse(callName)) + graph.addNode(callNode) + graph.addEdge(blockNode, callNode, EdgeTypes.AST) + graph.addEdge(methodNode, callNode, EdgeTypes.CONTAINS) + } + + def withMethodCall(calledMethod: String, callingMethod: String, code: Option[String] = None): MockCpg = + withCustom { (graph, cpg) => + val callingMethodNode = cpg.method(callingMethod).head + val calledMethodNode = cpg.method(calledMethod).head + val callNode = NewCall().name(calledMethod).code(code.getOrElse(calledMethod)) + graph.addEdge(callNode, calledMethodNode, EdgeTypes.CALL) + graph.addEdge(callingMethodNode, callNode, EdgeTypes.CONTAINS) + } + + def withLocalInMethod(methodName: String, localName: String): MockCpg = + withCustom { (graph, cpg) => + val methodNode = cpg.method(methodName).head + val blockNode = methodNode.block + val typeNode = NewType().name("alocaltype") + val localNode = NewLocal().name(localName).typeFullName("alocaltype") + graph.addNode(localNode) + graph.addNode(typeNode) + graph.addEdge(blockNode, localNode, EdgeTypes.AST) + graph.addEdge(localNode, typeNode, EdgeTypes.EVAL_TYPE) + } + + def withLiteralArgument(callName: String, literalCode: String): MockCpg = { + withCustom { (graph, cpg) => + val callNode = cpg.call(callName).head + val methodNode = callNode.method + val literalNode = NewLiteral().code(literalCode) + val typeDecl = NewTypeDecl() + .name("ATypeDecl") + .fullName("ATypeDecl") + + graph.addNode(typeDecl) + graph.addNode(literalNode) + graph.addEdge(callNode, literalNode, EdgeTypes.AST) + graph.addEdge(methodNode, literalNode, EdgeTypes.CONTAINS) + } + } + + def withIdentifierArgument(callName: String, name: String, index: Int = 1): MockCpg = + withArgument(callName, NewIdentifier().name(name).argumentIndex(index)) + + def withCallArgument(callName: String, callArgName: String, code: String = "", index: Int = 1): MockCpg = + withArgument(callName, NewCall().name(callArgName).code(code).argumentIndex(index)) + + def withArgument(callName: String, newNode: NewNode): MockCpg = withCustom { (graph, cpg) => + val callNode = cpg.call(callName).head + val methodNode = callNode.method + val typeDecl = NewTypeDecl().name("abc") + graph.addEdge(callNode, newNode, EdgeTypes.AST) + graph.addEdge(callNode, newNode, EdgeTypes.ARGUMENT) + graph.addEdge(methodNode, newNode, EdgeTypes.CONTAINS) + graph.addEdge(newNode, typeDecl, EdgeTypes.REF) + graph.addNode(newNode) + } + + def withCustom(f: (DiffGraphBuilder, Cpg) => Unit): MockCpg = { + val diffGraph = new DiffGraphBuilder(cpg.graph.schema) + f(diffGraph, cpg) + class MyPass extends CpgPass(cpg) { + override def run(builder: DiffGraphBuilder): Unit = { + builder.absorb(diffGraph) + } + } + new MyPass().createAndApply() + this + } +} diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/testing/package.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/testing/package.scala deleted file mode 100644 index 254e0aad4e64..000000000000 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/testing/package.scala +++ /dev/null @@ -1,233 +0,0 @@ -package io.shiftleft.semanticcpg - -import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.codepropertygraph.generated.nodes.* -import io.shiftleft.codepropertygraph.generated.{EdgeTypes, Languages, ModifierTypes} -import io.shiftleft.passes.CpgPass -import io.shiftleft.semanticcpg.language.* -import io.shiftleft.codepropertygraph.generated.DiffGraphBuilder - -package object testing { - - object MockCpg { - - def apply(): MockCpg = new MockCpg - - def apply(f: (DiffGraphBuilder, Cpg) => Unit): MockCpg = new MockCpg().withCustom(f) - } - - case class MockCpg(cpg: Cpg = Cpg.empty) { - - def withMetaData(language: String = Languages.C): MockCpg = withMetaData(language, Nil) - - def withMetaData(language: String, overlays: List[String]): MockCpg = { - withCustom { (diffGraph, _) => - diffGraph.addNode(NewMetaData().language(language).overlays(overlays)) - } - } - - def withFile(filename: String, content: Option[String] = None): MockCpg = - withCustom { (graph, _) => - val newFile = NewFile().name(filename) - content.foreach(newFile.content(_)) - graph.addNode(newFile) - } - - def withNamespace(name: String, inFile: Option[String] = None): MockCpg = - withCustom { (graph, _) => - { - val namespaceBlock = NewNamespaceBlock().name(name) - val namespace = NewNamespace().name(name) - graph.addNode(namespaceBlock) - graph.addNode(namespace) - graph.addEdge(namespaceBlock, namespace, EdgeTypes.REF) - if (inFile.isDefined) { - val fileNode = cpg.file.name(inFile.get).head - graph.addEdge(namespaceBlock, fileNode, EdgeTypes.SOURCE_FILE) - } - } - } - - def withTypeDecl( - name: String, - isExternal: Boolean = false, - inNamespace: Option[String] = None, - inFile: Option[String] = None, - offset: Option[Int] = None, - offsetEnd: Option[Int] = None - ): MockCpg = - withCustom { (graph, _) => - { - val typeNode = NewType().name(name) - val typeDeclNode = NewTypeDecl() - .name(name) - .fullName(name) - .isExternal(isExternal) - - offset.foreach(typeDeclNode.offset(_)) - offsetEnd.foreach(typeDeclNode.offsetEnd(_)) - - val member = NewMember().name("amember") - val modifier = NewModifier().modifierType(ModifierTypes.STATIC) - - graph.addNode(typeDeclNode) - graph.addNode(typeNode) - graph.addNode(member) - graph.addNode(modifier) - graph.addEdge(typeNode, typeDeclNode, EdgeTypes.REF) - graph.addEdge(typeDeclNode, member, EdgeTypes.AST) - graph.addEdge(member, modifier, EdgeTypes.AST) - - if (inNamespace.isDefined) { - val namespaceBlock = cpg.namespaceBlock(inNamespace.get).head - graph.addEdge(namespaceBlock, typeDeclNode, EdgeTypes.AST) - } - if (inFile.isDefined) { - val fileNode = cpg.file.name(inFile.get).head - graph.addEdge(typeDeclNode, fileNode, EdgeTypes.SOURCE_FILE) - } - } - } - - def withMethod( - name: String, - external: Boolean = false, - inTypeDecl: Option[String] = None, - fileName: String = "", - offset: Option[Int] = None, - offsetEnd: Option[Int] = None - ): MockCpg = - withCustom { (graph, _) => - val retParam = NewMethodReturn().typeFullName("int").order(10) - val param = NewMethodParameterIn().order(1).index(1).name("param1") - val paramType = NewType().name("paramtype") - val paramOut = NewMethodParameterOut().name("param1").order(1) - val method = - NewMethod().isExternal(external).name(name).fullName(name).signature("asignature").filename(fileName) - offset.foreach(method.offset(_)) - offsetEnd.foreach(method.offsetEnd(_)) - val block = NewBlock().typeFullName("int") - val modifier = NewModifier().modifierType("modifiertype") - - graph.addNode(method) - graph.addNode(retParam) - graph.addNode(param) - graph.addNode(paramType) - graph.addNode(paramOut) - graph.addNode(block) - graph.addNode(modifier) - graph.addEdge(method, retParam, EdgeTypes.AST) - graph.addEdge(method, param, EdgeTypes.AST) - graph.addEdge(param, paramOut, EdgeTypes.PARAMETER_LINK) - graph.addEdge(method, block, EdgeTypes.AST) - graph.addEdge(param, paramType, EdgeTypes.EVAL_TYPE) - graph.addEdge(paramOut, paramType, EdgeTypes.EVAL_TYPE) - graph.addEdge(method, modifier, EdgeTypes.AST) - - if (inTypeDecl.isDefined) { - val typeDeclNode = cpg.typeDecl.name(inTypeDecl.get).head - graph.addEdge(typeDeclNode, method, EdgeTypes.AST) - } - - if (fileName != "") { - val file = cpg.file - .nameExact(fileName) - .headOption - .getOrElse(throw new RuntimeException(s"file with name='$fileName' not found")) - graph.addEdge(method, file, EdgeTypes.SOURCE_FILE) - } - } - - def withTagsOnMethod( - methodName: String, - methodTags: List[(String, String)] = List(), - paramTags: List[(String, String)] = List() - ): MockCpg = - withCustom { (graph, cpg) => - implicit val diffGraph: DiffGraphBuilder = graph - methodTags.foreach { case (k, v) => - cpg.method.name(methodName).newTagNodePair(k, v).store()(diffGraph) - } - paramTags.foreach { case (k, v) => - cpg.method.name(methodName).parameter.newTagNodePair(k, v).store()(diffGraph) - } - } - - def withCallInMethod(methodName: String, callName: String, code: Option[String] = None): MockCpg = - withCustom { (graph, cpg) => - val methodNode = cpg.method.name(methodName).head - val blockNode = methodNode.block - val callNode = NewCall().name(callName).code(code.getOrElse(callName)) - graph.addNode(callNode) - graph.addEdge(blockNode, callNode, EdgeTypes.AST) - graph.addEdge(methodNode, callNode, EdgeTypes.CONTAINS) - } - - def withMethodCall(calledMethod: String, callingMethod: String, code: Option[String] = None): MockCpg = - withCustom { (graph, cpg) => - val callingMethodNode = cpg.method.name(callingMethod).head - val calledMethodNode = cpg.method.name(calledMethod).head - val callNode = NewCall().name(calledMethod).code(code.getOrElse(calledMethod)) - graph.addEdge(callNode, calledMethodNode, EdgeTypes.CALL) - graph.addEdge(callingMethodNode, callNode, EdgeTypes.CONTAINS) - } - - def withLocalInMethod(methodName: String, localName: String): MockCpg = - withCustom { (graph, cpg) => - val methodNode = cpg.method.name(methodName).head - val blockNode = methodNode.block - val typeNode = NewType().name("alocaltype") - val localNode = NewLocal().name(localName).typeFullName("alocaltype") - graph.addNode(localNode) - graph.addNode(typeNode) - graph.addEdge(blockNode, localNode, EdgeTypes.AST) - graph.addEdge(localNode, typeNode, EdgeTypes.EVAL_TYPE) - } - - def withLiteralArgument(callName: String, literalCode: String): MockCpg = { - withCustom { (graph, cpg) => - val callNode = cpg.call.name(callName).head - val methodNode = callNode.method - val literalNode = NewLiteral().code(literalCode) - val typeDecl = NewTypeDecl() - .name("ATypeDecl") - .fullName("ATypeDecl") - - graph.addNode(typeDecl) - graph.addNode(literalNode) - graph.addEdge(callNode, literalNode, EdgeTypes.AST) - graph.addEdge(methodNode, literalNode, EdgeTypes.CONTAINS) - } - } - - def withIdentifierArgument(callName: String, name: String, index: Int = 1): MockCpg = - withArgument(callName, NewIdentifier().name(name).argumentIndex(index)) - - def withCallArgument(callName: String, callArgName: String, code: String = "", index: Int = 1): MockCpg = - withArgument(callName, NewCall().name(callArgName).code(code).argumentIndex(index)) - - def withArgument(callName: String, newNode: NewNode): MockCpg = withCustom { (graph, cpg) => - val callNode = cpg.call.name(callName).head - val methodNode = callNode.method - val typeDecl = NewTypeDecl().name("abc") - graph.addEdge(callNode, newNode, EdgeTypes.AST) - graph.addEdge(callNode, newNode, EdgeTypes.ARGUMENT) - graph.addEdge(methodNode, newNode, EdgeTypes.CONTAINS) - graph.addEdge(newNode, typeDecl, EdgeTypes.REF) - graph.addNode(newNode) - } - - def withCustom(f: (DiffGraphBuilder, Cpg) => Unit): MockCpg = { - val diffGraph = Cpg.newDiffGraphBuilder - f(diffGraph, cpg) - class MyPass extends CpgPass(cpg) { - override def run(builder: DiffGraphBuilder): Unit = { - builder.absorb(diffGraph) - } - } - new MyPass().createAndApply() - this - } - } - -} diff --git a/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/NewNodeStepsTests.scala b/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/NewNodeStepsTests.scala index 20792c91f7f2..da5b1b4e803a 100644 --- a/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/NewNodeStepsTests.scala +++ b/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/NewNodeStepsTests.scala @@ -1,15 +1,13 @@ package io.shiftleft.semanticcpg.language import io.shiftleft.codepropertygraph.generated.{Cpg, DiffGraphBuilder} +import flatgraph.DiffGraphApplier.applyDiff import io.shiftleft.codepropertygraph.generated.nodes.* import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec -import overflowdb.BatchedUpdate.applyDiff - -import scala.jdk.CollectionConverters.* class NewNodeStepsTest extends AnyWordSpec with Matchers { - import io.shiftleft.semanticcpg.language.NewNodeNodeStepsTest._ + import io.shiftleft.semanticcpg.language.NewNodeNodeStepsTest.* "stores NewNodes" in { implicit val diffGraphBuilder: DiffGraphBuilder = Cpg.newDiffGraphBuilder @@ -17,9 +15,9 @@ class NewNodeStepsTest extends AnyWordSpec with Matchers { val cpg = Cpg.empty new NewNodeSteps(newNode.start).store() - cpg.graph.nodes.toList.size shouldBe 0 + cpg.all.size shouldBe 0 applyDiff(cpg.graph, diffGraphBuilder) - cpg.graph.nodes.toList.size shouldBe 1 + cpg.all.size shouldBe 1 } "can access the node label" in { @@ -29,17 +27,19 @@ class NewNodeStepsTest extends AnyWordSpec with Matchers { "stores containedNodes and connecting edge" when { "embedding a StoredNode and a NewNode" in { - implicit val diffGraphBuilder: DiffGraphBuilder = Cpg.newDiffGraphBuilder - val cpg = Cpg.empty - val existingContainedNode = cpg.graph.addNode(42L, "MODIFIER").asInstanceOf[StoredNode] - cpg.graph.V().asScala.toSet shouldBe Set(existingContainedNode) + val cpg = Cpg.empty + val newModifier = NewModifier() + applyDiff(cpg.graph, Cpg.newDiffGraphBuilder.addNode(newModifier)) + val existingContainedNode = newModifier.storedRef.get + cpg.graph.allNodes.toSet shouldBe Set(existingContainedNode) - val newContainedNode = newTestNode() - val newNode = newTestNode(evidence = List(existingContainedNode, newContainedNode)) + implicit val diffGraphBuilder: DiffGraphBuilder = Cpg.newDiffGraphBuilder + val newContainedNode = newTestNode() + val newNode = newTestNode(evidence = List(existingContainedNode, newContainedNode)) new NewNodeSteps(newNode.start).store() - cpg.graph.V().asScala.length shouldBe 1 + cpg.all.length shouldBe 1 applyDiff(cpg.graph, diffGraphBuilder) - cpg.graph.V().asScala.length shouldBe 3 + cpg.all.length shouldBe 3 } "embedding a NewNode recursively" in { @@ -49,9 +49,9 @@ class NewNodeStepsTest extends AnyWordSpec with Matchers { val newContainedNodeL0 = newTestNode(evidence = List(newContainedNodeL1)) val newNode = newTestNode(evidence = List(newContainedNodeL0)) new NewNodeSteps(newNode.start).store() - cpg.graph.V().asScala.size shouldBe 0 + cpg.all.size shouldBe 0 applyDiff(cpg.graph, diffGraphBuilder) - cpg.graph.V().asScala.size shouldBe 3 + cpg.all.size shouldBe 3 } } diff --git a/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/OverlaysTests.scala b/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/OverlaysTests.scala similarity index 100% rename from semanticcpg/src/test/scala/io/shiftleft/semanticcpg/OverlaysTests.scala rename to semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/OverlaysTests.scala diff --git a/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/StepsTest.scala b/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/StepsTest.scala index 8e9107fe4412..deafb4044809 100644 --- a/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/StepsTest.scala +++ b/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/StepsTest.scala @@ -1,18 +1,15 @@ package io.shiftleft.semanticcpg.language -import io.shiftleft.codepropertygraph.Cpg.docSearchPackages import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.codepropertygraph.generated.{NodeTypes, Properties} +import io.shiftleft.codepropertygraph.generated.nodes.* +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.testing.MockCpg +import flatgraph.help.Table.{AvailableWidthProvider, ConstantWidth} import org.json4s.* import org.json4s.native.JsonMethods.parse import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec -import overflowdb.traversal.help.Table.{AvailableWidthProvider, ConstantWidth} - -import java.util.Optional -import scala.jdk.CollectionConverters.IteratorHasAsScala class StepsTest extends AnyWordSpec with Matchers { @@ -52,7 +49,7 @@ class StepsTest extends AnyWordSpec with Matchers { val method: Method = cpg.method.head val results: List[Method] = cpg.method.id(method.id).toList results.size shouldBe 1 - results.head.underlying.id + results.head.id } "providing multiple" in { @@ -158,7 +155,7 @@ class StepsTest extends AnyWordSpec with Matchers { val nodeId = mainMethods.head.id val printed = mainMethods.p.head printed.should(startWith(s"""(METHOD,$nodeId):""")) - printed.should(include("IS_EXTERNAL: false")) + printed.should(include("SIGNATURE: asignature")) printed.should(include("FULL_NAME: woo")) } @@ -197,19 +194,19 @@ class StepsTest extends AnyWordSpec with Matchers { "show domain overview" in { val domainStartersHelp = Cpg.empty.help domainStartersHelp should include(".comment") - domainStartersHelp should include("All comments in source-based CPGs") + domainStartersHelp should include("A source code comment") domainStartersHelp should include(".arithmetic") domainStartersHelp should include("All arithmetic operations") } "provide node-specific overview" in { val methodStepsHelp = Cpg.empty.method.help - methodStepsHelp should include("Available steps for Method") + methodStepsHelp should include("Available steps for `Method`") methodStepsHelp should include(".namespace") methodStepsHelp should include(".depth") // from AstNode val methodStepsHelpVerbose = Cpg.empty.method.helpVerbose - methodStepsHelpVerbose should include("traversal name") + methodStepsHelpVerbose should include("implemented in") methodStepsHelpVerbose should include("structure.MethodTraversal") val assignmentStepsHelp = Cpg.empty.assignment.help @@ -283,7 +280,6 @@ class StepsTest extends AnyWordSpec with Matchers { def methodParameterOut = cpg.graph .nodes(NodeTypes.METHOD_PARAMETER_OUT) - .asScala .cast[MethodParameterOut] .name("param1") methodParameterOut.typ.name.head shouldBe "paramtype" @@ -305,7 +301,7 @@ class StepsTest extends AnyWordSpec with Matchers { file.typeDecl.name.head shouldBe "AClass" file.head.typeDecl.name.head shouldBe "AClass" - def block = cpg.graph.nodes(NodeTypes.BLOCK).asScala.cast[Block].typeFullName("int") + def block = cpg.graph.nodes(NodeTypes.BLOCK).cast[Block].typeFullName("int") block.local.name.size shouldBe 1 block.flatMap(_.local.name).size shouldBe 1 @@ -349,13 +345,6 @@ class StepsTest extends AnyWordSpec with Matchers { method.head.modifier.modifierType.toSetMutable shouldBe Set("modifiertype") } - "id starter step" in { - // only verifying what compiles and what doesn't... - // if it compiles, :shipit: - assertCompiles("cpg.id(1).out") - assertDoesNotCompile("cpg.id(1).outV") // `.outV` is only available on Traversal[Edge] - } - "property accessors" in { val cpg = MockCpg().withCustom { (diffGraph, _) => diffGraph @@ -370,21 +359,35 @@ class StepsTest extends AnyWordSpec with Matchers { val (Seq(emptyCall), Seq(callWithProperties)) = cpg.call.l.partition(_.argumentName.isEmpty) - emptyCall.propertyOption(Properties.TypeFullName) shouldBe Optional.of("") - emptyCall.propertyOption(Properties.TypeFullName.name) shouldBe Optional.of("") - emptyCall.propertyOption(Properties.ArgumentName) shouldBe Optional.empty - emptyCall.propertyOption(Properties.ArgumentName.name) shouldBe Optional.empty + // Cardinality.One + emptyCall.property(Properties.TypeFullName) shouldBe "" + emptyCall.propertyOption(Properties.TypeFullName) shouldBe Some("") + emptyCall.propertyOption(Properties.TypeFullName.name) shouldBe Some("") + // Cardinality.ZeroOrOne + emptyCall.property(Properties.ArgumentName) shouldBe None + emptyCall.propertyOption(Properties.ArgumentName) shouldBe None + emptyCall.propertyOption(Properties.ArgumentName.name) shouldBe None + // Cardinality.List // these ones are rather a historic accident it'd be better and more consistent to return `None` here - // we'll defer that change until after the flatgraph port though and just document it for now - emptyCall.propertyOption(Properties.DynamicTypeHintFullName) shouldBe Optional.of(Seq.empty) - emptyCall.propertyOption(Properties.DynamicTypeHintFullName.name) shouldBe Optional.of(Seq.empty) - - callWithProperties.propertyOption(Properties.TypeFullName) shouldBe Optional.of("aa") - callWithProperties.propertyOption(Properties.TypeFullName.name) shouldBe Optional.of("aa") - callWithProperties.propertyOption(Properties.ArgumentName) shouldBe Optional.of("bb") - callWithProperties.propertyOption(Properties.ArgumentName.name) shouldBe Optional.of("bb") - callWithProperties.propertyOption(Properties.DynamicTypeHintFullName) shouldBe Optional.of(Seq("cc", "dd")) - callWithProperties.propertyOption(Properties.DynamicTypeHintFullName.name) shouldBe Optional.of(Seq("cc", "dd")) + emptyCall.property(Properties.DynamicTypeHintFullName) shouldBe Seq.empty + emptyCall.propertyOption(Properties.DynamicTypeHintFullName) shouldBe Some(Seq.empty) + emptyCall.propertyOption(Properties.DynamicTypeHintFullName.name) shouldBe Some(Seq.empty) + + // Cardinality.One + callWithProperties.property(Properties.TypeFullName) shouldBe "aa" + callWithProperties.propertyOption(Properties.TypeFullName) shouldBe Some("aa") + callWithProperties.propertyOption(Properties.TypeFullName.name) shouldBe Some("aa") + + // Cardinality.ZeroOrOne + callWithProperties.property(Properties.ArgumentName) shouldBe Some("bb") + callWithProperties.propertyOption(Properties.ArgumentName) shouldBe Some("bb") + callWithProperties.propertyOption(Properties.ArgumentName.name) shouldBe Some("bb") + + // Cardinality.List + callWithProperties.property(Properties.DynamicTypeHintFullName) shouldBe Seq("cc", "dd") + callWithProperties.propertyOption(Properties.DynamicTypeHintFullName) shouldBe Some(Seq("cc", "dd")) + callWithProperties.propertyOption(Properties.DynamicTypeHintFullName.name) shouldBe Some(Seq("cc", "dd")) } } diff --git a/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/accesspath/AccessPathTests.scala b/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/accesspath/AccessPathTests.scala similarity index 100% rename from semanticcpg/src/test/scala/io/shiftleft/semanticcpg/accesspath/AccessPathTests.scala rename to semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/accesspath/AccessPathTests.scala diff --git a/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/operatorextension/OperatorExtensionTests.scala b/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/operatorextension/OperatorExtensionTests.scala index 573ab58d7f78..650914868147 100644 --- a/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/operatorextension/OperatorExtensionTests.scala +++ b/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/operatorextension/OperatorExtensionTests.scala @@ -4,7 +4,6 @@ import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.Operators import io.shiftleft.codepropertygraph.generated.nodes.Identifier import io.shiftleft.semanticcpg.language.* -import io.shiftleft.semanticcpg.language.operatorextension.OpNodes.ArrayAccess import io.shiftleft.semanticcpg.testing.MockCpg import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec diff --git a/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/utils/CountStatementsTests.scala b/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/utils/CountStatementsTests.scala similarity index 100% rename from semanticcpg/src/test/scala/io/shiftleft/semanticcpg/utils/CountStatementsTests.scala rename to semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/utils/CountStatementsTests.scala