flatgraph port (#344)

mpollmeier · web-flow · commit 4c5afdadb92d · 2024-07-23T08:49:49.000+02:00
* port to flatgraph

* upgrade to latest
diff --git a/build.sbt b/build.sbt
@@ -1,5 +1,5 @@
-val cpgVersion   = "1.6.11"
-val joernVersion = "2.0.335"
+val cpgVersion   = "1.7.3"
+val joernVersion = "4.0.8"
 
 val gitCommitString = SettingKey[String]("gitSha")
 
@@ -16,7 +16,7 @@ Global / excludeLintKeys += Fast / configuration
 Global / excludeLintKeys += gitCommitString
 
 lazy val commonSettings = Seq(
-  scalaVersion := "3.4.1",
+  scalaVersion := "3.4.2",
   organization := "io.shiftleft",
   scalacOptions ++= Seq("-Xtarget:8"),
   resolvers ++= Seq(
diff --git a/project/build.properties b/project/build.properties
@@ -1 +1 @@
-sbt.version=1.9.9
+sbt.version=1.10.1
diff --git a/src/main/scala/io/shiftleft/js2cpg/astcreation/AstCreator.scala b/src/main/scala/io/shiftleft/js2cpg/astcreation/AstCreator.scala
@@ -44,6 +44,7 @@ import com.oracle.js.parser.ir.{
   WithNode
 }
 import com.oracle.js.parser.ir.LiteralNode.ArrayLiteralNode
+import flatgraph.DiffGraphBuilder
 import io.shiftleft.codepropertygraph.generated.nodes.{
   NewBlock,
   NewCall,
@@ -64,7 +65,6 @@ import io.shiftleft.js2cpg.datastructures.scope._
 import io.shiftleft.js2cpg.passes.{Defines, EcmaBuiltins, PassHelpers}
 import io.shiftleft.js2cpg.passes.PassHelpers.ParamNodeInitKind
 import io.shiftleft.js2cpg.parser.{GeneralizingAstVisitor, JsSource}
-import overflowdb.BatchedUpdate.DiffGraphBuilder
 import org.slf4j.LoggerFactory
 
 import scala.collection.mutable
@@ -429,7 +429,7 @@ class AstCreator(diffGraph: DiffGraphBuilder, source: JsSource, usedIdentNodes:
     astNodeBuilder.createTypeNode(methodName, methodFullName)
 
     val astParentType     = parentNodeId.label
-    val astParentFullName = parentNodeId.properties("FULL_NAME").toString
+    val astParentFullName = parentNodeId.propertiesMap.get("FULL_NAME").toString
 
     val functionTypeDeclId =
       astNodeBuilder.createTypeDeclNode(methodName, methodFullName, astParentType, astParentFullName, Some(Defines.Any))
@@ -457,7 +457,7 @@ class AstCreator(diffGraph: DiffGraphBuilder, source: JsSource, usedIdentNodes:
     // need to be resolved first, we for now dont handle the class
     // hierarchy.
     val astParentType     = methodAstParentStack.head.label
-    val astParentFullName = methodAstParentStack.head.properties("FULL_NAME").toString
+    val astParentFullName = methodAstParentStack.head.propertiesMap.get("FULL_NAME").toString
 
     val typeDeclId =
       astNodeBuilder.createTypeDeclNode(typeName, typeFullName, astParentType, astParentFullName, inheritsFrom = None)
diff --git a/src/main/scala/io/shiftleft/js2cpg/astcreation/AstEdgeBuilder.scala b/src/main/scala/io/shiftleft/js2cpg/astcreation/AstEdgeBuilder.scala
@@ -1,9 +1,9 @@
 package io.shiftleft.js2cpg.astcreation
 
+import flatgraph.DiffGraphBuilder
 import io.shiftleft.codepropertygraph.generated.EdgeTypes
 import io.shiftleft.codepropertygraph.generated.nodes._
 import io.shiftleft.js2cpg.datastructures.OrderTracker
-import overflowdb.BatchedUpdate.DiffGraphBuilder
 import org.slf4j.LoggerFactory
 
 class AstEdgeBuilder(private val diffGraph: DiffGraphBuilder) {
diff --git a/src/main/scala/io/shiftleft/js2cpg/astcreation/AstNodeBuilder.scala b/src/main/scala/io/shiftleft/js2cpg/astcreation/AstNodeBuilder.scala
@@ -1,14 +1,14 @@
 package io.shiftleft.js2cpg.astcreation
 
 import com.oracle.js.parser.ir._
+import flatgraph.DiffGraphBuilder
 import io.shiftleft.codepropertygraph.generated.nodes._
 import io.shiftleft.codepropertygraph.generated.{DispatchTypes, EvaluationStrategies, Operators}
 import io.shiftleft.js2cpg.datastructures.{LineAndColumn, OrderTracker}
 import io.shiftleft.js2cpg.datastructures.scope.{MethodScope, Scope}
 import io.shiftleft.js2cpg.passes.Defines
 import io.shiftleft.js2cpg.parser.JsSource
 import io.shiftleft.js2cpg.parser.JsSource.shortenCode
-import overflowdb.BatchedUpdate.DiffGraphBuilder
 
 class AstNodeBuilder(
   private val diffGraph: DiffGraphBuilder,
diff --git a/src/main/scala/io/shiftleft/js2cpg/astcreation/NewCompositeNode.scala b/src/main/scala/io/shiftleft/js2cpg/astcreation/NewCompositeNode.scala
@@ -1,14 +1,14 @@
 package io.shiftleft.js2cpg.astcreation
 
-import io.shiftleft.codepropertygraph.generated.nodes.NewNode
+import flatgraph.BatchedUpdateInterface
+import io.shiftleft.codepropertygraph.generated.nodes.{NewNode, StoredNode}
 
+import java.util
 import scala.collection.mutable.ListBuffer
 
-class NewCompositeNode(underlying: ListBuffer[NewNode] = ListBuffer.empty[NewNode]) extends NewNode {
+class NewCompositeNode(underlying: ListBuffer[NewNode] = ListBuffer.empty[NewNode]) extends NewNode(nodeKind = -1) {
   override def label: String = "COMPOSITE"
 
-  override def properties: Map[String, Any] = ??? // we do not need this
-
   override def canEqual(that: Any): Boolean =
     that.isInstanceOf[NewCompositeNode]
 
@@ -29,8 +29,9 @@ class NewCompositeNode(underlying: ListBuffer[NewNode] = ListBuffer.empty[NewNod
     underlying.foreach(func)
   }
 
-  def isValidInNeighbor(edgeLabel: String, node: NewNode): Boolean = ??? // we do not need this
-
-  def isValidOutNeighbor(edgeLabel: String, node: NewNode): Boolean = ??? // we do not need this
-
+  override def isValidInNeighbor(edgeLabel: String, node: NewNode): Boolean     = ??? // we do not need this
+  override def isValidOutNeighbor(edgeLabel: String, node: NewNode): Boolean    = ??? // we do not need this
+  override def propertiesMap: util.Map[String, Any]                             = ??? // we do not need this
+  override def countAndVisitProperties(interface: BatchedUpdateInterface): Unit = ???
+  override type StoredNodeType = StoredNode
 }
diff --git a/src/main/scala/io/shiftleft/js2cpg/passes/AstCreationPass.scala b/src/main/scala/io/shiftleft/js2cpg/passes/AstCreationPass.scala
@@ -4,12 +4,12 @@ import java.nio.file.Path
 import better.files.File
 import com.oracle.js.parser.Source
 import com.oracle.js.parser.ir.FunctionNode
-import io.shiftleft.codepropertygraph.Cpg
+import io.shiftleft.codepropertygraph.generated.Cpg
 import io.shiftleft.js2cpg.core.Report
 import io.shiftleft.js2cpg.astcreation.AstCreator
 import io.shiftleft.js2cpg.io.{FileUtils, JsFileChecks}
 import io.shiftleft.js2cpg.parser.{JavaScriptParser, JsSource}
-import io.shiftleft.passes.ConcurrentWriterCpgPass
+import io.shiftleft.passes.ForkJoinParallelCpgPass
 import org.slf4j.LoggerFactory
 import io.shiftleft.js2cpg.utils.SourceWrapper._
 import io.shiftleft.js2cpg.utils.TimeUtils
@@ -20,7 +20,7 @@ import scala.util.{Failure, Success, Try}
   * in parallel.
   */
 class AstCreationPass(srcDir: File, filenames: List[(Path, Path)], cpg: Cpg, report: Report)
-    extends ConcurrentWriterCpgPass[(Path, Path)](cpg) {
+    extends ForkJoinParallelCpgPass[(Path, Path)](cpg) {
 
   private val logger = LoggerFactory.getLogger(getClass)
 
@@ -41,7 +41,7 @@ class AstCreationPass(srcDir: File, filenames: List[(Path, Path)], cpg: Cpg, rep
 
     parseResult.map { case (parseResult, usedIdentNodes) =>
       val (result, duration) = {
-        TimeUtils.time(generateCpg(parseResult, new DiffGraphBuilder, usedIdentNodes))
+        TimeUtils.time(generateCpg(parseResult, Cpg.newDiffGraphBuilder, usedIdentNodes))
       }
       val path = parseResult.jsSource.originalFilePath
       result match {
diff --git a/src/main/scala/io/shiftleft/js2cpg/passes/ConfigPass.scala b/src/main/scala/io/shiftleft/js2cpg/passes/ConfigPass.scala
@@ -1,19 +1,19 @@
 package io.shiftleft.js2cpg.passes
 
-import io.shiftleft.codepropertygraph.Cpg
+import io.shiftleft.codepropertygraph.generated.Cpg
 import io.shiftleft.codepropertygraph.generated.nodes.NewConfigFile
 import io.shiftleft.js2cpg.core.Report
 import io.shiftleft.js2cpg.io.FileDefaults
 import io.shiftleft.js2cpg.io.FileUtils
 import io.shiftleft.js2cpg.utils.TimeUtils
-import io.shiftleft.passes.ConcurrentWriterCpgPass
+import io.shiftleft.passes.ForkJoinParallelCpgPass
 import io.shiftleft.utils.IOUtils
 import org.slf4j.{Logger, LoggerFactory}
 
 import java.nio.file.Path
 
 class ConfigPass(filenames: List[(Path, Path)], cpg: Cpg, report: Report)
-    extends ConcurrentWriterCpgPass[(Path, Path)](cpg) {
+    extends ForkJoinParallelCpgPass[(Path, Path)](cpg) {
 
   private val logger: Logger = LoggerFactory.getLogger(getClass)
 
@@ -37,7 +37,7 @@ class ConfigPass(filenames: List[(Path, Path)], cpg: Cpg, report: Report)
       val fileName = relativeFile.toString
       val content  = fileContent(filePath)
       val (result, time) = TimeUtils.time {
-        val localDiff = new DiffGraphBuilder
+        val localDiff = Cpg.newDiffGraphBuilder
         logger.debug(s"Adding file '$relativeFile' as config file.")
         val configNode = NewConfigFile().name(fileName).content(content.mkString("\n"))
         report.addReportInfo(fileName, fileStatistics.linesOfCode, parsed = true, cpgGen = true, isConfig = true)
diff --git a/src/test/scala/io/shiftleft/js2cpg/io/ExcludeTest.scala b/src/test/scala/io/shiftleft/js2cpg/io/ExcludeTest.scala
@@ -40,7 +40,7 @@ class ExcludeTest extends AnyWordSpec with Matchers with TableDrivenPropertyChec
           defaultArgs.map(_.toArg).toArray
       )
 
-      val cpg = Cpg.withConfig(overflowdb.Config.withoutOverflow.withStorageLocation(cpgPath.pathAsString))
+      val cpg = Cpg.withStorage(cpgPath.pathAsString, deserializeOnClose = false)
 
       fileNames(cpg) should contain theSameElementsAs expectedFiles.map(_.replace("/", java.io.File.separator))
       cpg.close()
diff --git a/src/test/scala/io/shiftleft/js2cpg/io/PrivateModulesTest.scala b/src/test/scala/io/shiftleft/js2cpg/io/PrivateModulesTest.scala
@@ -24,7 +24,7 @@ class PrivateModulesTest extends AnyWordSpec with Matchers {
         val cpgPath = tmpDir / "cpg.bin.zip"
         Js2CpgMain.main(Array(tmpProjectPath.pathAsString, "--output", cpgPath.pathAsString, "--no-babel"))
 
-        val cpg = Cpg.withConfig(overflowdb.Config.withoutOverflow.withStorageLocation(cpgPath.pathAsString))
+        val cpg = Cpg.withStorage(cpgPath.pathAsString, deserializeOnClose = false)
 
         fileNames(cpg) should contain allElementsOf Set(
           s"@privateA${java.io.File.separator}a.js",
@@ -52,7 +52,7 @@ class PrivateModulesTest extends AnyWordSpec with Matchers {
           )
         )
 
-        val cpg = Cpg.withConfig(overflowdb.Config.withoutOverflow.withStorageLocation(cpgPath.pathAsString))
+        val cpg = Cpg.withStorage(cpgPath.pathAsString, deserializeOnClose = false)
 
         fileNames(cpg) should contain allElementsOf Set(
           s"@privateA${java.io.File.separator}a.js",
@@ -82,7 +82,7 @@ class PrivateModulesTest extends AnyWordSpec with Matchers {
           )
         )
 
-        val cpg = Cpg.withConfig(overflowdb.Config.withoutOverflow.withStorageLocation(cpgPath.pathAsString))
+        val cpg = Cpg.withStorage(cpgPath.pathAsString, deserializeOnClose = false)
 
         fileNames(cpg) should contain only s"@privateB${java.io.File.separator}b.js"
         cpg.close()
@@ -98,7 +98,7 @@ class PrivateModulesTest extends AnyWordSpec with Matchers {
         val cpgPath = tmpDir / "cpg.bin.zip"
         Js2CpgMain.main(Array(tmpProjectPath.pathAsString, "--output", cpgPath.pathAsString, "--no-babel"))
 
-        val cpg = Cpg.withConfig(overflowdb.Config.withoutOverflow.withStorageLocation(cpgPath.pathAsString))
+        val cpg = Cpg.withStorage(cpgPath.pathAsString, deserializeOnClose = false)
 
         fileNames(cpg) should contain only "index.js"
         cpg.close()
diff --git a/src/test/scala/io/shiftleft/js2cpg/passes/AbstractPassTest.scala b/src/test/scala/io/shiftleft/js2cpg/passes/AbstractPassTest.scala
@@ -7,8 +7,6 @@ import io.shiftleft.codepropertygraph.generated.nodes.Dependency
 import io.shiftleft.codepropertygraph.Cpg
 import io.shiftleft.js2cpg.core.Report
 import io.shiftleft.semanticcpg.language.*
-import overflowdb.Node
-import overflowdb.traversal.*
 import org.scalatest.matchers.should.Matchers
 import org.scalatest.wordspec.AnyWordSpec
 
@@ -18,7 +16,8 @@ abstract class AbstractPassTest extends AnyWordSpec with Matchers {
 
   protected abstract class Fixture
 
-  protected def getDependencies(cpg: Cpg): Traversal[Dependency] = cpg.dependency
+  protected def getDependencies(cpg: Cpg): Iterator[Dependency] =
+    cpg.dependency
 
   protected object AstFixture extends Fixture {
     def apply(code: String)(f: Cpg => Unit): Unit = {
diff --git a/src/test/scala/io/shiftleft/js2cpg/passes/BuiltinTypesPassTest.scala b/src/test/scala/io/shiftleft/js2cpg/passes/BuiltinTypesPassTest.scala
@@ -1,12 +1,12 @@
 package io.shiftleft.js2cpg.passes
 
-import io.shiftleft.codepropertygraph.Cpg
+import io.shiftleft.codepropertygraph.generated.Cpg
 import io.shiftleft.semanticcpg.language._
 
 class BuiltinTypesPassTest extends AbstractPassTest {
 
   "BuiltinTypesPass" should {
-    val cpg = Cpg.emptyCpg
+    val cpg = Cpg.empty
     new BuiltinTypesPass(cpg).createAndApply()
 
     "create a '<global>' NamespaceBlock" in {
diff --git a/src/test/scala/io/shiftleft/js2cpg/passes/CfgCreationPassTest.scala b/src/test/scala/io/shiftleft/js2cpg/passes/CfgCreationPassTest.scala
@@ -1,17 +1,14 @@
 package io.shiftleft.js2cpg.passes
 
 import better.files.File
-import io.shiftleft.codepropertygraph.Cpg
-import io.shiftleft.codepropertygraph.generated._
+import io.shiftleft.codepropertygraph.generated.*
 import io.shiftleft.js2cpg.core.Report
-import io.shiftleft.semanticcpg.language._
+import io.shiftleft.semanticcpg.language.*
 import io.joern.x2cpg.passes.controlflow.CfgCreationPass
-import io.joern.x2cpg.passes.controlflow.cfgcreation.Cfg._
-import io.shiftleft.codepropertygraph.generated.nodes.AstNodeBase
-import io.shiftleft.codepropertygraph.generated.nodes.Method
+import io.joern.x2cpg.passes.controlflow.cfgcreation.Cfg.*
+import io.shiftleft.codepropertygraph.generated.nodes.{AstNodeBase, CfgNode, Method}
 import org.scalatest.matchers.should.Matchers
 import org.scalatest.wordspec.AnyWordSpec
-import overflowdb.Node
 
 class CfgCreationPassTest extends AnyWordSpec with Matchers {
 
@@ -1290,7 +1287,7 @@ class CfgCreationPassTest extends AnyWordSpec with Matchers {
 
   private class CfgFixture(code: String) {
 
-    private val cpg: Cpg = Cpg.emptyCpg
+    private val cpg: Cpg = Cpg.empty
 
     File.usingTemporaryDirectory("js2cpgCfgIntegrationTest") { workspace =>
       val file = workspace / "test.js"
@@ -1300,7 +1297,7 @@ class CfgCreationPassTest extends AnyWordSpec with Matchers {
       new CfgCreationPass(cpg).createAndApply()
     }
 
-    private def matchCode(node: Node, code: String): Boolean = node match {
+    private def matchCode(node: CfgNode, code: String): Boolean = node match {
       case m: Method            => m.name == code
       case astNode: AstNodeBase => astNode.code == code
       case _                    => false
@@ -1325,7 +1322,7 @@ class CfgCreationPassTest extends AnyWordSpec with Matchers {
           }
           .lift(index)
           .getOrElse(fail(s"No node found for code = '$code' and index '$index'!"))
-        node.property(PropertyNames.CODE).toString
+        node.code
       }.toSet
     }
 
@@ -1337,14 +1334,14 @@ class CfgCreationPassTest extends AnyWordSpec with Matchers {
     def succOf(code: String, index: Int): Set[String] = {
       val nodes      = cpg.method.ast.isCfgNode.collect { case node if matchCode(node, code) => node }.l
       val node       = nodes.lift(index).getOrElse(fail(s"No node found for code = '$code' and index '$index'!"))
-      val successors = node._cfgOut.map(_.property(PropertyNames.CODE).toString)
+      val successors = node._cfgOut.cast[CfgNode].code
       successors.toSetImmutable
     }
 
     def succOf(code: String, nodeType: String): Set[String] = {
       val nodes = cpg.method.ast.isCfgNode.label(nodeType).collectFirst { case node if matchCode(node, code) => node }
       val node  = nodes.getOrElse(fail(s"No node found for code = '$code' and type '$nodeType'!"))
-      val successors = node._cfgOut.map(_.property(PropertyNames.CODE).toString)
+      val successors = node._cfgOut.cast[CfgNode].code
       successors.toSetImmutable
     }
 
diff --git a/src/test/scala/io/shiftleft/js2cpg/passes/ConfigPassTest.scala b/src/test/scala/io/shiftleft/js2cpg/passes/ConfigPassTest.scala
diff --git a/src/test/scala/io/shiftleft/js2cpg/passes/JsMetaDataPassTest.scala b/src/test/scala/io/shiftleft/js2cpg/passes/JsMetaDataPassTest.scala
diff --git a/src/test/scala/io/shiftleft/js2cpg/preprocessing/TranspilationRunnerTest.scala b/src/test/scala/io/shiftleft/js2cpg/preprocessing/TranspilationRunnerTest.scala

Original file line number	Diff line number	Diff line change
`@@ -1 +1 @@`
`1`		`-sbt.version=1.9.9`
	`1`	`+sbt.version=1.10.1`
Original file line number	Diff line number	Diff line change
`@@ -40,7 +40,7 @@ class ExcludeTest extends AnyWordSpec with Matchers with TableDrivenPropertyChec`
`40`	`40`	`defaultArgs.map(_.toArg).toArray`
`41`	`41`	`)`
`42`	`42`
`43`		`- val cpg = Cpg.withConfig(overflowdb.Config.withoutOverflow.withStorageLocation(cpgPath.pathAsString))`
	`43`	`+ val cpg = Cpg.withStorage(cpgPath.pathAsString, deserializeOnClose = false)`
`44`	`44`
`45`	`45`	`fileNames(cpg) should contain theSameElementsAs expectedFiles.map(_.replace("/", java.io.File.separator))`
`46`	`46`	`cpg.close()`