[x2cpg] Program Summary Mutable Merging

As pointed out in #4240, combining this nested immutable map-like structure has a quadratic performance, and the more performant strategy would be to use nested data-structures to merge.

For now, I've decided not to opt for a builder pattern, but rather keep the underlying structure mutable, and accessor methods return immutable structures.

dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/package.scala

@@ -16,7 +16,7 @@ package object dataflowengineoss {
  */
  def globalFromLiteral(lit: Literal, recursive: Boolean = true): Iterator[Expression] = lit.start
  .where(_.method.isModule)
- .flatMap(t => if (recursive) t.inAssignment else t.inCall.assignment)
+ .flatMap(t => if (recursive) t.inAssignment else t.inCall.isAssignment)
  .target
 
  def identifierToFirstUsages(node: Identifier): List[Identifier] = node.refsTo.flatMap(identifiersFromCapturedScopes).l

joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/declarations/AstForMethodsCreator.scala

@@ -42,6 +42,7 @@ import io.shiftleft.codepropertygraph.generated.EdgeTypes
 import com.github.javaparser.ast.Node
 import com.github.javaparser.symbolsolver.javaparsermodel.declarations.JavaParserParameterDeclaration
 import io.joern.javasrc2cpg.astcreation.declarations.AstForMethodsCreator.PartialConstructorDeclaration
+import io.joern.javasrc2cpg.util.Util
 
 private[declarations] trait AstForMethodsCreator { this: AstCreator =>
  def astForMethod(methodDeclaration: MethodDeclaration): Ast = {
@@ -51,7 +52,7 @@ private[declarations] trait AstForMethodsCreator { this: AstCreator =>
 
  val maybeResolved = tryWithSafeStackOverflow(methodDeclaration.resolve())
  val expectedReturnType = Try(symbolSolver.toResolvedType(methodDeclaration.getType, classOf[ResolvedType])).toOption
- val simpleMethodReturnType = methodDeclaration.getTypeAsString()
+ val simpleMethodReturnType = Util.stripGenericTypes(methodDeclaration.getTypeAsString())
  val returnTypeFullName = expectedReturnType
  .flatMap(typeInfoCalc.fullName)
  .orElse(scope.lookupType(simpleMethodReturnType))
@@ -218,19 +219,14 @@ private[declarations] trait AstForMethodsCreator { this: AstCreator =>
  }
 
  private def astForParameter(parameter: Parameter, childNum: Int): Ast = {
- val maybeArraySuffix = if (parameter.isVarArgs) "[]" else ""
+ val maybeArraySuffix = if (parameter.isVarArgs) "[]" else ""
+ val rawParameterTypeName = Util.stripGenericTypes(parameter.getTypeAsString)
  val typeFullName =
  typeInfoCalc
  .fullName(parameter.getType)
- .orElse(scope.lookupType(parameter.getTypeAsString))
- // In a scenario where we have an import of an external type e.g. `import foo.bar.Baz` and
- // this parameter's type is e.g. `Baz<String>`, the lookup will fail. However, if we lookup
- // for `Baz` instead (i.e. without type arguments), then the lookup will succeed.
- .orElse(
- Try(parameter.getType.asClassOrInterfaceType).toOption.flatMap(t => scope.lookupType(t.getNameAsString))
- )
+ .orElse(scope.lookupType(rawParameterTypeName))
  .map(_ ++ maybeArraySuffix)
- .getOrElse(s"${Defines.UnresolvedNamespace}.${parameter.getTypeAsString}")
+ .getOrElse(s"${Defines.UnresolvedNamespace}.$rawParameterTypeName")
  val evalStrat =
  if (parameter.getType.isPrimitiveType) EvaluationStrategies.BY_VALUE else EvaluationStrategies.BY_SHARING
  typeInfoCalc.registerType(typeFullName)

joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/declarations/AstForTypeDeclsCreator.scala

@@ -568,34 +568,16 @@ private[declarations] trait AstForTypeDeclsCreator { this: AstCreator =>
  // TODO: Should be able to find expected type here
  val annotations = fieldDeclaration.getAnnotations
 
- // variable can be declared with generic type, so we need to get rid of the <> part of it to get the package information
- // and append the <> when forming the typeFullName again
- // Ex - private Consumer<String, Integer> consumer;
- // From Consumer<String, Integer> we need to get to Consumer so splitting it by '<' and then combining with '<' to
- // form typeFullName as Consumer<String, Integer>
+ val rawTypeName = Util.stripGenericTypes(v.getTypeAsString)
 
- val typeFullNameWithoutGenericSplit = typeInfoCalc
+ val typeFullName = typeInfoCalc
  .fullName(v.getType)
- .orElse(scope.lookupType(v.getTypeAsString))
- .getOrElse(s"${Defines.UnresolvedNamespace}.${v.getTypeAsString}")
- val typeFullName = {
- // Check if the typeFullName is unresolved and if it has generic information to resolve the typeFullName
- if (
- typeFullNameWithoutGenericSplit
- .contains(Defines.UnresolvedNamespace) && v.getTypeAsString.contains(Defines.LeftAngularBracket)
- ) {
- val splitByLeftAngular = v.getTypeAsString.split(Defines.LeftAngularBracket)
- scope.lookupType(splitByLeftAngular.head) match {
- case Some(foundType) =>
- foundType + splitByLeftAngular
- .slice(1, splitByLeftAngular.size)
- .mkString(Defines.LeftAngularBracket, Defines.LeftAngularBracket, "")
- case None => typeFullNameWithoutGenericSplit
- }
- } else typeFullNameWithoutGenericSplit
- }
- val name = v.getName.toString
- val node = memberNode(v, name, s"$typeFullName $name", typeFullName)
+ .orElse(scope.lookupType(rawTypeName))
+ .getOrElse(s"${Defines.UnresolvedNamespace}.$rawTypeName")
+
+ val name = v.getName.toString
+ // Use type name without generics stripped in code
+ val node = memberNode(v, name, s"${v.getTypeAsString} $name", typeFullName)
  val memberAst = Ast(node)
  val annotationAsts = annotations.asScala.map(astForAnnotationExpr)
 

joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/expressions/AstForCallExpressionsCreator.scala

@@ -20,7 +20,7 @@ import io.joern.javasrc2cpg.astcreation.expressions.AstForCallExpressionsCreator
 import io.joern.javasrc2cpg.astcreation.{AstCreator, ExpectedType}
 import io.joern.javasrc2cpg.scope.Scope.typeFullName
 import io.joern.javasrc2cpg.typesolvers.TypeInfoCalculator.TypeConstants
-import io.joern.javasrc2cpg.util.NameConstants
+import io.joern.javasrc2cpg.util.{NameConstants, Util}
 import io.joern.javasrc2cpg.util.Util.{composeMethodFullName, composeMethodLikeSignature, composeUnresolvedSignature}
 import io.joern.x2cpg.utils.AstPropertiesUtil.*
 import io.joern.x2cpg.utils.NodeBuilders.{newIdentifierNode, newOperatorCallNode}
@@ -192,9 +192,10 @@ trait AstForCallExpressionsCreator { this: AstCreator =>
 
  val anonymousClassBody = expr.getAnonymousClassBody.toScala.map(_.asScala.toList)
  val nameSuffix = if (anonymousClassBody.isEmpty) "" else s"$$${scope.getNextAnonymousClassIndex()}"
- val typeName = s"${expr.getTypeAsString}$nameSuffix"
+ val rawType = Util.stripGenericTypes(expr.getTypeAsString)
+ val typeName = s"$rawType$nameSuffix"
 
- val baseTypeFromScope = scope.lookupScopeType(expr.getTypeAsString)
+ val baseTypeFromScope = scope.lookupScopeType(rawType)
  // These will be the same for non-anonymous type decls, but in that case only the typeFullName will be used.
  val baseTypeFullName =
  baseTypeFromScope
@@ -455,6 +456,7 @@ trait AstForCallExpressionsCreator { this: AstCreator =>
  }
 
  case objectCreationExpr: ObjectCreationExpr =>
+ // Use type name with generics for code
  val typeName = objectCreationExpr.getTypeAsString
  val argumentsString = getArgumentCodeString(objectCreationExpr.getArguments)
  someWithDotSuffix(s"new $typeName($argumentsString)")

joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/expressions/AstForNameExpressionsCreator.scala

@@ -97,22 +97,17 @@ trait AstForNameExpressionsCreator { this: AstCreator =>
  val variable = capturedVariable.variable
  val typeDeclChain = capturedVariable.typeDeclChain
 
- scope.enclosingMethod.map(_.lookupVariable("this")) match {
- case None | Some(NotInScope) | Some(CapturedVariable(_, _)) =>
+ scope.lookupVariable("this") match {
+ case NotInScope | CapturedVariable(_, _) =>
  logger.warn(
  s"Attempted to create AST for captured variable ${variable.name}, but could not find `this` param in direct scope."
  )
- Ast(NewUnknown().code(variable.name).lineNumber(line(nameExpr)).columnNumber(column(nameExpr)))
-
- case Some(SimpleVariable(ScopeParameter(thisNode: NewMethodParameterIn))) =>
- val thisIdentifier = identifierNode(
- nameExpr,
- thisNode.name,
- thisNode.code,
- thisNode.typeFullName,
- thisNode.dynamicTypeHintFullName
- )
- val thisAst = Ast(thisIdentifier).withRefEdge(thisIdentifier, thisNode)
+ Ast(identifierNode(nameExpr, variable.name, variable.name, variable.typeFullName))
+
+ case SimpleVariable(scopeVariable) =>
+ val thisIdentifier =
+ identifierNode(nameExpr, scopeVariable.name, scopeVariable.name, scopeVariable.typeFullName)
+ val thisAst = Ast(thisIdentifier).withRefEdge(thisIdentifier, scopeVariable.node)
 
  val lineNumber = line(nameExpr)
  val columnNumber = column(nameExpr)
@@ -139,12 +134,6 @@ trait AstForNameExpressionsCreator { this: AstCreator =>
 
  val captureFieldIdentifier = fieldIdentifierNode(nameExpr, variable.name, variable.name)
  callAst(finalFieldAccess, List(outerClassChain, Ast(captureFieldIdentifier)))
-
- case Some(SimpleVariable(thisNode)) =>
- logger.warn(
- s"Attempted to create AST for captured variable ${variable.name}, but found non-parameter `this`: ${thisNode}."
- )
- Ast(NewUnknown().code(variable.name).lineNumber(line(nameExpr)).columnNumber(column(nameExpr)))
  }
  }
 }

joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/expressions/AstForSimpleExpressionsCreator.scala

@@ -204,8 +204,13 @@ trait AstForSimpleExpressionsCreator { this: AstCreator =>
  val callNode = newOperatorCallNode(Operators.fieldAccess, expr.toString, someTypeFullName, line(expr), column(expr))
 
  val identifierType = typeInfoCalc.fullName(expr.getType)
- val identifier = identifierNode(expr, expr.getTypeAsString, expr.getTypeAsString, identifierType.getOrElse("ANY"))
- val idAst = Ast(identifier)
+ val identifier = identifierNode(
+ expr,
+ Util.stripGenericTypes(expr.getTypeAsString),
+ expr.getTypeAsString,
+ identifierType.getOrElse("ANY")
+ )
+ val idAst = Ast(identifier)
 
  val fieldIdentifier = NewFieldIdentifier()
  .canonicalName("class")
@@ -388,8 +393,9 @@ trait AstForSimpleExpressionsCreator { this: AstCreator =>
  private[expressions] def astForMethodReferenceExpr(expr: MethodReferenceExpr, expectedType: ExpectedType): Ast = {
  val typeFullName = expr.getScope match {
  case typeExpr: TypeExpr =>
+ val rawType = Util.stripGenericTypes(typeExpr.getTypeAsString)
  // JavaParser wraps the "type" scope of a MethodReferenceExpr in a TypeExpr, but this also catches variable names.
- scope.lookupVariableOrType(typeExpr.getTypeAsString).orElse(expressionReturnTypeFullName(typeExpr))
+ scope.lookupVariableOrType(rawType).orElse(expressionReturnTypeFullName(typeExpr))
  case scopeExpr => expressionReturnTypeFullName(scopeExpr)
  }
 

joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/expressions/AstForVarDeclAndAssignsCreator.scala

@@ -9,7 +9,7 @@ import com.github.javaparser.resolution.types.ResolvedType
 import io.joern.javasrc2cpg.astcreation.{AstCreator, ExpectedType}
 import io.joern.javasrc2cpg.scope.Scope.{NewVariableNode, typeFullName}
 import io.joern.javasrc2cpg.typesolvers.TypeInfoCalculator.TypeConstants
-import io.joern.javasrc2cpg.util.NameConstants
+import io.joern.javasrc2cpg.util.{NameConstants, Util}
 import io.joern.x2cpg.utils.AstPropertiesUtil.*
 import io.joern.x2cpg.Ast
 import io.shiftleft.codepropertygraph.generated.nodes.{
@@ -113,7 +113,7 @@ trait AstForVarDeclAndAssignsCreator { this: AstCreator =>
  case typ: ClassOrInterfaceType =>
  val typeParams = typ.getTypeArguments.toScala.map(_.asScala.flatMap(typeInfoCalc.fullName))
  (typ.getName.asString(), typeParams)
- case _ => (variableDeclarator.getTypeAsString, None)
+ case _ => (Util.stripGenericTypes(variableDeclarator.getTypeAsString), None)
  }
 
  val typeFullName = tryWithSafeStackOverflow(
@@ -131,6 +131,7 @@ trait AstForVarDeclAndAssignsCreator { this: AstCreator =>
  val declarationNode: Option[NewVariableNode] = if (originNode.isInstanceOf[FieldDeclaration]) {
  scope.lookupVariable(variableName).variableNode
  } else {
+ // Use type name with generics for code
  val localCode = s"${variableDeclarator.getTypeAsString} ${variableDeclarator.getNameAsString}"
 
  val local =
@@ -181,7 +182,7 @@ trait AstForVarDeclAndAssignsCreator { this: AstCreator =>
  Operators.assignment,
  "=",
  ExpectedType(typeFullName, expectedType),
- Some(variableDeclarator.getTypeAsString)
+ Some(Util.stripGenericTypes(variableDeclarator.getTypeAsString))
  )
  }
 

joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/util/Util.scala

@@ -50,6 +50,19 @@ object Util {
  s"${Defines.UnresolvedSignature}($paramCount)"
  }
 
+ def stripGenericTypes(typeName: String): String = {
+ if (typeName.startsWith(Defines.UnresolvedNamespace)) {
+ logger.warn(s"stripGenericTypes should not be used for javasrc2cpg type $typeName")
+ typeName.head +: takeUntilGenericStart(typeName.tail)
+ } else {
+ takeUntilGenericStart(typeName)
+ }
+ }
+
+ private def takeUntilGenericStart(typeName: String): String = {
+ typeName.takeWhile(char => char != '<')
+ }
+
  private def getAllParents(typ: ResolvedReferenceType, result: mutable.ArrayBuffer[ResolvedReferenceType]): Unit = {
  if (typ.isJavaLangObject) {
  Iterable.empty

joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/EnumTests.scala

@@ -59,7 +59,7 @@ class EnumTests extends JavaSrcCode2CpgFixture {
  cpg.typeDecl.name(".*Color.*").member.size shouldBe 3
  val List(r, b, l) = cpg.typeDecl.name(".*Color.*").member.l
 
- l.code shouldBe "java.lang.String label"
+ l.code shouldBe "String label"
 
  r.code shouldBe "RED(\"Red\")"
  r.astChildren.size shouldBe 0