From 0658c5917e4d67cade3188b8cd0dae7d25b87efa Mon Sep 17 00:00:00 2001 From: David Baker Effendi Date: Thu, 30 May 2024 17:10:27 +0200 Subject: [PATCH] [x2cpg] Program Summary Mutable Merging As pointed out in https://github.com/joernio/joern/pull/4240, combining this nested immutable map-like structure has a quadratic performance, and the more performant strategy would be to use nested data-structures to merge. For now, I've decided not to opt for a builder pattern, but rather keep the underlying structure mutable, and accessor methods return immutable structures. --- .../io/joern/dataflowengineoss/package.scala | 2 +- .../declarations/AstForMethodsCreator.scala | 16 +- .../declarations/AstForTypeDeclsCreator.scala | 34 +- .../AstForCallExpressionsCreator.scala | 8 +- .../AstForNameExpressionsCreator.scala | 27 +- .../AstForSimpleExpressionsCreator.scala | 12 +- .../AstForVarDeclAndAssignsCreator.scala | 7 +- .../io/joern/javasrc2cpg/util/Util.scala | 13 + .../javasrc2cpg/querying/EnumTests.scala | 2 +- .../javasrc2cpg/querying/GenericsTests.scala | 401 +++++++++++------- .../javasrc2cpg/querying/LambdaTests.scala | 68 +++ .../javasrc2cpg/querying/MemberTests.scala | 4 +- .../javasrc2cpg/querying/VarDeclTests.scala | 2 +- .../pysrc2cpg/dataflow/DataFlowTests.scala | 62 ++- .../types/expressions/CallTraversal.scala | 7 + 15 files changed, 448 insertions(+), 217 deletions(-) diff --git a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/package.scala b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/package.scala index 0652d87bbf8..dc7c253d491 100644 --- a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/package.scala +++ b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/package.scala @@ -16,7 +16,7 @@ package object dataflowengineoss { */ def globalFromLiteral(lit: Literal, recursive: Boolean = true): Iterator[Expression] = lit.start .where(_.method.isModule) - .flatMap(t => if (recursive) t.inAssignment else t.inCall.assignment) + .flatMap(t => if (recursive) t.inAssignment else t.inCall.isAssignment) .target def identifierToFirstUsages(node: Identifier): List[Identifier] = node.refsTo.flatMap(identifiersFromCapturedScopes).l diff --git a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/declarations/AstForMethodsCreator.scala b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/declarations/AstForMethodsCreator.scala index 6a1a5d5b91a..55dbb42ec4c 100644 --- a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/declarations/AstForMethodsCreator.scala +++ b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/declarations/AstForMethodsCreator.scala @@ -42,6 +42,7 @@ import io.shiftleft.codepropertygraph.generated.EdgeTypes import com.github.javaparser.ast.Node import com.github.javaparser.symbolsolver.javaparsermodel.declarations.JavaParserParameterDeclaration import io.joern.javasrc2cpg.astcreation.declarations.AstForMethodsCreator.PartialConstructorDeclaration +import io.joern.javasrc2cpg.util.Util private[declarations] trait AstForMethodsCreator { this: AstCreator => def astForMethod(methodDeclaration: MethodDeclaration): Ast = { @@ -51,7 +52,7 @@ private[declarations] trait AstForMethodsCreator { this: AstCreator => val maybeResolved = tryWithSafeStackOverflow(methodDeclaration.resolve()) val expectedReturnType = Try(symbolSolver.toResolvedType(methodDeclaration.getType, classOf[ResolvedType])).toOption - val simpleMethodReturnType = methodDeclaration.getTypeAsString() + val simpleMethodReturnType = Util.stripGenericTypes(methodDeclaration.getTypeAsString()) val returnTypeFullName = expectedReturnType .flatMap(typeInfoCalc.fullName) .orElse(scope.lookupType(simpleMethodReturnType)) @@ -218,19 +219,14 @@ private[declarations] trait AstForMethodsCreator { this: AstCreator => } private def astForParameter(parameter: Parameter, childNum: Int): Ast = { - val maybeArraySuffix = if (parameter.isVarArgs) "[]" else "" + val maybeArraySuffix = if (parameter.isVarArgs) "[]" else "" + val rawParameterTypeName = Util.stripGenericTypes(parameter.getTypeAsString) val typeFullName = typeInfoCalc .fullName(parameter.getType) - .orElse(scope.lookupType(parameter.getTypeAsString)) - // In a scenario where we have an import of an external type e.g. `import foo.bar.Baz` and - // this parameter's type is e.g. `Baz`, the lookup will fail. However, if we lookup - // for `Baz` instead (i.e. without type arguments), then the lookup will succeed. - .orElse( - Try(parameter.getType.asClassOrInterfaceType).toOption.flatMap(t => scope.lookupType(t.getNameAsString)) - ) + .orElse(scope.lookupType(rawParameterTypeName)) .map(_ ++ maybeArraySuffix) - .getOrElse(s"${Defines.UnresolvedNamespace}.${parameter.getTypeAsString}") + .getOrElse(s"${Defines.UnresolvedNamespace}.$rawParameterTypeName") val evalStrat = if (parameter.getType.isPrimitiveType) EvaluationStrategies.BY_VALUE else EvaluationStrategies.BY_SHARING typeInfoCalc.registerType(typeFullName) diff --git a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/declarations/AstForTypeDeclsCreator.scala b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/declarations/AstForTypeDeclsCreator.scala index 25733602ccb..f4445b10268 100644 --- a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/declarations/AstForTypeDeclsCreator.scala +++ b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/declarations/AstForTypeDeclsCreator.scala @@ -568,34 +568,16 @@ private[declarations] trait AstForTypeDeclsCreator { this: AstCreator => // TODO: Should be able to find expected type here val annotations = fieldDeclaration.getAnnotations - // variable can be declared with generic type, so we need to get rid of the <> part of it to get the package information - // and append the <> when forming the typeFullName again - // Ex - private Consumer consumer; - // From Consumer we need to get to Consumer so splitting it by '<' and then combining with '<' to - // form typeFullName as Consumer + val rawTypeName = Util.stripGenericTypes(v.getTypeAsString) - val typeFullNameWithoutGenericSplit = typeInfoCalc + val typeFullName = typeInfoCalc .fullName(v.getType) - .orElse(scope.lookupType(v.getTypeAsString)) - .getOrElse(s"${Defines.UnresolvedNamespace}.${v.getTypeAsString}") - val typeFullName = { - // Check if the typeFullName is unresolved and if it has generic information to resolve the typeFullName - if ( - typeFullNameWithoutGenericSplit - .contains(Defines.UnresolvedNamespace) && v.getTypeAsString.contains(Defines.LeftAngularBracket) - ) { - val splitByLeftAngular = v.getTypeAsString.split(Defines.LeftAngularBracket) - scope.lookupType(splitByLeftAngular.head) match { - case Some(foundType) => - foundType + splitByLeftAngular - .slice(1, splitByLeftAngular.size) - .mkString(Defines.LeftAngularBracket, Defines.LeftAngularBracket, "") - case None => typeFullNameWithoutGenericSplit - } - } else typeFullNameWithoutGenericSplit - } - val name = v.getName.toString - val node = memberNode(v, name, s"$typeFullName $name", typeFullName) + .orElse(scope.lookupType(rawTypeName)) + .getOrElse(s"${Defines.UnresolvedNamespace}.$rawTypeName") + + val name = v.getName.toString + // Use type name without generics stripped in code + val node = memberNode(v, name, s"${v.getTypeAsString} $name", typeFullName) val memberAst = Ast(node) val annotationAsts = annotations.asScala.map(astForAnnotationExpr) diff --git a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/expressions/AstForCallExpressionsCreator.scala b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/expressions/AstForCallExpressionsCreator.scala index 7fcd0db7d07..127eb6824b1 100644 --- a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/expressions/AstForCallExpressionsCreator.scala +++ b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/expressions/AstForCallExpressionsCreator.scala @@ -20,7 +20,7 @@ import io.joern.javasrc2cpg.astcreation.expressions.AstForCallExpressionsCreator import io.joern.javasrc2cpg.astcreation.{AstCreator, ExpectedType} import io.joern.javasrc2cpg.scope.Scope.typeFullName import io.joern.javasrc2cpg.typesolvers.TypeInfoCalculator.TypeConstants -import io.joern.javasrc2cpg.util.NameConstants +import io.joern.javasrc2cpg.util.{NameConstants, Util} import io.joern.javasrc2cpg.util.Util.{composeMethodFullName, composeMethodLikeSignature, composeUnresolvedSignature} import io.joern.x2cpg.utils.AstPropertiesUtil.* import io.joern.x2cpg.utils.NodeBuilders.{newIdentifierNode, newOperatorCallNode} @@ -192,9 +192,10 @@ trait AstForCallExpressionsCreator { this: AstCreator => val anonymousClassBody = expr.getAnonymousClassBody.toScala.map(_.asScala.toList) val nameSuffix = if (anonymousClassBody.isEmpty) "" else s"$$${scope.getNextAnonymousClassIndex()}" - val typeName = s"${expr.getTypeAsString}$nameSuffix" + val rawType = Util.stripGenericTypes(expr.getTypeAsString) + val typeName = s"$rawType$nameSuffix" - val baseTypeFromScope = scope.lookupScopeType(expr.getTypeAsString) + val baseTypeFromScope = scope.lookupScopeType(rawType) // These will be the same for non-anonymous type decls, but in that case only the typeFullName will be used. val baseTypeFullName = baseTypeFromScope @@ -455,6 +456,7 @@ trait AstForCallExpressionsCreator { this: AstCreator => } case objectCreationExpr: ObjectCreationExpr => + // Use type name with generics for code val typeName = objectCreationExpr.getTypeAsString val argumentsString = getArgumentCodeString(objectCreationExpr.getArguments) someWithDotSuffix(s"new $typeName($argumentsString)") diff --git a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/expressions/AstForNameExpressionsCreator.scala b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/expressions/AstForNameExpressionsCreator.scala index d3f17a2ad8f..8b8ad10c6a3 100644 --- a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/expressions/AstForNameExpressionsCreator.scala +++ b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/expressions/AstForNameExpressionsCreator.scala @@ -97,22 +97,17 @@ trait AstForNameExpressionsCreator { this: AstCreator => val variable = capturedVariable.variable val typeDeclChain = capturedVariable.typeDeclChain - scope.enclosingMethod.map(_.lookupVariable("this")) match { - case None | Some(NotInScope) | Some(CapturedVariable(_, _)) => + scope.lookupVariable("this") match { + case NotInScope | CapturedVariable(_, _) => logger.warn( s"Attempted to create AST for captured variable ${variable.name}, but could not find `this` param in direct scope." ) - Ast(NewUnknown().code(variable.name).lineNumber(line(nameExpr)).columnNumber(column(nameExpr))) - - case Some(SimpleVariable(ScopeParameter(thisNode: NewMethodParameterIn))) => - val thisIdentifier = identifierNode( - nameExpr, - thisNode.name, - thisNode.code, - thisNode.typeFullName, - thisNode.dynamicTypeHintFullName - ) - val thisAst = Ast(thisIdentifier).withRefEdge(thisIdentifier, thisNode) + Ast(identifierNode(nameExpr, variable.name, variable.name, variable.typeFullName)) + + case SimpleVariable(scopeVariable) => + val thisIdentifier = + identifierNode(nameExpr, scopeVariable.name, scopeVariable.name, scopeVariable.typeFullName) + val thisAst = Ast(thisIdentifier).withRefEdge(thisIdentifier, scopeVariable.node) val lineNumber = line(nameExpr) val columnNumber = column(nameExpr) @@ -139,12 +134,6 @@ trait AstForNameExpressionsCreator { this: AstCreator => val captureFieldIdentifier = fieldIdentifierNode(nameExpr, variable.name, variable.name) callAst(finalFieldAccess, List(outerClassChain, Ast(captureFieldIdentifier))) - - case Some(SimpleVariable(thisNode)) => - logger.warn( - s"Attempted to create AST for captured variable ${variable.name}, but found non-parameter `this`: ${thisNode}." - ) - Ast(NewUnknown().code(variable.name).lineNumber(line(nameExpr)).columnNumber(column(nameExpr))) } } } diff --git a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/expressions/AstForSimpleExpressionsCreator.scala b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/expressions/AstForSimpleExpressionsCreator.scala index d2d4c3677b4..566c6a18e9c 100644 --- a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/expressions/AstForSimpleExpressionsCreator.scala +++ b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/expressions/AstForSimpleExpressionsCreator.scala @@ -204,8 +204,13 @@ trait AstForSimpleExpressionsCreator { this: AstCreator => val callNode = newOperatorCallNode(Operators.fieldAccess, expr.toString, someTypeFullName, line(expr), column(expr)) val identifierType = typeInfoCalc.fullName(expr.getType) - val identifier = identifierNode(expr, expr.getTypeAsString, expr.getTypeAsString, identifierType.getOrElse("ANY")) - val idAst = Ast(identifier) + val identifier = identifierNode( + expr, + Util.stripGenericTypes(expr.getTypeAsString), + expr.getTypeAsString, + identifierType.getOrElse("ANY") + ) + val idAst = Ast(identifier) val fieldIdentifier = NewFieldIdentifier() .canonicalName("class") @@ -388,8 +393,9 @@ trait AstForSimpleExpressionsCreator { this: AstCreator => private[expressions] def astForMethodReferenceExpr(expr: MethodReferenceExpr, expectedType: ExpectedType): Ast = { val typeFullName = expr.getScope match { case typeExpr: TypeExpr => + val rawType = Util.stripGenericTypes(typeExpr.getTypeAsString) // JavaParser wraps the "type" scope of a MethodReferenceExpr in a TypeExpr, but this also catches variable names. - scope.lookupVariableOrType(typeExpr.getTypeAsString).orElse(expressionReturnTypeFullName(typeExpr)) + scope.lookupVariableOrType(rawType).orElse(expressionReturnTypeFullName(typeExpr)) case scopeExpr => expressionReturnTypeFullName(scopeExpr) } diff --git a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/expressions/AstForVarDeclAndAssignsCreator.scala b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/expressions/AstForVarDeclAndAssignsCreator.scala index 5445ed980a0..205dc5ef2ac 100644 --- a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/expressions/AstForVarDeclAndAssignsCreator.scala +++ b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/expressions/AstForVarDeclAndAssignsCreator.scala @@ -9,7 +9,7 @@ import com.github.javaparser.resolution.types.ResolvedType import io.joern.javasrc2cpg.astcreation.{AstCreator, ExpectedType} import io.joern.javasrc2cpg.scope.Scope.{NewVariableNode, typeFullName} import io.joern.javasrc2cpg.typesolvers.TypeInfoCalculator.TypeConstants -import io.joern.javasrc2cpg.util.NameConstants +import io.joern.javasrc2cpg.util.{NameConstants, Util} import io.joern.x2cpg.utils.AstPropertiesUtil.* import io.joern.x2cpg.Ast import io.shiftleft.codepropertygraph.generated.nodes.{ @@ -113,7 +113,7 @@ trait AstForVarDeclAndAssignsCreator { this: AstCreator => case typ: ClassOrInterfaceType => val typeParams = typ.getTypeArguments.toScala.map(_.asScala.flatMap(typeInfoCalc.fullName)) (typ.getName.asString(), typeParams) - case _ => (variableDeclarator.getTypeAsString, None) + case _ => (Util.stripGenericTypes(variableDeclarator.getTypeAsString), None) } val typeFullName = tryWithSafeStackOverflow( @@ -131,6 +131,7 @@ trait AstForVarDeclAndAssignsCreator { this: AstCreator => val declarationNode: Option[NewVariableNode] = if (originNode.isInstanceOf[FieldDeclaration]) { scope.lookupVariable(variableName).variableNode } else { + // Use type name with generics for code val localCode = s"${variableDeclarator.getTypeAsString} ${variableDeclarator.getNameAsString}" val local = @@ -181,7 +182,7 @@ trait AstForVarDeclAndAssignsCreator { this: AstCreator => Operators.assignment, "=", ExpectedType(typeFullName, expectedType), - Some(variableDeclarator.getTypeAsString) + Some(Util.stripGenericTypes(variableDeclarator.getTypeAsString)) ) } diff --git a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/util/Util.scala b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/util/Util.scala index ac849eb464e..8bc1e05e920 100644 --- a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/util/Util.scala +++ b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/util/Util.scala @@ -50,6 +50,19 @@ object Util { s"${Defines.UnresolvedSignature}($paramCount)" } + def stripGenericTypes(typeName: String): String = { + if (typeName.startsWith(Defines.UnresolvedNamespace)) { + logger.warn(s"stripGenericTypes should not be used for javasrc2cpg type $typeName") + typeName.head +: takeUntilGenericStart(typeName.tail) + } else { + takeUntilGenericStart(typeName) + } + } + + private def takeUntilGenericStart(typeName: String): String = { + typeName.takeWhile(char => char != '<') + } + private def getAllParents(typ: ResolvedReferenceType, result: mutable.ArrayBuffer[ResolvedReferenceType]): Unit = { if (typ.isJavaLangObject) { Iterable.empty diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/EnumTests.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/EnumTests.scala index 3d111c7ccfb..84e8ed5e3df 100644 --- a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/EnumTests.scala +++ b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/EnumTests.scala @@ -59,7 +59,7 @@ class EnumTests extends JavaSrcCode2CpgFixture { cpg.typeDecl.name(".*Color.*").member.size shouldBe 3 val List(r, b, l) = cpg.typeDecl.name(".*Color.*").member.l - l.code shouldBe "java.lang.String label" + l.code shouldBe "String label" r.code shouldBe "RED(\"Red\")" r.astChildren.size shouldBe 0 diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/GenericsTests.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/GenericsTests.scala index 8e7d24934c0..609e8789244 100644 --- a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/GenericsTests.scala +++ b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/GenericsTests.scala @@ -4,195 +4,302 @@ import io.joern.javasrc2cpg.testfixtures.JavaSrcCode2CpgFixture import io.shiftleft.semanticcpg.language._ class GenericsTests extends JavaSrcCode2CpgFixture { + "unresolved generic type declarations" should { + val cpg = code("""import box.Box; + | + |public class Foo { + | public static void test() { + | Box b = new Box<>(0); + | b.get(); + | } + |} + |""".stripMargin) - val cpg = code(""" - |import java.util.function.Function; - | - |// Box - |class Box { - | - | // java.lang.Object - | private T item; - | - | // Box.getItem:java.lang.Object() - | public T getItem() { - | return item; - | } - | - | public void setItem(T item) { - | this.item = item; - | } - | - | // Box.map:Box(java.util.function.Function) - | public Box map(Function f) { - | // java.util.function.Function.apply: java.lang.Object(java.lang.Object) - | G newValue = f.apply(item); - | // Box.:void() - | Box newBox = new Box(); - | return newBox.withValue(newValue); - | } - | - | // Box.withValue:Box(java.lang.Object) - | public Box withValue(T value) { - | this.item = value; - | return this; - | } - | - | public String toString() { - | return "Box(" + item.toString() + ")"; - | } - | - | // Box.idk:java.lang.Number(java.lang.Number) - | public static K idK(K item) { - | return item; - | } - | - | // Box.idKC:java.lang.Number(java.lang.Number) - | public static K idKC(K item) { - | return item; - | } - | - | // Box.idC:java.lang.Comparable(java.lang.Comparable) - | public static K idC(K item) { - | return item; - | } - | - | // Box.testWildCard:void(Box) - | public static void testWildCard(Box b) { - | System.out.println(b); - | } - | - | // Box.testWildCardLower:void(Box) - | public static void testWildCardLower(Box b) { - | System.out.println(b); - | } - |} - | - | - |// inheritsFrom Box - |public class Test extends Box {} - |""".stripMargin) + "use erased types for the method full name in the constructor invocation" in { + cpg.call.nameExact("").methodFullName.l shouldBe List("box.Box.:(1)") + } - "it should create the correct generic typeDecl name" in { - cpg.typeDecl.nameExact("Box").l match { - case decl :: Nil => decl.fullName shouldBe "Box" + "use erased types for the method full name in the get method invocation" in { + cpg.call.nameExact("get").methodFullName.l shouldBe List("box.Box.get:(0)") + } - case res => fail(s"Expected typeDecl Box but got $res") + "not include generic types in type full names of objects" in { + cpg.method.name("test").local.name("b").typeFullName.l shouldBe List("box.Box") } } - "it should default to Object for a simple generic type" in { - cpg.method.name("getItem").l match { - case method :: Nil => - method.fullName shouldBe "Box.getItem:java.lang.Object()" - method.signature shouldBe "java.lang.Object()" + "generic methods" should { + val cpg = code("""package foo; + | + |class Foo { + | public T foo(S s) { return null; } + | + | static void test(Foo f) { + | f.foo(0); + | } + |} + |""".stripMargin) + + "use erased types in the method fullName" in { + cpg.method.name("foo").fullName.l shouldBe List("foo.Foo.foo:java.lang.Object(java.lang.Object)") + } - case res => fail(s"Expected method getItem but got $res") + "use erased types in the call methodFullName" in { + cpg.method.name("test").call.name("foo").methodFullName.l shouldBe List( + "foo.Foo.foo:java.lang.Object(java.lang.Object)" + ) } } - "it should default to Object for simple generic parameters" in { - cpg.method.name("setItem").l match { - case method :: Nil => - method.fullName shouldBe "Box.setItem:void(java.lang.Object)" - method.signature shouldBe "void(java.lang.Object)" + "methods returning parameterized types" should { + val cpg = code("""package foo; + | + |class Box { + | public Box into() { return null; } + | + | public T get() { return null; } + | + | static void test(Box stringBox) { + | stringBox.into().get(); + | } + |} + |""".stripMargin) - case res => fail(s"Expected method setItem but got $res") + "not have the generic types in the methodFullName for calls" in { + cpg.call.name("into").methodFullName.l shouldBe List("foo.Box.into:foo.Box()") + + cpg.call.name("get").methodFullName.l shouldBe List("foo.Box.get:java.lang.Object()") } - cpg.method.name("setItem").parameter.name("item").l match { - case node :: Nil => - node.typeFullName shouldBe "java.lang.Object" + } + + "unresolved generic variable types" should { + val cpg = code("""package foo; + |import a.*; + |import b.*; + | + |class Foo { + | + | void foo(Bar b) { + | b.bar(); + | } + |} + |""".stripMargin) - case res => fail(s"Expected param item but got $res") + "contain generic type information in the variable typeFullName" in { + cpg.method.name("foo").parameter.name("b").typeFullName.l shouldBe List(".Bar") } - } - "it should erase generic types in parameters" in { - val List(method) = cpg.method.name("map").l - method.fullName shouldBe "Box.map:Box(java.util.function.Function)" - method.signature shouldBe "Box(java.util.function.Function)" + "not contain generic type information in the bar call methodFullName" in { + cpg.call.name("bar").methodFullName.l shouldBe List(".Bar.bar:(0)") + } - val List(param) = cpg.method.name("map").parameter.name("f").l - param.typeFullName shouldBe "java.util.function.Function" } - "it should create correct constructor calls" in { - cpg.method.name("map").call.nameExact(io.joern.x2cpg.Defines.ConstructorMethodName).l match { - case const :: Nil => - const.methodFullName shouldBe s"Box.${io.joern.x2cpg.Defines.ConstructorMethodName}:void()" - const.signature shouldBe "void()" + "fields with generic types" should { + val cpg = code(""" + |package foo; + |class Box {} + | + |class Foo { + | Box box; + |} + |""".stripMargin) - case res => fail(s"Expected call to but got $res") + "not have the generic types in field type full names" in { + cpg.typeDecl.name("Foo").member.name("box").typeFullName.l shouldBe List("foo.Box") } } - "it should correctly handle generic return types" in { - cpg.method.name("withValue").l match { - case method :: Nil => - method.fullName shouldBe "Box.withValue:Box(java.lang.Object)" - method.signature shouldBe "Box(java.lang.Object)" + "old generics tests" should { + val cpg = code("""import java.util.function.Function; + | + |// Box + |class Box { + | + | // java.lang.Object + | private T item; + | + | // Box.getItem:java.lang.Object() + | public T getItem() { + | return item; + | } + | + | public void setItem(T item) { + | this.item = item; + | } + | + | // Box.map:Box(java.util.function.Function) + | public Box map(Function f) { + | // java.util.function.Function.apply: java.lang.Object(java.lang.Object) + | G newValue = f.apply(item); + | // Box.:void() + | Box newBox = new Box(); + | return newBox.withValue(newValue); + | } + | + | // Box.withValue:Box(java.lang.Object) + | public Box withValue(T value) { + | this.item = value; + | return this; + | } + | + | public String toString() { + | return "Box(" + item.toString() + ")"; + | } + | + | // Box.idk:java.lang.Number(java.lang.Number) + | public static K idK(K item) { + | return item; + | } + | + | // Box.idKC:java.lang.Number(java.lang.Number) + | public static K idKC(K item) { + | return item; + | } + | + | // Box.idC:java.lang.Comparable(java.lang.Comparable) + | public static K idC(K item) { + | return item; + | } + | + | // Box.testWildCard:void(Box) + | public static void testWildCard(Box b) { + | System.out.println(b); + | } + | + | // Box.testWildCardLower:void(Box) + | public static void testWildCardLower(Box b) { + | System.out.println(b); + | } + |} + | + | + |// inheritsFrom Box + |public class Test extends Box {} + |""".stripMargin) - case res => fail(s"Expected method withValue but got $res") + "it should create the correct generic typeDecl name" in { + cpg.typeDecl.nameExact("Box").l match { + case decl :: Nil => decl.fullName shouldBe "Box" + + case res => fail(s"Expected typeDecl Box but got $res") + } } - } - "it should handle generics with upper bounds" in { - cpg.method.name("idK").l match { - case method :: Nil => - method.fullName shouldBe "Box.idK:java.lang.Number(java.lang.Number)" - method.signature shouldBe "java.lang.Number(java.lang.Number)" + "it should default to Object for a simple generic type" in { + cpg.method.name("getItem").l match { + case method :: Nil => + method.fullName shouldBe "Box.getItem:java.lang.Object()" + method.signature shouldBe "java.lang.Object()" - case res => fail(s"Expected method idK but found $res") + case res => fail(s"Expected method getItem but got $res") + } } - } - "it should handle generics with compound upper bounds" in { - cpg.method.name("idKC").l match { - case method :: Nil => - method.fullName shouldBe "Box.idKC:java.lang.Number(java.lang.Number)" - method.signature shouldBe "java.lang.Number(java.lang.Number)" + "it should default to Object for simple generic parameters" in { + cpg.method.name("setItem").l match { + case method :: Nil => + method.fullName shouldBe "Box.setItem:void(java.lang.Object)" + method.signature shouldBe "void(java.lang.Object)" + + case res => fail(s"Expected method setItem but got $res") + } + + cpg.method.name("setItem").parameter.name("item").l match { + case node :: Nil => + node.typeFullName shouldBe "java.lang.Object" - case res => fail(s"Expected method idKC but found $res") + case res => fail(s"Expected param item but got $res") + } } - } - "it should handle generics with an interface upper bound" in { - cpg.method.name("idC").l match { - case method :: Nil => - method.fullName shouldBe "Box.idC:java.lang.Comparable(java.lang.Comparable)" - method.signature shouldBe "java.lang.Comparable(java.lang.Comparable)" + "it should erase generic types in parameters" in { + val List(method) = cpg.method.name("map").l + method.fullName shouldBe "Box.map:Box(java.util.function.Function)" + method.signature shouldBe "Box(java.util.function.Function)" - case res => fail(s"Expected method idC but found $res") + val List(param) = cpg.method.name("map").parameter.name("f").l + param.typeFullName shouldBe "java.util.function.Function" } - } - "it should handle wildcard subclass generics" in { - cpg.method.name("testWildCard").l match { - case method :: Nil => - method.fullName shouldBe "Box.testWildCard:void(Box)" - method.signature shouldBe "void(Box)" + "it should create correct constructor calls" in { + cpg.method.name("map").call.nameExact(io.joern.x2cpg.Defines.ConstructorMethodName).l match { + case const :: Nil => + const.methodFullName shouldBe s"Box.${io.joern.x2cpg.Defines.ConstructorMethodName}:void()" + const.signature shouldBe "void()" - case res => fail(s"Expected method testWildCard but found $res") + case res => fail(s"Expected call to but got $res") + } } - } - "it should handle wildcard superclass generics" in { - cpg.method.name("testWildCardLower").l match { - case method :: Nil => - method.fullName shouldBe "Box.testWildCardLower:void(Box)" - method.signature shouldBe "void(Box)" + "it should correctly handle generic return types" in { + cpg.method.name("withValue").l match { + case method :: Nil => + method.fullName shouldBe "Box.withValue:Box(java.lang.Object)" + method.signature shouldBe "Box(java.lang.Object)" - case res => fail(s"Expected method testWildCardLower but found $res") + case res => fail(s"Expected method withValue but got $res") + } + } + + "it should handle generics with upper bounds" in { + cpg.method.name("idK").l match { + case method :: Nil => + method.fullName shouldBe "Box.idK:java.lang.Number(java.lang.Number)" + method.signature shouldBe "java.lang.Number(java.lang.Number)" + + case res => fail(s"Expected method idK but found $res") + } + } + + "it should handle generics with compound upper bounds" in { + cpg.method.name("idKC").l match { + case method :: Nil => + method.fullName shouldBe "Box.idKC:java.lang.Number(java.lang.Number)" + method.signature shouldBe "java.lang.Number(java.lang.Number)" + + case res => fail(s"Expected method idKC but found $res") + } + } + + "it should handle generics with an interface upper bound" in { + cpg.method.name("idC").l match { + case method :: Nil => + method.fullName shouldBe "Box.idC:java.lang.Comparable(java.lang.Comparable)" + method.signature shouldBe "java.lang.Comparable(java.lang.Comparable)" + + case res => fail(s"Expected method idC but found $res") + } + } + + "it should handle wildcard subclass generics" in { + cpg.method.name("testWildCard").l match { + case method :: Nil => + method.fullName shouldBe "Box.testWildCard:void(Box)" + method.signature shouldBe "void(Box)" + + case res => fail(s"Expected method testWildCard but found $res") + } + } + + "it should handle wildcard superclass generics" in { + cpg.method.name("testWildCardLower").l match { + case method :: Nil => + method.fullName shouldBe "Box.testWildCardLower:void(Box)" + method.signature shouldBe "void(Box)" + + case res => fail(s"Expected method testWildCardLower but found $res") + } } - } - "it should handle generic inheritance" in { - cpg.typeDecl.name("Test").l match { - case decl :: Nil => - decl.inheritsFromTypeFullName.head shouldBe "Box" + "it should handle generic inheritance" in { + cpg.typeDecl.name("Test").l match { + case decl :: Nil => + decl.inheritsFromTypeFullName.head shouldBe "Box" - case res => fail(s"Expected typeDecl Test but found $res") + case res => fail(s"Expected typeDecl Test but found $res") + } } } diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/LambdaTests.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/LambdaTests.scala index 7a67dd20ef0..aecb56aa7fd 100644 --- a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/LambdaTests.scala +++ b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/LambdaTests.scala @@ -742,4 +742,72 @@ class LambdaTests extends JavaSrcCode2CpgFixture { cpg.call.nameExact("").count(_.argument.isEmpty) shouldBe 0 } } + + "calls on captured variables in lambdas contained in anonymous classes" should { + val cpg = code(""" + | + |public class Foo { + | + | public static void sink(String s) {}; + | + | public static Object test(Bar captured) { + | Visitor v = new Visitor() { + | public void visit(Visited visited) { + | visited.getList().forEach(lambdaParam -> captured.remove(lambdaParam)); + | } + | }; + | } + |} + |""".stripMargin) + + // TODO: This behaviour isn't exactly correct, but is on par with how we currently handle field captures in lambdas. + "have the correct receiver ast" in { + inside(cpg.call.name("remove").receiver.l) { case List(fieldAccessCall: Call) => + fieldAccessCall.name shouldBe Operators.fieldAccess + + inside(fieldAccessCall.argument.l) { case List(identifier: Identifier, fieldIdentifier: FieldIdentifier) => + identifier.name shouldBe "this" + identifier.typeFullName shouldBe "Foo.test.Visitor$0" + + fieldIdentifier.canonicalName shouldBe "captured" + + fieldAccessCall.typeFullName shouldBe ".Bar" + } + } + } + } + + // TODO: These tests exist to document current behaviour, but the current behaviour is wrong. + "lambdas capturing parameters" should { + val cpg = code(""" + |import java.util.function.Consumer; + | + |public class Foo { + | public String capturedField; + | + | public void foo() { + | Consumer consumer = lambdaParam -> System.out.println(capturedField); + | } + |} + |""".stripMargin) + + "represent the captured field as a field access" in { + inside(cpg.method.name(".*lambda.*").call.name("println").argument.l) { case List(_, fieldAccessCall: Call) => + fieldAccessCall.name shouldBe Operators.fieldAccess + + inside(fieldAccessCall.argument.l) { case List(identifier: Identifier, fieldIdentifier: FieldIdentifier) => + identifier.name shouldBe "this" + identifier.typeFullName shouldBe "Foo" + + fieldIdentifier.canonicalName shouldBe "capturedField" + } + } + } + + // TODO: It should, but it doesn't. + "have a captured local for the enclosing class" in { + // There should be an `outerClass` local which captures the outer method `this`. + cpg.method.name(".*lambda.*").local.name("this").typeFullName(".*Foo.*").isEmpty shouldBe true + } + } } diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/MemberTests.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/MemberTests.scala index 0a82f5ac306..7ee3291c760 100644 --- a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/MemberTests.scala +++ b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/MemberTests.scala @@ -82,7 +82,7 @@ class NewMemberTests extends JavaSrcCode2CpgFixture { cpg.member .name("consumer") .typeFullName - .head shouldBe "org.apache.kafka.clients.consumer.Consumer" + .head shouldBe "org.apache.kafka.clients.consumer.Consumer" } "have a resolved package name in methodFullName" in { @@ -91,7 +91,7 @@ class NewMemberTests extends JavaSrcCode2CpgFixture { .methodFullName .head .split(":") - .head shouldBe "org.apache.kafka.clients.consumer.Consumer.poll" + .head shouldBe "org.apache.kafka.clients.consumer.Consumer.poll" } } diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/VarDeclTests.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/VarDeclTests.scala index b058a3db581..a14cfa4e478 100644 --- a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/VarDeclTests.scala +++ b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/VarDeclTests.scala @@ -203,7 +203,7 @@ class VarDeclTests extends JavaSrcCode2CpgFixture { .codeExact("new FlinkKafkaProducer(\"kafka-topic\", schema, kafkaProps)") .filterNot(_.name == Operators.alloc) .map(_.methodFullName) - .head shouldBe "org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer.:(3)" + .head shouldBe "org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer.:(3)" } } diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/dataflow/DataFlowTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/dataflow/DataFlowTests.scala index 40121307df0..244ef8bb61f 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/dataflow/DataFlowTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/dataflow/DataFlowTests.scala @@ -1,7 +1,7 @@ package io.joern.pysrc2cpg.dataflow import io.joern.dataflowengineoss.language.toExtendedCfgNode -import io.joern.dataflowengineoss.semanticsloader.FlowSemantic +import io.joern.dataflowengineoss.semanticsloader.{FlowMapping, FlowSemantic, PassThroughMapping} import io.joern.pysrc2cpg.PySrc2CpgFixture import io.shiftleft.codepropertygraph.Cpg import io.shiftleft.codepropertygraph.generated.nodes.{Literal, Member, Method} @@ -452,6 +452,66 @@ class DataFlowTests extends PySrc2CpgFixture(withOssDataflow = true) { flow shouldBe List(("a.run(\"X\")", 4), ("x = a.run(\"X\")", 4), ("tmp0[\"x\"] = x", 6)) } + "flow from literals in dictionary literal assignment and first argument to second argument" in { + val cpg = code(""" + |x = {'x': 10} + |print(1, x) + |""".stripMargin) + + def source = cpg.literal + def sink = cpg.call("print").argument(2) + val List(flow1, flow10) = sink.reachableByFlows(source).map(flowToResultPairs).sortBy(_.length).l + flow1 shouldBe List(("print(1, x)", 3)) + flow10 shouldBe List( + ("tmp0['x'] = 10", 2), + ("tmp0", 2), + ("x = tmp0 = {}\ntmp0['x'] = 10\ntmp0", 2), + ("print(1, x)", 3) + ) + } + + "flow from literal in dictionary literal assignment to second argument, using custom flows" in { + val cpg = code(""" + |x = {'x': 10} + |print(1, x) + |""".stripMargin) + .withExtraFlows(List(FlowSemantic(".*print", List(PassThroughMapping), true))) + + def source = cpg.literal + def sink = cpg.call("print").argument.argumentIndex(2) + val List(flow10) = sink.reachableByFlows(source).map(flowToResultPairs).l + flow10 shouldBe List( + ("tmp0['x'] = 10", 2), + ("tmp0", 2), + ("x = tmp0 = {}\ntmp0['x'] = 10\ntmp0", 2), + ("print(1, x)", 3) + ) + } + + "flow from literals in dictionary literal and first argument to second argument" in { + val cpg = code(""" + |print(1, {'x': 10}) + |""".stripMargin) + + def source = cpg.literal + def sink = cpg.call("print").argument.argumentIndex(2) + val List(flow10) = sink.reachableByFlows(source).map(flowToResultPairs).l + flow10 shouldBe List(("tmp0['x'] = 10", 2), ("tmp0", 2)) + } + + "flow from literals into a dictionary literal used as an argument to an external call" in { + val cpg = code(""" + |import bar + |x = 100 + |bar.foo('D').baz(A='A', B=b, C={'Property': x}) + |""".stripMargin) + + def source = cpg.literal + def sink = cpg.call("baz").argument.argumentName("C") + val List(flow100) = sink.reachableByFlows(source).map(flowToResultPairs).l + flow100 shouldBe List(("x = 100", 3), ("tmp0['Property'] = x", 4), ("tmp0", 4)) + } + "flow from global variable defined in imported file and used as argument to `print`" in { val cpg = code(""" |from models import FOOBAR diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/CallTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/CallTraversal.scala index ef233713389..33e47f74f37 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/CallTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/CallTraversal.scala @@ -2,6 +2,8 @@ package io.shiftleft.semanticcpg.language.types.expressions import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.semanticcpg.language.* +import io.shiftleft.semanticcpg.language.operatorextension.OpNodes.Assignment +import io.shiftleft.semanticcpg.language.operatorextension.allAssignmentTypes /** A call site */ @@ -17,6 +19,11 @@ class CallTraversal(val traversal: Iterator[Call]) extends AnyVal { def isDynamic: Iterator[Call] = traversal.dispatchType("DYNAMIC_DISPATCH") + /** Only assignment calls + */ + def isAssignment: Iterator[Assignment] = + traversal.methodFullNameExact(allAssignmentTypes.toSeq*).collectAll[Assignment] + /** The receiver of a call if the call has a receiver associated. */ def receiver: Iterator[Expression] =