Engine tweaks for performance. Removed logging from more places. (#114)

* Engine tweaks for performance. Removed logging from more places. Fixes #113.

Signed-off-by: Prabhu Subramanian <[email protected]>

* Remove logging

Signed-off-by: Prabhu Subramanian <[email protected]>

---------

Signed-off-by: Prabhu Subramanian <[email protected]>

Author: prabhu

build.sbt

@@ -1,6 +1,6 @@
 name                     := "chen"
 ThisBuild / organization := "io.appthreat"
-ThisBuild / version      := "2.3.11"
+ThisBuild / version      := "2.4.0"
 ThisBuild / scalaVersion := "3.6.2"
 
 val cpgVersion = "1.0.1"

ci/Dockerfile

@@ -4,7 +4,7 @@ LABEL maintainer="appthreat" \
       org.opencontainers.image.authors="Team AppThreat <[email protected]>" \
       org.opencontainers.image.source="https://github.com/appthreat/chen" \
       org.opencontainers.image.url="https://github.com/appthreat/chen" \
-      org.opencontainers.image.version="2.3.x" \
+      org.opencontainers.image.version="2.4.x" \
       org.opencontainers.image.vendor="appthreat" \
       org.opencontainers.image.licenses="Apache-2.0" \
       org.opencontainers.image.title="chen" \

codemeta.json

@@ -7,7 +7,7 @@
   "downloadUrl": "https://github.com/AppThreat/chen",
   "issueTracker": "https://github.com/AppThreat/chen/issues",
   "name": "chen",
-  "version": "2.3.11",
+  "version": "2.4.0",
   "description": "Code Hierarchy Exploration Net (chen) is an advanced exploration toolkit for your application source code and its dependency hierarchy.",
   "applicationCategory": "code-analysis",
   "keywords": [

dataflowengineoss/src/main/scala/io/appthreat/dataflowengineoss/passes/reachingdef/ReachingDefPass.scala

@@ -6,7 +6,6 @@ import io.shiftleft.codepropertygraph.Cpg
 import io.shiftleft.codepropertygraph.generated.nodes.*
 import io.shiftleft.passes.ForkJoinParallelCpgPass
 import io.shiftleft.semanticcpg.language.*
-import org.slf4j.{Logger, LoggerFactory}
 
 import scala.collection.mutable
 
@@ -15,21 +14,14 @@ import scala.collection.mutable
 class ReachingDefPass(cpg: Cpg, maxNumberOfDefinitions: Int = 4000)(implicit s: Semantics)
     extends ForkJoinParallelCpgPass[Method](cpg):
 
-  private val logger: Logger = LoggerFactory.getLogger(this.getClass)
   // If there are any regex method full names, load them early
   s.loadRegexSemantics(cpg)
 
   override def generateParts(): Array[Method] = cpg.method.toArray
 
   override def runOnPart(dstGraph: DiffGraphBuilder, method: Method): Unit =
-    logger.debug(
-      "Calculating reaching definitions for: {} in {}",
-      method.fullName,
-      method.filename
-    )
     val problem = ReachingDefProblem.create(method)
     if shouldBailOut(method, problem) then
-      logger.warn("Skipping.")
       return
 
     val solution     = new DataFlowSolver().calculateMopSolutionForwards(problem)
@@ -49,9 +41,7 @@ class ReachingDefPass(cpg: Cpg, maxNumberOfDefinitions: Int = 4000)(implicit s:
     // For each node, the `gen` map contains the list of definitions it generates
     // We add up the sizes of these lists to obtain the total number of definitions
     val numberOfDefinitions = transferFunction.gen.foldLeft(0)(_ + _._2.size)
-    logger.debug("Number of definitions for {}: {}", method.fullName, numberOfDefinitions)
     if numberOfDefinitions > maxNumberOfDefinitions then
-      logger.warn("{} has more than {} definitions", method.fullName, maxNumberOfDefinitions)
       true
     else
       false

dataflowengineoss/src/main/scala/io/appthreat/dataflowengineoss/passes/reachingdef/ReachingDefProblem.scala

@@ -39,8 +39,6 @@ object ReachingDefProblem:
   */
 class ReachingDefFlowGraph(val method: Method) extends FlowGraph[StoredNode]:
 
-  private val logger: Logger = LoggerFactory.getLogger(this.getClass)
-
   val entryNode: StoredNode = method
   val exitNode: StoredNode  = method.methodReturn
 
@@ -89,7 +87,6 @@ class ReachingDefFlowGraph(val method: Method) extends FlowGraph[StoredNode]:
           case paramOut: MethodParameterOut => paramOut -> nextParamOutOrExit(paramOut)
           case cfgNode: CfgNode             => cfgNode  -> cfgNextOrFirstOutParam(cfgNode)
           case n =>
-              logger.warn(s"Node type ${n.getClass.getSimpleName} should not be part of the CFG")
               n -> List()
       }.toMap
 
@@ -105,7 +102,6 @@ class ReachingDefFlowGraph(val method: Method) extends FlowGraph[StoredNode]:
           case n if n == exitNode     => n -> lastOutputParamOrLastNodeOfBody()
           case n @ (cfgNode: CfgNode) => n -> cfgNode.cfgPrev.l
           case n =>
-              logger.warn(s"Node type ${n.getClass.getSimpleName} should not be part of the CFG")
               n -> List()
       }.toMap
 

dataflowengineoss/src/main/scala/io/appthreat/dataflowengineoss/queryengine/AccessPathUsage.scala

@@ -43,13 +43,11 @@ object AccessPathUsage:
             // assume: MemberAccess.isGenericMemberAccessName(call.name)
             val argOne = memberAccess.argumentOption(1)
             if argOne.isEmpty then
-              logger.debug(s"Missing first argument on call ${memberAccess.code}.")
               return (TrackedUnknown, Nil)
             val (base, tail) = toTrackedBaseAndAccessPathInternal(argOne.get)
             val path         = AccessPathHandling.memberAccessToPath(memberAccess, tail)
             (base, path)
         case _ =>
-            logger.debug(s"Missing handling for node type ${node.getClass}.")
             (TrackedUnknown, Nil)
     end if
   end toTrackedBaseAndAccessPathInternal

dataflowengineoss/src/main/scala/io/appthreat/dataflowengineoss/queryengine/Engine.scala

@@ -8,7 +8,6 @@ import io.shiftleft.codepropertygraph.generated.nodes.*
 import io.shiftleft.codepropertygraph.generated.{EdgeTypes, Properties}
 import io.shiftleft.semanticcpg.language.*
 import io.shiftleft.semanticcpg.language.NoResolve
-import org.slf4j.{Logger, LoggerFactory}
 import overflowdb.Edge
 
 import java.util.concurrent.*
@@ -26,7 +25,6 @@ class Engine(context: EngineContext):
 
   import Engine.*
 
-  private val logger: Logger = LoggerFactory.getLogger(this.getClass)
   private val executorService: ExecutorService =
       Executors.newVirtualThreadPerTaskExecutor()
   private val completionService =
@@ -45,10 +43,6 @@ class Engine(context: EngineContext):
     * the analysis.
     */
   def backwards(sinks: List[CfgNode], sources: List[CfgNode]): List[TableEntry] =
-    if sources.isEmpty then
-      logger.debug("Attempting to determine flows from empty list of sources.")
-    if sinks.isEmpty then
-      logger.debug("Attempting to determine flows to empty list of sinks.")
     reset()
     val sourcesSet = sources.toSet
     val tasks      = createOneTaskPerSink(sinks)
@@ -103,24 +97,10 @@ class Engine(context: EngineContext):
                 numberOfTasksRunning -= 1
 
     submitTasks(tasks.toVector, sources)
-    val startTimeSec: Long = System.currentTimeMillis / 1000
     runUntilAllTasksAreSolved()
-    val taskFinishTimeSec: Long = System.currentTimeMillis / 1000
-    logger.debug(
-      "Time measurement -----> Task processing done in " +
-          (taskFinishTimeSec - startTimeSec) + " seconds"
-    )
     new HeldTaskCompletion(held.toList, mainResultTable).completeHeldTasks()
-    val dedupResult          = deduplicateFinal(extractResultsFromTable(sinks))
-    val allDoneTimeSec: Long = System.currentTimeMillis / 1000
-
-    logger.debug(
-      "Time measurement -----> Task processing: " +
-          (taskFinishTimeSec - startTimeSec) + " seconds" +
-          ", Deduplication: " + (allDoneTimeSec - taskFinishTimeSec) +
-          ", Deduped results size: " + dedupResult.length
-    )
-    dedupResult
+    // Deal with duplicates in downstream tools
+    extractResultsFromTable(sinks)
   end solveTasks
 
   private def submitTasks(tasks: Vector[ReachableByTask], sources: Set[CfgNode]): Unit =
@@ -314,11 +294,11 @@ case class EngineContext(
   *   max limit on number arguments for which tasks will be created for unresolved arguments
   */
 case class EngineConfig(
-  var maxCallDepth: Int = 4,
+  var maxCallDepth: Int = 3,
   initialTable: Option[mutable.Map[TaskFingerprint, Vector[ReachableByResult]]] = None,
   shareCacheBetweenTasks: Boolean = true,
-  maxArgsToAllow: Int = 1000,
-  maxOutputArgsExpansion: Int = 1000
+  maxArgsToAllow: Int = 100,
+  maxOutputArgsExpansion: Int = 100
 )
 
 /** Tracks various performance characteristics of the query engine.

dataflowengineoss/src/main/scala/io/appthreat/dataflowengineoss/queryengine/TaskCreator.scala

@@ -85,8 +85,6 @@ class TaskCreator(context: EngineContext):
   private def paramToArgs(param: MethodParameterIn): List[Expression] =
     val args = paramToArgsOfCallers(param) ++ paramToMethodRefCallReceivers(param)
     if args.size > context.config.maxArgsToAllow then
-      logger.warn(s"Too many arguments for parameter: ${args.size}. Not expanding")
-      logger.warn("Method name: " + param.method.fullName)
       List()
     else
       args
@@ -211,6 +209,5 @@ class TaskCreator(context: EngineContext):
       if l.size <= context.config.maxOutputArgsExpansion then
         l
       else
-        logger.warn("Too many new tasks in expansion of unresolved output arguments")
         Vector()
 end TaskCreator

meta.yaml

@@ -1,4 +1,4 @@
-{% set version = "2.3.11" %}
+{% set version = "2.4.0" %}
 
 package:
   name: chen

platform/frontends/ruby2atom/src/main/scala/io/appthreat/ruby2atom/astcreation/AstForExpressionsCreator.scala

@@ -591,6 +591,8 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode):
                 matchSplatExpression,
                 reassign(lhs, op, thenClause, transform)
               )(x.span)
+          case InClause(pattern, body) =>
+              InClause(pattern, reassign(lhs, op, body, transform))(x.span)
 
     rhs match
       case StatementList(statements) =>