Added repo tests

Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com>
AppThreat · Apr 2, 2023 · be48e2f · be48e2f
1 parent e9ac9a5
commit be48e2f
Show file tree

Hide file tree

Showing 7 changed files with 133 additions and 52 deletions.
diff --git a/.github/workflows/pythonpublish.yml b/.github/workflows/pythonpublish.yml
@@ -4,8 +4,6 @@ on:
   schedule:
   - cron: "0 */12 * * *"
   push:
-    branches:
-      - main
     tags:
       - 'v*'
   workflow_dispatch:

diff --git a/.github/workflows/repotests.yml b/.github/workflows/repotests.yml
@@ -68,15 +68,37 @@ jobs:
         run: |
           mkdir /tmp/all_cpgs
           docker build -t ghcr.io/appthreat/cpggen .
-          docker run --rm -it -v /tmp:/tmp -v $(pwd):/app:rw -t ghcr.io/appthreat/cpggen cpggen -i /app/repotests/shiftleft-java-example -o /tmp/all_cpgs/shiftleft-java-example
-          docker run --rm -it -v /tmp:/tmp -v $(pwd):/app:rw -t ghcr.io/appthreat/cpggen cpggen -i /app/repotests/shiftleft-ts-example -o /tmp/all_cpgs/shiftleft-ts-example
-          docker run --rm -it -v /tmp:/tmp -v $(pwd):/app:rw -t ghcr.io/appthreat/cpggen cpggen -i /app/repotests/shiftleft-go-example -o /tmp/all_cpgs/shiftleft-go-example --build
-          docker run --rm -it -v /tmp:/tmp -v $(pwd):/app:rw -t ghcr.io/appthreat/cpggen cpggen -i /app/repotests/shiftleft-scala-example -o /tmp/all_cpgs/shiftleft-scala-example
-          docker run --rm -it -v /tmp:/tmp -v $(pwd):/app:rw -t ghcr.io/appthreat/cpggen cpggen -i /app/repotests/vulnerable_net_core -o /tmp/all_cpgs/vulnerable_net_core --build
-          docker run --rm -it -v /tmp:/tmp -v $(pwd):/app:rw -t ghcr.io/appthreat/cpggen cpggen -i /app/repotests/Goatly.NET -o /tmp/all_cpgs/Goatly.NET --build
-          docker run --rm -it -v /tmp:/tmp -v $(pwd):/app:rw -t ghcr.io/appthreat/cpggen cpggen -i /app/repotests/DjanGoat -o /tmp/all_cpgs/DjanGoat
-          docker run --rm -it -v /tmp:/tmp -v $(pwd):/app:rw -t ghcr.io/appthreat/cpggen cpggen -i /app/repotests/Vulnerable-Web-Application -o /tmp/all_cpgs/Vulnerable-Web-Application
-          docker run --rm -it -v /tmp:/tmp -v $(pwd):/app:rw -t ghcr.io/appthreat/cpggen cpggen -i /app/repotests/microservices-demo -o /tmp/all_cpgs/microservices-demo
+          poetry run cpggen -i repotests/shiftleft-java-example -o /tmp/all_cpgs/shiftleft-java-example
+          if [ -e "/tmp/all_cpgs/shiftleft-java-example/java-cpg.bin.zip" ]; then
+            echo "Java direct test was successful"
+          else
+            echo "Java direct test was not successful"
+          fi
+          docker run --rm -v /tmp:/tmp -v $(pwd):/app:rw -t ghcr.io/appthreat/cpggen cpggen -i /app/repotests/shiftleft-java-example -o /tmp/all_cpgs/shiftleft-java-example
+          docker run --rm -v /tmp:/tmp -v $(pwd):/app:rw -t ghcr.io/appthreat/cpggen joern -J-Xmx7G --script /app/contrib/joern_scripts/cpg-methods.sc --params payload=/tmp/all_cpgs/shiftleft-java-example/java-cpg.bin.zip,resultFile=/tmp/all_cpgs/shiftleft-java-example/java-cpg-methods.json
+          if [ -e "/tmp/all_cpgs/shiftleft-java-example/java-cpg-methods.json" ]; then
+            echo "Java cpg test was successful"
+          else
+            echo "Java cpg test was not successful"
+          fi
+          poetry run cpggen -i repotests/shiftleft-ts-example -o /tmp/all_cpgs/shiftleft-ts-example
+          if [ -e "/tmp/all_cpgs/shiftleft-ts-example/js-cpg.bin.zip" ]; then
+            echo "JS direct test was successful"
+          else
+            echo "JS direct test was not successful"
+          fi
+          docker run --rm -v /tmp:/tmp -v $(pwd):/app:rw -t ghcr.io/appthreat/cpggen cpggen -i /app/repotests/shiftleft-ts-example -o /tmp/all_cpgs/shiftleft-ts-example
+          poetry run cpggen -i repotests/shiftleft-go-example -o /tmp/all_cpgs/shiftleft-go-example --build
+          docker run --rm -v /tmp:/tmp -v $(pwd):/app:rw -t ghcr.io/appthreat/cpggen cpggen -i /app/repotests/shiftleft-go-example -o /tmp/all_cpgs/shiftleft-go-example --build
+          poetry run cpggen -i repotests/shiftleft-scala-example -o /tmp/all_cpgs/shiftleft-scala-example --build
+          docker run --rm -v /tmp:/tmp -v $(pwd):/app:rw -t ghcr.io/appthreat/cpggen cpggen -i /app/repotests/shiftleft-scala-example -o /tmp/all_cpgs/shiftleft-scala-example --build
+          docker run --rm -v /tmp:/tmp -v $(pwd):/app:rw -t ghcr.io/appthreat/cpggen cpggen -i /app/repotests/vulnerable_net_core -o /tmp/all_cpgs/vulnerable_net_core --build
+          docker run --rm -v /tmp:/tmp -v $(pwd):/app:rw -t ghcr.io/appthreat/cpggen cpggen -i /app/repotests/Goatly.NET -o /tmp/all_cpgs/Goatly.NET --build
+          docker run --rm -v /tmp:/tmp -v $(pwd):/app:rw -t ghcr.io/appthreat/cpggen cpggen -i /app/repotests/DjanGoat -o /tmp/all_cpgs/DjanGoat
+          docker run --rm -v /tmp:/tmp -v $(pwd):/app:rw -t ghcr.io/appthreat/cpggen cpggen -i /app/repotests/Vulnerable-Web-Application -o /tmp/all_cpgs/Vulnerable-Web-Application
+          docker run --rm -v /tmp:/tmp -v $(pwd):/app:rw -t ghcr.io/appthreat/cpggen cpggen -i /app/repotests/microservices-demo -o /tmp/all_cpgs/microservices-demo
           ls -ltr /tmp/all_cpgs
         env:
           AT_DEBUG_MODE: debug
+          CPGGEN_CONTAINER_MEMORY: 7g
+          CPGGEN_MEMORY: 7G
diff --git a/Dockerfile b/Dockerfile
@@ -12,28 +12,36 @@ LABEL maintainer="appthreat" \
       org.opencontainers.docker.cmd="docker run --rm -it -v /tmp:/tmp -v $(pwd):/app:rw --cpus=4 --memory=16g -t ghcr.io/appthreat/cpggen cpggen -i /app -o /app/cpg_out"
 
 ENV JOERN_HOME=/opt/joern/joern-cli \
-    GOPATH=/opt/app-root/go \
+    GOROOT=/usr/local/go \
     GO_VERSION=1.19.7 \
-    CGO_ENABLED=0 \
+    SBT_VERSION=1.8.2 \
+    CGO_ENABLED=1 \
+    GO111MODULE="" \
+    GOOS="linux" \
     PYTHONUNBUFFERED=1 \
     DOTNET_CLI_TELEMETRY_OPTOUT=1 \
     JOERN_DATAFLOW_TRACKED_WIDTH=128 \
     JAVA_OPTS="-Xmx16G" \
     SL_CPG_OPTS="-J-Xms4g -J-Xmx16g" \
     SHIFTLEFT_JAVA_OPTS="-Xms4g -Xmx16g" \
-    PATH=${PATH}:/opt/joern/joern-cli:/opt/joern/joern-cli/bin:${GOPATH}/bin:/usr/local/go/bin:/usr/local/bin/:/root/.local/bin:
+    PATH=${PATH}:/opt/joern/joern-cli:/opt/joern/joern-cli/bin:${GOPATH}/bin:/usr/local/go/bin:/usr/local/bin/:/root/.local/bin:/opt/sbt/bin:/usr/local/go/pkg/tool/linux_amd64:
 
 COPY . /usr/local/src/
 
-RUN microdnf install -y gcc git-core php php-cli python3 python3-devel pcre2 which tar zip unzip sudo java-1.8.0-openjdk-headless \
+RUN echo -e "[nodejs]\nname=nodejs\nstream=18\nprofiles=\nstate=enabled\n" > /etc/dnf/modules.d/nodejs.module \
+    && microdnf install -y gcc git-core php php-cli python3 python3-devel pcre2 which tar zip unzip sudo java-1.8.0-openjdk-headless \
         java-17-openjdk-headless ncurses jq krb5-libs libicu openssl-libs compat-openssl11 zlib \
-        dotnet-sdk-7.0 dotnet-targeting-pack-7.0 dotnet-templates-7.0 dotnet-hostfxr-7.0 \
+        dotnet-sdk-7.0 dotnet-targeting-pack-7.0 dotnet-templates-7.0 dotnet-hostfxr-7.0 nodejs \
     && curl -LO https://github.com/joernio/joern/releases/latest/download/joern-install.sh \
     && chmod +x ./joern-install.sh \
     && ./joern-install.sh \
     && curl -LO "https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz" \
     && tar -C /usr/local -xzf go${GO_VERSION}.linux-amd64.tar.gz \
     && rm go${GO_VERSION}.linux-amd64.tar.gz \
+    && curl -LO "https://github.com/sbt/sbt/releases/download/v${SBT_VERSION}/sbt-${SBT_VERSION}.zip" \
+    && unzip -q sbt-${SBT_VERSION}.zip -d /opt/ \
+    && chmod +x /opt/sbt/bin/sbt \
+    && rm sbt-${SBT_VERSION}.zip \
     && curl -L $(curl -L https://www.shiftleft.io/download/java2cpg.json | jq -r ".downloadURL") -o /opt/joern/joern-cli/java2cpg.jar \
     && curl -L $(curl -L https://www.shiftleft.io/download/go2cpgmanifest-linux-x64.json | jq -r ".downloadURL") -o /opt/joern/joern-cli/go2cpg \
     && chmod +x /opt/joern/joern-cli/go2cpg && go2cpg version \
@@ -43,6 +51,7 @@ RUN microdnf install -y gcc git-core php php-cli python3 python3-devel pcre2 whi
     && mkdir -p /opt/joern/custom_scripts \
     && useradd -ms /bin/bash joern \
     && chown -R joern:joern /opt/joern \
+    && npm install -g @cyclonedx/cdxgen \
     && python -m pip install --no-cache-dir poetry==1.3.2 \
     && poetry config virtualenvs.create false \
     && cd /usr/local/src/ && poetry install --no-cache --without dev \

diff --git a/README.md b/README.md
@@ -26,6 +26,13 @@ docker pull ghcr.io/appthreat/cpggen
 # podman pull ghcr.io/appthreat/cpggen
 ```
 
+Or use the nightly to always get the latest joern and tools.
+
+```
+docker pull ghcr.io/appthreat/cpggen:nightly
+# podman pull ghcr.io/appthreat/cpggen:nightly
+```
+
 ## Usage
 
 To auto detect the language from the current directory and generate CPG.
@@ -87,3 +94,7 @@ curl "http://127.0.0.1:7072/cpg?url=https://github.com/HooliCorp/vulnerable-aws-
 | CPGGEN_CONTAINER_MEMORY | Memory units to use in container execution mode. Default 32g |
 | CPGGEN_MEMORY           | Heap memory to use for frontends. Default 32G                |
 | AT_DEBUG_MODE           | Set to debug to enable debug logging                         |
+
+## License
+
+Apache-2.0
diff --git a/contrib/joern_scripts/cpg-methods.sc b/contrib/joern_scripts/cpg-methods.sc
@@ -0,0 +1,9 @@
+@main def printCpgMethods(payload: String, resultFile: String) : Boolean = {
+    importCpg(payload)
+    if(!workspace.cpgExists(payload)) {
+        printf("[-] Failed to create CPG for %s\n", payload)
+        return false
+    }
+    cpg.method.toJsonPretty |> resultFile
+    return true
+}
diff --git a/cpggen/cli.py b/cpggen/cli.py
@@ -5,9 +5,10 @@
 from multiprocessing import Pool
 from pathlib import Path
 
+from quart import Quart, request
+
 from cpggen import executor, utils
 from cpggen.logger import LOG, console
-from quart import Quart, request
 
 try:
     os.environ["PYTHONIOENCODING"] = "utf-8"
@@ -138,7 +139,7 @@ async def generate_cpg():
     params = await request.get_json()
     url = None
     src = None
-    language = None
+    languages = None
     cpg_out_dir = None
     is_temp_dir = False
     if not params:
@@ -151,13 +152,13 @@ async def generate_cpg():
         cpg_out_dir = q.get("out_dir")
 
     if q.get("lang"):
-        language = q.get("lang")
+        languages = q.get("lang")
     if not url and params.get("url"):
         url = params.get("url")
     if not src and params.get("src"):
         src = params.get("src")
-    if not language and params.get("lang"):
-        language = params.get("lang")
+    if not languages and params.get("lang"):
+        languages = params.get("lang")
     if not cpg_out_dir and params.get("out_dir"):
         cpg_out_dir = params.get("out_dir")
     if not src and not url:
@@ -168,22 +169,31 @@ async def generate_cpg():
         is_temp_dir = True
     if cpg_out_dir and not os.path.exists(cpg_out_dir):
         os.makedirs(cpg_out_dir, exist_ok=True)
-    executor.exec_tool(
-        language,
-        src,
-        cpg_out_dir,
-        src,
-        joern_home=os.getenv(
-            "JOERN_HOME", str(Path.home() / "bin" / "joern" / "joern-cli")
-        ),
-    )
+    if not languages or languages == "autodetect":
+        languages = utils.detect_project_type(src)
+    else:
+        languages = languages.split(",")
+    for lang in languages:
+        executor.exec_tool(
+            lang,
+            src,
+            cpg_out_dir,
+            src,
+            joern_home=os.getenv(
+                "JOERN_HOME", str(Path.home() / "bin" / "joern" / "joern-cli")
+            ),
+        )
     if is_temp_dir:
         try:
             os.remove(src)
         except Exception:
             # Ignore cleanup errors
             pass
-    return {}
+    return {
+        "success": True,
+        "message": f"CPG generated successfully at {cpg_out_dir}",
+        "out_dir": cpg_out_dir,
+    }
 
 
 def init_worker():
@@ -228,11 +238,15 @@ def main():
     joern_home = args.joern_home
     use_container = args.use_container
     if not os.path.exists(joern_home):
-        use_container = True
-        console.print(
-            "Joern installation was not found. Please install joern by following the instructions at https://joern.io and set the environment variable JOERN_HOME to the directory containing the cli tools"
-        )
-        console.print("Fallback to using cpggen container image")
+        if utils.check_command("docker") or utils.check_command("podman"):
+            use_container = True
+        else:
+            console.print(
+                "Joern installation was not found. Please install joern by following the instructions at https://joern.io and set the environment variable JOERN_HOME to the directory containing the cli tools"
+            )
+            console.print(
+                "Alternatively, ensure docker or podman is available to use cpggen container image"
+            )
     is_temp_dir = False
     if src.startswith("http") or src.startswith("git"):
         clone_dir = tempfile.mkdtemp(prefix="cpggen")

diff --git a/cpggen/executor.py b/cpggen/executor.py
@@ -1,11 +1,12 @@
 import os
 import subprocess
+import tempfile
 from pathlib import Path
 
 from rich.progress import Progress
 
 from cpggen.logger import DEBUG, LOG, console
-from cpggen.utils import check_command, find_java_artifacts, find_csharp_artifacts
+from cpggen.utils import check_command, find_csharp_artifacts, find_java_artifacts
 
 runtimeValues = {}
 
@@ -46,6 +47,7 @@ def get(configName, default_value=None):
     "jar": "java -Xmx32G -jar %(joern_home)s/java2cpg.jar %(uber_jar)s -nojsp -nb --experimental-langs scala -su -o %(cpg_out)s",
     "scala": "java -Xmx32G -jar %(joern_home)s/java2cpg.jar %(uber_jar)s -nojsp -nb --experimental-langs scala -su -o %(cpg_out)s",
     "jsp": "java -Xmx32G -jar %(joern_home)s/java2cpg.jar %(uber_jar)s -nb --experimental-langs scala -su -o %(cpg_out)s",
+    "sbom": "cdxgen -t %(tool_lang)s -o %(sbom_out)s %(src)s",
 }
 
 build_tools_map = {
@@ -64,11 +66,7 @@ def get(configName, default_value=None):
         "maven": [get("MVN_CMD"), "compile"],
         "gradle": [get("GRADLE_CMD"), "compileGroovy"],
     },
-    "scala": {
-        "maven": [get("MVN_CMD"), "compile"],
-        "gradle": [get("GRADLE_CMD"), "compileScala"],
-        "sbt": ["sbt", "compile"],
-    },
+    "scala": ["sbt", "stage"],
     "nodejs": {
         "npm": ["npm", "install", "--prefer-offline", "--no-audit", "--progress=false"],
         "yarn": ["yarn", "install"],
@@ -115,19 +113,30 @@ def exec_tool(
             )
             cmd_with_args = cpg_tools_map.get(tool_lang)
             if not cmd_with_args:
-                return None
+                return None, None, None
             cpg_out = (
                 cpg_out_dir
                 if cpg_out_dir.endswith(".bin.zip")
                 or cpg_out_dir.endswith(".bin")
                 or cpg_out_dir.endswith(".cpg")
                 else os.path.join(cpg_out_dir, f"{tool_lang}-cpg.bin.zip")
             )
-            LOG.debug(f"CPG file for {tool_lang} can be found at {cpg_out}")
+            sbom_out = (
+                cpg_out.replace(".bin.zip", ".bom.json")
+                if cpg_out.endswith(".bin.zip")
+                else f"{cpg_out}.bom.json"
+            )
+            LOG.debug(f"CPG file for {tool_lang} is {cpg_out}")
             if use_container:
-                cmd_with_args = f"""docker run --rm -it -w {src} -v /tmp:/tmp -v {src}:{src}:rw -v {cpg_out_dir}:{cpg_out_dir}:rw --cpus={os.getenv("CPGGEN_CONTAINER_CPU", "2")} --memory={os.getenv("CPGGEN_CONTAINER_MEMORY", "32g")} -t {os.getenv("CPGGEN_IMAGE", "ghcr.io/appthreat/cpggen")} {cmd_with_args}"""
+                src = os.path.abspath(src)
+                container_cli = "docker"
+                if check_command("podman"):
+                    container_cli = "podman"
+                # cmd_with_args = f"""{container_cli} run --rm -w {os.path.abspath(src)} -v /tmp:/tmp -v {os.path.abspath(src)}:{os.path.abspath(src)}:rw -v {os.path.abspath(cpg_out_dir)}:{os.path.abspath(cpg_out_dir)}:rw --cpus={os.getenv("CPGGEN_CONTAINER_CPU", "2")} --memory={os.getenv("CPGGEN_CONTAINER_MEMORY", "32g")} -t {os.getenv("CPGGEN_IMAGE", "ghcr.io/appthreat/cpggen")} {cmd_with_args}"""
+                cmd_with_args = f"""{container_cli} run --rm -w {src} -v {tempfile.gettempdir()}:/tmp -v {src}:{src}:rw -v {os.path.abspath(cpg_out_dir)}:{os.path.abspath(cpg_out_dir)}:rw -t {os.getenv("CPGGEN_IMAGE", "ghcr.io/appthreat/cpggen")} {cmd_with_args}"""
                 # We need to fix joern_home to the directory inside the container
                 joern_home = "/opt/joern/joern-cli"
+                # We need to make src an absolute path since relative paths wouldn't work in container mode
             uber_jar = ""
             csharp_artifacts = ""
             # For languages like scala, jsp or jar we need to create a uber jar containing all jar, war files from the source directory
@@ -141,7 +150,7 @@ def exec_tool(
                 csharp_artifacts = find_csharp_artifacts(src)
                 if len(csharp_artifacts) == 1:
                     csharp_artifacts = csharp_artifacts[0]
-            if auto_build and tool_lang in ("csharp", "go"):
+            if auto_build and tool_lang in ("csharp", "go", "scala"):
                 build_args = build_tools_map[tool_lang]
                 LOG.info(
                     '⚡︎ Attempting to auto build {} "{}"'.format(
@@ -158,6 +167,11 @@ def exec_tool(
                     shell=False,
                     encoding="utf-8",
                 )
+                if cp:
+                    if cp.stdout:
+                        LOG.info(cp.stdout)
+                    if cp.stderr:
+                        LOG.info(cp.stderr)
             cmd_with_args = cmd_with_args % dict(
                 src=src,
                 cpg_out=cpg_out,
@@ -166,14 +180,16 @@ def exec_tool(
                 uber_jar=uber_jar,
                 csharp_artifacts=csharp_artifacts,
                 memory=os.getenv("CPGGEN_MEMORY", "32G"),
+                tool_lang=tool_lang,
+                sbom_out=sbom_out,
             )
             cmd_with_args = cmd_with_args.split(" ")
             lang_cmd = cmd_with_args[0]
             if not check_command(lang_cmd):
                 LOG.warn(
                     f"{lang_cmd} is not found. Try running cpggen with --use-container argument"
                 )
-                return None
+                return None, None, None
             LOG.info(
                 '⚡︎ Generating CPG for {} "{}"'.format(
                     tool_lang, " ".join(cmd_with_args)
@@ -206,14 +222,16 @@ def exec_tool(
                 LOG.info(
                     f"""CPG for {tool_lang} is {cpg_out}. You can import this in joern using importCpg("{cpg_out}")"""
                 )
-                return cp, cpg_out
+                return cp, cpg_out, sbom_out
             else:
                 LOG.info(f"CPG was not generated successfully for {tool_lang}")
-                LOG.info(cp.stdout)
-                LOG.info(cp.stderr)
-            return cp, None
+                if cp.stdout:
+                    LOG.info(cp.stdout)
+                if cp.stderr:
+                    LOG.info(cp.stderr)
+            return cp, None, None
         except Exception as e:
             if task:
                 progress.update(task, completed=20, total=10, visible=False)
             LOG.error(e)
-            return None
+            return None, None, None