Remove upstream astgen (#159)

* Remove upstream astgen Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com> * Remove upstream astgen Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com> * Remove upstream astgen Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com> * Update readme Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com> --------- Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com>
AppThreat · Sep 21, 2024 · bfabb8b · bfabb8b
1 parent 7a3b78a
commit bfabb8b
Show file tree

Hide file tree

Showing 6 changed files with 41 additions and 101 deletions.
diff --git a/.github/workflows/npm-release.yml b/.github/workflows/npm-release.yml
@@ -17,7 +17,7 @@ jobs:
     - name: Use Node.js
       uses: actions/setup-node@v4
       with:
-        node-version: 18.x
+        node-version: '22.x'
         registry-url: https://registry.npmjs.org/
     - uses: coursier/cache-action@v6
     - name: Set up JDK

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
@@ -38,12 +38,16 @@ jobs:
         uses: graalvm/setup-graalvm@v1
         with:
           distribution: 'graalvm-community'
-          java-version: '22'
+          java-version: '23'
           github-token: ${{ secrets.GITHUB_TOKEN }}
           native-image-job-reports: 'true'
           cache: 'sbt'
       - run: |
-          sbt stage astGenDlTask
+          sbt stage createDistribution
+          cd wrapper/nodejs
+          bash build.sh
+          npm install -g .
+          cd ../..
           ./atom.sh --remove-atom -o /tmp/java.atom -l java $GITHUB_WORKSPACE/repotests/shiftleft-java-example -Dlog4j.configurationFile=log4j2.xml
           ./atom.sh --remove-atom -o /tmp/juice.atom -l js $GITHUB_WORKSPACE/repotests/juice-shop -Dlog4j.configurationFile=log4j2.xml
           ./atom.sh --remove-atom -o /tmp/ts.atom -l js $GITHUB_WORKSPACE/repotests/shiftleft-ts-example -Dlog4j.configurationFile=log4j2.xml
@@ -72,11 +76,13 @@ jobs:
           ./atom.sh usages -o /tmp/DjanGoat.atom -l python $GITHUB_WORKSPACE/repotests/DjanGoat -Dlog4j.configurationFile=log4j2.xml --slice-outfile /tmp/DjanGoat.usages.json
         env:
           JAVA_TOOL_OPTIONS: "-Dfile.encoding=UTF-8"
-        if: runner.os != 'Windows'
       - run: |
           bash ci/native-image.sh
+          ./target/graalvm-native-image/atom --remove-atom -o /tmp/java.atom -l java $GITHUB_WORKSPACE/repotests/shiftleft-java-example -Dlog4j.configurationFile=log4j2.xml
+          ./target/graalvm-native-image/atom --remove-atom -o /tmp/juice.atom -l js $GITHUB_WORKSPACE/repotests/juice-shop -Dlog4j.configurationFile=log4j2.xml
         env:
           JAVA_TOOL_OPTIONS: "-Dfile.encoding=UTF-8"
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         shell: bash
       - name: Generate SBOM with cdxgen
         run: |

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -72,7 +72,7 @@ jobs:
           tar -xvf upx-4.2.2-amd64_linux.tar.xz
           chmod +x upx-4.2.2-amd64_linux/upx
           sudo cp upx-4.2.2-amd64_linux/upx /usr/local/bin/
-          sbt stage astGenDlTask assembly createDistribution
+          sbt stage assembly createDistribution
           sha512sum target/atom.zip > target/atom.zip.sha512
           bash ci/native-image.sh
           cp target/graalvm-native-image/atom target/graalvm-native-image/atom-amd64

diff --git a/.github/workflows/repotests.yml b/.github/workflows/repotests.yml
@@ -81,12 +81,16 @@ jobs:
         run: brew install sbt
         if: runner.os == 'macOS'
       - run: |
-          sbt stage astGenDlTask
+          sbt stage createDistribution
           python -m pip install atom-tools
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           JAVA_TOOL_OPTIONS: "-Dfile.encoding=UTF-8"
       - run: |
+          cd wrapper/nodejs
+          bash build.sh
+          npm install -g .
+          cd ../..
           ./atom.sh --remove-atom -o /tmp/java.atom -l java $GITHUB_WORKSPACE/repotests/shiftleft-java-example -Dlog4j.configurationFile=log4j2.xml
           ./atom.sh --remove-atom -o /tmp/juice.atom -l js $GITHUB_WORKSPACE/repotests/juice-shop -Dlog4j.configurationFile=log4j2.xml
           ./atom.sh --remove-atom -o /tmp/ts.atom -l js $GITHUB_WORKSPACE/repotests/shiftleft-ts-example -Dlog4j.configurationFile=log4j2.xml
@@ -115,6 +119,13 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           JAVA_TOOL_OPTIONS: "-Dfile.encoding=UTF-8"
       - run: |
+          cd wrapper/nodejs
+          copy ..\..\target\atom.zip plugins\
+          Expand-Archive -Path ..\..\target\atom.zip -DestinationPath plugins\ -Force
+          Remove-Item -Force plugins\atom.zip
+          npm install -g .
+          cd ../..
+          .\atom.bat usages --remove-atom -o $env:GITHUB_WORKSPACE\\repotests\\juice-shop\\jshop.atom -l js $env:GITHUB_WORKSPACE\\repotests\\juice-shop --slice-outfile $env:GITHUB_WORKSPACE\\repotests\\juice-shop\\jshop.usages.json
           .\atom.bat usages --extract-endpoints --remove-atom -o $env:GITHUB_WORKSPACE\\repotests\\DjanGoat\\py3.atom -l python $env:GITHUB_WORKSPACE\\repotests\\DjanGoat --slice-outfile $env:GITHUB_WORKSPACE\\repotests\\DjanGoat\\py.usages.json
           python -m json.tool $env:GITHUB_WORKSPACE\\repotests\\DjanGoat\\openapi.generated.json
         if: runner.os == 'Windows'

diff --git a/README.md b/README.md
@@ -1,6 +1,6 @@
 # Atom (⚛)
 
-Atom is a novel intermediate representation for applications and a standalone tool powered by the [chen](https://github.com/AppThreat/chen) library. The intermediate representation (a network with nodes and links) is optimized for operations typically used for application analytics and machine learning, including [slicing](./specification/docs/slices.md) and [vectoring](./specification/docs/vectors.md).
+Atom is a novel intermediate representation for applications and a standalone tool powered by the [chen](https://github.com/AppThreat/chen) library. The intermediate representation (a network with nodes and links) is optimized for operations typically used for application analytics and machine learning, including [slicing](./specification/docs/slices.md) and vectoring.
 
 Our vision is to make atom useful for many use cases such as:
 
@@ -15,7 +15,6 @@ and more.
 
 [![release](https://github.com/appthreat/atom/actions/workflows/npm-release.yml/badge.svg)](https://github.com/appthreat/atom/actions/workflows/npm-release.yml)
 ![npm](https://img.shields.io/npm/dw/@appthreat/atom)
-[![Discord](https://img.shields.io/badge/-Discord-lime?style=for-the-badge&logo=discord&logoColor=white&color=black)](https://discord.gg/tmmtjCEHNV)
 
 ![Atom logo](./specification/docs/Atom-logo.png)
 
@@ -34,7 +33,21 @@ Install cdxgen to generate a Software Bill-of-Materials which is required for re
 npm install -g @cyclonedx/cdxgen --omit=optional
 ```
 
-## atom native-image
+## container usage
+
+```shell
+docker run --rm -v /tmp:/tmp -v $HOME:$HOME -v $(pwd):/app:rw -it ghcr.io/appthreat/atom atom --help
+# podman run --rm -v /tmp:/tmp -v $HOME:$HOME -v $(pwd):/app:rw -it ghcr.io/appthreat/atom atom --help
+```
+
+Example for java project.
+
+```shell
+docker run --rm -v /tmp:/tmp -v $HOME:$HOME -v $(pwd):/app:rw -it ghcr.io/appthreat/atom atom -l java -o /app/app.atom /app
+# podman run --rm -v /tmp:/tmp -v $HOME:$HOME -v $(pwd):/app:rw -it ghcr.io/appthreat/atom atom -l java -o /app/app.atom /app
+```
+
+## atom native-image (Advanced users only)
 
 atom is available as a native image built using graalvm community edition.
 
@@ -51,7 +64,7 @@ curl -LO https://github.com/AppThreat/atom/releases/latest/download/atom.exe
 .\atom.exe --help
 ```
 
-NOTE: cdxgen is not bundled into the native image so needs to be installed separately.
+NOTE: Commands such as cdxgen, astgen, and phpastgen are not bundled into this native image. Therefore, the binary is quite limited in functionality.
 
 ## CLI Usage
 
@@ -167,20 +180,6 @@ To also compute and include data-dependency graph (DDG) information in the expor
 atom -o app.atom -l java --export-atom --export-dir <export dir> --with-data-deps <path to application>
 ```
 
-## container usage
-
-```shell
-docker run --rm -v /tmp:/tmp -v $HOME:$HOME -v $(pwd):/app:rw -it ghcr.io/appthreat/atom atom --help
-# podman run --rm -v /tmp:/tmp -v $HOME:$HOME -v $(pwd):/app:rw -it ghcr.io/appthreat/atom atom --help
-```
-
-Example for java project.
-
-```shell
-docker run --rm -v /tmp:/tmp -v $HOME:$HOME -v $(pwd):/app:rw -it ghcr.io/appthreat/atom atom -l java -o /app/app.atom /app
-# podman run --rm -v /tmp:/tmp -v $HOME:$HOME -v $(pwd):/app:rw -it ghcr.io/appthreat/atom atom -l java -o /app/app.atom /app
-```
-
 ## Languages supported
 
 - C/C++
@@ -253,7 +252,7 @@ Checkout [atom-tools](https://github.com/AppThreat/atom-tools) for some project
 
 ## Enterprise support
 
-Enterprise support including custom language development and integration services is available via AppThreat Ltd. Free community support is also available via [discord](https://discord.gg/tmmtjCEHNV).
+Enterprise support including custom language development and integration services is available via AppThreat Ltd.
 
 ## Sponsors
 

diff --git a/build.sbt b/build.sbt
@@ -7,8 +7,6 @@ val chenVersion = "2.1.8"
 
 lazy val atom = Projects.atom
 
-val astGenVersion = "3.5.0"
-
 libraryDependencies ++= Seq(
   "com.github.pathikrit" %% "better-files" % "3.9.2",
   "com.github.scopt"     %% "scopt"        % "4.1.0",
@@ -68,80 +66,6 @@ Universal / mappings := (Universal / mappings).value.filter {
 
 enablePlugins(JavaAppPackaging, ClasspathJarPlugin, GraalVMNativeImagePlugin)
 
-lazy val AstgenWin      = "astgen-win.exe"
-lazy val AstgenLinux    = "astgen-linux"
-lazy val AstgenLinuxArm = "astgen-linux-arm"
-lazy val AstgenMac      = "astgen-macos"
-lazy val AstgenMacArm   = "astgen-macos-arm"
-
-lazy val astGenDlUrl = settingKey[String]("astgen download url")
-astGenDlUrl := s"https://github.com/joernio/astgen/releases/download/v${astGenVersion}/"
-
-lazy val astGenBinaryNames = taskKey[Seq[String]]("astgen binary names")
-astGenBinaryNames := {
-    if (sys.props.get("ALL_PLATFORMS").contains("TRUE")) {
-        Seq(AstgenWin, AstgenLinux, AstgenMac, AstgenMacArm)
-    } else {
-        Environment.operatingSystem match {
-            case Environment.OperatingSystemType.Windows =>
-                Seq(AstgenWin)
-            case Environment.OperatingSystemType.Linux =>
-                Environment.architecture match {
-                    case Environment.ArchitectureType.X86 => Seq(AstgenLinux)
-                    case Environment.ArchitectureType.ARM => Seq(AstgenLinuxArm)
-                }
-            case Environment.OperatingSystemType.Mac =>
-                Environment.architecture match {
-                    case Environment.ArchitectureType.X86 => Seq(AstgenMac)
-                    case Environment.ArchitectureType.ARM => Seq(AstgenMacArm)
-                }
-            case Environment.OperatingSystemType.Unknown =>
-                Seq(AstgenWin, AstgenLinux, AstgenMac, AstgenMacArm)
-        }
-    }
-}
-
-lazy val astGenDlTask = taskKey[Unit](s"Download astgen binaries")
-astGenDlTask := {
-    val astGenDir = baseDirectory.value / "bin" / "astgen"
-    astGenDir.mkdirs()
-
-    astGenBinaryNames.value.foreach { fileName =>
-        val dest = astGenDir / fileName
-        if (!dest.exists) {
-            val url            = s"${astGenDlUrl.value}$fileName"
-            val downloadedFile = SimpleCache.downloadMaybe(url)
-            IO.copyFile(downloadedFile, dest)
-        }
-    }
-
-    val distDir = (Universal / stagingDirectory).value / "bin" / "astgen"
-    distDir.mkdirs()
-    IO.copyDirectory(astGenDir, distDir)
-
-    // permissions are lost during the download; need to set them manually
-    astGenDir.listFiles().foreach(_.setExecutable(true, false))
-    distDir.listFiles().foreach(_.setExecutable(true, false))
-}
-
-lazy val astGenSetAllPlatforms = taskKey[Unit](s"Set ALL_PLATFORMS")
-astGenSetAllPlatforms := { System.setProperty("ALL_PLATFORMS", "TRUE") }
-
-stage := Def
-    .sequential(astGenSetAllPlatforms, Universal / stage)
-    .andFinally(System.setProperty("ALL_PLATFORMS", "FALSE"))
-    .value
-
-// Also remove astgen binaries with clean, e.g., to allow for updating them.
-// Sadly, we can't define the bin/ folders globally,
-// as .value can only be used within a task or setting macro
-cleanFiles ++= Seq(
-  baseDirectory.value / "bin" / "astgen",
-  (Universal / stagingDirectory).value / "bin" / "astgen"
-) ++ astGenBinaryNames.value.map(fileName =>
-    SimpleCache.encodeFile(s"${astGenDlUrl.value}$fileName")
-)
-
 ThisBuild / licenses := List("Apache-2.0" -> url("http://www.apache.org/licenses/LICENSE-2.0"))
 
 Global / onChangedBuildSource := ReloadOnSourceChanges