change packaging: download frontends as binaries (joernio#581)

* upgrade cpg * remove fuzzyc/c2cpg as direct dependencies - will download binaries * WIP * keep fuzzyc as source dependency for JoernParse * download frontends as binary releases, use local cache redo the entire packaging * update packaging and installer * fix ghidra folder structure * upgrade deps * build setup: create dirs if not existent * restructure package * fix `stage` and restructure again * ghidra is now part of joern-cli/stage already
zu1kbackup · pull · Aug 25, 2021 · Aug 25, 2021 · Aug 25, 2021 · 83fe97531164a411e0088c3fde2c4dd7d7e45af0
commit 83fe97531164a411e0088c3fde2c4dd7d7e45af0
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -24,7 +24,7 @@ jobs:
           ./schema-extender/test.sh
           cd -
       - run: sbt ciReleaseTagNextVersion createDistribution
-      - run: sha512sum ./joern-cli.zip > ./joern-cli.zip.sha512
+      - run: sha512sum target/joern-cli.zip > target/joern-cli.zip.sha512
       - name: Export ENV vars
         run:
           echo "LATEST_TAG=$(git describe --tags --abbrev=0)" >> $GITHUB_ENV
@@ -53,7 +53,7 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
           upload_url: ${{ steps.create_release.outputs.upload_url }}
-          asset_path: ./joern-cli.zip
+          asset_path: target/joern-cli.zip
           asset_name: joern-cli.zip
           asset_content_type: application/zip
       - name: Upload joern-cli.zip.sha512
@@ -62,6 +62,6 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
           upload_url: ${{ steps.create_release.outputs.upload_url }}
-          asset_path: ./joern-cli.zip.sha512
+          asset_path: target/joern-cli.zip.sha512
           asset_name: joern-cli.zip.sha512
           asset_content_type: text/plain
diff --git a/build.sbt b/build.sbt
@@ -1,15 +1,13 @@
-import net.lingala.zip4j.model.ZipParameters
-
 enablePlugins(GitVersioning)
 
 name := "joern"
 organization := "io.shiftleft"
 ThisBuild / scalaVersion := "2.13.5"
 // don't upgrade to 2.13.6 until https://github.com/com-lihaoyi/Ammonite/issues/1182 is resolved
 ThisBuild /Test /fork := true
-val cpgVersion = "1.3.287"
-val ghidra2cpgVersion = "0.0.24"
-val js2cpgVersion = "0.2.2"
+val cpgVersion = "1.3.302"
+val ghidra2cpgVersion = "0.0.25"
+val js2cpgVersion = "0.2.3"
 
 ThisBuild / resolvers ++= Seq(
   Resolver.mavenLocal,
@@ -31,23 +29,27 @@ lazy val joerncli = Projects.joerncli
 lazy val ghidra2cpg = Projects.ghidra2cpg
 
 lazy val createDistribution = taskKey[File]("Create a complete Joern distribution")
+import net.lingala.zip4j._
+import net.lingala.zip4j.model.ZipParameters
 createDistribution := {
-  val joernCliZip = (joerncli/Universal/packageBin).value
-  val ghidraStaged = (ghidra2cpg/Universal/stage).value
-
-  val distributionZip = file("./joern-cli.zip")
-  IO.copyFile(joernCliZip, distributionZip,
-    CopyOptions(overwrite = true, preserveLastModified = true, preserveExecutable = true))
-
-  // add ghidra2cpg
-  import net.lingala.zip4j._
-  val params = new ZipParameters()
-  params.setRootFolderNameInZip("ghidra2cpg")
-  params.setIncludeRootFolder(false)
-  new ZipFile(distributionZip).addFolder(ghidraStaged, params)
-
-  println(s"created distribution - resulting files: $distributionZip")
-  distributionZip
+  val distributionFile = file("target/joern-cli.zip")
+  val distributionZip = new ZipFile(distributionFile)
+
+  distributionZip.addFile((joerncli/Universal/packageBin).value, withName("joern-cli.zip"))
+  distributionZip.addFile(Frontends.downloadC2CpgZip, withName("c2cpg.zip"))
+  distributionZip.addFile(Frontends.downloadFuzzyc2CpgZip, withName("fuzzyc2cpg.zip"))
+  distributionZip.addFile(Frontends.downloadJs2CpgZip, withName("js2cpg.zip"))
+
+  println(s"created distribution - resulting files: $distributionFile")
+  distributionFile
 }
 
+def withName(name: String): ZipParameters = {
+  val zipParams = new ZipParameters
+  zipParams.setFileNameInZip(name)
+  zipParams
+}
+
+
+
 Global / onChangedBuildSource := ReloadOnSourceChanges
diff --git a/joern-cli/build.sbt b/joern-cli/build.sbt
@@ -5,14 +5,12 @@ organization := "io.shiftleft"
 name := "joern-cli"
 
 libraryDependencies ++= Seq(
-  "io.shiftleft" %% "codepropertygraph" % Versions.cpgVersion,
-  "io.shiftleft" %% "semanticcpg" % Versions.cpgVersion,
-  "io.shiftleft" %% "console" % Versions.cpgVersion,
-  "io.shiftleft" %% "console" % Versions.cpgVersion % Test classifier "tests",
-  "io.shiftleft" %% "dataflowengineoss" % Versions.cpgVersion,
-  "io.shiftleft" %% "fuzzyc2cpg" % Versions.cpgVersion,
-  "io.shiftleft" %% "c2cpg" % Versions.cpgVersion,
-  "io.shiftleft" %% "js2cpg" % Versions.js2cpg,
+  "io.shiftleft" %% "codepropertygraph" % Versions.cpg,
+  "io.shiftleft" %% "semanticcpg" % Versions.cpg,
+  "io.shiftleft" %% "console" % Versions.cpg,
+  "io.shiftleft" %% "console" % Versions.cpg % Test classifier "tests",
+  "io.shiftleft" %% "dataflowengineoss" % Versions.cpg,
+  "io.shiftleft" %% "fuzzyc2cpg" % Versions.cpg, // only needed for joern-parse - TODO MP consider to remove?
   "io.github.plume-oss"    % "plume" % "0.5.12" exclude("io.github.plume-oss", "cpgconv"),
 
   "com.lihaoyi" %% "requests" % "0.6.5",
@@ -36,7 +34,7 @@ downloadFuzzyPreprocessor := {
   val log = streams.value.log
   val ppFilename = "fuzzyppcli.zip"
   val ppUrl = new URL(
-    s"https://github.com/ShiftLeftSecurity/codepropertygraph/releases/download/v${Versions.cpgVersion}/$ppFilename")
+    s"https://github.com/ShiftLeftSecurity/codepropertygraph/releases/download/v${Versions.cpg}/$ppFilename")
   val ppOutputDir = file("fuzzyppcli")
 
   log.info(s"trying to download fuzzypp from $ppUrl")
@@ -62,7 +60,9 @@ Universal/mappings ++= downloadFuzzyPreprocessor.value.map { fuzzyppdir =>
 lazy val cpgVersionFile = taskKey[File]("persist cpg version in file (e.g. for schema-extender)")
 cpgVersionFile := {
   val ret = target.value / "cpg-version"
-  better.files.File(ret.getPath).writeText(Versions.cpgVersion)
+  better.files.File(ret.getPath)
+    .createIfNotExists(createParents = true)
+    .writeText(Versions.cpg)
   ret
 }
 Universal/mappings += cpgVersionFile.value -> "schema-extender/cpg-version"
@@ -129,7 +129,22 @@ generateScaladocs := {
   out
 }
 
-import sbt.Path.directory
-Universal/packageBin/mappings ++= directory(new File("joern-cli/src/main/resources/scripts"))
+Universal/packageBin/mappings ++= sbt.Path.directory(new File("joern-cli/src/main/resources/scripts"))
+
+Universal/mappings ++= {
+  import net.lingala.zip4j.ZipFile
+  val frontendsDirRoot = s"target/cpg-${Versions.cpg}/"
+  val frontendsDir = s"$frontendsDirRoot/frontends"
+
+  new ZipFile(Frontends.downloadC2CpgZip).extractAll(s"$frontendsDir/c2cpg")
+  new ZipFile(Frontends.downloadFuzzyc2CpgZip).extractAll(s"$frontendsDir/fuzzyc2cpg")
+  new ZipFile(Frontends.downloadJs2CpgZip).extractAll(s"$frontendsDir/js2cpg")
+
+  NativePackagerHelper.contentOf(frontendsDirRoot)
+}
+
+Universal/mappings ++= NativePackagerHelper.contentOf((Projects.ghidra2cpg/stage).value).map {
+  case (file, str) => (file, s"frontends/ghidra2cpg/$str")
+}
 
 maintainer := "[email protected]"
diff --git a/joern-cli/src/main/scala/io/shiftleft/joern/C2Cpg.scala b/joern-cli/src/main/scala/io/shiftleft/joern/C2Cpg.scala
diff --git a/joern-cli/src/main/scala/io/shiftleft/joern/Fuzzyc2cpg.scala b/joern-cli/src/main/scala/io/shiftleft/joern/Fuzzyc2cpg.scala
diff --git a/joern-cli/src/main/scala/io/shiftleft/joern/Js2Cpg.scala b/joern-cli/src/main/scala/io/shiftleft/joern/Js2Cpg.scala
diff --git a/joern-cli/src/universal/c2cpg.sh b/joern-cli/src/universal/c2cpg.sh
@@ -7,7 +7,7 @@ else
 fi
 
 SCRIPT_ABS_DIR=$(dirname "$SCRIPT_ABS_PATH")
-SCRIPT="$SCRIPT_ABS_DIR"/bin/c-2-cpg
+SCRIPT="$SCRIPT_ABS_DIR"/frontends/c2cpg/bin/c2cpg
 
 if [ ! -f "$SCRIPT" ]; then
     echo "Unable to find $SCRIPT, have you created the distribution?"

diff --git a/joern-cli/src/universal/fuzzyc2cpg.sh b/joern-cli/src/universal/fuzzyc2cpg.sh
@@ -6,7 +6,7 @@ else
     SCRIPT_ABS_PATH=$(readlink -f "$0")
 fi
 SCRIPT_ABS_DIR=$(dirname "$SCRIPT_ABS_PATH")
-SCRIPT="$SCRIPT_ABS_DIR"/bin/fuzzyc-2-cpg
+SCRIPT="$SCRIPT_ABS_DIR"/frontends/fuzzyc2cpg/bin/fuzzyc2cpg
 
 if [ ! -f "$SCRIPT" ]; then
     echo "Unable to find $SCRIPT, have you created the distribution?"

diff --git a/joern-cli/src/universal/ghidra2cpg b/joern-cli/src/universal/ghidra2cpg
@@ -6,7 +6,7 @@ else
     SCRIPT_ABS_PATH=$(readlink -f "$0")
 fi
 SCRIPT_ABS_DIR=$(dirname "$SCRIPT_ABS_PATH")
-SCRIPT="$SCRIPT_ABS_DIR"/../ghidra2cpg/bin/ghidra2cpg
+SCRIPT="$SCRIPT_ABS_DIR"/frontends/ghidra2cpg/bin/ghidra2cpg
 
 if [ ! -f "$SCRIPT" ]; then
     echo "Unable to find $SCRIPT, have you created the distribution?"

diff --git a/joern-cli/src/universal/js2cpg.sh b/joern-cli/src/universal/js2cpg.sh
@@ -7,7 +7,7 @@ else
 fi
 
 SCRIPT_ABS_DIR=$(dirname "$SCRIPT_ABS_PATH")
-SCRIPT="$SCRIPT_ABS_DIR"/bin/js-2-cpg
+SCRIPT="$SCRIPT_ABS_DIR"/frontends/js2cpg/bin/js2cpg
 
 if [ ! -f "$SCRIPT" ]; then
     echo "Unable to find $SCRIPT, have you created the distribution?"

diff --git a/joern-install.sh b/joern-install.sh
@@ -162,6 +162,17 @@ fi
 unzip -qo -d "$JOERN_INSTALL_DIR" "$SCRIPT_ABS_DIR"/joern-cli.zip
 rm "$SCRIPT_ABS_DIR"/joern-cli.zip
 
+cd "$JOERN_INSTALL_DIR"
+unzip -qo joern-cli.zip
+FRONTENDS_DIR="$JOERN_INSTALL_DIR/joern-cli/frontends"
+mkdir -p "$FRONTENDS_DIR"
+unzip -qo c2cpg.zip -d "$FRONTENDS_DIR/c2cpg"
+unzip -qo fuzzyc2cpg.zip -d "$FRONTENDS_DIR/fuzzyc2cpg"
+unzip -qo js2cpg.zip -d "$FRONTENDS_DIR/js2cpg"
+rm joern-cli.zip c2cpg.zip fuzzyc2cpg.zip js2cpg.zip
+cd -
+
+
 if [ $INTERACTIVE = false ] && [ "$(whoami)" != "root" ]; then
   echo "==============================================================="
   echo "WARNING: you are NOT root and you are running non-interactively"

diff --git a/project/Frontends.scala b/project/Frontends.scala
@@ -0,0 +1,13 @@
+
+object Frontends {
+
+  def downloadC2CpgZip =
+    SimpleCache.downloadMaybe(s"https://github.com/ShiftLeftSecurity/codepropertygraph/releases/download/v${Versions.cpg}/c2cpg.zip")
+
+  def downloadFuzzyc2CpgZip =
+    SimpleCache.downloadMaybe(s"https://github.com/ShiftLeftSecurity/codepropertygraph/releases/download/v${Versions.cpg}/fuzzy2cpg.zip")
+
+  def downloadJs2CpgZip =
+    SimpleCache.downloadMaybe(s"https://github.com/ShiftLeftSecurity/js2cpg/releases/download/v${Versions.js2cpg}/js2cpg.zip")
+
+}
diff --git a/project/SimpleCache.scala b/project/SimpleCache.scala
@@ -0,0 +1,25 @@
+import sbt.{IO,URL}
+import java.io.File
+import java.net.URLEncoder
+
+object SimpleCache {
+  val LocalCacheDir = ".local"
+
+  /** download given url if not yet present in cache, then add to cache
+    * warning: only use for immutable artifacts, i.e. do not use for snapshot artifacts that may change 
+    */
+  def downloadMaybe(url: String): File = {
+    val urlEncoded = URLEncoder.encode(url, "UTF-8")
+    val localFile = new File(s"$LocalCacheDir/$urlEncoded")
+
+    if (!localFile.exists) {
+      println(s"downloading $url")
+      sbt.io.Using.urlInputStream(new URL(url)) { inputStream =>
+        IO.transfer(inputStream, localFile)
+      }
+    }
+
+    localFile
+  }
+
+}
diff --git a/project/Versions.scala b/project/Versions.scala
@@ -1,6 +1,6 @@
 /* reads version declarations from /build.sbt so that we can declare them in one place */
 object Versions {
-  val cpgVersion = parseVersion("cpgVersion")
+  val cpg = parseVersion("cpgVersion")
   val ghidra2cpg = parseVersion("ghidra2cpgVersion")
   val js2cpg = parseVersion("js2cpgVersion")