From c121b4378f9100680e131f509563d17505f7f556 Mon Sep 17 00:00:00 2001 From: yazgoo Date: Sun, 9 Jun 2024 23:04:37 +0200 Subject: [PATCH 01/28] dependencies analysis --- .../ch/epfl/scala/SubmitDependencyGraph.scala | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/sbt-plugin/src/main/scala/ch/epfl/scala/SubmitDependencyGraph.scala b/sbt-plugin/src/main/scala/ch/epfl/scala/SubmitDependencyGraph.scala index d4c96b8..5d6fea3 100644 --- a/sbt-plugin/src/main/scala/ch/epfl/scala/SubmitDependencyGraph.scala +++ b/sbt-plugin/src/main/scala/ch/epfl/scala/SubmitDependencyGraph.scala @@ -24,6 +24,10 @@ object SubmitDependencyGraph { val Generate = "githubGenerateSnapshot" private val GenerateUsage = s"""$Generate {"ignoredModules":[], "ignoredConfig":[]}""" private val GenerateDetail = "Generate the dependency graph of a set of projects and scala versions" + + val AnalyzeDependecies = "githubAnalyzeDependencies" + private val AnalyzeDependenciesUsage = s"""$AnalyzeDependecies [get|list] pattern""" + private val AnalyzeDependenciesDetail = "Analyze the dependencies base on a search pattern" private val GenerateInternal = s"${Generate}Internal" private val InternalOnly = "internal usage only" @@ -33,6 +37,7 @@ object SubmitDependencyGraph { val commands: Seq[Command] = Seq( Command(Generate, (GenerateUsage, GenerateDetail), GenerateDetail)(inputParser)(generate), + Command(AnalyzeDependecies, (AnalyzeDependenciesUsage, AnalyzeDependenciesDetail), AnalyzeDependenciesDetail)(extractPattern)(analyzeDependencies), Command.command(GenerateInternal, InternalOnly, InternalOnly)(generateInternal), Command.command(Submit, SubmitDetail, SubmitDetail)(submit) ) @@ -50,6 +55,32 @@ object SubmitDependencyGraph { .get }.failOnException + sealed trait AnalysisAction { + def name: String + } + object AnalysisAction { + case object Get extends AnalysisAction { + val name = "get" + } + case object List extends AnalysisAction { + val name = "list" + } + val values: Seq[AnalysisAction] = Seq(Get, List) + def fromString(str: String): Option[AnalysisAction] = values.find(_.name == str) + } + + + case class AnalysisParams(action: AnalysisAction, pattern: Option[String]) + + private def extractPattern(state: State): Parser[AnalysisParams] = + Parsers.any.*.map { raw => + raw.mkString.trim.split(" ").toSeq match { + case Seq(action, pattern) => + AnalysisParams(AnalysisAction.fromString(action).get, Some(pattern)) + } + }.failOnException + + private def generate(state: State, input: DependencySnapshotInput): State = { val loadedBuild = state.setting(Keys.loadedBuild) // all project refs that have a Scala version @@ -81,6 +112,43 @@ object SubmitDependencyGraph { commands.toList ::: initState } + private def analyzeDependencies(state: State, params: AnalysisParams): State = { + val action = params.action + params.pattern.foreach { pattern => + def getDeps(dependencies: Seq[String], pattern: String): Seq[String] = { + for { + dep <- dependencies.filter(_.contains(pattern)) + } yield dep + } + + def resolvedDeps(tabs: String, acc: Seq[String], resolvedByName: Map[String, DependencyNode], pattern: String): Seq[String] = { + acc ++ (for { + (name, resolved) <- resolvedByName.toSeq + matchingDependency <- getDeps(resolved.dependencies, pattern) + resolvedDep <- resolvedDeps(" " + tabs, acc ++ Seq(tabs + matchingDependency), resolvedByName, name) + } yield resolvedDep) + } + + val matches = (for { + manifests <- state.get(githubManifestsKey).toSeq + (_, manifest) <- manifests + } yield (manifest, resolvedDeps("", Nil, manifest.resolved, pattern))).toMap + + if (action == AnalysisAction.Get) { + matches.foreach { case (manifest, deps) => + println(s"Manifest: ${manifest.name}") + println(deps.mkString("\n")) + } + } + else if (action == AnalysisAction.List) { + println( + matches.flatMap { case (manifest, deps) => deps } + .filter(_.contains(pattern)).toSet.mkString("\n")) + } + } + state + } + private def generateInternal(state: State): State = { val snapshot = githubDependencySnapshot(state) val snapshotJson = CompactPrinter(Converter.toJsonUnsafe(snapshot)) From 9936d295c3e7a69f4d5b9bb84c11b593d90c3ce6 Mon Sep 17 00:00:00 2001 From: yazgoo Date: Wed, 12 Jun 2024 00:33:53 +0200 Subject: [PATCH 02/28] add getvulnerabilities --- .../ch/epfl/scala/SubmitDependencyGraph.scala | 94 +++++++++++++++++-- 1 file changed, 86 insertions(+), 8 deletions(-) diff --git a/sbt-plugin/src/main/scala/ch/epfl/scala/SubmitDependencyGraph.scala b/sbt-plugin/src/main/scala/ch/epfl/scala/SubmitDependencyGraph.scala index 5d6fea3..99f81da 100644 --- a/sbt-plugin/src/main/scala/ch/epfl/scala/SubmitDependencyGraph.scala +++ b/sbt-plugin/src/main/scala/ch/epfl/scala/SubmitDependencyGraph.scala @@ -65,18 +65,21 @@ object SubmitDependencyGraph { case object List extends AnalysisAction { val name = "list" } - val values: Seq[AnalysisAction] = Seq(Get, List) + case object Alerts extends AnalysisAction { + val name = "alerts" + } + val values: Seq[AnalysisAction] = Seq(Get, List, Alerts) def fromString(str: String): Option[AnalysisAction] = values.find(_.name == str) } - case class AnalysisParams(action: AnalysisAction, pattern: Option[String]) + case class AnalysisParams(action: AnalysisAction, arg: Option[String]) private def extractPattern(state: State): Parser[AnalysisParams] = Parsers.any.*.map { raw => raw.mkString.trim.split(" ").toSeq match { - case Seq(action, pattern) => - AnalysisParams(AnalysisAction.fromString(action).get, Some(pattern)) + case Seq(action, arg) => + AnalysisParams(AnalysisAction.fromString(action).get, Some(arg)) } }.failOnException @@ -112,9 +115,7 @@ object SubmitDependencyGraph { commands.toList ::: initState } - private def analyzeDependencies(state: State, params: AnalysisParams): State = { - val action = params.action - params.pattern.foreach { pattern => + private def analyzeDependenciesInternal(state: State, action: AnalysisAction, pattern: String) = { def getDeps(dependencies: Seq[String], pattern: String): Seq[String] = { for { dep <- dependencies.filter(_.contains(pattern)) @@ -137,7 +138,7 @@ object SubmitDependencyGraph { if (action == AnalysisAction.Get) { matches.foreach { case (manifest, deps) => println(s"Manifest: ${manifest.name}") - println(deps.mkString("\n")) + println(deps.map{ dep : String => s" ${dep}" }.mkString("\n")) } } else if (action == AnalysisAction.List) { @@ -145,6 +146,49 @@ object SubmitDependencyGraph { matches.flatMap { case (manifest, deps) => deps } .filter(_.contains(pattern)).toSet.mkString("\n")) } + } + + private def getGithubTokenFromGhConfigDir(): String = { + // use GH_CONFIG_DIR variable if it exists + val ghConfigDir = Properties.envOrElse("GH_CONFIG_DIR", Paths.get(System.getProperty("user.home"), ".config", "gh").toString) + val ghConfigFile = Paths.get(ghConfigDir).resolve("hosts.yml").toFile + if (ghConfigFile.exists()) { + val lines = IO.readLines(ghConfigFile) + val tokenLine = lines.find(_.contains("oauth_token")) + tokenLine match { + case Some(line) => line.split(":").last.trim + case None => throw new MessageOnlyException("No token found in gh config file") + } + } else { + throw new MessageOnlyException("No gh config file found") + } + } + + private def downloadAlerts(state: State, repo: String) = { + val snapshotUrl = s"https://api.github.com/repos/$repo/dependabot/alerts" + val request = Gigahorse + .url(snapshotUrl) + .get + .addHeaders( + "Authorization" -> s"token ${getGithubTokenFromGhConfigDir()}" + ) + state.log.info(s"Downloading alerts from $snapshotUrl") + for { + httpResp <- Try(Await.result(http.processFull(request), Duration.Inf)) + alerts <- getAlerts(httpResp) + } yield { + println(httpResp.bodyAsString) + } + } + + private def analyzeDependencies(state: State, params: AnalysisParams): State = { + val action = params.action + if (Seq(AnalysisAction.Get, AnalysisAction.List).contains(action)) { + params.arg.foreach { pattern => analyzeDependenciesInternal(state, action, pattern) } + } else if (action == AnalysisAction.Alerts) { + params.arg.foreach { repo => + downloadAlerts(state, repo) + } } state } @@ -201,6 +245,40 @@ object SubmitDependencyGraph { for (output <- githubOutput()) IO.writeLines(output, outputs.map { case (name, value) => s"${name}=${value}" }, append = true) + case class Vulnerability( + packageId: String, + vulnerableVersionRange: String, + firstPatchedVersion: String, + severity: String, + ) + + private def getVulnerabilities(httpResp: FullResponse): Try[Seq[Vulnerability]] = + httpResp.status match { + case status if status / 100 == 2 => + // here is the jq command: + // jq -r '.[]|select((.state == "open"))|.security_vulnerability|"\(.package.name);\(.vulnerable_version_range);\(.first_patched_version.identifier);\(.severity)"' | sort + // do the equivalent in scala and build a seq of Vulnerability, without a converter + val json = JsonParser.parseFromByteBuffer(httpResp.bodyAsByteBuffer).get + + + // fix line below, because "value asArray is not a member of sjsonnew.shaded.scalajson.ast.unsafe.JValue" + // val vulnerabilities = json.asArray.get.value.map { value => + val vulnerabilities = json.asArray.get.value.map { value => + val obj = value.asObject.get + val securityVulnerability = obj("security_vulnerability").get.asObject.get + Vulnerability( + securityVulnerability("package").get.asObject.get("name").get.asString.get, + securityVulnerability("vulnerable_version_range").get.asString.get, + securityVulnerability("first_patched_version").get.asObject.get("identifier").get.asString.get, + securityVulnerability("severity").get.asString.get + ) + } + case status => + val message = + s"Unexpected status $status ${httpResp.statusText} with body:\n${httpResp.bodyAsString}" + throw new MessageOnlyException(message) + } + private def getSnapshot(httpResp: FullResponse): Try[SnapshotResponse] = httpResp.status match { case status if status / 100 == 2 => From 212560d1da7c4729191f95a0b904a1e84418889f Mon Sep 17 00:00:00 2001 From: yazgoo Date: Wed, 12 Jun 2024 21:16:00 +0200 Subject: [PATCH 03/28] add cve analyzis --- .../scala/GithubDependencyGraphPlugin.scala | 1 + .../ch/epfl/scala/SubmitDependencyGraph.scala | 134 +++++++++++++++--- 2 files changed, 116 insertions(+), 19 deletions(-) diff --git a/sbt-plugin/src/main/scala/ch/epfl/scala/GithubDependencyGraphPlugin.scala b/sbt-plugin/src/main/scala/ch/epfl/scala/GithubDependencyGraphPlugin.scala index 245ffec..dd607e8 100644 --- a/sbt-plugin/src/main/scala/ch/epfl/scala/GithubDependencyGraphPlugin.scala +++ b/sbt-plugin/src/main/scala/ch/epfl/scala/GithubDependencyGraphPlugin.scala @@ -32,6 +32,7 @@ object GithubDependencyGraphPlugin extends AutoPlugin { val githubManifestsKey: AttributeKey[Map[String, githubapi.Manifest]] = AttributeKey("githubDependencyManifests") val githubProjectsKey: AttributeKey[Seq[ProjectRef]] = AttributeKey("githubProjectRefs") val githubSnapshotFileKey: AttributeKey[File] = AttributeKey("githubSnapshotFile") + val githubAlertsKey: AttributeKey[Seq[SubmitDependencyGraph.Vulnerability]] = AttributeKey("githubAlerts") val githubDependencyManifest: TaskKey[Option[githubapi.Manifest]] = taskKey( "The dependency manifest of the project" diff --git a/sbt-plugin/src/main/scala/ch/epfl/scala/SubmitDependencyGraph.scala b/sbt-plugin/src/main/scala/ch/epfl/scala/SubmitDependencyGraph.scala index 99f81da..61d08b1 100644 --- a/sbt-plugin/src/main/scala/ch/epfl/scala/SubmitDependencyGraph.scala +++ b/sbt-plugin/src/main/scala/ch/epfl/scala/SubmitDependencyGraph.scala @@ -17,7 +17,7 @@ import gigahorse.HttpClient import gigahorse.support.asynchttpclient.Gigahorse import sbt._ import sbt.internal.util.complete._ -import sjsonnew.shaded.scalajson.ast.unsafe.JValue +import sjsonnew.shaded.scalajson.ast.unsafe.{ JValue, JArray, JObject, JField, JString } import sjsonnew.support.scalajson.unsafe.{Parser => JsonParser, _} object SubmitDependencyGraph { @@ -68,7 +68,10 @@ object SubmitDependencyGraph { case object Alerts extends AnalysisAction { val name = "alerts" } - val values: Seq[AnalysisAction] = Seq(Get, List, Alerts) + case object Cves extends AnalysisAction { + val name = "cves" + } + val values: Seq[AnalysisAction] = Seq(Get, List, Alerts, Cves) def fromString(str: String): Option[AnalysisAction] = values.find(_.name == str) } @@ -164,7 +167,7 @@ object SubmitDependencyGraph { } } - private def downloadAlerts(state: State, repo: String) = { + private def downloadAlerts(state: State, repo: String) : Try[State] = { val snapshotUrl = s"https://api.github.com/repos/$repo/dependabot/alerts" val request = Gigahorse .url(snapshotUrl) @@ -175,22 +178,99 @@ object SubmitDependencyGraph { state.log.info(s"Downloading alerts from $snapshotUrl") for { httpResp <- Try(Await.result(http.processFull(request), Duration.Inf)) - alerts <- getAlerts(httpResp) + vulnerabilities <- getVulnerabilities(httpResp) } yield { - println(httpResp.bodyAsString) + vulnerabilities.foreach { v => + println(s"${v.packageId} ${v.vulnerableVersionRange} ${v.firstPatchedVersion} ${v.severity}") + } + state.put(githubAlertsKey, vulnerabilities) } } + private def getAllArtifacts(state: State): Seq[String] = { + for { + manifests <- state.get(githubManifestsKey).toSeq + (_, manifest) <- manifests + artifact <- manifest.resolved.values.toSeq + } yield artifact.package_url + } + + /* + # example alert + # [ "com.google.guava:guava", ">= 1.0, < 32.0.0-android", "32.0.0-android" ] + # example artifact + # "pkg:maven/com.google.guava/guava@31.1-jre" + */ + + private def vulnerabilityMatchesArtifact(alert: Vulnerability, artifact: String): String = { + val alertMavenPath = s"pkg:maven/${alert.packageId.replace(":", "/")}@" + if (artifact.startsWith(alertMavenPath)) { + val version = artifact.split("@").last + if (alert.vulnerableVersionRange.contains(",")) { + val range = alert.vulnerableVersionRange.split(",") + if (range.head <= version && version < range.last) { + "bad" + } else { + "good" + } + } else { + if (alert.vulnerableVersionRange <= version) { + "bad" + } else { + "good" + } + } + } else { + "no" + } + } + + private def vulnerabilityMatchesArtifacts(alert: Vulnerability, artifacts: Seq[String]): Map[String, Seq[String]] = { + artifacts.foldLeft(Map("good" -> Seq.empty[String], "bad" -> Seq.empty[String])) { (acc, artifact) => + val res = vulnerabilityMatchesArtifact(alert, artifact) + if (res != "no") { + acc.updated(res, acc(res) :+ artifact) + } else { + acc + } + } + } + + private def analyzeCves(state: State): State = { + val vulnerabilities = state.get(githubAlertsKey).get + val cves = vulnerabilities + val artifacts = getAllArtifacts(state) + cves.foreach { v => + val matches = vulnerabilityMatchesArtifacts(v, artifacts) + println(s"${v.packageId} ${v.vulnerableVersionRange} ${v.firstPatchedVersion} ${v.severity}") + if (matches("good").length + matches("bad").length > 0) { + matches("good").foreach { m => + println(s" 🟢 ${m}") + } + matches("bad").foreach { m => + println(s" 🔴 ${m}") + } + } else { + println(" 🎉 no match (dependency was probably removed)") + } + } + state + } + private def analyzeDependencies(state: State, params: AnalysisParams): State = { val action = params.action if (Seq(AnalysisAction.Get, AnalysisAction.List).contains(action)) { params.arg.foreach { pattern => analyzeDependenciesInternal(state, action, pattern) } + state } else if (action == AnalysisAction.Alerts) { - params.arg.foreach { repo => - downloadAlerts(state, repo) - } + params.arg.map { repo => + downloadAlerts(state, repo).get + }.get + } else if (action == AnalysisAction.Cves) { + analyzeCves(state) + } else { + state } - state } private def generateInternal(state: State): State = { @@ -254,24 +334,40 @@ object SubmitDependencyGraph { private def getVulnerabilities(httpResp: FullResponse): Try[Seq[Vulnerability]] = httpResp.status match { - case status if status / 100 == 2 => + case status if status / 100 == 2 => Try { // here is the jq command: // jq -r '.[]|select((.state == "open"))|.security_vulnerability|"\(.package.name);\(.vulnerable_version_range);\(.first_patched_version.identifier);\(.severity)"' | sort // do the equivalent in scala and build a seq of Vulnerability, without a converter - val json = JsonParser.parseFromByteBuffer(httpResp.bodyAsByteBuffer).get + val json : JValue = JsonParser.parseFromByteBuffer(httpResp.bodyAsByteBuffer).get + + // // fix line below, because "value asArray is not a member of sjsonnew.shaded.scalajson.ast.unsafe.JValue" // val vulnerabilities = json.asArray.get.value.map { value => - val vulnerabilities = json.asArray.get.value.map { value => - val obj = value.asObject.get - val securityVulnerability = obj("security_vulnerability").get.asObject.get - Vulnerability( - securityVulnerability("package").get.asObject.get("name").get.asString.get, - securityVulnerability("vulnerable_version_range").get.asString.get, - securityVulnerability("first_patched_version").get.asObject.get("identifier").get.asString.get, - securityVulnerability("severity").get.asString.get + json.asInstanceOf[JArray].value.map { value => + val obj = value.asInstanceOf[JObject].value + + // convert obj to map of string => JValue : + + val map = obj.map { case JField(k, v) => (k, v) }.toMap + + val securityVulnerability = map("security_vulnerability").asInstanceOf[JObject].value.map { case JField(k, v) => (k, v) }.toMap + + val packageObj = securityVulnerability("package").asInstanceOf[JObject].value.map { case JField(k, v) => (k, v) }.toMap + + val firstPatchedVersion = Try(securityVulnerability("first_patched_version").asInstanceOf[JObject].value.map { case JField(k, v) => (k, v) }.toMap).getOrElse(Map.empty) + + ( + map("state") == JString("open"), + Vulnerability( + packageObj("name").asInstanceOf[JString].value, + securityVulnerability("vulnerable_version_range").asInstanceOf[JString].value, + firstPatchedVersion.get("identifier").map { x => x.asInstanceOf[JString].value }.getOrElse(""), + securityVulnerability("severity").asInstanceOf[JString].value ) + ) + }.filter(_._1).map(_._2) } case status => val message = From 98c468bb11a749a674b493a9c9798d227dc3c8cd Mon Sep 17 00:00:00 2001 From: yazgoo Date: Wed, 12 Jun 2024 23:28:31 +0200 Subject: [PATCH 04/28] fix version matching --- .../ch/epfl/scala/SubmitDependencyGraph.scala | 44 ++++++++++++------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/sbt-plugin/src/main/scala/ch/epfl/scala/SubmitDependencyGraph.scala b/sbt-plugin/src/main/scala/ch/epfl/scala/SubmitDependencyGraph.scala index 61d08b1..2ba2db0 100644 --- a/sbt-plugin/src/main/scala/ch/epfl/scala/SubmitDependencyGraph.scala +++ b/sbt-plugin/src/main/scala/ch/epfl/scala/SubmitDependencyGraph.scala @@ -83,6 +83,8 @@ object SubmitDependencyGraph { raw.mkString.trim.split(" ").toSeq match { case Seq(action, arg) => AnalysisParams(AnalysisAction.fromString(action).get, Some(arg)) + case Seq(action) => + AnalysisParams(AnalysisAction.fromString(action).get, None) } }.failOnException @@ -202,27 +204,35 @@ object SubmitDependencyGraph { # "pkg:maven/com.google.guava/guava@31.1-jre" */ + // versionMatchesRange("31.1-jre", ">= 1.0, < 32.0.0-android") => true + // versionMatchesRange("2.8.5", "< 2.9.0") => true + // versionMatchesRange("2.9.0", "< 2.9.0") => false + + private def translateToSemVer(string: String): String = { + // if a version in the string has more than 3 digits, we assume it's a pre-release version + // ">= 1.0 <32.0.0.4" => ">= 1.0 < 32.0.0-4" + string.replaceAll("([0-9]+)\\.([0-9]+)\\.([0-9]+)\\.([0-9]+)", "$1.$2.$3-$4") + } + + private def versionMatchesRange(versionStr: String, rangeStr: String): Boolean = { + val range = rangeStr.replaceAll(" ", "").replace(",", " ") + VersionNumber(translateToSemVer(versionStr)).matchesSemVer(SemanticSelector(translateToSemVer(range))) + } + private def vulnerabilityMatchesArtifact(alert: Vulnerability, artifact: String): String = { val alertMavenPath = s"pkg:maven/${alert.packageId.replace(":", "/")}@" if (artifact.startsWith(alertMavenPath)) { val version = artifact.split("@").last - if (alert.vulnerableVersionRange.contains(",")) { - val range = alert.vulnerableVersionRange.split(",") - if (range.head <= version && version < range.last) { - "bad" - } else { - "good" - } - } else { - if (alert.vulnerableVersionRange <= version) { - "bad" - } else { - "good" - } - } - } else { - "no" - } + // vulnerableVersionRange can be ">= 1.0, < 32.0.0-android" or "< 2.9.0" + val bad = versionMatchesRange(version, alert.vulnerableVersionRange) + if (bad) { + "bad" + } else { + "good" + } + } else { + "no" + } } private def vulnerabilityMatchesArtifacts(alert: Vulnerability, artifacts: Seq[String]): Map[String, Seq[String]] = { From 8fa7da1e47d6547463f54efcd3d973e5677245d0 Mon Sep 17 00:00:00 2001 From: yazgoo Date: Thu, 13 Jun 2024 08:18:14 +0200 Subject: [PATCH 05/28] support name matching --- .../ch/epfl/scala/SubmitDependencyGraph.scala | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/sbt-plugin/src/main/scala/ch/epfl/scala/SubmitDependencyGraph.scala b/sbt-plugin/src/main/scala/ch/epfl/scala/SubmitDependencyGraph.scala index 2ba2db0..a19cee3 100644 --- a/sbt-plugin/src/main/scala/ch/epfl/scala/SubmitDependencyGraph.scala +++ b/sbt-plugin/src/main/scala/ch/epfl/scala/SubmitDependencyGraph.scala @@ -128,16 +128,30 @@ object SubmitDependencyGraph { } def resolvedDeps(tabs: String, acc: Seq[String], resolvedByName: Map[String, DependencyNode], pattern: String): Seq[String] = { + acc ++ (for { (name, resolved) <- resolvedByName.toSeq - matchingDependency <- getDeps(resolved.dependencies, pattern) - resolvedDep <- resolvedDeps(" " + tabs, acc ++ Seq(tabs + matchingDependency), resolvedByName, name) - } yield resolvedDep) + matchingDependencies = getDeps(resolved.dependencies, pattern) + resultDeps <- if (matchingDependencies.isEmpty) { + if (name.contains(pattern)) { + Seq(Seq(tabs + name)) + } else { + Nil + } + } else { + for { + matchingDependency <- matchingDependencies + } yield resolvedDeps(" " + tabs, acc ++ Seq(tabs + matchingDependency), resolvedByName, name) + } + resultDep <- resultDeps + } yield { + resultDep + }) } val matches = (for { manifests <- state.get(githubManifestsKey).toSeq - (_, manifest) <- manifests + (name, manifest) <- manifests } yield (manifest, resolvedDeps("", Nil, manifest.resolved, pattern))).toMap if (action == AnalysisAction.Get) { From 9d17482bbfd4df0b7d1fe254223cd759906d1276 Mon Sep 17 00:00:00 2001 From: yazgoo Date: Thu, 13 Jun 2024 13:19:04 +0200 Subject: [PATCH 06/28] improvements --- .../ch/epfl/scala/SubmitDependencyGraph.scala | 26 +++++++++++++------ 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/sbt-plugin/src/main/scala/ch/epfl/scala/SubmitDependencyGraph.scala b/sbt-plugin/src/main/scala/ch/epfl/scala/SubmitDependencyGraph.scala index a19cee3..8918aee 100644 --- a/sbt-plugin/src/main/scala/ch/epfl/scala/SubmitDependencyGraph.scala +++ b/sbt-plugin/src/main/scala/ch/epfl/scala/SubmitDependencyGraph.scala @@ -167,19 +167,27 @@ object SubmitDependencyGraph { } } + private def getGithubToken(ghConfigFile: File): Option[String] = { + println(s"extract token from ${ghConfigFile.getPath()}") + if (ghConfigFile.exists()) { + val lines = IO.readLines(ghConfigFile) + val tokenLine = lines.find(_.contains("oauth_token")) + tokenLine.map { line => line.split(":").last.trim } + } else { + None + } + } + private def getGithubTokenFromGhConfigDir(): String = { // use GH_CONFIG_DIR variable if it exists val ghConfigDir = Properties.envOrElse("GH_CONFIG_DIR", Paths.get(System.getProperty("user.home"), ".config", "gh").toString) val ghConfigFile = Paths.get(ghConfigDir).resolve("hosts.yml").toFile - if (ghConfigFile.exists()) { - val lines = IO.readLines(ghConfigFile) - val tokenLine = lines.find(_.contains("oauth_token")) - tokenLine match { - case Some(line) => line.split(":").last.trim - case None => throw new MessageOnlyException("No token found in gh config file") + getGithubToken(ghConfigFile).getOrElse { + val ghConfigPath = Properties.envOrElse("HUB_CONFIG", Paths.get(System.getProperty("user.home"), ".config", "hub").toString) + val hubConfigFile = Paths.get(ghConfigPath).toFile + getGithubToken(hubConfigFile).getOrElse { + githubToken() } - } else { - throw new MessageOnlyException("No gh config file found") } } @@ -204,11 +212,13 @@ object SubmitDependencyGraph { } private def getAllArtifacts(state: State): Seq[String] = { + { for { manifests <- state.get(githubManifestsKey).toSeq (_, manifest) <- manifests artifact <- manifest.resolved.values.toSeq } yield artifact.package_url + }.toSet.toSeq } /* From 21a3abd9cacf70b6d83ff7ba8ed6e4fb84b6db60 Mon Sep 17 00:00:00 2001 From: yazgoo Date: Thu, 13 Jun 2024 23:51:53 +0200 Subject: [PATCH 07/28] refactoring --- .../epfl/scala/AnalyzeDependencyGraph.scala | 319 ++++++++++++++++++ .../scala/GithubDependencyGraphPlugin.scala | 4 +- .../ch/epfl/scala/SubmitDependencyGraph.scala | 277 +-------------- 3 files changed, 322 insertions(+), 278 deletions(-) create mode 100644 sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala diff --git a/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala b/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala new file mode 100644 index 0000000..9289eb7 --- /dev/null +++ b/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala @@ -0,0 +1,319 @@ +package ch.epfl.scala + +import java.nio.file.Paths + +import scala.concurrent.Await +import scala.concurrent.duration.Duration +import scala.util.Properties +import scala.util.Try + +import ch.epfl.scala.GithubDependencyGraphPlugin.autoImport._ +import ch.epfl.scala.githubapi._ +import gigahorse.support.asynchttpclient.Gigahorse +import sbt._ +import sbt.internal.util.complete._ +import sjsonnew.shaded.scalajson.ast.unsafe.{ JValue, JArray, JObject, JField, JString } +import gigahorse.FullResponse +import gigahorse.HttpClient +import gigahorse.support.asynchttpclient.Gigahorse +import sjsonnew.support.scalajson.unsafe.{Parser => JsonParser} + +import scala.sys.process._ + +import sbt._ + +object AnalyzeDependencyGraph { + + val AnalyzeDependecies = "githubAnalyzeDependencies" + private val AnalyzeDependenciesUsage = s"""$AnalyzeDependecies [get|list|alerts|cves] pattern""" + private val AnalyzeDependenciesDetail = "Analyze the dependencies base on a search pattern" + + val commands: Seq[Command] = Seq( + Command(AnalyzeDependecies, (AnalyzeDependenciesUsage, AnalyzeDependenciesDetail), AnalyzeDependenciesDetail)(extractPattern)(analyzeDependencies), + ) + + private lazy val http: HttpClient = Gigahorse.http(Gigahorse.config) + + sealed trait AnalysisAction { + def name: String + } + object AnalysisAction { + case object Get extends AnalysisAction { + val name = "get" + } + case object List extends AnalysisAction { + val name = "list" + } + case object Alerts extends AnalysisAction { + val name = "alerts" + } + case object Cves extends AnalysisAction { + val name = "cves" + } + val values: Seq[AnalysisAction] = Seq(Get, List, Alerts, Cves) + def fromString(str: String): Option[AnalysisAction] = values.find(_.name == str) + } + + + case class AnalysisParams(action: AnalysisAction, arg: Option[String]) + + private def extractPattern(state: State): Parser[AnalysisParams] = + Parsers.any.*.map { raw => + raw.mkString.trim.split(" ").toSeq match { + case Seq(action, arg) => + AnalysisParams(AnalysisAction.fromString(action).get, Some(arg)) + case Seq(action) => + AnalysisParams(AnalysisAction.fromString(action).get, None) + } + }.failOnException + + private def analyzeDependenciesInternal(state: State, action: AnalysisAction, pattern: String) = { + def getDeps(dependencies: Seq[String], pattern: String): Seq[String] = { + for { + dep <- dependencies.filter(_.contains(pattern)) + } yield dep + } + + def resolvedDeps(tabs: String, acc: Seq[String], resolvedByName: Map[String, DependencyNode], pattern: String): Seq[String] = { + + acc ++ (for { + (name, resolved) <- resolvedByName.toSeq + matchingDependencies = getDeps(resolved.dependencies, pattern) + resultDeps <- if (matchingDependencies.isEmpty) { + if (name.contains(pattern)) { + Seq(Seq(tabs + name)) + } else { + Nil + } + } else { + for { + matchingDependency <- matchingDependencies + } yield resolvedDeps(" " + tabs, acc ++ Seq(tabs + matchingDependency), resolvedByName, name) + } + resultDep <- resultDeps + } yield { + resultDep + }) + } + + val matches = (for { + manifests <- state.get(githubManifestsKey).toSeq + (name, manifest) <- manifests + } yield (manifest, resolvedDeps("", Nil, manifest.resolved, pattern))).toMap + + if (action == AnalysisAction.Get) { + matches.foreach { case (manifest, deps) => + println(s"Manifest: ${manifest.name}") + println(deps.map{ dep : String => s" ${dep}" }.mkString("\n")) + } + } + else if (action == AnalysisAction.List) { + println( + matches.flatMap { case (manifest, deps) => deps } + .filter(_.contains(pattern)).toSet.mkString("\n")) + } + } + + private def getGithubToken(ghConfigFile: File): Option[String] = { + println(s"extract token from ${ghConfigFile.getPath()}") + if (ghConfigFile.exists()) { + val lines = IO.readLines(ghConfigFile) + val tokenLine = lines.find(_.contains("oauth_token")) + tokenLine.map { line => line.split(":").last.trim } + } else { + None + } + } + + private def getGithubTokenFromGhConfigDir(): String = { + // use GH_CONFIG_DIR variable if it exists + val ghConfigDir = Properties.envOrElse("GH_CONFIG_DIR", Paths.get(System.getProperty("user.home"), ".config", "gh").toString) + val ghConfigFile = Paths.get(ghConfigDir).resolve("hosts.yml").toFile + getGithubToken(ghConfigFile).getOrElse { + val ghConfigPath = Properties.envOrElse("HUB_CONFIG", Paths.get(System.getProperty("user.home"), ".config", "hub").toString) + val hubConfigFile = Paths.get(ghConfigPath).toFile + getGithubToken(hubConfigFile).getOrElse { + githubToken() + } + } + } + + private def downloadAlerts(state: State, repo: String) : Try[State] = { + val snapshotUrl = s"https://api.github.com/repos/$repo/dependabot/alerts" + val request = Gigahorse + .url(snapshotUrl) + .get + .addHeaders( + "Authorization" -> s"token ${getGithubTokenFromGhConfigDir()}" + ) + state.log.info(s"Downloading alerts from $snapshotUrl") + for { + httpResp <- Try(Await.result(http.processFull(request), Duration.Inf)) + vulnerabilities <- getVulnerabilities(httpResp) + } yield { + vulnerabilities.foreach { v => + println(s"${v.packageId} ${v.vulnerableVersionRange} ${v.firstPatchedVersion} ${v.severity}") + } + state.put(githubAlertsKey, vulnerabilities) + } + } + + private def getAllArtifacts(state: State): Seq[String] = { + { + for { + manifests <- state.get(githubManifestsKey).toSeq + (_, manifest) <- manifests + artifact <- manifest.resolved.values.toSeq + } yield artifact.package_url + }.toSet.toSeq + } + + /* + # example alert + # [ "com.google.guava:guava", ">= 1.0, < 32.0.0-android", "32.0.0-android" ] + # example artifact + # "pkg:maven/com.google.guava/guava@31.1-jre" + */ + + // versionMatchesRange("31.1-jre", ">= 1.0, < 32.0.0-android") => true + // versionMatchesRange("2.8.5", "< 2.9.0") => true + // versionMatchesRange("2.9.0", "< 2.9.0") => false + + private def translateToSemVer(string: String): String = { + // if a version in the string has more than 3 digits, we assume it's a pre-release version + // ">= 1.0 <32.0.0.4" => ">= 1.0 < 32.0.0-4" + string.replaceAll("([0-9]+)\\.([0-9]+)\\.([0-9]+)\\.([0-9]+)", "$1.$2.$3-$4") + } + + private def versionMatchesRange(versionStr: String, rangeStr: String): Boolean = { + val range = rangeStr.replaceAll(" ", "").replace(",", " ") + val result = VersionNumber(translateToSemVer(versionStr)).matchesSemVer(SemanticSelector(translateToSemVer(range))) + result + } + + private def vulnerabilityMatchesArtifact(alert: Vulnerability, artifact: String): String = { + val alertMavenPath = s"pkg:maven/${alert.packageId.replace(":", "/")}@" + if (artifact.startsWith(alertMavenPath)) { + val version = artifact.split("@").last + // vulnerableVersionRange can be ">= 1.0, < 32.0.0-android" or "< 2.9.0" + val bad = versionMatchesRange(version, alert.vulnerableVersionRange) + if (bad) { + "bad" + } else { + "good" + } + } else { + "no" + } + } + + private def vulnerabilityMatchesArtifacts(alert: Vulnerability, artifacts: Seq[String]): Map[String, Seq[String]] = { + artifacts.foldLeft(Map("good" -> Seq.empty[String], "bad" -> Seq.empty[String])) { (acc, artifact) => + val res = vulnerabilityMatchesArtifact(alert, artifact) + if (res != "no") { + acc.updated(res, acc(res) :+ artifact) + } else { + acc + } + } + } + + private def analyzeCves(state: State): State = { + val vulnerabilities = state.get(githubAlertsKey).get + val cves = vulnerabilities + val artifacts = getAllArtifacts(state) + cves.foreach { v => + val matches = vulnerabilityMatchesArtifacts(v, artifacts) + println(s"${v.packageId} ${v.vulnerableVersionRange} ${v.firstPatchedVersion} ${v.severity}") + if (matches("good").length + matches("bad").length > 0) { + matches("good").foreach { m => + println(s" 🟢 ${m}") + } + matches("bad").foreach { m => + println(s" 🔴 ${m}") + } + } else { + println(" 🎉 no match (dependency was probably removed)") + } + } + state + } + + def getGitHubRepo: Option[String] = { + val remoteUrl = "git config --get remote.origin.url".!!.trim + val repoPattern = """(?:https://|git@)github\.com[:/](.+/.+)\.git""".r + remoteUrl match { + case repoPattern(repo) => Some(repo) + case _ => None + } + } + + private def analyzeDependencies(state: State, params: AnalysisParams): State = { + val action = params.action + if (Seq(AnalysisAction.Get, AnalysisAction.List).contains(action)) { + params.arg.foreach { pattern => analyzeDependenciesInternal(state, action, pattern) } + state + } else if (action == AnalysisAction.Alerts) { + params.arg.orElse(getGitHubRepo).map { repo => + downloadAlerts(state, repo).get + }.get + } else if (action == AnalysisAction.Cves) { + analyzeCves(state) + } else { + state + } + } + + case class Vulnerability( + packageId: String, + vulnerableVersionRange: String, + firstPatchedVersion: String, + severity: String, + ) + + private def getVulnerabilities(httpResp: FullResponse): Try[Seq[Vulnerability]] = + httpResp.status match { + case status if status / 100 == 2 => Try { + // here is the jq command: + // jq -r '.[]|select((.state == "open"))|.security_vulnerability|"\(.package.name);\(.vulnerable_version_range);\(.first_patched_version.identifier);\(.severity)"' | sort + // do the equivalent in scala and build a seq of Vulnerability, without a converter + val json : JValue = JsonParser.parseFromByteBuffer(httpResp.bodyAsByteBuffer).get + + + + // + // fix line below, because "value asArray is not a member of sjsonnew.shaded.scalajson.ast.unsafe.JValue" + // val vulnerabilities = json.asArray.get.value.map { value => + json.asInstanceOf[JArray].value.map { value => + val obj = value.asInstanceOf[JObject].value + + // convert obj to map of string => JValue : + + val map = obj.map { case JField(k, v) => (k, v) }.toMap + + val securityVulnerability = map("security_vulnerability").asInstanceOf[JObject].value.map { case JField(k, v) => (k, v) }.toMap + + val packageObj = securityVulnerability("package").asInstanceOf[JObject].value.map { case JField(k, v) => (k, v) }.toMap + + val firstPatchedVersion = Try(securityVulnerability("first_patched_version").asInstanceOf[JObject].value.map { case JField(k, v) => (k, v) }.toMap).getOrElse(Map.empty) + + ( + map("state") == JString("open"), + Vulnerability( + packageObj("name").asInstanceOf[JString].value, + securityVulnerability("vulnerable_version_range").asInstanceOf[JString].value, + firstPatchedVersion.get("identifier").map { x => x.asInstanceOf[JString].value }.getOrElse(""), + securityVulnerability("severity").asInstanceOf[JString].value + ) + ) + }.filter(_._1).map(_._2) + } + case status => + val message = + s"Unexpected status $status ${httpResp.statusText} with body:\n${httpResp.bodyAsString}" + throw new MessageOnlyException(message) + } + + + private def githubToken(): String = Properties.envOrElse("GITHUB_TOKEN", "") +} diff --git a/sbt-plugin/src/main/scala/ch/epfl/scala/GithubDependencyGraphPlugin.scala b/sbt-plugin/src/main/scala/ch/epfl/scala/GithubDependencyGraphPlugin.scala index dd607e8..bf26e7f 100644 --- a/sbt-plugin/src/main/scala/ch/epfl/scala/GithubDependencyGraphPlugin.scala +++ b/sbt-plugin/src/main/scala/ch/epfl/scala/GithubDependencyGraphPlugin.scala @@ -32,7 +32,7 @@ object GithubDependencyGraphPlugin extends AutoPlugin { val githubManifestsKey: AttributeKey[Map[String, githubapi.Manifest]] = AttributeKey("githubDependencyManifests") val githubProjectsKey: AttributeKey[Seq[ProjectRef]] = AttributeKey("githubProjectRefs") val githubSnapshotFileKey: AttributeKey[File] = AttributeKey("githubSnapshotFile") - val githubAlertsKey: AttributeKey[Seq[SubmitDependencyGraph.Vulnerability]] = AttributeKey("githubAlerts") + val githubAlertsKey: AttributeKey[Seq[AnalyzeDependencyGraph.Vulnerability]] = AttributeKey("githubAlerts") val githubDependencyManifest: TaskKey[Option[githubapi.Manifest]] = taskKey( "The dependency manifest of the project" @@ -49,7 +49,7 @@ object GithubDependencyGraphPlugin extends AutoPlugin { override def globalSettings: Seq[Setting[_]] = Def.settings( githubStoreDependencyManifests := storeManifestsTask.evaluated, - Keys.commands ++= SubmitDependencyGraph.commands + Keys.commands ++= SubmitDependencyGraph.commands ++ AnalyzeDependencyGraph.commands ) override def projectSettings: Seq[Setting[_]] = Def.settings( diff --git a/sbt-plugin/src/main/scala/ch/epfl/scala/SubmitDependencyGraph.scala b/sbt-plugin/src/main/scala/ch/epfl/scala/SubmitDependencyGraph.scala index 8918aee..e093ce4 100644 --- a/sbt-plugin/src/main/scala/ch/epfl/scala/SubmitDependencyGraph.scala +++ b/sbt-plugin/src/main/scala/ch/epfl/scala/SubmitDependencyGraph.scala @@ -8,6 +8,7 @@ import scala.concurrent.duration.Duration import scala.util.Properties import scala.util.Try +import sjsonnew.shaded.scalajson.ast.unsafe.JValue import ch.epfl.scala.GithubDependencyGraphPlugin.autoImport._ import ch.epfl.scala.JsonProtocol._ import ch.epfl.scala.githubapi.JsonProtocol._ @@ -17,7 +18,6 @@ import gigahorse.HttpClient import gigahorse.support.asynchttpclient.Gigahorse import sbt._ import sbt.internal.util.complete._ -import sjsonnew.shaded.scalajson.ast.unsafe.{ JValue, JArray, JObject, JField, JString } import sjsonnew.support.scalajson.unsafe.{Parser => JsonParser, _} object SubmitDependencyGraph { @@ -25,10 +25,6 @@ object SubmitDependencyGraph { private val GenerateUsage = s"""$Generate {"ignoredModules":[], "ignoredConfig":[]}""" private val GenerateDetail = "Generate the dependency graph of a set of projects and scala versions" - val AnalyzeDependecies = "githubAnalyzeDependencies" - private val AnalyzeDependenciesUsage = s"""$AnalyzeDependecies [get|list] pattern""" - private val AnalyzeDependenciesDetail = "Analyze the dependencies base on a search pattern" - private val GenerateInternal = s"${Generate}Internal" private val InternalOnly = "internal usage only" @@ -37,7 +33,6 @@ object SubmitDependencyGraph { val commands: Seq[Command] = Seq( Command(Generate, (GenerateUsage, GenerateDetail), GenerateDetail)(inputParser)(generate), - Command(AnalyzeDependecies, (AnalyzeDependenciesUsage, AnalyzeDependenciesDetail), AnalyzeDependenciesDetail)(extractPattern)(analyzeDependencies), Command.command(GenerateInternal, InternalOnly, InternalOnly)(generateInternal), Command.command(Submit, SubmitDetail, SubmitDetail)(submit) ) @@ -55,39 +50,6 @@ object SubmitDependencyGraph { .get }.failOnException - sealed trait AnalysisAction { - def name: String - } - object AnalysisAction { - case object Get extends AnalysisAction { - val name = "get" - } - case object List extends AnalysisAction { - val name = "list" - } - case object Alerts extends AnalysisAction { - val name = "alerts" - } - case object Cves extends AnalysisAction { - val name = "cves" - } - val values: Seq[AnalysisAction] = Seq(Get, List, Alerts, Cves) - def fromString(str: String): Option[AnalysisAction] = values.find(_.name == str) - } - - - case class AnalysisParams(action: AnalysisAction, arg: Option[String]) - - private def extractPattern(state: State): Parser[AnalysisParams] = - Parsers.any.*.map { raw => - raw.mkString.trim.split(" ").toSeq match { - case Seq(action, arg) => - AnalysisParams(AnalysisAction.fromString(action).get, Some(arg)) - case Seq(action) => - AnalysisParams(AnalysisAction.fromString(action).get, None) - } - }.failOnException - private def generate(state: State, input: DependencySnapshotInput): State = { val loadedBuild = state.setting(Keys.loadedBuild) @@ -120,193 +82,6 @@ object SubmitDependencyGraph { commands.toList ::: initState } - private def analyzeDependenciesInternal(state: State, action: AnalysisAction, pattern: String) = { - def getDeps(dependencies: Seq[String], pattern: String): Seq[String] = { - for { - dep <- dependencies.filter(_.contains(pattern)) - } yield dep - } - - def resolvedDeps(tabs: String, acc: Seq[String], resolvedByName: Map[String, DependencyNode], pattern: String): Seq[String] = { - - acc ++ (for { - (name, resolved) <- resolvedByName.toSeq - matchingDependencies = getDeps(resolved.dependencies, pattern) - resultDeps <- if (matchingDependencies.isEmpty) { - if (name.contains(pattern)) { - Seq(Seq(tabs + name)) - } else { - Nil - } - } else { - for { - matchingDependency <- matchingDependencies - } yield resolvedDeps(" " + tabs, acc ++ Seq(tabs + matchingDependency), resolvedByName, name) - } - resultDep <- resultDeps - } yield { - resultDep - }) - } - - val matches = (for { - manifests <- state.get(githubManifestsKey).toSeq - (name, manifest) <- manifests - } yield (manifest, resolvedDeps("", Nil, manifest.resolved, pattern))).toMap - - if (action == AnalysisAction.Get) { - matches.foreach { case (manifest, deps) => - println(s"Manifest: ${manifest.name}") - println(deps.map{ dep : String => s" ${dep}" }.mkString("\n")) - } - } - else if (action == AnalysisAction.List) { - println( - matches.flatMap { case (manifest, deps) => deps } - .filter(_.contains(pattern)).toSet.mkString("\n")) - } - } - - private def getGithubToken(ghConfigFile: File): Option[String] = { - println(s"extract token from ${ghConfigFile.getPath()}") - if (ghConfigFile.exists()) { - val lines = IO.readLines(ghConfigFile) - val tokenLine = lines.find(_.contains("oauth_token")) - tokenLine.map { line => line.split(":").last.trim } - } else { - None - } - } - - private def getGithubTokenFromGhConfigDir(): String = { - // use GH_CONFIG_DIR variable if it exists - val ghConfigDir = Properties.envOrElse("GH_CONFIG_DIR", Paths.get(System.getProperty("user.home"), ".config", "gh").toString) - val ghConfigFile = Paths.get(ghConfigDir).resolve("hosts.yml").toFile - getGithubToken(ghConfigFile).getOrElse { - val ghConfigPath = Properties.envOrElse("HUB_CONFIG", Paths.get(System.getProperty("user.home"), ".config", "hub").toString) - val hubConfigFile = Paths.get(ghConfigPath).toFile - getGithubToken(hubConfigFile).getOrElse { - githubToken() - } - } - } - - private def downloadAlerts(state: State, repo: String) : Try[State] = { - val snapshotUrl = s"https://api.github.com/repos/$repo/dependabot/alerts" - val request = Gigahorse - .url(snapshotUrl) - .get - .addHeaders( - "Authorization" -> s"token ${getGithubTokenFromGhConfigDir()}" - ) - state.log.info(s"Downloading alerts from $snapshotUrl") - for { - httpResp <- Try(Await.result(http.processFull(request), Duration.Inf)) - vulnerabilities <- getVulnerabilities(httpResp) - } yield { - vulnerabilities.foreach { v => - println(s"${v.packageId} ${v.vulnerableVersionRange} ${v.firstPatchedVersion} ${v.severity}") - } - state.put(githubAlertsKey, vulnerabilities) - } - } - - private def getAllArtifacts(state: State): Seq[String] = { - { - for { - manifests <- state.get(githubManifestsKey).toSeq - (_, manifest) <- manifests - artifact <- manifest.resolved.values.toSeq - } yield artifact.package_url - }.toSet.toSeq - } - - /* - # example alert - # [ "com.google.guava:guava", ">= 1.0, < 32.0.0-android", "32.0.0-android" ] - # example artifact - # "pkg:maven/com.google.guava/guava@31.1-jre" - */ - - // versionMatchesRange("31.1-jre", ">= 1.0, < 32.0.0-android") => true - // versionMatchesRange("2.8.5", "< 2.9.0") => true - // versionMatchesRange("2.9.0", "< 2.9.0") => false - - private def translateToSemVer(string: String): String = { - // if a version in the string has more than 3 digits, we assume it's a pre-release version - // ">= 1.0 <32.0.0.4" => ">= 1.0 < 32.0.0-4" - string.replaceAll("([0-9]+)\\.([0-9]+)\\.([0-9]+)\\.([0-9]+)", "$1.$2.$3-$4") - } - - private def versionMatchesRange(versionStr: String, rangeStr: String): Boolean = { - val range = rangeStr.replaceAll(" ", "").replace(",", " ") - VersionNumber(translateToSemVer(versionStr)).matchesSemVer(SemanticSelector(translateToSemVer(range))) - } - - private def vulnerabilityMatchesArtifact(alert: Vulnerability, artifact: String): String = { - val alertMavenPath = s"pkg:maven/${alert.packageId.replace(":", "/")}@" - if (artifact.startsWith(alertMavenPath)) { - val version = artifact.split("@").last - // vulnerableVersionRange can be ">= 1.0, < 32.0.0-android" or "< 2.9.0" - val bad = versionMatchesRange(version, alert.vulnerableVersionRange) - if (bad) { - "bad" - } else { - "good" - } - } else { - "no" - } - } - - private def vulnerabilityMatchesArtifacts(alert: Vulnerability, artifacts: Seq[String]): Map[String, Seq[String]] = { - artifacts.foldLeft(Map("good" -> Seq.empty[String], "bad" -> Seq.empty[String])) { (acc, artifact) => - val res = vulnerabilityMatchesArtifact(alert, artifact) - if (res != "no") { - acc.updated(res, acc(res) :+ artifact) - } else { - acc - } - } - } - - private def analyzeCves(state: State): State = { - val vulnerabilities = state.get(githubAlertsKey).get - val cves = vulnerabilities - val artifacts = getAllArtifacts(state) - cves.foreach { v => - val matches = vulnerabilityMatchesArtifacts(v, artifacts) - println(s"${v.packageId} ${v.vulnerableVersionRange} ${v.firstPatchedVersion} ${v.severity}") - if (matches("good").length + matches("bad").length > 0) { - matches("good").foreach { m => - println(s" 🟢 ${m}") - } - matches("bad").foreach { m => - println(s" 🔴 ${m}") - } - } else { - println(" 🎉 no match (dependency was probably removed)") - } - } - state - } - - private def analyzeDependencies(state: State, params: AnalysisParams): State = { - val action = params.action - if (Seq(AnalysisAction.Get, AnalysisAction.List).contains(action)) { - params.arg.foreach { pattern => analyzeDependenciesInternal(state, action, pattern) } - state - } else if (action == AnalysisAction.Alerts) { - params.arg.map { repo => - downloadAlerts(state, repo).get - }.get - } else if (action == AnalysisAction.Cves) { - analyzeCves(state) - } else { - state - } - } - private def generateInternal(state: State): State = { val snapshot = githubDependencySnapshot(state) val snapshotJson = CompactPrinter(Converter.toJsonUnsafe(snapshot)) @@ -359,56 +134,6 @@ object SubmitDependencyGraph { for (output <- githubOutput()) IO.writeLines(output, outputs.map { case (name, value) => s"${name}=${value}" }, append = true) - case class Vulnerability( - packageId: String, - vulnerableVersionRange: String, - firstPatchedVersion: String, - severity: String, - ) - - private def getVulnerabilities(httpResp: FullResponse): Try[Seq[Vulnerability]] = - httpResp.status match { - case status if status / 100 == 2 => Try { - // here is the jq command: - // jq -r '.[]|select((.state == "open"))|.security_vulnerability|"\(.package.name);\(.vulnerable_version_range);\(.first_patched_version.identifier);\(.severity)"' | sort - // do the equivalent in scala and build a seq of Vulnerability, without a converter - val json : JValue = JsonParser.parseFromByteBuffer(httpResp.bodyAsByteBuffer).get - - - - // - // fix line below, because "value asArray is not a member of sjsonnew.shaded.scalajson.ast.unsafe.JValue" - // val vulnerabilities = json.asArray.get.value.map { value => - json.asInstanceOf[JArray].value.map { value => - val obj = value.asInstanceOf[JObject].value - - // convert obj to map of string => JValue : - - val map = obj.map { case JField(k, v) => (k, v) }.toMap - - val securityVulnerability = map("security_vulnerability").asInstanceOf[JObject].value.map { case JField(k, v) => (k, v) }.toMap - - val packageObj = securityVulnerability("package").asInstanceOf[JObject].value.map { case JField(k, v) => (k, v) }.toMap - - val firstPatchedVersion = Try(securityVulnerability("first_patched_version").asInstanceOf[JObject].value.map { case JField(k, v) => (k, v) }.toMap).getOrElse(Map.empty) - - ( - map("state") == JString("open"), - Vulnerability( - packageObj("name").asInstanceOf[JString].value, - securityVulnerability("vulnerable_version_range").asInstanceOf[JString].value, - firstPatchedVersion.get("identifier").map { x => x.asInstanceOf[JString].value }.getOrElse(""), - securityVulnerability("severity").asInstanceOf[JString].value - ) - ) - }.filter(_._1).map(_._2) - } - case status => - val message = - s"Unexpected status $status ${httpResp.statusText} with body:\n${httpResp.bodyAsString}" - throw new MessageOnlyException(message) - } - private def getSnapshot(httpResp: FullResponse): Try[SnapshotResponse] = httpResp.status match { case status if status / 100 == 2 => From a333479b1474efa63215c9678f88eaeb90389075 Mon Sep 17 00:00:00 2001 From: yazgoo Date: Thu, 13 Jun 2024 23:54:35 +0200 Subject: [PATCH 08/28] refacto --- .../src/main/scala/ch/epfl/scala/SubmitDependencyGraph.scala | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/sbt-plugin/src/main/scala/ch/epfl/scala/SubmitDependencyGraph.scala b/sbt-plugin/src/main/scala/ch/epfl/scala/SubmitDependencyGraph.scala index e093ce4..98e0b72 100644 --- a/sbt-plugin/src/main/scala/ch/epfl/scala/SubmitDependencyGraph.scala +++ b/sbt-plugin/src/main/scala/ch/epfl/scala/SubmitDependencyGraph.scala @@ -8,7 +8,6 @@ import scala.concurrent.duration.Duration import scala.util.Properties import scala.util.Try -import sjsonnew.shaded.scalajson.ast.unsafe.JValue import ch.epfl.scala.GithubDependencyGraphPlugin.autoImport._ import ch.epfl.scala.JsonProtocol._ import ch.epfl.scala.githubapi.JsonProtocol._ @@ -18,13 +17,14 @@ import gigahorse.HttpClient import gigahorse.support.asynchttpclient.Gigahorse import sbt._ import sbt.internal.util.complete._ +import sjsonnew.shaded.scalajson.ast.unsafe.JValue import sjsonnew.support.scalajson.unsafe.{Parser => JsonParser, _} object SubmitDependencyGraph { val Generate = "githubGenerateSnapshot" private val GenerateUsage = s"""$Generate {"ignoredModules":[], "ignoredConfig":[]}""" private val GenerateDetail = "Generate the dependency graph of a set of projects and scala versions" - + private val GenerateInternal = s"${Generate}Internal" private val InternalOnly = "internal usage only" @@ -50,7 +50,6 @@ object SubmitDependencyGraph { .get }.failOnException - private def generate(state: State, input: DependencySnapshotInput): State = { val loadedBuild = state.setting(Keys.loadedBuild) // all project refs that have a Scala version From 6e2dd9d24d4b576fae20504918daf89fe89d6aa4 Mon Sep 17 00:00:00 2001 From: yazgoo Date: Thu, 13 Jun 2024 23:55:49 +0200 Subject: [PATCH 09/28] refacto --- .../src/main/scala/ch/epfl/scala/SubmitDependencyGraph.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/sbt-plugin/src/main/scala/ch/epfl/scala/SubmitDependencyGraph.scala b/sbt-plugin/src/main/scala/ch/epfl/scala/SubmitDependencyGraph.scala index 98e0b72..f5c4971 100644 --- a/sbt-plugin/src/main/scala/ch/epfl/scala/SubmitDependencyGraph.scala +++ b/sbt-plugin/src/main/scala/ch/epfl/scala/SubmitDependencyGraph.scala @@ -24,7 +24,6 @@ object SubmitDependencyGraph { val Generate = "githubGenerateSnapshot" private val GenerateUsage = s"""$Generate {"ignoredModules":[], "ignoredConfig":[]}""" private val GenerateDetail = "Generate the dependency graph of a set of projects and scala versions" - private val GenerateInternal = s"${Generate}Internal" private val InternalOnly = "internal usage only" From bca7114fa6c66392580cdfef29572df64d9165b7 Mon Sep 17 00:00:00 2001 From: yazgoo Date: Thu, 13 Jun 2024 23:56:14 +0200 Subject: [PATCH 10/28] refacto --- .../src/main/scala/ch/epfl/scala/SubmitDependencyGraph.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/sbt-plugin/src/main/scala/ch/epfl/scala/SubmitDependencyGraph.scala b/sbt-plugin/src/main/scala/ch/epfl/scala/SubmitDependencyGraph.scala index f5c4971..d4c96b8 100644 --- a/sbt-plugin/src/main/scala/ch/epfl/scala/SubmitDependencyGraph.scala +++ b/sbt-plugin/src/main/scala/ch/epfl/scala/SubmitDependencyGraph.scala @@ -24,6 +24,7 @@ object SubmitDependencyGraph { val Generate = "githubGenerateSnapshot" private val GenerateUsage = s"""$Generate {"ignoredModules":[], "ignoredConfig":[]}""" private val GenerateDetail = "Generate the dependency graph of a set of projects and scala versions" + private val GenerateInternal = s"${Generate}Internal" private val InternalOnly = "internal usage only" From af486f423bf2b6b6f8307bae12ca218b5580e92d Mon Sep 17 00:00:00 2001 From: yazgoo Date: Fri, 14 Jun 2024 00:01:55 +0200 Subject: [PATCH 11/28] cleanup --- .../epfl/scala/AnalyzeDependencyGraph.scala | 26 +++---------------- 1 file changed, 4 insertions(+), 22 deletions(-) diff --git a/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala b/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala index 9289eb7..aabe032 100644 --- a/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala +++ b/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala @@ -25,7 +25,7 @@ import sbt._ object AnalyzeDependencyGraph { val AnalyzeDependecies = "githubAnalyzeDependencies" - private val AnalyzeDependenciesUsage = s"""$AnalyzeDependecies [get|list|alerts|cves] pattern""" + private val AnalyzeDependenciesUsage = s"""$AnalyzeDependecies [${AnalysisAction.values.map(_.name).mkString("|")}] [pattern]""" private val AnalyzeDependenciesDetail = "Analyze the dependencies base on a search pattern" val commands: Seq[Command] = Seq( @@ -168,23 +168,15 @@ object AnalyzeDependencyGraph { }.toSet.toSeq } - /* - # example alert - # [ "com.google.guava:guava", ">= 1.0, < 32.0.0-android", "32.0.0-android" ] - # example artifact - # "pkg:maven/com.google.guava/guava@31.1-jre" - */ - - // versionMatchesRange("31.1-jre", ">= 1.0, < 32.0.0-android") => true - // versionMatchesRange("2.8.5", "< 2.9.0") => true - // versionMatchesRange("2.9.0", "< 2.9.0") => false - private def translateToSemVer(string: String): String = { // if a version in the string has more than 3 digits, we assume it's a pre-release version // ">= 1.0 <32.0.0.4" => ">= 1.0 < 32.0.0-4" string.replaceAll("([0-9]+)\\.([0-9]+)\\.([0-9]+)\\.([0-9]+)", "$1.$2.$3-$4") } + // versionMatchesRange("31.1-jre", ">= 1.0, < 32.0.0-android") => true + // versionMatchesRange("2.8.5", "< 2.9.0") => true + // versionMatchesRange("2.9.0", "< 2.9.0") => false private def versionMatchesRange(versionStr: String, rangeStr: String): Boolean = { val range = rangeStr.replaceAll(" ", "").replace(",", " ") val result = VersionNumber(translateToSemVer(versionStr)).matchesSemVer(SemanticSelector(translateToSemVer(range))) @@ -274,21 +266,11 @@ object AnalyzeDependencyGraph { private def getVulnerabilities(httpResp: FullResponse): Try[Seq[Vulnerability]] = httpResp.status match { case status if status / 100 == 2 => Try { - // here is the jq command: - // jq -r '.[]|select((.state == "open"))|.security_vulnerability|"\(.package.name);\(.vulnerable_version_range);\(.first_patched_version.identifier);\(.severity)"' | sort - // do the equivalent in scala and build a seq of Vulnerability, without a converter val json : JValue = JsonParser.parseFromByteBuffer(httpResp.bodyAsByteBuffer).get - - - // - // fix line below, because "value asArray is not a member of sjsonnew.shaded.scalajson.ast.unsafe.JValue" - // val vulnerabilities = json.asArray.get.value.map { value => json.asInstanceOf[JArray].value.map { value => val obj = value.asInstanceOf[JObject].value - // convert obj to map of string => JValue : - val map = obj.map { case JField(k, v) => (k, v) }.toMap val securityVulnerability = map("security_vulnerability").asInstanceOf[JObject].value.map { case JField(k, v) => (k, v) }.toMap From b2a8236bd7042105be2dcfc05a0611bd36b31985 Mon Sep 17 00:00:00 2001 From: yazgoo Date: Fri, 14 Jun 2024 00:11:54 +0200 Subject: [PATCH 12/28] cleanup --- .../epfl/scala/AnalyzeDependencyGraph.scala | 28 ++++++++++++------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala b/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala index aabe032..754aa3e 100644 --- a/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala +++ b/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala @@ -183,26 +183,34 @@ object AnalyzeDependencyGraph { result } - private def vulnerabilityMatchesArtifact(alert: Vulnerability, artifact: String): String = { + // create an enum with good, bad, no + + sealed trait Vulnerable + + object Good extends Vulnerable + object Bad extends Vulnerable + object No extends Vulnerable + + private def vulnerabilityMatchesArtifact(alert: Vulnerability, artifact: String): Vulnerable = { val alertMavenPath = s"pkg:maven/${alert.packageId.replace(":", "/")}@" if (artifact.startsWith(alertMavenPath)) { val version = artifact.split("@").last // vulnerableVersionRange can be ">= 1.0, < 32.0.0-android" or "< 2.9.0" val bad = versionMatchesRange(version, alert.vulnerableVersionRange) if (bad) { - "bad" + Bad } else { - "good" + Good } } else { - "no" + No } } - private def vulnerabilityMatchesArtifacts(alert: Vulnerability, artifacts: Seq[String]): Map[String, Seq[String]] = { - artifacts.foldLeft(Map("good" -> Seq.empty[String], "bad" -> Seq.empty[String])) { (acc, artifact) => + private def vulnerabilityMatchesArtifacts(alert: Vulnerability, artifacts: Seq[String]): Map[Vulnerable, Seq[String]] = { + artifacts.foldLeft(Map(Good -> Seq.empty[String], Bad -> Seq.empty[String])) { (acc, artifact) => val res = vulnerabilityMatchesArtifact(alert, artifact) - if (res != "no") { + if (res != No) { acc.updated(res, acc(res) :+ artifact) } else { acc @@ -217,11 +225,11 @@ object AnalyzeDependencyGraph { cves.foreach { v => val matches = vulnerabilityMatchesArtifacts(v, artifacts) println(s"${v.packageId} ${v.vulnerableVersionRange} ${v.firstPatchedVersion} ${v.severity}") - if (matches("good").length + matches("bad").length > 0) { - matches("good").foreach { m => + if (matches(Good).length + matches(Bad).length > 0) { + matches(Good).foreach { m => println(s" 🟢 ${m}") } - matches("bad").foreach { m => + matches(Bad).foreach { m => println(s" 🔴 ${m}") } } else { From 1e6bb8dbfccfdc5b2bac531f5c127641f90c2911 Mon Sep 17 00:00:00 2001 From: yazgoo Date: Fri, 14 Jun 2024 08:12:04 +0200 Subject: [PATCH 13/28] cleanup --- .../epfl/scala/AnalyzeDependencyGraph.scala | 345 ++++++++++-------- .../scala/GithubDependencyGraphPlugin.scala | 2 +- 2 files changed, 187 insertions(+), 160 deletions(-) diff --git a/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala b/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala index 754aa3e..fee1871 100644 --- a/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala +++ b/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala @@ -24,94 +24,143 @@ import sbt._ object AnalyzeDependencyGraph { + object Model { + sealed trait AnalysisAction { + def name: String + def help: String + } + object AnalysisAction { + case object Get extends AnalysisAction { + val name = "get" + val help = "search for a pattern in the dependencies (requires githubGenerateSnapshot)" + } + case object List extends AnalysisAction { + val name = "list" + val help = "list all dependencies matching a pattern (requires githubGenerateSnapshot)" + } + case object Alerts extends AnalysisAction { + val name = "alerts" + val help = "download and display CVEs alerts from Github (use hub or gh local config or GIT_TOKEN env var to authenticate)" + } + case object Cves extends AnalysisAction { + val name = "cves" + val help = "analyze CVEs alerts against the dependencies (requires githubGenerateSnapshot and githubAnalyzeDependencies alerts)" + } + val values: Seq[AnalysisAction] = Seq(Get, List, Alerts, Cves) + def fromString(str: String): Option[AnalysisAction] = values.find(_.name == str) + + } + + def blue(str: String): String = s"\u001b[34m${str}\u001b[0m" + + case class Vulnerability( + packageId: String, + vulnerableVersionRange: String, + firstPatchedVersion: String, + severity: String, + ) { + + def severityColor: String = { + severity match { + case "critical" => "\u001b[31m" + case "high" => "\u001b[31m" + case "medium" => "\u001b[33m" + case "low" => "\u001b[32m" + case _ => "\u001b[0m" + } + } + + def coloredSeverity: String = { + s"${severityColor}${severity}\u001b[0m" + } + + override def toString: String = { + s"${blue(packageId)} [ ${vulnerableVersionRange} ] fixed: ${firstPatchedVersion} ${coloredSeverity}" + } + } + + case class AnalysisParams(action: AnalysisAction, arg: Option[String]) + + sealed trait Vulnerable + + object Good extends Vulnerable + object Bad extends Vulnerable + object No extends Vulnerable + } + + import Model._ + val AnalyzeDependecies = "githubAnalyzeDependencies" private val AnalyzeDependenciesUsage = s"""$AnalyzeDependecies [${AnalysisAction.values.map(_.name).mkString("|")}] [pattern]""" - private val AnalyzeDependenciesDetail = "Analyze the dependencies base on a search pattern" + private val AnalyzeDependenciesDetail = s"""Analyze the dependencies base on a search pattern: + ${AnalysisAction.values.map(a => s"${a.name}: ${a.help}").mkString("\n ")} + """ val commands: Seq[Command] = Seq( Command(AnalyzeDependecies, (AnalyzeDependenciesUsage, AnalyzeDependenciesDetail), AnalyzeDependenciesDetail)(extractPattern)(analyzeDependencies), - ) + ) private lazy val http: HttpClient = Gigahorse.http(Gigahorse.config) - sealed trait AnalysisAction { - def name: String - } - object AnalysisAction { - case object Get extends AnalysisAction { - val name = "get" - } - case object List extends AnalysisAction { - val name = "list" - } - case object Alerts extends AnalysisAction { - val name = "alerts" - } - case object Cves extends AnalysisAction { - val name = "cves" - } - val values: Seq[AnalysisAction] = Seq(Get, List, Alerts, Cves) - def fromString(str: String): Option[AnalysisAction] = values.find(_.name == str) - } - - - case class AnalysisParams(action: AnalysisAction, arg: Option[String]) - private def extractPattern(state: State): Parser[AnalysisParams] = Parsers.any.*.map { raw => raw.mkString.trim.split(" ").toSeq match { - case Seq(action, arg) => - AnalysisParams(AnalysisAction.fromString(action).get, Some(arg)) - case Seq(action) => - AnalysisParams(AnalysisAction.fromString(action).get, None) + case Seq(action, arg) => + AnalysisParams(AnalysisAction.fromString(action).get, Some(arg)) + case Seq(action) => + AnalysisParams(AnalysisAction.fromString(action).get, None) } }.failOnException - private def analyzeDependenciesInternal(state: State, action: AnalysisAction, pattern: String) = { - def getDeps(dependencies: Seq[String], pattern: String): Seq[String] = { - for { - dep <- dependencies.filter(_.contains(pattern)) - } yield dep - } + private def highlight(string: String, pattern: String): String = { + string.replaceAll(pattern, s"\u001b[32m${pattern}\u001b[0m") + } + + private def analyzeDependenciesInternal(state: State, action: AnalysisAction, pattern: String, originalPattern: String) = { + def getDeps(dependencies: Seq[String], pattern: String): Seq[String] = { + for { + dep <- dependencies.filter(_.contains(pattern)) + } yield highlight(dep, originalPattern) + } - def resolvedDeps(tabs: String, acc: Seq[String], resolvedByName: Map[String, DependencyNode], pattern: String): Seq[String] = { - - acc ++ (for { - (name, resolved) <- resolvedByName.toSeq - matchingDependencies = getDeps(resolved.dependencies, pattern) - resultDeps <- if (matchingDependencies.isEmpty) { - if (name.contains(pattern)) { - Seq(Seq(tabs + name)) - } else { - Nil - } + def resolvedDeps(tabs: String, acc: Seq[String], resolvedByName: Map[String, DependencyNode], pattern: String, originalPattern: String): Seq[String] = { + + acc ++ (for { + (name, resolved) <- resolvedByName.toSeq + matchingDependencies = getDeps(resolved.dependencies, pattern) + resultDeps <- if (matchingDependencies.isEmpty) { + if (name.contains(pattern)) { + Seq(Seq(tabs + highlight(name, originalPattern))) + } else { + Nil + } } else { for { matchingDependency <- matchingDependencies - } yield resolvedDeps(" " + tabs, acc ++ Seq(tabs + matchingDependency), resolvedByName, name) + } yield resolvedDeps(" " + tabs, acc ++ Seq(tabs + matchingDependency), resolvedByName, name, originalPattern) } resultDep <- resultDeps - } yield { - resultDep - }) - } + } yield { + resultDep + }) + } - val matches = (for { - manifests <- state.get(githubManifestsKey).toSeq - (name, manifest) <- manifests - } yield (manifest, resolvedDeps("", Nil, manifest.resolved, pattern))).toMap + val matches = (for { + manifests <- state.get(githubManifestsKey).toSeq + (name, manifest) <- manifests + } yield (manifest, resolvedDeps("", Nil, manifest.resolved, pattern, originalPattern = pattern))).toMap - if (action == AnalysisAction.Get) { - matches.foreach { case (manifest, deps) => - println(s"Manifest: ${manifest.name}") - println(deps.map{ dep : String => s" ${dep}" }.mkString("\n")) - } + if (action == AnalysisAction.Get) { + matches.foreach { case (manifest, deps) => + println(s"📁 ${blue(manifest.name)}") + println(deps.map{ dep : String => s" ${dep}" }.mkString("\n")) } - else if (action == AnalysisAction.List) { - println( + } + else if (action == AnalysisAction.List) { + println( matches.flatMap { case (manifest, deps) => deps } .filter(_.contains(pattern)).toSet.mkString("\n")) - } + } } private def getGithubToken(ghConfigFile: File): Option[String] = { @@ -139,23 +188,23 @@ object AnalyzeDependencyGraph { } private def downloadAlerts(state: State, repo: String) : Try[State] = { - val snapshotUrl = s"https://api.github.com/repos/$repo/dependabot/alerts" - val request = Gigahorse - .url(snapshotUrl) - .get - .addHeaders( - "Authorization" -> s"token ${getGithubTokenFromGhConfigDir()}" - ) + val snapshotUrl = s"https://api.github.com/repos/$repo/dependabot/alerts" + val request = Gigahorse + .url(snapshotUrl) + .get + .addHeaders( + "Authorization" -> s"token ${getGithubTokenFromGhConfigDir()}" + ) state.log.info(s"Downloading alerts from $snapshotUrl") for { httpResp <- Try(Await.result(http.processFull(request), Duration.Inf)) vulnerabilities <- getVulnerabilities(httpResp) - } yield { - vulnerabilities.foreach { v => - println(s"${v.packageId} ${v.vulnerableVersionRange} ${v.firstPatchedVersion} ${v.severity}") + } yield { + vulnerabilities.foreach { v => + println(v.toString) + } + state.put(githubAlertsKey, vulnerabilities) } - state.put(githubAlertsKey, vulnerabilities) - } } private def getAllArtifacts(state: State): Seq[String] = { @@ -168,55 +217,47 @@ object AnalyzeDependencyGraph { }.toSet.toSeq } - private def translateToSemVer(string: String): String = { - // if a version in the string has more than 3 digits, we assume it's a pre-release version - // ">= 1.0 <32.0.0.4" => ">= 1.0 < 32.0.0-4" - string.replaceAll("([0-9]+)\\.([0-9]+)\\.([0-9]+)\\.([0-9]+)", "$1.$2.$3-$4") - } - - // versionMatchesRange("31.1-jre", ">= 1.0, < 32.0.0-android") => true - // versionMatchesRange("2.8.5", "< 2.9.0") => true - // versionMatchesRange("2.9.0", "< 2.9.0") => false - private def versionMatchesRange(versionStr: String, rangeStr: String): Boolean = { - val range = rangeStr.replaceAll(" ", "").replace(",", " ") - val result = VersionNumber(translateToSemVer(versionStr)).matchesSemVer(SemanticSelector(translateToSemVer(range))) - result - } - - // create an enum with good, bad, no - - sealed trait Vulnerable - - object Good extends Vulnerable - object Bad extends Vulnerable - object No extends Vulnerable - - private def vulnerabilityMatchesArtifact(alert: Vulnerability, artifact: String): Vulnerable = { - val alertMavenPath = s"pkg:maven/${alert.packageId.replace(":", "/")}@" - if (artifact.startsWith(alertMavenPath)) { - val version = artifact.split("@").last - // vulnerableVersionRange can be ">= 1.0, < 32.0.0-android" or "< 2.9.0" - val bad = versionMatchesRange(version, alert.vulnerableVersionRange) - if (bad) { - Bad - } else { - Good - } - } else { - No - } - } - - private def vulnerabilityMatchesArtifacts(alert: Vulnerability, artifacts: Seq[String]): Map[Vulnerable, Seq[String]] = { - artifacts.foldLeft(Map(Good -> Seq.empty[String], Bad -> Seq.empty[String])) { (acc, artifact) => - val res = vulnerabilityMatchesArtifact(alert, artifact) - if (res != No) { - acc.updated(res, acc(res) :+ artifact) - } else { - acc - } - } - } + private def translateToSemVer(string: String): String = { + // if a version in the string has more than 3 digits, we assume it's a pre-release version + // ">= 1.0 <32.0.0.4" => ">= 1.0 < 32.0.0-4" + string.replaceAll("([0-9]+)\\.([0-9]+)\\.([0-9]+)\\.([0-9]+)", "$1.$2.$3-$4") + } + + // versionMatchesRange("31.1-jre", ">= 1.0, < 32.0.0-android") => true + // versionMatchesRange("2.8.5", "< 2.9.0") => true + // versionMatchesRange("2.9.0", "< 2.9.0") => false + private def versionMatchesRange(versionStr: String, rangeStr: String): Boolean = { + val range = rangeStr.replaceAll(" ", "").replace(",", " ") + val result = VersionNumber(translateToSemVer(versionStr)).matchesSemVer(SemanticSelector(translateToSemVer(range))) + result + } + + private def vulnerabilityMatchesArtifact(alert: Vulnerability, artifact: String): Vulnerable = { + val alertMavenPath = s"pkg:maven/${alert.packageId.replace(":", "/")}@" + if (artifact.startsWith(alertMavenPath)) { + val version = artifact.split("@").last + // vulnerableVersionRange can be ">= 1.0, < 32.0.0-android" or "< 2.9.0" + val bad = versionMatchesRange(version, alert.vulnerableVersionRange) + if (bad) { + Bad + } else { + Good + } + } else { + No + } + } + + private def vulnerabilityMatchesArtifacts(alert: Vulnerability, artifacts: Seq[String]): Map[Vulnerable, Seq[String]] = { + artifacts.foldLeft(Map(Good -> Seq.empty[String], Bad -> Seq.empty[String])) { (acc, artifact) => + val res = vulnerabilityMatchesArtifact(alert, artifact) + if (res != No) { + acc.updated(res, acc(res) :+ artifact) + } else { + acc + } + } + } private def analyzeCves(state: State): State = { val vulnerabilities = state.get(githubAlertsKey).get @@ -224,34 +265,34 @@ object AnalyzeDependencyGraph { val artifacts = getAllArtifacts(state) cves.foreach { v => val matches = vulnerabilityMatchesArtifacts(v, artifacts) - println(s"${v.packageId} ${v.vulnerableVersionRange} ${v.firstPatchedVersion} ${v.severity}") + println(v.toString) if (matches(Good).length + matches(Bad).length > 0) { matches(Good).foreach { m => - println(s" 🟢 ${m}") + println(s" 🟢 ${m.replaceAll(".*@", "")}") } matches(Bad).foreach { m => - println(s" 🔴 ${m}") + println(s" 🔴 ${m.replaceAll(".*@", "")}") + } + } else { + println(" 🎉 no match (dependency was probably removed)") } - } else { - println(" 🎉 no match (dependency was probably removed)") - } } state } - def getGitHubRepo: Option[String] = { - val remoteUrl = "git config --get remote.origin.url".!!.trim - val repoPattern = """(?:https://|git@)github\.com[:/](.+/.+)\.git""".r - remoteUrl match { - case repoPattern(repo) => Some(repo) - case _ => None - } - } + def getGitHubRepo: Option[String] = { + val remoteUrl = "git config --get remote.origin.url".!!.trim + val repoPattern = """(?:https://|git@)github\.com[:/](.+/.+)\.git""".r + remoteUrl match { + case repoPattern(repo) => Some(repo) + case _ => None + } + } private def analyzeDependencies(state: State, params: AnalysisParams): State = { val action = params.action if (Seq(AnalysisAction.Get, AnalysisAction.List).contains(action)) { - params.arg.foreach { pattern => analyzeDependenciesInternal(state, action, pattern) } + params.arg.foreach { pattern => analyzeDependenciesInternal(state, action, pattern, pattern) } state } else if (action == AnalysisAction.Alerts) { params.arg.orElse(getGitHubRepo).map { repo => @@ -263,47 +304,33 @@ object AnalyzeDependencyGraph { state } } - - case class Vulnerability( - packageId: String, - vulnerableVersionRange: String, - firstPatchedVersion: String, - severity: String, - ) - private def getVulnerabilities(httpResp: FullResponse): Try[Seq[Vulnerability]] = httpResp.status match { case status if status / 100 == 2 => Try { val json : JValue = JsonParser.parseFromByteBuffer(httpResp.bodyAsByteBuffer).get - json.asInstanceOf[JArray].value.map { value => val obj = value.asInstanceOf[JObject].value - val map = obj.map { case JField(k, v) => (k, v) }.toMap - val securityVulnerability = map("security_vulnerability").asInstanceOf[JObject].value.map { case JField(k, v) => (k, v) }.toMap - val packageObj = securityVulnerability("package").asInstanceOf[JObject].value.map { case JField(k, v) => (k, v) }.toMap - val firstPatchedVersion = Try(securityVulnerability("first_patched_version").asInstanceOf[JObject].value.map { case JField(k, v) => (k, v) }.toMap).getOrElse(Map.empty) ( map("state") == JString("open"), Vulnerability( - packageObj("name").asInstanceOf[JString].value, - securityVulnerability("vulnerable_version_range").asInstanceOf[JString].value, - firstPatchedVersion.get("identifier").map { x => x.asInstanceOf[JString].value }.getOrElse(""), - securityVulnerability("severity").asInstanceOf[JString].value + packageObj("name").asInstanceOf[JString].value, + securityVulnerability("vulnerable_version_range").asInstanceOf[JString].value, + firstPatchedVersion.get("identifier").map { x => x.asInstanceOf[JString].value }.getOrElse(""), + securityVulnerability("severity").asInstanceOf[JString].value + ) ) - ) }.filter(_._1).map(_._2) - } + } case status => val message = s"Unexpected status $status ${httpResp.statusText} with body:\n${httpResp.bodyAsString}" throw new MessageOnlyException(message) } - private def githubToken(): String = Properties.envOrElse("GITHUB_TOKEN", "") } diff --git a/sbt-plugin/src/main/scala/ch/epfl/scala/GithubDependencyGraphPlugin.scala b/sbt-plugin/src/main/scala/ch/epfl/scala/GithubDependencyGraphPlugin.scala index bf26e7f..448a48f 100644 --- a/sbt-plugin/src/main/scala/ch/epfl/scala/GithubDependencyGraphPlugin.scala +++ b/sbt-plugin/src/main/scala/ch/epfl/scala/GithubDependencyGraphPlugin.scala @@ -32,7 +32,7 @@ object GithubDependencyGraphPlugin extends AutoPlugin { val githubManifestsKey: AttributeKey[Map[String, githubapi.Manifest]] = AttributeKey("githubDependencyManifests") val githubProjectsKey: AttributeKey[Seq[ProjectRef]] = AttributeKey("githubProjectRefs") val githubSnapshotFileKey: AttributeKey[File] = AttributeKey("githubSnapshotFile") - val githubAlertsKey: AttributeKey[Seq[AnalyzeDependencyGraph.Vulnerability]] = AttributeKey("githubAlerts") + val githubAlertsKey: AttributeKey[Seq[AnalyzeDependencyGraph.Model.Vulnerability]] = AttributeKey("githubAlerts") val githubDependencyManifest: TaskKey[Option[githubapi.Manifest]] = taskKey( "The dependency manifest of the project" From edeffaf960d1d27b60d009ec21f218c96bfa2804 Mon Sep 17 00:00:00 2001 From: yazgoo Date: Fri, 14 Jun 2024 08:17:57 +0200 Subject: [PATCH 14/28] refacto --- .../epfl/scala/AnalyzeDependencyGraph.scala | 310 +++++++----------- 1 file changed, 119 insertions(+), 191 deletions(-) diff --git a/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala b/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala index fee1871..ceddda5 100644 --- a/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala +++ b/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala @@ -4,24 +4,18 @@ import java.nio.file.Paths import scala.concurrent.Await import scala.concurrent.duration.Duration -import scala.util.Properties -import scala.util.Try - +import scala.util.{Properties, Try} import ch.epfl.scala.GithubDependencyGraphPlugin.autoImport._ import ch.epfl.scala.githubapi._ import gigahorse.support.asynchttpclient.Gigahorse import sbt._ import sbt.internal.util.complete._ -import sjsonnew.shaded.scalajson.ast.unsafe.{ JValue, JArray, JObject, JField, JString } -import gigahorse.FullResponse -import gigahorse.HttpClient -import gigahorse.support.asynchttpclient.Gigahorse +import sjsonnew.shaded.scalajson.ast.unsafe.{JArray, JObject, JField, JString} +import gigahorse.{FullResponse, HttpClient} import sjsonnew.support.scalajson.unsafe.{Parser => JsonParser} import scala.sys.process._ -import sbt._ - object AnalyzeDependencyGraph { object Model { @@ -29,6 +23,7 @@ object AnalyzeDependencyGraph { def name: String def help: String } + object AnalysisAction { case object Get extends AnalysisAction { val name = "get" @@ -46,9 +41,10 @@ object AnalyzeDependencyGraph { val name = "cves" val help = "analyze CVEs alerts against the dependencies (requires githubGenerateSnapshot and githubAnalyzeDependencies alerts)" } + val values: Seq[AnalysisAction] = Seq(Get, List, Alerts, Cves) - def fromString(str: String): Option[AnalysisAction] = values.find(_.name == str) + def fromString(str: String): Option[AnalysisAction] = values.find(_.name == str) } def blue(str: String): String = s"\u001b[34m${str}\u001b[0m" @@ -57,32 +53,24 @@ object AnalyzeDependencyGraph { packageId: String, vulnerableVersionRange: String, firstPatchedVersion: String, - severity: String, - ) { - - def severityColor: String = { - severity match { - case "critical" => "\u001b[31m" - case "high" => "\u001b[31m" - case "medium" => "\u001b[33m" - case "low" => "\u001b[32m" - case _ => "\u001b[0m" - } - } + severity: String + ) { + def severityColor: String = severity match { + case "critical" => "\u001b[31m" + case "high" => "\u001b[31m" + case "medium" => "\u001b[33m" + case "low" => "\u001b[32m" + case _ => "\u001b[0m" + } - def coloredSeverity: String = { - s"${severityColor}${severity}\u001b[0m" - } + def coloredSeverity: String = s"${severityColor}${severity}\u001b[0m" - override def toString: String = { - s"${blue(packageId)} [ ${vulnerableVersionRange} ] fixed: ${firstPatchedVersion} ${coloredSeverity}" - } - } + override def toString: String = s"${blue(packageId)} [ $vulnerableVersionRange ] fixed: $firstPatchedVersion $coloredSeverity" + } case class AnalysisParams(action: AnalysisAction, arg: Option[String]) sealed trait Vulnerable - object Good extends Vulnerable object Bad extends Vulnerable object No extends Vulnerable @@ -90,192 +78,137 @@ object AnalyzeDependencyGraph { import Model._ - val AnalyzeDependecies = "githubAnalyzeDependencies" - private val AnalyzeDependenciesUsage = s"""$AnalyzeDependecies [${AnalysisAction.values.map(_.name).mkString("|")}] [pattern]""" - private val AnalyzeDependenciesDetail = s"""Analyze the dependencies base on a search pattern: + val AnalyzeDependencies = "githubAnalyzeDependencies" + private val AnalyzeDependenciesUsage = s"""$AnalyzeDependencies [${AnalysisAction.values.map(_.name).mkString("|")}] [pattern]""" + private val AnalyzeDependenciesDetail = s"""Analyze the dependencies based on a search pattern: ${AnalysisAction.values.map(a => s"${a.name}: ${a.help}").mkString("\n ")} """ val commands: Seq[Command] = Seq( - Command(AnalyzeDependecies, (AnalyzeDependenciesUsage, AnalyzeDependenciesDetail), AnalyzeDependenciesDetail)(extractPattern)(analyzeDependencies), - ) + Command(AnalyzeDependencies, (AnalyzeDependenciesUsage, AnalyzeDependenciesDetail), AnalyzeDependenciesDetail)(extractPattern)(analyzeDependencies) + ) private lazy val http: HttpClient = Gigahorse.http(Gigahorse.config) private def extractPattern(state: State): Parser[AnalysisParams] = Parsers.any.*.map { raw => raw.mkString.trim.split(" ").toSeq match { - case Seq(action, arg) => - AnalysisParams(AnalysisAction.fromString(action).get, Some(arg)) - case Seq(action) => - AnalysisParams(AnalysisAction.fromString(action).get, None) + case Seq(action, arg) => AnalysisParams(AnalysisAction.fromString(action).get, Some(arg)) + case Seq(action) => AnalysisParams(AnalysisAction.fromString(action).get, None) } }.failOnException - private def highlight(string: String, pattern: String): String = { + private def highlight(string: String, pattern: String): String = string.replaceAll(pattern, s"\u001b[32m${pattern}\u001b[0m") - } - private def analyzeDependenciesInternal(state: State, action: AnalysisAction, pattern: String, originalPattern: String) = { - def getDeps(dependencies: Seq[String], pattern: String): Seq[String] = { - for { - dep <- dependencies.filter(_.contains(pattern)) - } yield highlight(dep, originalPattern) - } + private def analyzeDependenciesInternal(state: State, action: AnalysisAction, pattern: String, originalPattern: String): Unit = { + def getDeps(dependencies: Seq[String], pattern: String): Seq[String] = + dependencies.filter(_.contains(pattern)).map(highlight(_, originalPattern)) def resolvedDeps(tabs: String, acc: Seq[String], resolvedByName: Map[String, DependencyNode], pattern: String, originalPattern: String): Seq[String] = { - - acc ++ (for { - (name, resolved) <- resolvedByName.toSeq - matchingDependencies = getDeps(resolved.dependencies, pattern) - resultDeps <- if (matchingDependencies.isEmpty) { - if (name.contains(pattern)) { - Seq(Seq(tabs + highlight(name, originalPattern))) - } else { - Nil - } - } else { - for { - matchingDependency <- matchingDependencies - } yield resolvedDeps(" " + tabs, acc ++ Seq(tabs + matchingDependency), resolvedByName, name, originalPattern) + acc ++ resolvedByName.toSeq.flatMap { case (name, resolved) => + val matchingDependencies = getDeps(resolved.dependencies, pattern) + if (matchingDependencies.isEmpty) { + if (name.contains(pattern)) Seq(tabs + highlight(name, originalPattern)) else Nil + } else { + matchingDependencies.flatMap { matchingDependency => + resolvedDeps(" " + tabs, acc :+ (tabs + matchingDependency), resolvedByName, name, originalPattern) } - resultDep <- resultDeps - } yield { - resultDep - }) + } + } } - val matches = (for { - manifests <- state.get(githubManifestsKey).toSeq - (name, manifest) <- manifests - } yield (manifest, resolvedDeps("", Nil, manifest.resolved, pattern, originalPattern = pattern))).toMap - - if (action == AnalysisAction.Get) { - matches.foreach { case (manifest, deps) => - println(s"📁 ${blue(manifest.name)}") - println(deps.map{ dep : String => s" ${dep}" }.mkString("\n")) + val matches = state.get(githubManifestsKey).toSeq.flatMap { manifests => + manifests.map { case (name, manifest) => + manifest -> resolvedDeps("", Nil, manifest.resolved, pattern, originalPattern = pattern) } - } - else if (action == AnalysisAction.List) { - println( - matches.flatMap { case (manifest, deps) => deps } - .filter(_.contains(pattern)).toSet.mkString("\n")) + }.toMap + + action match { + case AnalysisAction.Get => + matches.foreach { case (manifest, deps) => + println(s"📁 ${blue(manifest.name)}") + println(deps.map(dep => s" $dep").mkString("\n")) + } + case AnalysisAction.List => + println(matches.values.flatten.filter(_.contains(pattern)).toSet.mkString("\n")) + case _ => } } private def getGithubToken(ghConfigFile: File): Option[String] = { - println(s"extract token from ${ghConfigFile.getPath()}") + println(s"Extract token from ${ghConfigFile.getPath}") if (ghConfigFile.exists()) { - val lines = IO.readLines(ghConfigFile) - val tokenLine = lines.find(_.contains("oauth_token")) - tokenLine.map { line => line.split(":").last.trim } - } else { - None - } + IO.readLines(ghConfigFile).find(_.contains("oauth_token")).map(_.split(":").last.trim) + } else None } private def getGithubTokenFromGhConfigDir(): String = { - // use GH_CONFIG_DIR variable if it exists val ghConfigDir = Properties.envOrElse("GH_CONFIG_DIR", Paths.get(System.getProperty("user.home"), ".config", "gh").toString) val ghConfigFile = Paths.get(ghConfigDir).resolve("hosts.yml").toFile getGithubToken(ghConfigFile).getOrElse { val ghConfigPath = Properties.envOrElse("HUB_CONFIG", Paths.get(System.getProperty("user.home"), ".config", "hub").toString) val hubConfigFile = Paths.get(ghConfigPath).toFile - getGithubToken(hubConfigFile).getOrElse { - githubToken() - } + getGithubToken(hubConfigFile).getOrElse(githubToken()) } } - private def downloadAlerts(state: State, repo: String) : Try[State] = { + private def downloadAlerts(state: State, repo: String): Try[State] = { val snapshotUrl = s"https://api.github.com/repos/$repo/dependabot/alerts" - val request = Gigahorse - .url(snapshotUrl) - .get - .addHeaders( - "Authorization" -> s"token ${getGithubTokenFromGhConfigDir()}" - ) - state.log.info(s"Downloading alerts from $snapshotUrl") - for { - httpResp <- Try(Await.result(http.processFull(request), Duration.Inf)) - vulnerabilities <- getVulnerabilities(httpResp) - } yield { - vulnerabilities.foreach { v => - println(v.toString) - } - state.put(githubAlertsKey, vulnerabilities) - } + val request = Gigahorse.url(snapshotUrl).get.addHeaders("Authorization" -> s"token ${getGithubTokenFromGhConfigDir()}") + state.log.info(s"Downloading alerts from $snapshotUrl") + for { + httpResp <- Try(Await.result(http.processFull(request), Duration.Inf)) + vulnerabilities <- getVulnerabilities(httpResp) + } yield { + vulnerabilities.foreach(v => println(v.toString)) + state.put(githubAlertsKey, vulnerabilities) + } } private def getAllArtifacts(state: State): Seq[String] = { - { - for { - manifests <- state.get(githubManifestsKey).toSeq - (_, manifest) <- manifests - artifact <- manifest.resolved.values.toSeq - } yield artifact.package_url - }.toSet.toSeq + state.get(githubManifestsKey).toSeq.flatMap { manifests => + manifests.flatMap { case (_, manifest) => + manifest.resolved.values.toSeq.map(_.package_url) + } + }.distinct } - private def translateToSemVer(string: String): String = { - // if a version in the string has more than 3 digits, we assume it's a pre-release version - // ">= 1.0 <32.0.0.4" => ">= 1.0 < 32.0.0-4" + private def translateToSemVer(string: String): String = string.replaceAll("([0-9]+)\\.([0-9]+)\\.([0-9]+)\\.([0-9]+)", "$1.$2.$3-$4") - } - // versionMatchesRange("31.1-jre", ">= 1.0, < 32.0.0-android") => true - // versionMatchesRange("2.8.5", "< 2.9.0") => true - // versionMatchesRange("2.9.0", "< 2.9.0") => false private def versionMatchesRange(versionStr: String, rangeStr: String): Boolean = { val range = rangeStr.replaceAll(" ", "").replace(",", " ") - val result = VersionNumber(translateToSemVer(versionStr)).matchesSemVer(SemanticSelector(translateToSemVer(range))) - result + VersionNumber(translateToSemVer(versionStr)).matchesSemVer(SemanticSelector(translateToSemVer(range))) } private def vulnerabilityMatchesArtifact(alert: Vulnerability, artifact: String): Vulnerable = { val alertMavenPath = s"pkg:maven/${alert.packageId.replace(":", "/")}@" if (artifact.startsWith(alertMavenPath)) { - val version = artifact.split("@").last - // vulnerableVersionRange can be ">= 1.0, < 32.0.0-android" or "< 2.9.0" - val bad = versionMatchesRange(version, alert.vulnerableVersionRange) - if (bad) { - Bad - } else { - Good - } - } else { - No - } + val version = artifact.replaceAll(".*@", "") + if (versionMatchesRange(version, alert.vulnerableVersionRange)) Bad else Good + } else No } private def vulnerabilityMatchesArtifacts(alert: Vulnerability, artifacts: Seq[String]): Map[Vulnerable, Seq[String]] = { artifacts.foldLeft(Map(Good -> Seq.empty[String], Bad -> Seq.empty[String])) { (acc, artifact) => val res = vulnerabilityMatchesArtifact(alert, artifact) - if (res != No) { - acc.updated(res, acc(res) :+ artifact) - } else { - acc - } + if (res != No) acc.updated(res, acc(res) :+ artifact) else acc } } private def analyzeCves(state: State): State = { - val vulnerabilities = state.get(githubAlertsKey).get - val cves = vulnerabilities + val vulnerabilities = state.get(githubAlertsKey).getOrElse(Seq.empty) val artifacts = getAllArtifacts(state) - cves.foreach { v => + vulnerabilities.foreach { v => val matches = vulnerabilityMatchesArtifacts(v, artifacts) println(v.toString) - if (matches(Good).length + matches(Bad).length > 0) { - matches(Good).foreach { m => - println(s" 🟢 ${m.replaceAll(".*@", "")}") - } - matches(Bad).foreach { m => - println(s" 🔴 ${m.replaceAll(".*@", "")}") - } - } else { - println(" 🎉 no match (dependency was probably removed)") - } + if (matches(Good).nonEmpty || matches(Bad).nonEmpty) { + matches(Good).foreach(m => println(s" 🟢 ${m.replaceAll(".*@", "")}")) + matches(Bad).foreach(m => println(s" 🔴 ${m.replaceAll(".*@", "")}")) + } else { + println(" 🎉 no match (dependency was probably removed)") + } } state } @@ -283,54 +216,49 @@ object AnalyzeDependencyGraph { def getGitHubRepo: Option[String] = { val remoteUrl = "git config --get remote.origin.url".!!.trim val repoPattern = """(?:https://|git@)github\.com[:/](.+/.+)\.git""".r - remoteUrl match { - case repoPattern(repo) => Some(repo) - case _ => None - } + remoteUrl match { + case repoPattern(repo) => Some(repo) + case _ => None + } } private def analyzeDependencies(state: State, params: AnalysisParams): State = { - val action = params.action - if (Seq(AnalysisAction.Get, AnalysisAction.List).contains(action)) { - params.arg.foreach { pattern => analyzeDependenciesInternal(state, action, pattern, pattern) } - state - } else if (action == AnalysisAction.Alerts) { - params.arg.orElse(getGitHubRepo).map { repo => - downloadAlerts(state, repo).get - }.get - } else if (action == AnalysisAction.Cves) { - analyzeCves(state) - } else { - state + params.action match { + case AnalysisAction.Get | AnalysisAction.List => + params.arg.foreach(pattern => analyzeDependenciesInternal(state, params.action, pattern, pattern)) + state + case AnalysisAction.Alerts => + params.arg.orElse(getGitHubRepo).map(repo => downloadAlerts(state, repo).get).getOrElse(state) + case AnalysisAction.Cves => + analyzeCves(state) + case _ => + state } } - private def getVulnerabilities(httpResp: FullResponse): Try[Seq[Vulnerability]] = + + private def getVulnerabilities(httpResp: FullResponse): Try[Seq[Vulnerability]] = Try { httpResp.status match { - case status if status / 100 == 2 => Try { - val json : JValue = JsonParser.parseFromByteBuffer(httpResp.bodyAsByteBuffer).get - json.asInstanceOf[JArray].value.map { value => - val obj = value.asInstanceOf[JObject].value - val map = obj.map { case JField(k, v) => (k, v) }.toMap - val securityVulnerability = map("security_vulnerability").asInstanceOf[JObject].value.map { case JField(k, v) => (k, v) }.toMap - val packageObj = securityVulnerability("package").asInstanceOf[JObject].value.map { case JField(k, v) => (k, v) }.toMap - val firstPatchedVersion = Try(securityVulnerability("first_patched_version").asInstanceOf[JObject].value.map { case JField(k, v) => (k, v) }.toMap).getOrElse(Map.empty) - - ( - map("state") == JString("open"), + case status if status / 100 == 2 => + val json: JArray = JsonParser.parseFromByteBuffer(httpResp.bodyAsByteBuffer).get.asInstanceOf[JArray] + json.value.collect { + case obj: JObject if (obj.value.collectFirst { case JField("state", JString("open")) => true }.isDefined) => + val securityVulnerability = obj.value.collectFirst { case JField("security_vulnerability", secVuln: JObject) => secVuln }.get.value + val packageObj = securityVulnerability.collectFirst { case JField("package", pkg: JObject) => pkg }.get.value + val firstPatchedVersion = securityVulnerability.collectFirst { + case JField("first_patched_version", firstPatched: JObject) => firstPatched + }.map(_.value.collectFirst { case JField("identifier", JString(ident)) => ident }.getOrElse("")).getOrElse("") Vulnerability( - packageObj("name").asInstanceOf[JString].value, - securityVulnerability("vulnerable_version_range").asInstanceOf[JString].value, - firstPatchedVersion.get("identifier").map { x => x.asInstanceOf[JString].value }.getOrElse(""), - securityVulnerability("severity").asInstanceOf[JString].value + packageObj.collectFirst { case JField("name", JString(name)) => name }.get, + securityVulnerability.collectFirst { case JField("vulnerable_version_range", JString(range)) => range }.get, + firstPatchedVersion, + securityVulnerability.collectFirst { case JField("severity", JString(sev)) => sev }.get ) - ) - }.filter(_._1).map(_._2) - } - case status => - val message = - s"Unexpected status $status ${httpResp.statusText} with body:\n${httpResp.bodyAsString}" + } + case _ => + val message = s"Unexpected status ${httpResp.status} ${httpResp.statusText} with body:\n${httpResp.bodyAsString}" throw new MessageOnlyException(message) } + } - private def githubToken(): String = Properties.envOrElse("GITHUB_TOKEN", "") + private def githubToken(): String = Properties.envOrElse("GITHUB_TOKEN", "") } From b74fc1acb12fdd56fbffb97b5845b5e52de81f5e Mon Sep 17 00:00:00 2001 From: yazgoo Date: Fri, 14 Jun 2024 08:27:03 +0200 Subject: [PATCH 15/28] scalafmt --- .../epfl/scala/AnalyzeDependencyGraph.scala | 141 +++++++++++------- 1 file changed, 89 insertions(+), 52 deletions(-) diff --git a/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala b/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala index ceddda5..8f6c68b 100644 --- a/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala +++ b/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala @@ -35,11 +35,13 @@ object AnalyzeDependencyGraph { } case object Alerts extends AnalysisAction { val name = "alerts" - val help = "download and display CVEs alerts from Github (use hub or gh local config or GIT_TOKEN env var to authenticate)" + val help = + "download and display CVEs alerts from Github (use hub or gh local config or GIT_TOKEN env var to authenticate)" } case object Cves extends AnalysisAction { val name = "cves" - val help = "analyze CVEs alerts against the dependencies (requires githubGenerateSnapshot and githubAnalyzeDependencies alerts)" + val help = + "analyze CVEs alerts against the dependencies (requires githubGenerateSnapshot and githubAnalyzeDependencies alerts)" } val values: Seq[AnalysisAction] = Seq(Get, List, Alerts, Cves) @@ -50,10 +52,10 @@ object AnalyzeDependencyGraph { def blue(str: String): String = s"\u001b[34m${str}\u001b[0m" case class Vulnerability( - packageId: String, - vulnerableVersionRange: String, - firstPatchedVersion: String, - severity: String + packageId: String, + vulnerableVersionRange: String, + firstPatchedVersion: String, + severity: String ) { def severityColor: String = severity match { case "critical" => "\u001b[31m" @@ -65,7 +67,8 @@ object AnalyzeDependencyGraph { def coloredSeverity: String = s"${severityColor}${severity}\u001b[0m" - override def toString: String = s"${blue(packageId)} [ $vulnerableVersionRange ] fixed: $firstPatchedVersion $coloredSeverity" + override def toString: String = + s"${blue(packageId)} [ $vulnerableVersionRange ] fixed: $firstPatchedVersion $coloredSeverity" } case class AnalysisParams(action: AnalysisAction, arg: Option[String]) @@ -79,13 +82,16 @@ object AnalyzeDependencyGraph { import Model._ val AnalyzeDependencies = "githubAnalyzeDependencies" - private val AnalyzeDependenciesUsage = s"""$AnalyzeDependencies [${AnalysisAction.values.map(_.name).mkString("|")}] [pattern]""" + private val AnalyzeDependenciesUsage = + s"""$AnalyzeDependencies [${AnalysisAction.values.map(_.name).mkString("|")}] [pattern]""" private val AnalyzeDependenciesDetail = s"""Analyze the dependencies based on a search pattern: ${AnalysisAction.values.map(a => s"${a.name}: ${a.help}").mkString("\n ")} """ val commands: Seq[Command] = Seq( - Command(AnalyzeDependencies, (AnalyzeDependenciesUsage, AnalyzeDependenciesDetail), AnalyzeDependenciesDetail)(extractPattern)(analyzeDependencies) + Command(AnalyzeDependencies, (AnalyzeDependenciesUsage, AnalyzeDependenciesDetail), AnalyzeDependenciesDetail)( + extractPattern + )(analyzeDependencies) ) private lazy val http: HttpClient = Gigahorse.http(Gigahorse.config) @@ -98,37 +104,54 @@ object AnalyzeDependencyGraph { } }.failOnException - private def highlight(string: String, pattern: String): String = + private def highlight(string: String, pattern: String): String = string.replaceAll(pattern, s"\u001b[32m${pattern}\u001b[0m") - private def analyzeDependenciesInternal(state: State, action: AnalysisAction, pattern: String, originalPattern: String): Unit = { - def getDeps(dependencies: Seq[String], pattern: String): Seq[String] = + private def analyzeDependenciesInternal( + state: State, + action: AnalysisAction, + pattern: String, + originalPattern: String + ): Unit = { + def getDeps(dependencies: Seq[String], pattern: String): Seq[String] = dependencies.filter(_.contains(pattern)).map(highlight(_, originalPattern)) - def resolvedDeps(tabs: String, acc: Seq[String], resolvedByName: Map[String, DependencyNode], pattern: String, originalPattern: String): Seq[String] = { - acc ++ resolvedByName.toSeq.flatMap { case (name, resolved) => - val matchingDependencies = getDeps(resolved.dependencies, pattern) - if (matchingDependencies.isEmpty) { - if (name.contains(pattern)) Seq(tabs + highlight(name, originalPattern)) else Nil - } else { - matchingDependencies.flatMap { matchingDependency => - resolvedDeps(" " + tabs, acc :+ (tabs + matchingDependency), resolvedByName, name, originalPattern) + def resolvedDeps( + tabs: String, + acc: Seq[String], + resolvedByName: Map[String, DependencyNode], + pattern: String, + originalPattern: String + ): Seq[String] = + acc ++ resolvedByName.toSeq.flatMap { + case (name, resolved) => + val matchingDependencies = getDeps(resolved.dependencies, pattern) + if (matchingDependencies.isEmpty) { + if (name.contains(pattern)) Seq(tabs + highlight(name, originalPattern)) else Nil + } else { + matchingDependencies.flatMap { matchingDependency => + resolvedDeps(" " + tabs, acc :+ (tabs + matchingDependency), resolvedByName, name, originalPattern) + } } - } } - } - val matches = state.get(githubManifestsKey).toSeq.flatMap { manifests => - manifests.map { case (name, manifest) => - manifest -> resolvedDeps("", Nil, manifest.resolved, pattern, originalPattern = pattern) + val matches = state + .get(githubManifestsKey) + .toSeq + .flatMap { manifests => + manifests.map { + case (name, manifest) => + manifest -> resolvedDeps("", Nil, manifest.resolved, pattern, originalPattern = pattern) + } } - }.toMap + .toMap action match { case AnalysisAction.Get => - matches.foreach { case (manifest, deps) => - println(s"📁 ${blue(manifest.name)}") - println(deps.map(dep => s" $dep").mkString("\n")) + matches.foreach { + case (manifest, deps) => + println(s"📁 ${blue(manifest.name)}") + println(deps.map(dep => s" $dep").mkString("\n")) } case AnalysisAction.List => println(matches.values.flatten.filter(_.contains(pattern)).toSet.mkString("\n")) @@ -144,10 +167,12 @@ object AnalyzeDependencyGraph { } private def getGithubTokenFromGhConfigDir(): String = { - val ghConfigDir = Properties.envOrElse("GH_CONFIG_DIR", Paths.get(System.getProperty("user.home"), ".config", "gh").toString) + val ghConfigDir = + Properties.envOrElse("GH_CONFIG_DIR", Paths.get(System.getProperty("user.home"), ".config", "gh").toString) val ghConfigFile = Paths.get(ghConfigDir).resolve("hosts.yml").toFile getGithubToken(ghConfigFile).getOrElse { - val ghConfigPath = Properties.envOrElse("HUB_CONFIG", Paths.get(System.getProperty("user.home"), ".config", "hub").toString) + val ghConfigPath = + Properties.envOrElse("HUB_CONFIG", Paths.get(System.getProperty("user.home"), ".config", "hub").toString) val hubConfigFile = Paths.get(ghConfigPath).toFile getGithubToken(hubConfigFile).getOrElse(githubToken()) } @@ -155,7 +180,8 @@ object AnalyzeDependencyGraph { private def downloadAlerts(state: State, repo: String): Try[State] = { val snapshotUrl = s"https://api.github.com/repos/$repo/dependabot/alerts" - val request = Gigahorse.url(snapshotUrl).get.addHeaders("Authorization" -> s"token ${getGithubTokenFromGhConfigDir()}") + val request = + Gigahorse.url(snapshotUrl).get.addHeaders("Authorization" -> s"token ${getGithubTokenFromGhConfigDir()}") state.log.info(s"Downloading alerts from $snapshotUrl") for { httpResp <- Try(Await.result(http.processFull(request), Duration.Inf)) @@ -166,13 +192,17 @@ object AnalyzeDependencyGraph { } } - private def getAllArtifacts(state: State): Seq[String] = { - state.get(githubManifestsKey).toSeq.flatMap { manifests => - manifests.flatMap { case (_, manifest) => - manifest.resolved.values.toSeq.map(_.package_url) + private def getAllArtifacts(state: State): Seq[String] = + state + .get(githubManifestsKey) + .toSeq + .flatMap { manifests => + manifests.flatMap { + case (_, manifest) => + manifest.resolved.values.toSeq.map(_.package_url) + } } - }.distinct - } + .distinct private def translateToSemVer(string: String): String = string.replaceAll("([0-9]+)\\.([0-9]+)\\.([0-9]+)\\.([0-9]+)", "$1.$2.$3-$4") @@ -190,12 +220,14 @@ object AnalyzeDependencyGraph { } else No } - private def vulnerabilityMatchesArtifacts(alert: Vulnerability, artifacts: Seq[String]): Map[Vulnerable, Seq[String]] = { + private def vulnerabilityMatchesArtifacts( + alert: Vulnerability, + artifacts: Seq[String] + ): Map[Vulnerable, Seq[String]] = artifacts.foldLeft(Map(Good -> Seq.empty[String], Bad -> Seq.empty[String])) { (acc, artifact) => val res = vulnerabilityMatchesArtifact(alert, artifact) if (res != No) acc.updated(res, acc(res) :+ artifact) else acc } - } private def analyzeCves(state: State): State = { val vulnerabilities = state.get(githubAlertsKey).getOrElse(Seq.empty) @@ -218,11 +250,11 @@ object AnalyzeDependencyGraph { val repoPattern = """(?:https://|git@)github\.com[:/](.+/.+)\.git""".r remoteUrl match { case repoPattern(repo) => Some(repo) - case _ => None + case _ => None } } - private def analyzeDependencies(state: State, params: AnalysisParams): State = { + private def analyzeDependencies(state: State, params: AnalysisParams): State = params.action match { case AnalysisAction.Get | AnalysisAction.List => params.arg.foreach(pattern => analyzeDependenciesInternal(state, params.action, pattern, pattern)) @@ -234,31 +266,36 @@ object AnalyzeDependencyGraph { case _ => state } - } private def getVulnerabilities(httpResp: FullResponse): Try[Seq[Vulnerability]] = Try { httpResp.status match { case status if status / 100 == 2 => val json: JArray = JsonParser.parseFromByteBuffer(httpResp.bodyAsByteBuffer).get.asInstanceOf[JArray] json.value.collect { - case obj: JObject if (obj.value.collectFirst { case JField("state", JString("open")) => true }.isDefined) => - val securityVulnerability = obj.value.collectFirst { case JField("security_vulnerability", secVuln: JObject) => secVuln }.get.value - val packageObj = securityVulnerability.collectFirst { case JField("package", pkg: JObject) => pkg }.get.value - val firstPatchedVersion = securityVulnerability.collectFirst { - case JField("first_patched_version", firstPatched: JObject) => firstPatched - }.map(_.value.collectFirst { case JField("identifier", JString(ident)) => ident }.getOrElse("")).getOrElse("") + case obj: JObject if obj.value.collectFirst { case JField("state", JString("open")) => true }.isDefined => + val securityVulnerability = + obj.value.collectFirst { case JField("security_vulnerability", secVuln: JObject) => secVuln }.get.value + val packageObj = + securityVulnerability.collectFirst { case JField("package", pkg: JObject) => pkg }.get.value + val firstPatchedVersion = securityVulnerability + .collectFirst { case JField("first_patched_version", firstPatched: JObject) => firstPatched } + .map(_.value.collectFirst { case JField("identifier", JString(ident)) => ident }.getOrElse("")) + .getOrElse("") Vulnerability( packageObj.collectFirst { case JField("name", JString(name)) => name }.get, - securityVulnerability.collectFirst { case JField("vulnerable_version_range", JString(range)) => range }.get, + securityVulnerability.collectFirst { + case JField("vulnerable_version_range", JString(range)) => range + }.get, firstPatchedVersion, securityVulnerability.collectFirst { case JField("severity", JString(sev)) => sev }.get ) } case _ => - val message = s"Unexpected status ${httpResp.status} ${httpResp.statusText} with body:\n${httpResp.bodyAsString}" + val message = + s"Unexpected status ${httpResp.status} ${httpResp.statusText} with body:\n${httpResp.bodyAsString}" throw new MessageOnlyException(message) } } - private def githubToken(): String = Properties.envOrElse("GITHUB_TOKEN", "") + private def githubToken(): String = Properties.envOrElse("GITHUB_TOKEN", "") } From 16c8d461f0ad850b9f9a0a044016978bfa23c872 Mon Sep 17 00:00:00 2001 From: yazgoo Date: Tue, 25 Jun 2024 12:09:48 +0200 Subject: [PATCH 16/28] better version comparison --- .../main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala b/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala index 8f6c68b..0605c08 100644 --- a/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala +++ b/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala @@ -204,8 +204,9 @@ object AnalyzeDependencyGraph { } .distinct - private def translateToSemVer(string: String): String = - string.replaceAll("([0-9]+)\\.([0-9]+)\\.([0-9]+)\\.([0-9]+)", "$1.$2.$3-$4") + private def translateToSemVer(string: String): String = + string.replaceAll("([a-zA-Z]+)", "0").replaceAll("([0-9]+)\\.([0-9]+)\\.([0-9]+)\\.([0-9]+)", "$1.$2.$3-$4") + private def versionMatchesRange(versionStr: String, rangeStr: String): Boolean = { val range = rangeStr.replaceAll(" ", "").replace(",", " ") From 7366314c4712fb83fe4d84d022c5fb1edfe30da4 Mon Sep 17 00:00:00 2001 From: yazgoo Date: Tue, 25 Jun 2024 13:12:57 +0200 Subject: [PATCH 17/28] warn on state retrieval --- .../epfl/scala/AnalyzeDependencyGraph.scala | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala b/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala index 0605c08..af15da1 100644 --- a/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala +++ b/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala @@ -107,6 +107,15 @@ object AnalyzeDependencyGraph { private def highlight(string: String, pattern: String): String = string.replaceAll(pattern, s"\u001b[32m${pattern}\u001b[0m") + private def getStateOrWarn[T](state: State, key: AttributeKey[T], what: String, command: String): Option[T] = + state.get(key).orElse { + println(s"🟠 No $what found, please run '$command' first") + None + } + + private def getGithubManifest(state: State): Seq[Map[String, Manifest]] = + getStateOrWarn(state, githubManifestsKey, "dependencies", SubmitDependencyGraph.Generate).toSeq + private def analyzeDependenciesInternal( state: State, action: AnalysisAction, @@ -135,9 +144,7 @@ object AnalyzeDependencyGraph { } } - val matches = state - .get(githubManifestsKey) - .toSeq + val matches = getGithubManifest(state) .flatMap { manifests => manifests.map { case (name, manifest) => @@ -193,9 +200,7 @@ object AnalyzeDependencyGraph { } private def getAllArtifacts(state: State): Seq[String] = - state - .get(githubManifestsKey) - .toSeq + getGithubManifest(state) .flatMap { manifests => manifests.flatMap { case (_, manifest) => @@ -231,7 +236,7 @@ object AnalyzeDependencyGraph { } private def analyzeCves(state: State): State = { - val vulnerabilities = state.get(githubAlertsKey).getOrElse(Seq.empty) + val vulnerabilities = getStateOrWarn(state, githubAlertsKey, "artifcats", s"${AnalyzeDependencies} alerts").getOrElse(Seq.empty) val artifacts = getAllArtifacts(state) vulnerabilities.foreach { v => val matches = vulnerabilityMatchesArtifacts(v, artifacts) @@ -298,5 +303,5 @@ object AnalyzeDependencyGraph { } } - private def githubToken(): String = Properties.envOrElse("GITHUB_TOKEN", "") + private def githubToken( ): String = Properties.envOrElse("GITHUB_TOKEN", "") } From 2d0d1141f68d79c2a2bbea1e3e42386480712c2b Mon Sep 17 00:00:00 2001 From: yazgoo Date: Fri, 25 Oct 2024 14:35:16 +0200 Subject: [PATCH 18/28] remove list and get --- .../epfl/scala/AnalyzeDependencyGraph.scala | 68 +------------------ 1 file changed, 2 insertions(+), 66 deletions(-) diff --git a/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala b/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala index af15da1..fed27f1 100644 --- a/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala +++ b/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala @@ -25,14 +25,6 @@ object AnalyzeDependencyGraph { } object AnalysisAction { - case object Get extends AnalysisAction { - val name = "get" - val help = "search for a pattern in the dependencies (requires githubGenerateSnapshot)" - } - case object List extends AnalysisAction { - val name = "list" - val help = "list all dependencies matching a pattern (requires githubGenerateSnapshot)" - } case object Alerts extends AnalysisAction { val name = "alerts" val help = @@ -44,7 +36,7 @@ object AnalyzeDependencyGraph { "analyze CVEs alerts against the dependencies (requires githubGenerateSnapshot and githubAnalyzeDependencies alerts)" } - val values: Seq[AnalysisAction] = Seq(Get, List, Alerts, Cves) + val values: Seq[AnalysisAction] = Seq(Alerts, Cves) def fromString(str: String): Option[AnalysisAction] = values.find(_.name == str) } @@ -104,68 +96,15 @@ object AnalyzeDependencyGraph { } }.failOnException - private def highlight(string: String, pattern: String): String = - string.replaceAll(pattern, s"\u001b[32m${pattern}\u001b[0m") - private def getStateOrWarn[T](state: State, key: AttributeKey[T], what: String, command: String): Option[T] = state.get(key).orElse { println(s"🟠 No $what found, please run '$command' first") None } - private def getGithubManifest(state: State): Seq[Map[String, Manifest]] = + def getGithubManifest(state: State): Seq[Map[String, Manifest]] = getStateOrWarn(state, githubManifestsKey, "dependencies", SubmitDependencyGraph.Generate).toSeq - private def analyzeDependenciesInternal( - state: State, - action: AnalysisAction, - pattern: String, - originalPattern: String - ): Unit = { - def getDeps(dependencies: Seq[String], pattern: String): Seq[String] = - dependencies.filter(_.contains(pattern)).map(highlight(_, originalPattern)) - - def resolvedDeps( - tabs: String, - acc: Seq[String], - resolvedByName: Map[String, DependencyNode], - pattern: String, - originalPattern: String - ): Seq[String] = - acc ++ resolvedByName.toSeq.flatMap { - case (name, resolved) => - val matchingDependencies = getDeps(resolved.dependencies, pattern) - if (matchingDependencies.isEmpty) { - if (name.contains(pattern)) Seq(tabs + highlight(name, originalPattern)) else Nil - } else { - matchingDependencies.flatMap { matchingDependency => - resolvedDeps(" " + tabs, acc :+ (tabs + matchingDependency), resolvedByName, name, originalPattern) - } - } - } - - val matches = getGithubManifest(state) - .flatMap { manifests => - manifests.map { - case (name, manifest) => - manifest -> resolvedDeps("", Nil, manifest.resolved, pattern, originalPattern = pattern) - } - } - .toMap - - action match { - case AnalysisAction.Get => - matches.foreach { - case (manifest, deps) => - println(s"📁 ${blue(manifest.name)}") - println(deps.map(dep => s" $dep").mkString("\n")) - } - case AnalysisAction.List => - println(matches.values.flatten.filter(_.contains(pattern)).toSet.mkString("\n")) - case _ => - } - } - private def getGithubToken(ghConfigFile: File): Option[String] = { println(s"Extract token from ${ghConfigFile.getPath}") if (ghConfigFile.exists()) { @@ -262,9 +201,6 @@ object AnalyzeDependencyGraph { private def analyzeDependencies(state: State, params: AnalysisParams): State = params.action match { - case AnalysisAction.Get | AnalysisAction.List => - params.arg.foreach(pattern => analyzeDependenciesInternal(state, params.action, pattern, pattern)) - state case AnalysisAction.Alerts => params.arg.orElse(getGitHubRepo).map(repo => downloadAlerts(state, repo).get).getOrElse(state) case AnalysisAction.Cves => From 825c4d5b3182872cc2172385e0c77240ce09920d Mon Sep 17 00:00:00 2001 From: yazgoo Date: Fri, 25 Oct 2024 14:38:06 +0200 Subject: [PATCH 19/28] scalafix --- .../epfl/scala/AnalyzeDependencyGraph.scala | 37 ++++++++++--------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala b/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala index fed27f1..d05f4b2 100644 --- a/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala +++ b/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala @@ -4,18 +4,23 @@ import java.nio.file.Paths import scala.concurrent.Await import scala.concurrent.duration.Duration -import scala.util.{Properties, Try} +import scala.sys.process._ +import scala.util.Properties +import scala.util.Try + import ch.epfl.scala.GithubDependencyGraphPlugin.autoImport._ import ch.epfl.scala.githubapi._ +import gigahorse.FullResponse +import gigahorse.HttpClient import gigahorse.support.asynchttpclient.Gigahorse import sbt._ import sbt.internal.util.complete._ -import sjsonnew.shaded.scalajson.ast.unsafe.{JArray, JObject, JField, JString} -import gigahorse.{FullResponse, HttpClient} +import sjsonnew.shaded.scalajson.ast.unsafe.JArray +import sjsonnew.shaded.scalajson.ast.unsafe.JField +import sjsonnew.shaded.scalajson.ast.unsafe.JObject +import sjsonnew.shaded.scalajson.ast.unsafe.JString import sjsonnew.support.scalajson.unsafe.{Parser => JsonParser} -import scala.sys.process._ - object AnalyzeDependencyGraph { object Model { @@ -139,18 +144,15 @@ object AnalyzeDependencyGraph { } private def getAllArtifacts(state: State): Seq[String] = - getGithubManifest(state) - .flatMap { manifests => - manifests.flatMap { - case (_, manifest) => - manifest.resolved.values.toSeq.map(_.package_url) - } + getGithubManifest(state).flatMap { manifests => + manifests.flatMap { + case (_, manifest) => + manifest.resolved.values.toSeq.map(_.package_url) } - .distinct - - private def translateToSemVer(string: String): String = - string.replaceAll("([a-zA-Z]+)", "0").replaceAll("([0-9]+)\\.([0-9]+)\\.([0-9]+)\\.([0-9]+)", "$1.$2.$3-$4") + }.distinct + private def translateToSemVer(string: String): String = + string.replaceAll("([a-zA-Z]+)", "0").replaceAll("([0-9]+)\\.([0-9]+)\\.([0-9]+)\\.([0-9]+)", "$1.$2.$3-$4") private def versionMatchesRange(versionStr: String, rangeStr: String): Boolean = { val range = rangeStr.replaceAll(" ", "").replace(",", " ") @@ -175,7 +177,8 @@ object AnalyzeDependencyGraph { } private def analyzeCves(state: State): State = { - val vulnerabilities = getStateOrWarn(state, githubAlertsKey, "artifcats", s"${AnalyzeDependencies} alerts").getOrElse(Seq.empty) + val vulnerabilities = + getStateOrWarn(state, githubAlertsKey, "artifcats", s"${AnalyzeDependencies} alerts").getOrElse(Seq.empty) val artifacts = getAllArtifacts(state) vulnerabilities.foreach { v => val matches = vulnerabilityMatchesArtifacts(v, artifacts) @@ -239,5 +242,5 @@ object AnalyzeDependencyGraph { } } - private def githubToken( ): String = Properties.envOrElse("GITHUB_TOKEN", "") + private def githubToken(): String = Properties.envOrElse("GITHUB_TOKEN", "") } From 60a1348aabf9d030bd53f7a242af225a704673fb Mon Sep 17 00:00:00 2001 From: yazgoo Date: Tue, 29 Oct 2024 11:16:09 +0100 Subject: [PATCH 20/28] declare only one command group "alerts" and "cve" command in one command to make it easier to use. --- .../epfl/scala/AnalyzeDependencyGraph.scala | 44 ++++--------------- 1 file changed, 9 insertions(+), 35 deletions(-) diff --git a/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala b/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala index d05f4b2..5ae6899 100644 --- a/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala +++ b/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala @@ -24,27 +24,8 @@ import sjsonnew.support.scalajson.unsafe.{Parser => JsonParser} object AnalyzeDependencyGraph { object Model { - sealed trait AnalysisAction { - def name: String - def help: String - } - - object AnalysisAction { - case object Alerts extends AnalysisAction { - val name = "alerts" - val help = - "download and display CVEs alerts from Github (use hub or gh local config or GIT_TOKEN env var to authenticate)" - } - case object Cves extends AnalysisAction { - val name = "cves" - val help = - "analyze CVEs alerts against the dependencies (requires githubGenerateSnapshot and githubAnalyzeDependencies alerts)" - } - - val values: Seq[AnalysisAction] = Seq(Alerts, Cves) - - def fromString(str: String): Option[AnalysisAction] = values.find(_.name == str) - } + val help = + "download and display CVEs alerts from Github, and analyze them against dependencies (use hub or gh local config or GIT_TOKEN env var to authenticate, requires githubGenerateSnapshot and githubAnalyzeDependencies alerts)" def blue(str: String): String = s"\u001b[34m${str}\u001b[0m" @@ -68,7 +49,7 @@ object AnalyzeDependencyGraph { s"${blue(packageId)} [ $vulnerableVersionRange ] fixed: $firstPatchedVersion $coloredSeverity" } - case class AnalysisParams(action: AnalysisAction, arg: Option[String]) + case class AnalysisParams(arg: Option[String]) sealed trait Vulnerable object Good extends Vulnerable @@ -80,9 +61,9 @@ object AnalyzeDependencyGraph { val AnalyzeDependencies = "githubAnalyzeDependencies" private val AnalyzeDependenciesUsage = - s"""$AnalyzeDependencies [${AnalysisAction.values.map(_.name).mkString("|")}] [pattern]""" + s"""$AnalyzeDependencies [pattern]""" private val AnalyzeDependenciesDetail = s"""Analyze the dependencies based on a search pattern: - ${AnalysisAction.values.map(a => s"${a.name}: ${a.help}").mkString("\n ")} + $help """ val commands: Seq[Command] = Seq( @@ -96,8 +77,8 @@ object AnalyzeDependencyGraph { private def extractPattern(state: State): Parser[AnalysisParams] = Parsers.any.*.map { raw => raw.mkString.trim.split(" ").toSeq match { - case Seq(action, arg) => AnalysisParams(AnalysisAction.fromString(action).get, Some(arg)) - case Seq(action) => AnalysisParams(AnalysisAction.fromString(action).get, None) + case Seq("") | Nil => AnalysisParams(None) + case Seq(arg) => AnalysisParams(Some(arg)) } }.failOnException @@ -138,7 +119,7 @@ object AnalyzeDependencyGraph { httpResp <- Try(Await.result(http.processFull(request), Duration.Inf)) vulnerabilities <- getVulnerabilities(httpResp) } yield { - vulnerabilities.foreach(v => println(v.toString)) + state.log.info(s"Downloaded ${vulnerabilities.size} alerts") state.put(githubAlertsKey, vulnerabilities) } } @@ -203,14 +184,7 @@ object AnalyzeDependencyGraph { } private def analyzeDependencies(state: State, params: AnalysisParams): State = - params.action match { - case AnalysisAction.Alerts => - params.arg.orElse(getGitHubRepo).map(repo => downloadAlerts(state, repo).get).getOrElse(state) - case AnalysisAction.Cves => - analyzeCves(state) - case _ => - state - } + analyzeCves(params.arg.orElse(getGitHubRepo).map(repo => downloadAlerts(state, repo).get).getOrElse(state)) private def getVulnerabilities(httpResp: FullResponse): Try[Seq[Vulnerability]] = Try { httpResp.status match { From 8c0444a8e148e7b7f06435eabf4aa089a4d09b13 Mon Sep 17 00:00:00 2001 From: yazgoo Date: Wed, 30 Oct 2024 10:54:41 +0100 Subject: [PATCH 21/28] fix help --- .../src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala b/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala index 5ae6899..c7d2a4a 100644 --- a/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala +++ b/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala @@ -25,7 +25,7 @@ object AnalyzeDependencyGraph { object Model { val help = - "download and display CVEs alerts from Github, and analyze them against dependencies (use hub or gh local config or GIT_TOKEN env var to authenticate, requires githubGenerateSnapshot and githubAnalyzeDependencies alerts)" + "download and display CVEs alerts from Github, and analyze them against dependencies (use hub or gh local config or GIT_TOKEN env var to authenticate, requires githubGenerateSnapshot)" def blue(str: String): String = s"\u001b[34m${str}\u001b[0m" From d072011979b0ff52bde0b43c16a23bc7f9216d3e Mon Sep 17 00:00:00 2001 From: yazgoo Date: Tue, 5 Nov 2024 12:05:09 +0100 Subject: [PATCH 22/28] =?UTF-8?q?=F0=9F=91=8C=20review?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../epfl/scala/AnalyzeDependencyGraph.scala | 181 +++++++++--------- .../scala/GithubDependencyGraphPlugin.scala | 2 +- 2 files changed, 91 insertions(+), 92 deletions(-) diff --git a/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala b/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala index c7d2a4a..d8bd706 100644 --- a/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala +++ b/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala @@ -20,44 +20,20 @@ import sjsonnew.shaded.scalajson.ast.unsafe.JField import sjsonnew.shaded.scalajson.ast.unsafe.JObject import sjsonnew.shaded.scalajson.ast.unsafe.JString import sjsonnew.support.scalajson.unsafe.{Parser => JsonParser} +import scala.Console +import scala.util.{ Success, Failure } object AnalyzeDependencyGraph { - object Model { - val help = - "download and display CVEs alerts from Github, and analyze them against dependencies (use hub or gh local config or GIT_TOKEN env var to authenticate, requires githubGenerateSnapshot)" - - def blue(str: String): String = s"\u001b[34m${str}\u001b[0m" - - case class Vulnerability( - packageId: String, - vulnerableVersionRange: String, - firstPatchedVersion: String, - severity: String - ) { - def severityColor: String = severity match { - case "critical" => "\u001b[31m" - case "high" => "\u001b[31m" - case "medium" => "\u001b[33m" - case "low" => "\u001b[32m" - case _ => "\u001b[0m" - } - - def coloredSeverity: String = s"${severityColor}${severity}\u001b[0m" - - override def toString: String = - s"${blue(packageId)} [ $vulnerableVersionRange ] fixed: $firstPatchedVersion $coloredSeverity" - } + val help = + "download and display CVEs alerts from Github, and analyze them against dependencies (use hub or gh local config or GIT_TOKEN env var to authenticate, requires githubGenerateSnapshot)" - case class AnalysisParams(arg: Option[String]) + case class AnalysisParams(repository: Option[String]) - sealed trait Vulnerable - object Good extends Vulnerable - object Bad extends Vulnerable - object No extends Vulnerable - } - - import Model._ + sealed trait Vulnerable + object Good extends Vulnerable + object Bad extends Vulnerable + object No extends Vulnerable val AnalyzeDependencies = "githubAnalyzeDependencies" private val AnalyzeDependenciesUsage = @@ -67,14 +43,13 @@ object AnalyzeDependencyGraph { """ val commands: Seq[Command] = Seq( - Command(AnalyzeDependencies, (AnalyzeDependenciesUsage, AnalyzeDependenciesDetail), AnalyzeDependenciesDetail)( - extractPattern - )(analyzeDependencies) + Command(AnalyzeDependencies, + (AnalyzeDependenciesUsage, AnalyzeDependenciesDetail), + AnalyzeDependenciesDetail + )(parser)(analyzeDependencies) ) - private lazy val http: HttpClient = Gigahorse.http(Gigahorse.config) - - private def extractPattern(state: State): Parser[AnalysisParams] = + private def parser(state: State): Parser[AnalysisParams] = Parsers.any.*.map { raw => raw.mkString.trim.split(" ").toSeq match { case Seq("") | Nil => AnalysisParams(None) @@ -82,45 +57,96 @@ object AnalyzeDependencyGraph { } }.failOnException + private def analyzeDependencies(state: State, params: AnalysisParams) : State = + (for { + repo <- params.repository.orElse(getGitHubRepo) + vulnerabilities <- downloadAlerts(state, repo) match { + case Success(v) => Some(v) + case Failure(e) => + state.log.error(s"Failed to download alerts: ${e.getMessage}") + None + } + } yield (analyzeCves(state, vulnerabilities)) + ).getOrElse(state) + + private def analyzeCves(state: State, vulnerabilities: Seq[Vulnerability]): State = { + val artifacts = getAllArtifacts(state) + vulnerabilities.foreach { v => + val (goodMatches, badMatches) = vulnerabilityMatchesArtifacts(v, artifacts) + println(v.toString) + if (goodMatches.nonEmpty || badMatches.nonEmpty) { + goodMatches.foreach(m => println(s" 🟢 ${m.replaceAll(".*@", "")}")) + badMatches.foreach(m => println(s" 🔴 ${m.replaceAll(".*@", "")}")) + } else { + println(" 🎉 no match (dependency was probably removed)") + } + } + state + } + private def getStateOrWarn[T](state: State, key: AttributeKey[T], what: String, command: String): Option[T] = state.get(key).orElse { println(s"🟠 No $what found, please run '$command' first") None } + private def downloadAlerts(state: State, repo: String): Try[Seq[Vulnerability]] = { + val snapshotUrl = s"https://api.github.com/repos/$repo/dependabot/alerts" + val request = + Gigahorse.url(snapshotUrl).get.addHeaders("Authorization" -> s"token ${getGithubToken()}") + state.log.info(s"Downloading alerts from $snapshotUrl") + for { + httpResp <- Try(Await.result(http.processFull(request), Duration.Inf)) + vulnerabilities <- getVulnerabilities(httpResp) + } yield { + state.log.info(s"Downloaded ${vulnerabilities.size} alerts") + vulnerabilities + } + } + + case class Vulnerability( + packageId: String, + vulnerableVersionRange: String, + firstPatchedVersion: String, + severity: String + ) { + def severityColor: String = severity match { + case "critical" => Console.RED + case "high" => Console.RED + case "medium" => Console.YELLOW + case "low" => Console.GREEN + case _ => Console.RESET + } + + def coloredSeverity: String = s"${severityColor}${severity}${Console.RESET}" + + def coloredPackageId: String = s"${Console.BLUE}$packageId${Console.RESET}" + + override def toString: String = + s"${coloredPackageId} [ $vulnerableVersionRange ] fixed: $firstPatchedVersion $coloredSeverity" + } + + private lazy val http: HttpClient = Gigahorse.http(Gigahorse.config) + def getGithubManifest(state: State): Seq[Map[String, Manifest]] = getStateOrWarn(state, githubManifestsKey, "dependencies", SubmitDependencyGraph.Generate).toSeq - private def getGithubToken(ghConfigFile: File): Option[String] = { + private def getGithubTokenFromFile(ghConfigFile: File): Option[String] = { println(s"Extract token from ${ghConfigFile.getPath}") if (ghConfigFile.exists()) { IO.readLines(ghConfigFile).find(_.contains("oauth_token")).map(_.split(":").last.trim) } else None } - private def getGithubTokenFromGhConfigDir(): String = { + private def getGithubToken(): String = { val ghConfigDir = Properties.envOrElse("GH_CONFIG_DIR", Paths.get(System.getProperty("user.home"), ".config", "gh").toString) val ghConfigFile = Paths.get(ghConfigDir).resolve("hosts.yml").toFile - getGithubToken(ghConfigFile).getOrElse { + getGithubTokenFromFile(ghConfigFile).getOrElse { val ghConfigPath = Properties.envOrElse("HUB_CONFIG", Paths.get(System.getProperty("user.home"), ".config", "hub").toString) val hubConfigFile = Paths.get(ghConfigPath).toFile - getGithubToken(hubConfigFile).getOrElse(githubToken()) - } - } - - private def downloadAlerts(state: State, repo: String): Try[State] = { - val snapshotUrl = s"https://api.github.com/repos/$repo/dependabot/alerts" - val request = - Gigahorse.url(snapshotUrl).get.addHeaders("Authorization" -> s"token ${getGithubTokenFromGhConfigDir()}") - state.log.info(s"Downloading alerts from $snapshotUrl") - for { - httpResp <- Try(Await.result(http.processFull(request), Duration.Inf)) - vulnerabilities <- getVulnerabilities(httpResp) - } yield { - state.log.info(s"Downloaded ${vulnerabilities.size} alerts") - state.put(githubAlertsKey, vulnerabilities) + getGithubTokenFromFile(hubConfigFile).getOrElse(githubToken()) } } @@ -140,38 +166,14 @@ object AnalyzeDependencyGraph { VersionNumber(translateToSemVer(versionStr)).matchesSemVer(SemanticSelector(translateToSemVer(range))) } - private def vulnerabilityMatchesArtifact(alert: Vulnerability, artifact: String): Vulnerable = { + private def vulnerabilityMatchesArtifacts(alert: Vulnerability, artifacts: Seq[String]): (Seq[String], Seq[String]) = { val alertMavenPath = s"pkg:maven/${alert.packageId.replace(":", "/")}@" - if (artifact.startsWith(alertMavenPath)) { - val version = artifact.replaceAll(".*@", "") - if (versionMatchesRange(version, alert.vulnerableVersionRange)) Bad else Good - } else No - } - - private def vulnerabilityMatchesArtifacts( - alert: Vulnerability, - artifacts: Seq[String] - ): Map[Vulnerable, Seq[String]] = - artifacts.foldLeft(Map(Good -> Seq.empty[String], Bad -> Seq.empty[String])) { (acc, artifact) => - val res = vulnerabilityMatchesArtifact(alert, artifact) - if (res != No) acc.updated(res, acc(res) :+ artifact) else acc - } - - private def analyzeCves(state: State): State = { - val vulnerabilities = - getStateOrWarn(state, githubAlertsKey, "artifcats", s"${AnalyzeDependencies} alerts").getOrElse(Seq.empty) - val artifacts = getAllArtifacts(state) - vulnerabilities.foreach { v => - val matches = vulnerabilityMatchesArtifacts(v, artifacts) - println(v.toString) - if (matches(Good).nonEmpty || matches(Bad).nonEmpty) { - matches(Good).foreach(m => println(s" 🟢 ${m.replaceAll(".*@", "")}")) - matches(Bad).foreach(m => println(s" 🔴 ${m.replaceAll(".*@", "")}")) - } else { - println(" 🎉 no match (dependency was probably removed)") + artifacts + .filter(_.startsWith(alertMavenPath)) + .partition { artifact => + val version = artifact.replaceAll(".*@", "") + versionMatchesRange(version, alert.vulnerableVersionRange) } - } - state } def getGitHubRepo: Option[String] = { @@ -183,9 +185,6 @@ object AnalyzeDependencyGraph { } } - private def analyzeDependencies(state: State, params: AnalysisParams): State = - analyzeCves(params.arg.orElse(getGitHubRepo).map(repo => downloadAlerts(state, repo).get).getOrElse(state)) - private def getVulnerabilities(httpResp: FullResponse): Try[Seq[Vulnerability]] = Try { httpResp.status match { case status if status / 100 == 2 => @@ -216,5 +215,5 @@ object AnalyzeDependencyGraph { } } - private def githubToken(): String = Properties.envOrElse("GITHUB_TOKEN", "") + private def githubToken(): String = Properties.envOrElse("GITHUB_TOKEN", "") } diff --git a/sbt-plugin/src/main/scala/ch/epfl/scala/GithubDependencyGraphPlugin.scala b/sbt-plugin/src/main/scala/ch/epfl/scala/GithubDependencyGraphPlugin.scala index 448a48f..bf26e7f 100644 --- a/sbt-plugin/src/main/scala/ch/epfl/scala/GithubDependencyGraphPlugin.scala +++ b/sbt-plugin/src/main/scala/ch/epfl/scala/GithubDependencyGraphPlugin.scala @@ -32,7 +32,7 @@ object GithubDependencyGraphPlugin extends AutoPlugin { val githubManifestsKey: AttributeKey[Map[String, githubapi.Manifest]] = AttributeKey("githubDependencyManifests") val githubProjectsKey: AttributeKey[Seq[ProjectRef]] = AttributeKey("githubProjectRefs") val githubSnapshotFileKey: AttributeKey[File] = AttributeKey("githubSnapshotFile") - val githubAlertsKey: AttributeKey[Seq[AnalyzeDependencyGraph.Model.Vulnerability]] = AttributeKey("githubAlerts") + val githubAlertsKey: AttributeKey[Seq[AnalyzeDependencyGraph.Vulnerability]] = AttributeKey("githubAlerts") val githubDependencyManifest: TaskKey[Option[githubapi.Manifest]] = taskKey( "The dependency manifest of the project" From 4e839cf28e1e71d1a69cd4e7ca98fa5a9e657cb6 Mon Sep 17 00:00:00 2001 From: yazgoo Date: Tue, 5 Nov 2024 12:06:24 +0100 Subject: [PATCH 23/28] scalafix --- .../main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala b/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala index d8bd706..ff7f57c 100644 --- a/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala +++ b/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala @@ -2,10 +2,13 @@ package ch.epfl.scala import java.nio.file.Paths +import scala.Console import scala.concurrent.Await import scala.concurrent.duration.Duration import scala.sys.process._ +import scala.util.Failure import scala.util.Properties +import scala.util.Success import scala.util.Try import ch.epfl.scala.GithubDependencyGraphPlugin.autoImport._ @@ -20,8 +23,6 @@ import sjsonnew.shaded.scalajson.ast.unsafe.JField import sjsonnew.shaded.scalajson.ast.unsafe.JObject import sjsonnew.shaded.scalajson.ast.unsafe.JString import sjsonnew.support.scalajson.unsafe.{Parser => JsonParser} -import scala.Console -import scala.util.{ Success, Failure } object AnalyzeDependencyGraph { From 1c05a4e7e0ffb3bd9d37eb67a3a0879be15ad04f Mon Sep 17 00:00:00 2001 From: yazgoo Date: Tue, 5 Nov 2024 12:07:00 +0100 Subject: [PATCH 24/28] rm Vulnerable --- .../main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala | 5 ----- 1 file changed, 5 deletions(-) diff --git a/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala b/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala index ff7f57c..09d3498 100644 --- a/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala +++ b/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala @@ -31,11 +31,6 @@ object AnalyzeDependencyGraph { case class AnalysisParams(repository: Option[String]) - sealed trait Vulnerable - object Good extends Vulnerable - object Bad extends Vulnerable - object No extends Vulnerable - val AnalyzeDependencies = "githubAnalyzeDependencies" private val AnalyzeDependenciesUsage = s"""$AnalyzeDependencies [pattern]""" From 761d27a28fd2f0b1126c57e0a86a6918bdaf2a16 Mon Sep 17 00:00:00 2001 From: yazgoo Date: Tue, 5 Nov 2024 12:11:57 +0100 Subject: [PATCH 25/28] format --- .../epfl/scala/AnalyzeDependencyGraph.scala | 43 ++++++++++--------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala b/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala index 09d3498..948f815 100644 --- a/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala +++ b/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala @@ -39,10 +39,9 @@ object AnalyzeDependencyGraph { """ val commands: Seq[Command] = Seq( - Command(AnalyzeDependencies, - (AnalyzeDependenciesUsage, AnalyzeDependenciesDetail), - AnalyzeDependenciesDetail - )(parser)(analyzeDependencies) + Command(AnalyzeDependencies, (AnalyzeDependenciesUsage, AnalyzeDependenciesDetail), AnalyzeDependenciesDetail)( + parser + )(analyzeDependencies) ) private def parser(state: State): Parser[AnalysisParams] = @@ -53,7 +52,7 @@ object AnalyzeDependencyGraph { } }.failOnException - private def analyzeDependencies(state: State, params: AnalysisParams) : State = + private def analyzeDependencies(state: State, params: AnalysisParams): State = (for { repo <- params.repository.orElse(getGitHubRepo) vulnerabilities <- downloadAlerts(state, repo) match { @@ -62,8 +61,7 @@ object AnalyzeDependencyGraph { state.log.error(s"Failed to download alerts: ${e.getMessage}") None } - } yield (analyzeCves(state, vulnerabilities)) - ).getOrElse(state) + } yield analyzeCves(state, vulnerabilities)).getOrElse(state) private def analyzeCves(state: State, vulnerabilities: Seq[Vulnerability]): State = { val artifacts = getAllArtifacts(state) @@ -101,10 +99,10 @@ object AnalyzeDependencyGraph { } case class Vulnerability( - packageId: String, - vulnerableVersionRange: String, - firstPatchedVersion: String, - severity: String + packageId: String, + vulnerableVersionRange: String, + firstPatchedVersion: String, + severity: String ) { def severityColor: String = severity match { case "critical" => Console.RED @@ -162,15 +160,18 @@ object AnalyzeDependencyGraph { VersionNumber(translateToSemVer(versionStr)).matchesSemVer(SemanticSelector(translateToSemVer(range))) } - private def vulnerabilityMatchesArtifacts(alert: Vulnerability, artifacts: Seq[String]): (Seq[String], Seq[String]) = { - val alertMavenPath = s"pkg:maven/${alert.packageId.replace(":", "/")}@" - artifacts - .filter(_.startsWith(alertMavenPath)) - .partition { artifact => - val version = artifact.replaceAll(".*@", "") - versionMatchesRange(version, alert.vulnerableVersionRange) - } - } + private def vulnerabilityMatchesArtifacts( + alert: Vulnerability, + artifacts: Seq[String] + ): (Seq[String], Seq[String]) = { + val alertMavenPath = s"pkg:maven/${alert.packageId.replace(":", "/")}@" + artifacts + .filter(_.startsWith(alertMavenPath)) + .partition { artifact => + val version = artifact.replaceAll(".*@", "") + versionMatchesRange(version, alert.vulnerableVersionRange) + } + } def getGitHubRepo: Option[String] = { val remoteUrl = "git config --get remote.origin.url".!!.trim @@ -211,5 +212,5 @@ object AnalyzeDependencyGraph { } } - private def githubToken(): String = Properties.envOrElse("GITHUB_TOKEN", "") + private def githubToken(): String = Properties.envOrElse("GITHUB_TOKEN", "") } From 598d6282a11d961a8317dbbf82947a79c8e85270 Mon Sep 17 00:00:00 2001 From: yazgoo Date: Tue, 5 Nov 2024 12:13:41 +0100 Subject: [PATCH 26/28] fmt --- .../ch/epfl/scala/AnalyzeDependencyGraph.scala | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala b/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala index 948f815..e470bfb 100644 --- a/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala +++ b/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala @@ -163,15 +163,15 @@ object AnalyzeDependencyGraph { private def vulnerabilityMatchesArtifacts( alert: Vulnerability, artifacts: Seq[String] - ): (Seq[String], Seq[String]) = { - val alertMavenPath = s"pkg:maven/${alert.packageId.replace(":", "/")}@" - artifacts - .filter(_.startsWith(alertMavenPath)) - .partition { artifact => - val version = artifact.replaceAll(".*@", "") - versionMatchesRange(version, alert.vulnerableVersionRange) - } - } + ): (Seq[String], Seq[String]) = { + val alertMavenPath = s"pkg:maven/${alert.packageId.replace(":", "/")}@" + artifacts + .filter(_.startsWith(alertMavenPath)) + .partition { artifact => + val version = artifact.replaceAll(".*@", "") + versionMatchesRange(version, alert.vulnerableVersionRange) + } + } def getGitHubRepo: Option[String] = { val remoteUrl = "git config --get remote.origin.url".!!.trim From 486272108f8c058e8846208d390bd0c089d5f12a Mon Sep 17 00:00:00 2001 From: yazgoo Date: Mon, 25 Nov 2024 11:11:53 +0100 Subject: [PATCH 27/28] =?UTF-8?q?=F0=9F=91=8Creview?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../scala/ch/epfl/scala/AnalyzeDependencyGraph.scala | 11 ++++++----- .../ch/epfl/scala/GithubDependencyGraphPlugin.scala | 1 - 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala b/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala index e470bfb..693c76d 100644 --- a/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala +++ b/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala @@ -52,8 +52,8 @@ object AnalyzeDependencyGraph { } }.failOnException - private def analyzeDependencies(state: State, params: AnalysisParams): State = - (for { + private def analyzeDependencies(state: State, params: AnalysisParams): State = { + for { repo <- params.repository.orElse(getGitHubRepo) vulnerabilities <- downloadAlerts(state, repo) match { case Success(v) => Some(v) @@ -61,9 +61,11 @@ object AnalyzeDependencyGraph { state.log.error(s"Failed to download alerts: ${e.getMessage}") None } - } yield analyzeCves(state, vulnerabilities)).getOrElse(state) + } yield analyzeCves(state, vulnerabilities) + state + } - private def analyzeCves(state: State, vulnerabilities: Seq[Vulnerability]): State = { + private def analyzeCves(state: State, vulnerabilities: Seq[Vulnerability]): Unit = { val artifacts = getAllArtifacts(state) vulnerabilities.foreach { v => val (goodMatches, badMatches) = vulnerabilityMatchesArtifacts(v, artifacts) @@ -75,7 +77,6 @@ object AnalyzeDependencyGraph { println(" 🎉 no match (dependency was probably removed)") } } - state } private def getStateOrWarn[T](state: State, key: AttributeKey[T], what: String, command: String): Option[T] = diff --git a/sbt-plugin/src/main/scala/ch/epfl/scala/GithubDependencyGraphPlugin.scala b/sbt-plugin/src/main/scala/ch/epfl/scala/GithubDependencyGraphPlugin.scala index bf26e7f..975c3fa 100644 --- a/sbt-plugin/src/main/scala/ch/epfl/scala/GithubDependencyGraphPlugin.scala +++ b/sbt-plugin/src/main/scala/ch/epfl/scala/GithubDependencyGraphPlugin.scala @@ -32,7 +32,6 @@ object GithubDependencyGraphPlugin extends AutoPlugin { val githubManifestsKey: AttributeKey[Map[String, githubapi.Manifest]] = AttributeKey("githubDependencyManifests") val githubProjectsKey: AttributeKey[Seq[ProjectRef]] = AttributeKey("githubProjectRefs") val githubSnapshotFileKey: AttributeKey[File] = AttributeKey("githubSnapshotFile") - val githubAlertsKey: AttributeKey[Seq[AnalyzeDependencyGraph.Vulnerability]] = AttributeKey("githubAlerts") val githubDependencyManifest: TaskKey[Option[githubapi.Manifest]] = taskKey( "The dependency manifest of the project" From 5a403ebcfbbd054ebbaf7d90a16d940548a037ec Mon Sep 17 00:00:00 2001 From: yazgoo Date: Tue, 26 Nov 2024 09:43:26 +0100 Subject: [PATCH 28/28] scalafmt --- .../src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala b/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala index 693c76d..bbc5759 100644 --- a/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala +++ b/sbt-plugin/src/main/scala/ch/epfl/scala/AnalyzeDependencyGraph.scala @@ -61,7 +61,7 @@ object AnalyzeDependencyGraph { state.log.error(s"Failed to download alerts: ${e.getMessage}") None } - } yield analyzeCves(state, vulnerabilities) + } yield analyzeCves(state, vulnerabilities) state }