From e332db60cc0189f9d5753905267a3c2abbfba7f9 Mon Sep 17 00:00:00 2001 From: Sean Corfield Date: Sun, 10 Dec 2023 14:56:01 -0800 Subject: [PATCH 1/4] update dependencies --- .gitignore | 1 + deps.edn | 17 +++++++++-------- resources/dependency-check.properties | 19 ++++++++++++++++--- src/clj_watson/controller/deps.clj | 6 +++--- src/clj_watson/controller/remediate.clj | 4 ++-- src/clj_watson/diplomat/dependency.clj | 10 +++++----- 6 files changed, 36 insertions(+), 21 deletions(-) diff --git a/.gitignore b/.gitignore index 6d9e405..3cde320 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,4 @@ pom.xml.asc .clj-kondo/*/ target/ .lsp/ +.portal/ diff --git a/deps.edn b/deps.edn index 12c3164..085156c 100644 --- a/deps.edn +++ b/deps.edn @@ -1,16 +1,17 @@ {:deps {org.clojure/clojure {:mvn/version "1.11.1"} version-clj/version-clj {:mvn/version "2.0.2"} clj-http/clj-http {:mvn/version "3.12.3"} - cheshire/cheshire {:mvn/version "5.11.0"} + cheshire/cheshire {:mvn/version "5.12.0"} cli-matic/cli-matic {:mvn/version "0.5.4"} clj-time/clj-time {:mvn/version "0.15.2"} - selmer/selmer {:mvn/version "1.12.55"} - org.slf4j/slf4j-nop {:mvn/version "2.0.6"} - borkdude/edamame {:mvn/version "1.0.16"} - org.clojure/tools.deps.alpha {:mvn/version "0.15.1254"} - org.owasp/dependency-check-core {:mvn/version "7.4.4"} - org.apache.maven.resolver/maven-resolver-transport-http {:mvn/version "1.9.2"}} - + selmer/selmer {:mvn/version "1.12.59"} + org.slf4j/slf4j-nop {:mvn/version "2.0.9"} + borkdude/edamame {:mvn/version "1.3.23"} + ;; TODO: update to non-alpha + org.clojure/tools.deps {:mvn/version "0.18.1374"} + org.owasp/dependency-check-core {:mvn/version "9.0.4"} + org.apache.maven.resolver/maven-resolver-transport-http {:mvn/version "1.9.18"}} + :mvn/repos {"central" {:url "https://repo1.maven.org/maven2/"} "clojars" {:url "https://repo.clojars.org/"}} diff --git a/resources/dependency-check.properties b/resources/dependency-check.properties index 5c5c10c..cc30e97 100644 --- a/resources/dependency-check.properties +++ b/resources/dependency-check.properties @@ -1,5 +1,6 @@ -odc.application.name=${pom.name} -odc.application.version=${pom.version} +# replaced ${pom.*} with actual values: +odc.application.name=clj-watson +odc.application.version=4.1.3 odc.autoupdate=true odc.analysis.timeout=30 odc.settings.mask=.*password.*,.*token.* @@ -20,6 +21,18 @@ data.driver_name=org.h2.Driver proxy.disableSchemas=true +# nvd.api.key must be provided by the user: +#nvd.api.key=... +nvd.api.check.validforhours=12 +nvd.api.datafeed.startyear=2002 +nvd.api.datafeed.validfordays=7 +nvd.api.delay=2000 +nvd.api.max.retry.count=10 +# unused nvd.api.* keys: +#nvd.api.datafeed.url= +#nvd.api.datafeed.user= +#nvd.api.datafeed.password= + cve.url.modified.validfordays=7 cve.check.validforhours=12 cve.startyear=2002 @@ -103,4 +116,4 @@ analyzer.vulnerabilitysuppression.enabled=true updater.nvdcve.enabled=true updater.versioncheck.enabled=true analyzer.versionfilter.enabled=true -analyzer.artifactory.enabled=false \ No newline at end of file +analyzer.artifactory.enabled=false diff --git a/src/clj_watson/controller/deps.clj b/src/clj_watson/controller/deps.clj index cc10f39..b30a29a 100644 --- a/src/clj_watson/controller/deps.clj +++ b/src/clj_watson/controller/deps.clj @@ -1,8 +1,8 @@ (ns clj-watson.controller.deps (:require [clojure.set :refer [rename-keys]] - [clojure.tools.deps.alpha :as deps] - [clojure.tools.deps.alpha.util.maven :as maven] + [clojure.tools.deps :as deps] + [clojure.tools.deps.util.maven :as maven] [edamame.core :refer [parse-string]]) (:import (java.io File))) @@ -50,4 +50,4 @@ (dependencies-map->dependencies-vector dependencies-physical-location))})) (comment - (parse "resources/vulnerable-deps.edn" nil)) \ No newline at end of file + (parse "resources/vulnerable-deps.edn" nil)) diff --git a/src/clj_watson/controller/remediate.clj b/src/clj_watson/controller/remediate.clj index 8383c0d..c184700 100644 --- a/src/clj_watson/controller/remediate.clj +++ b/src/clj_watson/controller/remediate.clj @@ -2,7 +2,7 @@ (:require [clj-watson.diplomat.dependency :as diplomat.dependency] [clj-watson.logic.dependency :as logic.dependency] - [clojure.tools.deps.alpha.util.maven :as maven] + [clojure.tools.deps.util.maven :as maven] [version-clj.core :as version])) (defn ^:private parent-contains-child-version? @@ -62,4 +62,4 @@ :firstPatchedVersion {:identifier "2.14.2"}}], :secure-version {:mvn/version "2.14.2"}}]) - (scan vulnerable-dependencies {:mvn/repos maven/standard-repos})) \ No newline at end of file + (scan vulnerable-dependencies {:mvn/repos maven/standard-repos})) diff --git a/src/clj_watson/diplomat/dependency.clj b/src/clj_watson/diplomat/dependency.clj index 98fa122..9661a40 100644 --- a/src/clj_watson/diplomat/dependency.clj +++ b/src/clj_watson/diplomat/dependency.clj @@ -1,9 +1,9 @@ (ns clj-watson.diplomat.dependency (:require - [clojure.tools.deps.alpha :as deps] - [clojure.tools.deps.alpha.extensions :as ext] - [clojure.tools.deps.alpha.extensions.git :as git] - [clojure.tools.deps.alpha.util.maven :as maven] + [clojure.tools.deps :as deps] + [clojure.tools.deps.extensions :as ext] + [clojure.tools.deps.extensions.git :as git] + [clojure.tools.deps.util.maven :as maven] [clojure.tools.gitlibs :as gitlibs])) (defn ^:private append-sha-when-is-git-version [dependency version] @@ -47,4 +47,4 @@ (get-latest-version! 'org.clojure/clojure {:mvn/repos maven/standard-repos}) (get-latest-version! 'io.github.clj-holmes/clj-watson {:mvn/repos maven/standard-repos}) (resolve-dependency! {:deps {'io.github.clj-holmes/clj-watson {:git/tag "v2.1.3" :git/sha "19636f2"}} - :mvn/repos maven/standard-repos})) \ No newline at end of file + :mvn/repos maven/standard-repos})) From 1fb77898948d4dadee075b6d31f4e1384a05dd5a Mon Sep 17 00:00:00 2001 From: Sean Corfield Date: Sun, 10 Dec 2023 15:06:15 -0800 Subject: [PATCH 2/4] remove todo comment --- deps.edn | 1 - 1 file changed, 1 deletion(-) diff --git a/deps.edn b/deps.edn index 085156c..f21a279 100644 --- a/deps.edn +++ b/deps.edn @@ -7,7 +7,6 @@ selmer/selmer {:mvn/version "1.12.59"} org.slf4j/slf4j-nop {:mvn/version "2.0.9"} borkdude/edamame {:mvn/version "1.3.23"} - ;; TODO: update to non-alpha org.clojure/tools.deps {:mvn/version "0.18.1374"} org.owasp/dependency-check-core {:mvn/version "9.0.4"} org.apache.maven.resolver/maven-resolver-transport-http {:mvn/version "1.9.18"}} From de20312b84d7049dfba10b87b6e962ab9ffa876d Mon Sep 17 00:00:00 2001 From: Sean Corfield Date: Fri, 15 Dec 2023 22:31:23 -0800 Subject: [PATCH 3/4] add -w / --clj-watson-properties option this allows for a properties file that **merges** additional properties into the defaults, making it easier to customize watson's behavior --- README.md | 30 ++++++++++++++----- src/clj_watson/cli.clj | 4 +++ .../controller/dependency_check/scanner.clj | 17 ++++++----- src/clj_watson/entrypoint.clj | 7 +++-- 4 files changed, 42 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 4120679..a64241e 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ but you can allways allow a CVE for a limited period by adding a config file at ## Remediation suggestion #### The big difference from clj-watson to other tools. -Since fixing the found vulnerabilities manually could be truly frustrating `clj-watson` provides a way to suggest a remediation. +Since fixing the found vulnerabilities manually could be truly frustrating `clj-watson` provides a way to suggest a remediation. It basically lookups the whole dependency tree finding if the latest version of a parent dependency uses the secure version of the child dependency until it reaches the direct dependency. Given the following dependency tree, ``` @@ -51,7 +51,7 @@ In order to get the auto remediate suggestion it's necessary to provide a `--sug It's possible to install clj-watson as a clojure tool and invoke it. ```bash $ clojure -Ttools install io.github.clj-holmes/clj-watson '{:git/tag "v4.1.2" :git/sha "eb15492"}' :as clj-watson -$ clojure -Tclj-watson scan '{:output "stdout" :dependency-check-properties nil :fail-on-result true :deps-edn-path "deps.edn" :suggest-fix true :aliases ["*"] :database-strategy "dependency-check"}' +$ clojure -Tclj-watson scan '{:output "stdout" :fail-on-result true :deps-edn-path "deps.edn" :suggest-fix true :aliases ["*"] :database-strategy "dependency-check"}' ``` It can also be called directly. ```bash @@ -67,7 +67,7 @@ Or you can just add it to your project `deps.edn` # CLI Options ```bash -$ clojure -M:clj-watson scan -\? +$ clojure -M:clj-watson scan -\? NAME: clj-watson scan - Performs a scan on a deps.edn file @@ -79,12 +79,28 @@ OPTIONS: -o, --output edn|json|stdout|stdout-simple|sarif report Output type. -a, --aliases S Specify a alias that will have the dependencies analysed alongside with the project deps.It's possible to provide multiple aliases. If a * is provided all the aliases are going to be analysed. -d, --dependency-check-properties S [ONLY APPLIED IF USING DEPENDENCY-CHECK STRATEGY] Path of a dependency-check properties file. If not provided uses resources/dependency-check.properties. + -w, --clj-watson-properties S [ONLY APPLIED IF USING DEPENDENCY-CHECK STRATEGY] Path of an additional, optional properties file. -t, --database-strategy dependency-check|github-advisory dependency-check Vulnerability database strategy. -s, --[no-]suggest-fix Suggest a new deps.edn file fixing all vulnerabilities found. -f, --[no-]fail-on-result Enable or disable fail if results were found (useful for CI/CD). -?, --help ``` +By default, when using the DEPENDENCY-CHECK strategy, clj-watson will load +its own `dependency-check.properties` file, and then look for a +`clj-watson.properties` file on the classpath and load that if found, for +additional properties to apply to the dependency-check scan. + +If you provide `-d` (or `--dependency-check-properties`) then clj-watson will +load that file instead of its own `dependency-check.properties` file so it +needs to be a complete properties file, not just the properties you want to +override. + +If you provide `-w` (or `--clj-watson-properties`) then clj-watson will load +that file and apply those properties to the dependency-check scan. This is +in addition to the properties loaded from the `dependency-check.properties` +or the `-d` file. This can be useful to override just a few properties. + # Execution The minimum necessary to execute clj-watson is to provide the path to a `deps.edn` file, but it's recommended that you all provide the `-s` option so `clj-watson` will try to provide a remediation suggestion to the vulnerabilities. @@ -113,21 +129,21 @@ Vulnerabilities CVE: CVE-2022-1000000 CVSSV3: 7.5 CVSSV2: 5.0 -SUGGESTED BUMP: 1.55 +SUGGESTED BUMP: 1.55 CVE: CVE-2022-2000000 CVSSV3: 5.3 CVSSV2: 5.0 -SUGGESTED BUMP: 1.55 +SUGGESTED BUMP: 1.55 @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ ``` # Who uses it - [180 Seguros](https://180s.com.br) - [World Singles Networks](https://worldsinglesnetworks.com/) - + # Development ## nREPL -``` +``` clj -M:nREPL -m nrepl.cmdline ``` ## Build diff --git a/src/clj_watson/cli.clj b/src/clj_watson/cli.clj index b358ea8..37dea5c 100644 --- a/src/clj_watson/cli.clj +++ b/src/clj_watson/cli.clj @@ -25,6 +25,10 @@ :type :string :default nil :as "[ONLY APPLIED IF USING DEPENDENCY-CHECK STRATEGY] Path of a dependency-check properties file. If not provided uses resources/dependency-check.properties."} + {:option "clj-watson-properties" :short "w" + :type :string + :default nil + :as "[ONLY APPLIED IF USING DEPENDENCY-CHECK STRATEGY] Path of an additional, optional properties file."} {:option "database-strategy" :short "t" :type #{"dependency-check" "github-advisory"} :default "dependency-check" diff --git a/src/clj_watson/controller/dependency_check/scanner.clj b/src/clj_watson/controller/dependency_check/scanner.clj index efb2599..cc233bf 100644 --- a/src/clj_watson/controller/dependency_check/scanner.clj +++ b/src/clj_watson/controller/dependency_check/scanner.clj @@ -14,15 +14,18 @@ (.doUpdates engine) (println "Download/Update completed."))) -(defn ^:private create-settings [^String properties-file-path] +(defn ^:private create-settings [^String properties-file-path ^String additional-properties-file-path] (let [settings (Settings.)] (if properties-file-path (->> properties-file-path File. (.mergeProperties settings)) (->> "dependency-check.properties" io/resource slurp .getBytes ByteArrayInputStream. (.mergeProperties settings))) + (when additional-properties-file-path + (->> additional-properties-file-path File. (.mergeProperties settings)) + (some->> "clj-watson.properties" io/resource slurp .getBytes ByteArrayInputStream. (.mergeProperties settings))) settings)) -(defn ^:private build-engine [dependency-check-properties] - (let [settings (create-settings dependency-check-properties) +(defn ^:private build-engine [dependency-check-properties clj-watson-properties] + (let [settings (create-settings dependency-check-properties clj-watson-properties) engine (Engine. settings)] (update-download-database engine) engine)) @@ -30,8 +33,8 @@ (defn ^:private clojure-file? [dependency-path] (string/ends-with? dependency-path ".jar")) -(defn ^:private scan-jars [dependencies dependency-check-properties] - (let [engine (build-engine dependency-check-properties)] +(defn ^:private scan-jars [dependencies dependency-check-properties clj-watson-properties] + (let [engine (build-engine dependency-check-properties clj-watson-properties)] (->> dependencies (map :paths) (apply concat) @@ -41,7 +44,7 @@ (.analyzeDependencies engine) engine)) -(defn start! [dependencies dependency-check-properties] - (let [engine (scan-jars dependencies dependency-check-properties) +(defn start! [dependencies dependency-check-properties clj-watson-properties] + (let [engine (scan-jars dependencies dependency-check-properties clj-watson-properties) scanned-dependencies (->> engine .getDependencies Arrays/asList)] scanned-dependencies)) diff --git a/src/clj_watson/entrypoint.clj b/src/clj_watson/entrypoint.clj index aa5a898..78cdcea 100644 --- a/src/clj_watson/entrypoint.clj +++ b/src/clj_watson/entrypoint.clj @@ -22,10 +22,13 @@ (controller.remediate/scan vulnerable-dependencies deps) vulnerable-dependencies))) -(defmethod scan* :dependency-check [{:keys [deps-edn-path suggest-fix aliases dependency-check-properties]}] +(defmethod scan* :dependency-check [{:keys [deps-edn-path suggest-fix aliases + dependency-check-properties clj-watson-properties]}] (let [{:keys [deps dependencies]} (controller.deps/parse deps-edn-path aliases) repositories (select-keys deps [:mvn/repos]) - scanned-dependencies (controller.dc.scanner/start! dependencies dependency-check-properties) + scanned-dependencies (controller.dc.scanner/start! dependencies + dependency-check-properties + clj-watson-properties) vulnerable-dependencies (controller.dc.vulnerability/extract scanned-dependencies dependencies repositories)] (if suggest-fix (controller.remediate/scan vulnerable-dependencies deps) From 9adcb3071b340e0ab7c69b1e6a553db7d91f9ebc Mon Sep 17 00:00:00 2001 From: Sean Corfield Date: Sat, 16 Dec 2023 23:25:15 -0800 Subject: [PATCH 4/4] move toward v5.0.0 this uses the 9.0.x DependencyCheck and requires an API key provided via the clj-watson.properties file --- README.md | 22 ++++++++++++++++++++++ deps.edn | 14 +++++++------- resources/dependency-check.properties | 2 +- 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index a64241e..4fe0eab 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,28 @@ clj-watson supports two methods for vulnerabilities scan. ### dependency-check [dependency-check](https://github.com/jeremylong/DependencyCheck) is the most used method around the clojure/java sca tools, it downloads all vulnerabilities from nvd and stores it in a database (located in `/tmp/db`), compose a [cpe](https://nvd.nist.gov/products/cpe) based on the dependencies, scans all jars in the classpath and matches vulnerabilities using it. +#### NIST NVD API + +As of version v5.0.0, `clj-watson` uses +[`DependencyCheck` 9.0.x](https://github.com/jeremylong/DependencyCheck/tree/main?tab=readme-ov-file#900-upgrade-notice) +which switches from the earlier NVD data feeds to the new NIST NVD API. + +This new API heavily throttles anonymous requests, so it is +[highly recommended to get an API key](https://github.com/jeremylong/DependencyCheck/tree/main?tab=readme-ov-file#nvd-api-key-highly-recommended) +in order to use the API efficiently. + +Read the [NIST NVD announcement](https://nvd.nist.gov/general/news/API-Key-Announcement) for more information. + +Once you have an API key, you can provide it to `clj-watson` via the `nvd.api.key` +property in the optional `clj-watson.properties` file, either on the classpath +you use to run `clj-watson` or via the `-w` / `--clj-watson-properties` +command-line option: + +``` +# clj-watson.properties file +nvd.api.key=...your key here... +``` + ### Github advisory database [experimental] It doesn't need to download a database since it uses the [github advisory database](https://github.com/advisories) via the [graphql api](https://docs.github.com/en/graphql/reference/objects#securityvulnerability), matches are made via package name. But there's a requirements to use it, it's necessary to generate a [Github PAT (Personal Access Token)](https://docs.github.com/en/graphql/guides/forming-calls-with-graphql#authenticating-with-graphql) to access graphql api or if you use Github actions it's possible to use their Github token. diff --git a/deps.edn b/deps.edn index f21a279..2a2d455 100644 --- a/deps.edn +++ b/deps.edn @@ -1,15 +1,15 @@ {:deps {org.clojure/clojure {:mvn/version "1.11.1"} - version-clj/version-clj {:mvn/version "2.0.2"} - clj-http/clj-http {:mvn/version "3.12.3"} + borkdude/edamame {:mvn/version "1.3.23"} cheshire/cheshire {:mvn/version "5.12.0"} cli-matic/cli-matic {:mvn/version "0.5.4"} + clj-http/clj-http {:mvn/version "3.12.3"} clj-time/clj-time {:mvn/version "0.15.2"} - selmer/selmer {:mvn/version "1.12.59"} - org.slf4j/slf4j-nop {:mvn/version "2.0.9"} - borkdude/edamame {:mvn/version "1.3.23"} + org.apache.maven.resolver/maven-resolver-transport-http {:mvn/version "1.9.18"} org.clojure/tools.deps {:mvn/version "0.18.1374"} - org.owasp/dependency-check-core {:mvn/version "9.0.4"} - org.apache.maven.resolver/maven-resolver-transport-http {:mvn/version "1.9.18"}} + org.owasp/dependency-check-core {:mvn/version "9.0.6"} + org.slf4j/slf4j-nop {:mvn/version "2.0.9"} + selmer/selmer {:mvn/version "1.12.59"} + version-clj/version-clj {:mvn/version "2.0.2"}} :mvn/repos {"central" {:url "https://repo1.maven.org/maven2/"} "clojars" {:url "https://repo.clojars.org/"}} diff --git a/resources/dependency-check.properties b/resources/dependency-check.properties index cc30e97..f817b20 100644 --- a/resources/dependency-check.properties +++ b/resources/dependency-check.properties @@ -1,6 +1,6 @@ # replaced ${pom.*} with actual values: odc.application.name=clj-watson -odc.application.version=4.1.3 +odc.application.version=5.0.0 odc.autoupdate=true odc.analysis.timeout=30 odc.settings.mask=.*password.*,.*token.*