From 0c3746ed5388c33142644c0fd65b6922710e6aac Mon Sep 17 00:00:00 2001 From: Facundo Tuesca Date: Wed, 20 Jul 2022 07:04:19 +0200 Subject: [PATCH] tools: add support for using API key to vuln checking script This change adds a new parameter `--nvd-key` to `dep_checker`, which allows the user to specify a NVD API key with which to query the National Vulnerability Database. This increases the rate at which we are allowed to query the database, which speeds up the running time of the script. PR-URL: https://github.com/nodejs/node/pull/43909 Reviewed-By: Matteo Collina Reviewed-By: Rafael Gonzaga Reviewed-By: Mohammed Keyvanzadeh Reviewed-By: Richard Lau Reviewed-By: Michael Dawson --- tools/dep_checker/README.md | 24 +++++++++++++++++------- tools/dep_checker/main.py | 18 +++++++++++++++--- tools/dep_checker/requirements.txt | 2 +- 3 files changed, 33 insertions(+), 11 deletions(-) diff --git a/tools/dep_checker/README.md b/tools/dep_checker/README.md index 69dde6badaba85..84e1fab640d992 100644 --- a/tools/dep_checker/README.md +++ b/tools/dep_checker/README.md @@ -6,9 +6,17 @@ in Node's dependencies. ## How to use -In order to query the GitHub Advisory Database, -a [Personal Access Token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) -has to be created (no permissions need to be given to the token, since it's only used to query the public database). +### Database authentication + +- In order to query the GitHub Advisory Database, + a [Personal Access Token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) + has to be created (no permissions need to be given to the token, since it's only used to query the public database). +- The NVD can be queried without authentication, but it will be rate limited to one query every six seconds. In order to + remove + that limitation [request an API key](https://nvd.nist.gov/developers/request-an-api-key) and pass it as a parameter. + +### Running the script + Once acquired, the script can be run as follows: ```shell @@ -16,9 +24,11 @@ cd node/tools/dep_checker/ pip install -r requirements.txt # Python >= 3.9 required -python main.py --gh-token=$PERSONAL_ACCESS_TOKEN +python main.py --gh-token=$PERSONAL_ACCESS_TOKEN --nvd-key=$NVD_API_KEY -# or to skip querying the GitHub Advisory Database, simply run: +# The command can also be run without parameters +# This will skip querying the GitHub Advisory Database, and query the NVD +# using the anonymous (rate-limited) API python main.py ``` @@ -51,8 +61,8 @@ non-affected version. - The queries can return false positives ( see [this](https://github.com/nodejs/security-wg/issues/802#issuecomment-1144207417) comment for an example). These can be ignored by adding the vulnerability to the `ignore_list` in `dependencies.py` -- The script takes a while to finish (~2 min) because queries to the NVD - are [rate-limited](https://nvd.nist.gov/developers) +- If no NVD API key is provided, the script will take a while to finish (~2 min) because queries to the NVD + are [rate-limited](https://nvd.nist.gov/developers/start-here) - If any vulnerabilities are found, the script returns 1 and prints out a list with the ID and a link to a description of the vulnerability. This is the case except when the ID matches one in the ignore-list (inside `dependencies.py`) in diff --git a/tools/dep_checker/main.py b/tools/dep_checker/main.py index cccb435f8389f4..6675f48f570cfb 100644 --- a/tools/dep_checker/main.py +++ b/tools/dep_checker/main.py @@ -18,6 +18,7 @@ from gql.transport.aiohttp import AIOHTTPTransport from nvdlib import searchCVE # type: ignore from packaging.specifiers import SpecifierSet +from typing import Optional class Vulnerability: @@ -105,7 +106,7 @@ def query_ghad(gh_token: str) -> dict[str, list[Vulnerability]]: return found_vulnerabilities -def query_nvd() -> dict[str, list[Vulnerability]]: +def query_nvd(api_key: Optional[str]) -> dict[str, list[Vulnerability]]: """Queries the National Vulnerability Database for vulnerabilities reported for Node's dependencies. The database supports querying by CPE (Common Platform Enumeration) or by a keyword present in the CVE's @@ -121,7 +122,9 @@ def query_nvd() -> dict[str, list[Vulnerability]]: for name, dep in deps_in_nvd.items(): query_results = [ cve - for cve in searchCVE(cpeMatchString=dep.get_cpe(), keyword=dep.keyword) + for cve in searchCVE( + cpeMatchString=dep.get_cpe(), keyword=dep.keyword, key=api_key + ) if cve.id not in ignore_list ] if query_results: @@ -140,15 +143,24 @@ def main(): "--gh-token", help="the GitHub authentication token for querying the GH Advisory Database", ) + parser.add_argument( + "--nvd-key", + help="the NVD API key for querying the National Vulnerability Database", + ) gh_token = parser.parse_args().gh_token + nvd_key = parser.parse_args().nvd_key if gh_token is None: print( "Warning: GitHub authentication token not provided, skipping GitHub Advisory Database queries" ) + if nvd_key is None: + print( + "Warning: NVD API key not provided, queries will be slower due to rate limiting" + ) ghad_vulnerabilities: dict[str, list[Vulnerability]] = ( {} if gh_token is None else query_ghad(gh_token) ) - nvd_vulnerabilities = query_nvd() + nvd_vulnerabilities: dict[str, list[Vulnerability]] = query_nvd(nvd_key) if not ghad_vulnerabilities and not nvd_vulnerabilities: print(f"No new vulnerabilities found ({len(ignore_list)} ignored)") diff --git a/tools/dep_checker/requirements.txt b/tools/dep_checker/requirements.txt index 894840287e861f..3a41c5824b7f06 100644 --- a/tools/dep_checker/requirements.txt +++ b/tools/dep_checker/requirements.txt @@ -1,3 +1,3 @@ gql[aiohttp] -nvdlib +nvdlib==0.5.8 packaging