Merge pull request clearlydefined#537 from lumaxis/update-scancode-to…

…olkit

Update scancode-toolkit to latest version

.prettierignore

@@ -0,0 +1,5 @@
+.github/workflows/*
+.vscode/launch.json
+schemas/*.json
+test/**/*.json
+test/**/*.yaml

.vscode/launch.json

@@ -14,12 +14,16 @@
       "name": "ClearlyDefined",
       "program": "${workspaceRoot}/index.js",
       "cwd": "${workspaceRoot}",
-      "runtimeArgs": ["--nolazy"],
+      "runtimeArgs": [
+        "--nolazy"
+      ],
       "env": {
         "NODE_ENV": "localhost",
         "PORT": "5000"
       },
-      "skipFiles": ["<node_internals>/**/*.js"],
+      "skipFiles": [
+        "<node_internals>/**/*.js"
+      ],
       "showAsyncStacks": true
     },
     {
@@ -28,15 +32,23 @@
       "name": "Mocha",
       "program": "${workspaceRoot}/node_modules/mocha/bin/_mocha",
       "stopOnEntry": false,
-      "args": ["${workspaceRoot}/test/**/*.js", "--timeout", "999999"],
+      "args": [
+        "${workspaceRoot}/test/**/*.js",
+        "--timeout",
+        "999999"
+      ],
       "cwd": "${workspaceRoot}",
       "runtimeExecutable": null,
-      "runtimeArgs": ["--nolazy"],
+      "runtimeArgs": [
+        "--nolazy"
+      ],
       "env": {
         "NODE_ENV": "localhost"
       },
       "console": "internalConsole",
-      "skipFiles": ["<node_internals>/**/*.js"],
+      "skipFiles": [
+        "<node_internals>/**/*.js"
+      ],
       "showAsyncStacks": true
     }
   ]

DevDockerfile

@@ -16,12 +16,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends --no-install-su
   gem install bundler -v 2.5.4 --no-document
 
 # Scancode
-ARG SCANCODE_VERSION="30.1.0"
+ARG SCANCODE_VERSION="32.1.0"
 RUN pip3 install --upgrade pip setuptools wheel && \
   curl -Os https://raw.githubusercontent.com/nexB/scancode-toolkit/v$SCANCODE_VERSION/requirements.txt && \
   pip3 install --constraint requirements.txt scancode-toolkit==$SCANCODE_VERSION && \
   rm requirements.txt && \
-  scancode --reindex-licenses && \
+  scancode-reindex-licenses && \
   scancode --version
 
 ENV SCANCODE_HOME=/usr/local/bin

Dockerfile

@@ -1,14 +1,8 @@
 # Copyright (c) Microsoft Corporation and others. Licensed under the MIT license.
 # SPDX-License-Identifier: MIT
 
-#FROM fossology/fossology:3.4.0 as fossology
-#COPY fossology_init.sh fossology_init.sh
-#RUN ./fossology_init.sh
-
 FROM node:18-bullseye
 ENV APPDIR=/opt/service
-#RUN apk update && apk upgrade && \
-#    apk add --no-cache bash git openssh
 
 ARG BUILD_NUMBER=0
 ENV CRAWLER_BUILD_NUMBER=$BUILD_NUMBER
@@ -22,12 +16,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends --no-install-su
   gem install bundler -v 2.5.4 --no-document
 
 # Scancode
-ARG SCANCODE_VERSION="30.1.0"
+ARG SCANCODE_VERSION="32.1.0"
 RUN pip3 install --upgrade pip setuptools wheel && \
   curl -Os https://raw.githubusercontent.com/nexB/scancode-toolkit/v$SCANCODE_VERSION/requirements.txt && \
   pip3 install --constraint requirements.txt scancode-toolkit==$SCANCODE_VERSION && \
   rm requirements.txt && \
-  scancode --reindex-licenses && \
+  scancode-reindex-licenses && \
   scancode --version
 
 ENV SCANCODE_HOME=/usr/local/bin
@@ -45,32 +39,6 @@ RUN gem install nokogiri:1.16.0 --no-document && \
 RUN pip3 install setuptools
 RUN pip3 install reuse==3.0.1
 
-# FOSSology
-# WORKDIR /opt
-# RUN git clone https://github.com/fossology/fossology.git
-# RUN cd fossology && git checkout -b clearlydefined tags/3.4.0
-
-# See https://github.com/fossology/fossology/blob/faaaeedb9d08f00def00f9b8a68a5cffc5eaa657/utils/fo-installdeps#L103-L105
-# Additional libjsoncpp-dev https://github.com/fossology/fossology/blob/261d1a3e663b5fd20652a05b2d6360f4b31a17cb/src/copyright/mod_deps#L79-L80
-# RUN apt-get update && apt-get install -y --no-install-recommends --no-install-suggests \
-#  libmxml-dev curl libxml2-dev libcunit1-dev libjsoncpp-dev \
-#  build-essential libtext-template-perl subversion rpm librpm-dev libmagic-dev libglib2.0 libboost-regex-dev libboost-program-options-dev
-
-# WORKDIR /opt/fossology/src/nomos/agent
-# RUN make -f Makefile.sa
-# RUN echo $(./nomossa -V)
-
-# NOTE: must build copyright before Monk to cause libfossology to be built
-# WORKDIR /opt/fossology/src/copyright/agent
-# RUN make
-
-# WORKDIR /opt/fossology/src/monk/agent
-# RUN make
-# RUN echo $(./monk -V)
-# COPY --from=fossology /tmp/monk_knowledgebase .
-
-# ENV FOSSOLOGY_HOME=/opt/fossology/src
-
 # Crawler config
 ENV CRAWLER_DEADLETTER_PROVIDER=cd(azblob)
 ENV CRAWLER_NAME=cdcrawlerprod
@@ -84,7 +52,7 @@ RUN git config --global --add safe.directory '*'
 
 COPY package*.json /tmp/
 COPY patches /tmp/patches
-RUN cd /tmp && npm install --production
+RUN cd /tmp && npm install --omit=dev
 RUN mkdir -p "${APPDIR}" && cp -a /tmp/node_modules "${APPDIR}"
 
 WORKDIR "${APPDIR}"

config/cdConfig.js

@@ -77,20 +77,21 @@ module.exports = {
       installDir: config.get('SCANCODE_HOME'),
       options: [
         '--copyright',
-        '--license',
         '--info',
-        '--license-text',
-        '--is-license-text',
         '--package',
-        '--license-text-diagnostics',
         '--strip-root',
         '--email',
         '--url',
-        '--license-clarity-score',
         '--classify',
         '--generated',
+        '--license',
+        '--license-clarity-score',
+        '--license-references',
+        '--license-text',
+        '--license-text-diagnostics',
         '--summary',
-        '--summary-key-files'
+        '--tallies',
+        '--tallies-key-files'
         // '--quiet'
       ],
       timeout: 1000,

providers/process/scancode.js

@@ -72,7 +72,7 @@ class ScanCodeProcessor extends AbstractProcessor {
     // Pick files that represent whole packages. We can be reasonably agressive here
     // and the summarizers etc will further refine what makes it into the final definitions
     const packages = output.files.reduce((result, file) => {
-      file.packages.forEach(entry => {
+      file.package_data.forEach(entry => {
         // in this case the manifest_path contains a subpath pointing to the corresponding file
         if (file.type === 'directory' && entry.manifest_path)
           result.push(`${file.path ? file.path + '/' : ''}${entry.manifest_path}`)
@@ -113,8 +113,8 @@ class ScanCodeProcessor extends AbstractProcessor {
         this.logger.info('Detecting ScanCode version')
 
         const raw_output = result.stdout
-        const scancode_line = raw_output.match(/ScanCode version .*\n/)[0]
-        this._toolVersion = scancode_line.replace('ScanCode version ', '').trim()
+        const scancode_line = raw_output.match(/ScanCode version: .*\n/)[0]
+        this._toolVersion = scancode_line.replace('ScanCode version: ', '').trim()
         this._schemaVersion = this.aggregateVersions(
           [this._schemaVersion, this.toolVersion, this.configVersion],
           'Invalid ScanCode version'