From 67615345c349be3e4ab0ae5091739395d6bee574 Mon Sep 17 00:00:00 2001 From: Greg Guthe Date: Mon, 24 Sep 2018 16:35:23 -0400 Subject: [PATCH 1/7] readme: remove reference to nsp --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6b02864643..5b958b7aea 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ We apologize but we have no story for development on Windows (though the add-on ### Linting and Testing -`npm run test` will run tests as well as eslint and [nsp](https://www.npmjs.com/package/nsp). You can control the tests with the following shell/environment variables: +`npm run test` will run tests as well as eslint. You can control the tests with the following shell/environment variables: - `MOZ_HEADLESS` - when this variable is set, the Selenium tests will run in [headless mode](https://developer.mozilla.org/en-US/Firefox/Headless_mode). - `SCREENSHOTS_BACKEND` - the server where the addon will try to save shots if the default http://localhost:10080 is not available or desirable. From 17cfc1e9c24ebb838a5b212f5a607d6df0011428 Mon Sep 17 00:00:00 2001 From: Greg Guthe Date: Mon, 24 Sep 2018 16:41:20 -0400 Subject: [PATCH 2/7] remove comment from .nsprc to make it valid JSON --- .nsprc | 1 - 1 file changed, 1 deletion(-) diff --git a/.nsprc b/.nsprc index 9ef8d4e9d8..3997606f8c 100644 --- a/.nsprc +++ b/.nsprc @@ -1,5 +1,4 @@ { - // See https://github.com/mozilla-services/screenshots/issues/4397 "exceptions": [ "https://nodesecurity.io/advisories/566", "https://nodesecurity.io/advisories/577", From 4f4082eeae560b604cde74b354be5f2e119568f0 Mon Sep 17 00:00:00 2001 From: Greg Guthe Date: Mon, 24 Sep 2018 17:03:38 -0400 Subject: [PATCH 3/7] npmrc: add exceptions for ReDoS in dev deps istanbul middleware --- .nsprc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.nsprc b/.nsprc index 3997606f8c..8b533e837b 100644 --- a/.nsprc +++ b/.nsprc @@ -1,5 +1,7 @@ { "exceptions": [ + "https://nodesecurity.io/advisories/118", + "https://nodesecurity.io/advisories/534", "https://nodesecurity.io/advisories/566", "https://nodesecurity.io/advisories/577", "https://nodesecurity.io/advisories/598", From a59fd1dfecf82c7d1ebea0792b8fa93c86d4ddfc Mon Sep 17 00:00:00 2001 From: Greg Guthe Date: Mon, 24 Sep 2018 17:10:15 -0400 Subject: [PATCH 4/7] nsprc: add exception for adm-zip file write advisory https://www.npmjs.com/advisories/681 at paths: jpm > firefox-profile > adm-zip web-ext > firefox-profile > adm-zip These 1) run from build tools that already have access to local the local filesystem and 2) do not take input files outside the repo --- .nsprc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.nsprc b/.nsprc index 8b533e837b..8762f89d5f 100644 --- a/.nsprc +++ b/.nsprc @@ -6,6 +6,7 @@ "https://nodesecurity.io/advisories/577", "https://nodesecurity.io/advisories/598", "https://nodesecurity.io/advisories/663", - "https://nodesecurity.io/advisories/664" + "https://nodesecurity.io/advisories/664", + "https://nodesecurity.io/advisories/681" ] } From 18321d41e9008408a16e66cc7e6010830fb13030 Mon Sep 17 00:00:00 2001 From: Greg Guthe Date: Mon, 24 Sep 2018 16:43:12 -0400 Subject: [PATCH 5/7] add npm lint:dep script to run npm audit with filter in CI --- bin/filtered_npm_audit.sh | 307 ++++++++++++++++++++++++++++++++++++++ package.json | 1 + 2 files changed, 308 insertions(+) create mode 100755 bin/filtered_npm_audit.sh diff --git a/bin/filtered_npm_audit.sh b/bin/filtered_npm_audit.sh new file mode 100755 index 0000000000..63900426f9 --- /dev/null +++ b/bin/filtered_npm_audit.sh @@ -0,0 +1,307 @@ +#!/bin/bash + +set -ue + +# This script does platform detection, downloads audit-filter, and runs it. +# +# Platform detection and dependent functions from rustup-init.sh which is: +# +# Copyright 2016 The Rust Project Developers. See the COPYRIGHT +# file at the top-level directory of this distribution and at +# http://rust-lang.org/COPYRIGHT. +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + + +need_cmd() { + if ! check_cmd "$1" + then err "need '$1' (command not found)" + fi +} + +check_cmd() { + command -v "$1" > /dev/null 2>&1 + return $? +} + +err() { + say "$1" >&2 + exit 1 +} + +say() { + echo "filtered_npm_audit.sh: $1" +} + +assert_nz() { + if [ -z "$1" ]; then err "assert_nz $2"; fi +} + +# This wraps curl or wget. Try curl first, if not installed, +# use wget instead. +downloader() { + if check_cmd curl + then _dld=curl + elif check_cmd wget + then _dld=wget + else _dld='curl or wget' # to be used in error message of need_cmd + fi + + if [ "$1" = --check ] + then need_cmd "$_dld" + elif [ "$_dld" = curl ] + then curl -sSfL "$1" -o "$2" + elif [ "$_dld" = wget ] + then wget "$1" -O "$2" + else err "Unknown downloader" # should not reach here + fi +} + +get_bitness() { + need_cmd head + # Architecture detection without dependencies beyond coreutils. + # ELF files start out "\x7fELF", and the following byte is + # 0x01 for 32-bit and + # 0x02 for 64-bit. + # The printf builtin on some shells like dash only supports octal + # escape sequences, so we use those. + local _current_exe_head=$(head -c 5 /proc/self/exe ) + if [ "$_current_exe_head" = "$(printf '\177ELF\001')" ]; then + echo 32 + elif [ "$_current_exe_head" = "$(printf '\177ELF\002')" ]; then + echo 64 + else + err "unknown platform bitness" + fi +} + +get_endianness() { + local cputype=$1 + local suffix_eb=$2 + local suffix_el=$3 + + # detect endianness without od/hexdump, like get_bitness() does. + need_cmd head + need_cmd tail + + local _current_exe_endianness="$(head -c 6 /proc/self/exe | tail -c 1)" + if [ "$_current_exe_endianness" = "$(printf '\001')" ]; then + echo "${cputype}${suffix_el}" + elif [ "$_current_exe_endianness" = "$(printf '\002')" ]; then + echo "${cputype}${suffix_eb}" + else + err "unknown platform endianness" + fi +} + +get_architecture() { + + local _ostype="$(uname -s)" + local _cputype="$(uname -m)" + + if [ "$_ostype" = Linux ]; then + if [ "$(uname -o)" = Android ]; then + local _ostype=Android + fi + fi + + if [ "$_ostype" = Darwin -a "$_cputype" = i386 ]; then + # Darwin `uname -s` lies + if sysctl hw.optional.x86_64 | grep -q ': 1'; then + local _cputype=x86_64 + fi + fi + + case "$_ostype" in + + Android) + local _ostype=linux-android + ;; + + Linux) + local _ostype=unknown-linux-gnu + ;; + + FreeBSD) + local _ostype=unknown-freebsd + ;; + + NetBSD) + local _ostype=unknown-netbsd + ;; + + DragonFly) + local _ostype=unknown-dragonfly + ;; + + Darwin) + local _ostype=apple-darwin + ;; + + MINGW* | MSYS* | CYGWIN*) + local _ostype=pc-windows-gnu + ;; + + *) + err "unrecognized OS type: $_ostype" + ;; + + esac + + case "$_cputype" in + + i386 | i486 | i686 | i786 | x86) + local _cputype=i686 + ;; + + xscale | arm) + local _cputype=arm + if [ "$_ostype" = "linux-android" ]; then + local _ostype=linux-androideabi + fi + ;; + + armv6l) + local _cputype=arm + if [ "$_ostype" = "linux-android" ]; then + local _ostype=linux-androideabi + else + local _ostype="${_ostype}eabihf" + fi + ;; + + armv7l | armv8l) + local _cputype=armv7 + if [ "$_ostype" = "linux-android" ]; then + local _ostype=linux-androideabi + else + local _ostype="${_ostype}eabihf" + fi + ;; + + aarch64) + local _cputype=aarch64 + ;; + + x86_64 | x86-64 | x64 | amd64) + local _cputype=x86_64 + ;; + + mips) + local _cputype="$(get_endianness $_cputype "" 'el')" + ;; + + mips64) + local _bitness="$(get_bitness)" + if [ $_bitness = "32" ]; then + if [ $_ostype = "unknown-linux-gnu" ]; then + # 64-bit kernel with 32-bit userland + # endianness suffix is appended later + local _cputype=mips + fi + else + # only n64 ABI is supported for now + local _ostype="${_ostype}abi64" + fi + + local _cputype="$(get_endianness $_cputype "" 'el')" + ;; + + ppc) + local _cputype=powerpc + ;; + + ppc64) + local _cputype=powerpc64 + ;; + + ppc64le) + local _cputype=powerpc64le + ;; + + *) + err "unknown CPU type: $_cputype" + + esac + + # Detect 64-bit linux with 32-bit userland + if [ $_ostype = unknown-linux-gnu -a $_cputype = x86_64 ]; then + if [ "$(get_bitness)" = "32" ]; then + local _cputype=i686 + fi + fi + + # Detect armv7 but without the CPU features Rust needs in that build, + # and fall back to arm. + # See https://github.com/rust-lang-nursery/rustup.rs/issues/587. + if [ $_ostype = "unknown-linux-gnueabihf" -a $_cputype = armv7 ]; then + if ensure grep '^Features' /proc/cpuinfo | grep -q -v neon; then + # At least one processor does not have NEON. + local _cputype=arm + fi + fi + + local _arch="$_cputype-$_ostype" + + RETVAL="$_arch" +} + +check_ci() { + if [ -z ${CI+0} ]; then + echo "Skipping dep lint. (set CI != 0 to run locally)"; + exit 0 + fi +} + +main() { + downloader --check + need_cmd uname + need_cmd chmod + need_cmd npm + check_ci + + get_architecture || return 1 + local _sha256 + local _arch="$RETVAL" + assert_nz "$_arch" "arch" + + say "Detected architecture ${_arch}" + if [[ $_arch == x86_64-unknown-linux* ]]; then + _arch=x86_64-unknown-linux-musl + _sha256=0f0cbb0edb582a2d6c74b44e10e7ad1abf00497a4afbe791cdca88cca3c1cbdf + elif [[ $_arch == i686-unknown-linux* ]]; then + _arch=i686-unknown-linux-musl + _sha256=916a41c405e05d720f90a8137e92783dc4bcff01108878980e0f32dd87160000 + elif [[ $_arch != x86_64-apple-darwin ]]; then + err "Unsupported architecture $_arch" + else + # x86_64-apple-darwin + _sha256=be977a1937e6eb4542ac19896a3ec100c6dab3cba58b1632779f8e4b1f57ef11 + fi + say "Fetching tarball for architecture ${_arch} w/ sha256: ${_sha256}" + + local _tag=0.2.5 + local _basename=audit-filter-${_tag}-${_arch} + local _file=${_basename}.tar.gz + local _url=https://github.com/mozilla-services/audit-filter/releases/download/${_tag}/${_basename}.tar.gz + + cd bin/ + test -f ${_file} || downloader "$_url" "$_file" + test -f ${_basename}/audit-filter || tar xvzf ${_file} ${_basename}/audit-filter > /dev/null + echo "${_sha256} ${_basename}/audit-filter" | sha256sum -c - > /dev/null || exit 1 + + chmod +x ${_basename}/audit-filter + say "Extracted bin/${_file}/audit-filter with expected sha256" + cd - + + local _npm=$(npm prefix -g)/bin/npm + say "Using npm at: ${_npm} with version: $($_npm --version)" + set -v + $_npm audit --json | bin/${_basename}/audit-filter --nsp-config .nsprc +} + +main "$@" || exit 1 diff --git a/package.json b/package.json index 54ca5ba294..3d574c7da9 100644 --- a/package.json +++ b/package.json @@ -95,6 +95,7 @@ "lint:flake8": "make flake8", "lint:js": "eslint .", "lint:sass": "sass-lint -v -q", + "lint:deps": "./bin/filtered_npm_audit.sh", "postlint": "npm audit || true", "posttest": "npm run lint", "test": "bin/test-helpers/make_addon_with_unavailable_server && mocha test/ test/server/unit/*", From f09f0925d3fbaf62e1f93c1fa65691b4f2f47b09 Mon Sep 17 00:00:00 2001 From: Greg Guthe Date: Tue, 25 Sep 2018 15:11:52 -0400 Subject: [PATCH 6/7] ci: install npm@6.4.1 for audit support --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 04440e70b2..81e81620f7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -63,6 +63,7 @@ jobs: command: | . "$NVM_DIR/nvm.sh" nvm use 8.10.0 + npm install -g npm@6.4.1 npm run test:server # Disable code coverage (temporarily): https://github.com/mozilla-services/screenshots/issues/4728 # mkdir -p $CIRCLE_TEST_REPORTS/server-coverage From e4ffef76a938c329bbbab7ff553dc4da588929ec Mon Sep 17 00:00:00 2001 From: Greg Guthe Date: Tue, 25 Sep 2018 15:59:55 -0400 Subject: [PATCH 7/7] nsprc: switch to npmjs.org/advisories to fix filtering --- .nsprc | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.nsprc b/.nsprc index 8762f89d5f..416eaedc4f 100644 --- a/.nsprc +++ b/.nsprc @@ -1,12 +1,12 @@ { "exceptions": [ - "https://nodesecurity.io/advisories/118", - "https://nodesecurity.io/advisories/534", - "https://nodesecurity.io/advisories/566", - "https://nodesecurity.io/advisories/577", - "https://nodesecurity.io/advisories/598", - "https://nodesecurity.io/advisories/663", - "https://nodesecurity.io/advisories/664", - "https://nodesecurity.io/advisories/681" + "https://npmjs.com/advisories/118", + "https://npmjs.com/advisories/534", + "https://npmjs.com/advisories/566", + "https://npmjs.com/advisories/577", + "https://npmjs.com/advisories/598", + "https://npmjs.com/advisories/663", + "https://npmjs.com/advisories/664", + "https://npmjs.com/advisories/681" ] }