chore: remove mocks from npm audit tests

tap-snapshots/test/lib/commands/audit.js.test.cjs

@@ -0,0 +1,76 @@
+/* IMPORTANT
+ * This snapshot file is auto-generated, but designed for humans.
+ * It should be checked into source control and tracked carefully.
+ * Re-generate by setting TAP_SNAPSHOT=1 and running tests.
+ * Make sure to inspect the output below.  Do not ignore changes!
+ */
+'use strict'
+exports[`test/lib/commands/audit.js TAP audit fix > must match snapshot 1`] = `
+
+added 1 package, and audited 2 packages in 120ms
+
+found 0 vulnerabilities
+`
+
+exports[`test/lib/commands/audit.js TAP json audit > must match snapshot 1`] = `
+{
+  "auditReportVersion": 2,
+  "vulnerabilities": {
+    "test-dep-a": {
+      "name": "test-dep-a",
+      "severity": "high",
+      "isDirect": true,
+      "via": [
+        {
+          "source": 100,
+          "name": "test-dep-a",
+          "dependency": "test-dep-a",
+          "title": "Test advisory 100",
+          "url": "https://github.com/advisories/GHSA-100",
+          "severity": "high",
+          "range": "*"
+        }
+      ],
+      "effects": [],
+      "range": "*",
+      "nodes": [
+        "node_modules/test-dep-a"
+      ],
+      "fixAvailable": false
+    }
+  },
+  "metadata": {
+    "vulnerabilities": {
+      "info": 0,
+      "low": 0,
+      "moderate": 0,
+      "high": 1,
+      "critical": 0,
+      "total": 1
+    },
+    "dependencies": {
+      "prod": 2,
+      "dev": 0,
+      "optional": 0,
+      "peer": 0,
+      "peerOptional": 0,
+      "total": 1
+    }
+  }
+}
+`
+
+exports[`test/lib/commands/audit.js TAP normal audit > must match snapshot 1`] = `
+# npm audit report
+
+test-dep-a  *
+Severity: high
+Test advisory 100 - https://github.com/advisories/GHSA-100
+No fix available
+node_modules/test-dep-a
+
+1 high severity vulnerability
+
+Some issues need review, and may require choosing
+a different dependency.
+`

test/fixtures/mock-registry.js

@@ -44,6 +44,24 @@ class MockRegistry {
     this.nock.get('/-/whoami').reply(200, { username })
   }
 
+  advisory (advisory = {}) {
+    const id = advisory.id || parseInt(Math.random() * 1000000)
+    return {
+      id,
+      url: `https://github.com/advisories/GHSA-${id}`,
+      title: `Test advisory ${id}`,
+      severity: 'high',
+      vulnerable_versions: '*',
+      cwe: [
+        'cwe-0',
+      ],
+      cvss: {
+        score: 0,
+      },
+      ...advisory,
+    }
+  }
+
   async package ({ manifest, times = 1, query, tarballs }) {
     let nock = this.nock
     nock = nock.get(`/${manifest.name}`).times(times)
@@ -52,7 +70,8 @@ class MockRegistry {
     }
     nock = nock.reply(200, manifest)
     if (tarballs) {
-      for (const version in manifest.versions) {
+      for (const version in tarballs) {
+      // for (const version in manifest.versions) {
         const packument = manifest.versions[version]
         const dist = new URL(packument.dist.tarball)
         const tarball = await pacote.tarball(tarballs[version])

test/lib/commands/audit.js

@@ -1,139 +1,148 @@
 const t = require('tap')
-const { load: _loadMockNpm } = require('../../fixtures/mock-npm')
 
-t.test('should audit using Arborist', async t => {
-  let ARB_ARGS = null
-  let AUDIT_CALLED = false
-  let REIFY_FINISH_CALLED = false
-  let AUDIT_REPORT_CALLED = false
-  let ARB_OBJ = null
+const { load: loadMockNpm } = require('../../fixtures/mock-npm')
+const MockRegistry = require('../../fixtures/mock-registry.js')
+const util = require('util')
+const zlib = require('zlib')
+const gzip = util.promisify(zlib.gzip)
+const path = require('path')
 
-  const loadMockNpm = (t) => _loadMockNpm(t, {
-    mocks: {
-      'npm-audit-report': () => {
-        AUDIT_REPORT_CALLED = true
-        return {
-          report: 'there are vulnerabilities',
-          exitCode: 0,
-        }
+const tree = {
+  'package.json': JSON.stringify({
+    name: 'test-dep',
+    version: '1.0.0',
+    dependencies: {
+      'test-dep-a': '*',
+    },
+  }),
+  'package-lock.json': JSON.stringify({
+    name: 'test-dep',
+    version: '1.0.0',
+    lockfileVersion: 2,
+    requires: true,
+    packages: {
+      '': {
+        xname: 'scratch',
+        version: '1.0.0',
+        dependencies: {
+          'test-dep-a': '*',
+        },
+        devDependencies: {},
       },
-      '@npmcli/arborist': function (args) {
-        ARB_ARGS = args
-        ARB_OBJ = this
-        this.audit = () => {
-          AUDIT_CALLED = true
-          this.auditReport = {}
-        }
+      'node_modules/test-dep-a': {
+        name: 'test-dep-a',
+        version: '1.0.0',
       },
-      '../../lib/utils/reify-finish.js': (npm, arb) => {
-        if (arb !== ARB_OBJ) {
-          throw new Error('got wrong object passed to reify-output')
-        }
-
-        REIFY_FINISH_CALLED = true
+    },
+    dependencies: {
+      'test-dep-a': {
+        version: '1.0.0',
       },
     },
-  })
+  }),
+  'test-dep-a': {
+    'package.json': JSON.stringify({
+      name: 'test-dep-a',
+      version: '1.0.1',
+    }),
+    'fixed.txt': 'fixed test-dep-a',
+  },
+}
 
-  t.test('audit', async t => {
-    const { npm, outputs } = await loadMockNpm(t)
-    await npm.exec('audit', [])
-    t.match(ARB_ARGS, { audit: true, path: npm.prefix })
-    t.equal(AUDIT_CALLED, true, 'called audit')
-    t.equal(AUDIT_REPORT_CALLED, true, 'called audit report')
-    t.match(outputs, [['there are vulnerabilities']])
+t.test('normal audit', async t => {
+  const { npm, joinedOutput } = await loadMockNpm(t, {
+    prefixDir: tree,
+  })
+  const registry = new MockRegistry({
+    tap: t,
+    registry: npm.config.get('registry'),
   })
 
-  t.test('audit fix', async t => {
-    const { npm } = await loadMockNpm(t)
-    await npm.exec('audit', ['fix'])
-    t.equal(REIFY_FINISH_CALLED, true, 'called reify output')
+  const manifest = registry.manifest({
+    name: 'test-dep-a',
+    packuments: [{ version: '1.0.0' }, { version: '1.0.1' }],
   })
+  await registry.package({ manifest })
+  const advisory = registry.advisory({ id: 100 })
+  const bulkBody = await gzip(JSON.stringify({ 'test-dep-a': ['1.0.0'] }))
+  registry.nock.post('/-/npm/v1/security/advisories/bulk', bulkBody)
+    .reply(200, {
+      'test-dep-a': [advisory],
+    })
+
+  await npm.exec('audit', [])
+  t.ok(process.exitCode, 'would have exited uncleanly')
+  process.exitCode = 0
+  t.matchSnapshot(joinedOutput())
 })
 
-t.test('should audit - json', async t => {
-  t.plan(1)
-  const { npm } = await _loadMockNpm(t, {
-    mocks: {
-      'npm-audit-report': (_, opts) => {
-        t.match(opts.reporter, 'json')
-        return {
-          report: 'there are vulnerabilities',
-          exitCode: 0,
-        }
-      },
-      '@npmcli/arborist': function () {
-        this.audit = () => {
-          this.auditReport = {}
-        }
-      },
-      '../../lib/utils/reify-output.js': () => {},
-    },
+t.test('json audit', async t => {
+  const { npm, joinedOutput } = await loadMockNpm(t, {
+    prefixDir: tree,
     config: {
       json: true,
     },
   })
+  const registry = new MockRegistry({
+    tap: t,
+    registry: npm.config.get('registry'),
+  })
+
+  const manifest = registry.manifest({
+    name: 'test-dep-a',
+    packuments: [{ version: '1.0.0' }, { version: '1.0.1' }],
+  })
+  await registry.package({ manifest })
+  const advisory = registry.advisory({ id: 100 })
+  const bulkBody = await gzip(JSON.stringify({ 'test-dep-a': ['1.0.0'] }))
+  registry.nock.post('/-/npm/v1/security/advisories/bulk', bulkBody)
+    .reply(200, {
+      'test-dep-a': [advisory],
+    })
+
   await npm.exec('audit', [])
+  t.ok(process.exitCode, 'would have exited uncleanly')
+  process.exitCode = 0
+  t.matchSnapshot(joinedOutput())
 })
 
-t.test('report endpoint error', async t => {
-  const loadMockNpm = (t, options) => _loadMockNpm(t, {
-    mocks: {
-      'npm-audit-report': () => {
-        throw new Error('should not call audit report when there are errors')
-      },
-      '@npmcli/arborist': function () {
-        this.audit = () => {
-          this.auditReport = {
-            error: {
-              message: 'hello, this didnt work',
-              method: 'POST',
-              uri: 'https://example.com/',
-              headers: {
-                head: ['ers'],
-              },
-              statusCode: 420,
-              body: 'this is a string',
-            },
-          }
-        }
-      },
-      '../../lib/utils/reify-output.js': () => {},
-    },
-    ...options,
+t.test('audit fix', async t => {
+  const { npm, joinedOutput } = await loadMockNpm(t, {
+    prefixDir: tree,
   })
-
-  t.test('json=false', async t => {
-    const { npm, outputs, logs } = await loadMockNpm(t, { config: { json: false } })
-    await t.rejects(npm.exec('audit', []), 'audit endpoint returned an error')
-    t.match(logs.warn, [['audit', 'hello, this didnt work']])
-    t.strictSame(outputs, [['this is a string']])
+  const registry = new MockRegistry({
+    tap: t,
+    registry: npm.config.get('registry'),
   })
-
-  t.test('json=true', async t => {
-    const { npm, outputs, logs } = await loadMockNpm(t, { config: { json: true } })
-    await t.rejects(npm.exec('audit', []), 'audit endpoint returned an error')
-    t.match(logs.warn, [['audit', 'hello, this didnt work']])
-    t.strictSame(outputs, [[
-      '{\n' +
-      '  "message": "hello, this didnt work",\n' +
-      '  "method": "POST",\n' +
-      '  "uri": "https://example.com/",\n' +
-      '  "headers": {\n' +
-      '    "head": [\n' +
-      '      "ers"\n' +
-      '    ]\n' +
-      '  },\n' +
-      '  "statusCode": 420,\n' +
-      '  "body": "this is a string"\n' +
-      '}',
-    ],
-    ])
+  // with fix
+  const manifest = registry.manifest({
+    name: 'test-dep-a',
+    packuments: [{ version: '1.0.0' }, { version: '1.0.1' }],
+  })
+  await registry.package({
+    manifest,
+    tarballs: {
+      '1.0.1': path.join(npm.prefix, 'test-dep-a'),
+    },
   })
+  const advisory = registry.advisory({ id: 100, vulnerable_versions: '1.0.0' })
+  // Can't validate this request body because it changes with each node
+  // version/npm version and nock's body validation is not async, while
+  // zlib.gunzip is
+  registry.nock.post('/-/npm/v1/security/advisories/bulk')
+    .reply(200, { // first audit
+      'test-dep-a': [advisory],
+    })
+    .post('/-/npm/v1/security/advisories/bulk')
+    .reply(200, { // after fix
+      'test-dep-a': [advisory],
+    })
+  await npm.exec('audit', ['fix'])
+  t.matchSnapshot(joinedOutput())
 })
 
 t.test('completion', async t => {
-  const { npm } = await _loadMockNpm(t)
+  const { npm } = await loadMockNpm(t)
   const audit = await npm.cmd('audit')
   t.test('fix', async t => {
     await t.resolveMatch(