Fix proxying to an https registry

.github/workflows/ci-npmcli-smoke-tests.yml

@@ -35,7 +35,7 @@ jobs:
       - name: Setup Node
         uses: actions/setup-node@v3
         with:
-          node-version: 18.x
+          node-version: 20.x
           cache: npm
           check-latest: true
       - name: Check Git Status
@@ -64,12 +64,10 @@ jobs:
             os: windows-latest
             shell: cmd
         node-version:
-          - 14.17.0
-          - 14.x
-          - 16.13.0
-          - 16.x
-          - 18.0.0
+          - 18.17.0
           - 18.x
+          - 20.5.0
+          - 20.x
     runs-on: ${{ matrix.platform.os }}
     defaults:
       run:

DEPENDENCIES.md

@@ -720,6 +720,7 @@ graph LR;
   npmcli-smoke-tests-->npmcli-mock-registry["@npmcli/mock-registry"];
   npmcli-smoke-tests-->npmcli-promise-spawn["@npmcli/promise-spawn"];
   npmcli-smoke-tests-->npmcli-template-oss["@npmcli/template-oss"];
+  npmcli-smoke-tests-->proxy;
   npmcli-smoke-tests-->tap;
   npmcli-smoke-tests-->which;
   npmlog-->are-we-there-yet;

node_modules/@npmcli/agent/lib/agents.js

@@ -7,7 +7,7 @@ const tls = require('tls')
 const { once } = require('events')
 const { createTimeout, abortRace, urlify, appendPort, cacheAgent } = require('./util')
 const { normalizeOptions, cacheOptions } = require('./options')
-const { getProxy, getProxyType, isSecureProxy, proxyCache } = require('./proxy.js')
+const { getProxy, getProxyType, proxyCache } = require('./proxy.js')
 const Errors = require('./errors.js')
 
 const createAgent = (base, name) => {
@@ -43,18 +43,16 @@ const createAgent = (base, name) => {
         return
       }
 
-      const secure = isSecureProxy(proxy)
-
       return cacheAgent({
         key: cacheOptions({
           ...options,
           ...this.#options,
-          secure,
+          secure: SECURE,
           timeouts: this.#timeouts,
           proxy,
         }),
         cache: proxyCache,
-        secure,
+        secure: SECURE,
         proxies: this.#proxy.proxies,
       }, proxy, this.#options)
     }

node_modules/@npmcli/agent/lib/proxy.js

@@ -22,13 +22,6 @@ const PROXY_ENV = (() => {
 })()
 
 const SOCKS_PROTOCOLS = new Set(SocksProxyAgent.protocols)
-const SECURE_PROTOCOLS = new Set([...SocksProxyAgent.protocols, 'https'])
-
-const isSecureProxy = (url) => {
-  url = urlify(url)
-  const protocol = url.protocol.slice(0, -1)
-  return SECURE_PROTOCOLS.has(protocol)
-}
 
 const getProxyType = (url) => {
   url = urlify(url)
@@ -91,6 +84,5 @@ const getProxy = (url, {
 module.exports = {
   getProxyType,
   getProxy,
-  isSecureProxy,
   proxyCache: PROXY_CACHE,
 }

node_modules/@npmcli/agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@npmcli/agent",
-  "version": "2.1.0",
+  "version": "2.1.1",
   "description": "the http/https agent used by the npm cli",
   "main": "lib/index.js",
   "scripts": {
@@ -35,7 +35,8 @@
       "16.x",
       "18.0.0",
       "18.x"
-    ]
+    ],
+    "npmSpec": "next-9"
   },
   "dependencies": {
     "http-proxy-agent": "^7.0.0",

package-lock.json

@@ -2855,9 +2855,9 @@
       }
     },
     "node_modules/@npmcli/agent": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/@npmcli/agent/-/agent-2.1.0.tgz",
-      "integrity": "sha512-/HFJP3a/DzgIg+6TWVee3bQmnBcWeKKYE9DKQqS8SWpAV8oYDTn/zkDM8iQ7bWI6kDDgNfHOlEFZZpN/UXMwig==",
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/@npmcli/agent/-/agent-2.1.1.tgz",
+      "integrity": "sha512-6RlbiOAi6L6uUYF4/CDEkDZQnKw0XDsFJVrEpnib8rAx2WRMOsUyAdgnvDpX/fdkDWxtqE+NHwF465llI2wR0g==",
       "inBundle": true,
       "dependencies": {
         "http-proxy-agent": "^7.0.0",
@@ -4232,6 +4232,110 @@
       "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==",
       "dev": true
     },
+    "node_modules/args": {
+      "version": "5.0.3",
+      "resolved": "https://registry.npmjs.org/args/-/args-5.0.3.tgz",
+      "integrity": "sha512-h6k/zfFgusnv3i5TU08KQkVKuCPBtL/PWQbWkHUxvJrZ2nAyeaUupneemcrgn1xmqxPQsPIzwkUhOpoqPDRZuA==",
+      "dev": true,
+      "dependencies": {
+        "camelcase": "5.0.0",
+        "chalk": "2.4.2",
+        "leven": "2.1.0",
+        "mri": "1.1.4"
+      },
+      "engines": {
+        "node": ">= 6.0.0"
+      }
+    },
+    "node_modules/args/node_modules/ansi-styles": {
+      "version": "3.2.1",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
+      "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
+      "dev": true,
+      "dependencies": {
+        "color-convert": "^1.9.0"
+      },
+      "engines": {
+        "node": ">=4"
+      }
+    },
+    "node_modules/args/node_modules/camelcase": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.0.0.tgz",
+      "integrity": "sha512-faqwZqnWxbxn+F1d399ygeamQNy3lPp/H9H6rNrqYh4FSVCtcY+3cub1MxA8o9mDd55mM8Aghuu/kuyYA6VTsA==",
+      "dev": true,
+      "engines": {
+        "node": ">=6"
+      }
+    },
+    "node_modules/args/node_modules/chalk": {
+      "version": "2.4.2",
+      "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz",
+      "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==",
+      "dev": true,
+      "dependencies": {
+        "ansi-styles": "^3.2.1",
+        "escape-string-regexp": "^1.0.5",
+        "supports-color": "^5.3.0"
+      },
+      "engines": {
+        "node": ">=4"
+      }
+    },
+    "node_modules/args/node_modules/color-convert": {
+      "version": "1.9.3",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz",
+      "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==",
+      "dev": true,
+      "dependencies": {
+        "color-name": "1.1.3"
+      }
+    },
+    "node_modules/args/node_modules/color-name": {
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz",
+      "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==",
+      "dev": true
+    },
+    "node_modules/args/node_modules/escape-string-regexp": {
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz",
+      "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==",
+      "dev": true,
+      "engines": {
+        "node": ">=0.8.0"
+      }
+    },
+    "node_modules/args/node_modules/has-flag": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
+      "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==",
+      "dev": true,
+      "engines": {
+        "node": ">=4"
+      }
+    },
+    "node_modules/args/node_modules/mri": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/mri/-/mri-1.1.4.tgz",
+      "integrity": "sha512-6y7IjGPm8AzlvoUrwAaw1tLnUBudaS3752vcd8JtrpGGQn+rXIe63LFVHm/YMwtqAuh+LJPCFdlLYPWM1nYn6w==",
+      "dev": true,
+      "engines": {
+        "node": ">=4"
+      }
+    },
+    "node_modules/args/node_modules/supports-color": {
+      "version": "5.5.0",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz",
+      "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==",
+      "dev": true,
+      "dependencies": {
+        "has-flag": "^3.0.0"
+      },
+      "engines": {
+        "node": ">=4"
+      }
+    },
     "node_modules/array-buffer-byte-length": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/array-buffer-byte-length/-/array-buffer-byte-length-1.0.0.tgz",
@@ -4416,6 +4520,12 @@
       ],
       "inBundle": true
     },
+    "node_modules/basic-auth-parser": {
+      "version": "0.0.2-1",
+      "resolved": "https://registry.npmjs.org/basic-auth-parser/-/basic-auth-parser-0.0.2-1.tgz",
+      "integrity": "sha512-GFj8iVxo9onSU6BnnQvVwqvxh60UcSHJEDnIk3z4B6iOjsKSmqe+ibW0Rsz7YO7IE1HG3D3tqCNIidP46SZVdQ==",
+      "dev": true
+    },
     "node_modules/before-after-hook": {
       "version": "2.2.3",
       "resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-2.2.3.tgz",
@@ -8633,6 +8743,15 @@
         "node": ">=6"
       }
     },
+    "node_modules/leven": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/leven/-/leven-2.1.0.tgz",
+      "integrity": "sha512-nvVPLpIHUxCUoRLrFqTgSxXJ614d8AgQoWl7zPe/2VadE8+1dpU3LBhowRuBAcuwruWtOdD8oYC9jDNJjXDPyA==",
+      "dev": true,
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/levn": {
       "version": "0.4.1",
       "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz",
@@ -12192,6 +12311,20 @@
         "url": "https://github.com/sponsors/wooorm"
       }
     },
+    "node_modules/proxy": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/proxy/-/proxy-2.1.1.tgz",
+      "integrity": "sha512-nLgd7zdUAOpB3ZO/xCkU8gy74UER7P0aihU8DkUsDS5ZoFwVCX7u8dy+cv5tVK8UaB/yminU1GiLWE26TKPYpg==",
+      "dev": true,
+      "dependencies": {
+        "args": "^5.0.3",
+        "basic-auth-parser": "0.0.2-1",
+        "debug": "^4.3.4"
+      },
+      "engines": {
+        "node": ">= 14"
+      }
+    },
     "node_modules/psl": {
       "version": "1.9.0",
       "resolved": "https://registry.npmjs.org/psl/-/psl-1.9.0.tgz",
@@ -17243,11 +17376,12 @@
         "@npmcli/promise-spawn": "^7.0.0",
         "@npmcli/template-oss": "4.18.0",
         "http-proxy": "^1.18.1",
+        "proxy": "^2.1.1",
         "tap": "^16.3.8",
         "which": "^4.0.0"
       },
       "engines": {
-        "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
+        "node": "^18.17.0 || >=20.5.0"
       }
     },
     "workspaces/arborist": {

smoke-tests/package.json

@@ -23,6 +23,7 @@
     "@npmcli/promise-spawn": "^7.0.0",
     "@npmcli/template-oss": "4.18.0",
     "http-proxy": "^1.18.1",
+    "proxy": "^2.1.1",
     "tap": "^16.3.8",
     "which": "^4.0.0"
   },
@@ -31,7 +32,13 @@
   "templateOSS": {
     "//@npmcli/template-oss": "This file is partially managed by @npmcli/template-oss. Edits may be overwritten.",
     "version": "4.18.0",
-    "content": "../scripts/template-oss/index.js"
+    "content": "../scripts/template-oss/index.js",
+    "ciVersions": [
+      "18.17.0",
+      "18.x",
+      "20.5.0",
+      "20.x"
+    ]
   },
   "tap": {
     "no-coverage": true,
@@ -47,6 +54,6 @@
     "lib/"
   ],
   "engines": {
-    "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
+    "node": "^18.17.0 || >=20.5.0"
   }
 }

smoke-tests/test/fixtures/setup.js

@@ -10,6 +10,7 @@ const httpProxy = require('http-proxy')
 const { SMOKE_PUBLISH_NPM, SMOKE_PUBLISH_TARBALL, CI, PATH, Path, TAP_CHILD_ID = '0' } = process.env
 const PROXY_PORT = 12345 + (+TAP_CHILD_ID)
 const HTTP_PROXY = `http://localhost:${PROXY_PORT}/`
+const DEFAULT_REGISTRY = new URL('https://registry.npmjs.org/')
 
 const NODE_PATH = process.execPath
 const CLI_ROOT = resolve(process.cwd(), '..')
@@ -114,7 +115,11 @@ module.exports = async (t, { testdir = {}, debug, registry: _registry = {} } = {
     globalNodeModules: join(root, 'global', GLOBAL_NODE_MODULES),
   }
 
-  const registry = await createRegistry(t, { ..._registry, debug })
+  const liveRegistry = _registry === false
+  const USE_PROXY = !liveRegistry
+  const registry = liveRegistry
+    ? DEFAULT_REGISTRY
+    : await createRegistry(t, { ..._registry, debug })
 
   // update notifier should never be written
   t.afterEach((t) => {
@@ -178,7 +183,7 @@ module.exports = async (t, { testdir = {}, debug, registry: _registry = {} } = {
   }
 
   const baseNpm = async (...a) => {
-    const [{ cwd, cmd, argv = [], proxy = true, ...opts }, args] = getOpts(...a)
+    const [{ cwd, cmd, argv = [], proxy = USE_PROXY, ...opts }, args] = getOpts(...a)
 
     const isGlobal = args.some(arg => ['-g', '--global', '--global=true'].includes(arg))
 
@@ -262,3 +267,4 @@ module.exports.WINDOWS = WINDOWS
 module.exports.SMOKE_PUBLISH = !!SMOKE_PUBLISH_NPM
 module.exports.SMOKE_PUBLISH_TARBALL = SMOKE_PUBLISH_TARBALL
 module.exports.HTTP_PROXY = HTTP_PROXY
+module.exports.PROXY_PORT = PROXY_PORT

smoke-tests/test/proxy.js

@@ -0,0 +1,29 @@
+const t = require('tap')
+const setup = require('./fixtures/setup.js')
+const { createProxy } = require('proxy')
+const http = require('http')
+
+t.test('basic', async t => {
+  const PORT = setup.PROXY_PORT
+  const { npm, readFile } = await setup(t, {
+    registry: false,
+    testdir: {
+      home: {
+        '.npmrc': `proxy = "http://localhost:${PORT}/"`,
+      },
+    },
+  })
+
+  const server = createProxy(http.createServer())
+  await new Promise(res => server.listen(PORT, res))
+
+  t.teardown(() => server.close())
+
+  await t.test('npm install prodDep@version', async t => {
+    await npm('install', 'abbrev@1.0.4')
+
+    t.strictSame(await readFile('package.json'), {
+      dependencies: { abbrev: '^1.0.4' },
+    })
+  })
+})