diff --git a/README.md b/README.md index c541c0d0e..27d14c034 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ - 💾 Upload data to LHCI server - 🔔 Slack notification - 😻 GitHub notification +- 🛳️ Automatic check of Netlify preview urls Lighthouse CI Action @@ -174,6 +175,23 @@ Use `error` value to send notifications only for failed CI checks. logLevel: 'error' ``` +### netlifySite + +Name of the site defined in Netlify console. + +> Users can specify their custom domains, so Netlif site name can be different from URL origin. + +It enables run LHCI check against [Netlify deploy preview](https://www.netlify.com/blog/2016/07/20/introducing-deploy-previews-in-netlify/) to make sure your changes are not gonna slow down production. + +```yml +netlifySite: 'your-netlify-preview-url.netlify.com' +``` + +> Before start checking, action ping Netlify preview each minute to make sure site was deployed. +> In case site wasn't deployed in 5 minutes it fails. + +[Read more](#recipes) about detailed configuration. + ## Recipes
@@ -283,6 +301,64 @@ Make a `budget.json` file with [budgets syntax](https://web.dev/use-lighthouse-f
+
+ Run LHCI against Netlify preview site
+ +Create `.github/workflows/main.yml` with the list of URLs, enable notifications to audit +and identify a budget with `budgetPath`. + +#### main.yml + +```yml +name: LHCI-assert-netlify-on-budget-notification +on: pull_request +jobs: + # This pass/fails a build with a budgets.json. + assert-on-budget: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - name: Run Lighthouse on Netlify urls and validate with budgets.json + uses: ./ + with: + urls: | + https://your-netlify-preview-url.netlify.com + https://your-netlify-preview-url.netlify.com/products/ + https://your-netlify-preview-url.netlify.com/contact/ + netlifySite: 'your-netlify-preview-url.netlify.com' + budgetPath: '.github/lighthouse/budget.json' + slackWebhookUrl: ${{ secrets.SLACK_WEBHOOK_URL }} + applicationGithubToken: ${{ secrets.GITHUB_TOKEN }} + personalGithubToken: ${{ secrets.PERSONAL_GITHUB_TOKEN }} + logLevel: 'error' +``` + +Make a `budget.json` file with [budgets syntax](https://web.dev/use-lighthouse-for-performance-budgets/). + +> **Note**: Under the hood, this will be transformed into LHCI assertions. + +#### budgets.json + +```json +[ + { + "path": "/*", + "resourceSizes": [ + { + "resourceType": "document", + "budget": 18 + }, + { + "resourceType": "total", + "budget": 200 + } + ] + } +] +``` + +
+
Run Lighthouse and validate against LHCI assertions.
diff --git a/action.yml b/action.yml index f1be00338..d2b7a6013 100644 --- a/action.yml +++ b/action.yml @@ -21,6 +21,8 @@ inputs: description: 'GitHub Application access token for ' personalGithubToken: description: 'GitHub access token' + netlifySite: + description: 'Netlify site name' upload.serverBaseUrl: description: 'Address of a LHCI server' upload.token: diff --git a/node_modules/@octokit/request/node_modules/node-fetch/package.json b/node_modules/@octokit/request/node_modules/node-fetch/package.json deleted file mode 100644 index 8e5c883b2..000000000 --- a/node_modules/@octokit/request/node_modules/node-fetch/package.json +++ /dev/null @@ -1,66 +0,0 @@ -{ - "name": "node-fetch", - "version": "2.6.0", - "description": "A light-weight module that brings window.fetch to node.js", - "main": "lib/index", - "browser": "./browser.js", - "module": "lib/index.mjs", - "files": [ - "lib/index.js", - "lib/index.mjs", - "lib/index.es.js", - "browser.js" - ], - "engines": { - "node": "4.x || >=6.0.0" - }, - "scripts": { - "build": "cross-env BABEL_ENV=rollup rollup -c", - "prepare": "npm run build", - "test": "cross-env BABEL_ENV=test mocha --require babel-register --throw-deprecation test/test.js", - "report": "cross-env BABEL_ENV=coverage nyc --reporter lcov --reporter text mocha -R spec test/test.js", - "coverage": "cross-env BABEL_ENV=coverage nyc --reporter json --reporter text mocha -R spec test/test.js && codecov -f coverage/coverage-final.json" - }, - "repository": { - "type": "git", - "url": "https://github.com/bitinn/node-fetch.git" - }, - "keywords": [ - "fetch", - "http", - "promise" - ], - "author": "David Frank", - "license": "MIT", - "bugs": { - "url": "https://github.com/bitinn/node-fetch/issues" - }, - "homepage": "https://github.com/bitinn/node-fetch", - "devDependencies": { - "@ungap/url-search-params": "^0.1.2", - "abort-controller": "^1.1.0", - "abortcontroller-polyfill": "^1.3.0", - "babel-core": "^6.26.3", - "babel-plugin-istanbul": "^4.1.6", - "babel-preset-env": "^1.6.1", - "babel-register": "^6.16.3", - "chai": "^3.5.0", - "chai-as-promised": "^7.1.1", - "chai-iterator": "^1.1.1", - "chai-string": "~1.3.0", - "codecov": "^3.3.0", - "cross-env": "^5.2.0", - "form-data": "^2.3.3", - "is-builtin-module": "^1.0.0", - "mocha": "^5.0.0", - "nyc": "11.9.0", - "parted": "^0.1.1", - "promise": "^8.0.3", - "resumer": "0.0.0", - "rollup": "^0.63.4", - "rollup-plugin-babel": "^3.0.7", - "string-to-arraybuffer": "^1.0.2", - "whatwg-url": "^5.0.0" - }, - "dependencies": {} -} diff --git a/node_modules/@types/node-fetch/node_modules/@types/node/package.json b/node_modules/@types/node-fetch/node_modules/@types/node/package.json new file mode 100644 index 000000000..427489f5a --- /dev/null +++ b/node_modules/@types/node-fetch/node_modules/@types/node/package.json @@ -0,0 +1,216 @@ +{ + "name": "@types/node", + "version": "12.7.5", + "description": "TypeScript definitions for Node.js", + "license": "MIT", + "contributors": [ + { + "name": "Microsoft TypeScript", + "url": "https://github.com/Microsoft", + "githubUsername": "Microsoft" + }, + { + "name": "DefinitelyTyped", + "url": "https://github.com/DefinitelyTyped", + "githubUsername": "DefinitelyTyped" + }, + { + "name": "Alberto Schiabel", + "url": "https://github.com/jkomyno", + "githubUsername": "jkomyno" + }, + { + "name": "Alexander T.", + "url": "https://github.com/a-tarasyuk", + "githubUsername": "a-tarasyuk" + }, + { + "name": "Alvis HT Tang", + "url": "https://github.com/alvis", + "githubUsername": "alvis" + }, + { + "name": "Andrew Makarov", + "url": "https://github.com/r3nya", + "githubUsername": "r3nya" + }, + { + "name": "Benjamin Toueg", + "url": "https://github.com/btoueg", + "githubUsername": "btoueg" + }, + { + "name": "Bruno Scheufler", + "url": "https://github.com/brunoscheufler", + "githubUsername": "brunoscheufler" + }, + { + "name": "Chigozirim C.", + "url": "https://github.com/smac89", + "githubUsername": "smac89" + }, + { + "name": "Christian Vaagland Tellnes", + "url": "https://github.com/tellnes", + "githubUsername": "tellnes" + }, + { + "name": "David Junger", + "url": "https://github.com/touffy", + "githubUsername": "touffy" + }, + { + "name": "Deividas Bakanas", + "url": "https://github.com/DeividasBakanas", + "githubUsername": "DeividasBakanas" + }, + { + "name": "Eugene Y. Q. Shen", + "url": "https://github.com/eyqs", + "githubUsername": "eyqs" + }, + { + "name": "Flarna", + "url": "https://github.com/Flarna", + "githubUsername": "Flarna" + }, + { + "name": "Hannes Magnusson", + "url": "https://github.com/Hannes-Magnusson-CK", + "githubUsername": "Hannes-Magnusson-CK" + }, + { + "name": "Hoàng Văn Khải", + "url": "https://github.com/KSXGitHub", + "githubUsername": "KSXGitHub" + }, + { + "name": "Huw", + "url": "https://github.com/hoo29", + "githubUsername": "hoo29" + }, + { + "name": "Kelvin Jin", + "url": "https://github.com/kjin", + "githubUsername": "kjin" + }, + { + "name": "Klaus Meinhardt", + "url": "https://github.com/ajafff", + "githubUsername": "ajafff" + }, + { + "name": "Lishude", + "url": "https://github.com/islishude", + "githubUsername": "islishude" + }, + { + "name": "Mariusz Wiktorczyk", + "url": "https://github.com/mwiktorczyk", + "githubUsername": "mwiktorczyk" + }, + { + "name": "Matthieu Sieben", + "url": "https://github.com/matthieusieben", + "githubUsername": "matthieusieben" + }, + { + "name": "Mohsen Azimi", + "url": "https://github.com/mohsen1", + "githubUsername": "mohsen1" + }, + { + "name": "Nicolas Even", + "url": "https://github.com/n-e", + "githubUsername": "n-e" + }, + { + "name": "Nicolas Voigt", + "url": "https://github.com/octo-sniffle", + "githubUsername": "octo-sniffle" + }, + { + "name": "Parambir Singh", + "url": "https://github.com/parambirs", + "githubUsername": "parambirs" + }, + { + "name": "Sebastian Silbermann", + "url": "https://github.com/eps1lon", + "githubUsername": "eps1lon" + }, + { + "name": "Simon Schick", + "url": "https://github.com/SimonSchick", + "githubUsername": "SimonSchick" + }, + { + "name": "Thomas den Hollander", + "url": "https://github.com/ThomasdenH", + "githubUsername": "ThomasdenH" + }, + { + "name": "Wilco Bakker", + "url": "https://github.com/WilcoBakker", + "githubUsername": "WilcoBakker" + }, + { + "name": "wwwy3y3", + "url": "https://github.com/wwwy3y3", + "githubUsername": "wwwy3y3" + }, + { + "name": "Zane Hannan AU", + "url": "https://github.com/ZaneHannanAU", + "githubUsername": "ZaneHannanAU" + }, + { + "name": "Samuel Ainsworth", + "url": "https://github.com/samuela", + "githubUsername": "samuela" + }, + { + "name": "Kyle Uehlein", + "url": "https://github.com/kuehlein", + "githubUsername": "kuehlein" + }, + { + "name": "Jordi Oliveras Rovira", + "url": "https://github.com/j-oliveras", + "githubUsername": "j-oliveras" + }, + { + "name": "Thanik Bhongbhibhat", + "url": "https://github.com/bhongy", + "githubUsername": "bhongy" + }, + { + "name": "Marcin Kopacz", + "url": "https://github.com/chyzwar", + "githubUsername": "chyzwar" + }, + { + "name": "Trivikram Kamat", + "url": "https://github.com/trivikr", + "githubUsername": "trivikr" + } + ], + "main": "", + "types": "index", + "typesVersions": { + ">=3.2.0-0": { + "*": [ + "ts3.2/*" + ] + } + }, + "repository": { + "type": "git", + "url": "https://github.com/DefinitelyTyped/DefinitelyTyped.git", + "directory": "types/node" + }, + "scripts": {}, + "dependencies": {}, + "typesPublisherContentHash": "f0d8295c97f3f4bafe268435e951340e846630f23bf920def70d6d0d454de5f3", + "typeScriptVersion": "2.0" +} \ No newline at end of file diff --git a/node_modules/@types/node-fetch/package.json b/node_modules/@types/node-fetch/package.json new file mode 100644 index 000000000..e89e3b511 --- /dev/null +++ b/node_modules/@types/node-fetch/package.json @@ -0,0 +1,77 @@ +{ + "name": "@types/node-fetch", + "version": "2.5.5", + "description": "TypeScript definitions for node-fetch", + "license": "MIT", + "contributors": [ + { + "name": "Torsten Werner", + "url": "https://github.com/torstenwerner", + "githubUsername": "torstenwerner" + }, + { + "name": "Niklas Lindgren", + "url": "https://github.com/nikcorg", + "githubUsername": "nikcorg" + }, + { + "name": "Vinay Bedre", + "url": "https://github.com/vinaybedre", + "githubUsername": "vinaybedre" + }, + { + "name": "Antonio Román", + "url": "https://github.com/kyranet", + "githubUsername": "kyranet" + }, + { + "name": "Andrew Leedham", + "url": "https://github.com/AndrewLeedham", + "githubUsername": "AndrewLeedham" + }, + { + "name": "Jason Li", + "url": "https://github.com/JasonLi914", + "githubUsername": "JasonLi914" + }, + { + "name": "Brandon Wilson", + "url": "https://github.com/wilsonianb", + "githubUsername": "wilsonianb" + }, + { + "name": "Steve Faulkner", + "url": "https://github.com/southpolesteve", + "githubUsername": "southpolesteve" + }, + { + "name": "ExE Boss", + "url": "https://github.com/ExE-Boss", + "githubUsername": "ExE-Boss" + }, + { + "name": "Alex Savin", + "url": "https://github.com/alexandrusavin", + "githubUsername": "alexandrusavin" + }, + { + "name": "Alexis Tyler", + "url": "https://github.com/OmgImAlexis", + "githubUsername": "OmgImAlexis" + } + ], + "main": "", + "types": "index.d.ts", + "repository": { + "type": "git", + "url": "https://github.com/DefinitelyTyped/DefinitelyTyped.git", + "directory": "types/node-fetch" + }, + "scripts": {}, + "dependencies": { + "@types/node": "*", + "form-data": "^3.0.0" + }, + "typesPublisherContentHash": "c438da001acaaae04b1a33df4b2ea9a62bd3da7967c9741b61684d5500f7c75a", + "typeScriptVersion": "2.8" +} \ No newline at end of file diff --git a/node_modules/mime-types/node_modules/mime-db/db.json b/node_modules/compressible/node_modules/mime-db/db.json similarity index 98% rename from node_modules/mime-types/node_modules/mime-db/db.json rename to node_modules/compressible/node_modules/mime-db/db.json index a5fc98708..f63a57ca6 100644 --- a/node_modules/mime-types/node_modules/mime-db/db.json +++ b/node_modules/compressible/node_modules/mime-db/db.json @@ -452,6 +452,9 @@ "application/fits": { "source": "iana" }, + "application/flexfec": { + "source": "iana" + }, "application/font-sfnt": { "source": "iana" }, @@ -813,6 +816,9 @@ "application/mikey": { "source": "iana" }, + "application/mipc": { + "source": "iana" + }, "application/mmt-aei+xml": { "source": "iana", "compressible": true @@ -1346,6 +1352,9 @@ "application/simplesymbolcontainer": { "source": "iana" }, + "application/sipc": { + "source": "iana" + }, "application/slate": { "source": "iana" }, @@ -1411,6 +1420,10 @@ "source": "iana", "compressible": true }, + "application/swid+xml": { + "source": "iana", + "compressible": true + }, "application/tamp-apex-update": { "source": "iana" }, @@ -1484,6 +1497,10 @@ "application/tnauthlist": { "source": "iana" }, + "application/toml": { + "compressible": true, + "extensions": ["toml"] + }, "application/trickle-ice-sdpfrag": { "source": "iana" }, @@ -1640,6 +1657,10 @@ "source": "iana", "compressible": true }, + "application/vnd.3gpp.mcvideo-info+xml": { + "source": "iana", + "compressible": true + }, "application/vnd.3gpp.mcvideo-location-info+xml": { "source": "iana", "compressible": true @@ -1812,6 +1833,9 @@ "source": "iana", "compressible": true }, + "application/vnd.android.ota": { + "source": "iana" + }, "application/vnd.android.package-archive": { "source": "apache", "compressible": false, @@ -1917,6 +1941,9 @@ "application/vnd.banana-accounting": { "source": "iana" }, + "application/vnd.bbf.usp.error": { + "source": "iana" + }, "application/vnd.bbf.usp.msg": { "source": "iana" }, @@ -1952,6 +1979,12 @@ "source": "iana", "extensions": ["bmi"] }, + "application/vnd.bpf": { + "source": "iana" + }, + "application/vnd.bpf3": { + "source": "iana" + }, "application/vnd.businessobjects": { "source": "iana", "extensions": ["rep"] @@ -1991,6 +2024,9 @@ "source": "iana", "extensions": ["mmd"] }, + "application/vnd.ciedi": { + "source": "iana" + }, "application/vnd.cinderella": { "source": "iana", "extensions": ["cdy"] @@ -2107,6 +2143,13 @@ "compressible": true, "extensions": ["wbs"] }, + "application/vnd.cryptii.pipe+json": { + "source": "iana", + "compressible": true + }, + "application/vnd.crypto-shade-file": { + "source": "iana" + }, "application/vnd.ctc-posml": { "source": "iana", "extensions": ["pml"] @@ -2540,6 +2583,10 @@ "application/vnd.ffsns": { "source": "iana" }, + "application/vnd.ficlab.flb+zip": { + "source": "iana", + "compressible": false + }, "application/vnd.filmit.zfc": { "source": "iana" }, @@ -3004,6 +3051,10 @@ "source": "iana", "extensions": ["fcs"] }, + "application/vnd.iso11783-10+zip": { + "source": "iana", + "compressible": false + }, "application/vnd.jam": { "source": "iana", "extensions": ["jam"] @@ -3103,6 +3154,9 @@ "source": "iana", "extensions": ["sse"] }, + "application/vnd.las": { + "source": "iana" + }, "application/vnd.las.las+json": { "source": "iana", "compressible": true @@ -3112,6 +3166,9 @@ "compressible": true, "extensions": ["lasxml"] }, + "application/vnd.laszip": { + "source": "iana" + }, "application/vnd.leap+json": { "source": "iana", "compressible": true @@ -3129,6 +3186,13 @@ "compressible": true, "extensions": ["lbe"] }, + "application/vnd.logipipe.circuit+zip": { + "source": "iana", + "compressible": false + }, + "application/vnd.loom": { + "source": "iana" + }, "application/vnd.lotus-1-2-3": { "source": "iana", "extensions": ["123"] @@ -4583,6 +4647,9 @@ "source": "iana", "extensions": ["semf"] }, + "application/vnd.shade-save-file": { + "source": "iana" + }, "application/vnd.shana.informed.formdata": { "source": "iana", "extensions": ["ifm"] @@ -4603,6 +4670,10 @@ "source": "iana", "compressible": true }, + "application/vnd.shopkick+json": { + "source": "iana", + "compressible": true + }, "application/vnd.sigrok.session": { "source": "iana" }, @@ -4920,6 +4991,9 @@ "application/vnd.veryant.thin": { "source": "iana" }, + "application/vnd.ves.encrypted": { + "source": "iana" + }, "application/vnd.vidsoft.vidconference": { "source": "iana" }, @@ -5988,6 +6062,9 @@ "audio/evs": { "source": "iana" }, + "audio/flexfec": { + "source": "iana" + }, "audio/fwdred": { "source": "iana" }, @@ -6474,6 +6551,7 @@ }, "font/ttf": { "source": "iana", + "compressible": true, "extensions": ["ttf"] }, "font/woff": { @@ -6544,6 +6622,14 @@ "source": "iana", "extensions": ["heifs"] }, + "image/hej2k": { + "source": "iana", + "extensions": ["hej2"] + }, + "image/hsj2": { + "source": "iana", + "extensions": ["hsj2"] + }, "image/ief": { "source": "iana", "extensions": ["ief"] @@ -6562,6 +6648,14 @@ "compressible": false, "extensions": ["jpeg","jpg","jpe"] }, + "image/jph": { + "source": "iana", + "extensions": ["jph"] + }, + "image/jphc": { + "source": "iana", + "extensions": ["jhc"] + }, "image/jpm": { "source": "iana", "compressible": false, @@ -6576,6 +6670,30 @@ "source": "iana", "extensions": ["jxr"] }, + "image/jxra": { + "source": "iana", + "extensions": ["jxra"] + }, + "image/jxrs": { + "source": "iana", + "extensions": ["jxrs"] + }, + "image/jxs": { + "source": "iana", + "extensions": ["jxs"] + }, + "image/jxsc": { + "source": "iana", + "extensions": ["jxsc"] + }, + "image/jxsi": { + "source": "iana", + "extensions": ["jxsi"] + }, + "image/jxss": { + "source": "iana", + "extensions": ["jxss"] + }, "image/ktx": { "source": "iana", "extensions": ["ktx"] @@ -6689,6 +6807,9 @@ "image/vnd.mozilla.apng": { "source": "iana" }, + "image/vnd.ms-dds": { + "extensions": ["dds"] + }, "image/vnd.ms-modi": { "source": "iana", "extensions": ["mdi"] @@ -7041,8 +7162,7 @@ "source": "iana" }, "multipart/mixed": { - "source": "iana", - "compressible": false + "source": "iana" }, "multipart/multilingual": { "source": "iana" @@ -7120,6 +7240,9 @@ "text/enriched": { "source": "iana" }, + "text/flexfec": { + "source": "iana" + }, "text/fwdred": { "source": "iana" }, @@ -7306,6 +7429,9 @@ "text/vnd.esmertec.theme-descriptor": { "source": "iana" }, + "text/vnd.ficlab.flt": { + "source": "iana" + }, "text/vnd.fly": { "source": "iana", "extensions": ["fly"] @@ -7359,6 +7485,9 @@ "text/vnd.si.uricatalogue": { "source": "iana" }, + "text/vnd.sosi": { + "source": "iana" + }, "text/vnd.sun.j2me.app-descriptor": { "source": "iana", "extensions": ["jad"] @@ -7511,6 +7640,9 @@ "video/encaprtp": { "source": "iana" }, + "video/flexfec": { + "source": "iana" + }, "video/h261": { "source": "iana", "extensions": ["h261"] @@ -7750,6 +7882,9 @@ "source": "iana", "extensions": ["viv"] }, + "video/vnd.youtube.yt": { + "source": "iana" + }, "video/vp8": { "source": "iana" }, diff --git a/node_modules/mime-types/node_modules/mime-db/index.js b/node_modules/compressible/node_modules/mime-db/index.js similarity index 100% rename from node_modules/mime-types/node_modules/mime-db/index.js rename to node_modules/compressible/node_modules/mime-db/index.js diff --git a/node_modules/mime-types/node_modules/mime-db/package.json b/node_modules/compressible/node_modules/mime-db/package.json similarity index 76% rename from node_modules/mime-types/node_modules/mime-db/package.json rename to node_modules/compressible/node_modules/mime-db/package.json index 07db1ecce..ee06d82ef 100644 --- a/node_modules/mime-types/node_modules/mime-db/package.json +++ b/node_modules/compressible/node_modules/mime-db/package.json @@ -1,7 +1,7 @@ { "name": "mime-db", "description": "Media Type Database", - "version": "1.40.0", + "version": "1.42.0", "contributors": [ "Douglas Christopher Wilson ", "Jonathan Ong (http://jongleberry.com)", @@ -19,20 +19,20 @@ ], "repository": "jshttp/mime-db", "devDependencies": { - "bluebird": "3.5.4", + "bluebird": "3.5.5", "co": "4.6.0", "cogent": "1.0.1", - "csv-parse": "4.3.4", - "eslint": "5.16.0", - "eslint-config-standard": "12.0.0", - "eslint-plugin-import": "2.16.0", - "eslint-plugin-node": "8.0.1", - "eslint-plugin-promise": "4.1.1", - "eslint-plugin-standard": "4.0.0", + "csv-parse": "4.4.6", + "eslint": "6.4.0", + "eslint-config-standard": "14.1.0", + "eslint-plugin-import": "2.18.2", + "eslint-plugin-node": "10.0.0", + "eslint-plugin-promise": "4.2.1", + "eslint-plugin-standard": "4.0.1", "gnode": "0.1.2", - "mocha": "6.1.4", - "nyc": "14.0.0", - "raw-body": "2.3.3", + "mocha": "6.2.0", + "nyc": "14.1.1", + "raw-body": "2.4.1", "stream-to-array": "2.3.0" }, "files": [ diff --git a/node_modules/form-data/README.md.bak b/node_modules/form-data/README.md.bak index 0524d6028..e9195bdb4 100644 --- a/node_modules/form-data/README.md.bak +++ b/node_modules/form-data/README.md.bak @@ -6,13 +6,12 @@ The API of this library is inspired by the [XMLHttpRequest-2 FormData Interface] [xhr2-fd]: http://dev.w3.org/2006/webapi/XMLHttpRequest-2/Overview.html#the-formdata-interface -[![Linux Build](https://img.shields.io/travis/form-data/form-data/master.svg?label=linux:4.x-9.x)](https://travis-ci.org/form-data/form-data) -[![MacOS Build](https://img.shields.io/travis/form-data/form-data/master.svg?label=macos:4.x-9.x)](https://travis-ci.org/form-data/form-data) -[![Windows Build](https://img.shields.io/appveyor/ci/alexindigo/form-data/master.svg?label=windows:4.x-9.x)](https://ci.appveyor.com/project/alexindigo/form-data) +[![Linux Build](https://img.shields.io/travis/form-data/form-data/master.svg?label=linux:6.x-12.x)](https://travis-ci.org/form-data/form-data) +[![MacOS Build](https://img.shields.io/travis/form-data/form-data/master.svg?label=macos:6.x-12.x)](https://travis-ci.org/form-data/form-data) +[![Windows Build](https://img.shields.io/travis/form-data/form-data/master.svg?label=windows:6.x-12.x)](https://travis-ci.org/form-data/form-data) [![Coverage Status](https://img.shields.io/coveralls/form-data/form-data/master.svg?label=code+coverage)](https://coveralls.io/github/form-data/form-data?branch=master) [![Dependency Status](https://img.shields.io/david/form-data/form-data.svg)](https://david-dm.org/form-data/form-data) -[![bitHound Overall Score](https://www.bithound.io/github/form-data/form-data/badges/score.svg)](https://www.bithound.io/github/form-data/form-data) ## Install @@ -185,6 +184,102 @@ form.submit({ }); ``` +### Methods + +- [_Void_ append( **String** _field_, **Mixed** _value_ [, **Mixed** _options_] )](https://github.com/form-data/form-data#void-append-string-field-mixed-value--mixed-options-). +- [_Headers_ getHeaders( [**Headers** _userHeaders_] )](https://github.com/form-data/form-data#array-getheaders-array-userheaders-) +- [_String_ getBoundary()](https://github.com/form-data/form-data#string-getboundary) +- [_Buffer_ getBuffer()](https://github.com/form-data/form-data#buffer-getbuffer) +- [_Integer_ getLengthSync()](https://github.com/form-data/form-data#integer-getlengthsync) +- [_Integer_ getLength( **function** _callback_ )](https://github.com/form-data/form-data#integer-getlength-function-callback-) +- [_Boolean_ hasKnownLength()](https://github.com/form-data/form-data#boolean-hasknownlength) +- [_Request_ submit( _params_, **function** _callback_ )](https://github.com/form-data/form-data#request-submit-params-function-callback-) +- [_String_ toString()](https://github.com/form-data/form-data#string-tostring) + +#### _Void_ append( **String** _field_, **Mixed** _value_ [, **Mixed** _options_] ) +Append data to the form. You can submit about any format (string, integer, boolean, buffer, etc.). However, Arrays are not supported and need to be turned into strings by the user. +```javascript +var form = new FormData(); +form.append( 'my_string', 'my value' ); +form.append( 'my_integer', 1 ); +form.append( 'my_boolean', true ); +form.append( 'my_buffer', new Buffer(10) ); +form.append( 'my_array_as_json', JSON.stringify( ['bird','cute'] ) ) +``` + +You may provide a string for options, or an object. +```javascript +// Set filename by providing a string for options +form.append( 'my_file', fs.createReadStream('/foo/bar.jpg'), 'bar.jpg' ); + +// provide an object. +form.append( 'my_file', fs.createReadStream('/foo/bar.jpg'), {filename: 'bar.jpg', contentType: 'image/jpeg', knownLength: 19806} ); +``` + +#### _Headers_ getHeaders( [**Headers** _userHeaders_] ) +This method ads the correct `content-type` header to the provided array of `userHeaders`. + +#### _String_ getBoundary() +Return the boundary of the formData. A boundary consists of 26 `-` followed by 24 numbers +for example: +```javascript +--------------------------515890814546601021194782 +``` +_Note: The boundary must be unique and may not appear in the data._ + +#### _Buffer_ getBuffer() +Return the full formdata request package, as a Buffer. You can insert this Buffer in e.g. Axios to send multipart data. +```javascript +var form = new FormData(); +form.append( 'my_buffer', Buffer.from([0x4a,0x42,0x20,0x52,0x6f,0x63,0x6b,0x73]) ); +form.append( 'my_file', fs.readFileSync('/foo/bar.jpg') ); + +axios.post( 'https://example.com/path/to/api', + form.getBuffer(), + form.getHeaders() + ) +``` +**Note:** Because the output is of type Buffer, you can only append types that are accepted by Buffer: *string, Buffer, ArrayBuffer, Array, or Array-like Object*. A ReadStream for example will result in an error. + +#### _Integer_ getLengthSync() +Same as `getLength` but synchronous. + +_Note: getLengthSync __doesn't__ calculate streams length._ + +#### _Integer_ getLength( **function** _callback_ ) +Returns the `Content-Length` async. The callback is used to handle errors and continue once the length has been calculated +```javascript +this.getLength(function(err, length) { + if (err) { + this._error(err); + return; + } + + // add content length + request.setHeader('Content-Length', length); + + ... +}.bind(this)); +``` + +#### _Boolean_ hasKnownLength() +Checks if the length of added values is known. + +#### _Request_ submit( _params_, **function** _callback_ ) +Submit the form to a web application. +```javascript +var form = new FormData(); +form.append( 'my_string', 'Hello World' ); + +form.submit( 'http://example.com/', function(err, res) { + // res – response object (http.IncomingMessage) // + res.resume(); +} ); +``` + +#### _String_ toString() +Returns the form data as a string. Don't use this if you are sending files or buffers, use `getBuffer()` instead. + ### Integration with other libraries #### Request @@ -224,10 +319,32 @@ fetch('http://example.com', { method: 'POST', body: form }) }); ``` +#### axios + +In Node.js you can post a file using [axios](https://github.com/axios/axios): +```javascript +const form = new FormData(); +const stream = fs.createReadStream(PATH_TO_FILE); + +form.append('image', stream); + +// In Node.js environment you need to set boundary in the header field 'Content-Type' by calling method `getHeaders` +const formHeaders = form.getHeaders(); + +axios.post('http://example.com', form, { + headers: { + ...formHeaders, + }, +}) +.then(response => response) +.catch(error => error) +``` + ## Notes - ```getLengthSync()``` method DOESN'T calculate length for streams, use ```knownLength``` options as workaround. - Starting version `2.x` FormData has dropped support for `node@0.10.x`. +- Starting version `3.x` FormData has dropped support for `node@4.x`. ## License diff --git a/node_modules/form-data/lib/form_data.js b/node_modules/form-data/lib/form_data.js index 3a1bb82b1..ddfae2e34 100644 --- a/node_modules/form-data/lib/form_data.js +++ b/node_modules/form-data/lib/form_data.js @@ -25,7 +25,7 @@ util.inherits(FormData, CombinedStream); */ function FormData(options) { if (!(this instanceof FormData)) { - return new FormData(); + return new FormData(options); } this._overheadLength = 0; @@ -230,7 +230,7 @@ FormData.prototype._getContentDisposition = function(value, options) { filename = path.basename(options.filename || value.name || value.path); } else if (value.readable && value.hasOwnProperty('httpVersion')) { // or try http response - filename = path.basename(value.client._httpMessage.path); + filename = path.basename(value.client._httpMessage.path || ''); } if (filename) { @@ -313,6 +313,32 @@ FormData.prototype.getBoundary = function() { return this._boundary; }; +FormData.prototype.getBuffer = function() { + var dataBuffer = new Buffer.alloc( 0 ); + var boundary = this.getBoundary(); + + // Create the form content. Add Line breaks to the end of data. + for (var i = 0, len = this._streams.length; i < len; i++) { + if (typeof this._streams[i] !== 'function') { + + // Add content to the buffer. + if(Buffer.isBuffer(this._streams[i])) { + dataBuffer = Buffer.concat( [dataBuffer, this._streams[i]]); + }else { + dataBuffer = Buffer.concat( [dataBuffer, Buffer.from(this._streams[i])]); + } + + // Add break after content. + if (typeof this._streams[i] !== 'string' || this._streams[i].substring( 2, boundary.length + 2 ) !== boundary) { + dataBuffer = Buffer.concat( [dataBuffer, Buffer.from(FormData.LINE_BREAK)] ); + } + } + } + + // Add the footer and return the Buffer object. + return Buffer.concat( [dataBuffer, Buffer.from(this._lastBoundary())] ); +}; + FormData.prototype._generateBoundary = function() { // This generates a 50 character boundary similar to those used by Firefox. // They are optimized for boyer-moore parsing. @@ -436,8 +462,19 @@ FormData.prototype.submit = function(params, cb) { this.pipe(request); if (cb) { - request.on('error', cb); - request.on('response', cb.bind(this, null)); + var onResponse; + + var callback = function (error, responce) { + request.removeListener('error', callback); + request.removeListener('response', onResponse); + + return cb.call(this, error, responce); + }; + + onResponse = callback.bind(this, null); + + request.on('error', callback); + request.on('response', onResponse); } }.bind(this)); diff --git a/node_modules/form-data/package.json b/node_modules/form-data/package.json index adacbae78..6f1ebf05e 100644 --- a/node_modules/form-data/package.json +++ b/node_modules/form-data/package.json @@ -2,20 +2,21 @@ "author": "Felix Geisendörfer (http://debuggable.com/)", "name": "form-data", "description": "A library to create readable \"multipart/form-data\" streams. Can be used to submit forms and file uploads to other web applications.", - "version": "2.3.3", + "version": "3.0.0", "repository": { "type": "git", "url": "git://github.com/form-data/form-data.git" }, "main": "./lib/form_data", "browser": "./lib/browser", + "typings": "./index.d.ts", "scripts": { "pretest": "rimraf coverage test/tmp", "test": "istanbul cover test/run.js", "posttest": "istanbul report lcov text", "lint": "eslint lib/*.js test/*.js test/integration/*.js", "report": "istanbul report lcov text", - "ci-lint": "is-node-modern 6 && npm run lint || is-node-not-modern 6", + "ci-lint": "is-node-modern 8 && npm run lint || is-node-not-modern 8", "ci-test": "npm run test && npm run browser && npm run report", "predebug": "rimraf coverage test/tmp", "debug": "verbose=1 ./test/run.js", @@ -34,19 +35,20 @@ "check" ], "engines": { - "node": ">= 0.12" + "node": ">= 6" }, "dependencies": { "asynckit": "^0.4.0", - "combined-stream": "^1.0.6", + "combined-stream": "^1.0.8", "mime-types": "^2.1.12" }, "devDependencies": { + "@types/node": "^12.0.10", "browserify": "^13.1.1", "browserify-istanbul": "^2.0.0", - "coveralls": "^2.11.14", - "cross-spawn": "^4.0.2", - "eslint": "^3.9.1", + "coveralls": "^3.0.4", + "cross-spawn": "^6.0.5", + "eslint": "^6.0.1", "fake": "^0.2.2", "far": "^0.0.7", "formidable": "^1.0.17", @@ -54,12 +56,13 @@ "is-node-modern": "^1.0.0", "istanbul": "^0.4.5", "obake": "^0.1.2", - "phantomjs-prebuilt": "^2.1.13", + "puppeteer": "^1.19.0", "pkgfiles": "^2.3.0", "pre-commit": "^1.1.3", - "request": "2.76.0", - "rimraf": "^2.5.4", - "tape": "^4.6.2" + "request": "^2.88.0", + "rimraf": "^2.7.1", + "tape": "^4.6.2", + "typescript": "^3.5.2" }, "license": "MIT" } diff --git a/node_modules/node-fetch/index.js b/node_modules/isomorphic-fetch/node_modules/node-fetch/index.js similarity index 100% rename from node_modules/node-fetch/index.js rename to node_modules/isomorphic-fetch/node_modules/node-fetch/index.js diff --git a/node_modules/node-fetch/lib/body.js b/node_modules/isomorphic-fetch/node_modules/node-fetch/lib/body.js similarity index 100% rename from node_modules/node-fetch/lib/body.js rename to node_modules/isomorphic-fetch/node_modules/node-fetch/lib/body.js diff --git a/node_modules/node-fetch/lib/fetch-error.js b/node_modules/isomorphic-fetch/node_modules/node-fetch/lib/fetch-error.js similarity index 100% rename from node_modules/node-fetch/lib/fetch-error.js rename to node_modules/isomorphic-fetch/node_modules/node-fetch/lib/fetch-error.js diff --git a/node_modules/node-fetch/lib/headers.js b/node_modules/isomorphic-fetch/node_modules/node-fetch/lib/headers.js similarity index 100% rename from node_modules/node-fetch/lib/headers.js rename to node_modules/isomorphic-fetch/node_modules/node-fetch/lib/headers.js diff --git a/node_modules/@octokit/request/node_modules/node-fetch/lib/index.js b/node_modules/isomorphic-fetch/node_modules/node-fetch/lib/index.js similarity index 55% rename from node_modules/@octokit/request/node_modules/node-fetch/lib/index.js rename to node_modules/isomorphic-fetch/node_modules/node-fetch/lib/index.js index daa44bcaf..f10085472 100644 --- a/node_modules/@octokit/request/node_modules/node-fetch/lib/index.js +++ b/node_modules/isomorphic-fetch/node_modules/node-fetch/lib/index.js @@ -2,31 +2,29 @@ Object.defineProperty(exports, '__esModule', { value: true }); -function _interopDefault (ex) { return (ex && (typeof ex === 'object') && 'default' in ex) ? ex['default'] : ex; } - -var Stream = _interopDefault(require('stream')); -var http = _interopDefault(require('http')); -var Url = _interopDefault(require('url')); -var https = _interopDefault(require('https')); -var zlib = _interopDefault(require('zlib')); - // Based on https://github.com/tmpvar/jsdom/blob/aa85b2abf07766ff7bf5c1f6daafb3726f2f2db5/lib/jsdom/living/blob.js - -// fix for "Readable" isn't a named export issue -const Readable = Stream.Readable; +// (MIT licensed) const BUFFER = Symbol('buffer'); const TYPE = Symbol('type'); +const CLOSED = Symbol('closed'); class Blob { constructor() { + Object.defineProperty(this, Symbol.toStringTag, { + value: 'Blob', + writable: false, + enumerable: false, + configurable: true + }); + + this[CLOSED] = false; this[TYPE] = ''; const blobParts = arguments[0]; const options = arguments[1]; const buffers = []; - let size = 0; if (blobParts) { const a = blobParts; @@ -45,7 +43,6 @@ class Blob { } else { buffer = Buffer.from(typeof element === 'string' ? element : String(element)); } - size += buffer.length; buffers.push(buffer); } } @@ -58,28 +55,13 @@ class Blob { } } get size() { - return this[BUFFER].length; + return this[CLOSED] ? 0 : this[BUFFER].length; } get type() { return this[TYPE]; } - text() { - return Promise.resolve(this[BUFFER].toString()); - } - arrayBuffer() { - const buf = this[BUFFER]; - const ab = buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength); - return Promise.resolve(ab); - } - stream() { - const readable = new Readable(); - readable._read = function () {}; - readable.push(this[BUFFER]); - readable.push(null); - return readable; - } - toString() { - return '[object Blob]'; + get isClosed() { + return this[CLOSED]; } slice() { const size = this.size; @@ -107,18 +89,16 @@ class Blob { const slicedBuffer = buffer.slice(relativeStart, relativeStart + span); const blob = new Blob([], { type: arguments[2] }); blob[BUFFER] = slicedBuffer; + blob[CLOSED] = this[CLOSED]; return blob; } + close() { + this[CLOSED] = true; + } } -Object.defineProperties(Blob.prototype, { - size: { enumerable: true }, - type: { enumerable: true }, - slice: { enumerable: true } -}); - Object.defineProperty(Blob.prototype, Symbol.toStringTag, { - value: 'Blob', + value: 'BlobPrototype', writable: false, enumerable: false, configurable: true @@ -157,28 +137,36 @@ FetchError.prototype = Object.create(Error.prototype); FetchError.prototype.constructor = FetchError; FetchError.prototype.name = 'FetchError'; +/** + * body.js + * + * Body interface provides common methods for Request and Response + */ + +const Stream = require('stream'); + +var _require$1 = require('stream'); + +const PassThrough$1 = _require$1.PassThrough; + + +const DISTURBED = Symbol('disturbed'); + let convert; try { convert = require('encoding').convert; } catch (e) {} -const INTERNALS = Symbol('Body internals'); - -// fix an issue where "PassThrough" isn't a named export for node <10 -const PassThrough = Stream.PassThrough; - /** - * Body mixin + * Body class * - * Ref: https://fetch.spec.whatwg.org/#body + * Cannot use ES6 class because Body must be called with .call(). * * @param Stream body Readable stream * @param Object opts Response options * @return Void */ function Body(body) { - var _this = this; - var _ref = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : {}, _ref$size = _ref.size; @@ -189,43 +177,30 @@ function Body(body) { if (body == null) { // body is undefined or null body = null; + } else if (typeof body === 'string') { + // body is string } else if (isURLSearchParams(body)) { // body is a URLSearchParams - body = Buffer.from(body.toString()); - } else if (isBlob(body)) ; else if (Buffer.isBuffer(body)) ; else if (Object.prototype.toString.call(body) === '[object ArrayBuffer]') { - // body is ArrayBuffer - body = Buffer.from(body); - } else if (ArrayBuffer.isView(body)) { - // body is ArrayBufferView - body = Buffer.from(body.buffer, body.byteOffset, body.byteLength); - } else if (body instanceof Stream) ; else { + } else if (body instanceof Blob) { + // body is blob + } else if (Buffer.isBuffer(body)) { + // body is buffer + } else if (body instanceof Stream) { + // body is stream + } else { // none of the above - // coerce to string then buffer - body = Buffer.from(String(body)); + // coerce to string + body = String(body); } - this[INTERNALS] = { - body, - disturbed: false, - error: null - }; + this.body = body; + this[DISTURBED] = false; this.size = size; this.timeout = timeout; - - if (body instanceof Stream) { - body.on('error', function (err) { - const error = err.name === 'AbortError' ? err : new FetchError(`Invalid response body while trying to fetch ${_this.url}: ${err.message}`, 'system', err); - _this[INTERNALS].error = error; - }); - } } Body.prototype = { - get body() { - return this[INTERNALS].body; - }, - get bodyUsed() { - return this[INTERNALS].disturbed; + return this[DISTURBED]; }, /** @@ -263,13 +238,13 @@ Body.prototype = { * @return Promise */ json() { - var _this2 = this; + var _this = this; return consumeBody.call(this).then(function (buffer) { try { return JSON.parse(buffer.toString()); } catch (err) { - return Body.Promise.reject(new FetchError(`invalid json response body at ${_this2.url} reason: ${err.message}`, 'invalid-json')); + return Body.Promise.reject(new FetchError(`invalid json response body at ${_this.url} reason: ${err.message}`, 'invalid-json')); } }); }, @@ -301,23 +276,14 @@ Body.prototype = { * @return Promise */ textConverted() { - var _this3 = this; + var _this2 = this; return consumeBody.call(this).then(function (buffer) { - return convertBody(buffer, _this3.headers); + return convertBody(buffer, _this2.headers); }); } -}; -// In browsers, all properties are enumerable. -Object.defineProperties(Body.prototype, { - body: { enumerable: true }, - bodyUsed: { enumerable: true }, - arrayBuffer: { enumerable: true }, - blob: { enumerable: true }, - json: { enumerable: true }, - text: { enumerable: true } -}); +}; Body.mixIn = function (proto) { for (const name of Object.getOwnPropertyNames(Body.prototype)) { @@ -330,44 +296,41 @@ Body.mixIn = function (proto) { }; /** - * Consume and convert an entire Body to a Buffer. - * - * Ref: https://fetch.spec.whatwg.org/#concept-body-consume-body + * Decode buffers into utf-8 string * * @return Promise */ -function consumeBody() { - var _this4 = this; +function consumeBody(body) { + var _this3 = this; - if (this[INTERNALS].disturbed) { - return Body.Promise.reject(new TypeError(`body used already for: ${this.url}`)); + if (this[DISTURBED]) { + return Body.Promise.reject(new Error(`body used already for: ${this.url}`)); } - this[INTERNALS].disturbed = true; - - if (this[INTERNALS].error) { - return Body.Promise.reject(this[INTERNALS].error); - } - - let body = this.body; + this[DISTURBED] = true; // body is null - if (body === null) { + if (this.body === null) { return Body.Promise.resolve(Buffer.alloc(0)); } + // body is string + if (typeof this.body === 'string') { + return Body.Promise.resolve(Buffer.from(this.body)); + } + // body is blob - if (isBlob(body)) { - body = body.stream(); + if (this.body instanceof Blob) { + return Body.Promise.resolve(this.body[BUFFER]); } // body is buffer - if (Buffer.isBuffer(body)) { - return Body.Promise.resolve(body); + if (Buffer.isBuffer(this.body)) { + return Body.Promise.resolve(this.body); } // istanbul ignore if: should never happen - if (!(body instanceof Stream)) { + if (!(this.body instanceof Stream)) { return Body.Promise.resolve(Buffer.alloc(0)); } @@ -381,33 +344,26 @@ function consumeBody() { let resTimeout; // allow timeout on slow response body - if (_this4.timeout) { + if (_this3.timeout) { resTimeout = setTimeout(function () { abort = true; - reject(new FetchError(`Response timeout while trying to fetch ${_this4.url} (over ${_this4.timeout}ms)`, 'body-timeout')); - }, _this4.timeout); + reject(new FetchError(`Response timeout while trying to fetch ${_this3.url} (over ${_this3.timeout}ms)`, 'body-timeout')); + }, _this3.timeout); } - // handle stream errors - body.on('error', function (err) { - if (err.name === 'AbortError') { - // if the request was aborted, reject with this Error - abort = true; - reject(err); - } else { - // other errors, such as incorrect content-encoding - reject(new FetchError(`Invalid response body while trying to fetch ${_this4.url}: ${err.message}`, 'system', err)); - } + // handle stream error, such as incorrect content-encoding + _this3.body.on('error', function (err) { + reject(new FetchError(`Invalid response body while trying to fetch ${_this3.url}: ${err.message}`, 'system', err)); }); - body.on('data', function (chunk) { + _this3.body.on('data', function (chunk) { if (abort || chunk === null) { return; } - if (_this4.size && accumBytes + chunk.length > _this4.size) { + if (_this3.size && accumBytes + chunk.length > _this3.size) { abort = true; - reject(new FetchError(`content size at ${_this4.url} over limit: ${_this4.size}`, 'max-size')); + reject(new FetchError(`content size at ${_this3.url} over limit: ${_this3.size}`, 'max-size')); return; } @@ -415,19 +371,13 @@ function consumeBody() { accum.push(chunk); }); - body.on('end', function () { + _this3.body.on('end', function () { if (abort) { return; } clearTimeout(resTimeout); - - try { - resolve(Buffer.concat(accum, accumBytes)); - } catch (err) { - // handle streams that have accumulated too much data (issue #414) - reject(new FetchError(`Could not create Buffer from response body for ${_this4.url}: ${err.message}`, 'system', err)); - } + resolve(Buffer.concat(accum)); }); }); } @@ -508,15 +458,6 @@ function isURLSearchParams(obj) { return obj.constructor.name === 'URLSearchParams' || Object.prototype.toString.call(obj) === '[object URLSearchParams]' || typeof obj.sort === 'function'; } -/** - * Check if `obj` is a W3C `Blob` object (which `File` inherits from) - * @param {*} obj - * @return {boolean} - */ -function isBlob(obj) { - return typeof obj === 'object' && typeof obj.arrayBuffer === 'function' && typeof obj.type === 'string' && typeof obj.stream === 'function' && typeof obj.constructor === 'function' && typeof obj.constructor.name === 'string' && /^(Blob|File)$/.test(obj.constructor.name) && /^(Blob|File)$/.test(obj[Symbol.toStringTag]); -} - /** * Clone body given Res/Req instance * @@ -536,12 +477,12 @@ function clone(instance) { // note: we can't clone the form-data object without having it as a dependency if (body instanceof Stream && typeof body.getBoundary !== 'function') { // tee instance body - p1 = new PassThrough(); - p2 = new PassThrough(); + p1 = new PassThrough$1(); + p2 = new PassThrough$1(); body.pipe(p1); body.pipe(p2); // set instance body to teed body and return the other teed body - instance[INTERNALS].body = p1; + instance.body = p1; body = p2; } @@ -553,11 +494,16 @@ function clone(instance) { * specified in the specification: * https://fetch.spec.whatwg.org/#concept-bodyinit-extract * - * This function assumes that instance.body is present. + * This function assumes that instance.body is present and non-null. * - * @param Mixed instance Any options.body input + * @param Mixed instance Response or Request instance */ -function extractContentType(body) { +function extractContentType(instance) { + const body = instance.body; + + // istanbul ignore if: Currently, because of a guard in Request, body + // can never be null. Included here for completeness. + if (body === null) { // body is null return null; @@ -567,48 +513,38 @@ function extractContentType(body) { } else if (isURLSearchParams(body)) { // body is a URLSearchParams return 'application/x-www-form-urlencoded;charset=UTF-8'; - } else if (isBlob(body)) { + } else if (body instanceof Blob) { // body is blob return body.type || null; } else if (Buffer.isBuffer(body)) { // body is buffer return null; - } else if (Object.prototype.toString.call(body) === '[object ArrayBuffer]') { - // body is ArrayBuffer - return null; - } else if (ArrayBuffer.isView(body)) { - // body is ArrayBufferView - return null; } else if (typeof body.getBoundary === 'function') { // detect form data input from form-data module return `multipart/form-data;boundary=${body.getBoundary()}`; - } else if (body instanceof Stream) { + } else { // body is stream // can't really do much about this return null; - } else { - // Body constructor defaults other things to string - return 'text/plain;charset=UTF-8'; } } -/** - * The Fetch Standard treats this as if "total bytes" is a property on the body. - * For us, we have to explicitly get it with a function. - * - * ref: https://fetch.spec.whatwg.org/#concept-body-total-bytes - * - * @param Body instance Instance of Body - * @return Number? Number of bytes, or null if not possible - */ function getTotalBytes(instance) { const body = instance.body; + // istanbul ignore if: included for completion if (body === null) { // body is null return 0; - } else if (isBlob(body)) { + } else if (typeof body === 'string') { + // body is string + return Buffer.byteLength(body); + } else if (isURLSearchParams(body)) { + // body is URLSearchParams + return Buffer.byteLength(String(body)); + } else if (body instanceof Blob) { + // body is blob return body.size; } else if (Buffer.isBuffer(body)) { // body is buffer @@ -623,16 +559,11 @@ function getTotalBytes(instance) { return null; } else { // body is stream + // can't really do much about this return null; } } -/** - * Write a Body to a Node.js WritableStream (e.g. http.Request) object. - * - * @param Body instance Instance of Body - * @return Void - */ function writeToStream(dest, instance) { const body = instance.body; @@ -640,8 +571,18 @@ function writeToStream(dest, instance) { if (body === null) { // body is null dest.end(); - } else if (isBlob(body)) { - body.stream().pipe(dest); + } else if (typeof body === 'string') { + // body is string + dest.write(body); + dest.end(); + } else if (isURLSearchParams(body)) { + // body is URLSearchParams + dest.write(Buffer.from(String(body))); + dest.end(); + } else if (body instanceof Blob) { + // body is blob + dest.write(body[BUFFER]); + dest.end(); } else if (Buffer.isBuffer(body)) { // body is buffer dest.write(body); @@ -655,45 +596,115 @@ function writeToStream(dest, instance) { // expose Promise Body.Promise = global.Promise; +/** + * A set of utilities borrowed from Node.js' _http_common.js + */ + +/** + * Verifies that the given val is a valid HTTP token + * per the rules defined in RFC 7230 + * See https://tools.ietf.org/html/rfc7230#section-3.2.6 + * + * Allowed characters in an HTTP token: + * ^_`a-z 94-122 + * A-Z 65-90 + * - 45 + * 0-9 48-57 + * ! 33 + * #$%&' 35-39 + * *+ 42-43 + * . 46 + * | 124 + * ~ 126 + * + * This implementation of checkIsHttpToken() loops over the string instead of + * using a regular expression since the former is up to 180% faster with v8 4.9 + * depending on the string length (the shorter the string, the larger the + * performance difference) + * + * Additionally, checkIsHttpToken() is currently designed to be inlinable by v8, + * so take care when making changes to the implementation so that the source + * code size does not exceed v8's default max_inlined_source_size setting. + **/ +/* istanbul ignore next */ +function isValidTokenChar(ch) { + if (ch >= 94 && ch <= 122) return true; + if (ch >= 65 && ch <= 90) return true; + if (ch === 45) return true; + if (ch >= 48 && ch <= 57) return true; + if (ch === 34 || ch === 40 || ch === 41 || ch === 44) return false; + if (ch >= 33 && ch <= 46) return true; + if (ch === 124 || ch === 126) return true; + return false; +} +/* istanbul ignore next */ +function checkIsHttpToken(val) { + if (typeof val !== 'string' || val.length === 0) return false; + if (!isValidTokenChar(val.charCodeAt(0))) return false; + const len = val.length; + if (len > 1) { + if (!isValidTokenChar(val.charCodeAt(1))) return false; + if (len > 2) { + if (!isValidTokenChar(val.charCodeAt(2))) return false; + if (len > 3) { + if (!isValidTokenChar(val.charCodeAt(3))) return false; + for (var i = 4; i < len; i++) { + if (!isValidTokenChar(val.charCodeAt(i))) return false; + } + } + } + } + return true; +} +/** + * True if val contains an invalid field-vchar + * field-value = *( field-content / obs-fold ) + * field-content = field-vchar [ 1*( SP / HTAB ) field-vchar ] + * field-vchar = VCHAR / obs-text + * + * checkInvalidHeaderChar() is currently designed to be inlinable by v8, + * so take care when making changes to the implementation so that the source + * code size does not exceed v8's default max_inlined_source_size setting. + **/ +/* istanbul ignore next */ +function checkInvalidHeaderChar(val) { + val += ''; + if (val.length < 1) return false; + var c = val.charCodeAt(0); + if (c <= 31 && c !== 9 || c > 255 || c === 127) return true; + if (val.length < 2) return false; + c = val.charCodeAt(1); + if (c <= 31 && c !== 9 || c > 255 || c === 127) return true; + if (val.length < 3) return false; + c = val.charCodeAt(2); + if (c <= 31 && c !== 9 || c > 255 || c === 127) return true; + for (var i = 3; i < val.length; ++i) { + c = val.charCodeAt(i); + if (c <= 31 && c !== 9 || c > 255 || c === 127) return true; + } + return false; +} + /** * headers.js * * Headers class offers convenient helpers */ -const invalidTokenRegex = /[^\^_`a-zA-Z\-0-9!#$%&'*+.|~]/; -const invalidHeaderCharRegex = /[^\t\x20-\x7e\x80-\xff]/; - -function validateName(name) { - name = `${name}`; - if (invalidTokenRegex.test(name) || name === '') { +function sanitizeName(name) { + name += ''; + if (!checkIsHttpToken(name)) { throw new TypeError(`${name} is not a legal HTTP header name`); } + return name.toLowerCase(); } -function validateValue(value) { - value = `${value}`; - if (invalidHeaderCharRegex.test(value)) { +function sanitizeValue(value) { + value += ''; + if (checkInvalidHeaderChar(value)) { throw new TypeError(`${value} is not a legal HTTP header value`); } -} - -/** - * Find the key in the map object given a header name. - * - * Returns undefined if not found. - * - * @param String name Header name - * @return String|Undefined - */ -function find(map, name) { - name = name.toLowerCase(); - for (const key in map) { - if (key.toLowerCase() === name) { - return key; - } - } - return undefined; + return value; } const MAP = Symbol('map'); @@ -724,7 +735,9 @@ class Headers { // We don't worry about converting prop to ByteString here as append() // will handle it. - if (init == null) ; else if (typeof init === 'object') { + if (init == null) { + // no op + } else if (typeof init === 'object') { const method = init[Symbol.iterator]; if (method != null) { if (typeof method !== 'function') { @@ -757,23 +770,28 @@ class Headers { } else { throw new TypeError('Provided initializer must be an object'); } + + Object.defineProperty(this, Symbol.toStringTag, { + value: 'Headers', + writable: false, + enumerable: false, + configurable: true + }); } /** - * Return combined header value given name + * Return first header value given name * * @param String name Header name * @return Mixed */ get(name) { - name = `${name}`; - validateName(name); - const key = find(this[MAP], name); - if (key === undefined) { + const list = this[MAP][sanitizeName(name)]; + if (!list) { return null; } - return this[MAP][key].join(', '); + return list.join(', '); } /** @@ -786,7 +804,7 @@ class Headers { forEach(callback) { let thisArg = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : undefined; - let pairs = getHeaders(this); + let pairs = getHeaderPairs(this); let i = 0; while (i < pairs.length) { var _pairs$i = pairs[i]; @@ -794,7 +812,7 @@ class Headers { value = _pairs$i[1]; callback.call(thisArg, value, name, this); - pairs = getHeaders(this); + pairs = getHeaderPairs(this); i++; } } @@ -807,12 +825,7 @@ class Headers { * @return Void */ set(name, value) { - name = `${name}`; - value = `${value}`; - validateName(name); - validateValue(value); - const key = find(this[MAP], name); - this[MAP][key !== undefined ? key : name] = [value]; + this[MAP][sanitizeName(name)] = [sanitizeValue(value)]; } /** @@ -823,16 +836,12 @@ class Headers { * @return Void */ append(name, value) { - name = `${name}`; - value = `${value}`; - validateName(name); - validateValue(value); - const key = find(this[MAP], name); - if (key !== undefined) { - this[MAP][key].push(value); - } else { - this[MAP][name] = [value]; + if (!this.has(name)) { + this.set(name, value); + return; } + + this[MAP][sanitizeName(name)].push(sanitizeValue(value)); } /** @@ -842,9 +851,7 @@ class Headers { * @return Boolean */ has(name) { - name = `${name}`; - validateName(name); - return find(this[MAP], name) !== undefined; + return !!this[MAP][sanitizeName(name)]; } /** @@ -854,12 +861,7 @@ class Headers { * @return Void */ delete(name) { - name = `${name}`; - validateName(name); - const key = find(this[MAP], name); - if (key !== undefined) { - delete this[MAP][key]; - } + delete this[MAP][sanitizeName(name)]; } /** @@ -903,34 +905,18 @@ class Headers { Headers.prototype.entries = Headers.prototype[Symbol.iterator]; Object.defineProperty(Headers.prototype, Symbol.toStringTag, { - value: 'Headers', + value: 'HeadersPrototype', writable: false, enumerable: false, configurable: true }); -Object.defineProperties(Headers.prototype, { - get: { enumerable: true }, - forEach: { enumerable: true }, - set: { enumerable: true }, - append: { enumerable: true }, - has: { enumerable: true }, - delete: { enumerable: true }, - keys: { enumerable: true }, - values: { enumerable: true }, - entries: { enumerable: true } -}); - -function getHeaders(headers) { - let kind = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : 'key+value'; - +function getHeaderPairs(headers, kind) { const keys = Object.keys(headers[MAP]).sort(); return keys.map(kind === 'key' ? function (k) { - return k.toLowerCase(); - } : kind === 'value' ? function (k) { - return headers[MAP][k].join(', '); + return [k]; } : function (k) { - return [k.toLowerCase(), headers[MAP][k].join(', ')]; + return [k, headers.get(k)]; }); } @@ -958,7 +944,7 @@ const HeadersIteratorPrototype = Object.setPrototypeOf({ kind = _INTERNAL.kind, index = _INTERNAL.index; - const values = getHeaders(target, kind); + const values = getHeaderPairs(target, kind); const len = values.length; if (index >= len) { return { @@ -967,10 +953,20 @@ const HeadersIteratorPrototype = Object.setPrototypeOf({ }; } + const pair = values[index]; this[INTERNAL].index = index + 1; + let result; + if (kind === 'key') { + result = pair[0]; + } else if (kind === 'value') { + result = pair[1]; + } else { + result = pair; + } + return { - value: values[index], + value: result, done: false }; } @@ -984,59 +980,14 @@ Object.defineProperty(HeadersIteratorPrototype, Symbol.toStringTag, { }); /** - * Export the Headers object in a form that Node.js can consume. - * - * @param Headers headers - * @return Object - */ -function exportNodeCompatibleHeaders(headers) { - const obj = Object.assign({ __proto__: null }, headers[MAP]); - - // http.request() only supports string as Host header. This hack makes - // specifying custom Host header possible. - const hostHeaderKey = find(headers[MAP], 'Host'); - if (hostHeaderKey !== undefined) { - obj[hostHeaderKey] = obj[hostHeaderKey][0]; - } - - return obj; -} - -/** - * Create a Headers object from an object of headers, ignoring those that do - * not conform to HTTP grammar productions. + * response.js * - * @param Object obj Object of headers - * @return Headers + * Response class provides content decoding */ -function createHeadersLenient(obj) { - const headers = new Headers(); - for (const name of Object.keys(obj)) { - if (invalidTokenRegex.test(name)) { - continue; - } - if (Array.isArray(obj[name])) { - for (const val of obj[name]) { - if (invalidHeaderCharRegex.test(val)) { - continue; - } - if (headers[MAP][name] === undefined) { - headers[MAP][name] = [val]; - } else { - headers[MAP][name].push(val); - } - } - } else if (!invalidHeaderCharRegex.test(obj[name])) { - headers[MAP][name] = [obj[name]]; - } - } - return headers; -} -const INTERNALS$1 = Symbol('Response internals'); +var _require$2 = require('http'); -// fix an issue where "STATUS_CODES" aren't a named export for node <10 -const STATUS_CODES = http.STATUS_CODES; +const STATUS_CODES = _require$2.STATUS_CODES; /** * Response class @@ -1045,6 +996,7 @@ const STATUS_CODES = http.STATUS_CODES; * @param Object opts Response options * @return Void */ + class Response { constructor() { let body = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : null; @@ -1052,50 +1004,25 @@ class Response { Body.call(this, body, opts); - const status = opts.status || 200; - const headers = new Headers(opts.headers); - - if (body != null && !headers.has('Content-Type')) { - const contentType = extractContentType(body); - if (contentType) { - headers.append('Content-Type', contentType); - } - } - - this[INTERNALS$1] = { - url: opts.url, - status, - statusText: opts.statusText || STATUS_CODES[status], - headers, - counter: opts.counter - }; - } + this.url = opts.url; + this.status = opts.status || 200; + this.statusText = opts.statusText || STATUS_CODES[this.status]; - get url() { - return this[INTERNALS$1].url || ''; - } + this.headers = new Headers(opts.headers); - get status() { - return this[INTERNALS$1].status; + Object.defineProperty(this, Symbol.toStringTag, { + value: 'Response', + writable: false, + enumerable: false, + configurable: true + }); } /** * Convenience property representing if the request ended normally */ get ok() { - return this[INTERNALS$1].status >= 200 && this[INTERNALS$1].status < 300; - } - - get redirected() { - return this[INTERNALS$1].counter > 0; - } - - get statusText() { - return this[INTERNALS$1].statusText; - } - - get headers() { - return this[INTERNALS$1].headers; + return this.status >= 200 && this.status < 300; } /** @@ -1104,58 +1031,39 @@ class Response { * @return Response */ clone() { + return new Response(clone(this), { url: this.url, status: this.status, statusText: this.statusText, headers: this.headers, - ok: this.ok, - redirected: this.redirected + ok: this.ok }); } } Body.mixIn(Response.prototype); -Object.defineProperties(Response.prototype, { - url: { enumerable: true }, - status: { enumerable: true }, - ok: { enumerable: true }, - redirected: { enumerable: true }, - statusText: { enumerable: true }, - headers: { enumerable: true }, - clone: { enumerable: true } -}); - Object.defineProperty(Response.prototype, Symbol.toStringTag, { - value: 'Response', + value: 'ResponsePrototype', writable: false, enumerable: false, configurable: true }); -const INTERNALS$2 = Symbol('Request internals'); - -// fix an issue where "format", "parse" aren't a named export for node <10 -const parse_url = Url.parse; -const format_url = Url.format; - -const streamDestructionSupported = 'destroy' in Stream.Readable.prototype; - /** - * Check if a value is an instance of Request. + * request.js * - * @param Mixed input - * @return Boolean + * Request class contains server only options */ -function isRequest(input) { - return typeof input === 'object' && typeof input[INTERNALS$2] === 'object'; -} -function isAbortSignal(signal) { - const proto = signal && typeof signal === 'object' && Object.getPrototypeOf(signal); - return !!(proto && proto.constructor.name === 'AbortSignal'); -} +var _require$3 = require('url'); + +const format_url = _require$3.format; +const parse_url = _require$3.parse; + + +const PARSED_URL = Symbol('url'); /** * Request class @@ -1171,7 +1079,7 @@ class Request { let parsedURL; // normalize input - if (!isRequest(input)) { + if (!(input instanceof Request)) { if (input && input.href) { // in order to support Node.js' Url objects; though WHATWG's URL objects // will fall into this branch also (since their `toString()` will return @@ -1187,68 +1095,47 @@ class Request { } let method = init.method || input.method || 'GET'; - method = method.toUpperCase(); - if ((init.body != null || isRequest(input) && input.body !== null) && (method === 'GET' || method === 'HEAD')) { + if ((init.body != null || input instanceof Request && input.body !== null) && (method === 'GET' || method === 'HEAD')) { throw new TypeError('Request with GET/HEAD method cannot have body'); } - let inputBody = init.body != null ? init.body : isRequest(input) && input.body !== null ? clone(input) : null; + let inputBody = init.body != null ? init.body : input instanceof Request && input.body !== null ? clone(input) : null; Body.call(this, inputBody, { timeout: init.timeout || input.timeout || 0, size: init.size || input.size || 0 }); - const headers = new Headers(init.headers || input.headers || {}); + // fetch spec options + this.method = method.toUpperCase(); + this.redirect = init.redirect || input.redirect || 'follow'; + this.headers = new Headers(init.headers || input.headers || {}); - if (inputBody != null && !headers.has('Content-Type')) { - const contentType = extractContentType(inputBody); - if (contentType) { - headers.append('Content-Type', contentType); + if (init.body != null) { + const contentType = extractContentType(this); + if (contentType !== null && !this.headers.has('Content-Type')) { + this.headers.append('Content-Type', contentType); } } - let signal = isRequest(input) ? input.signal : null; - if ('signal' in init) signal = init.signal; - - if (signal != null && !isAbortSignal(signal)) { - throw new TypeError('Expected signal to be an instanceof AbortSignal'); - } - - this[INTERNALS$2] = { - method, - redirect: init.redirect || input.redirect || 'follow', - headers, - parsedURL, - signal - }; - - // node-fetch-only options + // server only options this.follow = init.follow !== undefined ? init.follow : input.follow !== undefined ? input.follow : 20; this.compress = init.compress !== undefined ? init.compress : input.compress !== undefined ? input.compress : true; this.counter = init.counter || input.counter || 0; this.agent = init.agent || input.agent; - } - get method() { - return this[INTERNALS$2].method; + this[PARSED_URL] = parsedURL; + Object.defineProperty(this, Symbol.toStringTag, { + value: 'Request', + writable: false, + enumerable: false, + configurable: true + }); } get url() { - return format_url(this[INTERNALS$2].parsedURL); - } - - get headers() { - return this[INTERNALS$2].headers; - } - - get redirect() { - return this[INTERNALS$2].redirect; - } - - get signal() { - return this[INTERNALS$2].signal; + return format_url(this[PARSED_URL]); } /** @@ -1264,32 +1151,17 @@ class Request { Body.mixIn(Request.prototype); Object.defineProperty(Request.prototype, Symbol.toStringTag, { - value: 'Request', + value: 'RequestPrototype', writable: false, enumerable: false, configurable: true }); -Object.defineProperties(Request.prototype, { - method: { enumerable: true }, - url: { enumerable: true }, - headers: { enumerable: true }, - redirect: { enumerable: true }, - clone: { enumerable: true }, - signal: { enumerable: true } -}); - -/** - * Convert a Request to Node.js http request options. - * - * @param Request A Request instance - * @return Object The options object to be passed to http.request - */ function getNodeRequestOptions(request) { - const parsedURL = request[INTERNALS$2].parsedURL; - const headers = new Headers(request[INTERNALS$2].headers); + const parsedURL = request[PARSED_URL]; + const headers = new Headers(request.headers); - // fetch step 1.3 + // fetch step 3 if (!headers.has('Accept')) { headers.set('Accept', '*/*'); } @@ -1303,11 +1175,7 @@ function getNodeRequestOptions(request) { throw new TypeError('Only HTTP(S) protocols are supported'); } - if (request.signal && request.body instanceof Stream.Readable && !streamDestructionSupported) { - throw new Error('Cancellation of streamed requests with AbortSignal is not supported in node < 8'); - } - - // HTTP-network-or-cache fetch steps 2.4-2.7 + // HTTP-network-or-cache fetch steps 5-9 let contentLengthValue = null; if (request.body == null && /^(POST|PUT)$/i.test(request.method)) { contentLengthValue = '0'; @@ -1322,64 +1190,47 @@ function getNodeRequestOptions(request) { headers.set('Content-Length', contentLengthValue); } - // HTTP-network-or-cache fetch step 2.11 + // HTTP-network-or-cache fetch step 12 if (!headers.has('User-Agent')) { headers.set('User-Agent', 'node-fetch/1.0 (+https://github.com/bitinn/node-fetch)'); } - // HTTP-network-or-cache fetch step 2.15 - if (request.compress && !headers.has('Accept-Encoding')) { + // HTTP-network-or-cache fetch step 16 + if (request.compress) { headers.set('Accept-Encoding', 'gzip,deflate'); } - - let agent = request.agent; - if (typeof agent === 'function') { - agent = agent(parsedURL); - } - - if (!headers.has('Connection') && !agent) { + if (!headers.has('Connection') && !request.agent) { headers.set('Connection', 'close'); } - // HTTP-network fetch step 4.2 + // HTTP-network fetch step 4 // chunked encoding is handled by Node.js return Object.assign({}, parsedURL, { method: request.method, - headers: exportNodeCompatibleHeaders(headers), - agent + headers: headers.raw(), + agent: request.agent }); } /** - * abort-error.js + * index.js * - * AbortError interface for cancelled requests + * a request API compatible with window.fetch */ -/** - * Create AbortError instance - * - * @param String message Error message for human - * @return AbortError - */ -function AbortError(message) { - Error.call(this, message); +const http = require('http'); +const https = require('https'); - this.type = 'aborted'; - this.message = message; +var _require = require('stream'); - // hide custom error implementation details from end-users - Error.captureStackTrace(this, this.constructor); -} +const PassThrough = _require.PassThrough; + +var _require2 = require('url'); -AbortError.prototype = Object.create(Error.prototype); -AbortError.prototype.constructor = AbortError; -AbortError.prototype.name = 'AbortError'; +const resolve_url = _require2.resolve; -// fix an issue where "PassThrough", "resolve" aren't a named export for node <10 -const PassThrough$1 = Stream.PassThrough; -const resolve_url = Url.resolve; +const zlib = require('zlib'); /** * Fetch function @@ -1404,157 +1255,93 @@ function fetch(url, opts) { const options = getNodeRequestOptions(request); const send = (options.protocol === 'https:' ? https : http).request; - const signal = request.signal; - let response = null; - - const abort = function abort() { - let error = new AbortError('The user aborted a request.'); - reject(error); - if (request.body && request.body instanceof Stream.Readable) { - request.body.destroy(error); - } - if (!response || !response.body) return; - response.body.emit('error', error); - }; - - if (signal && signal.aborted) { - abort(); - return; + // http.request only support string as host header, this hack make custom host header possible + if (options.headers.host) { + options.headers.host = options.headers.host[0]; } - const abortAndFinalize = function abortAndFinalize() { - abort(); - finalize(); - }; - // send request const req = send(options); let reqTimeout; - if (signal) { - signal.addEventListener('abort', abortAndFinalize); - } - - function finalize() { - req.abort(); - if (signal) signal.removeEventListener('abort', abortAndFinalize); - clearTimeout(reqTimeout); - } - if (request.timeout) { req.once('socket', function (socket) { reqTimeout = setTimeout(function () { + req.abort(); reject(new FetchError(`network timeout at: ${request.url}`, 'request-timeout')); - finalize(); }, request.timeout); }); } req.on('error', function (err) { + clearTimeout(reqTimeout); reject(new FetchError(`request to ${request.url} failed, reason: ${err.message}`, 'system', err)); - finalize(); }); req.on('response', function (res) { clearTimeout(reqTimeout); - const headers = createHeadersLenient(res.headers); - - // HTTP fetch step 5 - if (fetch.isRedirect(res.statusCode)) { - // HTTP fetch step 5.2 - const location = headers.get('Location'); - - // HTTP fetch step 5.3 - const locationURL = location === null ? null : resolve_url(request.url, location); - - // HTTP fetch step 5.5 - switch (request.redirect) { - case 'error': - reject(new FetchError(`redirect mode is set to error: ${request.url}`, 'no-redirect')); - finalize(); - return; - case 'manual': - // node-fetch-specific step: make manual redirect a bit easier to use by setting the Location header value to the resolved URL. - if (locationURL !== null) { - // handle corrupted header - try { - headers.set('Location', locationURL); - } catch (err) { - // istanbul ignore next: nodejs server prevent invalid response headers, we can't test this through normal request - reject(err); - } - } - break; - case 'follow': - // HTTP-redirect fetch step 2 - if (locationURL === null) { - break; - } - - // HTTP-redirect fetch step 5 - if (request.counter >= request.follow) { - reject(new FetchError(`maximum redirect reached at: ${request.url}`, 'max-redirect')); - finalize(); - return; - } - - // HTTP-redirect fetch step 6 (counter increment) - // Create a new Request object. - const requestOpts = { - headers: new Headers(request.headers), - follow: request.follow, - counter: request.counter + 1, - agent: request.agent, - compress: request.compress, - method: request.method, - body: request.body, - signal: request.signal, - timeout: request.timeout - }; - - // HTTP-redirect fetch step 9 - if (res.statusCode !== 303 && request.body && getTotalBytes(request) === null) { - reject(new FetchError('Cannot follow redirect with body being a readable stream', 'unsupported-redirect')); - finalize(); - return; - } - - // HTTP-redirect fetch step 11 - if (res.statusCode === 303 || (res.statusCode === 301 || res.statusCode === 302) && request.method === 'POST') { - requestOpts.method = 'GET'; - requestOpts.body = undefined; - requestOpts.headers.delete('content-length'); - } - - // HTTP-redirect fetch step 15 - resolve(fetch(new Request(locationURL, requestOpts))); - finalize(); - return; + // handle redirect + if (fetch.isRedirect(res.statusCode) && request.redirect !== 'manual') { + if (request.redirect === 'error') { + reject(new FetchError(`redirect mode is set to error: ${request.url}`, 'no-redirect')); + return; } + + if (request.counter >= request.follow) { + reject(new FetchError(`maximum redirect reached at: ${request.url}`, 'max-redirect')); + return; + } + + if (!res.headers.location) { + reject(new FetchError(`redirect location header missing at: ${request.url}`, 'invalid-redirect')); + return; + } + + // per fetch spec, for POST request with 301/302 response, or any request with 303 response, use GET when following redirect + if (res.statusCode === 303 || (res.statusCode === 301 || res.statusCode === 302) && request.method === 'POST') { + request.method = 'GET'; + request.body = null; + request.headers.delete('content-length'); + } + + request.counter++; + + resolve(fetch(resolve_url(request.url, res.headers.location), request)); + return; } - // prepare response - res.once('end', function () { - if (signal) signal.removeEventListener('abort', abortAndFinalize); - }); - let body = res.pipe(new PassThrough$1()); + // normalize location header for manual redirect mode + const headers = new Headers(); + for (const name of Object.keys(res.headers)) { + if (Array.isArray(res.headers[name])) { + for (const val of res.headers[name]) { + headers.append(name, val); + } + } else { + headers.append(name, res.headers[name]); + } + } + if (request.redirect === 'manual' && headers.has('location')) { + headers.set('location', resolve_url(request.url, headers.get('location'))); + } + // prepare response + let body = res.pipe(new PassThrough()); const response_options = { url: request.url, status: res.statusCode, statusText: res.statusMessage, headers: headers, size: request.size, - timeout: request.timeout, - counter: request.counter + timeout: request.timeout }; - // HTTP-network fetch step 12.1.1.3 + // HTTP-network fetch step 16.1.2 const codings = headers.get('Content-Encoding'); - // HTTP-network fetch step 12.1.1.4: handle content codings + // HTTP-network fetch step 16.1.3: handle content codings // in following scenarios we ignore compression support // 1. compression support is disabled @@ -1563,8 +1350,7 @@ function fetch(url, opts) { // 4. no content response (204) // 5. content not modified response (304) if (!request.compress || request.method === 'HEAD' || codings === null || res.statusCode === 204 || res.statusCode === 304) { - response = new Response(body, response_options); - resolve(response); + resolve(new Response(body, response_options)); return; } @@ -1581,8 +1367,7 @@ function fetch(url, opts) { // for gzip if (codings == 'gzip' || codings == 'x-gzip') { body = body.pipe(zlib.createGunzip(zlibOptions)); - response = new Response(body, response_options); - resolve(response); + resolve(new Response(body, response_options)); return; } @@ -1590,7 +1375,7 @@ function fetch(url, opts) { if (codings == 'deflate' || codings == 'x-deflate') { // handle the infamous raw deflate response from old servers // a hack for old IIS and Apache servers - const raw = res.pipe(new PassThrough$1()); + const raw = res.pipe(new PassThrough()); raw.once('data', function (chunk) { // see http://stackoverflow.com/questions/37519828 if ((chunk[0] & 0x0F) === 0x08) { @@ -1598,28 +1383,19 @@ function fetch(url, opts) { } else { body = body.pipe(zlib.createInflateRaw()); } - response = new Response(body, response_options); - resolve(response); + resolve(new Response(body, response_options)); }); return; } - // for br - if (codings == 'br' && typeof zlib.createBrotliDecompress === 'function') { - body = body.pipe(zlib.createBrotliDecompress()); - response = new Response(body, response_options); - resolve(response); - return; - } - // otherwise, use response as-is - response = new Response(body, response_options); - resolve(response); + resolve(new Response(body, response_options)); }); writeToStream(req, request); }); } + /** * Redirect code matching * @@ -1634,8 +1410,6 @@ fetch.isRedirect = function (code) { fetch.Promise = global.Promise; module.exports = exports = fetch; -Object.defineProperty(exports, "__esModule", { value: true }); -exports.default = exports; exports.Headers = Headers; exports.Request = Request; exports.Response = Response; diff --git a/node_modules/node-fetch/lib/request.js b/node_modules/isomorphic-fetch/node_modules/node-fetch/lib/request.js similarity index 100% rename from node_modules/node-fetch/lib/request.js rename to node_modules/isomorphic-fetch/node_modules/node-fetch/lib/request.js diff --git a/node_modules/node-fetch/lib/response.js b/node_modules/isomorphic-fetch/node_modules/node-fetch/lib/response.js similarity index 100% rename from node_modules/node-fetch/lib/response.js rename to node_modules/isomorphic-fetch/node_modules/node-fetch/lib/response.js diff --git a/node_modules/isomorphic-fetch/node_modules/node-fetch/package.json b/node_modules/isomorphic-fetch/node_modules/node-fetch/package.json new file mode 100644 index 000000000..6bf8e40e8 --- /dev/null +++ b/node_modules/isomorphic-fetch/node_modules/node-fetch/package.json @@ -0,0 +1,42 @@ +{ + "name": "node-fetch", + "version": "1.7.3", + "description": "A light-weight module that brings window.fetch to node.js and io.js", + "main": "index.js", + "scripts": { + "test": "mocha test/test.js", + "report": "istanbul cover _mocha -- -R spec test/test.js", + "coverage": "istanbul cover _mocha --report lcovonly -- -R spec test/test.js && codecov" + }, + "repository": { + "type": "git", + "url": "https://github.com/bitinn/node-fetch.git" + }, + "keywords": [ + "fetch", + "http", + "promise" + ], + "author": "David Frank", + "license": "MIT", + "bugs": { + "url": "https://github.com/bitinn/node-fetch/issues" + }, + "homepage": "https://github.com/bitinn/node-fetch", + "devDependencies": { + "bluebird": "^3.3.4", + "chai": "^3.5.0", + "chai-as-promised": "^5.2.0", + "codecov": "^1.0.1", + "form-data": ">=1.0.0", + "istanbul": "^0.4.2", + "mocha": "^2.1.0", + "parted": "^0.1.1", + "promise": "^7.1.1", + "resumer": "0.0.0" + }, + "dependencies": { + "encoding": "^0.1.11", + "is-stream": "^1.0.1" + } +} diff --git a/node_modules/mime-db/db.json b/node_modules/mime-db/db.json index f63a57ca6..a5fc98708 100644 --- a/node_modules/mime-db/db.json +++ b/node_modules/mime-db/db.json @@ -452,9 +452,6 @@ "application/fits": { "source": "iana" }, - "application/flexfec": { - "source": "iana" - }, "application/font-sfnt": { "source": "iana" }, @@ -816,9 +813,6 @@ "application/mikey": { "source": "iana" }, - "application/mipc": { - "source": "iana" - }, "application/mmt-aei+xml": { "source": "iana", "compressible": true @@ -1352,9 +1346,6 @@ "application/simplesymbolcontainer": { "source": "iana" }, - "application/sipc": { - "source": "iana" - }, "application/slate": { "source": "iana" }, @@ -1420,10 +1411,6 @@ "source": "iana", "compressible": true }, - "application/swid+xml": { - "source": "iana", - "compressible": true - }, "application/tamp-apex-update": { "source": "iana" }, @@ -1497,10 +1484,6 @@ "application/tnauthlist": { "source": "iana" }, - "application/toml": { - "compressible": true, - "extensions": ["toml"] - }, "application/trickle-ice-sdpfrag": { "source": "iana" }, @@ -1657,10 +1640,6 @@ "source": "iana", "compressible": true }, - "application/vnd.3gpp.mcvideo-info+xml": { - "source": "iana", - "compressible": true - }, "application/vnd.3gpp.mcvideo-location-info+xml": { "source": "iana", "compressible": true @@ -1833,9 +1812,6 @@ "source": "iana", "compressible": true }, - "application/vnd.android.ota": { - "source": "iana" - }, "application/vnd.android.package-archive": { "source": "apache", "compressible": false, @@ -1941,9 +1917,6 @@ "application/vnd.banana-accounting": { "source": "iana" }, - "application/vnd.bbf.usp.error": { - "source": "iana" - }, "application/vnd.bbf.usp.msg": { "source": "iana" }, @@ -1979,12 +1952,6 @@ "source": "iana", "extensions": ["bmi"] }, - "application/vnd.bpf": { - "source": "iana" - }, - "application/vnd.bpf3": { - "source": "iana" - }, "application/vnd.businessobjects": { "source": "iana", "extensions": ["rep"] @@ -2024,9 +1991,6 @@ "source": "iana", "extensions": ["mmd"] }, - "application/vnd.ciedi": { - "source": "iana" - }, "application/vnd.cinderella": { "source": "iana", "extensions": ["cdy"] @@ -2143,13 +2107,6 @@ "compressible": true, "extensions": ["wbs"] }, - "application/vnd.cryptii.pipe+json": { - "source": "iana", - "compressible": true - }, - "application/vnd.crypto-shade-file": { - "source": "iana" - }, "application/vnd.ctc-posml": { "source": "iana", "extensions": ["pml"] @@ -2583,10 +2540,6 @@ "application/vnd.ffsns": { "source": "iana" }, - "application/vnd.ficlab.flb+zip": { - "source": "iana", - "compressible": false - }, "application/vnd.filmit.zfc": { "source": "iana" }, @@ -3051,10 +3004,6 @@ "source": "iana", "extensions": ["fcs"] }, - "application/vnd.iso11783-10+zip": { - "source": "iana", - "compressible": false - }, "application/vnd.jam": { "source": "iana", "extensions": ["jam"] @@ -3154,9 +3103,6 @@ "source": "iana", "extensions": ["sse"] }, - "application/vnd.las": { - "source": "iana" - }, "application/vnd.las.las+json": { "source": "iana", "compressible": true @@ -3166,9 +3112,6 @@ "compressible": true, "extensions": ["lasxml"] }, - "application/vnd.laszip": { - "source": "iana" - }, "application/vnd.leap+json": { "source": "iana", "compressible": true @@ -3186,13 +3129,6 @@ "compressible": true, "extensions": ["lbe"] }, - "application/vnd.logipipe.circuit+zip": { - "source": "iana", - "compressible": false - }, - "application/vnd.loom": { - "source": "iana" - }, "application/vnd.lotus-1-2-3": { "source": "iana", "extensions": ["123"] @@ -4647,9 +4583,6 @@ "source": "iana", "extensions": ["semf"] }, - "application/vnd.shade-save-file": { - "source": "iana" - }, "application/vnd.shana.informed.formdata": { "source": "iana", "extensions": ["ifm"] @@ -4670,10 +4603,6 @@ "source": "iana", "compressible": true }, - "application/vnd.shopkick+json": { - "source": "iana", - "compressible": true - }, "application/vnd.sigrok.session": { "source": "iana" }, @@ -4991,9 +4920,6 @@ "application/vnd.veryant.thin": { "source": "iana" }, - "application/vnd.ves.encrypted": { - "source": "iana" - }, "application/vnd.vidsoft.vidconference": { "source": "iana" }, @@ -6062,9 +5988,6 @@ "audio/evs": { "source": "iana" }, - "audio/flexfec": { - "source": "iana" - }, "audio/fwdred": { "source": "iana" }, @@ -6551,7 +6474,6 @@ }, "font/ttf": { "source": "iana", - "compressible": true, "extensions": ["ttf"] }, "font/woff": { @@ -6622,14 +6544,6 @@ "source": "iana", "extensions": ["heifs"] }, - "image/hej2k": { - "source": "iana", - "extensions": ["hej2"] - }, - "image/hsj2": { - "source": "iana", - "extensions": ["hsj2"] - }, "image/ief": { "source": "iana", "extensions": ["ief"] @@ -6648,14 +6562,6 @@ "compressible": false, "extensions": ["jpeg","jpg","jpe"] }, - "image/jph": { - "source": "iana", - "extensions": ["jph"] - }, - "image/jphc": { - "source": "iana", - "extensions": ["jhc"] - }, "image/jpm": { "source": "iana", "compressible": false, @@ -6670,30 +6576,6 @@ "source": "iana", "extensions": ["jxr"] }, - "image/jxra": { - "source": "iana", - "extensions": ["jxra"] - }, - "image/jxrs": { - "source": "iana", - "extensions": ["jxrs"] - }, - "image/jxs": { - "source": "iana", - "extensions": ["jxs"] - }, - "image/jxsc": { - "source": "iana", - "extensions": ["jxsc"] - }, - "image/jxsi": { - "source": "iana", - "extensions": ["jxsi"] - }, - "image/jxss": { - "source": "iana", - "extensions": ["jxss"] - }, "image/ktx": { "source": "iana", "extensions": ["ktx"] @@ -6807,9 +6689,6 @@ "image/vnd.mozilla.apng": { "source": "iana" }, - "image/vnd.ms-dds": { - "extensions": ["dds"] - }, "image/vnd.ms-modi": { "source": "iana", "extensions": ["mdi"] @@ -7162,7 +7041,8 @@ "source": "iana" }, "multipart/mixed": { - "source": "iana" + "source": "iana", + "compressible": false }, "multipart/multilingual": { "source": "iana" @@ -7240,9 +7120,6 @@ "text/enriched": { "source": "iana" }, - "text/flexfec": { - "source": "iana" - }, "text/fwdred": { "source": "iana" }, @@ -7429,9 +7306,6 @@ "text/vnd.esmertec.theme-descriptor": { "source": "iana" }, - "text/vnd.ficlab.flt": { - "source": "iana" - }, "text/vnd.fly": { "source": "iana", "extensions": ["fly"] @@ -7485,9 +7359,6 @@ "text/vnd.si.uricatalogue": { "source": "iana" }, - "text/vnd.sosi": { - "source": "iana" - }, "text/vnd.sun.j2me.app-descriptor": { "source": "iana", "extensions": ["jad"] @@ -7640,9 +7511,6 @@ "video/encaprtp": { "source": "iana" }, - "video/flexfec": { - "source": "iana" - }, "video/h261": { "source": "iana", "extensions": ["h261"] @@ -7882,9 +7750,6 @@ "source": "iana", "extensions": ["viv"] }, - "video/vnd.youtube.yt": { - "source": "iana" - }, "video/vp8": { "source": "iana" }, diff --git a/node_modules/mime-db/package.json b/node_modules/mime-db/package.json index ee06d82ef..07db1ecce 100644 --- a/node_modules/mime-db/package.json +++ b/node_modules/mime-db/package.json @@ -1,7 +1,7 @@ { "name": "mime-db", "description": "Media Type Database", - "version": "1.42.0", + "version": "1.40.0", "contributors": [ "Douglas Christopher Wilson ", "Jonathan Ong (http://jongleberry.com)", @@ -19,20 +19,20 @@ ], "repository": "jshttp/mime-db", "devDependencies": { - "bluebird": "3.5.5", + "bluebird": "3.5.4", "co": "4.6.0", "cogent": "1.0.1", - "csv-parse": "4.4.6", - "eslint": "6.4.0", - "eslint-config-standard": "14.1.0", - "eslint-plugin-import": "2.18.2", - "eslint-plugin-node": "10.0.0", - "eslint-plugin-promise": "4.2.1", - "eslint-plugin-standard": "4.0.1", + "csv-parse": "4.3.4", + "eslint": "5.16.0", + "eslint-config-standard": "12.0.0", + "eslint-plugin-import": "2.16.0", + "eslint-plugin-node": "8.0.1", + "eslint-plugin-promise": "4.1.1", + "eslint-plugin-standard": "4.0.0", "gnode": "0.1.2", - "mocha": "6.2.0", - "nyc": "14.1.1", - "raw-body": "2.4.1", + "mocha": "6.1.4", + "nyc": "14.0.0", + "raw-body": "2.3.3", "stream-to-array": "2.3.0" }, "files": [ diff --git a/node_modules/@octokit/request/node_modules/node-fetch/browser.js b/node_modules/node-fetch/browser.js similarity index 100% rename from node_modules/@octokit/request/node_modules/node-fetch/browser.js rename to node_modules/node-fetch/browser.js diff --git a/node_modules/@octokit/request/node_modules/node-fetch/lib/index.es.js b/node_modules/node-fetch/lib/index.es.js similarity index 100% rename from node_modules/@octokit/request/node_modules/node-fetch/lib/index.es.js rename to node_modules/node-fetch/lib/index.es.js diff --git a/node_modules/node-fetch/lib/index.js b/node_modules/node-fetch/lib/index.js index f10085472..daa44bcaf 100644 --- a/node_modules/node-fetch/lib/index.js +++ b/node_modules/node-fetch/lib/index.js @@ -2,29 +2,31 @@ Object.defineProperty(exports, '__esModule', { value: true }); +function _interopDefault (ex) { return (ex && (typeof ex === 'object') && 'default' in ex) ? ex['default'] : ex; } + +var Stream = _interopDefault(require('stream')); +var http = _interopDefault(require('http')); +var Url = _interopDefault(require('url')); +var https = _interopDefault(require('https')); +var zlib = _interopDefault(require('zlib')); + // Based on https://github.com/tmpvar/jsdom/blob/aa85b2abf07766ff7bf5c1f6daafb3726f2f2db5/lib/jsdom/living/blob.js -// (MIT licensed) + +// fix for "Readable" isn't a named export issue +const Readable = Stream.Readable; const BUFFER = Symbol('buffer'); const TYPE = Symbol('type'); -const CLOSED = Symbol('closed'); class Blob { constructor() { - Object.defineProperty(this, Symbol.toStringTag, { - value: 'Blob', - writable: false, - enumerable: false, - configurable: true - }); - - this[CLOSED] = false; this[TYPE] = ''; const blobParts = arguments[0]; const options = arguments[1]; const buffers = []; + let size = 0; if (blobParts) { const a = blobParts; @@ -43,6 +45,7 @@ class Blob { } else { buffer = Buffer.from(typeof element === 'string' ? element : String(element)); } + size += buffer.length; buffers.push(buffer); } } @@ -55,13 +58,28 @@ class Blob { } } get size() { - return this[CLOSED] ? 0 : this[BUFFER].length; + return this[BUFFER].length; } get type() { return this[TYPE]; } - get isClosed() { - return this[CLOSED]; + text() { + return Promise.resolve(this[BUFFER].toString()); + } + arrayBuffer() { + const buf = this[BUFFER]; + const ab = buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength); + return Promise.resolve(ab); + } + stream() { + const readable = new Readable(); + readable._read = function () {}; + readable.push(this[BUFFER]); + readable.push(null); + return readable; + } + toString() { + return '[object Blob]'; } slice() { const size = this.size; @@ -89,16 +107,18 @@ class Blob { const slicedBuffer = buffer.slice(relativeStart, relativeStart + span); const blob = new Blob([], { type: arguments[2] }); blob[BUFFER] = slicedBuffer; - blob[CLOSED] = this[CLOSED]; return blob; } - close() { - this[CLOSED] = true; - } } +Object.defineProperties(Blob.prototype, { + size: { enumerable: true }, + type: { enumerable: true }, + slice: { enumerable: true } +}); + Object.defineProperty(Blob.prototype, Symbol.toStringTag, { - value: 'BlobPrototype', + value: 'Blob', writable: false, enumerable: false, configurable: true @@ -137,36 +157,28 @@ FetchError.prototype = Object.create(Error.prototype); FetchError.prototype.constructor = FetchError; FetchError.prototype.name = 'FetchError'; -/** - * body.js - * - * Body interface provides common methods for Request and Response - */ - -const Stream = require('stream'); - -var _require$1 = require('stream'); - -const PassThrough$1 = _require$1.PassThrough; - - -const DISTURBED = Symbol('disturbed'); - let convert; try { convert = require('encoding').convert; } catch (e) {} +const INTERNALS = Symbol('Body internals'); + +// fix an issue where "PassThrough" isn't a named export for node <10 +const PassThrough = Stream.PassThrough; + /** - * Body class + * Body mixin * - * Cannot use ES6 class because Body must be called with .call(). + * Ref: https://fetch.spec.whatwg.org/#body * * @param Stream body Readable stream * @param Object opts Response options * @return Void */ function Body(body) { + var _this = this; + var _ref = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : {}, _ref$size = _ref.size; @@ -177,30 +189,43 @@ function Body(body) { if (body == null) { // body is undefined or null body = null; - } else if (typeof body === 'string') { - // body is string } else if (isURLSearchParams(body)) { // body is a URLSearchParams - } else if (body instanceof Blob) { - // body is blob - } else if (Buffer.isBuffer(body)) { - // body is buffer - } else if (body instanceof Stream) { - // body is stream - } else { + body = Buffer.from(body.toString()); + } else if (isBlob(body)) ; else if (Buffer.isBuffer(body)) ; else if (Object.prototype.toString.call(body) === '[object ArrayBuffer]') { + // body is ArrayBuffer + body = Buffer.from(body); + } else if (ArrayBuffer.isView(body)) { + // body is ArrayBufferView + body = Buffer.from(body.buffer, body.byteOffset, body.byteLength); + } else if (body instanceof Stream) ; else { // none of the above - // coerce to string - body = String(body); + // coerce to string then buffer + body = Buffer.from(String(body)); } - this.body = body; - this[DISTURBED] = false; + this[INTERNALS] = { + body, + disturbed: false, + error: null + }; this.size = size; this.timeout = timeout; + + if (body instanceof Stream) { + body.on('error', function (err) { + const error = err.name === 'AbortError' ? err : new FetchError(`Invalid response body while trying to fetch ${_this.url}: ${err.message}`, 'system', err); + _this[INTERNALS].error = error; + }); + } } Body.prototype = { + get body() { + return this[INTERNALS].body; + }, + get bodyUsed() { - return this[DISTURBED]; + return this[INTERNALS].disturbed; }, /** @@ -238,13 +263,13 @@ Body.prototype = { * @return Promise */ json() { - var _this = this; + var _this2 = this; return consumeBody.call(this).then(function (buffer) { try { return JSON.parse(buffer.toString()); } catch (err) { - return Body.Promise.reject(new FetchError(`invalid json response body at ${_this.url} reason: ${err.message}`, 'invalid-json')); + return Body.Promise.reject(new FetchError(`invalid json response body at ${_this2.url} reason: ${err.message}`, 'invalid-json')); } }); }, @@ -276,15 +301,24 @@ Body.prototype = { * @return Promise */ textConverted() { - var _this2 = this; + var _this3 = this; return consumeBody.call(this).then(function (buffer) { - return convertBody(buffer, _this2.headers); + return convertBody(buffer, _this3.headers); }); } - }; +// In browsers, all properties are enumerable. +Object.defineProperties(Body.prototype, { + body: { enumerable: true }, + bodyUsed: { enumerable: true }, + arrayBuffer: { enumerable: true }, + blob: { enumerable: true }, + json: { enumerable: true }, + text: { enumerable: true } +}); + Body.mixIn = function (proto) { for (const name of Object.getOwnPropertyNames(Body.prototype)) { // istanbul ignore else: future proof @@ -296,41 +330,44 @@ Body.mixIn = function (proto) { }; /** - * Decode buffers into utf-8 string + * Consume and convert an entire Body to a Buffer. + * + * Ref: https://fetch.spec.whatwg.org/#concept-body-consume-body * * @return Promise */ -function consumeBody(body) { - var _this3 = this; +function consumeBody() { + var _this4 = this; - if (this[DISTURBED]) { - return Body.Promise.reject(new Error(`body used already for: ${this.url}`)); + if (this[INTERNALS].disturbed) { + return Body.Promise.reject(new TypeError(`body used already for: ${this.url}`)); } - this[DISTURBED] = true; + this[INTERNALS].disturbed = true; - // body is null - if (this.body === null) { - return Body.Promise.resolve(Buffer.alloc(0)); + if (this[INTERNALS].error) { + return Body.Promise.reject(this[INTERNALS].error); } - // body is string - if (typeof this.body === 'string') { - return Body.Promise.resolve(Buffer.from(this.body)); + let body = this.body; + + // body is null + if (body === null) { + return Body.Promise.resolve(Buffer.alloc(0)); } // body is blob - if (this.body instanceof Blob) { - return Body.Promise.resolve(this.body[BUFFER]); + if (isBlob(body)) { + body = body.stream(); } // body is buffer - if (Buffer.isBuffer(this.body)) { - return Body.Promise.resolve(this.body); + if (Buffer.isBuffer(body)) { + return Body.Promise.resolve(body); } // istanbul ignore if: should never happen - if (!(this.body instanceof Stream)) { + if (!(body instanceof Stream)) { return Body.Promise.resolve(Buffer.alloc(0)); } @@ -344,26 +381,33 @@ function consumeBody(body) { let resTimeout; // allow timeout on slow response body - if (_this3.timeout) { + if (_this4.timeout) { resTimeout = setTimeout(function () { abort = true; - reject(new FetchError(`Response timeout while trying to fetch ${_this3.url} (over ${_this3.timeout}ms)`, 'body-timeout')); - }, _this3.timeout); + reject(new FetchError(`Response timeout while trying to fetch ${_this4.url} (over ${_this4.timeout}ms)`, 'body-timeout')); + }, _this4.timeout); } - // handle stream error, such as incorrect content-encoding - _this3.body.on('error', function (err) { - reject(new FetchError(`Invalid response body while trying to fetch ${_this3.url}: ${err.message}`, 'system', err)); + // handle stream errors + body.on('error', function (err) { + if (err.name === 'AbortError') { + // if the request was aborted, reject with this Error + abort = true; + reject(err); + } else { + // other errors, such as incorrect content-encoding + reject(new FetchError(`Invalid response body while trying to fetch ${_this4.url}: ${err.message}`, 'system', err)); + } }); - _this3.body.on('data', function (chunk) { + body.on('data', function (chunk) { if (abort || chunk === null) { return; } - if (_this3.size && accumBytes + chunk.length > _this3.size) { + if (_this4.size && accumBytes + chunk.length > _this4.size) { abort = true; - reject(new FetchError(`content size at ${_this3.url} over limit: ${_this3.size}`, 'max-size')); + reject(new FetchError(`content size at ${_this4.url} over limit: ${_this4.size}`, 'max-size')); return; } @@ -371,13 +415,19 @@ function consumeBody(body) { accum.push(chunk); }); - _this3.body.on('end', function () { + body.on('end', function () { if (abort) { return; } clearTimeout(resTimeout); - resolve(Buffer.concat(accum)); + + try { + resolve(Buffer.concat(accum, accumBytes)); + } catch (err) { + // handle streams that have accumulated too much data (issue #414) + reject(new FetchError(`Could not create Buffer from response body for ${_this4.url}: ${err.message}`, 'system', err)); + } }); }); } @@ -458,6 +508,15 @@ function isURLSearchParams(obj) { return obj.constructor.name === 'URLSearchParams' || Object.prototype.toString.call(obj) === '[object URLSearchParams]' || typeof obj.sort === 'function'; } +/** + * Check if `obj` is a W3C `Blob` object (which `File` inherits from) + * @param {*} obj + * @return {boolean} + */ +function isBlob(obj) { + return typeof obj === 'object' && typeof obj.arrayBuffer === 'function' && typeof obj.type === 'string' && typeof obj.stream === 'function' && typeof obj.constructor === 'function' && typeof obj.constructor.name === 'string' && /^(Blob|File)$/.test(obj.constructor.name) && /^(Blob|File)$/.test(obj[Symbol.toStringTag]); +} + /** * Clone body given Res/Req instance * @@ -477,12 +536,12 @@ function clone(instance) { // note: we can't clone the form-data object without having it as a dependency if (body instanceof Stream && typeof body.getBoundary !== 'function') { // tee instance body - p1 = new PassThrough$1(); - p2 = new PassThrough$1(); + p1 = new PassThrough(); + p2 = new PassThrough(); body.pipe(p1); body.pipe(p2); // set instance body to teed body and return the other teed body - instance.body = p1; + instance[INTERNALS].body = p1; body = p2; } @@ -494,16 +553,11 @@ function clone(instance) { * specified in the specification: * https://fetch.spec.whatwg.org/#concept-bodyinit-extract * - * This function assumes that instance.body is present and non-null. + * This function assumes that instance.body is present. * - * @param Mixed instance Response or Request instance + * @param Mixed instance Any options.body input */ -function extractContentType(instance) { - const body = instance.body; - - // istanbul ignore if: Currently, because of a guard in Request, body - // can never be null. Included here for completeness. - +function extractContentType(body) { if (body === null) { // body is null return null; @@ -513,38 +567,48 @@ function extractContentType(instance) { } else if (isURLSearchParams(body)) { // body is a URLSearchParams return 'application/x-www-form-urlencoded;charset=UTF-8'; - } else if (body instanceof Blob) { + } else if (isBlob(body)) { // body is blob return body.type || null; } else if (Buffer.isBuffer(body)) { // body is buffer return null; + } else if (Object.prototype.toString.call(body) === '[object ArrayBuffer]') { + // body is ArrayBuffer + return null; + } else if (ArrayBuffer.isView(body)) { + // body is ArrayBufferView + return null; } else if (typeof body.getBoundary === 'function') { // detect form data input from form-data module return `multipart/form-data;boundary=${body.getBoundary()}`; - } else { + } else if (body instanceof Stream) { // body is stream // can't really do much about this return null; + } else { + // Body constructor defaults other things to string + return 'text/plain;charset=UTF-8'; } } +/** + * The Fetch Standard treats this as if "total bytes" is a property on the body. + * For us, we have to explicitly get it with a function. + * + * ref: https://fetch.spec.whatwg.org/#concept-body-total-bytes + * + * @param Body instance Instance of Body + * @return Number? Number of bytes, or null if not possible + */ function getTotalBytes(instance) { const body = instance.body; - // istanbul ignore if: included for completion if (body === null) { // body is null return 0; - } else if (typeof body === 'string') { - // body is string - return Buffer.byteLength(body); - } else if (isURLSearchParams(body)) { - // body is URLSearchParams - return Buffer.byteLength(String(body)); - } else if (body instanceof Blob) { - // body is blob + } else if (isBlob(body)) { return body.size; } else if (Buffer.isBuffer(body)) { // body is buffer @@ -559,11 +623,16 @@ function getTotalBytes(instance) { return null; } else { // body is stream - // can't really do much about this return null; } } +/** + * Write a Body to a Node.js WritableStream (e.g. http.Request) object. + * + * @param Body instance Instance of Body + * @return Void + */ function writeToStream(dest, instance) { const body = instance.body; @@ -571,18 +640,8 @@ function writeToStream(dest, instance) { if (body === null) { // body is null dest.end(); - } else if (typeof body === 'string') { - // body is string - dest.write(body); - dest.end(); - } else if (isURLSearchParams(body)) { - // body is URLSearchParams - dest.write(Buffer.from(String(body))); - dest.end(); - } else if (body instanceof Blob) { - // body is blob - dest.write(body[BUFFER]); - dest.end(); + } else if (isBlob(body)) { + body.stream().pipe(dest); } else if (Buffer.isBuffer(body)) { // body is buffer dest.write(body); @@ -596,115 +655,45 @@ function writeToStream(dest, instance) { // expose Promise Body.Promise = global.Promise; -/** - * A set of utilities borrowed from Node.js' _http_common.js - */ - -/** - * Verifies that the given val is a valid HTTP token - * per the rules defined in RFC 7230 - * See https://tools.ietf.org/html/rfc7230#section-3.2.6 - * - * Allowed characters in an HTTP token: - * ^_`a-z 94-122 - * A-Z 65-90 - * - 45 - * 0-9 48-57 - * ! 33 - * #$%&' 35-39 - * *+ 42-43 - * . 46 - * | 124 - * ~ 126 - * - * This implementation of checkIsHttpToken() loops over the string instead of - * using a regular expression since the former is up to 180% faster with v8 4.9 - * depending on the string length (the shorter the string, the larger the - * performance difference) - * - * Additionally, checkIsHttpToken() is currently designed to be inlinable by v8, - * so take care when making changes to the implementation so that the source - * code size does not exceed v8's default max_inlined_source_size setting. - **/ -/* istanbul ignore next */ -function isValidTokenChar(ch) { - if (ch >= 94 && ch <= 122) return true; - if (ch >= 65 && ch <= 90) return true; - if (ch === 45) return true; - if (ch >= 48 && ch <= 57) return true; - if (ch === 34 || ch === 40 || ch === 41 || ch === 44) return false; - if (ch >= 33 && ch <= 46) return true; - if (ch === 124 || ch === 126) return true; - return false; -} -/* istanbul ignore next */ -function checkIsHttpToken(val) { - if (typeof val !== 'string' || val.length === 0) return false; - if (!isValidTokenChar(val.charCodeAt(0))) return false; - const len = val.length; - if (len > 1) { - if (!isValidTokenChar(val.charCodeAt(1))) return false; - if (len > 2) { - if (!isValidTokenChar(val.charCodeAt(2))) return false; - if (len > 3) { - if (!isValidTokenChar(val.charCodeAt(3))) return false; - for (var i = 4; i < len; i++) { - if (!isValidTokenChar(val.charCodeAt(i))) return false; - } - } - } - } - return true; -} -/** - * True if val contains an invalid field-vchar - * field-value = *( field-content / obs-fold ) - * field-content = field-vchar [ 1*( SP / HTAB ) field-vchar ] - * field-vchar = VCHAR / obs-text - * - * checkInvalidHeaderChar() is currently designed to be inlinable by v8, - * so take care when making changes to the implementation so that the source - * code size does not exceed v8's default max_inlined_source_size setting. - **/ -/* istanbul ignore next */ -function checkInvalidHeaderChar(val) { - val += ''; - if (val.length < 1) return false; - var c = val.charCodeAt(0); - if (c <= 31 && c !== 9 || c > 255 || c === 127) return true; - if (val.length < 2) return false; - c = val.charCodeAt(1); - if (c <= 31 && c !== 9 || c > 255 || c === 127) return true; - if (val.length < 3) return false; - c = val.charCodeAt(2); - if (c <= 31 && c !== 9 || c > 255 || c === 127) return true; - for (var i = 3; i < val.length; ++i) { - c = val.charCodeAt(i); - if (c <= 31 && c !== 9 || c > 255 || c === 127) return true; - } - return false; -} - /** * headers.js * * Headers class offers convenient helpers */ -function sanitizeName(name) { - name += ''; - if (!checkIsHttpToken(name)) { +const invalidTokenRegex = /[^\^_`a-zA-Z\-0-9!#$%&'*+.|~]/; +const invalidHeaderCharRegex = /[^\t\x20-\x7e\x80-\xff]/; + +function validateName(name) { + name = `${name}`; + if (invalidTokenRegex.test(name) || name === '') { throw new TypeError(`${name} is not a legal HTTP header name`); } - return name.toLowerCase(); } -function sanitizeValue(value) { - value += ''; - if (checkInvalidHeaderChar(value)) { +function validateValue(value) { + value = `${value}`; + if (invalidHeaderCharRegex.test(value)) { throw new TypeError(`${value} is not a legal HTTP header value`); } - return value; +} + +/** + * Find the key in the map object given a header name. + * + * Returns undefined if not found. + * + * @param String name Header name + * @return String|Undefined + */ +function find(map, name) { + name = name.toLowerCase(); + for (const key in map) { + if (key.toLowerCase() === name) { + return key; + } + } + return undefined; } const MAP = Symbol('map'); @@ -735,9 +724,7 @@ class Headers { // We don't worry about converting prop to ByteString here as append() // will handle it. - if (init == null) { - // no op - } else if (typeof init === 'object') { + if (init == null) ; else if (typeof init === 'object') { const method = init[Symbol.iterator]; if (method != null) { if (typeof method !== 'function') { @@ -770,28 +757,23 @@ class Headers { } else { throw new TypeError('Provided initializer must be an object'); } - - Object.defineProperty(this, Symbol.toStringTag, { - value: 'Headers', - writable: false, - enumerable: false, - configurable: true - }); } /** - * Return first header value given name + * Return combined header value given name * * @param String name Header name * @return Mixed */ get(name) { - const list = this[MAP][sanitizeName(name)]; - if (!list) { + name = `${name}`; + validateName(name); + const key = find(this[MAP], name); + if (key === undefined) { return null; } - return list.join(', '); + return this[MAP][key].join(', '); } /** @@ -804,7 +786,7 @@ class Headers { forEach(callback) { let thisArg = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : undefined; - let pairs = getHeaderPairs(this); + let pairs = getHeaders(this); let i = 0; while (i < pairs.length) { var _pairs$i = pairs[i]; @@ -812,7 +794,7 @@ class Headers { value = _pairs$i[1]; callback.call(thisArg, value, name, this); - pairs = getHeaderPairs(this); + pairs = getHeaders(this); i++; } } @@ -825,7 +807,12 @@ class Headers { * @return Void */ set(name, value) { - this[MAP][sanitizeName(name)] = [sanitizeValue(value)]; + name = `${name}`; + value = `${value}`; + validateName(name); + validateValue(value); + const key = find(this[MAP], name); + this[MAP][key !== undefined ? key : name] = [value]; } /** @@ -836,12 +823,16 @@ class Headers { * @return Void */ append(name, value) { - if (!this.has(name)) { - this.set(name, value); - return; + name = `${name}`; + value = `${value}`; + validateName(name); + validateValue(value); + const key = find(this[MAP], name); + if (key !== undefined) { + this[MAP][key].push(value); + } else { + this[MAP][name] = [value]; } - - this[MAP][sanitizeName(name)].push(sanitizeValue(value)); } /** @@ -851,7 +842,9 @@ class Headers { * @return Boolean */ has(name) { - return !!this[MAP][sanitizeName(name)]; + name = `${name}`; + validateName(name); + return find(this[MAP], name) !== undefined; } /** @@ -861,7 +854,12 @@ class Headers { * @return Void */ delete(name) { - delete this[MAP][sanitizeName(name)]; + name = `${name}`; + validateName(name); + const key = find(this[MAP], name); + if (key !== undefined) { + delete this[MAP][key]; + } } /** @@ -905,18 +903,34 @@ class Headers { Headers.prototype.entries = Headers.prototype[Symbol.iterator]; Object.defineProperty(Headers.prototype, Symbol.toStringTag, { - value: 'HeadersPrototype', + value: 'Headers', writable: false, enumerable: false, configurable: true }); -function getHeaderPairs(headers, kind) { +Object.defineProperties(Headers.prototype, { + get: { enumerable: true }, + forEach: { enumerable: true }, + set: { enumerable: true }, + append: { enumerable: true }, + has: { enumerable: true }, + delete: { enumerable: true }, + keys: { enumerable: true }, + values: { enumerable: true }, + entries: { enumerable: true } +}); + +function getHeaders(headers) { + let kind = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : 'key+value'; + const keys = Object.keys(headers[MAP]).sort(); return keys.map(kind === 'key' ? function (k) { - return [k]; + return k.toLowerCase(); + } : kind === 'value' ? function (k) { + return headers[MAP][k].join(', '); } : function (k) { - return [k, headers.get(k)]; + return [k.toLowerCase(), headers[MAP][k].join(', ')]; }); } @@ -944,7 +958,7 @@ const HeadersIteratorPrototype = Object.setPrototypeOf({ kind = _INTERNAL.kind, index = _INTERNAL.index; - const values = getHeaderPairs(target, kind); + const values = getHeaders(target, kind); const len = values.length; if (index >= len) { return { @@ -953,20 +967,10 @@ const HeadersIteratorPrototype = Object.setPrototypeOf({ }; } - const pair = values[index]; this[INTERNAL].index = index + 1; - let result; - if (kind === 'key') { - result = pair[0]; - } else if (kind === 'value') { - result = pair[1]; - } else { - result = pair; - } - return { - value: result, + value: values[index], done: false }; } @@ -980,14 +984,59 @@ Object.defineProperty(HeadersIteratorPrototype, Symbol.toStringTag, { }); /** - * response.js + * Export the Headers object in a form that Node.js can consume. + * + * @param Headers headers + * @return Object + */ +function exportNodeCompatibleHeaders(headers) { + const obj = Object.assign({ __proto__: null }, headers[MAP]); + + // http.request() only supports string as Host header. This hack makes + // specifying custom Host header possible. + const hostHeaderKey = find(headers[MAP], 'Host'); + if (hostHeaderKey !== undefined) { + obj[hostHeaderKey] = obj[hostHeaderKey][0]; + } + + return obj; +} + +/** + * Create a Headers object from an object of headers, ignoring those that do + * not conform to HTTP grammar productions. * - * Response class provides content decoding + * @param Object obj Object of headers + * @return Headers */ +function createHeadersLenient(obj) { + const headers = new Headers(); + for (const name of Object.keys(obj)) { + if (invalidTokenRegex.test(name)) { + continue; + } + if (Array.isArray(obj[name])) { + for (const val of obj[name]) { + if (invalidHeaderCharRegex.test(val)) { + continue; + } + if (headers[MAP][name] === undefined) { + headers[MAP][name] = [val]; + } else { + headers[MAP][name].push(val); + } + } + } else if (!invalidHeaderCharRegex.test(obj[name])) { + headers[MAP][name] = [obj[name]]; + } + } + return headers; +} -var _require$2 = require('http'); +const INTERNALS$1 = Symbol('Response internals'); -const STATUS_CODES = _require$2.STATUS_CODES; +// fix an issue where "STATUS_CODES" aren't a named export for node <10 +const STATUS_CODES = http.STATUS_CODES; /** * Response class @@ -996,7 +1045,6 @@ const STATUS_CODES = _require$2.STATUS_CODES; * @param Object opts Response options * @return Void */ - class Response { constructor() { let body = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : null; @@ -1004,25 +1052,50 @@ class Response { Body.call(this, body, opts); - this.url = opts.url; - this.status = opts.status || 200; - this.statusText = opts.statusText || STATUS_CODES[this.status]; + const status = opts.status || 200; + const headers = new Headers(opts.headers); - this.headers = new Headers(opts.headers); + if (body != null && !headers.has('Content-Type')) { + const contentType = extractContentType(body); + if (contentType) { + headers.append('Content-Type', contentType); + } + } - Object.defineProperty(this, Symbol.toStringTag, { - value: 'Response', - writable: false, - enumerable: false, - configurable: true - }); + this[INTERNALS$1] = { + url: opts.url, + status, + statusText: opts.statusText || STATUS_CODES[status], + headers, + counter: opts.counter + }; + } + + get url() { + return this[INTERNALS$1].url || ''; + } + + get status() { + return this[INTERNALS$1].status; } /** * Convenience property representing if the request ended normally */ get ok() { - return this.status >= 200 && this.status < 300; + return this[INTERNALS$1].status >= 200 && this[INTERNALS$1].status < 300; + } + + get redirected() { + return this[INTERNALS$1].counter > 0; + } + + get statusText() { + return this[INTERNALS$1].statusText; + } + + get headers() { + return this[INTERNALS$1].headers; } /** @@ -1031,39 +1104,58 @@ class Response { * @return Response */ clone() { - return new Response(clone(this), { url: this.url, status: this.status, statusText: this.statusText, headers: this.headers, - ok: this.ok + ok: this.ok, + redirected: this.redirected }); } } Body.mixIn(Response.prototype); +Object.defineProperties(Response.prototype, { + url: { enumerable: true }, + status: { enumerable: true }, + ok: { enumerable: true }, + redirected: { enumerable: true }, + statusText: { enumerable: true }, + headers: { enumerable: true }, + clone: { enumerable: true } +}); + Object.defineProperty(Response.prototype, Symbol.toStringTag, { - value: 'ResponsePrototype', + value: 'Response', writable: false, enumerable: false, configurable: true }); -/** - * request.js - * - * Request class contains server only options - */ +const INTERNALS$2 = Symbol('Request internals'); -var _require$3 = require('url'); +// fix an issue where "format", "parse" aren't a named export for node <10 +const parse_url = Url.parse; +const format_url = Url.format; -const format_url = _require$3.format; -const parse_url = _require$3.parse; +const streamDestructionSupported = 'destroy' in Stream.Readable.prototype; +/** + * Check if a value is an instance of Request. + * + * @param Mixed input + * @return Boolean + */ +function isRequest(input) { + return typeof input === 'object' && typeof input[INTERNALS$2] === 'object'; +} -const PARSED_URL = Symbol('url'); +function isAbortSignal(signal) { + const proto = signal && typeof signal === 'object' && Object.getPrototypeOf(signal); + return !!(proto && proto.constructor.name === 'AbortSignal'); +} /** * Request class @@ -1079,7 +1171,7 @@ class Request { let parsedURL; // normalize input - if (!(input instanceof Request)) { + if (!isRequest(input)) { if (input && input.href) { // in order to support Node.js' Url objects; though WHATWG's URL objects // will fall into this branch also (since their `toString()` will return @@ -1095,47 +1187,68 @@ class Request { } let method = init.method || input.method || 'GET'; + method = method.toUpperCase(); - if ((init.body != null || input instanceof Request && input.body !== null) && (method === 'GET' || method === 'HEAD')) { + if ((init.body != null || isRequest(input) && input.body !== null) && (method === 'GET' || method === 'HEAD')) { throw new TypeError('Request with GET/HEAD method cannot have body'); } - let inputBody = init.body != null ? init.body : input instanceof Request && input.body !== null ? clone(input) : null; + let inputBody = init.body != null ? init.body : isRequest(input) && input.body !== null ? clone(input) : null; Body.call(this, inputBody, { timeout: init.timeout || input.timeout || 0, size: init.size || input.size || 0 }); - // fetch spec options - this.method = method.toUpperCase(); - this.redirect = init.redirect || input.redirect || 'follow'; - this.headers = new Headers(init.headers || input.headers || {}); + const headers = new Headers(init.headers || input.headers || {}); - if (init.body != null) { - const contentType = extractContentType(this); - if (contentType !== null && !this.headers.has('Content-Type')) { - this.headers.append('Content-Type', contentType); + if (inputBody != null && !headers.has('Content-Type')) { + const contentType = extractContentType(inputBody); + if (contentType) { + headers.append('Content-Type', contentType); } } - // server only options + let signal = isRequest(input) ? input.signal : null; + if ('signal' in init) signal = init.signal; + + if (signal != null && !isAbortSignal(signal)) { + throw new TypeError('Expected signal to be an instanceof AbortSignal'); + } + + this[INTERNALS$2] = { + method, + redirect: init.redirect || input.redirect || 'follow', + headers, + parsedURL, + signal + }; + + // node-fetch-only options this.follow = init.follow !== undefined ? init.follow : input.follow !== undefined ? input.follow : 20; this.compress = init.compress !== undefined ? init.compress : input.compress !== undefined ? input.compress : true; this.counter = init.counter || input.counter || 0; this.agent = init.agent || input.agent; + } - this[PARSED_URL] = parsedURL; - Object.defineProperty(this, Symbol.toStringTag, { - value: 'Request', - writable: false, - enumerable: false, - configurable: true - }); + get method() { + return this[INTERNALS$2].method; } get url() { - return format_url(this[PARSED_URL]); + return format_url(this[INTERNALS$2].parsedURL); + } + + get headers() { + return this[INTERNALS$2].headers; + } + + get redirect() { + return this[INTERNALS$2].redirect; + } + + get signal() { + return this[INTERNALS$2].signal; } /** @@ -1151,17 +1264,32 @@ class Request { Body.mixIn(Request.prototype); Object.defineProperty(Request.prototype, Symbol.toStringTag, { - value: 'RequestPrototype', + value: 'Request', writable: false, enumerable: false, configurable: true }); +Object.defineProperties(Request.prototype, { + method: { enumerable: true }, + url: { enumerable: true }, + headers: { enumerable: true }, + redirect: { enumerable: true }, + clone: { enumerable: true }, + signal: { enumerable: true } +}); + +/** + * Convert a Request to Node.js http request options. + * + * @param Request A Request instance + * @return Object The options object to be passed to http.request + */ function getNodeRequestOptions(request) { - const parsedURL = request[PARSED_URL]; - const headers = new Headers(request.headers); + const parsedURL = request[INTERNALS$2].parsedURL; + const headers = new Headers(request[INTERNALS$2].headers); - // fetch step 3 + // fetch step 1.3 if (!headers.has('Accept')) { headers.set('Accept', '*/*'); } @@ -1175,7 +1303,11 @@ function getNodeRequestOptions(request) { throw new TypeError('Only HTTP(S) protocols are supported'); } - // HTTP-network-or-cache fetch steps 5-9 + if (request.signal && request.body instanceof Stream.Readable && !streamDestructionSupported) { + throw new Error('Cancellation of streamed requests with AbortSignal is not supported in node < 8'); + } + + // HTTP-network-or-cache fetch steps 2.4-2.7 let contentLengthValue = null; if (request.body == null && /^(POST|PUT)$/i.test(request.method)) { contentLengthValue = '0'; @@ -1190,47 +1322,64 @@ function getNodeRequestOptions(request) { headers.set('Content-Length', contentLengthValue); } - // HTTP-network-or-cache fetch step 12 + // HTTP-network-or-cache fetch step 2.11 if (!headers.has('User-Agent')) { headers.set('User-Agent', 'node-fetch/1.0 (+https://github.com/bitinn/node-fetch)'); } - // HTTP-network-or-cache fetch step 16 - if (request.compress) { + // HTTP-network-or-cache fetch step 2.15 + if (request.compress && !headers.has('Accept-Encoding')) { headers.set('Accept-Encoding', 'gzip,deflate'); } - if (!headers.has('Connection') && !request.agent) { + + let agent = request.agent; + if (typeof agent === 'function') { + agent = agent(parsedURL); + } + + if (!headers.has('Connection') && !agent) { headers.set('Connection', 'close'); } - // HTTP-network fetch step 4 + // HTTP-network fetch step 4.2 // chunked encoding is handled by Node.js return Object.assign({}, parsedURL, { method: request.method, - headers: headers.raw(), - agent: request.agent + headers: exportNodeCompatibleHeaders(headers), + agent }); } /** - * index.js + * abort-error.js * - * a request API compatible with window.fetch + * AbortError interface for cancelled requests */ -const http = require('http'); -const https = require('https'); - -var _require = require('stream'); +/** + * Create AbortError instance + * + * @param String message Error message for human + * @return AbortError + */ +function AbortError(message) { + Error.call(this, message); -const PassThrough = _require.PassThrough; + this.type = 'aborted'; + this.message = message; -var _require2 = require('url'); + // hide custom error implementation details from end-users + Error.captureStackTrace(this, this.constructor); +} -const resolve_url = _require2.resolve; +AbortError.prototype = Object.create(Error.prototype); +AbortError.prototype.constructor = AbortError; +AbortError.prototype.name = 'AbortError'; -const zlib = require('zlib'); +// fix an issue where "PassThrough", "resolve" aren't a named export for node <10 +const PassThrough$1 = Stream.PassThrough; +const resolve_url = Url.resolve; /** * Fetch function @@ -1255,93 +1404,157 @@ function fetch(url, opts) { const options = getNodeRequestOptions(request); const send = (options.protocol === 'https:' ? https : http).request; + const signal = request.signal; - // http.request only support string as host header, this hack make custom host header possible - if (options.headers.host) { - options.headers.host = options.headers.host[0]; + let response = null; + + const abort = function abort() { + let error = new AbortError('The user aborted a request.'); + reject(error); + if (request.body && request.body instanceof Stream.Readable) { + request.body.destroy(error); + } + if (!response || !response.body) return; + response.body.emit('error', error); + }; + + if (signal && signal.aborted) { + abort(); + return; } + const abortAndFinalize = function abortAndFinalize() { + abort(); + finalize(); + }; + // send request const req = send(options); let reqTimeout; + if (signal) { + signal.addEventListener('abort', abortAndFinalize); + } + + function finalize() { + req.abort(); + if (signal) signal.removeEventListener('abort', abortAndFinalize); + clearTimeout(reqTimeout); + } + if (request.timeout) { req.once('socket', function (socket) { reqTimeout = setTimeout(function () { - req.abort(); reject(new FetchError(`network timeout at: ${request.url}`, 'request-timeout')); + finalize(); }, request.timeout); }); } req.on('error', function (err) { - clearTimeout(reqTimeout); reject(new FetchError(`request to ${request.url} failed, reason: ${err.message}`, 'system', err)); + finalize(); }); req.on('response', function (res) { clearTimeout(reqTimeout); - // handle redirect - if (fetch.isRedirect(res.statusCode) && request.redirect !== 'manual') { - if (request.redirect === 'error') { - reject(new FetchError(`redirect mode is set to error: ${request.url}`, 'no-redirect')); - return; - } - - if (request.counter >= request.follow) { - reject(new FetchError(`maximum redirect reached at: ${request.url}`, 'max-redirect')); - return; - } - - if (!res.headers.location) { - reject(new FetchError(`redirect location header missing at: ${request.url}`, 'invalid-redirect')); - return; - } - - // per fetch spec, for POST request with 301/302 response, or any request with 303 response, use GET when following redirect - if (res.statusCode === 303 || (res.statusCode === 301 || res.statusCode === 302) && request.method === 'POST') { - request.method = 'GET'; - request.body = null; - request.headers.delete('content-length'); - } - - request.counter++; - - resolve(fetch(resolve_url(request.url, res.headers.location), request)); - return; - } - - // normalize location header for manual redirect mode - const headers = new Headers(); - for (const name of Object.keys(res.headers)) { - if (Array.isArray(res.headers[name])) { - for (const val of res.headers[name]) { - headers.append(name, val); - } - } else { - headers.append(name, res.headers[name]); + const headers = createHeadersLenient(res.headers); + + // HTTP fetch step 5 + if (fetch.isRedirect(res.statusCode)) { + // HTTP fetch step 5.2 + const location = headers.get('Location'); + + // HTTP fetch step 5.3 + const locationURL = location === null ? null : resolve_url(request.url, location); + + // HTTP fetch step 5.5 + switch (request.redirect) { + case 'error': + reject(new FetchError(`redirect mode is set to error: ${request.url}`, 'no-redirect')); + finalize(); + return; + case 'manual': + // node-fetch-specific step: make manual redirect a bit easier to use by setting the Location header value to the resolved URL. + if (locationURL !== null) { + // handle corrupted header + try { + headers.set('Location', locationURL); + } catch (err) { + // istanbul ignore next: nodejs server prevent invalid response headers, we can't test this through normal request + reject(err); + } + } + break; + case 'follow': + // HTTP-redirect fetch step 2 + if (locationURL === null) { + break; + } + + // HTTP-redirect fetch step 5 + if (request.counter >= request.follow) { + reject(new FetchError(`maximum redirect reached at: ${request.url}`, 'max-redirect')); + finalize(); + return; + } + + // HTTP-redirect fetch step 6 (counter increment) + // Create a new Request object. + const requestOpts = { + headers: new Headers(request.headers), + follow: request.follow, + counter: request.counter + 1, + agent: request.agent, + compress: request.compress, + method: request.method, + body: request.body, + signal: request.signal, + timeout: request.timeout + }; + + // HTTP-redirect fetch step 9 + if (res.statusCode !== 303 && request.body && getTotalBytes(request) === null) { + reject(new FetchError('Cannot follow redirect with body being a readable stream', 'unsupported-redirect')); + finalize(); + return; + } + + // HTTP-redirect fetch step 11 + if (res.statusCode === 303 || (res.statusCode === 301 || res.statusCode === 302) && request.method === 'POST') { + requestOpts.method = 'GET'; + requestOpts.body = undefined; + requestOpts.headers.delete('content-length'); + } + + // HTTP-redirect fetch step 15 + resolve(fetch(new Request(locationURL, requestOpts))); + finalize(); + return; } } - if (request.redirect === 'manual' && headers.has('location')) { - headers.set('location', resolve_url(request.url, headers.get('location'))); - } // prepare response - let body = res.pipe(new PassThrough()); + res.once('end', function () { + if (signal) signal.removeEventListener('abort', abortAndFinalize); + }); + let body = res.pipe(new PassThrough$1()); + const response_options = { url: request.url, status: res.statusCode, statusText: res.statusMessage, headers: headers, size: request.size, - timeout: request.timeout + timeout: request.timeout, + counter: request.counter }; - // HTTP-network fetch step 16.1.2 + // HTTP-network fetch step 12.1.1.3 const codings = headers.get('Content-Encoding'); - // HTTP-network fetch step 16.1.3: handle content codings + // HTTP-network fetch step 12.1.1.4: handle content codings // in following scenarios we ignore compression support // 1. compression support is disabled @@ -1350,7 +1563,8 @@ function fetch(url, opts) { // 4. no content response (204) // 5. content not modified response (304) if (!request.compress || request.method === 'HEAD' || codings === null || res.statusCode === 204 || res.statusCode === 304) { - resolve(new Response(body, response_options)); + response = new Response(body, response_options); + resolve(response); return; } @@ -1367,7 +1581,8 @@ function fetch(url, opts) { // for gzip if (codings == 'gzip' || codings == 'x-gzip') { body = body.pipe(zlib.createGunzip(zlibOptions)); - resolve(new Response(body, response_options)); + response = new Response(body, response_options); + resolve(response); return; } @@ -1375,7 +1590,7 @@ function fetch(url, opts) { if (codings == 'deflate' || codings == 'x-deflate') { // handle the infamous raw deflate response from old servers // a hack for old IIS and Apache servers - const raw = res.pipe(new PassThrough()); + const raw = res.pipe(new PassThrough$1()); raw.once('data', function (chunk) { // see http://stackoverflow.com/questions/37519828 if ((chunk[0] & 0x0F) === 0x08) { @@ -1383,19 +1598,28 @@ function fetch(url, opts) { } else { body = body.pipe(zlib.createInflateRaw()); } - resolve(new Response(body, response_options)); + response = new Response(body, response_options); + resolve(response); }); return; } + // for br + if (codings == 'br' && typeof zlib.createBrotliDecompress === 'function') { + body = body.pipe(zlib.createBrotliDecompress()); + response = new Response(body, response_options); + resolve(response); + return; + } + // otherwise, use response as-is - resolve(new Response(body, response_options)); + response = new Response(body, response_options); + resolve(response); }); writeToStream(req, request); }); } - /** * Redirect code matching * @@ -1410,6 +1634,8 @@ fetch.isRedirect = function (code) { fetch.Promise = global.Promise; module.exports = exports = fetch; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.default = exports; exports.Headers = Headers; exports.Request = Request; exports.Response = Response; diff --git a/node_modules/@octokit/request/node_modules/node-fetch/lib/index.mjs b/node_modules/node-fetch/lib/index.mjs similarity index 100% rename from node_modules/@octokit/request/node_modules/node-fetch/lib/index.mjs rename to node_modules/node-fetch/lib/index.mjs diff --git a/node_modules/node-fetch/package.json b/node_modules/node-fetch/package.json index 6bf8e40e8..8e5c883b2 100644 --- a/node_modules/node-fetch/package.json +++ b/node_modules/node-fetch/package.json @@ -1,12 +1,25 @@ { "name": "node-fetch", - "version": "1.7.3", - "description": "A light-weight module that brings window.fetch to node.js and io.js", - "main": "index.js", + "version": "2.6.0", + "description": "A light-weight module that brings window.fetch to node.js", + "main": "lib/index", + "browser": "./browser.js", + "module": "lib/index.mjs", + "files": [ + "lib/index.js", + "lib/index.mjs", + "lib/index.es.js", + "browser.js" + ], + "engines": { + "node": "4.x || >=6.0.0" + }, "scripts": { - "test": "mocha test/test.js", - "report": "istanbul cover _mocha -- -R spec test/test.js", - "coverage": "istanbul cover _mocha --report lcovonly -- -R spec test/test.js && codecov" + "build": "cross-env BABEL_ENV=rollup rollup -c", + "prepare": "npm run build", + "test": "cross-env BABEL_ENV=test mocha --require babel-register --throw-deprecation test/test.js", + "report": "cross-env BABEL_ENV=coverage nyc --reporter lcov --reporter text mocha -R spec test/test.js", + "coverage": "cross-env BABEL_ENV=coverage nyc --reporter json --reporter text mocha -R spec test/test.js && codecov -f coverage/coverage-final.json" }, "repository": { "type": "git", @@ -24,19 +37,30 @@ }, "homepage": "https://github.com/bitinn/node-fetch", "devDependencies": { - "bluebird": "^3.3.4", + "@ungap/url-search-params": "^0.1.2", + "abort-controller": "^1.1.0", + "abortcontroller-polyfill": "^1.3.0", + "babel-core": "^6.26.3", + "babel-plugin-istanbul": "^4.1.6", + "babel-preset-env": "^1.6.1", + "babel-register": "^6.16.3", "chai": "^3.5.0", - "chai-as-promised": "^5.2.0", - "codecov": "^1.0.1", - "form-data": ">=1.0.0", - "istanbul": "^0.4.2", - "mocha": "^2.1.0", + "chai-as-promised": "^7.1.1", + "chai-iterator": "^1.1.1", + "chai-string": "~1.3.0", + "codecov": "^3.3.0", + "cross-env": "^5.2.0", + "form-data": "^2.3.3", + "is-builtin-module": "^1.0.0", + "mocha": "^5.0.0", + "nyc": "11.9.0", "parted": "^0.1.1", - "promise": "^7.1.1", - "resumer": "0.0.0" + "promise": "^8.0.3", + "resumer": "0.0.0", + "rollup": "^0.63.4", + "rollup-plugin-babel": "^3.0.7", + "string-to-arraybuffer": "^1.0.2", + "whatwg-url": "^5.0.0" }, - "dependencies": { - "encoding": "^0.1.11", - "is-stream": "^1.0.1" - } + "dependencies": {} } diff --git a/node_modules/request/node_modules/form-data/README.md.bak b/node_modules/request/node_modules/form-data/README.md.bak new file mode 100644 index 000000000..0524d6028 --- /dev/null +++ b/node_modules/request/node_modules/form-data/README.md.bak @@ -0,0 +1,234 @@ +# Form-Data [![NPM Module](https://img.shields.io/npm/v/form-data.svg)](https://www.npmjs.com/package/form-data) [![Join the chat at https://gitter.im/form-data/form-data](http://form-data.github.io/images/gitterbadge.svg)](https://gitter.im/form-data/form-data) + +A library to create readable ```"multipart/form-data"``` streams. Can be used to submit forms and file uploads to other web applications. + +The API of this library is inspired by the [XMLHttpRequest-2 FormData Interface][xhr2-fd]. + +[xhr2-fd]: http://dev.w3.org/2006/webapi/XMLHttpRequest-2/Overview.html#the-formdata-interface + +[![Linux Build](https://img.shields.io/travis/form-data/form-data/master.svg?label=linux:4.x-9.x)](https://travis-ci.org/form-data/form-data) +[![MacOS Build](https://img.shields.io/travis/form-data/form-data/master.svg?label=macos:4.x-9.x)](https://travis-ci.org/form-data/form-data) +[![Windows Build](https://img.shields.io/appveyor/ci/alexindigo/form-data/master.svg?label=windows:4.x-9.x)](https://ci.appveyor.com/project/alexindigo/form-data) + +[![Coverage Status](https://img.shields.io/coveralls/form-data/form-data/master.svg?label=code+coverage)](https://coveralls.io/github/form-data/form-data?branch=master) +[![Dependency Status](https://img.shields.io/david/form-data/form-data.svg)](https://david-dm.org/form-data/form-data) +[![bitHound Overall Score](https://www.bithound.io/github/form-data/form-data/badges/score.svg)](https://www.bithound.io/github/form-data/form-data) + +## Install + +``` +npm install --save form-data +``` + +## Usage + +In this example we are constructing a form with 3 fields that contain a string, +a buffer and a file stream. + +``` javascript +var FormData = require('form-data'); +var fs = require('fs'); + +var form = new FormData(); +form.append('my_field', 'my value'); +form.append('my_buffer', new Buffer(10)); +form.append('my_file', fs.createReadStream('/foo/bar.jpg')); +``` + +Also you can use http-response stream: + +``` javascript +var FormData = require('form-data'); +var http = require('http'); + +var form = new FormData(); + +http.request('http://nodejs.org/images/logo.png', function(response) { + form.append('my_field', 'my value'); + form.append('my_buffer', new Buffer(10)); + form.append('my_logo', response); +}); +``` + +Or @mikeal's [request](https://github.com/request/request) stream: + +``` javascript +var FormData = require('form-data'); +var request = require('request'); + +var form = new FormData(); + +form.append('my_field', 'my value'); +form.append('my_buffer', new Buffer(10)); +form.append('my_logo', request('http://nodejs.org/images/logo.png')); +``` + +In order to submit this form to a web application, call ```submit(url, [callback])``` method: + +``` javascript +form.submit('http://example.org/', function(err, res) { + // res – response object (http.IncomingMessage) // + res.resume(); +}); + +``` + +For more advanced request manipulations ```submit()``` method returns ```http.ClientRequest``` object, or you can choose from one of the alternative submission methods. + +### Custom options + +You can provide custom options, such as `maxDataSize`: + +``` javascript +var FormData = require('form-data'); + +var form = new FormData({ maxDataSize: 20971520 }); +form.append('my_field', 'my value'); +form.append('my_buffer', /* something big */); +``` + +List of available options could be found in [combined-stream](https://github.com/felixge/node-combined-stream/blob/master/lib/combined_stream.js#L7-L15) + +### Alternative submission methods + +You can use node's http client interface: + +``` javascript +var http = require('http'); + +var request = http.request({ + method: 'post', + host: 'example.org', + path: '/upload', + headers: form.getHeaders() +}); + +form.pipe(request); + +request.on('response', function(res) { + console.log(res.statusCode); +}); +``` + +Or if you would prefer the `'Content-Length'` header to be set for you: + +``` javascript +form.submit('example.org/upload', function(err, res) { + console.log(res.statusCode); +}); +``` + +To use custom headers and pre-known length in parts: + +``` javascript +var CRLF = '\r\n'; +var form = new FormData(); + +var options = { + header: CRLF + '--' + form.getBoundary() + CRLF + 'X-Custom-Header: 123' + CRLF + CRLF, + knownLength: 1 +}; + +form.append('my_buffer', buffer, options); + +form.submit('http://example.com/', function(err, res) { + if (err) throw err; + console.log('Done'); +}); +``` + +Form-Data can recognize and fetch all the required information from common types of streams (```fs.readStream```, ```http.response``` and ```mikeal's request```), for some other types of streams you'd need to provide "file"-related information manually: + +``` javascript +someModule.stream(function(err, stdout, stderr) { + if (err) throw err; + + var form = new FormData(); + + form.append('file', stdout, { + filename: 'unicycle.jpg', // ... or: + filepath: 'photos/toys/unicycle.jpg', + contentType: 'image/jpeg', + knownLength: 19806 + }); + + form.submit('http://example.com/', function(err, res) { + if (err) throw err; + console.log('Done'); + }); +}); +``` + +The `filepath` property overrides `filename` and may contain a relative path. This is typically used when uploading [multiple files from a directory](https://wicg.github.io/entries-api/#dom-htmlinputelement-webkitdirectory). + +For edge cases, like POST request to URL with query string or to pass HTTP auth credentials, object can be passed to `form.submit()` as first parameter: + +``` javascript +form.submit({ + host: 'example.com', + path: '/probably.php?extra=params', + auth: 'username:password' +}, function(err, res) { + console.log(res.statusCode); +}); +``` + +In case you need to also send custom HTTP headers with the POST request, you can use the `headers` key in first parameter of `form.submit()`: + +``` javascript +form.submit({ + host: 'example.com', + path: '/surelynot.php', + headers: {'x-test-header': 'test-header-value'} +}, function(err, res) { + console.log(res.statusCode); +}); +``` + +### Integration with other libraries + +#### Request + +Form submission using [request](https://github.com/request/request): + +```javascript +var formData = { + my_field: 'my_value', + my_file: fs.createReadStream(__dirname + '/unicycle.jpg'), +}; + +request.post({url:'http://service.com/upload', formData: formData}, function(err, httpResponse, body) { + if (err) { + return console.error('upload failed:', err); + } + console.log('Upload successful! Server responded with:', body); +}); +``` + +For more details see [request readme](https://github.com/request/request#multipartform-data-multipart-form-uploads). + +#### node-fetch + +You can also submit a form using [node-fetch](https://github.com/bitinn/node-fetch): + +```javascript +var form = new FormData(); + +form.append('a', 1); + +fetch('http://example.com', { method: 'POST', body: form }) + .then(function(res) { + return res.json(); + }).then(function(json) { + console.log(json); + }); +``` + +## Notes + +- ```getLengthSync()``` method DOESN'T calculate length for streams, use ```knownLength``` options as workaround. +- Starting version `2.x` FormData has dropped support for `node@0.10.x`. + +## License + +Form-Data is released under the [MIT](License) license. diff --git a/node_modules/request/node_modules/form-data/lib/browser.js b/node_modules/request/node_modules/form-data/lib/browser.js new file mode 100644 index 000000000..09e7c70e6 --- /dev/null +++ b/node_modules/request/node_modules/form-data/lib/browser.js @@ -0,0 +1,2 @@ +/* eslint-env browser */ +module.exports = typeof self == 'object' ? self.FormData : window.FormData; diff --git a/node_modules/request/node_modules/form-data/lib/form_data.js b/node_modules/request/node_modules/form-data/lib/form_data.js new file mode 100644 index 000000000..3a1bb82b1 --- /dev/null +++ b/node_modules/request/node_modules/form-data/lib/form_data.js @@ -0,0 +1,457 @@ +var CombinedStream = require('combined-stream'); +var util = require('util'); +var path = require('path'); +var http = require('http'); +var https = require('https'); +var parseUrl = require('url').parse; +var fs = require('fs'); +var mime = require('mime-types'); +var asynckit = require('asynckit'); +var populate = require('./populate.js'); + +// Public API +module.exports = FormData; + +// make it a Stream +util.inherits(FormData, CombinedStream); + +/** + * Create readable "multipart/form-data" streams. + * Can be used to submit forms + * and file uploads to other web applications. + * + * @constructor + * @param {Object} options - Properties to be added/overriden for FormData and CombinedStream + */ +function FormData(options) { + if (!(this instanceof FormData)) { + return new FormData(); + } + + this._overheadLength = 0; + this._valueLength = 0; + this._valuesToMeasure = []; + + CombinedStream.call(this); + + options = options || {}; + for (var option in options) { + this[option] = options[option]; + } +} + +FormData.LINE_BREAK = '\r\n'; +FormData.DEFAULT_CONTENT_TYPE = 'application/octet-stream'; + +FormData.prototype.append = function(field, value, options) { + + options = options || {}; + + // allow filename as single option + if (typeof options == 'string') { + options = {filename: options}; + } + + var append = CombinedStream.prototype.append.bind(this); + + // all that streamy business can't handle numbers + if (typeof value == 'number') { + value = '' + value; + } + + // https://github.com/felixge/node-form-data/issues/38 + if (util.isArray(value)) { + // Please convert your array into string + // the way web server expects it + this._error(new Error('Arrays are not supported.')); + return; + } + + var header = this._multiPartHeader(field, value, options); + var footer = this._multiPartFooter(); + + append(header); + append(value); + append(footer); + + // pass along options.knownLength + this._trackLength(header, value, options); +}; + +FormData.prototype._trackLength = function(header, value, options) { + var valueLength = 0; + + // used w/ getLengthSync(), when length is known. + // e.g. for streaming directly from a remote server, + // w/ a known file a size, and not wanting to wait for + // incoming file to finish to get its size. + if (options.knownLength != null) { + valueLength += +options.knownLength; + } else if (Buffer.isBuffer(value)) { + valueLength = value.length; + } else if (typeof value === 'string') { + valueLength = Buffer.byteLength(value); + } + + this._valueLength += valueLength; + + // @check why add CRLF? does this account for custom/multiple CRLFs? + this._overheadLength += + Buffer.byteLength(header) + + FormData.LINE_BREAK.length; + + // empty or either doesn't have path or not an http response + if (!value || ( !value.path && !(value.readable && value.hasOwnProperty('httpVersion')) )) { + return; + } + + // no need to bother with the length + if (!options.knownLength) { + this._valuesToMeasure.push(value); + } +}; + +FormData.prototype._lengthRetriever = function(value, callback) { + + if (value.hasOwnProperty('fd')) { + + // take read range into a account + // `end` = Infinity –> read file till the end + // + // TODO: Looks like there is bug in Node fs.createReadStream + // it doesn't respect `end` options without `start` options + // Fix it when node fixes it. + // https://github.com/joyent/node/issues/7819 + if (value.end != undefined && value.end != Infinity && value.start != undefined) { + + // when end specified + // no need to calculate range + // inclusive, starts with 0 + callback(null, value.end + 1 - (value.start ? value.start : 0)); + + // not that fast snoopy + } else { + // still need to fetch file size from fs + fs.stat(value.path, function(err, stat) { + + var fileSize; + + if (err) { + callback(err); + return; + } + + // update final size based on the range options + fileSize = stat.size - (value.start ? value.start : 0); + callback(null, fileSize); + }); + } + + // or http response + } else if (value.hasOwnProperty('httpVersion')) { + callback(null, +value.headers['content-length']); + + // or request stream http://github.com/mikeal/request + } else if (value.hasOwnProperty('httpModule')) { + // wait till response come back + value.on('response', function(response) { + value.pause(); + callback(null, +response.headers['content-length']); + }); + value.resume(); + + // something else + } else { + callback('Unknown stream'); + } +}; + +FormData.prototype._multiPartHeader = function(field, value, options) { + // custom header specified (as string)? + // it becomes responsible for boundary + // (e.g. to handle extra CRLFs on .NET servers) + if (typeof options.header == 'string') { + return options.header; + } + + var contentDisposition = this._getContentDisposition(value, options); + var contentType = this._getContentType(value, options); + + var contents = ''; + var headers = { + // add custom disposition as third element or keep it two elements if not + 'Content-Disposition': ['form-data', 'name="' + field + '"'].concat(contentDisposition || []), + // if no content type. allow it to be empty array + 'Content-Type': [].concat(contentType || []) + }; + + // allow custom headers. + if (typeof options.header == 'object') { + populate(headers, options.header); + } + + var header; + for (var prop in headers) { + if (!headers.hasOwnProperty(prop)) continue; + header = headers[prop]; + + // skip nullish headers. + if (header == null) { + continue; + } + + // convert all headers to arrays. + if (!Array.isArray(header)) { + header = [header]; + } + + // add non-empty headers. + if (header.length) { + contents += prop + ': ' + header.join('; ') + FormData.LINE_BREAK; + } + } + + return '--' + this.getBoundary() + FormData.LINE_BREAK + contents + FormData.LINE_BREAK; +}; + +FormData.prototype._getContentDisposition = function(value, options) { + + var filename + , contentDisposition + ; + + if (typeof options.filepath === 'string') { + // custom filepath for relative paths + filename = path.normalize(options.filepath).replace(/\\/g, '/'); + } else if (options.filename || value.name || value.path) { + // custom filename take precedence + // formidable and the browser add a name property + // fs- and request- streams have path property + filename = path.basename(options.filename || value.name || value.path); + } else if (value.readable && value.hasOwnProperty('httpVersion')) { + // or try http response + filename = path.basename(value.client._httpMessage.path); + } + + if (filename) { + contentDisposition = 'filename="' + filename + '"'; + } + + return contentDisposition; +}; + +FormData.prototype._getContentType = function(value, options) { + + // use custom content-type above all + var contentType = options.contentType; + + // or try `name` from formidable, browser + if (!contentType && value.name) { + contentType = mime.lookup(value.name); + } + + // or try `path` from fs-, request- streams + if (!contentType && value.path) { + contentType = mime.lookup(value.path); + } + + // or if it's http-reponse + if (!contentType && value.readable && value.hasOwnProperty('httpVersion')) { + contentType = value.headers['content-type']; + } + + // or guess it from the filepath or filename + if (!contentType && (options.filepath || options.filename)) { + contentType = mime.lookup(options.filepath || options.filename); + } + + // fallback to the default content type if `value` is not simple value + if (!contentType && typeof value == 'object') { + contentType = FormData.DEFAULT_CONTENT_TYPE; + } + + return contentType; +}; + +FormData.prototype._multiPartFooter = function() { + return function(next) { + var footer = FormData.LINE_BREAK; + + var lastPart = (this._streams.length === 0); + if (lastPart) { + footer += this._lastBoundary(); + } + + next(footer); + }.bind(this); +}; + +FormData.prototype._lastBoundary = function() { + return '--' + this.getBoundary() + '--' + FormData.LINE_BREAK; +}; + +FormData.prototype.getHeaders = function(userHeaders) { + var header; + var formHeaders = { + 'content-type': 'multipart/form-data; boundary=' + this.getBoundary() + }; + + for (header in userHeaders) { + if (userHeaders.hasOwnProperty(header)) { + formHeaders[header.toLowerCase()] = userHeaders[header]; + } + } + + return formHeaders; +}; + +FormData.prototype.getBoundary = function() { + if (!this._boundary) { + this._generateBoundary(); + } + + return this._boundary; +}; + +FormData.prototype._generateBoundary = function() { + // This generates a 50 character boundary similar to those used by Firefox. + // They are optimized for boyer-moore parsing. + var boundary = '--------------------------'; + for (var i = 0; i < 24; i++) { + boundary += Math.floor(Math.random() * 10).toString(16); + } + + this._boundary = boundary; +}; + +// Note: getLengthSync DOESN'T calculate streams length +// As workaround one can calculate file size manually +// and add it as knownLength option +FormData.prototype.getLengthSync = function() { + var knownLength = this._overheadLength + this._valueLength; + + // Don't get confused, there are 3 "internal" streams for each keyval pair + // so it basically checks if there is any value added to the form + if (this._streams.length) { + knownLength += this._lastBoundary().length; + } + + // https://github.com/form-data/form-data/issues/40 + if (!this.hasKnownLength()) { + // Some async length retrievers are present + // therefore synchronous length calculation is false. + // Please use getLength(callback) to get proper length + this._error(new Error('Cannot calculate proper length in synchronous way.')); + } + + return knownLength; +}; + +// Public API to check if length of added values is known +// https://github.com/form-data/form-data/issues/196 +// https://github.com/form-data/form-data/issues/262 +FormData.prototype.hasKnownLength = function() { + var hasKnownLength = true; + + if (this._valuesToMeasure.length) { + hasKnownLength = false; + } + + return hasKnownLength; +}; + +FormData.prototype.getLength = function(cb) { + var knownLength = this._overheadLength + this._valueLength; + + if (this._streams.length) { + knownLength += this._lastBoundary().length; + } + + if (!this._valuesToMeasure.length) { + process.nextTick(cb.bind(this, null, knownLength)); + return; + } + + asynckit.parallel(this._valuesToMeasure, this._lengthRetriever, function(err, values) { + if (err) { + cb(err); + return; + } + + values.forEach(function(length) { + knownLength += length; + }); + + cb(null, knownLength); + }); +}; + +FormData.prototype.submit = function(params, cb) { + var request + , options + , defaults = {method: 'post'} + ; + + // parse provided url if it's string + // or treat it as options object + if (typeof params == 'string') { + + params = parseUrl(params); + options = populate({ + port: params.port, + path: params.pathname, + host: params.hostname, + protocol: params.protocol + }, defaults); + + // use custom params + } else { + + options = populate(params, defaults); + // if no port provided use default one + if (!options.port) { + options.port = options.protocol == 'https:' ? 443 : 80; + } + } + + // put that good code in getHeaders to some use + options.headers = this.getHeaders(params.headers); + + // https if specified, fallback to http in any other case + if (options.protocol == 'https:') { + request = https.request(options); + } else { + request = http.request(options); + } + + // get content length and fire away + this.getLength(function(err, length) { + if (err) { + this._error(err); + return; + } + + // add content length + request.setHeader('Content-Length', length); + + this.pipe(request); + if (cb) { + request.on('error', cb); + request.on('response', cb.bind(this, null)); + } + }.bind(this)); + + return request; +}; + +FormData.prototype._error = function(err) { + if (!this.error) { + this.error = err; + this.pause(); + this.emit('error', err); + } +}; + +FormData.prototype.toString = function () { + return '[object FormData]'; +}; diff --git a/node_modules/request/node_modules/form-data/lib/populate.js b/node_modules/request/node_modules/form-data/lib/populate.js new file mode 100644 index 000000000..4d35738dd --- /dev/null +++ b/node_modules/request/node_modules/form-data/lib/populate.js @@ -0,0 +1,10 @@ +// populates missing values +module.exports = function(dst, src) { + + Object.keys(src).forEach(function(prop) + { + dst[prop] = dst[prop] || src[prop]; + }); + + return dst; +}; diff --git a/node_modules/request/node_modules/form-data/package.json b/node_modules/request/node_modules/form-data/package.json new file mode 100644 index 000000000..adacbae78 --- /dev/null +++ b/node_modules/request/node_modules/form-data/package.json @@ -0,0 +1,65 @@ +{ + "author": "Felix Geisendörfer (http://debuggable.com/)", + "name": "form-data", + "description": "A library to create readable \"multipart/form-data\" streams. Can be used to submit forms and file uploads to other web applications.", + "version": "2.3.3", + "repository": { + "type": "git", + "url": "git://github.com/form-data/form-data.git" + }, + "main": "./lib/form_data", + "browser": "./lib/browser", + "scripts": { + "pretest": "rimraf coverage test/tmp", + "test": "istanbul cover test/run.js", + "posttest": "istanbul report lcov text", + "lint": "eslint lib/*.js test/*.js test/integration/*.js", + "report": "istanbul report lcov text", + "ci-lint": "is-node-modern 6 && npm run lint || is-node-not-modern 6", + "ci-test": "npm run test && npm run browser && npm run report", + "predebug": "rimraf coverage test/tmp", + "debug": "verbose=1 ./test/run.js", + "browser": "browserify -t browserify-istanbul test/run-browser.js | obake --coverage", + "check": "istanbul check-coverage coverage/coverage*.json", + "files": "pkgfiles --sort=name", + "get-version": "node -e \"console.log(require('./package.json').version)\"", + "update-readme": "sed -i.bak 's/\\/master\\.svg/\\/v'$(npm --silent run get-version)'.svg/g' README.md", + "restore-readme": "mv README.md.bak README.md", + "prepublish": "in-publish && npm run update-readme || not-in-publish", + "postpublish": "npm run restore-readme" + }, + "pre-commit": [ + "lint", + "ci-test", + "check" + ], + "engines": { + "node": ">= 0.12" + }, + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.6", + "mime-types": "^2.1.12" + }, + "devDependencies": { + "browserify": "^13.1.1", + "browserify-istanbul": "^2.0.0", + "coveralls": "^2.11.14", + "cross-spawn": "^4.0.2", + "eslint": "^3.9.1", + "fake": "^0.2.2", + "far": "^0.0.7", + "formidable": "^1.0.17", + "in-publish": "^2.0.0", + "is-node-modern": "^1.0.0", + "istanbul": "^0.4.5", + "obake": "^0.1.2", + "phantomjs-prebuilt": "^2.1.13", + "pkgfiles": "^2.3.0", + "pre-commit": "^1.1.3", + "request": "2.76.0", + "rimraf": "^2.5.4", + "tape": "^4.6.2" + }, + "license": "MIT" +} diff --git a/node_modules/form-data/yarn.lock b/node_modules/request/node_modules/form-data/yarn.lock similarity index 100% rename from node_modules/form-data/yarn.lock rename to node_modules/request/node_modules/form-data/yarn.lock diff --git a/package.json b/package.json index c4a307d6c..707c5940f 100644 --- a/package.json +++ b/package.json @@ -12,16 +12,18 @@ }, "dependencies": { "@actions/core": "^1.2.0", - "@lhci/cli": "0.3.7", - "is-windows": "^1.0.2", "@actions/github": "^2.1.0", + "@lhci/cli": "0.3.7", "@slack/webhook": "^5.0.2", - "lodash": "^4.17.15" + "is-windows": "^1.0.2", + "lodash": "^4.17.15", + "node-fetch": "^2.6.0" }, "devDependencies": { - "@types/lodash": "^4.14.149", "@types/is-windows": "^1.0.0", + "@types/lodash": "^4.14.149", "@types/node": "12", + "@types/node-fetch": "^2.5.5", "prettier": "^1.19.1", "typescript": "^3.7.3" } diff --git a/src/index.js b/src/index.js index 38176d69d..6d0704e34 100644 --- a/src/index.js +++ b/src/index.js @@ -11,17 +11,71 @@ const nodePathParts = [ process.env.NODE_PATH = nodePathParts.join(nodePathDelim) const core = require('@actions/core') +const fetch = require('node-fetch').default const childProcess = require('child_process') const lhciCliPath = require.resolve('@lhci/cli/src/cli.js') const input = require('./input.js') const output = require('./output.js') +const netlifyBuildWaitingTime = 60000 + // audit urls with Lighthouse CI async function main() { + let status core.startGroup('Action config') console.log('Input args:', input) core.endGroup() // Action config + /*******************************WAITING FOR PRECONDITIONS***********************************/ + if (input.netlifySite) { + core.startGroup('Waiting for Netlify site') + const retryNumber = 5 + let runs = 0 + const waitingSeconds = parseInt((netlifyBuildWaitingTime / 1000).toString()) + const waitForNetlifyBuild = async () => { + console.log(`Waiting additional ${waitingSeconds} seconds for Netlify build to be done.`) + return new Promise(r => setTimeout(r, netlifyBuildWaitingTime)) + } + /** + * @return {Promise} + */ + const resolveNetlifyBuildURL = async () => { + runs += 1 + try { + const res = await Promise.race( + input.urls.map(async url => { + console.log(`Pinging Netlify site ${url}`) + return fetch(url) + }) + ) + if (res.status === 200) { + console.log('Netlify site finished build, continue audit...') + return Promise.resolve(0) + } else if (runs > retryNumber) { + console.log('No HTTP 200 OK from Netlify within 5 minute timeout.') + return Promise.resolve(1) + } + await waitForNetlifyBuild() + return await resolveNetlifyBuildURL() + } catch (e) { + if (runs > retryNumber) { + console.log('Resolve Netlify site error', e) + return Promise.resolve(1) + } + await waitForNetlifyBuild() + return await resolveNetlifyBuildURL() + } + } + + status = await resolveNetlifyBuildURL() + + if (status !== 0) { + throw new Error('Could not reach Netlify site within 5 minutes.') + } + + core.endGroup() // Action config + } + /*******************************COLLECTING***********************************/ core.startGroup(`Collecting`) let args = [] @@ -45,7 +99,7 @@ async function main() { } // else, no args and will default to 3 in LHCI. - let status = await runChildCommand('collect', args) + status = await runChildCommand('collect', args) if (status !== 0) { throw new Error(`LHCI 'collect' has encountered a problem.`) } diff --git a/src/input.js b/src/input.js index 5dc25f97e..7f2b8d9d6 100644 --- a/src/input.js +++ b/src/input.js @@ -1,5 +1,8 @@ const core = require('@actions/core') +const { get, trim, isNumber } = require('lodash') +const { context } = require('@actions/github') const { readFileSync } = require('fs') +const { parse: urlParse } = require('url') function getArgs() { // Make sure we don't have LHCI xor API token @@ -64,6 +67,7 @@ function getArgs() { numberOfRuns: getIntArg('runs'), applicationGithubToken: getArg('applicationGithubToken'), personalGithubToken: getArg('personalGithubToken'), + netlifySite: getArg('netlifySite'), serverBaseUrl, token, rcCollect, @@ -105,14 +109,11 @@ function getList(arg, separator = '\n') { } /** - * Takes a set of URL strings and interpolates - * any declared ENV vars into them - * * @param {string[]} urls * @return {string[]} */ function interpolateProcessIntoURLs(urls) { - return urls.map(url => { + urls = urls.map(url => { if (!url.includes('$')) return url Object.keys(process.env).forEach(key => { if (url.includes(`${key}`)) { @@ -121,6 +122,33 @@ function interpolateProcessIntoURLs(urls) { }) return url }) + + const ref = get(context, 'ref', '').split('/')[2] + const netlifySite = getArg('netlifySite') + let origin = '' + + if (ref) { + origin = `https://${ref}--${netlifySite}` + } + + if (isNumber(Number.parseInt(ref))) { + origin = `https://deploy-preview-${ref}--${netlifySite}` + } + + if (origin && netlifySite) { + return urls.map( + /** + * @param {string} url + * @return {string} + */ + url => { + let { path } = urlParse(url) + path = path || '' + return `${trim(origin, '/')}/${trim(path, '/')}` + } + ) + } + return urls } module.exports = getArgs() diff --git a/yarn.lock b/yarn.lock index d158065ed..5122f7256 100644 --- a/yarn.lock +++ b/yarn.lock @@ -179,6 +179,14 @@ resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.149.tgz#1342d63d948c6062838fbf961012f74d4e638440" integrity sha512-ijGqzZt/b7BfzcK9vTrS6MFljQRPn5BFWOx8oE0GYxribu6uV+aA9zZuXI1zc/etK9E8nrgdoF2+LgUw7+9tJQ== +"@types/node-fetch@^2.5.5": + version "2.5.5" + resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.5.5.tgz#cd264e20a81f4600a6c52864d38e7fef72485e92" + integrity sha512-IWwjsyYjGw+em3xTvWVQi5MgYKbRs0du57klfTaZkv/B24AEQ/p/IopNeqIYNy3EsfHOpg8ieQSDomPcsYMHpA== + dependencies: + "@types/node" "*" + form-data "^3.0.0" + "@types/node@*": version "12.7.5" resolved "https://registry.yarnpkg.com/@types/node/-/node-12.7.5.tgz#e19436e7f8e9b4601005d73673b6dc4784ffcc2f" @@ -515,7 +523,7 @@ color-name@1.1.3: version "1.1.3" resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.3.tgz#a7d0558bd89c42f795dd42328f740831ca53bc25" -combined-stream@^1.0.6, combined-stream@~1.0.6: +combined-stream@^1.0.6, combined-stream@^1.0.8, combined-stream@~1.0.6: version "1.0.8" resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.8.tgz#c3d45a8b34fd730631a110a8a2520682b31d5a7f" dependencies: @@ -879,6 +887,15 @@ forever-agent@~0.6.1: version "0.6.1" resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91" +form-data@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/form-data/-/form-data-3.0.0.tgz#31b7e39c85f1355b7139ee0c647cf0de7f83c682" + integrity sha512-CKMFDglpbMi6PyN+brwB9Q/GOw0eAnsrEZDgcsH5Krhz5Od/haKHAX0NmQfha2zPPz0JpWzA7GJHGSnvCRLWsg== + dependencies: + asynckit "^0.4.0" + combined-stream "^1.0.8" + mime-types "^2.1.12" + form-data@~2.3.2: version "2.3.3" resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.3.3.tgz#dcce52c05f644f298c6a7ab936bd724ceffbf3a6" @@ -1580,7 +1597,7 @@ node-fetch@^1.0.1: encoding "^0.1.11" is-stream "^1.0.1" -node-fetch@^2.3.0: +node-fetch@^2.3.0, node-fetch@^2.6.0: version "2.6.0" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.0.tgz#e633456386d4aa55863f676a7ab0daa8fdecb0fd" integrity sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==