From 24ae03eb19db61511993bc13828513ab6617e761 Mon Sep 17 00:00:00 2001 From: Gaurav Nelson Date: Thu, 11 Feb 2021 13:44:03 +1000 Subject: [PATCH 1/2] Updates and config file issue fix --- .github/workflows/publish-snap.yml | 16 --- Dockerfile | 13 +- README.md | 20 +-- asciidoc-link-check | 224 ++++++++++++++++++----------- index.js | 127 ++++++++++++---- package.json | 16 +-- snapcraft.yaml | 28 ---- test.md | 3 + 8 files changed, 272 insertions(+), 175 deletions(-) delete mode 100644 .github/workflows/publish-snap.yml delete mode 100644 snapcraft.yaml create mode 100644 test.md diff --git a/.github/workflows/publish-snap.yml b/.github/workflows/publish-snap.yml deleted file mode 100644 index 98ebf96..0000000 --- a/.github/workflows/publish-snap.yml +++ /dev/null @@ -1,16 +0,0 @@ -name: Publish to Snapcraft - -on: [push] - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: snapcore/action-build@v1 - id: build - - uses: snapcore/action-publish@v1 - with: - store_login: ${{ secrets.STORE_LOGIN }} - snap: ${{ steps.build.outputs.snap }} - release: stable diff --git a/Dockerfile b/Dockerfile index e8286df..3fdf3e6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,17 @@ -FROM node:latest +FROM node:alpine LABEL maintainer="https://github.com/gaurav-nelson" +# Use the Open Container Image Annotation Spec (https://github.com/opencontainers/image-spec/blob/master/annotations.md) +LABEL org.opencontainers.image.title="asciidoc-link-check" +LABEL org.opencontainers.image.description="Checks if all hyperlinks in an asciidoc file are alive(or dead)." +LABEL org.opencontainers.image.documentation="https://github.com/gaurav-nelson/asciidoc-link-check/blob/master/README.md" +LABEL org.opencontainers.image.source="https://github.com/gaurav-nelson/asciidoc-link-check" + # Install app dependencies -COPY package.json /src/package.json +COPY package.json /src/ WORKDIR /src RUN set -ex; \ - npm install \ - && npm ls + npm install # Bundle app source COPY . /src ENTRYPOINT [ "/src/asciidoc-link-check" ] diff --git a/README.md b/README.md index bd69615..a801ddb 100644 --- a/README.md +++ b/README.md @@ -5,14 +5,7 @@ Local Installation ================== -[![Get it from the Snap Store](https://snapcraft.io/static/images/badges/en/snap-store-black.svg)](https://snapcraft.io/asciidoc-link-check) - - -```bash -sudo snap install asciidoc-link-check -``` - -Or install with `npm` run: +Install with `npm` run: ``` bash npm install -g asciidoc-link-check @@ -72,6 +65,12 @@ find -name \*.adoc -exec asciidoc-link-check -p {} \; asciidoc-link-check README.adoc -c ``` +> **NOTE** +> +> To see other options that `asciidoc-link-check` supports, see +> `markdown-link-check` +> config file format at https://github.com/tcort/markdown-link-check#config-file-format + Using in your node project -------------------------- @@ -95,9 +94,10 @@ Docker Run docker build --tag asciidoc-link-check . -4. Pipe any asciidoc file `test.adoc` to `docker run`: +4. Add current directory with your `test.adoc` file as read only volume to the + `docker run` command: - docker run --rm -i asciidoc-link-check < test.adoc + docker run -v ${PWD}:/tmp:ro --rm -i asciidoc-link-check /tmp/test.adoc API === diff --git a/asciidoc-link-check b/asciidoc-link-check index 9380184..989bfbe 100755 --- a/asciidoc-link-check +++ b/asciidoc-link-check @@ -2,62 +2,110 @@ 'use strict'; -var chalk = require('chalk'); -var fs = require('fs'); -var asciidocLinkCheck = require('./'); -var program = require('commander'); +const chalk = require('chalk'); +const fs = require('fs'); +const asciidocLinkCheck = require('./'); +const program = require('commander'); const axios = require('axios'); -var url = require('url'); -var path = require('path'); - -var statusLabels = { - alive: chalk.green('✓'), - dead: chalk.red('✖'), - ignored: chalk.gray('~') +const url = require('url'); +const path = require('path'); + +const statusLabels = { + alive: chalk.green('✓'), + dead: chalk.red('✖'), + ignored: chalk.blue('/'), + error: chalk.yellow('⚠'), }; -var error = false; -var opts = {}; -var filenameOutput = ""; -var stream = process.stdin; // read from stdin unless a filename is given +const opts = {}; +let filenameForOutput = ''; +let stream = process.stdin; // read from stdin unless a filename is given + +function commaSeparatedCodesList(value, dummyPrevious) { + return value.split(',').map(function(item) { + return parseInt(item, 10); + }); +} + program -.option('-p, --progress', 'Show progress bar') -.option('-c, --config [config]', 'Use a JSON config file') -.option('-q, --quiet', 'Display errors only') -.arguments('[filenameOrUrl]') -.action(function (filenameOrUrl) { - filenameOutput = filenameOrUrl; - if (/https?:/.test(filenameOrUrl)) { - stream = axios.get(filenameOrUrl); - try { // extract baseUrl from supplied URL - var parsed = url.parse(filenameOrUrl); - delete parsed.search; - delete parsed.hash; - if (parsed.pathname.lastIndexOf('/') !== -1) { - parsed.pathname = parsed.pathname.substr(0, parsed.pathname.lastIndexOf('/') + 1); - } - opts.baseUrl = url.format(parsed); - } catch (err) { /* ignore error */ } - } else { - opts.baseUrl = 'file://' + path.dirname(path.resolve(filenameOrUrl)); - stream = fs.createReadStream(filenameOrUrl); - } - }) -.parse(process.argv); + .option('-p, --progress', 'show progress bar') + .option('-c, --config [config]', 'apply a config file (JSON), holding e.g. url specific header configuration') + .option('-q, --quiet', 'displays errors only') + .option('-v, --verbose', 'displays detailed error information') + .option('-a, --alive ', 'comma separated list of HTTP codes to be considered as alive', commaSeparatedCodesList) + .option('-r, --retry', 'retry after the duration indicated in \'retry-after\' header when HTTP code is 429') + .arguments('[filenameOrUrl]') + .action(function (filenameOrUrl) { + filenameForOutput = filenameOrUrl; + if (/https?:/.test(filenameOrUrl)) { + axios(filenameOrUrl, function (error, response, body) { + if (error) { + console.error(chalk.red('\nERROR: Unable to connect! Please provide a valid URL as an argument.')); + process.exit(1); + } + else if (response.statusCode === 404){ + console.error(chalk.red('\nERROR: 404 - File not found! Please provide a valid URL as an argument.')); + process.exit(1); + } else { + stream = axios.get(filenameOrUrl); + } -opts.showProgressBar = (program.progress === true); // force true or undefined to be true or false. -opts.quiet = (program.quiet === true); + }); + try { // extract baseUrl from supplied URL + const parsed = url.parse(filenameOrUrl); + delete parsed.search; + delete parsed.hash; + if (parsed.pathname.lastIndexOf('/') !== -1) { + parsed.pathname = parsed.pathname.substr(0, parsed.pathname.lastIndexOf('/') + 1); + } + opts.baseUrl = url.format(parsed); + } catch (err) { /* ignore error */ + } + } else { + fs.stat(filenameOrUrl, function(error , stats){ + if (!error && stats.isDirectory()){ + console.error(chalk.red('\nERROR: ' + filenameOrUrl + ' is a directory! Please provide a valid filename as an argument.')); + process.exit(1); + } + }); + opts.baseUrl = 'file://' + path.dirname(path.resolve(filenameOrUrl)); + stream = fs.createReadStream(filenameOrUrl); + } -var asciidoc = ''; // collect the asciidoc data, then process it -stream.on('data', function (chunk) { - asciidoc += chunk.toString(); -}).on('end', function () { - console.log(chalk.cyan('\nFILE: ' + filenameOutput)); +}).parse(process.argv); - if (program.config) { - fs.access(program.config, fs.constants.R_OK, function (err) { +const options = program.opts(); + +opts.showProgressBar = (program.progress === true); // force true or undefined to be true or false. +opts.quiet = (program.quiet === true); +opts.verbose = (program.verbose === true); +opts.retryOn429 = (program.retry === true); +opts.aliveStatusCodes = program.alive; +// set the projectBaseUrl to the current working directory, so that `{{BASEURL}}` can be resolved to the project root. +opts.projectBaseUrl = `file://${process.cwd()}`; + +let asciidoc = ''; // collect the asciidoc data, then process it + +stream + .on('data', function (chunk) { + asciidoc += chunk.toString(); + }) + .on('error', function(error) { + if (error.code === 'ENOENT') { + console.error(chalk.red('\nERROR: File not found! Please provide a valid filename as an argument.')); + } else { + console.error(chalk.red(error)); + } + return process.exit(1); + }) + .on('end', function () { + if (filenameForOutput) { + console.log(chalk.cyan('\nFILE: ' + filenameForOutput)); + } + if (options.config) { + fs.access(options.config, (fs.constants || fs).R_OK, function (err) { if (!err) { - let configStream = fs.createReadStream(program.config); + let configStream = fs.createReadStream(options.config); let configData = ''; configStream.on('data', function (chunk) { @@ -66,12 +114,20 @@ stream.on('data', function (chunk) { let config = JSON.parse(configData); opts.ignorePatterns = config.ignorePatterns; + opts.replacementPatterns = config.replacementPatterns; + opts.httpHeaders = config.httpHeaders; + opts.timeout = config.timeout; + opts.ignoreDisable = config.ignoreDisable; + opts.retryOn429 = config.retryOn429; + opts.retryCount = config.retryCount; + opts.fallbackRetryDelay = config.fallbackRetryDelay; + opts.aliveStatusCodes = config.aliveStatusCodes; runasciidocLinkCheck(asciidoc, opts); }); } else { - console.error(chalk.red('\nERROR: Can\'t access configuration file.')); + console.error(chalk.red('\nERROR: Config file not accessible.')); process.exit(1); } }); @@ -82,37 +138,41 @@ stream.on('data', function (chunk) { }); function runasciidocLinkCheck(asciidoc, opts) { + asciidocLinkCheck(asciidoc, opts, function (err, results) { + if (err) { + console.error(chalk.red('\nERROR: something went wrong!')); + console.error(err.stack); + process.exit(1); + } + + if (results.length === 0 && !opts.quiet) { + console.log(chalk.yellow('No hyperlinks found!')); + } + results.forEach(function (result) { + // Skip messages for non-deadlinks in quiet mode. + if (opts.quiet && result.status !== 'dead') { + return; + } -asciidocLinkCheck(asciidoc, opts, function (err, results) { - - if (results.length === 0) { - console.log(chalk.yellow('No hyperlinks found!\n')); - } - results.forEach(function (result) { - if (result.status === 'dead') { - error = true; - } - - // Skip messages for non-deadlinks in quiet mode. - if (opts.quiet && result.status !== 'dead') { - return; - } - - switch(result.status) { - case 'dead': - console.log('[%s] %s', statusLabels[result.status], chalk.red(result.link)); - break; - case 'ignored': - console.log('[%s] %s', statusLabels[result.status], chalk.gray(result.link)); - break; - default: - console.log('[%s] %s', statusLabels[result.status], result.link); - } - - }); - if (error) { - console.error(chalk.red('ERROR: dead links found!\n')); - process.exit(1); - } -}); -} \ No newline at end of file + if (opts.verbose) { + if (result.err) { + console.log('[%s] %s → Status: %s %s', statusLabels[result.status], result.link, result.statusCode, result.err); + } else { + console.log('[%s] %s → Status: %s', statusLabels[result.status], result.link, result.statusCode); + } + } + else { + console.log('[%s] %s', statusLabels[result.status], result.link); + } + }); + console.log('\n%s links checked.', results.length); + if (results.some((result) => result.status === 'dead')) { + let deadLinks = results.filter(result => { return result.status === 'dead'; }); + console.error(chalk.red('\nERROR: %s dead links found!'), deadLinks.length); + deadLinks.forEach(function (result) { + console.log('[%s] %s → Status: %s', statusLabels[result.status], result.link, result.statusCode); + }); + process.exit(1); + } + }); +} diff --git a/index.js b/index.js index 6091833..1f62770 100644 --- a/index.js +++ b/index.js @@ -1,13 +1,47 @@ -(function () { - 'use strict'; - // this function is strict... - }()); -var _ = require('lodash'); -var async = require('async'); -var linkCheck = require('link-check'); -var asciidocLinkExtractor = require('asciidoc-link-extractor'); -var ProgressBar = require('progress'); +'use strict'; + +const _ = require('lodash'); +const async = require('async'); +const linkCheck = require('link-check'); +const LinkCheckResult = require('link-check').LinkCheckResult; +const asciidocLinkExtractor = require('asciidoc-link-extractor'); +const ProgressBar = require('progress'); + +const envVarPatternMatcher = /(?{{env\.(?[a-zA-Z0-9\-_]+)}})/; + +/* + * Performs some special replacements for the following patterns: + * - {{BASEURL}} - to be replaced with opts.projectBaseUrl + * - {{env.}} - to be replaced with the environment variable specified with + */ +function performSpecialReplacements(str, opts) { + // replace the `{{BASEURL}}` with the opts.projectBaseUrl. Helpful to build absolute urls "relative" to project roots + str = str.replace('{{BASEURL}}', opts.projectBaseUrl); + + // replace {{env.}} with the corresponding environment variable or an empty string if none is set. + var envVarMatch; + do { + envVarMatch = envVarPatternMatcher.exec(str); + + if(!envVarMatch) { + break; + } + + var envVarPattern = envVarMatch.groups.pattern; + var envVarName = envVarMatch.groups.name; + + var envVarPatternReplacement = ''; + + if(envVarName in process.env) { + envVarPatternReplacement = process.env[envVarName]; + } + + str = str.replace(envVarPattern, envVarPatternReplacement); + } while (true); + + return str; +} module.exports = function asciidocLinkCheck(asciidoc, opts, callback) { if (arguments.length === 2 && typeof opts === 'function') { @@ -16,42 +50,81 @@ module.exports = function asciidocLinkCheck(asciidoc, opts, callback) { opts = {}; } - var bar; - var linksCollection = _.uniq(asciidocLinkExtractor(asciidoc)); - if (opts.showProgressBar) { - bar = new ProgressBar('Checking... [:bar] :percent', { + if(!opts.ignoreDisable) { + asciidoc = [ + /([\S\s]*?)/mg, + /([\S\s]*(?!))/mg, + /(\r?\n[^\r\n]*)/mg, + /([^\r\n]*[^\r\n]*)/mg + ].reduce(function(_asciidoc, disablePattern) { + return _asciidoc.replace(new RegExp(disablePattern), ''); + }, asciidoc); + } + + const linksCollection = _.uniq(asciidocLinkExtractor(asciidoc)); + const bar = (opts.showProgressBar) ? + new ProgressBar('Checking... [:bar] :percent', { complete: '=', incomplete: ' ', width: 25, total: linksCollection.length - }); - } + }) : undefined; async.mapLimit(linksCollection, 2, function (link, callback) { if (opts.ignorePatterns) { - let shouldIgnore = opts.ignorePatterns.some(function(ignorePattern) { + const shouldIgnore = opts.ignorePatterns.some(function(ignorePattern) { return ignorePattern.pattern instanceof RegExp ? ignorePattern.pattern.test(link) : (new RegExp(ignorePattern.pattern)).test(link) ? true : false; }); - + if (shouldIgnore) { - let linkCheckResult = {}; + const result = new LinkCheckResult(opts, link, 0, undefined); + result.status = 'ignored'; // custom status for ignored links + callback(null, result); + return; + } + } + + if (opts.replacementPatterns) { + for (let replacementPattern of opts.replacementPatterns) { + let pattern = replacementPattern.pattern instanceof RegExp ? replacementPattern.pattern : new RegExp(replacementPattern.pattern); + link = link.replace(pattern, performSpecialReplacements(replacementPattern.replacement, opts)); + } + } - linkCheckResult.link = link; - linkCheckResult.statusCode = 0; - linkCheckResult.status = 'ignored'; - if (opts.showProgressBar) { - bar.tick(); + // Make sure it is not undefined and that the appropriate headers are always recalculated for a given link. + opts.headers = {}; + + if (opts.httpHeaders) { + for (const httpHeader of opts.httpHeaders) { + if (httpHeader.headers) { + for (const header of Object.keys(httpHeader.headers)) { + httpHeader.headers[header] = performSpecialReplacements(httpHeader.headers[header], opts); + } + } + + for (const url of httpHeader.urls) { + if (link.startsWith(url)) { + Object.assign(opts.headers, httpHeader.headers); + + // The headers of this httpHeader has been applied, the other URLs of this httpHeader don't need to be evaluated any further. + break; + } } - callback(null, linkCheckResult); - return; } } - + linkCheck(link, opts, function (err, result) { + if (opts.showProgressBar) { bar.tick(); } - callback(err, result); + + if (err) { + result = new LinkCheckResult(opts, link, 500, err); + result.status = 'error'; // custom status for errored links + } + + callback(null, result); }); }, callback); }; diff --git a/package.json b/package.json index 3ccf7d9..2c4e3a3 100644 --- a/package.json +++ b/package.json @@ -31,15 +31,15 @@ "homepage": "https://github.com/gaurav-nelson/asciidoc-link-check#readme", "dependencies": { "asciidoc-link-extractor": ">=1.0.3", - "async": ">=2.6.1", - "chalk": "^2.4.2", - "commander": ">=2.19.0", - "link-check": ">=4.4.4", - "lodash": ">=4.17.11", - "progress": "^2.0.3", - "axios": ">=0.21.1" + "async": ">=3.2.0", + "axios": ">=0.21.1", + "chalk": "^4.1.0", + "commander": ">=7.0.0", + "link-check": ">=4.5.4", + "lodash": ">=4.17.20", + "progress": "^2.0.3" }, "devDependencies": { - "jshint": "^2.9.7" + "jshint": "^2.12.0" } } diff --git a/snapcraft.yaml b/snapcraft.yaml deleted file mode 100644 index d7e5a3a..0000000 --- a/snapcraft.yaml +++ /dev/null @@ -1,28 +0,0 @@ -name: asciidoc-link-check -version: git -summary: Checks if all hyperlinks in an asciidoc file are alive(or dead). -description: | - Check links for a local file: - $ asciidoc-link-check README.adoc --progress - Check links for an online file: - $ asciidoc-link-check https://github.com/gaurav-nelson/asciidoc-link-check/blob/master/README.adoc - Check links from standard input: - $ cat .adoc | asciidoc-link-check -p - Check links in all asciidoc files in the current directory: - $ find . -name \*.adoc -exec asciidoc-link-check -p {} \; - Ignore specific links: - $ asciidoc-link-check README.adoc -c -version: 1.0.14 -grade: stable - -base: core18 -confinement: strict - -parts: - asciidoc-link-check: - plugin: nodejs - source: . - -apps: - asciidoc-link-check: - command: asciidoc-link-check \ No newline at end of file diff --git a/test.md b/test.md new file mode 100644 index 0000000..60dceb5 --- /dev/null +++ b/test.md @@ -0,0 +1,3 @@ +[This is](http://www.methods.co.nz/asciidoc/) + +[Another](https://github.abcd/) \ No newline at end of file From 6eb277a8479fa769e341ca0ca0f8d657c1adc0f1 Mon Sep 17 00:00:00 2001 From: Gaurav Nelson Date: Thu, 11 Feb 2021 13:54:07 +1000 Subject: [PATCH 2/2] 1.0.15 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 2c4e3a3..bc39863 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "asciidoc-link-check", - "version": "1.0.14", + "version": "1.0.15", "description": "Checks if all hyperlinks in an asciidoc file are alive(or dead).", "bin": { "asciidoc-link-check": "asciidoc-link-check"