Skip to content

Commit

Permalink
tools: lint shell scripts
Browse files Browse the repository at this point in the history
PR-URL: #36099
Reviewed-By: Rich Trott <rtrott@gmail.com>
  • Loading branch information
aduh95 authored and targos committed Dec 21, 2020
1 parent eb29a16 commit 7b8d373
Show file tree
Hide file tree
Showing 12 changed files with 255 additions and 54 deletions.
7 changes: 7 additions & 0 deletions .github/workflows/linters.yml
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,13 @@ jobs:
run: |
make lint-py-build || true
NODE=$(command -v node) make lint-py
lint-sh:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v2
- run: shellcheck -V
- name: Lint Shell scripts
run: tools/lint-sh.js .

lint-codeowners:
runs-on: ubuntu-latest
Expand Down
11 changes: 6 additions & 5 deletions tools/actions/commit-queue.sh
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#!/bin/bash
#!/bin/sh

set -xe

Expand All @@ -14,19 +14,19 @@ API_URL=https://api.github.com
COMMIT_QUEUE_LABEL='commit-queue'
COMMIT_QUEUE_FAILED_LABEL='commit-queue-failed'

function issueUrl() {
issueUrl() {
echo "$API_URL/repos/${OWNER}/${REPOSITORY}/issues/${1}"
}

function labelsUrl() {
labelsUrl() {
echo "$(issueUrl "${1}")/labels"
}

function commentsUrl() {
commentsUrl() {
echo "$(issueUrl "${1}")/comments"
}

function gitHubCurl() {
gitHubCurl() {
url=$1
method=$2
shift 2
Expand Down Expand Up @@ -67,6 +67,7 @@ for pr in "$@"; do
if ! tail -n 10 output | grep '. Post "Landed in .*/pull/'"${pr}"; then
gitHubCurl "$(labelsUrl "$pr")" POST --data '{"labels": ["'"${COMMIT_QUEUE_FAILED_LABEL}"'"]}'

# shellcheck disable=SC2154
cqurl="${GITHUB_SERVER_URL}/${OWNER}/${REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
jq -n --arg content "<details><summary>Commit Queue failed</summary><pre>$(cat output)</pre><a href='$cqurl'>$cqurl</a></details>" '{body: $content}' > output.json
cat output.json
Expand Down
10 changes: 5 additions & 5 deletions tools/actions/start-ci.sh
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#!/bin/bash
#!/bin/sh

set -xe

Expand All @@ -10,15 +10,15 @@ REQUEST_CI_LABEL='request-ci'
REQUEST_CI_FAILED_LABEL='request-ci-failed'
shift 3

function issueUrl() {
issueUrl() {
echo "$API_URL/repos/${OWNER}/${REPOSITORY}/issues/${1}"
}

function labelsUrl() {
labelsUrl() {
echo "$(issueUrl "${1}")/labels"
}

function commentsUrl() {
commentsUrl() {
echo "$(issueUrl "${1}")/comments"
}

Expand All @@ -33,7 +33,7 @@ for pr in "$@"; do
ncu-ci run "$pr" >output 2>&1 || ci_started=no
cat output

if [ "$ci_started" == "no" ]; then
if [ "$ci_started" = "no" ]; then
# Do we need to reset?
curl -sL --request PUT \
--url "$(labelsUrl "$pr")" \
Expand Down
8 changes: 4 additions & 4 deletions tools/create_expfile.sh
Original file line number Diff line number Diff line change
Expand Up @@ -37,15 +37,15 @@
echo "Searching $1 to write out expfile to $2"

# This special sequence must be at the start of the exp file.
echo "#!." > $2.tmp
echo "#!." > "$2.tmp"

# Pull the symbols from the .a files.
find $1 -name "*.a" | grep -v gtest \
find "$1" -name "*.a" | grep -v gtest \
| xargs nm -Xany -BCpg \
| awk '{
if ((($2 == "T") || ($2 == "D") || ($2 == "B")) &&
(substr($3,1,1) != ".")) { print $3 }
}' \
| sort -u >> $2.tmp
| sort -u >> "$2.tmp"

mv -f $2.tmp $2
mv -f "$2.tmp" "$2"
10 changes: 5 additions & 5 deletions tools/lint-pr-commit-message.sh
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#!/usr/bin/env bash
#!/bin/sh

# Shell script to lint the message of the first commit in a pull request.
#
Expand All @@ -13,7 +13,7 @@ PR_ID=$1;
if [ -z "${PR_ID}" ]; then
# Attempt to work out the PR number based on current HEAD
if HEAD_COMMIT="$( git rev-parse HEAD )"; then
if SEARCH_RESULTS="$( curl -s ${GH_API_URL}/search/issues?q=sha:${HEAD_COMMIT}+type:pr+repo:nodejs/node )"; then
if SEARCH_RESULTS="$( curl -s "${GH_API_URL}/search/issues?q=sha:${HEAD_COMMIT}+type:pr+repo:nodejs/node" )"; then
if FOUND_PR="$( node -p 'JSON.parse(process.argv[1]).items[0].number' "${SEARCH_RESULTS}" 2> /dev/null )"; then
PR_ID=${FOUND_PR}
fi
Expand All @@ -26,13 +26,13 @@ if [ -z "${PR_ID}" ]; then
exit 1
fi

PATCH=$( curl -sL https://github.com/nodejs/node/pull/${PR_ID}.patch | grep '^From ' )
PATCH=$( curl -sL "https://github.com/nodejs/node/pull/${PR_ID}.patch" | grep '^From ' )
if FIRST_COMMIT="$( echo "$PATCH" | awk '/^From [0-9a-f]{40} / { if (count++ == 0) print $2 }' )"; then
MESSAGE=$( git show --quiet --format='format:%B' $FIRST_COMMIT )
MESSAGE=$( git show --quiet --format='format:%B' "$FIRST_COMMIT" )
echo "
*** Linting the first commit message for pull request ${PR_ID}
*** according to the guidelines at https://goo.gl/p2fr5Q.
*** Commit message for $(echo $FIRST_COMMIT | cut -c 1-10) is:
*** Commit message for $(echo "$FIRST_COMMIT" | cut -c 1-10) is:
${MESSAGE}
"
npx -q core-validate-commit --no-validate-metadata "${FIRST_COMMIT}"
Expand Down
185 changes: 185 additions & 0 deletions tools/lint-sh.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,185 @@
#!/usr/bin/env node
'use strict';

const { execSync, spawn } = require('child_process');
const { promises: fs, readdirSync, statSync } = require('fs');
const { extname, join, relative, resolve } = require('path');

const FIX_MODE_ENABLED = process.argv.includes('--fix');
const USE_NPX = process.argv.includes('--from-npx');

const SHELLCHECK_EXE_NAME = 'shellcheck';
const SHELLCHECK_OPTIONS = ['--shell=sh', '--severity=info', '--enable=all'];
if (FIX_MODE_ENABLED) SHELLCHECK_OPTIONS.push('--format=diff');
else if (process.env.GITHUB_ACTIONS) SHELLCHECK_OPTIONS.push('--format=json');

const SPAWN_OPTIONS = {
cwd: null,
shell: false,
stdio: ['pipe', 'pipe', 'inherit'],
};

function* findScriptFilesRecursively(dirPath) {
const entries = readdirSync(dirPath, { withFileTypes: true });

for (const entry of entries) {
const path = join(dirPath, entry.name);

if (
entry.isDirectory() &&
entry.name !== 'build' &&
entry.name !== 'changelogs' &&
entry.name !== 'deps' &&
entry.name !== 'fixtures' &&
entry.name !== 'gyp' &&
entry.name !== 'inspector_protocol' &&
entry.name !== 'node_modules' &&
entry.name !== 'out' &&
entry.name !== 'tmp'
) {
yield* findScriptFilesRecursively(path);
} else if (entry.isFile() && extname(entry.name) === '.sh') {
yield path;
}
}
}

const expectedHashBang = Buffer.from('#!/bin/sh\n');
async function hasInvalidHashBang(fd) {
const { length } = expectedHashBang;

const actual = Buffer.allocUnsafe(length);
await fd.read(actual, 0, length, 0);

return Buffer.compare(actual, expectedHashBang);
}

async function checkFiles(...files) {
const flags = FIX_MODE_ENABLED ? 'r+' : 'r';
await Promise.all(
files.map(async (file) => {
const fd = await fs.open(file, flags);
if (await hasInvalidHashBang(fd)) {
if (FIX_MODE_ENABLED) {
const file = await fd.readFile();

const fileContent =
file[0] === '#'.charCodeAt() ?
file.subarray(file.indexOf('\n') + 1) :
file;

const toWrite = Buffer.concat([expectedHashBang, fileContent]);
await fd.truncate(toWrite.length);
await fd.write(toWrite, 0, toWrite.length, 0);
} else {
if (!process.exitCode) process.exitCode = 1;
console.error(
(process.env.GITHUB_ACTIONS ?
`::error file=${file},line=1,col=1::` :
'Fixable with --fix: ') +
`Invalid hashbang for ${file} (expected /bin/sh).`
);
}
}
await fd.close();
})
);

const stdout = await new Promise((resolve, reject) => {
const SHELLCHECK_EXE =
process.env.SHELLCHECK ||
execSync('command -v ' + (USE_NPX ? 'npx' : SHELLCHECK_EXE_NAME))
.toString()
.trim();
const NPX_OPTIONS = USE_NPX ? [SHELLCHECK_EXE_NAME] : [];

const shellcheck = spawn(
SHELLCHECK_EXE,
[
...NPX_OPTIONS,
...SHELLCHECK_OPTIONS,
...(FIX_MODE_ENABLED ?
files.map((filePath) => relative(SPAWN_OPTIONS.cwd, filePath)) :
files),
],
SPAWN_OPTIONS
);
shellcheck.once('error', reject);

let json = '';
let childProcess = shellcheck;
if (FIX_MODE_ENABLED) {
const GIT_EXE =
process.env.GIT || execSync('command -v git').toString().trim();

const gitApply = spawn(GIT_EXE, ['apply'], SPAWN_OPTIONS);
shellcheck.stdout.pipe(gitApply.stdin);
shellcheck.once('exit', (code) => {
if (!process.exitCode && code) process.exitCode = code;
});
gitApply.stdout.pipe(process.stdout);

gitApply.once('error', reject);
childProcess = gitApply;
} else if (process.env.GITHUB_ACTIONS) {
shellcheck.stdout.on('data', (chunk) => {
json += chunk;
});
} else {
shellcheck.stdout.pipe(process.stdout);
}
childProcess.once('exit', (code) => {
if (!process.exitCode && code) process.exitCode = code;
resolve(json);
});
});

if (!FIX_MODE_ENABLED && process.env.GITHUB_ACTIONS) {
const data = JSON.parse(stdout);
for (const { file, line, column, message } of data) {
console.error(
`::error file=${file},line=${line},col=${column}::` +
`${file}:${line}:${column}: ${message}`
);
}
}
}

const USAGE_STR =
`Usage: ${process.argv[1]} <path> [--fix] [--from-npx]\n` +
'\n' +
'Environment variables:\n' +
' - SHELLCHECK: absolute path to `shellcheck`. If not provided, the\n' +
' script will use the result of `command -v shellcheck`, or\n' +
' `$(command -v npx) shellcheck` if the flag `--from-npx` is provided\n' +
' (may require an internet connection).\n' +
' - GIT: absolute path to `git`. If not provided, the \n' +
' script will use the result of `command -v git`.\n';

if (
process.argv.length < 3 ||
process.argv.includes('-h') ||
process.argv.includes('--help')
) {
console.log(USAGE_STR);
} else {
console.log('Running Shell scripts checker...');
const entryPoint = resolve(process.argv[2]);
const stats = statSync(entryPoint, { throwIfNoEntry: false });

function onError(e) {
console.log(USAGE_STR);
console.error(e);
process.exitCode = 1;
}
if (stats?.isDirectory()) {
SPAWN_OPTIONS.cwd = entryPoint;
checkFiles(...findScriptFilesRecursively(entryPoint)).catch(onError);
} else if (stats?.isFile()) {
SPAWN_OPTIONS.cwd = process.cwd();
checkFiles(entryPoint).catch(onError);
} else {
onError(new Error('You must provide a valid directory or file path. ' +
`Received '${process.argv[2]}'.`));
}
}
2 changes: 1 addition & 1 deletion tools/macos-firewall.sh
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#!/bin/bash
#!/bin/sh
# Script that adds rules to Mac OS X Socket Firewall to avoid
# popups asking to accept incoming network connections when
# running tests.
Expand Down
30 changes: 17 additions & 13 deletions tools/make-v8.sh
Original file line number Diff line number Diff line change
@@ -1,37 +1,41 @@
#!/bin/bash -xe
#!/bin/sh

set -xe

BUILD_ARCH_TYPE=$1
V8_BUILD_OPTIONS=$2

cd deps/v8
find . -type d -name .git | xargs rm -rf
cd deps/v8 || exit
find . -type d -name .git -print0 | xargs -0 rm -rf
tools/node/fetch_deps.py .

ARCH="`arch`"
if [[ "$ARCH" == "s390x" ]] || [[ "$ARCH" == "ppc64le" ]]; then
if [ "$ARCH" = "s390x" ] || [ "$ARCH" = "ppc64le" ]; then
TARGET_ARCH=$ARCH
if [[ "$ARCH" == "ppc64le" ]]; then
if [ "$ARCH" = "ppc64le" ]; then
TARGET_ARCH="ppc64"
fi
# set paths manually for now to use locally installed gn
export BUILD_TOOLS=/home/iojs/build-tools
export LD_LIBRARY_PATH=$BUILD_TOOLS:$LD_LIBRARY_PATH
# Avoid linking to ccache symbolic links as ccache decides which
# binary to run based on the name of the link (we always name them gcc/g++).
CC_PATH=`which -a $CC gcc | grep -v ccache | head -n 1`
CXX_PATH=`which -a $CXX g++ | grep -v ccache | head -n 1`
# shellcheck disable=SC2154
CC_PATH=`command -v "$CC" gcc | grep -v ccache | head -n 1`
# shellcheck disable=SC2154
CXX_PATH=`command -v "$CXX" g++ | grep -v ccache | head -n 1`
rm -f "$BUILD_TOOLS/g++"
rm -f "$BUILD_TOOLS/gcc"
ln -s $CXX_PATH "$BUILD_TOOLS/g++"
ln -s $CC_PATH "$BUILD_TOOLS/gcc"
ln -s "$CXX_PATH" "$BUILD_TOOLS/g++"
ln -s "$CC_PATH" "$BUILD_TOOLS/gcc"
export PATH=$BUILD_TOOLS:$PATH

g++ --version
gcc --version
export PKG_CONFIG_PATH=$BUILD_TOOLS/pkg-config
gn gen -v out.gn/$BUILD_ARCH_TYPE --args="is_component_build=false is_debug=false use_goma=false goma_dir=\"None\" use_custom_libcxx=false v8_target_cpu=\"$TARGET_ARCH\" target_cpu=\"$TARGET_ARCH\" v8_enable_backtrace=true"
ninja -v -C out.gn/$BUILD_ARCH_TYPE d8 cctest inspector-test
gn gen -v "out.gn/$BUILD_ARCH_TYPE" --args="is_component_build=false is_debug=false use_goma=false goma_dir=\"None\" use_custom_libcxx=false v8_target_cpu=\"$TARGET_ARCH\" target_cpu=\"$TARGET_ARCH\" v8_enable_backtrace=true"
ninja -v -C "out.gn/$BUILD_ARCH_TYPE" d8 cctest inspector-test
else
PATH=~/_depot_tools:$PATH tools/dev/v8gen.py $BUILD_ARCH_TYPE --no-goma $V8_BUILD_OPTIONS
PATH=~/_depot_tools:$PATH ninja -C out.gn/$BUILD_ARCH_TYPE/ d8 cctest inspector-test
PATH=~/_depot_tools:$PATH tools/dev/v8gen.py "$BUILD_ARCH_TYPE" --no-goma "$V8_BUILD_OPTIONS"
PATH=~/_depot_tools:$PATH ninja -C "out.gn/$BUILD_ARCH_TYPE/" d8 cctest inspector-test
fi
2 changes: 1 addition & 1 deletion tools/osx-pkg-postinstall.sh
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#!/bin/sh
# TODO Can this be done inside the .pmdoc?
# TODO Can we extract $PREFIX from the installer?
cd /usr/local/bin
cd /usr/local/bin || exit
ln -sf ../lib/node_modules/npm/bin/npm-cli.js npm
ln -sf ../lib/node_modules/npm/bin/npx-cli.js npx
Loading

0 comments on commit 7b8d373

Please sign in to comment.