From 8a9f679534492cbcb4e371905d8aeb9658fe138f Mon Sep 17 00:00:00 2001 From: mrumph Date: Thu, 27 Feb 2020 13:57:02 -0800 Subject: [PATCH 01/11] [Bug 5021] Add a script to fix spelling errors with codespell Run "scripts/spell-check.sh" from the Squid tree root directory. For details, see bug report 5021: - https://bugs.squid-cache.org/show_bug.cgi?id=5021 Changes to be committed: new file: scripts/codespell-whitelist.txt new file: scripts/spell-check.sh (cherry picked from commit 0678534211b10341437559ea70ae2728c83fa030) --- scripts/codespell-whitelist.txt | 57 +++++++++++++++++++++++++ scripts/spell-check.sh | 76 +++++++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+) create mode 100644 scripts/codespell-whitelist.txt create mode 100755 scripts/spell-check.sh diff --git a/scripts/codespell-whitelist.txt b/scripts/codespell-whitelist.txt new file mode 100644 index 00000000000..0c2005845e5 --- /dev/null +++ b/scripts/codespell-whitelist.txt @@ -0,0 +1,57 @@ +actuall +agains +aline +alloced +anid +ans +aparent +backword +backwords +cachable +cas +childs +commend +crypted +dont +fo +followings +formater +hight +hist +iff +inactivate +initate +nd +neeed +nnumber +normall +othere +pasttime +performes +pevents +pointes +preceed +querys +readed +referer +retuned +sence +sheme +tage +te +tha +ther +therefor +thru +thur +tim +tthe +ue +uint +upto +vaid +valuse +whan +whe +wil +wnat diff --git a/scripts/spell-check.sh b/scripts/spell-check.sh new file mode 100755 index 00000000000..f4d20df905e --- /dev/null +++ b/scripts/spell-check.sh @@ -0,0 +1,76 @@ +#!/bin/sh +# +## Copyright (C) 2020 The Squid Software Foundation and contributors +## +## Squid software is distributed under GPLv2+ license and includes +## contributions from numerous individuals and organizations. +## Please see the COPYING and CONTRIBUTORS files for details. +## + +# +# This script runs codespell against selected files. +# +# Use -q or --quiet option to run quietly +# + +CODESPELL_LOC=`which codespell` +if test "${CODESPELL_LOC}" = ""; then + echo "This script requires codespell which was not found." + exit 1 +fi + +CODESPELL_VER=`codespell --version 2>&1` +echo "The codespell version is ${CODESPELL_VER}." + +UNSTAGED_CHANGES=`git diff | wc -l` +if test "${UNSTAGED_CHANGES}" != "0"; then + echo "There are unstaged changes. Stage these first to prevent conflict." + exit 1 +fi + +WHITE_LIST=scripts/codespell-whitelist.txt +if test ! -f "${WHITE_LIST}"; then + echo "${WHITE_LIST} does not exist" + exit 1 +fi + +QUIET=0 +while test "$1" != ""; do + case $1 in + -q | --quiet ) + QUIET=1 + ;; + esac + shift +done + +# +# Scan for file-specific actions +# + +for FILENAME in `git ls-files`; do + # skip subdirectories, git ls-files is recursive + test -d $FILENAME && continue + + case ${FILENAME} in + + doc/*.txt|doc/*/*.txt) + ;; + + *.h|*.c|*.cc|*.cci|*.pl|*.sh|*.pre|*.pl.in|*.pm|*.dox|*.html|*.txt|*.sql|errors/templates/ERR_*|INSTALL|README|QUICKSTART) + # + # Run codespell against specific file + # + if test "${QUIET}" = "0"; then + echo "Running codespell for ${FILENAME}" + fi + codespell -d -q 3 -w -I ${WHITE_LIST} ${FILENAME} + if test "$?" != "0"; then + echo "codespell failed for ${FILENAME}" + exit 1 + fi + ;; + esac +done + +exit 0 From bc08d0fc0656b8189aabfbdba13e5a7e82f2171d Mon Sep 17 00:00:00 2001 From: mrumph Date: Thu, 5 Mar 2020 12:53:47 -0800 Subject: [PATCH 02/11] Bug 5021: Improve the spell check script Addressed comments from the pull request #565. Changes to be committed: modified: scripts/spell-check.sh --- scripts/spell-check.sh | 43 ++++++++++++++---------------------------- 1 file changed, 14 insertions(+), 29 deletions(-) diff --git a/scripts/spell-check.sh b/scripts/spell-check.sh index f4d20df905e..9996ab83015 100755 --- a/scripts/spell-check.sh +++ b/scripts/spell-check.sh @@ -10,20 +10,16 @@ # # This script runs codespell against selected files. # -# Use -q or --quiet option to run quietly -# -CODESPELL_LOC=`which codespell` -if test "${CODESPELL_LOC}" = ""; then +set -e + +echo -n "Codespell version: " +if ! codespell --version; then echo "This script requires codespell which was not found." exit 1 fi -CODESPELL_VER=`codespell --version 2>&1` -echo "The codespell version is ${CODESPELL_VER}." - -UNSTAGED_CHANGES=`git diff | wc -l` -if test "${UNSTAGED_CHANGES}" != "0"; then +if ! git diff --quiet; then echo "There are unstaged changes. Stage these first to prevent conflict." exit 1 fi @@ -34,20 +30,6 @@ if test ! -f "${WHITE_LIST}"; then exit 1 fi -QUIET=0 -while test "$1" != ""; do - case $1 in - -q | --quiet ) - QUIET=1 - ;; - esac - shift -done - -# -# Scan for file-specific actions -# - for FILENAME in `git ls-files`; do # skip subdirectories, git ls-files is recursive test -d $FILENAME && continue @@ -57,15 +39,18 @@ for FILENAME in `git ls-files`; do doc/*.txt|doc/*/*.txt) ;; - *.h|*.c|*.cc|*.cci|*.pl|*.sh|*.pre|*.pl.in|*.pm|*.dox|*.html|*.txt|*.sql|errors/templates/ERR_*|INSTALL|README|QUICKSTART) + *.h|*.c|*.cc|*.cci|\ + *.pl|*.sh|\ + *.pre|*.pl.in|*.pm|\ + *.dox|*.html|*.txt|\ + *.sql|\ + errors/templates/ERR_*|\ + INSTALL|README|QUICKSTART) # # Run codespell against specific file # - if test "${QUIET}" = "0"; then - echo "Running codespell for ${FILENAME}" - fi - codespell -d -q 3 -w -I ${WHITE_LIST} ${FILENAME} - if test "$?" != "0"; then + echo "Running codespell for ${FILENAME}" + if ! codespell -d -q 3 -w -I "${WHITE_LIST}" ${FILENAME}; then echo "codespell failed for ${FILENAME}" exit 1 fi From acd49d30ca6c810d45711a2e8dc30793fa19d34b Mon Sep 17 00:00:00 2001 From: Alex Rousskov Date: Sun, 22 Mar 2020 22:26:47 -0400 Subject: [PATCH 03/11] fixup: Removed trailing whitespace (detected by Semaphore CI tests) --- scripts/spell-check.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/spell-check.sh b/scripts/spell-check.sh index 9996ab83015..5c9c330dbf7 100755 --- a/scripts/spell-check.sh +++ b/scripts/spell-check.sh @@ -22,7 +22,7 @@ fi if ! git diff --quiet; then echo "There are unstaged changes. Stage these first to prevent conflict." exit 1 -fi +fi WHITE_LIST=scripts/codespell-whitelist.txt if test ! -f "${WHITE_LIST}"; then @@ -37,7 +37,7 @@ for FILENAME in `git ls-files`; do case ${FILENAME} in doc/*.txt|doc/*/*.txt) - ;; + ;; *.h|*.c|*.cc|*.cci|\ *.pl|*.sh|\ From 0cf053fbe90e667f281b1ca1f084768ee6e4aa21 Mon Sep 17 00:00:00 2001 From: Alex Rousskov Date: Sun, 22 Mar 2020 22:28:58 -0400 Subject: [PATCH 04/11] fixup: DRY --- scripts/spell-check.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/scripts/spell-check.sh b/scripts/spell-check.sh index 5c9c330dbf7..d8130241881 100755 --- a/scripts/spell-check.sh +++ b/scripts/spell-check.sh @@ -46,9 +46,6 @@ for FILENAME in `git ls-files`; do *.sql|\ errors/templates/ERR_*|\ INSTALL|README|QUICKSTART) - # - # Run codespell against specific file - # echo "Running codespell for ${FILENAME}" if ! codespell -d -q 3 -w -I "${WHITE_LIST}" ${FILENAME}; then echo "codespell failed for ${FILENAME}" From 5523abb8bfd10a24e3433bcefa71f3799a6aa939 Mon Sep 17 00:00:00 2001 From: Alex Rousskov Date: Sun, 22 Mar 2020 22:31:46 -0400 Subject: [PATCH 05/11] fixup: Do not be verbose by default. The caller can use "sh -x" for that --- scripts/spell-check.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/spell-check.sh b/scripts/spell-check.sh index d8130241881..66cc1b255fa 100755 --- a/scripts/spell-check.sh +++ b/scripts/spell-check.sh @@ -46,7 +46,6 @@ for FILENAME in `git ls-files`; do *.sql|\ errors/templates/ERR_*|\ INSTALL|README|QUICKSTART) - echo "Running codespell for ${FILENAME}" if ! codespell -d -q 3 -w -I "${WHITE_LIST}" ${FILENAME}; then echo "codespell failed for ${FILENAME}" exit 1 From 9b3822b55dd3b92db392c022b758be12abfa56c9 Mon Sep 17 00:00:00 2001 From: Alex Rousskov Date: Sun, 22 Mar 2020 22:34:37 -0400 Subject: [PATCH 06/11] fixup: Group all Perl-related filename masks together --- scripts/spell-check.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/spell-check.sh b/scripts/spell-check.sh index 66cc1b255fa..e26db61e1b8 100755 --- a/scripts/spell-check.sh +++ b/scripts/spell-check.sh @@ -40,8 +40,9 @@ for FILENAME in `git ls-files`; do ;; *.h|*.c|*.cc|*.cci|\ - *.pl|*.sh|\ - *.pre|*.pl.in|*.pm|\ + *.sh|\ + *.pre|\ + *.pl|*.pl.in|*.pm|\ *.dox|*.html|*.txt|\ *.sql|\ errors/templates/ERR_*|\ From f44a8645444af89e64caea7e06662cf5fc71aa4a Mon Sep 17 00:00:00 2001 From: Alex Rousskov Date: Sun, 22 Mar 2020 22:49:22 -0400 Subject: [PATCH 07/11] Do process doc/*.txt files by default doc/HTTP-codes.txt has a spelling error. Still exclude doc/debug-sections.txt because it is generated. Still exclude doc/*/*.txt because they are imported/foreign. --- scripts/spell-check.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/spell-check.sh b/scripts/spell-check.sh index e26db61e1b8..1bf8f282503 100755 --- a/scripts/spell-check.sh +++ b/scripts/spell-check.sh @@ -36,9 +36,15 @@ for FILENAME in `git ls-files`; do case ${FILENAME} in - doc/*.txt|doc/*/*.txt) + # skip (some) generated files with otherwise-checked extensions + doc/debug-sections.txt) ;; + # skip imported/foreign files with otherwise-checked extensions + doc/*/*.txt) + ;; + + # check all these *.h|*.c|*.cc|*.cci|\ *.sh|\ *.pre|\ From 37649dc28985329f407b8f9ab503079cf13b8c66 Mon Sep 17 00:00:00 2001 From: Alex Rousskov Date: Mon, 23 Mar 2020 16:58:56 -0400 Subject: [PATCH 08/11] Whitelist tread to work around codespell inability to interpret \t I could not find any more cases where \x was misinterpreted (for any x). --- scripts/codespell-whitelist.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/codespell-whitelist.txt b/scripts/codespell-whitelist.txt index 0c2005845e5..50a498c1f90 100644 --- a/scripts/codespell-whitelist.txt +++ b/scripts/codespell-whitelist.txt @@ -45,6 +45,7 @@ therefor thru thur tim +tread tthe ue uint From 2c18b644c822b1cdbeca1ca682c907ee621945a9 Mon Sep 17 00:00:00 2001 From: Alex Rousskov Date: Mon, 23 Mar 2020 17:10:47 -0400 Subject: [PATCH 09/11] Allow spell-check.sh to be applied to a subset of files This is very useful when working on a small subset of files because spell checking the whole tree takes a while: git diff $base --name-only | xargs ./scripts/spell-check.sh --- scripts/spell-check.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/spell-check.sh b/scripts/spell-check.sh index 1bf8f282503..32730093e89 100755 --- a/scripts/spell-check.sh +++ b/scripts/spell-check.sh @@ -30,7 +30,7 @@ if test ! -f "${WHITE_LIST}"; then exit 1 fi -for FILENAME in `git ls-files`; do +for FILENAME in `git ls-files "$@"`; do # skip subdirectories, git ls-files is recursive test -d $FILENAME && continue From 94a20432ba903c32e6ff9213f7c3578864c22fb7 Mon Sep 17 00:00:00 2001 From: Alex Rousskov Date: Tue, 24 Mar 2020 10:02:43 -0400 Subject: [PATCH 10/11] fixup: Documented usage --- scripts/spell-check.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/scripts/spell-check.sh b/scripts/spell-check.sh index 32730093e89..b4ea05d1c08 100755 --- a/scripts/spell-check.sh +++ b/scripts/spell-check.sh @@ -8,7 +8,15 @@ ## # -# This script runs codespell against selected files. +# This script uses codespell to automatically fix a subset of common spelling +# mistakes in the current git-controlled workspace. +# +# Usage: ./scripts/spell-check.sh [target]... +# ... where "target" is a git-controlled file or directory name to be fixed. +# +# By default, a hand-picked subset of Squid repository sources is fixed. +# +# See ${WHITE_LIST} below for the list of allowed misspellings. # set -e From 21a54ee6e0f943ea8b3af23e5b6c5ba86ae07a43 Mon Sep 17 00:00:00 2001 From: Alex Rousskov Date: Tue, 24 Mar 2020 10:12:38 -0400 Subject: [PATCH 11/11] fixup: Clarified why folks need to stage workspace changes. The script cannot, technically, create "conflicts" (at least not in a git sense of that word). The script might wipe out or corrupt unstaged changes, and we want to protect users from that. A very useful side effect of our protection is that any script changes are very easy to _review_ using `git diff --word-diff`. Any developer ought to review codespell changes. The script itself is probably not the right place to document/discuss all that. --- scripts/spell-check.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/spell-check.sh b/scripts/spell-check.sh index b4ea05d1c08..337c1040a3e 100755 --- a/scripts/spell-check.sh +++ b/scripts/spell-check.sh @@ -28,7 +28,8 @@ if ! codespell --version; then fi if ! git diff --quiet; then - echo "There are unstaged changes. Stage these first to prevent conflict." + echo "There are unstaged changes. This script may modify sources." + echo "Stage changes to avoid permanent losses when things go bad." exit 1 fi