WIP: Update readtags to support escape sequences and fix & improve its testing

Makefile.am

@@ -17,6 +17,7 @@ bin_PROGRAMS = ctags
 if USE_READCMD
 bin_PROGRAMS+= readtags
 readtags_CPPFLAGS = -I. -I$(srcdir) -I$(srcdir)/main -I$(srcdir)/read
+readtags_CFLAGS = $(COVERAGE_CFLAGS)
 dist_readtags_SOURCES = $(READTAGS_SRCS) $(READTAGS_HEADS)
 readtags_CPPFLAGS += -DQUALIFIER -I$(srcdir)/dsl
 dist_readtags_SOURCES += $(QUALIFIER_SRCS) $(QUALIFIER_HEADS)

makefiles/testing.mak

@@ -1,7 +1,7 @@
 # -*- makefile -*-
 .PHONY: check units fuzz noise tmain tinst clean-units clean-tmain clean-gcov run-gcov codecheck cppcheck dicts cspell
 
-check: tmain units
+check: tmain units roundtrip
 
 clean-local: clean-units clean-tmain
 
@@ -18,13 +18,21 @@ LANGUAGES=
 CATEGORIES=
 UNITS=
 
+SILENT = $(SILENT_@AM_V@)
+SILENT_ = $(SILENT_@AM_DEFAULT_V@)
+SILENT_0 = @
+
+V_RUN = $(V_RUN_@AM_V@)
+V_RUN_ = $(V_RUN_@AM_DEFAULT_V@)
+V_RUN_0 = @echo "  RUN      $@";
+
 #
 # FUZZ Target
 #
 # SHELL must be dash or bash.
 #
 fuzz: $(CTAGS_TEST)
-	@ \
+	$(V_RUN) \
 	if test -n "$${ZSH_VERSION+set}"; then set -o SH_WORD_SPLIT; fi; \
 	if test x$(VG) = x1; then		\
 		VALGRIND=--with-valgrind;	\
@@ -40,7 +48,7 @@ fuzz: $(CTAGS_TEST)
 # NOISE Target
 #
 noise: $(CTAGS_TEST)
-	@ \
+	$(V_RUN) \
 	if test -n "$${ZSH_VERSION+set}"; then set -o SH_WORD_SPLIT; fi; \
 	if test x$(VG) = x1; then		\
 		VALGRIND=--with-valgrind;	\
@@ -56,7 +64,7 @@ noise: $(CTAGS_TEST)
 # CHOP Target
 #
 chop: $(CTAGS_TEST)
-	@ \
+	$(V_RUN) \
 	if test -n "$${ZSH_VERSION+set}"; then set -o SH_WORD_SPLIT; fi; \
 	if test x$(VG) = x1; then		\
 		VALGRIND=--with-valgrind;	\
@@ -68,7 +76,7 @@ chop: $(CTAGS_TEST)
 		--with-timeout=$(TIMEOUT)"; \
 	$(SHELL) $${c} $(srcdir)/Units
 slap: $(CTAGS_TEST)
-	@ \
+	$(V_RUN) \
 	if test -n "$${ZSH_VERSION+set}"; then set -o SH_WORD_SPLIT; fi; \
 	if test x$(VG) = x1; then		\
 		VALGRIND=--with-valgrind;	\
@@ -84,7 +92,7 @@ slap: $(CTAGS_TEST)
 # UNITS Target
 #
 units: $(CTAGS_TEST)
-	@ \
+	$(V_RUN) \
 	if test -n "$${ZSH_VERSION+set}"; then set -o SH_WORD_SPLIT; fi; \
 	if test x$(VG) = x1; then		\
 		VALGRIND=--with-valgrind;	\
@@ -115,8 +123,7 @@ clean-units:
 # Test main part, not parsers
 #
 tmain: $(CTAGS_TEST)
-	@ \
-	\
+	$(V_RUN) \
 	if test -n "$${ZSH_VERSION+set}"; then set -o SH_WORD_SPLIT; fi; \
 	if test x$(VG) = x1; then		\
 		VALGRIND=--with-valgrind;	\
@@ -144,22 +151,24 @@ clean-tmain:
 # Test installation
 #
 tinst:
-	@ \
-	\
+	$(V_RUN) \
 	builddir=$$(pwd); \
 	rm -rf $$builddir/$(TINST_ROOT); \
-	\
 	$(SHELL) $(srcdir)/misc/tinst $(srcdir) $$builddir/$(TINST_ROOT)
 
 #
 # Test readtags
 #
+if USE_READCMD
 roundtrip: $(READ_TEST)
-	@ \
-	\
+	$(V_RUN) \
 	builddir=$$(pwd); \
-	\
 	$(SHELL) $(srcdir)/misc/roundtrip $(READ_TEST) $${builddir}/Units
+else
+# apparently Automake doesn't like adding an extra dependency in a conditional,
+# so add it unconditionally and simply have it not do anything if it's disabled.
+roundtrip:
+endif
 
 #
 # Checking code in ctags own rules

misc/roundtrip

@@ -15,15 +15,13 @@
 #   You should have received a copy of the GNU General Public License
 #   along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+# Avoid trouble with weird bytes on non-C locales
+export LC_ALL=C
+
 READTAGS=${1:-./readtags}
 UNITS=${2:-./Units}
 s=0
 
-sink()
-{
-    cat > /dev/null
-}
-
 if ! [ -f "${READTAGS}" ]; then
     echo "No such file: ${READTAGS}" 1>&2
     exit 1
@@ -40,16 +38,91 @@ if ! [ -x "${UNITS}" ]; then
 fi
 
 
+# Expand CTags escape sequences (and keep the original, too)
+# See docs/format.rst
+expandEscapeSequences()
+{
+    sed -e '
+# early out if there is no escape sequence (common case)
+/\\/!q
+
+# loop until we handled all occurrences of \.
+# as it is not possible to do all replacements at once and we need not to
+# ever use the result of an replacement to perform another one, use two
+# reserved placeholders.
+:again
+    s/\\/__BACKSLASH__/
+    T out
+    s/__BACKSLASH__\\/__LITBACKSLASH__/;	t again
+    s/__BACKSLASH__t/	/;			t again
+    s/__BACKSLASH__r/'"$(printf '\r')"'/;	t again
+    s/__BACKSLASH__n/\n/;			t again
+    s/__BACKSLASH__a//;			t again
+    s/__BACKSLASH__b/'"$(printf '\b')"'/;	t again
+    s/__BACKSLASH__v//;			t again
+    s/__BACKSLASH__f//;			t again
+    s/__BACKSLASH__x01/
/;			t again
+    s/__BACKSLASH__x02//;			t again
+    s/__BACKSLASH__x03//;			t again
+    s/__BACKSLASH__x04//;			t again
+    s/__BACKSLASH__x05//;			t again
+    s/__BACKSLASH__x06//;			t again
+    s/__BACKSLASH__x07//;			t again
+    s/__BACKSLASH__x08//;			t again
+    s/__BACKSLASH__x09/	/;			t again
+    s/__BACKSLASH__x0[aA]/\n/;			t again
+    s/__BACKSLASH__x0[bB]//;			t again
+    s/__BACKSLASH__x0[cC]//;			t again
+    s/__BACKSLASH__x0[dD]/'"$(printf '\r')"'/;	t again
+    s/__BACKSLASH__x0[eE]//;			t again
+    s/__BACKSLASH__x0[fF]//;			t again
+    s/__BACKSLASH__x10//;			t again
+    s/__BACKSLASH__x11//;			t again
+    s/__BACKSLASH__x12//;			t again
+    s/__BACKSLASH__x13//;			t again
+    s/__BACKSLASH__x14//;			t again
+    s/__BACKSLASH__x15//;			t again
+    s/__BACKSLASH__x16//;			t again
+    s/__BACKSLASH__x17//;			t again
+    s/__BACKSLASH__x18//;			t again
+    s/__BACKSLASH__x19//;			t again
+    s/__BACKSLASH__x1[aA]//;			t again
+    s/__BACKSLASH__x1[bB]//;			t again
+    s/__BACKSLASH__x1[cC]//;			t again
+    s/__BACKSLASH__x1[dD]//;			t again
+    s/__BACKSLASH__x1[eE]//;			t again
+    s/__BACKSLASH__x1[fF]//;			t again
+    s/__BACKSLASH__x21/!/;			t again
+    s/__BACKSLASH__x20/ /;			t again
+    s/__BACKSLASH__x7[fF]//;			t again
+
+:out
+    # replace lonely "\"es
+    s/__BACKSLASH__/\\/g
+    # replace literal "\"es ("\\" from the input)
+    s/__LITBACKSLASH__/\\/g
+'
+}
+
+# disable path expansion, because we don't need it and it's applied on the
+# result of the commands used to loop on, and we don't want that.
+set -f
+# remove space from IFS as it's valid in tag names
+OLD_IFS="$IFS"
+IFS='
+'
+set -x
 for tags in $(find "$UNITS" -name expected.tags); do
-    for t in $(sed -e 's/^\([^\t]*\)\t.*/\1/' "$tags"); do
-	if r=$( "${READTAGS}" -t "$tags" - "$t" 2>/dev/null ); then
-	    :
-	else
-	    printf "FAILED[%3d]: " $?
-	    echo "${READTAGS}" -t "$tags" - "$t"
+    for name in $(sed -e 's/^\([^	]*\)	.*/\1/' "$tags"); do
+	# don't expand in the loop to handle "\n" sequence properly
+	t="$(printf '%s' "$name" | expandEscapeSequences)"
+	if [ 1 -gt $("${READTAGS}" -t "$tags" - "$t" 2>/dev/null | wc -l) ]; then
+	    printf 'FAILED: "%s" -t "%s" - "%s"\n' "${READTAGS}" "$tags" "$t"
 	    s=1
 	fi
-    done 
+    done
 done
 
+IFS="$OLD_IFS"
+
 exit $s