From aea6442a38b6efd7258da986bc602b0611865481 Mon Sep 17 00:00:00 2001 From: Kirill 'kkm' Katsnelson Date: Mon, 16 Nov 2020 07:33:59 -0800 Subject: [PATCH] [tools] Prefer tarballs from GitHub for reliability (#4336) * Software distributed on GitHub is downloaded from GitHub: - cub (official): rework rules to avoid excessive re-downloads, and change tarball format to .tar.gz to match other software. - SCTK (official) - sph2pipe (mirror at burrmill, stable) * Fix build of sph2pipe. Some tarballs of the same version 2.5 did not accept the patch from extras/, and make -j N for large N stopped on a prompt from patch(1) that has scrolled far up, creating an impression that make had hung. The currently used mirror does not require a patch, and cleanly builds with its own Makefile. Rework build rules to use the Makefile, and remove the patch. * Retire 'openblas' rule: it no longer invokes install_openblas.sh and directs the user to use the script instead. The rule was incorrect, because it ignored OPENBLAS_VERSION manifestly set in the Makefile * Remove a patch for SCTK: its only purpose had been to add missing #include's and declarations, which generated warnings in C code. Instead, suppress all compilation warnings during build. Fixes: #4311 Fixes: #4323 --- tools/Makefile | 206 ++++++++++++++++++++---------------- tools/extras/sctk.patch | 34 ------ tools/extras/sph2pipe.patch | 23 ---- 3 files changed, 116 insertions(+), 147 deletions(-) delete mode 100644 tools/extras/sctk.patch delete mode 100644 tools/extras/sph2pipe.patch diff --git a/tools/Makefile b/tools/Makefile index b30847a5f18..4c4c5622716 100644 --- a/tools/Makefile +++ b/tools/Makefile @@ -11,13 +11,15 @@ WGET ?= wget # e.g. g++ >= 4.7, Apple clang >= 5.0 or LLVM clang >= 3.3. OPENFST_VERSION ?= 1.6.7 CUB_VERSION ?= 1.8.0 -OPENBLAS_VERSION ?= 0.3.5 -SCTK_VERSION_PARTIAL ?= 2.4.10 -SCTK_VERSION ?= $(SCTK_VERSION_PARTIAL)-20151007-1312Z -SPH2PIPE_VERSION = v2.5 +# No '?=', since there exists only one version of sph2pipe. +SPH2PIPE_VERSION = 2.5 +# SCTK official repo does not have version tags. Here's the mapping: +# 2.4.9 = 659bc36; 2.4.10 = d914e1b; 2.4.11 = 20159b5. +SCTK_GITHASH = 20159b5 # Default features configured for OpenFST; can be overridden in the make command line. -OPENFST_CONFIGURE ?= --enable-static --enable-shared --enable-far --enable-ngram-fsts --enable-lookahead-fsts --with-pic +OPENFST_CONFIGURE ?= --enable-static --enable-shared --enable-far \ + --enable-ngram-fsts --enable-lookahead-fsts --with-pic OPENFST_VER_NUM := $(shell echo $(OPENFST_VERSION) | sed 's/\./ /g' | xargs printf "%d%02d%02d") ifeq ("$(shell expr $(OPENFST_VER_NUM) \< 10600)","1") @@ -25,52 +27,47 @@ ifeq ("$(shell expr $(OPENFST_VER_NUM) \< 10600)","1") Supported versions: >= 1.6.0) endif -all: check_required_programs sph2pipe sclite openfst cub +.PHONY: all clean distclean +all: check_required_programs cub openfst sctk_made sph2pipe @echo -e "\n\n" @echo "Warning: IRSTLM is not installed by default anymore. If you need IRSTLM" @echo "Warning: use the script extras/install_irstlm.sh" @echo "All done OK." # make sure check_required_programs runs before anything else: -sph2pipe sclite openfst sctk: | check_required_programs +all cub openfst sctk_made sph2pipe: | check_required_programs check_required_programs: extras/check_dependencies.sh -clean: openfst_cleaned sclite_cleaned +clean: openfst_cleaned sctk_cleaned -openfst_cleaned: - -$(MAKE) -C openfst-$(OPENFST_VERSION) clean - -sclite_cleaned: - -$(MAKE) -C sctk clean - -distclean: - rm -rf openfst-$(OPENFST_VERSION)/ - rm -rf sctk-$(SCTK_VERSION_PARTIAL)/ - rm -rf sctk - rm -rf sph2pipe_$(SPH2PIPE_VERSION)/ - rm -rf sph2pipe_$(SPH2PIPE_VERSION).tar.gz - rm -rf sctk-$(SCTK_VERSION).tar.bz2 - rm -rf openfst-$(OPENFST_VERSION).tar.gz - rm -f openfst - rm -rf libsndfile-1.0.25{,.tar.gz} BeamformIt-3.51{,.tgz} - rm -f cub-$(CUB_VERSION).zip - rm -rf cub-$(CUB_VERSION) - rm -f cub +distclean: clean + rm -rf openfst openfst-$(OPENFST_VERSION) \ + openfst-$(OPENFST_VERSION).tar.gz + rm -rf sctk sctk-$(SCTK_GITHASH) \ + sctk-$(SCTK_GITHASH).tar.gz + rm -rf sph2pipe sph2pipe-$(SPH2PIPE_VERSION) \ + sph2pipe-$(SPH2PIPE_VERSION).tar.gz + rm -rf libsndfile-1.0.25 libsndfile-1.0.25.tar.gz \ + BeamformIt-3.51 BeamformIt-3.51.tgz + rm -rf cub cub-$(CUB_VERSION) \ + cub-$(CUB_VERSION).zip cub-$(CUB_VERSION).tar.gz +#=== OpenFST =================================================================== + +.PHONY: openfst openfst_compiled openfst_cleaned -.PHONY: openfst # so target will be made even though "openfst" exists. openfst: openfst_compiled openfst-$(OPENFST_VERSION)/lib - -rm -f openfst - -ln -s openfst-$(OPENFST_VERSION) openfst + rm -f openfst + ln -s openfst-$(OPENFST_VERSION) openfst -.PHONY: openfst_compiled openfst_compiled: openfst-$(OPENFST_VERSION)/Makefile $(MAKE) -C openfst-$(OPENFST_VERSION) install MAKEOVERRIDES= openfst-$(OPENFST_VERSION)/lib: | openfst-$(OPENFST_VERSION)/Makefile - -cd openfst-$(OPENFST_VERSION) && [ -d lib64 ] && [ ! -d lib ] && ln -s lib64 lib + - cd openfst-$(OPENFST_VERSION) && \ + [ -d lib64 ] && [ ! -d lib ] && ln -s lib64 lib # Add the -O flag to CXXFLAGS on cygwin as it can fix the compilation error # "file too big". @@ -84,91 +81,120 @@ else openfst_add_CXXFLAGS = -g -O2 endif -openfst-$(OPENFST_VERSION)/Makefile: openfst-$(OPENFST_VERSION) | check_required_programs +openfst-$(OPENFST_VERSION)/Makefile: openfst-$(OPENFST_VERSION) cd openfst-$(OPENFST_VERSION)/ && \ - ./configure --prefix=`pwd` $(OPENFST_CONFIGURE) CXX="$(CXX)" CXXFLAGS="$(CXXFLAGS) $(openfst_add_CXXFLAGS)" LDFLAGS="$(LDFLAGS)" LIBS="-ldl" - + ./configure --prefix=`pwd` $(OPENFST_CONFIGURE) CXX="$(CXX)" \ + CXXFLAGS="$(CXXFLAGS) $(openfst_add_CXXFLAGS)" \ + LDFLAGS="$(LDFLAGS)" LIBS="-ldl" openfst-$(OPENFST_VERSION): openfst-$(OPENFST_VERSION).tar.gz tar xozf openfst-$(OPENFST_VERSION).tar.gz openfst-$(OPENFST_VERSION).tar.gz: if [ -d "$(DOWNLOAD_DIR)" ]; then \ - cp -p "$(DOWNLOAD_DIR)/openfst-$(OPENFST_VERSION).tar.gz" .; \ + cp -p "$(DOWNLOAD_DIR)/openfst-$(OPENFST_VERSION).tar.gz" .; \ else \ - $(WGET) -T 10 -t 1 http://www.openfst.org/twiki/pub/FST/FstDownload/openfst-$(OPENFST_VERSION).tar.gz || \ - $(WGET) -T 10 -t 3 -c https://www.openslr.org/resources/2/openfst-$(OPENFST_VERSION).tar.gz; \ + $(WGET) -nv -T 10 -t 1 http://www.openfst.org/twiki/pub/FST/FstDownload/openfst-$(OPENFST_VERSION).tar.gz || \ + $(WGET) -nv -T 10 -t 3 -c https://www.openslr.org/resources/2/openfst-$(OPENFST_VERSION).tar.gz; \ fi -sclite: sclite_compiled - -.PHONY: sclite_compiled -sclite_compiled: sctk sctk_configured - cd sctk; \ - $(MAKE) CC="$(CC)" CXX="$(CXX)" all && $(MAKE) install && $(MAKE) doc - -sctk_configured: sctk sctk/.configured sctk/.patched - -sctk/.patched: sctk - cd sctk; patch -p1 < ../extras/sctk.patch - touch sctk/.patched - -sctk/.configured: sctk - cd sctk; $(MAKE) config - touch sctk/.configured - -.PHONY: sctk -sctk: sctk-$(SCTK_VERSION).tar.bz2 - tar xojf sctk-$(SCTK_VERSION).tar.bz2 || \ - tar --exclude '*NONE*html' -xvojf sctk-$(SCTK_VERSION).tar.bz2 - rm -rf sctk && ln -s sctk-$(SCTK_VERSION_PARTIAL) sctk - -sctk-$(SCTK_VERSION).tar.bz2: - if [ -d "$(DOWNLOAD_DIR)" ]; then \ - cp -p "$(DOWNLOAD_DIR)/sctk-$(SCTK_VERSION).tar.bz2" .; \ +openfst_cleaned: + -for d in openfst/ openfst-*/; do \ + [ ! -f $$d/Makefile ] || $(MAKE) -C $$d clean; \ + done + +#== SCTK ======================================================================= + +SCTK_CXFLAGS = -w -march=native +SCTK_MKENV = CFLAGS="$(CFLAGS) $(SCTK_CXFLAGS)" \ + CXXFLAGS="$(CXXFLAGS) $(SCTK_CXFLAGS)" \ + +# Keep the existing target 'sclite' to avoid breaking the users who might have +# scripted it in. +.PHONY: sclite sctk_cleaned sctk_made + +sclite sctk_made: sctk/.compiled + +sctk/.compiled: sctk + rm -f sctk/.compiled + $(SCTK_MKENV) $(MAKE) -C sctk config + $(SCTK_MKENV) $(MAKE) -C sctk all doc + $(MAKE) -C sctk install + touch sctk/.compiled + +# The GitHub archive unpacks into SCTK-{40-character-long-hash}/ +sctk: sctk-$(SCTK_GITHASH).tar.gz + tar --exclude '*NONE*html' -xmaf sctk-$(SCTK_GITHASH).tar.gz + rm -rf sctk-$(SCTK_GITHASH) sctk + mv SCTK-$(SCTK_GITHASH)* sctk-$(SCTK_GITHASH) + ln -s sctk-$(SCTK_GITHASH) sctk + touch sctk-$(SCTK_GITHASH).tar.gz + +sctk-$(SCTK_GITHASH).tar.gz: + if [ -d '$(DOWNLOAD_DIR)' ]; then \ + cp -p '$(DOWNLOAD_DIR)/sctk-$(SCTK_GITHASH).tar.gz' .; \ else \ - $(WGET) -T 10 https://www.openslr.org/resources/4/sctk-$(SCTK_VERSION).tar.bz2; \ + $(WGET) -nv -T 10 -t 3 -O sctk-$(SCTK_GITHASH).tar.gz \ + https://github.com/usnistgov/SCTK/archive/$(SCTK_GITHASH).tar.gz; \ fi -sph2pipe: sph2pipe_compiled +sctk_cleaned: + -for d in sctk/ sctk-*/; do \ + [ ! -f $$d/.compiled ] || $(MAKE) -C $$d clean; \ + rm -f $$d/.compiled; \ + done -sph2pipe_compiled: sph2pipe_patched sph2pipe_$(SPH2PIPE_VERSION)/sph2pipe +#== sph2pipe =================================================================== -sph2pipe_patched: sph2pipe_$(SPH2PIPE_VERSION)/.patched +# Note the naming subtlety: the tarball extracts into a subdirectory +# 'sph2pipe-2.5', but over 9000 scripts call for 'sph2pipe_v2.5'. -sph2pipe_$(SPH2PIPE_VERSION)/sph2pipe: sph2pipe_$(SPH2PIPE_VERSION)/.patched | sph2pipe_$(SPH2PIPE_VERSION) - cd sph2pipe_$(SPH2PIPE_VERSION)/ && \ - $(CC) -o sph2pipe *.c -lm +sph2pipe: sph2pipe_v$(SPH2PIPE_VERSION)/sph2pipe + rm -f sph2pipe + ln -s sph2pipe_v$(SPH2PIPE_VERSION) sph2pipe + touch -r $@ -c $^ -sph2pipe_$(SPH2PIPE_VERSION)/.patched: sph2pipe_$(SPH2PIPE_VERSION) - cd sph2pipe_$(SPH2PIPE_VERSION); patch -p1 < ../extras/sph2pipe.patch - touch sph2pipe_$(SPH2PIPE_VERSION)/.patched +sph2pipe_v$(SPH2PIPE_VERSION)/sph2pipe: sph2pipe_v$(SPH2PIPE_VERSION)/Makefile + $(MAKE) -C sph2pipe_v$(SPH2PIPE_VERSION) -sph2pipe_$(SPH2PIPE_VERSION): sph2pipe_$(SPH2PIPE_VERSION).tar.gz - tar --no-same-owner -xzf sph2pipe_$(SPH2PIPE_VERSION).tar.gz +sph2pipe_v$(SPH2PIPE_VERSION)/Makefile: sph2pipe-$(SPH2PIPE_VERSION).tar.gz + rm -rf sph2pipe_v* + tar -xmzf sph2pipe-$(SPH2PIPE_VERSION).tar.gz + mv sph2pipe-$(SPH2PIPE_VERSION) sph2pipe_v$(SPH2PIPE_VERSION) -sph2pipe_$(SPH2PIPE_VERSION).tar.gz: +sph2pipe-$(SPH2PIPE_VERSION).tar.gz: if [ -d "$(DOWNLOAD_DIR)" ]; then \ - cp -p "$(DOWNLOAD_DIR)/sph2pipe_$(SPH2PIPE_VERSION).tar.gz" .; \ + cp -p "$(DOWNLOAD_DIR)/sph2pipe-v$(SPH2PIPE_VERSION).tar.gz" \ + sph2pipe-$(SPH2PIPE_VERSION).tar.gz; \ else \ - $(WGET) -T 10 -t 3 https://www.openslr.org/resources/3/sph2pipe_$(SPH2PIPE_VERSION).tar.gz || \ - $(WGET) -T 10 -c https://sourceforge.net/projects/kaldi/files/sph2pipe_$(SPH2PIPE_VERSION).tar.gz; \ + $(WGET) -nv -T 10 -t 3 -O sph2pipe-$(SPH2PIPE_VERSION).tar.gz \ + https://github.com/burrmill/sph2pipe/archive/$(SPH2PIPE_VERSION).tar.gz; \ fi +#== CUB ======================================================================== + .PHONY: cub -cub: +cub: cub/common.mk + +cub/common.mk: cub-$(CUB_VERSION).tar.gz + rm -f cub + tar -xmzf cub-$(CUB_VERSION).tar.gz + ln -s cub-$(CUB_VERSION) cub + +cub-$(CUB_VERSION).tar.gz: if [ -d "$(DOWNLOAD_DIR)" ]; then \ - cp -p "$(DOWNLOAD_DIR)/cub-$(CUB_VERSION).zip" .; \ + cp -p "$(DOWNLOAD_DIR)/cub-$(CUB_VERSION).tar.gz" .; \ else \ - $(WGET) -T 10 -t 3 -O cub-$(CUB_VERSION).zip https://github.com/NVlabs/cub/archive/$(CUB_VERSION).zip; \ + $(WGET) -nv -T 10 -t 3 -O cub-$(CUB_VERSION).tar.gz \ + https://github.com/NVlabs/cub/archive/$(CUB_VERSION).tar.gz; \ fi - unzip -oq cub-$(CUB_VERSION).zip - rm -f cub - ln -s cub-$(CUB_VERSION) cub -# OpenBLAS is not compiled by default; you can run `make openblas` to build it, -# but you should probably just call extras/install_openblas.sh; this is -# here for legacy reasons. +#== No OpenBLAS ================================================================ + .PHONY: openblas openblas: - extras/install_openblas.sh + @echo "##===============================================##" + @echo "## WARNING: The 'openblas' target is obsolete. ##" + @echo "## Run 'extras/install_openblas.sh' instead. ##" + @echo "##===============================================##" + @false diff --git a/tools/extras/sctk.patch b/tools/extras/sctk.patch deleted file mode 100644 index 88f9788e6ec..00000000000 --- a/tools/extras/sctk.patch +++ /dev/null @@ -1,34 +0,0 @@ -diff --git a/src/sclite/align.c b/src/sclite/align.c -index b80dcd3..52bcbf0 100644 ---- a/src/sclite/align.c -+++ b/src/sclite/align.c -@@ -1,3 +1,4 @@ -+#include - #include "sctk.h" - - -diff --git a/src/sclite/sclite.c b/src/sclite/sclite.c -index 93b77ed..58c894a 100644 ---- a/src/sclite/sclite.c -+++ b/src/sclite/sclite.c -@@ -6,6 +6,7 @@ - - void do_exit(char *desc, char *prog, int ret); - void proc_args(int argc, char **argv, char *prog, char **rname, char **rfmt, char **hname, char **hfmt, int *nhyps, enum id_types *id_type, int *case_sens, int *outputs, char **title, int *feedback, int *linewidth, int *use_diff, char **out_dir, char **out_name,int *char_align, int *pipeout, int *pipein, int *infered_wordseg, char **lexicon, int *frag_correct, int *opt_del, int *inf_flags, int *stm2ctm_reduce, int *time_align, int *conf_outputs, int *left_to_right, char **wwl_file, char **lm_file); -+int TEXT_set_lang_prof(char *lprof); - - #define OUT_SUM 0x0001 - #define OUT_RSUM 0x0002 -diff --git a/src/sclite/sctkUnit.c b/src/sclite/sctkUnit.c -index 0b44fc1..4660391 100644 ---- a/src/sclite/sctkUnit.c -+++ b/src/sclite/sctkUnit.c -@@ -4,6 +4,8 @@ - static int v = 1; - static int exitOnError = 1; - -+int TEXT_set_lang_prof(char *lprof); -+ - int test_separate_chars(TEXT *from, TEXT *exp, int flag){ - TEXT *buff; - int size = 10; diff --git a/tools/extras/sph2pipe.patch b/tools/extras/sph2pipe.patch deleted file mode 100644 index 98f322c601d..00000000000 --- a/tools/extras/sph2pipe.patch +++ /dev/null @@ -1,23 +0,0 @@ -diff --git a/sph2pipe.c b/sph2pipe.c -index d0d502b..928739c 100644 ---- a/sph2pipe.c -+++ b/sph2pipe.c -@@ -103,6 +103,7 @@ - - #define _SPH_CONVERT_MAIN_ - -+#include - #include "sph_convert.h" - #include "ulaw.h" - -diff --git a/ulaw.h b/ulaw.h -index 990c833..62ff96e 100644 ---- a/ulaw.h -+++ b/ulaw.h -@@ -82,3 +82,6 @@ GLOBAL short int alaw2pcm[256] - } - #endif - ; -+ -+uchar pcm2ulaw( short int sample ); -+uchar pcm2alaw( short int pcmval );