diff --git a/.circleci/config.yml b/.circleci/config.yml index 035177edc7..65bb3c2275 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,6 +7,8 @@ jobs: # preinstalled to reduce installation time. docker: - image: fbopensource/zstd-circleci-primary:0.0.1 + # TODO: Re-enable aarch64 build: + # make aarch64build && make clean steps: - checkout - run: @@ -14,12 +16,11 @@ jobs: command: | ./tests/test-license.py cc -v; CFLAGS="-O0 -Werror -pedantic" make all && make clean - make c99build ; make clean - make c11build ; make clean - make aarch64build ; make clean - make -j regressiontest; make clean - make shortest ; make clean - make cxxtest ; make clean + make c99build && make clean + make c11build && make clean + make -j regressiontest&& make clean + make shortest && make clean + make cxxtest && make clean # the second half of the jobs are in this test short-tests-1: docker: diff --git a/.github/workflows/dev-short-tests.yml b/.github/workflows/dev-short-tests.yml index cdc9d7fe3b..5d7e54b51a 100644 --- a/.github/workflows/dev-short-tests.yml +++ b/.github/workflows/dev-short-tests.yml @@ -200,6 +200,15 @@ jobs: make clean && make -j all MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS" make clean && make check MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS" + test-variants: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: make all variants & validate + run: | + make -j -C programs allVariants MOREFLAGS=-O0 + ./tests/test-variants.sh + qemu-consistency: name: QEMU ${{ matrix.name }} @@ -263,7 +272,6 @@ jobs: run: | LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make clean check - # This test currently fails on Github Actions specifically. # Possible reason : TTY emulation. # Note that the same test works fine locally and on travisCI. diff --git a/Makefile b/Makefile index b9265e7f85..ff49fc0d8b 100644 --- a/Makefile +++ b/Makefile @@ -217,7 +217,7 @@ armbuild: clean CC=arm-linux-gnueabi-gcc CFLAGS="-Werror" $(MAKE) allzstd aarch64build: clean - CC=aarch64-linux-gnu-gcc CFLAGS="-Werror" $(MAKE) allzstd + CC=aarch64-linux-gnu-gcc CFLAGS="-Werror -O0" $(MAKE) allzstd ppcbuild: clean CC=powerpc-linux-gnu-gcc CFLAGS="-m32 -Wno-attributes -Werror" $(MAKE) -j allzstd @@ -381,23 +381,23 @@ cmakebuild: c89build: clean $(CC) -v - CFLAGS="-std=c89 -Werror" $(MAKE) allmost # will fail, due to missing support for `long long` + CFLAGS="-std=c89 -Werror -O0" $(MAKE) allmost # will fail, due to missing support for `long long` gnu90build: clean $(CC) -v - CFLAGS="-std=gnu90 -Werror" $(MAKE) allmost + CFLAGS="-std=gnu90 -Werror -O0" $(MAKE) allmost c99build: clean $(CC) -v - CFLAGS="-std=c99 -Werror" $(MAKE) allmost + CFLAGS="-std=c99 -Werror -O0" $(MAKE) allmost gnu99build: clean $(CC) -v - CFLAGS="-std=gnu99 -Werror" $(MAKE) allmost + CFLAGS="-std=gnu99 -Werror -O0" $(MAKE) allmost c11build: clean $(CC) -v - CFLAGS="-std=c11 -Werror" $(MAKE) allmost + CFLAGS="-std=c11 -Werror -O0" $(MAKE) allmost bmix64build: clean $(CC) -v diff --git a/build/cmake/lib/CMakeLists.txt b/build/cmake/lib/CMakeLists.txt index 5f756652e7..d91ebe1f06 100644 --- a/build/cmake/lib/CMakeLists.txt +++ b/build/cmake/lib/CMakeLists.txt @@ -7,7 +7,7 @@ # in the COPYING file in the root directory of this source tree). # ################################################################ -project(libzstd C) +project(libzstd C ASM) set(CMAKE_INCLUDE_CURRENT_DIR TRUE) option(ZSTD_BUILD_STATIC "BUILD STATIC LIBRARIES" ON) @@ -22,7 +22,7 @@ include_directories(${LIBRARY_DIR} ${LIBRARY_DIR}/common) file(GLOB CommonSources ${LIBRARY_DIR}/common/*.c) file(GLOB CompressSources ${LIBRARY_DIR}/compress/*.c) -file(GLOB DecompressSources ${LIBRARY_DIR}/decompress/*.c) +file(GLOB DecompressSources ${LIBRARY_DIR}/decompress/*.c ${LIBRARY_DIR}/decompress/*.S) file(GLOB DictBuilderSources ${LIBRARY_DIR}/dictBuilder/*.c) set(Sources diff --git a/contrib/freestanding_lib/freestanding.py b/contrib/freestanding_lib/freestanding.py index a191f592fa..cd9d637749 100755 --- a/contrib/freestanding_lib/freestanding.py +++ b/contrib/freestanding_lib/freestanding.py @@ -460,7 +460,8 @@ def _log(self, *args, **kwargs): print(*args, **kwargs) def _copy_file(self, lib_path): - if not (lib_path.endswith(".c") or lib_path.endswith(".h")): + suffixes = [".c", ".h", ".S"] + if not any((lib_path.endswith(suffix) for suffix in suffixes)): return if lib_path in SKIPPED_FILES: self._log(f"\tSkipping file: {lib_path}") diff --git a/contrib/linux-kernel/test/Makefile b/contrib/linux-kernel/test/Makefile index 2908839fd5..dc76a5f401 100644 --- a/contrib/linux-kernel/test/Makefile +++ b/contrib/linux-kernel/test/Makefile @@ -18,9 +18,13 @@ CPPFLAGS += -DZSTD_ASAN_DONT_POISON_WORKSPACE LINUX_ZSTD_MODULE := $(wildcard $(LINUX_ZSTDLIB)/*.c) LINUX_ZSTD_COMMON := $(wildcard $(LINUX_ZSTDLIB)/common/*.c) LINUX_ZSTD_COMPRESS := $(wildcard $(LINUX_ZSTDLIB)/compress/*.c) -LINUX_ZSTD_DECOMPRESS := $(wildcard $(LINUX_ZSTDLIB)/decompress/*.c) +LINUX_ZSTD_DECOMPRESS := $(wildcard $(LINUX_ZSTDLIB)/decompress/*.c $(LINUX_ZSTDLIB)/decompress/*.S) LINUX_ZSTD_FILES := $(LINUX_ZSTD_MODULE) $(LINUX_ZSTD_COMMON) $(LINUX_ZSTD_COMPRESS) $(LINUX_ZSTD_DECOMPRESS) -LINUX_ZSTD_OBJECTS := $(LINUX_ZSTD_FILES:.c=.o) +LINUX_ZSTD_OBJECTS0 := $(LINUX_ZSTD_FILES:.c=.o) +LINUX_ZSTD_OBJECTS := $(LINUX_ZSTD_OBJECTS0:.S=.o) + +%.o: %.S + $(CC) -c $(CPPFLAGS) $(CFLAGS) $^ -o $@ liblinuxzstd.a: $(LINUX_ZSTD_OBJECTS) $(AR) $(ARFLAGS) $@ $^ diff --git a/contrib/pzstd/Makefile b/contrib/pzstd/Makefile index 25265e79ad..3d930cca93 100644 --- a/contrib/pzstd/Makefile +++ b/contrib/pzstd/Makefile @@ -57,19 +57,6 @@ LD_COMMAND = $(CXX) $^ $(ALL_LDFLAGS) $(LIBS) -pthread -o $@ CC_COMMAND = $(CC) $(DEPFLAGS) $(ALL_CFLAGS) -c $< -o $@ CXX_COMMAND = $(CXX) $(DEPFLAGS) $(ALL_CXXFLAGS) -c $< -o $@ -# Get a list of all zstd files so we rebuild the static library when we need to -ZSTDCOMMON_FILES := $(wildcard $(ZSTDDIR)/common/*.c) \ - $(wildcard $(ZSTDDIR)/common/*.h) -ZSTDCOMP_FILES := $(wildcard $(ZSTDDIR)/compress/*.c) \ - $(wildcard $(ZSTDDIR)/compress/*.h) -ZSTDDECOMP_FILES := $(wildcard $(ZSTDDIR)/decompress/*.c) \ - $(wildcard $(ZSTDDIR)/decompress/*.h) -ZSTDPROG_FILES := $(wildcard $(PROGDIR)/*.c) \ - $(wildcard $(PROGDIR)/*.h) -ZSTD_FILES := $(wildcard $(ZSTDDIR)/*.h) \ - $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) \ - $(ZSTDPROG_FILES) - # List all the pzstd source files so we can determine their dependencies PZSTD_SRCS := $(wildcard *.cpp) PZSTD_TESTS := $(wildcard test/*.cpp) @@ -189,7 +176,8 @@ roundtrip: test/RoundTripTest$(EXT) # Use the static library that zstd builds for simplicity and # so we get the compiler options correct -$(ZSTDDIR)/libzstd.a: $(ZSTD_FILES) +.PHONY: $(ZSTDDIR)/libzstd.a +$(ZSTDDIR)/libzstd.a: CFLAGS="$(ALL_CFLAGS)" LDFLAGS="$(ALL_LDFLAGS)" $(MAKE) -C $(ZSTDDIR) libzstd.a # Rules to build the tests diff --git a/lib/Makefile b/lib/Makefile index 376a6690cd..9b74e708cb 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -8,110 +8,13 @@ # You may select, at your option, one of the above-listed licenses. # ################################################################ -# Note: by default, the static library is built single-threaded and dynamic library is built -# multi-threaded. It is possible to force multi or single threaded builds by appending -# -mt or -nomt to the build target (like lib-mt for multi-threaded, lib-nomt for single-threaded). -.PHONY: default -default: lib-release - -# define silent mode as default (verbose mode with V=1 or VERBOSE=1) -$(V)$(VERBOSE).SILENT: - -# When cross-compiling from linux to windows, -# one might need to specify TARGET_SYSTEM as "Windows." -# Building from Fedora fails without it. -# (but Ubuntu and Debian don't need to set anything) -TARGET_SYSTEM ?= $(OS) - -# Version numbers -LIBVER_MAJOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ./zstd.h` -LIBVER_MINOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ./zstd.h` -LIBVER_PATCH_SCRIPT:=`sed -n '/define ZSTD_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ./zstd.h` -LIBVER_SCRIPT:= $(LIBVER_MAJOR_SCRIPT).$(LIBVER_MINOR_SCRIPT).$(LIBVER_PATCH_SCRIPT) -LIBVER_MAJOR := $(shell echo $(LIBVER_MAJOR_SCRIPT)) -LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT)) -LIBVER_PATCH := $(shell echo $(LIBVER_PATCH_SCRIPT)) -LIBVER := $(shell echo $(LIBVER_SCRIPT)) -VERSION?= $(LIBVER) -CCVER := $(shell $(CC) --version) - -# ZSTD_LIB_MINIFY is a helper variable that -# configures a bunch of other variables to space-optimized defaults. -ZSTD_LIB_MINIFY ?= 0 -ifneq ($(ZSTD_LIB_MINIFY), 0) - HAVE_CC_OZ ?= $(shell echo "" | $(CC) -Oz -x c -c - -o /dev/null 2> /dev/null && echo 1 || echo 0) - ZSTD_LEGACY_SUPPORT ?= 0 - ZSTD_LIB_DEPRECATED ?= 0 - HUF_FORCE_DECOMPRESS_X1 ?= 1 - ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT ?= 1 - ZSTD_NO_INLINE ?= 1 - ZSTD_STRIP_ERROR_STRINGS ?= 1 -ifneq ($(HAVE_CC_OZ), 0) - # Some compilers (clang) support an even more space-optimized setting. - CFLAGS += -Oz -else - CFLAGS += -Os -endif - CFLAGS += -fno-stack-protector -fomit-frame-pointer -fno-ident \ - -DDYNAMIC_BMI2=0 -DNDEBUG -else - CFLAGS += -O3 -endif - -DEBUGLEVEL ?= 0 -CPPFLAGS += -DXXH_NAMESPACE=ZSTD_ -DDEBUGLEVEL=$(DEBUGLEVEL) -ifeq ($(TARGET_SYSTEM),Windows_NT) # MinGW assumed - CPPFLAGS += -D__USE_MINGW_ANSI_STDIO # compatibility with %zu formatting -endif -DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ - -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ - -Wstrict-prototypes -Wundef -Wpointer-arith \ - -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \ - -Wredundant-decls -Wmissing-prototypes -Wc++-compat -CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) -FLAGS = $(CPPFLAGS) $(CFLAGS) - -CPPFLAGS_DYNLIB = -DZSTD_MULTITHREAD # dynamic library build defaults to multi-threaded -LDFLAGS_DYNLIB = -pthread -CPPFLAGS_STATLIB = # static library build defaults to single-threaded - -HAVE_COLORNEVER = $(shell echo a | grep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0) -GREP_OPTIONS ?= -ifeq ($HAVE_COLORNEVER, 1) - GREP_OPTIONS += --color=never -endif -GREP = grep $(GREP_OPTIONS) -SED_ERE_OPT ?= -E - -ZSTDCOMMON_FILES := $(sort $(wildcard common/*.c)) -ZSTDCOMP_FILES := $(sort $(wildcard compress/*.c)) -ZSTDDECOMP_FILES := $(sort $(wildcard decompress/*.c)) -ZDICT_FILES := $(sort $(wildcard dictBuilder/*.c)) -ZDEPR_FILES := $(sort $(wildcard deprecated/*.c)) -ZSTD_FILES := $(ZSTDCOMMON_FILES) - -ifeq ($(findstring GCC,$(CCVER)),GCC) -decompress/zstd_decompress_block.o : CFLAGS+=-fno-tree-vectorize -endif - # Modules ZSTD_LIB_COMPRESSION ?= 1 ZSTD_LIB_DECOMPRESSION ?= 1 ZSTD_LIB_DICTBUILDER ?= 1 ZSTD_LIB_DEPRECATED ?= 0 -# Legacy support -ZSTD_LEGACY_SUPPORT ?= 5 -ZSTD_LEGACY_MULTITHREADED_API ?= 0 - -# Build size optimizations -HUF_FORCE_DECOMPRESS_X1 ?= 0 -HUF_FORCE_DECOMPRESS_X2 ?= 0 -ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT ?= 0 -ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG ?= 0 -ZSTD_NO_INLINE ?= 0 -ZSTD_STRIP_ERROR_STRINGS ?= 0 - +# Input variables for libzstd.mk ifeq ($(ZSTD_LIB_COMPRESSION), 0) ZSTD_LIB_DICTBUILDER = 0 ZSTD_LIB_DEPRECATED = 0 @@ -122,86 +25,46 @@ ifeq ($(ZSTD_LIB_DECOMPRESSION), 0) ZSTD_LIB_DEPRECATED = 0 endif +include libzstd.mk + +ZSTD_FILES := $(ZSTD_COMMON_FILES) $(ZSTD_LEGACY_FILES) + ifneq ($(ZSTD_LIB_COMPRESSION), 0) - ZSTD_FILES += $(ZSTDCOMP_FILES) + ZSTD_FILES += $(ZSTD_COMPRESS_FILES) endif ifneq ($(ZSTD_LIB_DECOMPRESSION), 0) - ZSTD_FILES += $(ZSTDDECOMP_FILES) + ZSTD_FILES += $(ZSTD_DECOMPRESS_FILES) endif ifneq ($(ZSTD_LIB_DEPRECATED), 0) - ZSTD_FILES += $(ZDEPR_FILES) + ZSTD_FILES += $(ZSTD_DEPRECATED_FILES) endif ifneq ($(ZSTD_LIB_DICTBUILDER), 0) - ZSTD_FILES += $(ZDICT_FILES) -endif - -ifneq ($(HUF_FORCE_DECOMPRESS_X1), 0) - CFLAGS += -DHUF_FORCE_DECOMPRESS_X1 -endif - -ifneq ($(HUF_FORCE_DECOMPRESS_X2), 0) - CFLAGS += -DHUF_FORCE_DECOMPRESS_X2 -endif - -ifneq ($(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT), 0) - CFLAGS += -DZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT -endif - -ifneq ($(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG), 0) - CFLAGS += -DZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG -endif - -ifneq ($(ZSTD_NO_INLINE), 0) - CFLAGS += -DZSTD_NO_INLINE + ZSTD_FILES += $(ZSTD_DICTBUILDER_FILES) endif -ifneq ($(ZSTD_STRIP_ERROR_STRINGS), 0) - CFLAGS += -DZSTD_STRIP_ERROR_STRINGS -endif - -ifneq ($(ZSTD_LEGACY_MULTITHREADED_API), 0) - CFLAGS += -DZSTD_LEGACY_MULTITHREADED_API -endif - -ifneq ($(ZSTD_LEGACY_SUPPORT), 0) -ifeq ($(shell test $(ZSTD_LEGACY_SUPPORT) -lt 8; echo $$?), 0) - ZSTD_FILES += $(shell ls legacy/*.c | $(GREP) 'v0[$(ZSTD_LEGACY_SUPPORT)-7]') -endif -endif -CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) - ZSTD_LOCAL_SRC := $(notdir $(ZSTD_FILES)) -ZSTD_LOCAL_OBJ := $(ZSTD_LOCAL_SRC:.c=.o) +ZSTD_LOCAL_OBJ0 := $(ZSTD_LOCAL_SRC:.c=.o) +ZSTD_LOCAL_OBJ := $(ZSTD_LOCAL_OBJ0:.S=.o) -ZSTD_SUBDIR := common compress decompress dictBuilder legacy deprecated -vpath %.c $(ZSTD_SUBDIR) +VERSION := $(ZSTD_VERSION) -UNAME := $(shell uname) +# Note: by default, the static library is built single-threaded and dynamic library is built +# multi-threaded. It is possible to force multi or single threaded builds by appending +# -mt or -nomt to the build target (like lib-mt for multi-threaded, lib-nomt for single-threaded). +.PHONY: default +default: lib-release + +CPPFLAGS_DYNLIB = -DZSTD_MULTITHREAD # dynamic library build defaults to multi-threaded +LDFLAGS_DYNLIB = -pthread +CPPFLAGS_STATLIB = # static library build defaults to single-threaded -ifndef BUILD_DIR -ifeq ($(UNAME), Darwin) - ifeq ($(shell md5 < /dev/null > /dev/null; echo $$?), 0) - HASH ?= md5 - endif -else ifeq ($(UNAME), FreeBSD) - HASH ?= gmd5sum -else ifeq ($(UNAME), NetBSD) - HASH ?= md5 -n -else ifeq ($(UNAME), OpenBSD) - HASH ?= md5 -endif -HASH ?= md5sum -HASH_DIR = conf_$(shell echo $(CC) $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(ZSTD_FILES) | $(HASH) | cut -f 1 -d " " ) -HAVE_HASH :=$(shell echo 1 | $(HASH) > /dev/null && echo 1 || echo 0) -ifeq ($(HAVE_HASH),0) - $(info warning : could not find HASH ($(HASH)), needed to differentiate builds using different flags) - BUILD_DIR := obj/generic_noconf +ifeq ($(findstring GCC,$(CCVER)),GCC) +decompress/zstd_decompress_block.o : CFLAGS+=-fno-tree-vectorize endif -endif # BUILD_DIR # macOS linker doesn't support -soname, and use different extension @@ -218,13 +81,6 @@ else SHARED_EXT_VER = $(SHARED_EXT).$(LIBVER) endif -SET_CACHE_DIRECTORY = \ - +$(MAKE) --no-print-directory $@ \ - BUILD_DIR=obj/$(HASH_DIR) \ - CPPFLAGS="$(CPPFLAGS)" \ - CFLAGS="$(CFLAGS)" \ - LDFLAGS="$(LDFLAGS)" - .PHONY: all all: lib @@ -233,6 +89,13 @@ all: lib .PHONY: libzstd.a # must be run every time libzstd.a: CPPFLAGS += $(CPPFLAGS_STATLIB) +SET_CACHE_DIRECTORY = \ + +$(MAKE) --no-print-directory $@ \ + BUILD_DIR=obj/$(HASH_DIR) \ + CPPFLAGS="$(CPPFLAGS)" \ + CFLAGS="$(CFLAGS)" \ + LDFLAGS="$(LDFLAGS)" + ifndef BUILD_DIR # determine BUILD_DIR from compilation flags @@ -343,6 +206,14 @@ $(ZSTD_STATLIB_DIR)/%.o : %.c $(ZSTD_STATLIB_DIR)/%.d | $(ZSTD_STATLIB_DIR) @echo CC $@ $(COMPILE.c) $(DEPFLAGS) $(ZSTD_STATLIB_DIR)/$*.d $(OUTPUT_OPTION) $< +$(ZSTD_DYNLIB_DIR)/%.o : %.S | $(ZSTD_DYNLIB_DIR) + @echo AS $@ + $(COMPILE.c) $(OUTPUT_OPTION) $< + +$(ZSTD_STATLIB_DIR)/%.o : %.S | $(ZSTD_STATLIB_DIR) + @echo AS $@ + $(COMPILE.c) $(OUTPUT_OPTION) $< + MKDIR ?= mkdir $(BUILD_DIR) $(ZSTD_DYNLIB_DIR) $(ZSTD_STATLIB_DIR): $(MKDIR) -p $@ diff --git a/lib/compress/huf_compress.c b/lib/compress/huf_compress.c index db11ab3dcf..915778eb12 100644 --- a/lib/compress/huf_compress.c +++ b/lib/compress/huf_compress.c @@ -445,7 +445,7 @@ typedef struct { /* RANK_POSITION_DISTINCT_COUNT_CUTOFF == Cutoff point in HUF_sort() buckets for which we use log2 bucketing. * Strategy is to use as many buckets as possible for representing distinct * counts while using the remainder to represent all "large" counts. - * + * * To satisfy this requirement for 192 buckets, we can do the following: * Let buckets 0-166 represent distinct counts of [0, 166] * Let buckets 166 to 192 represent all remaining counts up to RANK_POSITION_MAX_COUNT_LOG using log2 bucketing. @@ -517,7 +517,7 @@ static int HUF_quickSortPartition(nodeElt arr[], int const low, int const high) } /* Classic quicksort by descending with partially iterative calls - * to reduce worst case callstack size. + * to reduce worst case callstack size. */ static void HUF_simpleQuickSort(nodeElt arr[], int low, int high) { int const kInsertionSortThreshold = 8; @@ -813,6 +813,7 @@ FORCE_INLINE_TEMPLATE void HUF_addBits(HUF_CStream_t* bitC, HUF_CElt elt, int id assert(((elt >> dirtyBits) << (dirtyBits + nbBits)) == 0); /* We didn't overwrite any bits in the bit container. */ assert(!kFast || (bitC->bitPos[idx] & 0xFF) <= HUF_BITS_IN_CONTAINER); + (void)dirtyBits; } #endif } diff --git a/lib/libzstd.mk b/lib/libzstd.mk new file mode 100644 index 0000000000..45ee1ce629 --- /dev/null +++ b/lib/libzstd.mk @@ -0,0 +1,185 @@ +# ################################################################ +# Copyright (c) Yann Collet, Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under both the BSD-style license (found in the +# LICENSE file in the root directory of this source tree) and the GPLv2 (found +# in the COPYING file in the root directory of this source tree). +# You may select, at your option, one of the above-listed licenses. +# ################################################################ + +################################################################## +# Input Variables +################################################################## + +# Zstd lib directory +LIBZSTD ?= ./ + +# Legacy support +ZSTD_LEGACY_SUPPORT ?= 5 +ZSTD_LEGACY_MULTITHREADED_API ?= 0 + +# Build size optimizations +HUF_FORCE_DECOMPRESS_X1 ?= 0 +HUF_FORCE_DECOMPRESS_X2 ?= 0 +ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT ?= 0 +ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG ?= 0 +ZSTD_NO_INLINE ?= 0 +ZSTD_STRIP_ERROR_STRINGS ?= 0 + +# Assembly support +ZSTD_NO_ASM ?= 0 + +################################################################## +# libzstd helpers +################################################################## + +# Make 4.3 doesn't support '\#' anymore (https://lwn.net/Articles/810071/) +NUM_SYMBOL := \# + +# define silent mode as default (verbose mode with V=1 or VERBOSE=1) +$(V)$(VERBOSE).SILENT: + +# When cross-compiling from linux to windows, +# one might need to specify TARGET_SYSTEM as "Windows." +# Building from Fedora fails without it. +# (but Ubuntu and Debian don't need to set anything) +TARGET_SYSTEM ?= $(OS) + +# Version numbers +LIBVER_SRC := $(LIBZSTD)/zstd.h +LIBVER_MAJOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)` +LIBVER_MINOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)` +LIBVER_PATCH_SCRIPT:=`sed -n '/define ZSTD_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)` +LIBVER_SCRIPT:= $(LIBVER_MAJOR_SCRIPT).$(LIBVER_MINOR_SCRIPT).$(LIBVER_PATCH_SCRIPT) +LIBVER_MAJOR := $(shell echo $(LIBVER_MAJOR_SCRIPT)) +LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT)) +LIBVER_PATCH := $(shell echo $(LIBVER_PATCH_SCRIPT)) +LIBVER := $(shell echo $(LIBVER_SCRIPT)) +CCVER := $(shell $(CC) --version) +ZSTD_VERSION?= $(LIBVER) + +# ZSTD_LIB_MINIFY is a helper variable that +# configures a bunch of other variables to space-optimized defaults. +ZSTD_LIB_MINIFY ?= 0 +ifneq ($(ZSTD_LIB_MINIFY), 0) + HAVE_CC_OZ ?= $(shell echo "" | $(CC) -Oz -x c -c - -o /dev/null 2> /dev/null && echo 1 || echo 0) + ZSTD_LEGACY_SUPPORT ?= 0 + ZSTD_LIB_DEPRECATED ?= 0 + HUF_FORCE_DECOMPRESS_X1 ?= 1 + ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT ?= 1 + ZSTD_NO_INLINE ?= 1 + ZSTD_STRIP_ERROR_STRINGS ?= 1 +ifneq ($(HAVE_CC_OZ), 0) + # Some compilers (clang) support an even more space-optimized setting. + CFLAGS += -Oz +else + CFLAGS += -Os +endif + CFLAGS += -fno-stack-protector -fomit-frame-pointer -fno-ident \ + -DDYNAMIC_BMI2=0 -DNDEBUG +else + CFLAGS += -O3 +endif + +DEBUGLEVEL ?= 0 +CPPFLAGS += -DXXH_NAMESPACE=ZSTD_ -DDEBUGLEVEL=$(DEBUGLEVEL) +ifeq ($(TARGET_SYSTEM),Windows_NT) # MinGW assumed + CPPFLAGS += -D__USE_MINGW_ANSI_STDIO # compatibility with %zu formatting +endif +DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ + -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ + -Wstrict-prototypes -Wundef -Wpointer-arith \ + -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \ + -Wredundant-decls -Wmissing-prototypes -Wc++-compat +CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) +LDFLAGS += $(MOREFLAGS) +FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) + +HAVE_COLORNEVER = $(shell echo a | grep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0) +GREP_OPTIONS ?= +ifeq ($HAVE_COLORNEVER, 1) + GREP_OPTIONS += --color=never +endif +GREP = grep $(GREP_OPTIONS) +SED_ERE_OPT ?= -E + +ZSTD_COMMON_FILES := $(sort $(wildcard $(LIBZSTD)/common/*.c)) +ZSTD_COMPRESS_FILES := $(sort $(wildcard $(LIBZSTD)/compress/*.c)) +ZSTD_DECOMPRESS_FILES := $(sort $(wildcard $(LIBZSTD)/decompress/*.c)) +ZSTD_DICTBUILDER_FILES := $(sort $(wildcard $(LIBZSTD)/dictBuilder/*.c)) +ZSTD_DEPRECATED_FILES := $(sort $(wildcard $(LIBZSTD)/deprecated/*.c)) +ZSTD_LEGACY_FILES := + +ZSTD_DECOMPRESS_AMD64_ASM_FILES := $(sort $(wildcard $(LIBZSTD)/decompress/*_amd64.S)) + +ifneq ($(ZSTD_NO_ASM), 0) + CPPFLAGS += -DHUF_DISABLE_ASM +else + # Unconditionally add the ASM files they are disabled by + # macros in the .S file. + ZSTD_DECOMPRESS_FILES += $(ZSTD_DECOMPRESS_AMD64_ASM_FILES) +endif + +ifneq ($(HUF_FORCE_DECOMPRESS_X1), 0) + CFLAGS += -DHUF_FORCE_DECOMPRESS_X1 +endif + +ifneq ($(HUF_FORCE_DECOMPRESS_X2), 0) + CFLAGS += -DHUF_FORCE_DECOMPRESS_X2 +endif + +ifneq ($(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT), 0) + CFLAGS += -DZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT +endif + +ifneq ($(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG), 0) + CFLAGS += -DZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG +endif + +ifneq ($(ZSTD_NO_INLINE), 0) + CFLAGS += -DZSTD_NO_INLINE +endif + +ifneq ($(ZSTD_STRIP_ERROR_STRINGS), 0) + CFLAGS += -DZSTD_STRIP_ERROR_STRINGS +endif + +ifneq ($(ZSTD_LEGACY_MULTITHREADED_API), 0) + CFLAGS += -DZSTD_LEGACY_MULTITHREADED_API +endif + +ifneq ($(ZSTD_LEGACY_SUPPORT), 0) +ifeq ($(shell test $(ZSTD_LEGACY_SUPPORT) -lt 8; echo $$?), 0) + ZSTD_LEGACY_FILES += $(shell ls $(LIBZSTD)/legacy/*.c | $(GREP) 'v0[$(ZSTD_LEGACY_SUPPORT)-7]') +endif +endif +CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) + +UNAME := $(shell uname) + +ifndef BUILD_DIR +ifeq ($(UNAME), Darwin) + ifeq ($(shell md5 < /dev/null > /dev/null; echo $$?), 0) + HASH ?= md5 + endif +else ifeq ($(UNAME), FreeBSD) + HASH ?= gmd5sum +else ifeq ($(UNAME), NetBSD) + HASH ?= md5 -n +else ifeq ($(UNAME), OpenBSD) + HASH ?= md5 +endif +HASH ?= md5sum + +HASH_DIR = conf_$(shell echo $(CC) $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(ZSTD_FILES) | $(HASH) | cut -f 1 -d " " ) +HAVE_HASH :=$(shell echo 1 | $(HASH) > /dev/null && echo 1 || echo 0) +ifeq ($(HAVE_HASH),0) + $(info warning : could not find HASH ($(HASH)), needed to differentiate builds using different flags) + BUILD_DIR := obj/generic_noconf +endif +endif # BUILD_DIR + +ZSTD_SUBDIR := $(LIBZSTD)/common $(LIBZSTD)/compress $(LIBZSTD)/decompress $(LIBZSTD)/dictBuilder $(LIBZSTD)/legacy $(LIBZSTD)/deprecated +vpath %.c $(ZSTD_SUBDIR) +vpath %.S $(ZSTD_SUBDIR) diff --git a/programs/Makefile b/programs/Makefile index ad05dbe1db..ef094fb743 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -9,7 +9,7 @@ # ########################################################################## # zstd : Command Line Utility, supporting gzip-like arguments # zstd32 : Same as zstd, but forced to compile in 32-bits mode -# zstd_nolegacy : zstd without support of decompression of legacy versions +# zstd-nolegacy : zstd without support of decompression of legacy versions # zstd-small : minimal zstd without dictionary builder and benchmark # zstd-compress : compressor-only version of zstd # zstd-decompress : decompressor-only version of zstd @@ -18,31 +18,9 @@ .PHONY: default default: zstd-release -# silent mode by default; verbose can be triggered by V=1 or VERBOSE=1 -$(V)$(VERBOSE).SILENT: +LIBZSTD := ../lib - -ZSTDDIR := ../lib - -# Version numbers -LIBVER_SRC := $(ZSTDDIR)/zstd.h -LIBVER_MAJOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)` -LIBVER_MINOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)` -LIBVER_PATCH_SCRIPT:=`sed -n '/define ZSTD_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)` -LIBVER_SCRIPT:= $(LIBVER_MAJOR_SCRIPT).$(LIBVER_MINOR_SCRIPT).$(LIBVER_PATCH_SCRIPT) -LIBVER_MAJOR := $(shell echo $(LIBVER_MAJOR_SCRIPT)) -LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT)) -LIBVER_PATCH := $(shell echo $(LIBVER_PATCH_SCRIPT)) -LIBVER := $(shell echo $(LIBVER_SCRIPT)) - -ZSTD_VERSION = $(LIBVER) - -HAVE_COLORNEVER = $(shell echo a | grep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0) -GREP_OPTIONS ?= -ifeq ($HAVE_COLORNEVER, 1) - GREP_OPTIONS += --color=never -endif -GREP = grep $(GREP_OPTIONS) +include $(LIBZSTD)/libzstd.mk ifeq ($(shell $(CC) -v 2>&1 | $(GREP) -c "gcc version "), 1) ALIGN_LOOP = -falign-loops=32 @@ -50,78 +28,25 @@ else ALIGN_LOOP = endif -DEBUGLEVEL ?= 0 -CPPFLAGS += -DXXH_NAMESPACE=ZSTD_ -DDEBUGLEVEL=$(DEBUGLEVEL) -ifeq ($(OS),Windows_NT) # MinGW assumed - CPPFLAGS += -D__USE_MINGW_ANSI_STDIO # compatibility with %zu formatting -endif -CFLAGS ?= -O3 -DEBUGFLAGS+=-Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ - -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ - -Wstrict-prototypes -Wundef -Wpointer-arith \ - -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \ - -Wredundant-decls -Wmissing-prototypes -Wc++-compat -CFLAGS += $(DEBUGFLAGS) -CPPFLAGS += $(MOREFLAGS) -LDFLAGS += $(MOREFLAGS) -FLAGS = $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) - -ZSTDLIB_COMMON := $(ZSTDDIR)/common -ZSTDLIB_COMPRESS := $(ZSTDDIR)/compress -ZSTDLIB_DECOMPRESS := $(ZSTDDIR)/decompress -ZDICT_DIR := $(ZSTDDIR)/dictBuilder -ZSTDLEGACY_DIR := $(ZSTDDIR)/legacy - -vpath %.c $(ZSTDLIB_COMMON) $(ZSTDLIB_COMPRESS) $(ZSTDLIB_DECOMPRESS) $(ZDICT_DIR) $(ZSTDLEGACY_DIR) - -ZSTDLIB_COMMON_C := $(wildcard $(ZSTDLIB_COMMON)/*.c) -ZSTDLIB_COMPRESS_C := $(wildcard $(ZSTDLIB_COMPRESS)/*.c) -ZSTDLIB_DECOMPRESS_C := $(wildcard $(ZSTDLIB_DECOMPRESS)/*.c) -ZSTDLIB_CORE_SRC := $(ZSTDLIB_DECOMPRESS_C) $(ZSTDLIB_COMMON_C) $(ZSTDLIB_COMPRESS_C) -ZDICT_SRC := $(wildcard $(ZDICT_DIR)/*.c) - -ZSTD_LEGACY_SUPPORT ?= 5 -ZSTDLEGACY_SRC := -ifneq ($(ZSTD_LEGACY_SUPPORT), 0) -ifeq ($(shell test $(ZSTD_LEGACY_SUPPORT) -lt 8; echo $$?), 0) - ZSTDLEGACY_SRC += $(shell ls $(ZSTDLEGACY_DIR)/*.c | $(GREP) 'v0[$(ZSTD_LEGACY_SUPPORT)-7]') -endif -endif +ZSTDLIB_COMMON_SRC := $(sort $(ZSTD_COMMON_FILES)) +ZSTDLIB_COMPRESS_SRC := $(sort $(ZSTD_COMPRESS_FILES)) +ZSTDLIB_DECOMPRESS_SRC := $(sort $(ZSTD_DECOMPRESS_FILES)) +ZSTDLIB_CORE_SRC := $(sort $(ZSTD_DECOMPRESS_FILES) $(ZSTD_COMMON_FILES) $(ZSTD_COMPRESS_FILES)) +ZDICT_SRC := $(sort $(ZSTD_DICTBUILDER_FILES)) +ZSTDLEGACY_SRC := $(sort $(ZSTD_LEGACY_FILES)) # Sort files in alphabetical order for reproducible builds ZSTDLIB_FULL_SRC = $(sort $(ZSTDLIB_CORE_SRC) $(ZSTDLEGACY_SRC) $(ZDICT_SRC)) ZSTDLIB_LOCAL_SRC = $(notdir $(ZSTDLIB_FULL_SRC)) -ZSTDLIB_LOCAL_OBJ := $(ZSTDLIB_LOCAL_SRC:.c=.o) +ZSTDLIB_LOCAL_OBJ0 := $(ZSTDLIB_LOCAL_SRC:.c=.o) +ZSTDLIB_LOCAL_OBJ := $(ZSTDLIB_LOCAL_OBJ0:.S=.o) ZSTD_CLI_SRC := $(wildcard *.c) ZSTD_CLI_OBJ := $(ZSTD_CLI_SRC:.c=.o) ZSTD_ALL_SRC = $(ZSTDLIB_LOCAL_SRC) $(ZSTD_CLI_SRC) -ZSTD_ALL_OBJ := $(ZSTD_ALL_SRC:.c=.o) - -UNAME := $(shell uname) - -ifndef BUILD_DIR -ifeq ($(UNAME), Darwin) - ifeq ($(shell md5 < /dev/null > /dev/null; echo $$?), 0) - HASH ?= md5 - endif -else ifeq ($(UNAME), FreeBSD) - HASH ?= gmd5sum -else ifeq ($(UNAME), NetBSD) - HASH ?= md5 -n -else ifeq ($(UNAME), OpenBSD) - HASH ?= md5 -endif -HASH ?= md5sum -HAVE_HASH :=$(shell echo 1 | $(HASH) > /dev/null && echo 1 || echo 0) - -HASH_DIR = conf_$(shell echo $(CC) $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(LDLIBS) $(ZSTD_FILES) | $(HASH) | cut -f 1 -d " ") -ifeq ($(HAVE_HASH),0) - $(info warning : could not find HASH ($(HASH)), needed to differentiate builds using different flags) - BUILD_DIR := obj/generic_noconf -endif -endif # BUILD_DIR +ZSTD_ALL_OBJ0 := $(ZSTD_ALL_SRC:.c=.o) +ZSTD_ALL_OBJ := $(ZSTD_ALL_OBJ0:.S=.o) # Define *.exe as extension for Windows systems ifneq (,$(filter Windows%,$(OS))) @@ -139,9 +64,6 @@ endif VOID = /dev/null -# Make 4.3 doesn't support '\#' anymore (https://lwn.net/Articles/810071/) -NUM_SYMBOL := \# - # thread detection NO_THREAD_MSG := ==> no threads, building without multithreading support HAVE_PTHREAD := $(shell printf '$(NUM_SYMBOL)include \nint main(void) { return 0; }' > have_pthread.c && $(CC) $(FLAGS) -o have_pthread$(EXT) have_pthread.c -pthread 2> $(VOID) && rm have_pthread$(EXT) && echo 1 || echo 0; rm have_pthread.c) @@ -212,7 +134,7 @@ SET_CACHE_DIRECTORY = \ all: zstd .PHONY: allVariants -allVariants: zstd zstd-compress zstd-decompress zstd-small zstd-nolegacy zstd-dictBuilder +allVariants: zstd zstd-compress zstd-decompress zstd-small zstd-frugal zstd-nolegacy zstd-dictBuilder .PHONY: zstd # must always be run zstd : CPPFLAGS += $(THREAD_CPP) $(ZLIBCPP) $(LZMACPP) $(LZ4CPP) @@ -276,6 +198,7 @@ zstd32 : $(ZSTDLIB_FULL_SRC) $(ZSTD_CLI_SRC) ## zstd-nolegacy: same scope as zstd, with just support of legacy formats removed zstd-nolegacy : LDFLAGS += $(THREAD_LD) $(ZLIBLD) $(LZMALD) $(LZ4LD) $(DEBUGFLAGS_LD) +zstd-nolegacy : CPPFLAGS += -UZSTD_LEGACY_SUPPORT -DZSTD_LEGACY_SUPPORT=0 zstd-nolegacy : $(ZSTDLIB_CORE_SRC) $(ZDICT_SRC) $(ZSTD_CLI_OBJ) $(CC) $(FLAGS) $^ -o $@$(EXT) $(LDFLAGS) @@ -299,7 +222,7 @@ zstd-noxz : zstd ## zstd-dll: zstd executable linked to dynamic library libzstd (must have same version) .PHONY: zstd-dll -zstd-dll : LDFLAGS+= -L$(ZSTDDIR) +zstd-dll : LDFLAGS+= -L$(LIBZSTD) zstd-dll : LDLIBS += -lzstd zstd-dll : ZSTDLIB_LOCAL_SRC = xxhash.c zstd-dll : zstd @@ -323,18 +246,17 @@ zstd-pgo : ## zstd-small: minimal target, supporting only zstd compression and decompression. no bench. no legacy. no other format. zstd-small: CFLAGS = -Os -s zstd-frugal zstd-small: $(ZSTDLIB_CORE_SRC) zstdcli.c util.c timefn.c fileio.c - $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOTRACE $^ -o $@$(EXT) + $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOTRACE -UZSTD_LEGACY_SUPPORT -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT) -zstd-decompress: $(ZSTDLIB_COMMON_C) $(ZSTDLIB_DECOMPRESS_C) zstdcli.c util.c timefn.c fileio.c - $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOCOMPRESS -DZSTD_NOTRACE $^ -o $@$(EXT) +zstd-decompress: $(ZSTDLIB_COMMON_SRC) $(ZSTDLIB_DECOMPRESS_SRC) zstdcli.c util.c timefn.c fileio.c + $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOCOMPRESS -DZSTD_NOTRACE -UZSTD_LEGACY_SUPPORT -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT) -zstd-compress: $(ZSTDLIB_COMMON_C) $(ZSTDLIB_COMPRESS_C) zstdcli.c util.c timefn.c fileio.c - $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS -DZSTD_NOTRACE $^ -o $@$(EXT) +zstd-compress: $(ZSTDLIB_COMMON_SRC) $(ZSTDLIB_COMPRESS_SRC) zstdcli.c util.c timefn.c fileio.c + $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS -DZSTD_NOTRACE -UZSTD_LEGACY_SUPPORT -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT) ## zstd-dictBuilder: executable supporting dictionary creation and compression (only) -zstd-dictBuilder: CPPFLAGS += -DZSTD_NOBENCH -DZSTD_NODECOMPRESS -DZSTD_NOTRACE -zstd-dictBuilder: $(ZSTDLIB_COMMON_C) $(ZSTDLIB_COMPRESS_C) $(ZDICT_SRC) zstdcli.c util.c timefn.c fileio.c dibio.c - $(CC) $(FLAGS) $^ -o $@$(EXT) +zstd-dictBuilder: $(ZSTDLIB_COMMON_SRC) $(ZSTDLIB_COMPRESS_SRC) $(ZDICT_SRC) zstdcli.c util.c timefn.c fileio.c dibio.c + $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODECOMPRESS -DZSTD_NOTRACE $^ -o $@$(EXT) zstdmt: zstd ln -sf zstd zstdmt @@ -398,6 +320,10 @@ $(BUILD_DIR)/%.o : %.c $(BUILD_DIR)/%.d | $(BUILD_DIR) @echo CC $@ $(COMPILE.c) $(DEPFLAGS) $(BUILD_DIR)/$*.d $(OUTPUT_OPTION) $< +$(BUILD_DIR)/%.o : %.S | $(BUILD_DIR) + @echo AS $@ + $(COMPILE.c) $(OUTPUT_OPTION) $< + MKDIR ?= mkdir $(BUILD_DIR): ; $(MKDIR) -p $@ diff --git a/tests/Makefile b/tests/Makefile index 85553007d9..56c4483404 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -1,4 +1,5 @@ -# ################################################################ + + # ################################################################ # Copyright (c) Yann Collet, Facebook, Inc. # All rights reserved. # @@ -19,46 +20,43 @@ # zstreamtest32: Same as zstreamtest, but forced to compile in 32-bits mode # ########################################################################## -ZSTDDIR = ../lib +LIBZSTD = ../lib + +ZSTD_LEGACY_SUPPORT ?= 0 + +DEBUGLEVEL ?= 2 +export DEBUGLEVEL # transmit value to sub-makefiles + +include $(LIBZSTD)/libzstd.mk + +ZSTDDIR = $(LIBZSTD) PRGDIR = ../programs PYTHON ?= python3 TESTARTEFACT := versionsTest -DEBUGLEVEL ?= 2 -export DEBUGLEVEL # transmit value to sub-makefiles -DEBUGFLAGS = -g -DDEBUGLEVEL=$(DEBUGLEVEL) +DEBUGFLAGS += -g -Wno-c++-compat CPPFLAGS += -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \ -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR) \ -DZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY=1 -ifeq ($(OS),Windows_NT) # MinGW assumed -CPPFLAGS += -D__USE_MINGW_ANSI_STDIO # compatibility with %zu formatting -endif -CFLAGS ?= -O3 -CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ - -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ - -Wstrict-prototypes -Wundef \ - -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \ - -Wredundant-decls -Wmissing-prototypes -Wno-deprecated-declarations -CFLAGS += $(DEBUGFLAGS) -CPPFLAGS += $(MOREFLAGS) - - -ZSTDCOMMON_FILES := $(ZSTDDIR)/common/*.c -ZSTDCOMP_FILES := $(ZSTDDIR)/compress/*.c -ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/*.c + +ZSTDCOMMON_FILES := $(sort $(ZSTD_COMMON_FILES)) +ZSTDCOMP_FILES := $(sort $(ZSTD_COMPRESS_FILES)) +ZSTDDECOMP_FILES := $(sort $(ZSTD_DECOMPRESS_FILES)) ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) -ZDICT_FILES := $(ZSTDDIR)/dictBuilder/*.c +ZDICT_FILES := $(sort $(ZSTD_DICTBUILDER_FILES)) ZSTD_F1 := $(wildcard $(ZSTD_FILES)) ZSTD_OBJ1 := $(subst $(ZSTDDIR)/common/,zstdm_,$(ZSTD_F1)) ZSTD_OBJ2 := $(subst $(ZSTDDIR)/compress/,zstdc_,$(ZSTD_OBJ1)) ZSTD_OBJ3 := $(subst $(ZSTDDIR)/decompress/,zstdd_,$(ZSTD_OBJ2)) -ZSTD_OBJECTS := $(ZSTD_OBJ3:.c=.o) +ZSTD_OBJ4 := $(ZSTD_OBJ3:.c=.o) +ZSTD_OBJECTS := $(ZSTD_OBJ4:.S=.o) ZSTDMT_OBJ1 := $(subst $(ZSTDDIR)/common/,zstdmt_m_,$(ZSTD_F1)) ZSTDMT_OBJ2 := $(subst $(ZSTDDIR)/compress/,zstdmt_c_,$(ZSTDMT_OBJ1)) ZSTDMT_OBJ3 := $(subst $(ZSTDDIR)/decompress/,zstdmt_d_,$(ZSTDMT_OBJ2)) -ZSTDMT_OBJECTS := $(ZSTDMT_OBJ3:.c=.o) +ZSTDMT_OBJ4 := $(ZSTDMT_OBJ3:.c=.o) +ZSTDMT_OBJECTS := $(ZSTDMT_OBJ4:.S=.o) # Define *.exe as extension for Windows systems ifneq (,$(filter Windows%,$(OS))) @@ -119,6 +117,9 @@ zstdc_%.o : $(ZSTDDIR)/compress/%.c zstdd_%.o : $(ZSTDDIR)/decompress/%.c $(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@ +zstdd_%.o : $(ZSTDDIR)/decompress/%.S + $(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@ + zstdmt%.o : CPPFLAGS += $(MULTITHREAD_CPP) zstdmt_m_%.o : $(ZSTDDIR)/common/%.c @@ -130,6 +131,9 @@ zstdmt_c_%.o : $(ZSTDDIR)/compress/%.c zstdmt_d_%.o : $(ZSTDDIR)/decompress/%.c $(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@ +zstdmt_d_%.o : $(ZSTDDIR)/decompress/%.S + $(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@ + fullbench32: CPPFLAGS += -m32 fullbench fullbench32 : CPPFLAGS += $(MULTITHREAD_CPP) -Wno-deprecated-declarations fullbench fullbench32 : LDFLAGS += $(MULTITHREAD_LD) @@ -201,7 +205,7 @@ bigdict: $(ZSTDMT_OBJECTS) $(PRGDIR)/datagen.c bigdict.c invalidDictionaries : $(ZSTD_OBJECTS) invalidDictionaries.c -legacy : CPPFLAGS += -I$(ZSTDDIR)/legacy -DZSTD_LEGACY_SUPPORT=4 +legacy : CPPFLAGS += -I$(ZSTDDIR)/legacy -UZSTD_LEGACY_SUPPORT -DZSTD_LEGACY_SUPPORT=4 legacy : $(ZSTD_FILES) $(wildcard $(ZSTDDIR)/legacy/*.c) legacy.c decodecorpus : LDLIBS += -lm diff --git a/tests/fuzz/Makefile b/tests/fuzz/Makefile index 5c54ccd77d..72dbccb577 100644 --- a/tests/fuzz/Makefile +++ b/tests/fuzz/Makefile @@ -23,6 +23,12 @@ else endif CORPORA_URL_PREFIX:=https://github.com/facebook/zstd/releases/download/fuzz-corpora/ +LIBZSTD = ../../lib +DEBUGLEVEL ?= 2 +ZSTD_LEGACY_SUPPORT ?= 1 + +include $(LIBZSTD)/libzstd.mk + ZSTDDIR = ../../lib PRGDIR = ../../programs CONTRIBDIR = ../../contrib @@ -34,7 +40,7 @@ FUZZ_EXTRA_FLAGS := -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ -Wstrict-prototypes -Wundef \ -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \ - -Wredundant-decls \ + -Wredundant-decls -Wno-deprecated-declarations \ -g -fno-omit-frame-pointer FUZZ_CFLAGS := $(FUZZ_EXTRA_FLAGS) $(CFLAGS) FUZZ_CXXFLAGS := $(FUZZ_EXTRA_FLAGS) -std=c++11 $(CXXFLAGS) @@ -50,11 +56,11 @@ FUZZ_SRC := $(PRGDIR)/util.c ./fuzz_helpers.c ./zstd_helpers.c ./fuzz_data_produ SEEKABLE_HEADERS = $(CONTRIBDIR)/seekable_format/zstd_seekable.h SEEKABLE_OBJS = $(CONTRIBDIR)/seekable_format/zstdseek_compress.c $(CONTRIBDIR)/seekable_format/zstdseek_decompress.c -ZSTDCOMMON_SRC := $(ZSTDDIR)/common/*.c -ZSTDCOMP_SRC := $(ZSTDDIR)/compress/*.c -ZSTDDECOMP_SRC := $(ZSTDDIR)/decompress/*.c -ZSTDDICT_SRC := $(ZSTDDIR)/dictBuilder/*.c -ZSTDLEGACY_SRC := $(ZSTDDIR)/legacy/*.c +ZSTDCOMMON_SRC := $(ZSTD_COMMON_FILES) +ZSTDCOMP_SRC := $(ZSTD_COMPRESS_FILES) +ZSTDDECOMP_SRC := $(ZSTD_DECOMPRESS_FILES) +ZSTDDICT_SRC := $(ZSTD_DICTBUILDER_FILES) +ZSTDLEGACY_SRC := $(ZSTD_LEGACY_FILES) FUZZ_SRC := \ $(FUZZ_SRC) \ $(ZSTDDECOMP_SRC) \ @@ -71,7 +77,8 @@ FUZZ_D_OBJ4 := $(subst $(ZSTDDIR)/dictBuilder/,d_lib_dictBuilder_,$(FUZZ_D_OBJ3) FUZZ_D_OBJ5 := $(subst $(ZSTDDIR)/legacy/,d_lib_legacy_,$(FUZZ_D_OBJ4)) FUZZ_D_OBJ6 := $(subst $(PRGDIR)/,d_prg_,$(FUZZ_D_OBJ5)) FUZZ_D_OBJ7 := $(subst $\./,d_fuzz_,$(FUZZ_D_OBJ6)) -FUZZ_DECOMPRESS_OBJ := $(FUZZ_D_OBJ7:.c=.o) +FUZZ_D_OBJ8 := $(FUZZ_D_OBJ7:.c=.o) +FUZZ_DECOMPRESS_OBJ := $(FUZZ_D_OBJ8:.S=.o) FUZZ_RT_OBJ1 := $(subst $(ZSTDDIR)/common/,rt_lib_common_,$(FUZZ_SRC)) FUZZ_RT_OBJ2 := $(subst $(ZSTDDIR)/compress/,rt_lib_compress_,$(FUZZ_RT_OBJ1)) @@ -80,7 +87,8 @@ FUZZ_RT_OBJ4 := $(subst $(ZSTDDIR)/dictBuilder/,rt_lib_dictBuilder_,$(FUZZ_RT_OB FUZZ_RT_OBJ5 := $(subst $(ZSTDDIR)/legacy/,rt_lib_legacy_,$(FUZZ_RT_OBJ4)) FUZZ_RT_OBJ6 := $(subst $(PRGDIR)/,rt_prg_,$(FUZZ_RT_OBJ5)) FUZZ_RT_OBJ7 := $(subst $\./,rt_fuzz_,$(FUZZ_RT_OBJ6)) -FUZZ_ROUND_TRIP_OBJ := $(FUZZ_RT_OBJ7:.c=.o) +FUZZ_RT_OBJ8 := $(FUZZ_RT_OBJ7:.c=.o) +FUZZ_ROUND_TRIP_OBJ := $(FUZZ_RT_OBJ8:.S=.o) .PHONY: default all clean cleanall @@ -117,6 +125,9 @@ rt_lib_compress_%.o: $(ZSTDDIR)/compress/%.c rt_lib_decompress_%.o: $(ZSTDDIR)/decompress/%.c $(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $(FUZZ_ROUND_TRIP_FLAGS) $< -c -o $@ +rt_lib_decompress_%.o: $(ZSTDDIR)/decompress/%.S + $(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $(FUZZ_ROUND_TRIP_FLAGS) $< -c -o $@ + rt_lib_dictBuilder_%.o: $(ZSTDDIR)/dictBuilder/%.c $(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $(FUZZ_ROUND_TRIP_FLAGS) $< -c -o $@ @@ -138,6 +149,9 @@ d_lib_compress_%.o: $(ZSTDDIR)/compress/%.c d_lib_decompress_%.o: $(ZSTDDIR)/decompress/%.c $(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $< -c -o $@ +d_lib_decompress_%.o: $(ZSTDDIR)/decompress/%.S + $(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $< -c -o $@ + d_lib_dictBuilder_%.o: $(ZSTDDIR)/dictBuilder/%.c $(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $< -c -o $@ diff --git a/tests/fuzzer.c b/tests/fuzzer.c index ce79f53acf..c4de54247b 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -1528,7 +1528,7 @@ static int basicUnitTests(U32 const seed, double compressibility) DISPLAYLEVEL(3, "test%3i : resize context to full CCtx size : ", testNb++); staticCCtx = ZSTD_initStaticCStream(staticCCtxBuffer, staticCCtxSize); - DISPLAYLEVEL(4, "staticCCtxBuffer = %p, staticCCtx = %p , ", staticCCtxBuffer, staticCCtx); + DISPLAYLEVEL(4, "staticCCtxBuffer = %p, staticCCtx = %p , ", staticCCtxBuffer, (void*)staticCCtx); if (staticCCtx == NULL) goto _output_error; DISPLAYLEVEL(3, "OK \n"); diff --git a/tests/regression/Makefile b/tests/regression/Makefile index 3a5a64e5b0..a440c6c942 100644 --- a/tests/regression/Makefile +++ b/tests/regression/Makefile @@ -46,6 +46,7 @@ result.o: result.c result.h test.o: test.c data.h config.h method.h $(CC) $(REGRESSION_CFLAGS) $(REGRESSION_CPPFLAGS) $< -c -o $@ +.PHONY: libzstd.a libzstd.a: $(MAKE) -C $(LIBDIR) libzstd.a-mt cp $(LIBDIR)/libzstd.a . diff --git a/tests/test-variants.sh b/tests/test-variants.sh new file mode 100755 index 0000000000..f3a9e065bd --- /dev/null +++ b/tests/test-variants.sh @@ -0,0 +1,115 @@ +#!/bin/sh +set -e +set -u +set -x + + +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +PROG_DIR="$SCRIPT_DIR/../programs" + +ZSTD="$PROG_DIR/zstd" +ZSTD_COMPRESS="$PROG_DIR/zstd-compress" +ZSTD_DECOMPRESS="$PROG_DIR/zstd-decompress" +ZSTD_NOLEGACY="$PROG_DIR/zstd-nolegacy" +ZSTD_DICTBUILDER="$PROG_DIR/zstd-dictBuilder" +ZSTD_FRUGAL="$PROG_DIR/zstd-frugal" +ZSTD_NOMT="$PROG_DIR/zstd-nomt" + +println() { + printf '%b\n' "${*}" +} + +die() { + println "$@" 1>&2 + exit 1 +} + +symbol_present() { + (nm $1 || echo "symbol_present $@ failed") | grep $2 +} + +symbol_not_present() { + symbol_present $@ && die "Binary '$1' mistakenly contains symbol '$2'" ||: +} + +compress_not_present() { + symbol_not_present "$1" ZSTD_compress +} + +decompress_not_present() { + symbol_not_present "$1" ZSTD_decompress +} + +dict_not_present() { + symbol_not_present "$1" ZDICT_ + symbol_not_present "$1" COVER_ +} + +cliextra_not_present() { + symbol_not_present "$1" TRACE_ + symbol_not_present "$1" BMK_ +} + +legacy_not_present() { + symbol_not_present "$1" ZSTDv0 +} + +test_help() { + "$1" --help | grep -- "$2" +} + +test_no_help() { + test_help $@ && die "'$1' supports '$2' when it shouldn't" ||: +} + +extras_not_present() { + dict_not_present $@ + legacy_not_present $@ + cliextra_not_present $@ + test_no_help $@ "--train" + test_no_help $@ "-b#" +} + +test_compress() { + echo "hello" | "$1" | "$ZSTD" -t +} + +test_decompress() { + echo "hello" | "$ZSTD" | "$1" -t +} + +test_zstd() { + test_compress $@ + test_decompress $@ +} + +extras_not_present "$ZSTD_FRUGAL" +extras_not_present "$ZSTD_COMPRESS" +extras_not_present "$ZSTD_DECOMPRESS" + +compress_not_present "$ZSTD_DECOMPRESS" + +decompress_not_present "$ZSTD_COMPRESS" +decompress_not_present "$ZSTD_DICTBUILDER" + +cliextra_not_present "$ZSTD_DICTBUILDER" + +legacy_not_present "$ZSTD_DICTBUILDER" +legacy_not_present "$ZSTD_NOLEGACY" + +symbol_not_present "$ZSTD" ZSTDv01 +symbol_not_present "$ZSTD" ZSTDv02 +symbol_not_present "$ZSTD" ZSTDv03 +symbol_not_present "$ZSTD" ZSTDv04 + +test_compress "$ZSTD_COMPRESS" +test_decompress "$ZSTD_DECOMPRESS" + +test_zstd "$ZSTD_FRUGAL" +test_zstd "$ZSTD_NOLEGACY" + +test_help "$ZSTD" '-b#' +test_help "$ZSTD" --train +test_help "$ZSTD_DICTBUILDER" --train + +println "Success!"