diff --git a/.github/workflows/build-dev.yml b/.github/workflows/build-dev.yml new file mode 100644 index 0000000..e1c27d3 --- /dev/null +++ b/.github/workflows/build-dev.yml @@ -0,0 +1,43 @@ +name: build-dev + +on: + push: + branches: + - dev + - master + paths: + - 'src/python/requirements.txt' + - 'build-libs.sh' + - 'Dockerfile' + - 'Makefile' + pull_request: + branches: + - dev + - master + paths: + - 'src/python/requirements.txt' + - 'build-libs.sh' + - 'Dockerfile' + - 'Makefile' + types: + - opened + - edited + - synchronize + - reopened + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout close + uses: actions/checkout@v2 + - name: Build new Docker image + working-directory: ${{ github.workspace }} + run: docker build -t mtgupf/essentia-emscripten:dev . + - name: Build library + working-directory: ${{ github.workspace }} + run: | + docker run --rm -v `pwd`:/essentia/ mtgupf/essentia-emscripten:dev /essentia/build-libs.sh + shell: bash + + # unit-test: diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 15db032..50f7c41 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -10,7 +10,7 @@ on: # - 'test/**' - 'build-libs.sh' - 'Dockerfile' - - 'Makefile.essentiajs' + - 'Makefile' - 'rollup*.js' - 'tsconfig.json' - '!docs/**' diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index a6aefdf..a287779 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -10,7 +10,7 @@ on: # - 'test/**' - 'build-libs.sh' - 'Dockerfile' - - 'Makefile.essentiajs' + - 'Makefile' - 'rollup*.js' - 'tsconfig.json' - '!docs/**' @@ -24,7 +24,7 @@ on: # - 'test/**' - 'build-libs.sh' - 'Dockerfile' - - 'Makefile.essentiajs' + - 'Makefile' - 'rollup*.js' - 'tsconfig.json' - '!docs/**' diff --git a/Dockerfile b/Dockerfile index af0eb04..0fb2aa7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,17 +1,17 @@ -ARG EMSCRIPTEN_VERSION=1.39.19 +ARG EMSCRIPTEN_VERSION=3.1.24 FROM emscripten/emsdk:${EMSCRIPTEN_VERSION} ENV LANG C.UTF-8 # compile essentia with emscripten and selected third-party dependencies RUN apt-get update \ - && apt-get install -y cmake curl nano python-dev python-numpy-dev libpython2.7 python-pip libeigen3-dev \ + && apt-get install -y cmake curl nano python3-dev python3-numpy-dev python3-numpy python3-yaml python3-six libeigen3-dev python3-pip \ && mkdir /essentia && cd /essentia && git clone https://github.com/MTG/essentia.git \ && cd /essentia/essentia/packaging/debian_3rdparty \ && bash -C "./build_eigen3.sh" && cd ../../ && chmod +x waf \ - && emconfigure sh -c './waf configure --prefix=$EMSCRIPTEN/system/local/ --build-static --fft=KISS --emscripten --static-dependencies' \ - && emmake ./waf && emmake ./waf install \ - && apt-get remove -y python-dev libeigen3-dev \ + && emconfigure sh -c 'python3 waf configure --prefix=$EMSCRIPTEN/system/local/ --build-static --fft=KISS --emscripten --static-dependencies' \ + && emmake python3 waf && emmake python3 waf install \ + && apt-get remove -y python3-dev libeigen3-dev \ && apt-get autoremove -y \ && apt-get clean -y \ && rm -rf /var/lib/apt/lists/* \ @@ -19,8 +19,8 @@ RUN apt-get update \ # copy and install python dependencies COPY src/python/requirements.txt /tmp/requirements.txt -RUN pip install --upgrade setuptools \ - && pip install --no-cache-dir -r /tmp/requirements.txt \ +RUN pip3 install --upgrade setuptools \ + && pip3 install --no-cache-dir -r /tmp/requirements.txt \ && rm /tmp/requirements.txt # add latest eigen3 header files for linking the essentia.js binding builds @@ -30,6 +30,6 @@ RUN wget -P /usr/local/include/ "https://gitlab.com/libeigen/eigen/-/archive/3. && rm -f eigen-${EIGEN_VERSION}.tar.gz && mv eigen-${EIGEN_VERSION} eigen3 ENV EIGEN_PATH /usr/local/include/eigen3 -ENV EMSCRIPTEN /emsdk/upstream/emscripten +ENV EMSCRIPTEN /emsdk/upstream/emscripten WORKDIR /essentia/ \ No newline at end of file diff --git a/Makefile.essentiajs b/Makefile similarity index 75% rename from Makefile.essentiajs rename to Makefile index 04ff393..e05539e 100644 --- a/Makefile.essentiajs +++ b/Makefile @@ -1,6 +1,7 @@ ESSENTIAJS_VERSION=0.1.3 ## Path to libs for Emscripten LIB_DIR_ESSENTIA=$(EMSCRIPTEN)/system/local/lib +INCLUDE_DIR_ESSENTIA=$(EMSCRIPTEN)/system/local/include EIGEN_PATH=/usr/local/include/eigen3 ## Pass custom build and dist directories using system environment variables BUILD_DIR_ES := $(or $(ESSENTIAJS_WASM_BUILDS_DIR),builds) @@ -22,25 +23,18 @@ POST_JS_UMD_WASM=src/js/wasm.umd.post.js codegen: @echo "Generating cpp source code from essentia python bindings ..." - @cd src/python && python configure_bindings.py + @cd src/python && python3 configure_bindings.py build: @mkdir -p $(BUILD_DIR_ES) - @echo "Compiling the emscripten embind cpp bindings to bitcode ..." + @echo "Compiling + linking emscripten embind cpp bindings directly to js, wasm files ..." - @emcc -I $(EIGEN_PATH) \ - --bind -Oz $(BINDING_ESSENTIAJS) $(INCLUDE_ESSENTIAJS) \ - -o $(BUILD_DIR_ES)/essentiajs.bc \ - -s EXCEPTION_DEBUG \ - -s ASSERTIONS=2 \ - -s DISABLE_EXCEPTION_CATCHING=2 || exit 1 - @echo "Done ..." - - @echo "Linking and compiling the bindings with essentia to js, wasm files ..." - @echo "compiling async builds..." - @emcc --bind -Oz $(BUILD_DIR_ES)/essentiajs.bc ${LIB_DIR_ESSENTIA}/essentia.a \ - -s WASM=1 \ + @echo "... compiling async builds..." + @emcc -lembind -Oz $(BINDING_ESSENTIAJS) $(INCLUDE_ESSENTIAJS) \ + -lembind -lessentia -L $(LIB_DIR_ESSENTIA) \ + -I $(EIGEN_PATH) -I $(INCLUDE_DIR_ESSENTIA)\ + -s WASM=1 \ -o $(ESSENTIA_JS_WEB) \ -s EXCEPTION_DEBUG \ -s ASSERTIONS=2 \ @@ -51,8 +45,10 @@ build: -s ALLOW_MEMORY_GROWTH=1 || exit 1 @echo "Done ..." - @echo "compiling sync builds..." - @emcc --bind -Oz $(BUILD_DIR_ES)/essentiajs.bc ${LIB_DIR_ESSENTIA}/essentia.a \ + @echo "... compiling sync builds..." + @emcc --bind -Oz $(BINDING_ESSENTIAJS) $(INCLUDE_ESSENTIAJS) \ + -lembind -lessentia -L $(LIB_DIR_ESSENTIA) \ + -I $(EIGEN_PATH) -I $(INCLUDE_DIR_ESSENTIA)\ -s WASM=1 \ -o $(ESSENTIA_JS_MODULE) \ -s BINARYEN_ASYNC_COMPILATION=0 \ @@ -78,9 +74,6 @@ build: @echo "Done ..." - @echo "Removing unnecessary files ..." - @rm $(BUILD_DIR_ES)/essentiajs.bc - @echo "Builds ..." @ls $(BUILD_DIR_ES) diff --git a/build-libs.sh b/build-libs.sh index 154c21d..299a798 100755 --- a/build-libs.sh +++ b/build-libs.sh @@ -4,26 +4,26 @@ set -e -x # Custom path to the node installation in the essentia-emscripten docker image # provided along with emsdk. This can be removed once the # entrypoint issues in the essentia-emscripten docker image got resolved. -NPM_PATH=/emsdk/node/12.18.1_64bit/bin/npm -NODE_PATH=/emsdk/node/12.18.1_64bit/bin/node +# NPM_PATH=/emsdk/node/12.18.1_64bit/bin/npm +# NODE_PATH=/emsdk/node/12.18.1_64bit/bin/node # set permission -ln -sf $NODE_PATH /usr/bin/node +# ln -sf $NODE_PATH /usr/bin/node # Generate Essentia.js source code from Essentia docs -$NPM_PATH run gen-code +npm run gen-code # Build Essentia WASM backend -$NPM_PATH run build-wasm +npm run build-wasm # Install essentia.js node dependecies -$NPM_PATH install +npm install # Build essentia.js JS API and add-on modules -$NPM_PATH run build-js-api +npm run build-js-api # Minified build of JS API and add-on modules -$NPM_PATH run build-js-api rollup.config.min.js +npm run build-js-api rollup.config.min.js # Run tests echo "Running tests ..." -$NPM_PATH test \ No newline at end of file +npm test \ No newline at end of file diff --git a/docs/tutorials/4. Building from source.md b/docs/tutorials/4. Building from source.md index 0f523c9..a1dc754 100644 --- a/docs/tutorials/4. Building from source.md +++ b/docs/tutorials/4. Building from source.md @@ -63,7 +63,7 @@ You need to have a local installation of docker. docker run --rm -v `pwd`:/srv/workspace/ \ mtgupf/essentia-emscripten:latest \ /srv/workspace/build-bindings.sh \ - Makefile.essentiajs + Makefile ``` OR @@ -101,7 +101,7 @@ OR Spawn a subshell inside the emscripten `emconfigure` in order to properlly access the emscripten variables. ```bash - emconfigure sh -c './build-bindings.sh Makefile.essentiajs' + emconfigure sh -c './build-bindings.sh Makefile' ``` > Note: make you added the emscripten env variables to your bash profile. diff --git a/package.json b/package.json index 6476e23..ba1b5c6 100644 --- a/package.json +++ b/package.json @@ -15,8 +15,8 @@ "CHANGELOG.md" ], "scripts": { - "gen-code": "make -f Makefile.essentiajs codegen", - "build-wasm": "make -f Makefile.essentiajs build", + "gen-code": "make -f Makefile codegen", + "build-wasm": "make -f Makefile build", "build-js-api": "rollup --config", "build-api-docs": "./build-docs.sh", "test": "mocha" diff --git a/src/python/bindings.cog b/src/python/bindings.cog index a5cc673..4a04044 100644 --- a/src/python/bindings.cog +++ b/src/python/bindings.cog @@ -35,7 +35,7 @@ EMSCRIPTEN_BINDINGS(CLASS_EssentiaJS) { /*[[[cog import logging import cog - from .code_generator import TO_INCLUDE_ALGOS + from code_generator import TO_INCLUDE_ALGOS logging.basicConfig(level='INFO') logging.info("Generating emscripten bindings for the essentia...") for algo_name in TO_INCLUDE_ALGOS: diff --git a/src/python/configure_bindings.py b/src/python/configure_bindings.py index 23f6111..9f5c914 100644 --- a/src/python/configure_bindings.py +++ b/src/python/configure_bindings.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +from __future__ import print_function +from builtins import str import os import argparse import essentia.standard as estd @@ -30,6 +32,7 @@ def savelist_to_file(path_list, filename): # requires Gaia and Tensorflow dependencies 'GaiaTransform', 'MusicExtractorSVM', 'TensorflowPredict', 'TensorflowPredictMusiCNN', 'TensorflowPredictVGGish', + 'TensorNormalize', 'TensorTranspose', # these algortihms expect a matrix_real input or output types which are not yet supported for the JS bindings 'BpmHistogram', 'FadeDetection', 'HumDetector', 'Onsets', 'Panning', 'SBic', 'SingleGaussian', # these algorithms expect std::complex** type for wither input, parameters or outputs, which are not yet supported for the JS bindings diff --git a/src/python/excluded_algos.md b/src/python/excluded_algos.md index 68580b5..3c406db 100644 --- a/src/python/excluded_algos.md +++ b/src/python/excluded_algos.md @@ -29,6 +29,8 @@ MusicExtractorSVM TensorflowPredict TensorflowPredictMusiCNN TensorflowPredictVGGish +TensorNormalize +TensorTranspose BpmHistogram FadeDetection HumDetector diff --git a/src/python/header.cog b/src/python/header.cog index e7c866d..3d0835a 100644 --- a/src/python/header.cog +++ b/src/python/header.cog @@ -42,7 +42,7 @@ class EssentiaJS { // NOTE: The following code snippets are machine generated. Do not edit. /*[[[cog import cog - from .code_generator import generate_headers, TO_INCLUDE_ALGOS + from code_generator import generate_headers, TO_INCLUDE_ALGOS cog.outl(" ") cog.outl("// class property which stores all the list of essentia algorithm names available in essentia.js") c_algo_list = str(TO_INCLUDE_ALGOS) diff --git a/src/python/included_algos.md b/src/python/included_algos.md index 65a69d5..3c478e7 100644 --- a/src/python/included_algos.md +++ b/src/python/included_algos.md @@ -179,6 +179,7 @@ TempoTapDegara TempoTapMaxAgreement TempoTapTicks TensorflowInputMusiCNN +TensorflowInputTempoCNN TensorflowInputVGGish TonalExtractor TonicIndianArtMusic diff --git a/src/python/js_wrapper.cog b/src/python/js_wrapper.cog index 8388a94..d7d5aa1 100644 --- a/src/python/js_wrapper.cog +++ b/src/python/js_wrapper.cog @@ -198,7 +198,7 @@ class Essentia { // NOTE: The following code snippets are machine generated. Do not edit. /*[[[cog import cog - from .code_generator import generate_typescript_wrapper + from code_generator import generate_typescript_wrapper algos = generate_typescript_wrapper() for algo in algos: for ln in algo: diff --git a/src/python/library.cog b/src/python/library.cog index 74058c0..7cdc98e 100644 --- a/src/python/library.cog +++ b/src/python/library.cog @@ -161,7 +161,7 @@ val EssentiaJS::LoudnessEBUR128(std::vector& left_channel, std::vector