Filecrawler release build #14
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # GitHub Action workflow to build and run filecrawler's tests | |
| # | |
| name: PyPI Build filecrawler | |
| on: | |
| release: | |
| types: [published] | |
| env: | |
| LATEST_TAG: filecrawler:latest | |
| jobs: | |
| lint: | |
| name: Check syntaxs errors and warnings | |
| runs-on: ubuntu-latest | |
| if: | |
| github.event_name == 'push' || github.event.pull_request.head.repo.full_name != | |
| github.repository | |
| steps: | |
| - name: Checkout filecrawler | |
| uses: actions/checkout@v3 | |
| - name: Setup Python 3.9 | |
| uses: actions/setup-python@v3 | |
| with: | |
| python-version: 3.9 | |
| - name: Install Python dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| python -m pip install flake8 | |
| - name: Check syntax errors | |
| run: | | |
| flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics | |
| - name: Check PEP8 warnings | |
| run: | | |
| flake8 . --count --ignore=E1,E2,E3,E501,W291,W293 --exit-zero --max-complexity=65 --max-line-length=127 --statistics | |
| binutils_dependencies: | |
| name: Build external dependencies | |
| needs: lint | |
| runs-on: ubuntu-latest | |
| if: | |
| github.event_name == 'push' || github.event.pull_request.head.repo.full_name != | |
| github.repository | |
| strategy: | |
| fail-fast: false | |
| steps: | |
| - name: Get last release files | |
| id: checkurls | |
| run: | | |
| url1=$(curl -s https://api.github.com/repos/helviojunior/filecrawler/releases/latest | jq -r '.assets[] | select(.name|match("filecrawler_extra_libs.zip$")) | .browser_download_url') | |
| echo "BINUTILS_URL=${url1}" >> $GITHUB_OUTPUT | |
| - name: Install dependencies | |
| if: | |
| steps.checkurls.outputs.BINUTILS_URL == '' | |
| uses: awalsh128/cache-apt-pkgs-action@latest | |
| with: | |
| packages: wget curl jq zip | |
| version: 1.0 | |
| - name: Create temp path | |
| if: | |
| steps.checkurls.outputs.BINUTILS_URL == '' | |
| run: | | |
| mkdir -p /tmp/filecrawler/bin | |
| - name: Get tika server .jar | |
| if: | |
| steps.checkurls.outputs.BINUTILS_URL == '' | |
| run: | | |
| VER=$(curl -s "https://raw.githubusercontent.com/chrismattmann/tika-python/master/tika/tika.py" | grep 'TIKA_VERSION' | grep -oE '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}' || echo '2.6.0') | |
| SERVER_HASH=$(curl -s "http://search.maven.org/remotecontent?filepath=org/apache/tika/tika-server-standard/$VER/tika-server-standard-$VER.jar.sha1") | |
| wget -nv -O "/tmp/filecrawler/bin/tika-server.jar" "http://search.maven.org/remotecontent?filepath=org/apache/tika/tika-server-standard/$VER/tika-server-standard-$VER.jar" | |
| echo "${SERVER_HASH} /tmp/filecrawler/bin/tika-server.jar" | sha1sum -c - || exit 1 | |
| - name: Get JADX | |
| if: | |
| steps.checkurls.outputs.BINUTILS_URL == '' | |
| run: | | |
| VER=$(curl -s "https://api.github.com/repos/skylot/jadx/tags" | jq -r '.[0].name' | grep -oE '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}') | |
| wget -nv -O "/tmp/jadx.zip" "https://github.com/skylot/jadx/releases/download/v$VER/jadx-$VER.zip" | |
| mkdir -p /tmp/jadx/ | |
| unzip -o /tmp/jadx.zip -d /tmp/jadx/ | |
| FILE=$(find /tmp/jadx/ -name "jadx*.jar") | |
| mv "$FILE" "/tmp/filecrawler/bin/jadx.jar" | |
| - name: Get JADX | |
| if: | |
| steps.checkurls.outputs.BINUTILS_URL == '' | |
| run: | | |
| VER=$(curl -s "https://api.github.com/repos/iBotPeaches/Apktool/tags" | jq -r '.[0].name' | grep -oE '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}') | |
| wget -nv -O "/tmp/apktool.jar" "https://github.com/iBotPeaches/Apktool/releases/download/v$VER/apktool_$VER.jar" | |
| mv "/tmp/apktool.jar" "/tmp/filecrawler/bin/" | |
| - name: Create bin zip | |
| if: | |
| steps.checkurls.outputs.BINUTILS_URL == '' | |
| run: | | |
| zip -j -5 /tmp/filecrawler_extra_libs.zip /tmp/filecrawler/bin/* | |
| - name: Upload binutils release asset | |
| if: | |
| steps.checkurls.outputs.BINUTILS_URL == '' | |
| id: filecrawler_extra_libs | |
| uses: actions/upload-release-asset@v1 | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| with: | |
| upload_url: ${{ github.event.release.upload_url }} | |
| asset_path: /tmp/filecrawler_extra_libs.zip | |
| asset_name: filecrawler_extra_libs.zip | |
| asset_content_type: application/zip | |
| build: | |
| name: Build | |
| needs: binutils_dependencies | |
| runs-on: ubuntu-latest | |
| if: | |
| github.event_name == 'push' || github.event.pull_request.head.repo.full_name != | |
| github.repository | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.8", "3.9", "3.10"] | |
| experimental: [false] | |
| include: | |
| - python-version: "3.11" | |
| experimental: true | |
| continue-on-error: ${{ matrix.experimental }} | |
| steps: | |
| - name: Checkout filecrawler | |
| uses: actions/checkout@v3 | |
| - name: Setup Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v3 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install Python dependencies | |
| run: | | |
| python -m pip install --upgrade pip wheel | |
| python -m pip install -r requirements.txt -r tests/requirements-test.txt | |
| - name: Build artifact | |
| run: | | |
| python setup.py sdist | |
| docker_build: | |
| name: Check docker build and run | |
| needs: build | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v2 | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| #- name: Set up Docker Buildx | |
| # uses: docker/setup-buildx-action@v2 | |
| #- name: Build and export to Docker | |
| # uses: docker/build-push-action@v4 | |
| # with: | |
| # context: . | |
| # push: true | |
| # tags: ${{ env.LATEST_TAG }} | |
| #- name: Test | |
| # run: | | |
| # docker run --rm ${{ env.LATEST_TAG }} -h | |
| publish: | |
| name: Publish on Live Environment | |
| needs: docker_build | |
| runs-on: ubuntu-latest | |
| if: | |
| github.event_name == 'push' || github.event.pull_request.head.repo.full_name != | |
| github.repository | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.9"] | |
| experimental: [false] | |
| continue-on-error: ${{ matrix.experimental }} | |
| steps: | |
| - name: Checkout filecrawler | |
| uses: actions/checkout@v3 | |
| - name: Install dependencies | |
| uses: awalsh128/cache-apt-pkgs-action@latest | |
| with: | |
| packages: wget curl jq zip | |
| version: 1.0 | |
| - name: Setup Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v3 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install Python dependencies | |
| run: | | |
| python -m pip install --upgrade pip wheel | |
| python -m pip install -r requirements.txt -r tests/requirements-test.txt | |
| - name: Build artifact | |
| run: | | |
| python setup.py sdist | |
| - name: Publish package in Live Environment | |
| #if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') | |
| uses: pypa/gh-action-pypi-publish@release/v1 | |
| with: | |
| password: ${{ secrets.prod_pypi_password }} |