v0.1.14 #17
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # GitHub Action workflow to build and run filecrawler's tests | |
| # | |
| name: Filecrawler release build | |
| on: | |
| release: | |
| types: [published] | |
| env: | |
| LATEST_TAG: filecrawler:latest | |
| jobs: | |
| lint: | |
| name: Check syntaxs errors and warnings | |
| runs-on: ubuntu-latest | |
| if: | |
| github.event_name == 'push' || github.event.pull_request.head.repo.full_name != | |
| github.repository | |
| steps: | |
| - name: Checkout filecrawler | |
| uses: actions/checkout@v3 | |
| - name: Setup Python 3.9 | |
| uses: actions/setup-python@v3 | |
| with: | |
| python-version: 3.9 | |
| - name: Install Python dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| python -m pip install flake8 | |
| - name: Check syntax errors | |
| run: | | |
| flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics | |
| - name: Check PEP8 warnings | |
| run: | | |
| flake8 . --count --ignore=E1,E2,E3,E501,W291,W293 --exit-zero --max-complexity=65 --max-line-length=127 --statistics | |
| binutils_dependencies: | |
| name: Build external dependencies | |
| needs: lint | |
| runs-on: ubuntu-latest | |
| if: | |
| github.event_name == 'push' || github.event.pull_request.head.repo.full_name != | |
| github.repository | |
| strategy: | |
| fail-fast: false | |
| steps: | |
| - name: Install dependencies | |
| uses: awalsh128/cache-apt-pkgs-action@latest | |
| with: | |
| packages: wget curl jq zip | |
| version: 1.0 | |
| - name: Create temp path | |
| run: | | |
| mkdir -p /tmp/filecrawler/bin | |
| - name: Get tika server .jar | |
| run: | | |
| VER=$(curl -s "https://raw.githubusercontent.com/chrismattmann/tika-python/master/tika/tika.py" | grep 'TIKA_VERSION' | grep -oE '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}' || echo '2.6.0') | |
| SERVER_HASH=$(curl -s "http://search.maven.org/remotecontent?filepath=org/apache/tika/tika-server-standard/$VER/tika-server-standard-$VER.jar.sha1") | |
| wget -nv -O "/tmp/filecrawler/bin/tika-server.jar" "http://search.maven.org/remotecontent?filepath=org/apache/tika/tika-server-standard/$VER/tika-server-standard-$VER.jar" | |
| echo "${SERVER_HASH} /tmp/filecrawler/bin/tika-server.jar" | sha1sum -c - || exit 1 | |
| - name: Get JADX | |
| run: | | |
| VER=$(curl -s "https://api.github.com/repos/skylot/jadx/tags" | jq -r '.[0].name' | grep -oE '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}') | |
| wget -nv -O "/tmp/jadx.zip" "https://github.com/skylot/jadx/releases/download/v$VER/jadx-$VER.zip" | |
| mkdir -p /tmp/jadx/ | |
| unzip -o /tmp/jadx.zip -d /tmp/jadx/ | |
| FILE=$(find /tmp/jadx/ -name "jadx*.jar") | |
| mv "$FILE" "/tmp/filecrawler/bin/jadx.jar" | |
| - name: Get APKTool | |
| run: | | |
| VER=$(curl -s "https://api.github.com/repos/iBotPeaches/Apktool/tags" | jq -r '.[0].name' | grep -oE '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}') | |
| wget -nv -O "/tmp/apktool.jar" "https://github.com/iBotPeaches/Apktool/releases/download/v$VER/apktool_$VER.jar" | |
| mv "/tmp/apktool.jar" "/tmp/filecrawler/bin/" | |
| - name: Create bin zip | |
| run: | | |
| zip -j -5 /tmp/filecrawler_extra_libs.zip /tmp/filecrawler/bin/* | |
| - name: Upload binutils release asset | |
| id: filecrawler_extra_libs | |
| uses: actions/upload-release-asset@v1 | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| with: | |
| upload_url: ${{ github.event.release.upload_url }} | |
| asset_path: /tmp/filecrawler_extra_libs.zip | |
| asset_name: filecrawler_extra_libs.zip | |
| asset_content_type: application/zip | |
| build: | |
| name: Build | |
| needs: binutils_dependencies | |
| runs-on: ubuntu-latest | |
| if: | |
| github.event_name == 'push' || github.event.pull_request.head.repo.full_name != | |
| github.repository | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.8", "3.9", "3.10"] | |
| experimental: [false] | |
| include: | |
| - python-version: "3.11" | |
| experimental: true | |
| continue-on-error: ${{ matrix.experimental }} | |
| steps: | |
| - name: Checkout filecrawler | |
| uses: actions/checkout@v3 | |
| - name: Setup Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v3 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install Python dependencies | |
| run: | | |
| python -m pip install --upgrade pip wheel | |
| python -m pip install -r requirements.txt -r tests/requirements-test.txt | |
| - name: Install filecrawler | |
| run: | | |
| pip install . | |
| - name: Filecrawler basic test | |
| run: | | |
| filecrawler -h | |
| - name: Build artifact | |
| run: | | |
| python setup.py sdist | |
| docker_build: | |
| name: Check docker build and run | |
| needs: build | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v2 | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| #- name: Set up Docker Buildx | |
| # uses: docker/setup-buildx-action@v2 | |
| #- name: Build and export to Docker | |
| # uses: docker/build-push-action@v4 | |
| # with: | |
| # context: . | |
| # push: true | |
| # tags: ${{ env.LATEST_TAG }} | |
| #- name: Test | |
| # run: | | |
| # docker run --rm ${{ env.LATEST_TAG }} -h | |
| publish: | |
| name: Publish on Live Environment | |
| needs: docker_build | |
| runs-on: ubuntu-latest | |
| if: | |
| github.event_name == 'push' || github.event.pull_request.head.repo.full_name != | |
| github.repository | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.9"] | |
| experimental: [false] | |
| continue-on-error: ${{ matrix.experimental }} | |
| steps: | |
| - name: Checkout filecrawler | |
| uses: actions/checkout@v3 | |
| - name: Install dependencies | |
| uses: awalsh128/cache-apt-pkgs-action@latest | |
| with: | |
| packages: wget curl jq zip | |
| version: 1.0 | |
| - name: Setup Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v3 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install Python dependencies | |
| run: | | |
| python -m pip install --upgrade pip wheel | |
| python -m pip install -r requirements.txt -r tests/requirements-test.txt | |
| - name: Build artifact | |
| run: | | |
| python setup.py sdist | |
| - name: Publish package in Live Environment | |
| #if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') | |
| uses: pypa/gh-action-pypi-publish@release/v1 | |
| with: | |
| password: ${{ secrets.prod_pypi_password }} |