(scio-smb) Support mixed FileOperations per BucketedInput #1097
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This file was automatically generated by sbt-github-actions using the | |
| # githubWorkflowGenerate task. You should add and commit this file to | |
| # your git repository. It goes without saying that you shouldn't edit | |
| # this file by hand! Instead, if you wish to make changes, you should | |
| # change your sbt build configuration to revise the workflow description | |
| # to meet your needs, then regenerate this file. | |
| name: Continuous Integration | |
| on: | |
| pull_request: | |
| branches: [main] | |
| push: | |
| branches: [main] | |
| tags: [v*] | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| concurrency: | |
| group: ${{ github.workflow }} @ ${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| build: | |
| name: Build and Test | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest] | |
| scala: [2.13, 2.12] | |
| java: [corretto@11, corretto@17, corretto@21] | |
| exclude: | |
| - scala: 2.12 | |
| java: corretto@17 | |
| - scala: 2.12 | |
| java: corretto@21 | |
| runs-on: ${{ matrix.os }} | |
| timeout-minutes: 60 | |
| steps: | |
| - name: Checkout current branch (full) | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Setup Java (corretto@11) | |
| id: setup-java-corretto-11 | |
| if: matrix.java == 'corretto@11' | |
| uses: actions/setup-java@v3 | |
| with: | |
| distribution: corretto | |
| java-version: 11 | |
| cache: sbt | |
| - name: sbt update | |
| if: matrix.java == 'corretto@11' && steps.setup-java-corretto-11.outputs.cache-hit == 'false' | |
| run: sbt +update | |
| - name: Setup Java (corretto@17) | |
| id: setup-java-corretto-17 | |
| if: matrix.java == 'corretto@17' | |
| uses: actions/setup-java@v3 | |
| with: | |
| distribution: corretto | |
| java-version: 17 | |
| cache: sbt | |
| - name: sbt update | |
| if: matrix.java == 'corretto@17' && steps.setup-java-corretto-17.outputs.cache-hit == 'false' | |
| run: sbt +update | |
| - name: Setup Java (corretto@21) | |
| id: setup-java-corretto-21 | |
| if: matrix.java == 'corretto@21' | |
| uses: actions/setup-java@v3 | |
| with: | |
| distribution: corretto | |
| java-version: 21 | |
| cache: sbt | |
| - name: sbt update | |
| if: matrix.java == 'corretto@21' && steps.setup-java-corretto-21.outputs.cache-hit == 'false' | |
| run: sbt +update | |
| - name: Check that workflows are up to date | |
| run: sbt githubWorkflowCheck | |
| - name: Check headers and formatting | |
| if: matrix.java == 'corretto@11' && matrix.os == 'ubuntu-latest' | |
| run: sbt '++ ${{ matrix.scala }}' headerCheckAll scalafmtCheckAll 'project /' scalafmtSbtCheck | |
| - name: Test | |
| run: sbt '++ ${{ matrix.scala }}' test | |
| - name: Check binary compatibility | |
| if: matrix.java == 'corretto@11' && matrix.os == 'ubuntu-latest' | |
| run: sbt '++ ${{ matrix.scala }}' mimaReportBinaryIssues | |
| - name: Generate API documentation | |
| if: matrix.java == 'corretto@11' && matrix.os == 'ubuntu-latest' | |
| run: sbt '++ ${{ matrix.scala }}' doc | |
| - name: Check dependencies | |
| run: sbt '++ ${{ matrix.scala }}' undeclaredCompileDependenciesTest unusedCompileDependenciesTest | |
| - name: Make target directories | |
| if: github.event_name != 'pull_request' && (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main') | |
| run: mkdir -p integration/target scio-tensorflow/target site/target scio-cassandra/cassandra3/target scio-jmh/target scio-elasticsearch/es8/target scio-jdbc/target scio-macros/target scio-grpc/target scio-elasticsearch/common/target scio-test/target scio-avro/target scio-elasticsearch/es7/target scio-redis/target scio-extra/target scio-google-cloud-platform/target scio-smb/target scio-neo4j/target scio-parquet/target scio-core/target scio-repl/target project/target | |
| - name: Compress target directories | |
| if: github.event_name != 'pull_request' && (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main') | |
| run: tar cf targets.tar integration/target scio-tensorflow/target site/target scio-cassandra/cassandra3/target scio-jmh/target scio-elasticsearch/es8/target scio-jdbc/target scio-macros/target scio-grpc/target scio-elasticsearch/common/target scio-test/target scio-avro/target scio-elasticsearch/es7/target scio-redis/target scio-extra/target scio-google-cloud-platform/target scio-smb/target scio-neo4j/target scio-parquet/target scio-core/target scio-repl/target project/target | |
| - name: Upload target directories | |
| if: github.event_name != 'pull_request' && (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main') | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: target-${{ matrix.os }}-${{ matrix.java }}-${{ matrix.scala }} | |
| path: targets.tar | |
| publish: | |
| name: Publish Artifacts | |
| needs: [build] | |
| if: github.event_name != 'pull_request' && (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main') | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest] | |
| java: [corretto@11] | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - name: Checkout current branch (full) | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Setup Java (corretto@11) | |
| id: setup-java-corretto-11 | |
| if: matrix.java == 'corretto@11' | |
| uses: actions/setup-java@v3 | |
| with: | |
| distribution: corretto | |
| java-version: 11 | |
| cache: sbt | |
| - name: sbt update | |
| if: matrix.java == 'corretto@11' && steps.setup-java-corretto-11.outputs.cache-hit == 'false' | |
| run: sbt +update | |
| - name: Setup Java (corretto@17) | |
| id: setup-java-corretto-17 | |
| if: matrix.java == 'corretto@17' | |
| uses: actions/setup-java@v3 | |
| with: | |
| distribution: corretto | |
| java-version: 17 | |
| cache: sbt | |
| - name: sbt update | |
| if: matrix.java == 'corretto@17' && steps.setup-java-corretto-17.outputs.cache-hit == 'false' | |
| run: sbt +update | |
| - name: Setup Java (corretto@21) | |
| id: setup-java-corretto-21 | |
| if: matrix.java == 'corretto@21' | |
| uses: actions/setup-java@v3 | |
| with: | |
| distribution: corretto | |
| java-version: 21 | |
| cache: sbt | |
| - name: sbt update | |
| if: matrix.java == 'corretto@21' && steps.setup-java-corretto-21.outputs.cache-hit == 'false' | |
| run: sbt +update | |
| - name: Download target directories (2.13) | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: target-${{ matrix.os }}-${{ matrix.java }}-2.13 | |
| - name: Inflate target directories (2.13) | |
| run: | | |
| tar xf targets.tar | |
| rm targets.tar | |
| - name: Download target directories (2.12) | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: target-${{ matrix.os }}-${{ matrix.java }}-2.12 | |
| - name: Inflate target directories (2.12) | |
| run: | | |
| tar xf targets.tar | |
| rm targets.tar | |
| - name: Package repl | |
| run: sbt scio-repl/assembly | |
| - name: Publish | |
| env: | |
| SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }} | |
| SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }} | |
| SONATYPE_CREDENTIAL_HOST: ${{ secrets.SONATYPE_CREDENTIAL_HOST }} | |
| run: sbt tlCiRelease | |
| - name: Upload Repl | |
| uses: softprops/action-gh-release@v1 | |
| with: | |
| files: scio-repl/target/scala-2.13/scio-repl.jar | |
| draft: true | |
| dependency-submission: | |
| name: Submit Dependencies | |
| if: github.event_name != 'pull_request' | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest] | |
| java: [corretto@11] | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - name: Checkout current branch (full) | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Setup Java (corretto@11) | |
| id: setup-java-corretto-11 | |
| if: matrix.java == 'corretto@11' | |
| uses: actions/setup-java@v3 | |
| with: | |
| distribution: corretto | |
| java-version: 11 | |
| cache: sbt | |
| - name: sbt update | |
| if: matrix.java == 'corretto@11' && steps.setup-java-corretto-11.outputs.cache-hit == 'false' | |
| run: sbt +update | |
| - name: Setup Java (corretto@17) | |
| id: setup-java-corretto-17 | |
| if: matrix.java == 'corretto@17' | |
| uses: actions/setup-java@v3 | |
| with: | |
| distribution: corretto | |
| java-version: 17 | |
| cache: sbt | |
| - name: sbt update | |
| if: matrix.java == 'corretto@17' && steps.setup-java-corretto-17.outputs.cache-hit == 'false' | |
| run: sbt +update | |
| - name: Setup Java (corretto@21) | |
| id: setup-java-corretto-21 | |
| if: matrix.java == 'corretto@21' | |
| uses: actions/setup-java@v3 | |
| with: | |
| distribution: corretto | |
| java-version: 21 | |
| cache: sbt | |
| - name: sbt update | |
| if: matrix.java == 'corretto@21' && steps.setup-java-corretto-21.outputs.cache-hit == 'false' | |
| run: sbt +update | |
| - name: Submit Dependencies | |
| uses: scalacenter/sbt-dependency-submission@v2 | |
| with: | |
| modules-ignore: scio_2.13 scio_2.12 scio-examples_2.13 scio-examples_2.12 | |
| configs-ignore: test scala-tool scala-doc-tool test-internal | |
| validate-steward: | |
| name: Validate Steward Config | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest] | |
| java: [temurin@11] | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - name: Checkout current branch (fast) | |
| uses: actions/checkout@v4 | |
| - name: Setup Java (temurin@11) | |
| id: setup-java-temurin-11 | |
| if: matrix.java == 'temurin@11' | |
| uses: actions/setup-java@v3 | |
| with: | |
| distribution: temurin | |
| java-version: 11 | |
| - uses: coursier/setup-action@v1 | |
| with: | |
| apps: scala-steward | |
| - run: scala-steward validate-repo-config .scala-steward.conf | |
| coverage: | |
| name: Test Coverage | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest] | |
| scala: [2.13] | |
| java: [corretto@11] | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - name: Checkout current branch (full) | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Setup Java (corretto@11) | |
| id: setup-java-corretto-11 | |
| if: matrix.java == 'corretto@11' | |
| uses: actions/setup-java@v3 | |
| with: | |
| distribution: corretto | |
| java-version: 11 | |
| cache: sbt | |
| - name: sbt update | |
| if: matrix.java == 'corretto@11' && steps.setup-java-corretto-11.outputs.cache-hit == 'false' | |
| run: sbt +update | |
| - name: Test coverage | |
| run: sbt '++ ${{ matrix.scala }}' coverage test coverageAggregate | |
| - name: Upload coverage report | |
| run: 'bash <(curl -s https://codecov.io/bash)' | |
| it-test: | |
| name: Integration Test | |
| if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main' | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest] | |
| scala: [2.13] | |
| java: [corretto@11] | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - name: Checkout current branch (full) | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Setup Java (corretto@11) | |
| id: setup-java-corretto-11 | |
| if: matrix.java == 'corretto@11' | |
| uses: actions/setup-java@v3 | |
| with: | |
| distribution: corretto | |
| java-version: 11 | |
| cache: sbt | |
| - name: sbt update | |
| if: matrix.java == 'corretto@11' && steps.setup-java-corretto-11.outputs.cache-hit == 'false' | |
| run: sbt +update | |
| - name: gcloud auth | |
| uses: google-github-actions/auth@v2 | |
| with: | |
| credentials_json: ${{ secrets.GCP_CREDENTIALS }} | |
| export_environment_variables: true | |
| create_credentials_file: true | |
| - name: Setup GitHub Action | |
| env: | |
| BQ_READ_TIMEOUT: 30000 | |
| CLOUDSQL_SQLSERVER_PASSWORD: ${{ secrets.CLOUDSQL_SQLSERVER_PASSWORD }} | |
| run: scripts/gha_setup.sh | |
| - name: Check headers and formatting | |
| run: sbt '++ ${{ matrix.scala }}' integration/headerCheckAll integration/scalafmtCheckAll | |
| - name: Test | |
| run: sbt '++ ${{ matrix.scala }}' integration/test | |
| site: | |
| name: Generate Site | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest] | |
| scala: [2.13] | |
| java: [corretto@11] | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - name: Checkout current branch (full) | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Setup Java (corretto@11) | |
| id: setup-java-corretto-11 | |
| if: matrix.java == 'corretto@11' | |
| uses: actions/setup-java@v3 | |
| with: | |
| distribution: corretto | |
| java-version: 11 | |
| cache: sbt | |
| - name: sbt update | |
| if: matrix.java == 'corretto@11' && steps.setup-java-corretto-11.outputs.cache-hit == 'false' | |
| run: sbt +update | |
| - name: gcloud auth | |
| uses: google-github-actions/auth@v2 | |
| with: | |
| credentials_json: ${{ secrets.GCP_CREDENTIALS }} | |
| export_environment_variables: true | |
| create_credentials_file: true | |
| - name: Setup GitHub Action | |
| run: scripts/gha_setup.sh | |
| - name: Generate site | |
| env: | |
| SOCCO: true | |
| run: sbt '++ ${{ matrix.scala }}' scio-examples/compile site/makeSite | |
| - name: Publish site | |
| if: github.event_name != 'pull_request' && startsWith(github.ref, 'refs/tags/v') | |
| env: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| publish_dir: site/target | |
| keep_files: true | |
| uses: peaceiris/actions-gh-pages@v3.9.3 |