[NFT Metadata Crawler] Add CDN Prefix to CDN JSON/URI (#9441)

.github/actions/file-change-determinator/action.yaml

@@ -0,0 +1,16 @@
+name: File Change Determinator
+description: Runs the file change determinator (to identify which files changed in a pull request)
+outputs:
+  only_docs_changed:
+    description: "Returns true if only docs files were changed in the pull request"
+    value: ${{ steps.doc_change_determinator.outputs.should_skip }}
+
+runs:
+  using: composite
+  steps:
+    # Run the docs only change determinator
+    - id: doc_change_determinator
+      continue-on-error: true # Avoid skipping any checks if this job fails (see: https://github.com/fkirc/skip-duplicate-actions/issues/301)
+      uses: fkirc/skip-duplicate-actions@v5
+      with:
+        paths_ignore: '["**/*.md", "developer-docs-site/**"]'

.github/actions/general-lints/action.yaml

@@ -0,0 +1,56 @@
+name: General Lints
+description: Runs all general lints. This includes all linters except rust and docs lints.
+inputs:
+  GIT_CREDENTIALS:
+    description: "Optional credentials to pass to git. Useful if you need to pull private repos for dependencies"
+    required: false
+
+runs:
+  using: composite
+  steps:
+    # Checkout the repository
+    - uses: actions/checkout@v3
+
+    # Install shellcheck and run it on the dev_setup.sh script
+    - name: Run shell lints
+      run: |
+        sudo apt-get install shellcheck --assume-yes --no-install-recommends
+        shellcheck scripts/dev_setup.sh
+      shell: bash
+
+    # Run the python setup and run the python SDK linters
+    - name: Run python setup
+      uses: ./.github/actions/python-setup
+      with:
+        pyproject_directory: ecosystem/python/sdk
+
+    # Run the python SDK linters
+    - name: Run ecosystem python lint
+      run: make -C ecosystem/python/sdk fmt && ./scripts/fail_if_modified_files.sh
+      shell: bash
+
+    # Run the python lints and tests
+    - name: Run python lints and tests
+      uses: ./.github/actions/python-lint-tests
+
+    # Setup node
+    - name: Setup node
+      uses: actions/setup-node@v3
+      with:
+        node-version-file: .node-version
+
+    # Setup pnpm
+    - name: Setup pnpm
+      uses: pnpm/action-setup@v2
+      with:
+        version: 8.2.0
+
+    # Install packages for examples and run package build, lint and tests
+    - name: Run ecosystem lint
+      run: |
+        cd ./ecosystem/typescript/sdk/examples/typescript && pnpm install && cd -
+        cd ./ecosystem/typescript/sdk/examples/javascript && pnpm install && cd -
+        cd ./ecosystem/typescript/sdk && pnpm install && cd -
+        cd ./ecosystem/typescript/sdk && pnpm lint && cd -
+        cd ./ecosystem/typescript/sdk && pnpm fmt:check && cd -
+      shell: bash

.github/actions/python-lint-tests/action.yaml

@@ -0,0 +1,48 @@
+name: Run Python Tests
+description: Runs all Python tests
+inputs:
+  GIT_SHA:
+    description: "Optional git sha to checkout"
+    required: false
+    type: string
+
+runs:
+  using: composite
+  steps:
+    - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # pin@v3
+      with:
+        ref: ${{ inputs.GIT_SHA }}
+        # Get enough commits to compare to
+        fetch-depth: 100
+
+    - name: Get changed files
+      id: changed-files
+      uses: tj-actions/changed-files@60f4aabced9b4718c75acef86d42ffb631c4403a # pin@v29.0.3
+
+    - uses: ./.github/actions/python-setup
+      with:
+        pyproject_directory: testsuite
+
+    - name: Should run tests
+      run: ./testrun determinator.py changed-files --github-output-key SHOULD_RUN --pattern 'testsuite/.*py' ${{steps.changed-files.outputs.all_changed_files }}
+      id: should-run-tests
+      working-directory: testsuite
+      shell: bash
+
+    - name: Run python static type checker
+      if: steps.should-run-tests.outputs.SHOULD_RUN == 'true'
+      run: poetry run pyright
+      working-directory: testsuite
+      shell: bash
+
+    - name: Run python fmt
+      if: steps.should-run-tests.outputs.SHOULD_RUN == 'true'
+      run: poetry run black --check --diff .
+      working-directory: testsuite
+      shell: bash
+
+    - name: Run python unit tests
+      if: steps.should-run-tests.outputs.SHOULD_RUN == 'true'
+      run: find . -name '*test.py' | xargs poetry run python -m unittest
+      working-directory: testsuite
+      shell: bash

.github/actions/rust-lints/action.yaml

@@ -0,0 +1,27 @@
+name: Rust Lints
+description: Runs all Rust linters
+inputs:
+  GIT_CREDENTIALS:
+    description: "Optional credentials to pass to git. Useful if you need to pull private repos for dependencies"
+    required: false
+
+runs:
+  using: composite
+  steps:
+    # Checkout the repository and setup the rust toolchain
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0 # get all the history because cargo xtest --change-since origin/main requires it.
+    - uses: aptos-labs/aptos-core/.github/actions/rust-setup@main
+      with:
+        GIT_CREDENTIALS: ${{ inputs.GIT_CREDENTIALS }}
+
+    # Run the pre-commit checks
+    - uses: pre-commit/action@v3.0.0
+
+    # Run the rust linters and cargo checks
+    - name: Run cargo sort and rust lint checks
+      shell: bash
+      run: |
+        cargo install cargo-sort
+        scripts/rust_lint.sh --check

.github/actions/rust-setup/action.yaml

@@ -21,7 +21,7 @@ runs:
 
     - name: install protoc and related tools
       shell: bash
-      run: scripts/dev_setup.sh -b -r
+      run: scripts/dev_setup.sh -b -r -y -P -J
 
     - run: echo "/home/runner/.cargo/bin" | tee -a $GITHUB_PATH
       shell: bash

.github/actions/rust-smoke-tests/action.yaml

@@ -0,0 +1,49 @@
+name: Rust Smoke Tests
+description: Runs all Rust smoke tests
+inputs:
+  GIT_CREDENTIALS:
+    description: "Optional credentials to pass to git. Useful if you need to pull private repos for dependencies"
+    required: false
+
+runs:
+  using: composite
+  steps:
+    # Checkout the repository and setup the rust toolchain
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0 # get all the history because cargo xtest --change-since origin/main requires it.
+    - uses: aptos-labs/aptos-core/.github/actions/rust-setup@main
+      with:
+        GIT_CREDENTIALS: ${{ inputs.GIT_CREDENTIALS }}
+
+    # Install nextest
+    - uses: taiki-e/install-action@v1.5.6
+      with:
+        tool: nextest
+
+    # Run a postgres database
+    - name: Run postgres database
+      run: docker run --detach -p 5432:5432 cimg/postgres:14.2
+      shell: bash
+
+    # Run the rust smoke tests
+    - name: Run rust smoke tests
+      # Prebuild the aptos-node binary so that tests don't start before the node is built.
+      # Also, prebuild the aptos-node binary as a separate step to avoid feature unification issues.
+      # Note: --test-threads is intentionally set to reduce resource contention in ci jobs. Increasing this, increases job failures and retries.
+      run: cargo build --locked --package=aptos-node --features=failpoints,indexer --release && LOCAL_SWARM_NODE_RELEASE=1 cargo nextest run --release --profile ci --package smoke-test --test-threads 6 --retries 3
+      shell: bash
+      env:
+        INDEXER_DATABASE_URL: postgresql://postgres@localhost/postgres
+
+    # We always try to create the artifact, but it only creates on flaky or failed smoke tests -- when the directories are empty.
+    - name: Upload smoke test logs for failed and flaky tests
+      if: ${{ failure() || success() }}
+      uses: actions/upload-artifact@v3
+      with:
+        name: failed-smoke-test-logs
+        # Retain all smoke test data except for the db (which may be large).
+        path: |
+          /tmp/.tmp*
+          !/tmp/.tmp*/**/db/
+        retention-days: 14

.github/actions/rust-unit-tests/action.yaml

@@ -0,0 +1,51 @@
+name: Rust Unit Tests
+description: Runs all Rust unit tests
+inputs:
+  GIT_CREDENTIALS:
+    description: "Optional credentials to pass to git. Useful if you need to pull private repos for dependencies"
+    required: false
+
+runs:
+  using: composite
+  steps:
+    # Checkout the repository and setup the rust toolchain
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0 # get all the history because cargo xtest --change-since origin/main requires it.
+    - uses: aptos-labs/aptos-core/.github/actions/rust-setup@main
+      with:
+        GIT_CREDENTIALS: ${{ inputs.GIT_CREDENTIALS }}
+
+    # Check the VM features
+    - name: Check VM features
+      run: cargo test --locked --features check-vm-features -p aptos-node
+      shell: bash
+
+    # Run the rust doc tests
+    - name: Run rust doc tests
+      run: cargo test --locked --doc --workspace --exclude aptos-node-checker
+      shell: bash
+
+    # Install nextest
+    - uses: taiki-e/install-action@v1.5.6
+      with:
+        tool: nextest
+
+    # Run a postgres database
+    - name: Run postgres database
+      run: docker run --detach -p 5432:5432 cimg/postgres:14.2
+      shell: bash
+
+    # Run the rust unit tests
+    - name: Run all unit tests
+      run: cargo nextest run --profile ci --locked --workspace --exclude smoke-test --exclude aptos-testcases --retries 3 --no-fail-fast
+      shell: bash
+      env:
+        INDEXER_DATABASE_URL: postgresql://postgres@localhost/postgres
+        RUST_MIN_STACK: 4297152
+        MVP_TEST_ON_CI: true
+        SOLC_EXE: /home/runner/bin/solc
+        Z3_EXE: /home/runner/bin/z3
+        CVC5_EXE: /home/runner/bin/cvc5
+        DOTNET_ROOT: /home/runner/.dotnet
+        BOOGIE_EXE: /home/runner/.dotnet/tools/boogie