branched

blbadger · Nov 8, 2023 · 7e93f18 · 7e93f18
1 parent 4987bde
commit 7e93f18
Show file tree

Hide file tree

Showing 976 changed files with 51,755 additions and 6,039 deletions.
diff --git a/.circleci/create_circleci_config.py b/.circleci/create_circleci_config.py
@@ -127,6 +127,8 @@ def to_dict(self):
             },
         ]
         steps.extend([{"run": l} for l in self.install_steps])
+        steps.extend([{"run": 'pip install "fsspec>=2023.5.0,<2023.10.0"'}])
+        steps.extend([{"run": "pip install pytest-subtests"}])
         steps.append(
             {
                 "save_cache": {

diff --git a/.github/conda/meta.yaml b/.github/conda/meta.yaml
@@ -26,6 +26,8 @@ requirements:
     - protobuf
     - tokenizers >=0.11.1,!=0.11.3,<0.13
     - pyyaml >=5.1
+    - safetensors
+    - fsspec
   run:
     - python
     - numpy >=1.17
@@ -40,6 +42,8 @@ requirements:
     - protobuf
     - tokenizers >=0.11.1,!=0.11.3,<0.13
     - pyyaml >=5.1
+    - safetensors
+    - fsspec
 
 test:
   imports:

diff --git a/.github/workflows/build_documentation.yml b/.github/workflows/build_documentation.yml
@@ -15,7 +15,11 @@ jobs:
       commit_sha: ${{ github.sha }}
       package: transformers
       notebook_folder: transformers_doc
+<<<<<<< HEAD
       languages: de en es fr it ko pt zh ja
+=======
+      languages: de en es fr hi it ko pt tr zh ja te
+>>>>>>> 75d0f3122195beedf1f127d8a91bd7fbe64c9109
     secrets:
       token: ${{ secrets.HUGGINGFACE_PUSH }}
       hf_token: ${{ secrets.HF_DOC_BUILD_PUSH }}
diff --git a/.github/workflows/build_pr_documentation.yml b/.github/workflows/build_pr_documentation.yml
@@ -14,4 +14,8 @@ jobs:
       commit_sha: ${{ github.event.pull_request.head.sha }}
       pr_number: ${{ github.event.number }}
       package: transformers
+<<<<<<< HEAD
       languages: de en es fr it ko pt zh ja
+=======
+      languages: de en es fr hi it ko pt tr zh ja te
+>>>>>>> 75d0f3122195beedf1f127d8a91bd7fbe64c9109
diff --git a/.github/workflows/doctests.yml b/.github/workflows/doctests.yml
@@ -20,7 +20,11 @@ env:
 
 jobs:
   run_doctests:
+<<<<<<< HEAD
     runs-on: [single-gpu, nvidia-gpu, t4, doctest-ci]
+=======
+    runs-on: [single-gpu, nvidia-gpu, t4, ci]
+>>>>>>> 75d0f3122195beedf1f127d8a91bd7fbe64c9109
     container:
       image: huggingface/transformers-all-latest-gpu
       options: --gpus 0 --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/

diff --git a/.github/workflows/self-nightly-scheduled.yml b/.github/workflows/self-nightly-scheduled.yml
@@ -21,6 +21,7 @@ env:
   RUN_PT_TF_CROSS_TESTS: 1
 
 jobs:
+<<<<<<< HEAD
   check_runner_status:
     name: Check Runner Status
     runs-on: ubuntu-latest
@@ -48,9 +49,10 @@ jobs:
         run: |
           nvidia-smi
 
+=======
+>>>>>>> 75d0f3122195beedf1f127d8a91bd7fbe64c9109
   setup:
     name: Setup
-    needs: check_runners
     strategy:
       matrix:
         machine_type: [single-gpu, multi-gpu]
@@ -276,8 +278,6 @@ jobs:
     runs-on: ubuntu-latest
     if: always()
     needs: [
-      check_runner_status,
-      check_runners,
       setup,
       run_tests_single_gpu,
       run_tests_multi_gpu,
@@ -288,8 +288,6 @@ jobs:
         shell: bash
         # For the meaning of these environment variables, see the job `Setup`
         run: |
-          echo "Runner availability: ${{ needs.check_runner_status.result }}"
-          echo "Runner status: ${{ needs.check_runners.result }}"
           echo "Setup status: ${{ needs.setup.result }}"
 
       - uses: actions/checkout@v3
@@ -303,8 +301,6 @@ jobs:
           CI_SLACK_REPORT_CHANNEL_ID: ${{ secrets.CI_SLACK_CHANNEL_ID_PAST_FUTURE }}
           ACCESS_REPO_INFO_TOKEN: ${{ secrets.ACCESS_REPO_INFO_TOKEN }}
           CI_EVENT: Nightly CI
-          RUNNER_STATUS: ${{ needs.check_runner_status.result }}
-          RUNNER_ENV_STATUS: ${{ needs.check_runners.result }}
           SETUP_STATUS: ${{ needs.setup.result }}
         # We pass `needs.setup.outputs.matrix` as the argument. A processing in `notification_service.py` to change
         # `models/bert` to `models_bert` is required, as the artifact names use `_` instead of `/`.

diff --git a/.github/workflows/self-past.yml b/.github/workflows/self-past.yml
@@ -32,6 +32,7 @@ env:
   RUN_PT_TF_CROSS_TESTS: 1
 
 jobs:
+<<<<<<< HEAD
   check_runner_status:
     name: Check Runner Status
     runs-on: ubuntu-latest
@@ -59,9 +60,10 @@ jobs:
         run: |
           nvidia-smi
 
+=======
+>>>>>>> 75d0f3122195beedf1f127d8a91bd7fbe64c9109
   setup:
     name: Setup
-    needs: check_runners
     strategy:
       matrix:
         machine_type: [single-gpu, multi-gpu]
@@ -319,8 +321,6 @@ jobs:
     runs-on: ubuntu-latest
     if: always()
     needs: [
-      check_runner_status,
-      check_runners,
       setup,
       run_tests_single_gpu,
       run_tests_multi_gpu,
@@ -331,8 +331,6 @@ jobs:
         shell: bash
         # For the meaning of these environment variables, see the job `Setup`
         run: |
-          echo "Runner availability: ${{ needs.check_runner_status.result }}"
-          echo "Runner status: ${{ needs.check_runners.result }}"
           echo "Setup status: ${{ needs.setup.result }}"
 
       - uses: actions/checkout@v3
@@ -351,8 +349,6 @@ jobs:
           CI_SLACK_REPORT_CHANNEL_ID: ${{ secrets.CI_SLACK_CHANNEL_ID_PAST_FUTURE }}
           ACCESS_REPO_INFO_TOKEN: ${{ secrets.ACCESS_REPO_INFO_TOKEN }}
           CI_EVENT: Past CI - ${{ inputs.framework }}-${{ inputs.version }}
-          RUNNER_STATUS: ${{ needs.check_runner_status.result }}
-          RUNNER_ENV_STATUS: ${{ needs.check_runners.result }}
           SETUP_STATUS: ${{ needs.setup.result }}
         # We pass `needs.setup.outputs.matrix` as the argument. A processing in `notification_service.py` to change
         # `models/bert` to `models_bert` is required, as the artifact names use `_` instead of `/`.

diff --git a/.github/workflows/self-push-amd-mi210-caller.yml b/.github/workflows/self-push-amd-mi210-caller.yml
@@ -0,0 +1,25 @@
+name: Self-hosted runner (AMD mi210 CI caller)
+
+on:
+  workflow_run:
+    workflows: ["Self-hosted runner (push-caller)"]
+    branches: ["main"]
+    types: [completed]
+  push:
+    branches:
+      - run_amd_push_ci_caller*
+    paths:
+      - "src/**"
+      - "tests/**"
+      - ".github/**"
+      - "templates/**"
+      - "utils/**"
+
+jobs:
+  run_amd_ci:
+    name: AMD mi210
+    if: (cancelled() != true) && ((github.event_name != 'schedule') || ((github.event_name == 'push') && startsWith(github.ref_name, 'run_amd_push_ci_caller')))
+    uses: ./.github/workflows/self-push-amd.yml
+    with:
+      gpu_flavor: mi210
+    secrets: inherit
diff --git a/.github/workflows/self-push-amd-mi250-caller.yml b/.github/workflows/self-push-amd-mi250-caller.yml
@@ -0,0 +1,25 @@
+name: Self-hosted runner (AMD mi250 CI caller)
+
+on:
+  workflow_run:
+    workflows: ["Self-hosted runner (push-caller)"]
+    branches: ["main"]
+    types: [completed]
+  push:
+    branches:
+      - run_amd_push_ci_caller*
+    paths:
+      - "src/**"
+      - "tests/**"
+      - ".github/**"
+      - "templates/**"
+      - "utils/**"
+
+jobs:
+  run_amd_ci:
+    name: AMD mi250
+    if: (cancelled() != true) && ((github.event_name != 'schedule') || ((github.event_name == 'push') && startsWith(github.ref_name, 'run_amd_push_ci_caller')))
+    uses: ./.github/workflows/self-push-amd.yml
+    with:
+      gpu_flavor: mi250
+    secrets: inherit
diff --git a/.github/workflows/self-push-amd.yml b/.github/workflows/self-push-amd.yml
@@ -1,6 +1,7 @@
 name: Self-hosted runner AMD GPU (push)
 
 on:
+<<<<<<< HEAD
   workflow_run:
     workflows: ["Self-hosted runner (push-caller)"]
     branches: ["main"]
@@ -16,6 +17,13 @@ on:
       - "templates/**"
       - "utils/**"
   repository_dispatch:
+=======
+  workflow_call:
+    inputs:
+      gpu_flavor:
+        required: true
+        type: string
+>>>>>>> 75d0f3122195beedf1f127d8a91bd7fbe64c9109
 
 env:
   HF_HOME: /mnt/cache
@@ -45,8 +53,12 @@ jobs:
     strategy:
       matrix:
         machine_type: [single-gpu, multi-gpu]
+<<<<<<< HEAD
         gpu_flavor: [mi210]
     runs-on: [self-hosted, docker-gpu, amd-gpu, '${{ matrix.machine_type }}', '${{ matrix.gpu_flavor }}']
+=======
+    runs-on: [self-hosted, docker-gpu, amd-gpu, '${{ matrix.machine_type }}', '${{ inputs.gpu_flavor }}']
+>>>>>>> 75d0f3122195beedf1f127d8a91bd7fbe64c9109
     container:
       image: huggingface/transformers-pytorch-amd-gpu-push-ci  # <--- We test only for PyTorch for now
       options: --device /dev/kfd --device /dev/dri --env HIP_VISIBLE_DEVICES --env ROCR_VISIBLE_DEVICES --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
@@ -65,8 +77,12 @@ jobs:
     strategy:
       matrix:
         machine_type: [single-gpu, multi-gpu]
+<<<<<<< HEAD
         gpu_flavor: [mi210]
     runs-on: [self-hosted, docker-gpu, amd-gpu, '${{ matrix.machine_type }}', '${{ matrix.gpu_flavor }}']
+=======
+    runs-on: [self-hosted, docker-gpu, amd-gpu, '${{ matrix.machine_type }}', '${{ inputs.gpu_flavor }}']
+>>>>>>> 75d0f3122195beedf1f127d8a91bd7fbe64c9109
     container:
       image: huggingface/transformers-pytorch-amd-gpu-push-ci  # <--- We test only for PyTorch for now
       options: --device /dev/kfd --device /dev/dri --env HIP_VISIBLE_DEVICES --env ROCR_VISIBLE_DEVICES --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
@@ -164,8 +180,12 @@ jobs:
       matrix:
         folders: ${{ fromJson(needs.setup_gpu.outputs.matrix) }}
         machine_type: [single-gpu, multi-gpu]
+<<<<<<< HEAD
         gpu_flavor: [mi210]
     runs-on: [self-hosted, docker-gpu, amd-gpu, '${{ matrix.machine_type }}', '${{ matrix.gpu_flavor }}']
+=======
+    runs-on: [self-hosted, docker-gpu, amd-gpu, '${{ matrix.machine_type }}', '${{ inputs.gpu_flavor }}']
+>>>>>>> 75d0f3122195beedf1f127d8a91bd7fbe64c9109
     container:
       image: huggingface/transformers-pytorch-amd-gpu-push-ci  # <--- We test only for PyTorch for now
       options: --device /dev/kfd --device /dev/dri --env HIP_VISIBLE_DEVICES --env ROCR_VISIBLE_DEVICES --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
@@ -321,7 +341,11 @@ jobs:
           CI_SLACK_CHANNEL_DUMMY_TESTS: ${{ secrets.CI_SLACK_CHANNEL_DUMMY_TESTS }}
           CI_SLACK_REPORT_CHANNEL_ID: ${{ secrets.CI_SLACK_CHANNEL_ID_AMD }}
           ACCESS_REPO_INFO_TOKEN: ${{ secrets.ACCESS_REPO_INFO_TOKEN }}
+<<<<<<< HEAD
           CI_EVENT: push
+=======
+          CI_EVENT: Push CI (AMD) - ${{ inputs.gpu_flavor }}
+>>>>>>> 75d0f3122195beedf1f127d8a91bd7fbe64c9109
           CI_TITLE_PUSH: ${{ github.event.head_commit.message }}
           CI_TITLE_WORKFLOW_RUN: ${{ github.event.workflow_run.head_commit.message }}
           CI_SHA: ${{ env.CI_SHA }}

diff --git a/.github/workflows/self-push.yml b/.github/workflows/self-push.yml
@@ -27,6 +27,7 @@ env:
   RUN_PT_TF_CROSS_TESTS: 1
 
 jobs:
+<<<<<<< HEAD
   check_runner_status:
     name: Check Runner Status
     runs-on: ubuntu-latest
@@ -54,9 +55,10 @@ jobs:
         run: |
           nvidia-smi
 
+=======
+>>>>>>> 75d0f3122195beedf1f127d8a91bd7fbe64c9109
   setup:
     name: Setup
-    needs: check_runners
     strategy:
       matrix:
         machine_type: [single-gpu, multi-gpu]
@@ -521,8 +523,6 @@ jobs:
     runs-on: ubuntu-latest
     if: always()
     needs: [
-        check_runner_status,
-        check_runners,
         setup,
         run_tests_single_gpu,
         run_tests_multi_gpu,
@@ -534,9 +534,7 @@ jobs:
         shell: bash
         # For the meaning of these environment variables, see the job `Setup`
         run: |
-          echo "Runner availability: ${{ needs.check_runner_status.result }}"
           echo "Setup status: ${{ needs.setup.result }}"
-          echo "Runner status: ${{ needs.check_runners.result }}"
 
       # Necessary to get the correct branch name and commit SHA for `workflow_run` event
       # We also take into account the `push` event (we might want to test some changes in a branch)
@@ -589,8 +587,6 @@ jobs:
           CI_TITLE_PUSH: ${{ github.event.head_commit.message }}
           CI_TITLE_WORKFLOW_RUN: ${{ github.event.workflow_run.head_commit.message }}
           CI_SHA: ${{ env.CI_SHA }}
-          RUNNER_STATUS: ${{ needs.check_runner_status.result }}
-          RUNNER_ENV_STATUS: ${{ needs.check_runners.result }}
           SETUP_STATUS: ${{ needs.setup.result }}
 
         # We pass `needs.setup.outputs.matrix` as the argument. A processing in `notification_service.py` to change

diff --git a/.github/workflows/self-scheduled.yml b/.github/workflows/self-scheduled.yml
@@ -25,6 +25,7 @@ env:
   RUN_PT_TF_CROSS_TESTS: 1
 
 jobs:
+<<<<<<< HEAD
   check_runner_status:
     name: Check Runner Status
     runs-on: ubuntu-latest
@@ -52,9 +53,10 @@ jobs:
         run: |
           nvidia-smi
 
+=======
+>>>>>>> 75d0f3122195beedf1f127d8a91bd7fbe64c9109
   setup:
     name: Setup
-    needs: check_runners
     strategy:
       matrix:
         machine_type: [single-gpu, multi-gpu]
@@ -430,8 +432,6 @@ jobs:
     runs-on: ubuntu-latest
     if: always()
     needs: [
-      check_runner_status,
-      check_runners,
       setup,
       run_tests_single_gpu,
       run_tests_multi_gpu,
@@ -480,8 +480,6 @@ jobs:
     runs-on: ubuntu-latest
     if: always()
     needs: [
-      check_runner_status,
-      check_runners,
       setup,
       run_tests_single_gpu,
       run_tests_multi_gpu,
@@ -496,8 +494,6 @@ jobs:
         shell: bash
         # For the meaning of these environment variables, see the job `Setup`
         run: |
-          echo "Runner availability: ${{ needs.check_runner_status.result }}"
-          echo "Runner status: ${{ needs.check_runners.result }}"
           echo "Setup status: ${{ needs.setup.result }}"
 
       - uses: actions/checkout@v3
@@ -513,8 +509,6 @@ jobs:
           CI_EVENT: scheduled
           CI_SHA: ${{ github.sha }}
           CI_WORKFLOW_REF: ${{ github.workflow_ref }}
-          RUNNER_STATUS: ${{ needs.check_runner_status.result }}
-          RUNNER_ENV_STATUS: ${{ needs.check_runners.result }}
           SETUP_STATUS: ${{ needs.setup.result }}
         # We pass `needs.setup.outputs.matrix` as the argument. A processing in `notification_service.py` to change
         # `models/bert` to `models_bert` is required, as the artifact names use `_` instead of `/`.