[iOS] added workflows for libtorchvision_ops.a binary build

ghstack-source-id: e57ef604e70bc5e4b2b360cc2385087b03ade085 Pull Request resolved: #3582
pytorch · Mar 29, 2021 · 8560e84 · 8560e84
1 parent 5fe1449
commit 8560e84
Show file tree

Hide file tree

Showing 8 changed files with 638 additions and 2 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -46,6 +46,59 @@ commands:
             fi
             echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV}
 
+  brew_update:
+    description: "Update Homebrew and install base formulae"
+    steps:
+      - run:
+          name: Update Homebrew
+          no_output_timeout: "10m"
+          command: |
+            set -ex
+
+            # Update repositories manually.
+            # Running `brew update` produces a comparison between the
+            # current checkout and the updated checkout, which takes a
+            # very long time because the existing checkout is 2y old.
+            for path in $(find /usr/local/Homebrew -type d -name .git)
+            do
+            cd $path/..
+            git fetch --depth=1 origin
+            git reset --hard origin/master
+            done
+
+            export HOMEBREW_NO_AUTO_UPDATE=1
+
+            # Install expect and moreutils so that we can call `unbuffer` and `ts`.
+            # moreutils installs a `parallel` executable by default, which conflicts
+            # with the executable from the GNU `parallel`, so we must unlink GNU
+            # `parallel` first, and relink it afterwards.
+            brew install coreutils
+            brew unlink parallel
+            brew install moreutils
+            brew link parallel --overwrite
+            brew install expect
+
+  brew_install:
+    description: "Install Homebrew formulae"
+    parameters:
+      formulae:
+        type: string
+        default: ""
+    steps:
+      - run:
+          name: Install << parameters.formulae >>
+          no_output_timeout: "10m"
+          command: |
+            set -ex
+            export HOMEBREW_NO_AUTO_UPDATE=1
+            brew install << parameters.formulae >>
+
+  run_brew_for_ios_build:
+    steps:
+      - brew_update
+      - brew_install:
+          formulae: libtool
+
 binary_common: &binary_common
   parameters:
     # Edit these defaults to do a release
@@ -83,6 +136,22 @@ binary_common: &binary_common
     UNICODE_ABI: << parameters.unicode_abi >>
     CU_VERSION: << parameters.cu_version >>
 
+torchvision_ios_params: &torchvision_ios_params
+  parameters:
+    build_environment:
+      type: string
+      default: ""
+    ios_arch:
+      type: string
+      default: ""
+    ios_platform:
+      type: string
+      default: ""
+  environment:
+    BUILD_ENVIRONMENT: << parameters.build_environment >>
+    IOS_ARCH: << parameters.ios_arch >>
+    IOS_PLATFORM: << parameters.ios_platform >>
+
 smoke_test_common: &smoke_test_common
   <<: *binary_common
   docker:
@@ -278,6 +347,43 @@ jobs:
           paths:
             - "*"
 
+  binary_ios_build:
+    <<: *torchvision_ios_params
+    macos:
+      xcode: "12.0"
+    steps:
+    - attach_workspace:
+        at: ~/workspace
+    - checkout
+    - run_brew_for_ios_build
+    - run:
+        name: Build
+        no_output_timeout: "1h"
+        command: |
+          script="/Users/distiller/project/.circleci/unittest/ios/scripts/binary_ios_build.sh"
+          cat "$script"
+          source "$script"
+    - persist_to_workspace:
+        root: /Users/distiller/workspace/
+        paths: ios
+
+  binary_ios_upload:
+    <<: *torchvision_ios_params
+    macos:
+      xcode: "12.0"
+    steps:
+    - attach_workspace:
+        at: ~/workspace
+    - checkout
+    - run_brew_for_ios_build
+    - run:
+        name: Upload
+        no_output_timeout: "1h"
+        command: |
+          script="/Users/distiller/project/.circleci/unittest/ios/scripts/binary_ios_upload.sh"
+          cat "$script"
+          source "$script"
+
   binary_macos_conda:
     <<: *binary_common
     macos:
@@ -595,7 +701,7 @@ jobs:
 
           keys:
             - env-v1-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
- 
+
       - run:
           name: Setup
           command: .circleci/unittest/windows/scripts/setup_env.sh
@@ -1356,6 +1462,18 @@ workflows:
       - clang_format
       - torchhub_test
       - torch_onnx_test
+      - binary_ios_build:
+          build_environment: binary-libtorchvision_ops-ios-12.0.0-x86_64
+          context: org-member
+          ios_arch: x86_64
+          ios_platform: SIMULATOR
+          name: binary_libtorchvision_ops_ios_12.0.0_x86_64
+      - binary_ios_build:
+          build_environment: binary-libtorchvision_ops-ios-12.0.0-arm64
+          context: org-member
+          ios_arch: arm64
+          ios_platform: OS
+          name: binary_libtorchvision_ops_ios_12.0.0_arm64
 
   unittest:
     jobs:
@@ -1503,6 +1621,36 @@ workflows:
       - clang_format
       - torchhub_test
       - torch_onnx_test
+      - binary_ios_build:
+          build_environment: nightly-binary-libtorchvision_ops-ios-12.0.0-x86_64
+          context: org-member
+          filters:
+            branches:
+              only:
+              - nightly
+          ios_arch: x86_64
+          ios_platform: SIMULATOR
+          name: nightly_binary_libtorchvision_ops_ios_12.0.0_x86_64
+      - binary_ios_build:
+          build_environment: nightly-binary-libtorchvision_ops-ios-12.0.0-arm64
+          context: org-member
+          filters:
+            branches:
+              only:
+              - nightly
+          ios_arch: arm64
+          ios_platform: OS
+          name: nightly_binary_libtorchvision_ops_ios_12.0.0_arm64
+      - binary_ios_upload:
+          build_environment: nightly-binary-libtorchvision_ops-ios-12.0.0-upload
+          context: org-member
+          filters:
+            branches:
+              only:
+              - nightly
+          requires:
+          - nightly_binary_libtorchvision_ops_ios_12.0.0_x86_64
+          - nightly_binary_libtorchvision_ops_ios_12.0.0_arm64
       - binary_linux_wheel:
           conda_docker_image: pytorch/conda-builder:cpu
           cu_version: cpu

diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in
@@ -46,6 +46,59 @@ commands:
             fi
             echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV}
 
+  brew_update:
+    description: "Update Homebrew and install base formulae"
+    steps:
+      - run:
+          name: Update Homebrew
+          no_output_timeout: "10m"
+          command: |
+            set -ex
+
+            # Update repositories manually.
+            # Running `brew update` produces a comparison between the
+            # current checkout and the updated checkout, which takes a
+            # very long time because the existing checkout is 2y old.
+            for path in $(find /usr/local/Homebrew -type d -name .git)
+            do
+            cd $path/..
+            git fetch --depth=1 origin
+            git reset --hard origin/master
+            done
+
+            export HOMEBREW_NO_AUTO_UPDATE=1
+
+            # Install expect and moreutils so that we can call `unbuffer` and `ts`.
+            # moreutils installs a `parallel` executable by default, which conflicts
+            # with the executable from the GNU `parallel`, so we must unlink GNU
+            # `parallel` first, and relink it afterwards.
+            brew install coreutils
+            brew unlink parallel
+            brew install moreutils
+            brew link parallel --overwrite
+            brew install expect
+
+  brew_install:
+    description: "Install Homebrew formulae"
+    parameters:
+      formulae:
+        type: string
+        default: ""
+    steps:
+      - run:
+          name: Install << parameters.formulae >>
+          no_output_timeout: "10m"
+          command: |
+            set -ex
+            export HOMEBREW_NO_AUTO_UPDATE=1
+            brew install << parameters.formulae >>
+
+  run_brew_for_ios_build:
+    steps:
+      - brew_update
+      - brew_install:
+          formulae: libtool
+
 binary_common: &binary_common
   parameters:
     # Edit these defaults to do a release
@@ -83,6 +136,22 @@ binary_common: &binary_common
     UNICODE_ABI: << parameters.unicode_abi >>
     CU_VERSION: << parameters.cu_version >>
 
+torchvision_ios_params: &torchvision_ios_params
+  parameters:
+    build_environment:
+      type: string
+      default: ""
+    ios_arch:
+      type: string
+      default: ""
+    ios_platform:
+      type: string
+      default: ""
+  environment:
+    BUILD_ENVIRONMENT: << parameters.build_environment >>
+    IOS_ARCH: << parameters.ios_arch >>
+    IOS_PLATFORM: << parameters.ios_platform >>
+
 smoke_test_common: &smoke_test_common
   <<: *binary_common
   docker:
@@ -278,6 +347,43 @@ jobs:
           paths:
             - "*"
 
+  binary_ios_build:
+    <<: *torchvision_ios_params
+    macos:
+      xcode: "12.0"
+    steps:
+    - attach_workspace:
+        at: ~/workspace
+    - checkout
+    - run_brew_for_ios_build
+    - run:
+        name: Build
+        no_output_timeout: "1h"
+        command: |
+          script="/Users/distiller/project/.circleci/unittest/ios/scripts/binary_ios_build.sh"
+          cat "$script"
+          source "$script"
+    - persist_to_workspace:
+        root: /Users/distiller/workspace/
+        paths: ios
+
+  binary_ios_upload:
+    <<: *torchvision_ios_params
+    macos:
+      xcode: "12.0"
+    steps:
+    - attach_workspace:
+        at: ~/workspace
+    - checkout
+    - run_brew_for_ios_build
+    - run:
+        name: Upload
+        no_output_timeout: "1h"
+        command: |
+          script="/Users/distiller/project/.circleci/unittest/ios/scripts/binary_ios_upload.sh"
+          cat "$script"
+          source "$script"
+
   binary_macos_conda:
     <<: *binary_common
     macos:
@@ -595,7 +701,7 @@ jobs:
           {% raw %}
           keys:
             - env-v1-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
-          {% endraw %} 
+          {% endraw %}
       - run:
           name: Setup
           command: .circleci/unittest/windows/scripts/setup_env.sh
@@ -813,6 +919,7 @@ workflows:
       - clang_format
       - torchhub_test
       - torch_onnx_test
+      {{ ios_workflows() }}
 
   unittest:
     jobs:
@@ -832,6 +939,7 @@ workflows:
       - clang_format
       - torchhub_test
       - torch_onnx_test
+      {{ ios_workflows(nightly=True) }}
       {{ build_workflows(prefix="nightly_", filter_branch="nightly", upload=True) }}
   docker_build:
     triggers:

diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py
@@ -261,6 +261,36 @@ def cmake_workflows(indentation=6):
     return indent(indentation, jobs)
 
 
+def ios_workflows(indentation=6, nightly=False):
+    jobs = []
+    build_job_names = []
+    name_prefix = "nightly_" if nightly else ""
+    env_prefix = "nightly-" if nightly else ""
+    for arch, platform in [('x86_64', 'SIMULATOR'), ('arm64', 'OS')]:
+        name = f'{name_prefix}binary_libtorchvision_ops_ios_12.0.0_{arch}'
+        build_job_names.append(name)
+        build_job = {
+            'build_environment': f'{env_prefix}binary-libtorchvision_ops-ios-12.0.0-{arch}',
+            'context': 'org-member',
+            'ios_arch': arch,
+            'ios_platform': platform,
+            'name': name,
+        }
+        if nightly:
+            build_job['filters'] = gen_filter_branch_tree('nightly')
+        jobs.append({'binary_ios_build': build_job})
+
+    if nightly:
+        upload_job = {
+            'build_environment': f'{env_prefix}binary-libtorchvision_ops-ios-12.0.0-upload',
+            'context': 'org-member',
+            'filters': gen_filter_branch_tree('nightly'),
+            'requires': build_job_names,
+        }
+        jobs.append({'binary_ios_upload': upload_job})
+    return indent(indentation, jobs)
+
+
 if __name__ == "__main__":
     d = os.path.dirname(__file__)
     env = jinja2.Environment(
@@ -275,4 +305,5 @@ def cmake_workflows(indentation=6):
             build_workflows=build_workflows,
             unittest_workflows=unittest_workflows,
             cmake_workflows=cmake_workflows,
+            ios_workflows=ios_workflows,
         ))