azure-pipelines.yml

# We use the "Secure Development Tools" which needs to be installed in any DevOps organization that use 
# this YAML file. The free public Azure Pipelines for OSS includes these tools by default already.

# TODO: add ESLint once it's added to the "Secure Development Tools". TSLint is depricated.

variables:
- name: ubuntu_image
  value: ubuntu-18.04
- name: mac_image
  value: macOS-10.14
- name: windows_image
  value: windows-2019

jobs:
- job: 'Build_ebm_native'
  strategy:
    matrix:
      Linux:
        image.name: '${{ variables.ubuntu_image }}'
      Mac:
        image.name: '${{ variables.mac_image }}'
      Windows:
        image.name: '${{ variables.windows_image }}'
    maxParallel: 3
  pool:
    vmImage: '$(image.name)'
  variables:
    runCodesignValidationInjection: false
  steps:
  - script: |
      /bin/sh ./build.sh -32bit
    condition: startsWith(variables['image.name'], 'ubuntu')
    displayName: 'Building ebm_native code (Linux)'
  - script: |
      /bin/sh ./build.sh
    condition: startsWith(variables['image.name'], 'macOS')
    displayName: 'Building ebm_native code (Mac)'
  - script: |
      set PATH=%PATH%;C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin\
      .\build.bat -32bit
    condition: startsWith(variables['image.name'], 'windows')
    displayName: 'Building ebm_native code (Windows)'
  - task: ComponentGovernanceComponentDetection@0
    displayName: 'Scan dependencies for incompatible licenses and security issues'
  - task: CredScan@2
    displayName: 'Scan codebase for leaked secrets'
    condition: startsWith(variables['image.name'], 'windows')
#  - task: BinSkim@3
#    displayName: 'Scan DLLs for compiler/linker security improvements'
#    condition: startsWith(variables['image.name'], 'windows')
#    inputs:
#      InputType: Basic
#      AnalyzeTarget: 'staging\*.dll'
#      AnalyzeSymPath: 'staging'
#      AnalyzeVerbose: true
#  - task: AntiMalware@3
#    displayName: 'AntiMalware scan'
  - task: SdtReport@1
    displayName: 'Generate security report'
    condition: startsWith(variables['image.name'], 'windows')
    inputs:
      TsvFile: false
      AllTools: false
      BinSkim: false
      CredScan: true
      MSRD: false
      RoslynAnalyzers: false
      TSLint: false
      ToolLogsNotFoundAction: 'Standard'
  - task: PublishSecurityAnalysisLogs@2
    displayName: 'Publish security report'
    condition: startsWith(variables['image.name'], 'windows')
    inputs:
      ArtifactName: 'CodeAnalysisLogs'
      ArtifactType: 'Container'
      AllTools: true
      ToolLogsNotFoundAction: 'Standard'
  - task: PostAnalysis@1
    displayName: 'Stop on security errors'
    condition: startsWith(variables['image.name'], 'windows')
    inputs:
      AllTools: false
      BinSkim: false
      CredScan: true
      RoslynAnalyzers: false
      TSLint: false
      ToolLogsNotFoundAction: 'Standard'
  - task: CopyFiles@2
    condition: succeeded()
    inputs:
      sourceFolder: staging
      contents: '*'
      targetFolder: '$(Build.ArtifactStagingDirectory)'
    displayName: 'Move binary to staging'
  - task: PublishBuildArtifacts@1
    condition: succeeded()
    inputs:
      pathtoPublish: $(Build.ArtifactStagingDirectory)
      artifactName: ebm_native-$(image.name)
    displayName: 'Publish ebm_native library'

- job: 'Test_ebm_native'
  dependsOn: 'Build_ebm_native'
  strategy:
    matrix:
      Linux:
        image.name: '${{ variables.ubuntu_image }}'
      Mac:
        image.name: '${{ variables.mac_image }}'
      Windows:
        image.name: '${{ variables.windows_image }}'
    maxParallel: 3
  pool:
    vmImage: '$(image.name)'
  variables:
    runCodesignValidationInjection: false
  steps:
  - task: DownloadBuildArtifacts@0
    inputs:
      buildType: 'current'
      downloadType: 'specific'
    displayName: 'Download build artifacts'
  - task: CopyFiles@2
    condition: succeeded()
    inputs:
      sourceFolder: '$(System.ArtifactsDirectory)'
      contents: '**/lib_ebm_native_*'
      targetFolder: 'staging'
      flattenFolders: true
    displayName: 'Move binary to staging directory'
  - script: |
      /bin/sh ./shared/ebm_native/ebm_native_test/ebm_native_test.sh -nobuildebmnative
    condition: startsWith(variables['image.name'], 'ubuntu')
    displayName: 'Testing ebm_native code (Linux)'
  - script: |
      /bin/sh ./shared/ebm_native/ebm_native_test/ebm_native_test.sh -nobuildebmnative
    condition: startsWith(variables['image.name'], 'macOS')
    displayName: 'Testing ebm_native code (Mac)'
  - script: |
      set PATH=%PATH%;C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin\
      .\shared\ebm_native\ebm_native_test\ebm_native_test.bat -nobuildebmnative
    condition: startsWith(variables['image.name'], 'windows')
    displayName: 'Testing ebm_native code (Windows)'

- job: 'Build_JS_Inline'
  pool:
    vmImage: '${{ variables.ubuntu_image }}'
  variables:
    runCodesignValidationInjection: false
  steps:
  - task: UsePythonVersion@0
    inputs:
      versionSpec: '3.6'
      architecture: 'x64'
  - script: python build-js-bundles.py devops
    workingDirectory: 'python/scripts'
    displayName: 'Build JS bundle'
  - task: CopyFiles@2
    condition: succeeded()
    inputs:
      sourceFolder: 'python/interpret-core/interpret/lib'
      contents: 'interpret-inline.js'
      targetFolder: '$(Build.ArtifactStagingDirectory)'
    displayName: 'Move assets to staging'
  - task: PublishBuildArtifacts@1
    condition: succeeded()
    inputs:
      pathtoPublish: $(Build.ArtifactStagingDirectory)
      artifactName: interpret-inline-bundle
    displayName: 'Publish interpret-inline.js library'

- job: 'Build_Package'
  dependsOn: ['Build_ebm_native', 'Build_JS_Inline']
  pool:
    vmImage: '${{ variables.ubuntu_image }}'
  variables:
    runCodesignValidationInjection: false
  steps:
  - task: DownloadBuildArtifacts@0
    inputs:
      buildType: 'current'
      downloadType: 'specific'
    displayName: 'Download build artifacts'
  - task: UsePythonVersion@0
    condition: succeeded()
    inputs:
      versionSpec: '3.6'
      architecture: 'x64'
  - script: python -m pip install --upgrade pip setuptools wheel
    condition: succeeded()
    displayName: 'Install tools'
  - task: CopyFiles@2
    condition: succeeded()
    inputs:
      sourceFolder: '$(System.ArtifactsDirectory)'
      contents: '**/lib_ebm_native_*_x64*'
      targetFolder: 'python/interpret-core/interpret/lib'
      flattenFolders: true
    displayName: 'Move binary to Python layer'
  - task: CopyFiles@2
    condition: succeeded()
    inputs:
      sourceFolder: '$(System.ArtifactsDirectory)'
      contents: '**/interpret-inline.js'
      targetFolder: 'python/interpret-core/interpret/lib'
      flattenFolders: true
    displayName: 'Move interpret-inline.js to Python layer'
  - script: python setup.py bdist_wheel -d ../../staging
    condition: succeeded()
    workingDirectory: 'python/interpret-core'
    displayName: 'Build wheel (interpret-core)'
  - script: python setup.py bdist_wheel -d ../../staging
    condition: succeeded()
    workingDirectory: 'python/interpret'
    displayName: 'Build wheel (interpret)'
  - task: CopyFiles@2
    condition: succeeded()
    inputs:
      sourceFolder: staging
      contents: '*.whl'
      targetFolder: '$(Build.ArtifactStagingDirectory)/wheel'
    displayName: 'Move wheel for Build Artifact'
  - task: PublishBuildArtifacts@1
    condition: succeeded()
    inputs:
      pathtoPublish: '$(Build.ArtifactStagingDirectory)/wheel'
      artifactName: 'wheel'
    displayName: 'Publish wheel as Build Artifact'

- job: 'Test'
  dependsOn: 'Build_Package'
  # NOTE: Python 3.9 not supported yet in Azure DevOps.
  strategy:
    matrix:
      LinuxPython36:
        python.version: '3.6'
        image.name: '${{ variables.ubuntu_image }}'
      LinuxPython37:
        python.version: '3.7'
        image.name: '${{ variables.ubuntu_image }}'
      LinuxPython38:
        python.version: '3.8'
        image.name: '${{ variables.ubuntu_image }}'
      # LinuxPython39:
      #   python.version: '3.9'
      #   image.name: '${{ variables.ubuntu_image }}'
      WindowsPython36:
        python.version: '3.6'
        image.name: '${{ variables.windows_image }}'
      WindowsPython37:
        python.version: '3.7'
        image.name: '${{ variables.windows_image }}'
      # NOTE: Temporary disable due to upstream dependencies.
      # WindowsPython38:
      #   python.version: '3.8'
      #   image.name: '${{ variables.windows_image }}'
      # WindowsPython39:
      #   python.version: '3.9'
      #   image.name: '${{ variables.windows_image }}'
      MacPython36:
        python.version: '3.6'
        image.name: '${{ variables.mac_image }}'
      MacPython37:
        python.version: '3.7'
        image.name: '${{ variables.mac_image }}'
      MacPython38:
        python.version: '3.8'
        image.name: '${{ variables.mac_image }}'
      # MacPython39:
      #   python.version: '3.9'
      #   image.name: '${{ variables.mac_image }}'
    maxParallel: 9
  pool:
    vmImage: '$(image.name)'
  variables:
    runCodesignValidationInjection: false
  steps:
  - task: DownloadBuildArtifacts@0
    inputs:
      buildType: 'current'
      downloadType: 'specific'
    displayName: 'Download build artifacts'
  - task: CopyFiles@2
    condition: succeeded()
    inputs:
      sourceFolder: '$(System.ArtifactsDirectory)'
      contents: '**/lib_ebm_native_*'
      targetFolder: 'python/interpret-core/interpret/lib'
      flattenFolders: true
    displayName: 'Move binary to Python layer'
  - task: CopyFiles@2
    condition: succeeded()
    inputs:
      sourceFolder: '$(System.ArtifactsDirectory)'
      contents: '**/interpret-inline.js'
      targetFolder: 'python/interpret-core/interpret/lib'
      flattenFolders: true
    displayName: 'Move Interpret inline to Python layer'
  - task: UsePythonVersion@0
    condition: succeeded()
    inputs:
      versionSpec: '$(python.version)'
      architecture: 'x64'
  - script: python -m pip install --upgrade pip setuptools wheel
    condition: succeeded()
    displayName: 'Install tools'
  - script: python -m pip install numpy scipy
    condition: succeeded()
    workingDirectory: python/interpret-core
    displayName: 'Install numpy/scipy first for mis-specified pip packages.'
  - script: |
      mkdir -p ~/.matplotlib
      echo "backend: TkAgg" >> ~/.matplotlib/matplotlibrc
    condition: startsWith(variables['image.name'], 'macOS')
    displayName: 'Matplotlib patch for mac.'
  - script: python -m pip install -r dev-requirements.txt
    condition: succeeded()
    workingDirectory: python/interpret-core
    displayName: 'Install requirements (Linux/Mac/Windows)'
  - script: |
      set PATH=%PATH%;%GeckoWebDriver%
      python -m pytest -vv -n auto --runslow --runselenium --doctest-modules --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html
    workingDirectory: python/interpret-core
    condition: and(startsWith(variables['image.name'], 'windows'), or(eq(variables['Build.Reason'], 'Schedule'), in(variables['Build.SourceBranch'], 'refs/heads/master', 'refs/heads/develop')))
    displayName: 'Run pytest (Windows scheduled / master / develop)'
  - script: |
      python -m pytest -vv -n auto --runslow --doctest-modules --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html
    workingDirectory: python/interpret-core
    condition: not(and(startsWith(variables['image.name'], 'windows'), or(eq(variables['Build.Reason'], 'Schedule'), in(variables['Build.SourceBranch'], 'refs/heads/master', 'refs/heads/develop'))))
    displayName: 'Run pytest (Rest)'
  - task: PublishTestResults@2
    condition: succeededOrFailed()
    inputs:
      testResultsFiles: '**/test-*.xml'
      testRunTitle: 'Publish test results for Python $(python.version) at $(image.name)'
    displayName: 'Publish test results'
  - task: PublishCodeCoverageResults@1
    inputs:
      codeCoverageTool: Cobertura
      summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/coverage.xml'
      reportDirectory: '$(System.DefaultWorkingDirectory)/**/htmlcov'
    condition: startsWith(variables['image.name'], 'windows')
    displayName: 'Publish test coverage results'

- job: 'Publish_Package'
  dependsOn: ['Test_ebm_native', 'Test']
  pool:
    vmImage: '${{ variables.ubuntu_image }}'
  variables:
    runCodesignValidationInjection: false
  steps:
  - task: DownloadBuildArtifacts@0
    inputs:
      buildType: 'current'
      downloadType: 'single'
      artifactName: 'wheel'
  - task: DownloadBuildArtifacts@0
    inputs:
      buildType: 'current'
      downloadType: 'single'
      artifactName: 'interpret-inline-bundle'
  - task: PublishPipelineArtifact@0
    inputs:
      sourceFolder: '$(System.ArtifactsDirectory)'
      artifactName: 'drop'

schedules:
- cron: "0 0 * * *"
  displayName: Daily midnight build
  branches:
    include:
    - develop
  always: true