diff --git a/doc/dev/conda-builds.md b/doc/dev/conda-builds.md index aacebc5127ac..152156bf67ad 100644 --- a/doc/dev/conda-builds.md +++ b/doc/dev/conda-builds.md @@ -16,6 +16,21 @@ A Conda Artifact defines: ## How to Build an Azure SDK Conda Package Locally +### Set up your conda environment + +You will notice that all the azure-sdk conda distributions have the **same** version number and requirement set. This is due to the fact that the azure-sdk team pushes our conda packages out in waves. To support this, all versions are set via a common environment variable `AZURESDK_CONDA_VERSION`. + +We keep this environment variable set properly across all our builds by using a common `conda_env.yml` when creating our build environment. This environment definition ensures that: + +1. Our channel `https://azuresdkconda.blob.core.windows.net/channel1/` is added to the set to download packages +2. The environment variable `AZURESDK_CONDA_VERSION` will be set exactly once. + + +Reference the `conda_env.yml` in your local build by pass `-f ` when you create your conda environment. + +``` +conda env create --yes --quiet --name ${{ artifact.name }} -f $(Build.SourcesDirectory)/eng/conda_env.yml +``` ### Create Your Build Directory Given how Conda packages are comprised of multiple source distributions _combined_, the buildable source does not exist directly within the azure-sdk-for-python repo. Currently, there is _some_ manual work that needs to be done. @@ -84,6 +99,8 @@ python `build_conda_artifacts.py` -r "azure/storage" -n "azure-storage" -s "storage" + -e "/eng/conda_env.yml" + -c "" ``` ### Generate the Conda Package diff --git a/eng/conda_env.yml b/eng/conda_env.yml new file mode 100644 index 000000000000..960c090a9339 --- /dev/null +++ b/eng/conda_env.yml @@ -0,0 +1,2 @@ +variables: + AZURESDK_CONDA_VERSION: '2021.05.01' diff --git a/eng/pipelines/templates/steps/build-conda-artifacts.yml b/eng/pipelines/templates/steps/build-conda-artifacts.yml index 024898a4ea66..bcaec17a22e3 100644 --- a/eng/pipelines/templates/steps/build-conda-artifacts.yml +++ b/eng/pipelines/templates/steps/build-conda-artifacts.yml @@ -53,21 +53,28 @@ steps: displayName: 'Build Source Distribution for ${{ artifact.name }}' inputs: scriptPath: 'scripts/devops_tasks/build_conda_artifacts.py' - arguments: '-d "$(conda.output)" -b "$(conda.build)" -m "$(Build.SourcesDirectory)/sdk/${{ parameters.ServiceDirectory }}/${{ artifact.meta_source }}" -r "${{ artifact.common_root }}" -n "${{ artifact.name }}" -s "${{ parameters.ServiceDirectory }}" -o "${{ upper(parameters.ServiceDirectory) }}_SOURCE_DISTRIBUTION"' + arguments: >- + -d "$(conda.output)" + -b "$(conda.build)" + -m "$(Build.SourcesDirectory)/sdk/${{ parameters.ServiceDirectory }}/${{ artifact.meta_source }}" + -r "${{ artifact.common_root }}" + -n "${{ artifact.name }}" + -s "${{ parameters.ServiceDirectory }}" + -e "$(Build.SourcesDirectory)/eng/conda_env.yml" + -c "$(Build.SourcesDirectory)/sdk/${{ parameters.ServiceDirectory }}/ci.yml" - bash: | echo "##vso[task.prependpath]$CONDA/bin" displayName: 'Prepend PATH with Conda and INIT' - bash: | - conda create --yes --quiet --name ${{ artifact.name }} - source activate ${{ artifact.name }} + conda env create --name ${{ artifact.name }} --file $(Build.SourcesDirectory)/eng/conda_env.yml conda install --yes --quiet --name ${{ artifact.name }} conda-build displayName: 'Prepare Conda Environment for building ${{ artifact.name }}' - bash: | source activate ${{ artifact.name }} - conda-build . --output-folder "$(Agent.BuildDirectory)/conda/output" -c https://azuresdkconda.blob.core.windows.net/channel1/ + conda-build . --output-folder "$(Agent.BuildDirectory)/conda/output" -c $(AzureSDKCondaChannel) displayName: 'Activate Conda Environment and Build ${{ artifact.name }}' workingDirectory: $(Build.SourcesDirectory)/sdk/${{ parameters.ServiceDirectory }} diff --git a/eng/pipelines/templates/steps/get-tagged-code.yml b/eng/pipelines/templates/steps/get-tagged-code.yml index e8182f5a5319..34a0e0bf2033 100644 --- a/eng/pipelines/templates/steps/get-tagged-code.yml +++ b/eng/pipelines/templates/steps/get-tagged-code.yml @@ -15,9 +15,7 @@ parameters: steps: - pwsh: | $targetPath = "$(Agent.TempDirectory)/${{ parameters.Package }}" - if (!(Test-Path $targetPath)) { - mkdir $targetPath - } + mkdir -p $targetPath Write-Host "##vso[task.setvariable variable=Package.Clone]$targetPath" displayName: 'Prep for Sparse Checkout' diff --git a/eng/pipelines/templates/variables/globals.yml b/eng/pipelines/templates/variables/globals.yml index a9e52c921478..a949290954e9 100644 --- a/eng/pipelines/templates/variables/globals.yml +++ b/eng/pipelines/templates/variables/globals.yml @@ -1,3 +1,4 @@ variables: PythonVersion: '3.6' skipComponentGovernanceDetection: true + AzureSDKCondaChannel: https://azuresdkconda.blob.core.windows.net/channel1/ \ No newline at end of file diff --git a/scripts/devops_tasks/build_conda_artifacts.py b/scripts/devops_tasks/build_conda_artifacts.py index 8882c8ce0edf..090932f56192 100644 --- a/scripts/devops_tasks/build_conda_artifacts.py +++ b/scripts/devops_tasks/build_conda_artifacts.py @@ -25,12 +25,15 @@ import os import shutil import re +import yaml from common_tasks import process_glob_string, run_check_call, str_to_bool, parse_setup from subprocess import check_call from distutils.dir_util import copy_tree -VERSION_REGEX = re.compile(r"\{\%\s*set\s*version\s*=\s*\"(.*)\"\s*\%\}") +VERSION_REGEX = re.compile(r"\s*AZURESDK_CONDA_VERSION\s*:\s*[\'](.*)[\']\s*") + +SUMMARY_TEMPLATE = "- Generated from {}." NAMESPACE_EXTENSION_TEMPLATE = """__path__ = __import__('pkgutil').extend_path(__path__, __name__) # type: str """ @@ -143,8 +146,8 @@ def create_sdist_skeleton(build_directory, artifact_name, common_root): shutil.copytree(src, dest) -def get_version_from_meta(meta_yaml_location): - with open(os.path.abspath((meta_yaml_location)), "r") as f: +def get_version_from_config(environment_config): + with open(os.path.abspath((environment_config)), "r") as f: lines = f.readlines() for line in lines: result = VERSION_REGEX.match(line) @@ -165,7 +168,9 @@ def get_manifest_includes(common_root): return breadcrumbs -def create_setup_files(build_directory, common_root, artifact_name, service, meta_yaml): +def create_setup_files( + build_directory, common_root, artifact_name, service, meta_yaml, environment_config +): sdist_directory = os.path.join(build_directory, artifact_name) setup_location = os.path.join(sdist_directory, "setup.py") manifest_location = os.path.join(sdist_directory, "MANIFEST.in") @@ -173,7 +178,7 @@ def create_setup_files(build_directory, common_root, artifact_name, service, met setup_template = CONDA_PKG_SETUP_TEMPLATE.format( conda_package_name=artifact_name, - version=get_version_from_meta(meta_yaml), + version=get_version_from_config(environment_config), service=service, package_excludes="'azure', 'tests', '{}'".format(common_root.replace("/", ".")), ) @@ -182,7 +187,9 @@ def create_setup_files(build_directory, common_root, artifact_name, service, met f.write(setup_template) manifest_template = MANIFEST_TEMPLATE.format( - namespace_includes="\n".join(["include " + ns for ns in get_manifest_includes(common_root)]) + namespace_includes="\n".join( + ["include " + ns for ns in get_manifest_includes(common_root)] + ) ) with open(manifest_location, "w") as f: @@ -193,7 +200,13 @@ def create_setup_files(build_directory, common_root, artifact_name, service, met def create_combined_sdist( - output_directory, build_directory, artifact_name, common_root, service, meta_yaml + output_directory, + build_directory, + artifact_name, + common_root, + service, + meta_yaml, + environment_config, ): singular_dependency = ( len(get_pkgs_from_build_directory(build_directory, artifact_name)) == 0 @@ -202,7 +215,12 @@ def create_combined_sdist( if not singular_dependency: create_sdist_skeleton(build_directory, artifact_name, common_root) create_setup_files( - build_directory, common_root, artifact_name, service, meta_yaml + build_directory, + common_root, + artifact_name, + service, + meta_yaml, + environment_config, ) sdist_location = os.path.join(build_directory, artifact_name) @@ -222,6 +240,28 @@ def create_combined_sdist( return output_location +def get_summary(ci_yml, artifact_name): + pkg_list = [] + with open(ci_yml, "r") as f: + data = f.read() + + config = yaml.safe_load(data) + + conda_artifact = [ + conda_artifact + for conda_artifact in config["extends"]["parameters"]["CondaArtifacts"] + if conda_artifact["name"] == artifact_name + ] + + if conda_artifact: + dependencies = conda_artifact[0]["checkout"] + + for dep in dependencies: + pkg_list.append("{}=={}".format(dep["package"], dep["version"])) + + return SUMMARY_TEMPLATE.format(", ".join(pkg_list)) + + if __name__ == "__main__": parser = argparse.ArgumentParser( description="Build a Conda Package, given a properly formatted build directory, and input configuration. This script assumes that the build directory has been set up w/ the necessary sdists in each location." @@ -276,11 +316,19 @@ def create_combined_sdist( ) parser.add_argument( - "-o", - "--output_var", - dest="output_var", - help="The name of the environment variable that will be set in azure devops. The contents will be the final location of the output artifact. Local users will need to grab this value and set their env manually.", - required=False, + "-e", + "--environment_config", + dest="environment_config", + help="The location of the yml config file used to create the conda environments. This file has necessary common configuration information within.", + required=True, + ) + + parser.add_argument( + "-c", + "--ci_yml", + dest="ci_yml", + help="The location of the ci.yml that is used to define our conda artifacts. Used when to easily grab summary information.", + required=True, ) args = parser.parse_args() @@ -291,11 +339,21 @@ def create_combined_sdist( args.common_root, args.service, args.meta_yml, + args.environment_config, ) - if args.output_var: + summary = get_summary(args.ci_yml, args.artifact_name) + + if output_source_location: + print( + "##vso[task.setvariable variable={}]{}".format( + args.service.upper() + "_SOURCE_DISTRIBUTION", output_source_location + ) + ) + + if summary: print( "##vso[task.setvariable variable={}]{}".format( - args.output_var, output_source_location + args.service.upper() + "_SUMMARY", summary ) ) diff --git a/sdk/core/meta.yaml b/sdk/core/meta.yaml index 57260f435fa7..7d0ca118dd8a 100644 --- a/sdk/core/meta.yaml +++ b/sdk/core/meta.yaml @@ -1,9 +1,8 @@ {% set name = "azure-core" %} -{% set version = "2021.05.01" %} package: name: "{{ name|lower }}" - version: "{{ version }}" + version: {{ environ.get('AZURESDK_CONDA_VERSION', '0.0.0') }} source: url: {{ environ.get('CORE_SOURCE_DISTRIBUTION', '') }} @@ -39,10 +38,10 @@ about: license: MIT license_family: MIT license_file: - summary: "Microsoft Azure Core Library for Python" + summary: Microsoft Azure Core Library for Python{{ environ.get('CORE_SUMMARY', '') }} doc_url: dev_url: extra: recipe-maintainers: - - lmazuel,xiangyan99 + - lmazuel,xiangyan99,scbedd diff --git a/sdk/storage/meta.yaml b/sdk/storage/meta.yaml index 676f37b987a9..4f311a6e090e 100644 --- a/sdk/storage/meta.yaml +++ b/sdk/storage/meta.yaml @@ -1,9 +1,8 @@ {% set name = "azure-storage" %} -{% set version = "2021.05.01" %} package: name: "{{ name|lower }}" - version: "{{ version }}" + version: {{ environ.get('AZURESDK_CONDA_VERSION', '0.0.0') }} source: url: {{ environ.get('STORAGE_SOURCE_DISTRIBUTION', '') }} @@ -17,14 +16,14 @@ build: requirements: host: - - azure-core >=2021.05.01 + - azure-core >={{ environ.get('AZURESDK_CONDA_VERSION', '0.0.0') }} - cryptography >=2.1.4 - msrest >=2021.05.01 - pip - python - aiohttp run: - - azure-core >=2021.05.01 + - azure-core >={{ environ.get('AZURESDK_CONDA_VERSION', '0.0.0') }} - cryptography >=2.1.4 - msrest >=2021.05.01 - python @@ -42,11 +41,11 @@ about: home: "https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/storage" license: MIT license_family: MIT - license_file: - summary: "Microsoft Azure Storage Client Library for Python" + license_file: + summary: Microsoft Azure Storage Client Library for Python{{ environ.get('STORAGE_SUMMARY', '') }} doc_url: dev_url: extra: recipe-maintainers: - - your-github-id-here + - lmazuel,xiangyan99,scbedd