diff --git a/.github/workflows/build-pipeline.yml b/.github/workflows/build-pipeline.yml index c28f1a0e..62e6b393 100644 --- a/.github/workflows/build-pipeline.yml +++ b/.github/workflows/build-pipeline.yml @@ -243,6 +243,19 @@ jobs: pull: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + + - name: Deploy Harmony + env: + ENV: ${{ env.venue }} + CMR_USER: ${{ secrets.CMR_USER }} + CMR_PASS: ${{ secrets.CMR_PASS }} + if: | + github.ref == 'refs/heads/main' || + startsWith(github.ref, 'refs/heads/release') + working-directory: deployment + run: + poetry run python harmony_deploy.py --tag ${{ env.software_version }} + # As of 2023/01/23 these steps below for scanning the Docker image with Snyk are failing. I've tried both the official Snyk # action https://github.com/snyk/actions/tree/master/docker and this method below of manually calling the CLI. # The error when using the official Snyk action is diff --git a/CHANGELOG.md b/CHANGELOG.md index b676846a..69746444 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,11 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added -### Changed + - Update Github Actions + - Added harmony deployment into github actions. ### Changed ### Deprecated ### Removed ### Fixed + - Variable Merging + - Fixed way we merge variables when granules in a collection have varying variables. ## [0.9.0] diff --git a/deployment/harmony_deploy.py b/deployment/harmony_deploy.py new file mode 100644 index 00000000..38035bef --- /dev/null +++ b/deployment/harmony_deploy.py @@ -0,0 +1,67 @@ +import os +import requests +import json +import logging +import argparse +from requests.auth import HTTPBasicAuth + +# Environment variables +ENV = os.getenv('ENV') +CMR_USER = os.getenv('CMR_USER') +CMR_PASS = os.getenv('CMR_PASS') + +def bearer_token() -> str: + tokens = [] + headers = {'Accept': 'application/json'} + url = f"https://{'uat.' if ENV == 'uat' else ''}urs.earthdata.nasa.gov/api/users" + + # First just try to get a token that already exists + try: + resp = requests.get(url + "/tokens", headers=headers, auth=HTTPBasicAuth(CMR_USER, CMR_PASS)) + response_content = json.loads(resp.content) + + for x in response_content: + tokens.append(x['access_token']) + + except Exception: # noqa E722 + logging.warning("Error getting the token - check user name and password", exc_info=True) + + # No tokens exist, try to create one + if not tokens: + try: + resp = requests.post(url + "/token", headers=headers, auth=HTTPBasicAuth(CMR_USER, CMR_PASS)) + response_content = json.loads(resp.content) + tokens.append(response_content['access_token']) + except Exception: # noqa E722 + logging.warning("Error getting the token - check user name and password", exc_info=True) + + # If still no token, then we can't do anything + if not tokens: + raise RuntimeError("Unable to get bearer token from EDL") + + return next(iter(tokens)) + +if __name__ == "__main__": + + parser = argparse.ArgumentParser(description="Update the service image tag.") + parser.add_argument("--tag", help="The new tag version to update.", required=True) + args = parser.parse_args() + + url = f"https://harmony.{'uat.' if ENV == 'uat' else ''}earthdata.nasa.gov/service-image-tag/podaac-concise" + token = bearer_token() + + headers = { + "Authorization": f"Bearer {token}", + "Content-type": "application/json" + } + data = { + "tag": args.tag + } + + response = requests.put(url, headers=headers, json=data) + + print(response.status_code) + try: + print(response.json()) + except json.JSONDecodeError: + print("Response content is not in JSON format") diff --git a/podaac/merger/preprocess_worker.py b/podaac/merger/preprocess_worker.py index 4bedba33..7dd98022 100644 --- a/podaac/merger/preprocess_worker.py +++ b/podaac/merger/preprocess_worker.py @@ -236,14 +236,10 @@ def _run_multi_core(file_list: list[Path], if var_info is None: var_info = result['var_info'] elif var_info != result['var_info']: - if set(var_info.keys()).difference(result['var_info']): - # If not all variables match, only compare variables that intersect - intersecting_vars = set(var_info).intersection(result['var_info']) - if list( - map(var_info.get, intersecting_vars) - ) != list(map(result['var_info'].get, intersecting_vars)): - raise RuntimeError('Variable schemas are inconsistent between granules') - var_info.update(result['var_info']) + intersecting_vars = set(var_info).intersection(result['var_info']) + if any(var_info[var] != result['var_info'][var] for var in intersecting_vars): + raise RuntimeError('Variable schemas are inconsistent between granules') + var_info.update(result['var_info']) # The following data requires accumulation methods merge_max_dims(max_dims, result['max_dims'])