diff --git a/.github/workflows/add-collection-test.yml b/.github/workflows/add-collection-test.yml deleted file mode 100644 index 9cd69436..00000000 --- a/.github/workflows/add-collection-test.yml +++ /dev/null @@ -1,156 +0,0 @@ -# This is a test pipeline, that verifies and updates the associations in cmr -name: Add Collection Test -# Controls when the workflow will run -on: - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - -jobs: - # First job in the workflow installs and verifies the software - build: - name: Test Execution - # The type of runner that the job will run on - runs-on: ubuntu-latest - steps: - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: 3.9 - - name: Create Output Directory - run: mkdir test_output - - name: Set environment variables - run: | - echo "UAT_OUTPUT_FILE=test_output/output/uat_output_json.txt" >> $GITHUB_ENV - echo "OPS_OUTPUT_FILE=test_output/output/ops_output_json.txt" >> $GITHUB_ENV - echo "OUTPUT_DIR=test_output/output" >> $GITHUB_ENV - echo "UAT_SUCCESS_FILE=uat_success.txt" >> $GITHUB_ENV - echo "OPS_SUCCESS_FILE=ops_success.txt" >> $GITHUB_ENV - echo "UAT_FAILURE_FILE=uat_fail.txt" >> $GITHUB_ENV - echo "OPS_FAILURE_FILE=ops_fail.txt" >> $GITHUB_ENV - echo "SLACK_REPORT_CONTENT=" >> $GITHUB_ENV - - name: Set environment variables for notebook - run: | - echo "OPS_USERNAME=${{ secrets.EDL_USER }}" >> $GITHUB_ENV - echo "OPS_PASSWORD=${{ secrets.EDL_PASS }}" >> $GITHUB_ENV - echo "UAT_USERNAME=${{ secrets.EDL_USER }}" >> $GITHUB_ENV - echo "UAT_PASSWORD=${{ secrets.EDL_PASS }}" >> $GITHUB_ENV - - name: Create prerequisites - run: | - mkdir ${{ env.OUTPUT_DIR }} - realpath ${{ env.OUTPUT_DIR }} - - name: Install dependencies - run: | - pip3 install --upgrade pip - pip3 install black - pip3 install netCDF4 - pip3 install git+https://github.com/nasa/harmony-py.git - pip3 install git+https://github.com/podaac/cmr-umm-updater.git - pip3 install git+https://github.com/podaac/cmr-association-diff.git - pip3 install python-dateutil --upgrade - - name: Run CMR Association diff scripts - run: | - cmr_association_diff -e uat -t service -a "cmr/uat_associations.txt" -p POCLOUD -n 'PODAAC Concise' -o ${{ env.UAT_OUTPUT_FILE }} --token ${{ secrets.LAUNCHPAD_TOKEN_UAT }} - cmr_association_diff -e ops -t service -a "cmr/ops_associations.txt" -p POCLOUD -n 'PODAAC Concise' -o ${{ env.OPS_OUTPUT_FILE }} --token ${{ secrets.LAUNCHPAD_TOKEN_OPS }} - - name: Run Add Collection Test - run: | - python3 add_collection_test.py -e uat -i ${{ env.UAT_OUTPUT_FILE }} -o ${{ env.OUTPUT_DIR }} - python3 add_collection_test.py -e ops -i ${{ env.OPS_OUTPUT_FILE }} -o ${{ env.OUTPUT_DIR }} - - name: Check UAT files - id: check_UAT_output_files - run: | - echo "::set-output name=SUCCESS_EXISTS::$(if [ -f ${{ env.OUTPUT_DIR }}/${{ env.UAT_SUCCESS_FILE }} ]; then echo "true"; else echo "false"; fi)" - echo "::set-output name=FAIL_EXISTS::$(if [ -f ${{ env.OUTPUT_DIR }}/${{ env.UAT_FAILURE_FILE }} ]; then echo "true"; else echo "false"; fi)" - echo "::set-output name=SUCCESS_CONTENT::$(cat ${{ env.OUTPUT_DIR }}/${{ env.UAT_SUCCESS_FILE }} | sed 's/$/,/' | tr '\n' ' ')" - echo "::set-output name=FAIL_CONTENT::$(cat ${{ env.OUTPUT_DIR }}/${{ env.UAT_FAILURE_FILE }} | sed 's/$/,/' | tr '\n' ' ')" - echo - echo - echo "Success File: $(if [ -f ${{ env.OUTPUT_DIR }}/${{ env.UAT_SUCCESS_FILE }} ]; then echo "true"; else echo "false"; fi)" - echo "Failed File: $(if [ -f ${{ env.OUTPUT_DIR }}/${{ env.UAT_FAILURE_FILE }} ]; then echo "true"; else echo "false"; fi)" - echo "Success Content: $(cat ${{ env.OUTPUT_DIR }}/${{ env.UAT_SUCCESS_FILE }} | sed 's/$/,/' | tr '\n' ' ')" - echo - echo "Failed Content: $(cat ${{ env.OUTPUT_DIR }}/${{ env.UAT_FAILURE_FILE }} | sed 's/$/,/' | tr '\n' ' ')" - - name: Check OPS files - id: check_OPS_output_files - run: | - echo "::set-output name=SUCCESS_EXISTS::$(if [ -f ${{ env.OUTPUT_DIR }}/${{ env.OPS_SUCCESS_FILE }} ]; then echo "true"; else echo "false"; fi)" - echo "::set-output name=FAIL_EXISTS::$(if [ -f ${{ env.OUTPUT_DIR }}/${{ env.OPS_FAILURE_FILE }} ]; then echo "true"; else echo "false"; fi)" - echo "::set-output name=SUCCESS_CONTENT::$(cat ${{ env.OUTPUT_DIR }}/${{ env.OPS_SUCCESS_FILE }} | sed 's/$/,/' | tr '\n' ' ')" - echo "::set-output name=FAIL_CONTENT::$(cat ${{ env.OUTPUT_DIR }}/${{ env.OPS_FAILURE_FILE }} | sed 's/$/,/' | tr '\n' ' ')" - echo - echo - echo "Success File: $(if [ -f ${{ env.OUTPUT_DIR }}/${{ env.OPS_SUCCESS_FILE }} ]; then echo "true"; else echo "false"; fi)" - echo "Failed File: $(if [ -f ${{ env.OUTPUT_DIR }}/${{ env.OPS_FAILURE_FILE }} ]; then echo "true"; else echo "false"; fi)" - echo "Success Content: $(cat ${{ env.OUTPUT_DIR }}/${{ env.OPS_SUCCESS_FILE }} | sed 's/$/,/' | tr '\n' ' ')" - echo - echo "Failed Content: $(cat ${{ env.OUTPUT_DIR }}/${{ env.OPS_FAILURE_FILE }} | sed 's/$/,/' | tr '\n' ' ')" - - name: Slack Report update - Add info for UAT check - SUCCESS - if: | - steps.check_UAT_output_files.outputs.SUCCESS_EXISTS == 'true' - run: | - echo "SLACK_REPORT_CONTENT=${{ env.SLACK_REPORT_CONTENT }}\ncollection-test UAT result: SUCCESS\nSuccessfully ran Concise Notebook on the following collections:\n${{ steps.check_UAT_output_files.outputs.SUCCESS_CONTENT }}\n" >> $GITHUB_ENV - - name: Slack Report update - Add info for UAT check - FAILURE - if: | - steps.check_UAT_output_files.outputs.FAIL_EXISTS == 'true' - run: | - echo "SLACK_REPORT_CONTENT=${{ env.SLACK_REPORT_CONTENT }}\ncollection-test UAT result: FAILURE\nFailed to run Concise Notebook on the following collections:\n${{ steps.check_UAT_output_files.outputs.FAIL_CONTENT }}\n" >> $GITHUB_ENV - - name: Slack Report update - Add info for OPS check - SUCCESS - if: | - steps.check_OPS_output_files.outputs.SUCCESS_EXISTS == 'true' - run: | - echo "SLACK_REPORT_CONTENT=${{ env.SLACK_REPORT_CONTENT }}\ncollection-test OPS result: SUCCESS\nSuccessfully ran Concise Notebook on the following collections:\n${{ steps.check_OPS_output_files.outputs.SUCCESS_CONTENT }}\n" >> $GITHUB_ENV - - name: Slack Report update - Add info for OPS check - FAILURE - if: | - steps.check_OPS_output_files.outputs.FAIL_EXISTS == 'true' - run: | - echo "SLACK_REPORT_CONTENT=${{ env.SLACK_REPORT_CONTENT }}\ncollection-test OPS result: FAILURE\nFailed to run Concise Notebook on the following collections:\n${{ steps.check_OPS_output_files.outputs.FAIL_CONTENT }}\n" >> $GITHUB_ENV - - name: Slack Report update - Add action link on Failure - if: | - steps.check_UAT_output_files.outputs.FAIL_EXISTS == 'true' || - steps.check_OPS_output_files.outputs.FAIL_EXISTS == 'true' - run: | - echo "SLACK_REPORT_CONTENT=${{ env.SLACK_REPORT_CONTENT }}\nExecution link:\ngithub.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" >> $GITHUB_ENV - echo ${{ env.SLACK_REPORT_CONTENT }} - - name: Send Report to Slack - uses: slackapi/slack-github-action@v1.21.0 - if: | - env.SLACK_REPORT_CONTENT != '' - with: - payload: | - { - "status": "${{ env.SLACK_REPORT_CONTENT }}" - } - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_BOT_WEBHOOK }} - - name: Remove UAT collections on FAIL - if: | - steps.check_UAT_output_files.outputs.FAIL_EXISTS == 'true' - run: | - python3 "remove_association.py" -p POCLOUD -n 'PODAAC Concise' -e uat -i '${{ env.OUTPUT_DIR }}/${{ env.UAT_FAILURE_FILE }}' --token ${{ secrets.LAUNCHPAD_TOKEN_UAT }} - - name: Remove OPS collections on FAIL - if: | - steps.check_OPS_output_files.outputs.FAIL_EXISTS == 'true' - run: | - python3 "remove_association.py" -p POCLOUD -n 'PODAAC Concise' -e ops -i '${{ env.OUTPUT_DIR }}/${{ env.OPS_FAILURE_FILE }}' --token ${{ secrets.LAUNCHPAD_TOKEN_OPS }} - - name: Update UAT collections on SUCCESS - if: | - steps.check_UAT_output_files.outputs.SUCCESS_EXISTS == 'true' - # Add a new line to the EOF if there isn't any, so the merging will start in a new line not after the last character - run: | - vi -escwq ./cmr/uat_associations.txt - cat ${{ env.OUTPUT_DIR }}/${{ env.UAT_SUCCESS_FILE }} >> ./cmr/uat_associations.txt - git config user.name "${GITHUB_ACTOR}" - git config user.email "${GITHUB_ACTOR}@users.noreply.github.com" - git commit -am "Update uat_associations.txt with new collections" - git push - - name: Update OPS collections on SUCCESS - if: | - steps.check_OPS_output_files.outputs.SUCCESS_EXISTS == 'true' - # Add a new line to the EOF if there isn't any, so the merging will start in a new line not after the last character - run: | - vi -escwq ./cmr/ops_associations.txt - cat ${{ env.OUTPUT_DIR }}/${{ env.OPS_SUCCESS_FILE }} >> ./cmr/ops_associations.txt - git config user.name "${GITHUB_ACTOR}" - git config user.email "${GITHUB_ACTOR}@users.noreply.github.com" - git commit -am "Update ops_associations.txt with new collections" - git push diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a3884ba..3c8e867c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Updated python libraries ### Deprecated ### Removed + - Removed CMR testing. Now done in [concise-autotest](https://github.com/podaac/concise-autotest) repo + - Moved add_collection_test.py test to verify_collection.py (in concise-autotest repo) ### Fixed diff --git a/add_collection_test.py b/add_collection_test.py deleted file mode 100644 index a3bc1db4..00000000 --- a/add_collection_test.py +++ /dev/null @@ -1,307 +0,0 @@ -import os -from os import path -from urllib.parse import urlparse -import itertools -import unittest -import numpy as np -import netCDF4 as nc -import requests -import json -from harmony import BBox, Client, Collection, Request, Environment -import argparse -from utils import FileHandler -from utils.enums import Venue - - -def parse_args(): - """ - Parses the program arguments - Returns - ------- - args - """ - - parser = argparse.ArgumentParser( - description='Update CMR with latest profile', - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - - parser.add_argument('-e', '--env', - help='CMR environment used to pull results from.', - required=True, - choices=["uat", "ops", "ngap_uat", "ngap_ops"], - metavar='uat or ops') - - parser.add_argument('-n', '--notebook', - help='Notebook to run', - required=False, - metavar='') - - parser.add_argument('-i', '--input_file', - help='File of json collections', - required=True, - metavar='') - - parser.add_argument('-o', '--output_path', - help='output path for success and fails', - required=False, - metavar='') - - args = parser.parse_args() - return args - - -def get_username_and_password(venue): - if venue.lower() == "uat": - return os.environ.get("UAT_USERNAME"), os.environ.get("UAT_PASSWORD") - elif venue.lower() == "ops": - return os.environ.get('OPS_USERNAME'), os.environ.get('OPS_PASSWORD') - else: - raise ValueError("Invalid venue") - - -def get_x_y_variables(variables): - x_var_candidates = ["lon", "longitude", "beam_clon", "sp_lon", "cellon"] - y_var_candidates = ["lat", "latitude", "beam_clat", "sp_lat", "cellat"] - - x_var, y_var = None, None - for var in variables: - if x_var is None and var in x_var_candidates: - x_var = var - if y_var is None and var in y_var_candidates: - y_var = var - if x_var and y_var: - break - - return x_var, y_var - - -def verify_dims(merged_group, origin_group, both_merged): - for dim in origin_group.dimensions: - if both_merged: - unittest.TestCase().assertEqual(merged_group.dimensions[dim].size, origin_group.dimensions[dim].size) - else: - unittest.TestCase().assertGreaterEqual(merged_group.dimensions[dim].size, origin_group.dimensions[dim].size) - - -def verify_attrs(merged_obj, origin_obj, both_merged): - ignore_attributes = [ - 'request-bounding-box', 'request-bounding-box-description', 'PODAAC-dataset-shortname', - 'PODAAC-persistent-ID', 'time_coverage_end', 'time_coverage_start' - ] - - merged_attrs = merged_obj.ncattrs() - origin_attrs = origin_obj.ncattrs() - - for attr in origin_attrs: - if attr in ignore_attributes: - # Skip attributes which are present in the Java implementation, - # but not (currently) present in the Python implementation - continue - - if not both_merged and attr not in merged_attrs: - # Skip attributes which are not present in both merged and origin. - # This is normal operation as some attributes may be omited b/c - # they're inconsistent between granules - continue - - merged_attr = merged_obj.getncattr(attr) - if both_merged and isinstance(merged_attr, int): - # Skip integer values - the Java implementation seems to omit - # these values due to its internal handling of all values as - # Strings - continue - - origin_attr = origin_obj.getncattr(attr) - if isinstance(origin_attr, np.ndarray): - unittest.TestCase().assertTrue(np.array_equal(merged_attr, origin_attr)) - else: - if attr != "history_json": - unittest.TestCase().assertEqual(merged_attr, origin_attr) - - -def verify_variables(merged_group, origin_group, subset_index, both_merged): - for var in origin_group.variables: - merged_var = merged_group.variables[var] - origin_var = origin_group.variables[var] - - verify_attrs(merged_var, origin_var, both_merged) - - if both_merged: - # both groups require subset indexes - merged_data = merged_var[subset_index[0]] - origin_data = origin_var[subset_index[1]] - else: - # merged group requires a subset index - merged_data = np.resize(merged_var[subset_index], origin_var.shape) - origin_data = origin_var - - equal_nan = True - if merged_data.dtype.kind == 'S': - equal_nan = False - - # verify variable data - if isinstance(origin_data, str): - unittest.TestCase().assertEqual(merged_data, origin_data) - else: - unittest.TestCase().assertTrue(np.array_equal(merged_data, origin_data, equal_nan=equal_nan)) - - -def verify_groups(merged_group, origin_group, subset_index, file=None, both_merged=False): - if file: - print("verifying groups ....." + file) - - verify_dims(merged_group, origin_group, both_merged) - verify_attrs(merged_group, origin_group, both_merged) - verify_variables(merged_group, origin_group, subset_index, both_merged) - - for child_group in origin_group.groups: - merged_subgroup = merged_group[child_group] - origin_subgroup = origin_group[child_group] - verify_groups(merged_subgroup, origin_subgroup, subset_index, both_merged=both_merged) - - -# GET TOKEN FROM CMR -def get_token(cmr_root, username, password): - token_api = "https://{}/api/users/tokens".format(cmr_root) - response = requests.get(token_api, auth=(username, password)) - content = response.json() - if len(content) > 0: - return content[0].get('access_token') - else: - create_token_api = "https://{}/api/users/token".format(cmr_root) - response = requests.post(create_token_api, auth=(username, password)) - content = response.json() - return content.get('access_token') - - -def download_file(url, local_path, headers): - response = requests.get(url, stream=True, headers=headers) - if response.status_code == 200: - with open(local_path, 'wb') as file: - for chunk in response.iter_content(chunk_size=8192): - file.write(chunk) - print("Original File downloaded successfully. " + local_path) - else: - print(f"Failed to download the file. Status code: {response.status_code}") - - -def test(collection_id, venue): - - max_results = 2 - - username, password = get_username_and_password(venue) - environment = Environment.UAT if venue.lower() == "uat" else Environment.PROD - harmony_client = Client(auth=(username, password), env=environment) - - collection = Collection(id=collection_id) - - request = Request( - collection=collection, - concatenate=True, - max_results=max_results, - skip_preview=True, - format="application/x-netcdf4", - ) - - request.is_valid() - - print(harmony_client.request_as_curl(request)) - - job1_id = harmony_client.submit(request) - - print(f'\n{job1_id}') - - print(harmony_client.status(job1_id)) - - print('\nWaiting for the job to finish') - - results = harmony_client.result_json(job1_id) - - print('\nDownloading results:') - - futures = harmony_client.download_all(job1_id) - file_names = [f.result() for f in futures] - print('\nDone downloading.') - - filename = file_names[0] - - # Handle time dimension and variables dropping - merge_dataset = nc.Dataset(filename, 'r') - - cmr_base_url = "https://cmr.earthdata.nasa.gov/search/granules.umm_json?readable_granule_name=" - edl_root = 'urs.earthdata.nasa.gov' - - if venue.lower() == 'uat': - cmr_base_url = "https://cmr.uat.earthdata.nasa.gov/search/granules.umm_json?readable_granule_name=" - edl_root = 'uat.urs.earthdata.nasa.gov' - - token = get_token(edl_root, username, password) - headers = { - "Authorization": f"Bearer {token}" - } - - original_files = merge_dataset.variables['subset_files'] - history_json = json.loads(merge_dataset.history_json) - assert len(original_files) == max_results - - for url in history_json[0].get("derived_from"): - local_file_name = os.path.basename(url) - download_file(url, local_file_name, headers) - - for i, file in enumerate(original_files): - origin_dataset = nc.Dataset(file) - verify_groups(merge_dataset, origin_dataset, i, file=file) - - -def run(): - """ - Run from command line. - - Returns - ------- - """ - - _args = parse_args() - environment = _args.env - notebook = _args.notebook - inputFile = _args.input_file - output_location = _args.output_path if _args.output_path else '.' - - success = [] - fails = [] - - if path.exists(inputFile): - venue = Venue.from_str(environment) - collections = FileHandler.get_file_content_list_per_line(inputFile) - print(collections) - # limit number of collections tested to 1 - for collection in itertools.islice(collections, 1): - if "POCLOUD" not in collection and venue == "uat": - continue - - try: - print(collection) - test(collection, venue.name) - success.append(collection) - except Exception as ex: - print(ex) - fails.append(collection) - - # Create output files - if output_location: - success_outfile = path.realpath(f'{output_location}/{_args.env}_success.txt') - fail_outfile = path.realpath(f'{output_location}/{_args.env}_fail.txt') - - if success: - with open(success_outfile, 'w') as the_file: - the_file.writelines(x + '\n' for x in success) - - if fails: - with open(fail_outfile, 'w') as the_file: - the_file.writelines(x + '\n' for x in fails) - - -if __name__ == '__main__': - print("Start running test .......") - run() diff --git a/cmr/ops_associations.txt b/cmr/ops_associations.txt deleted file mode 100644 index 3ff0104d..00000000 --- a/cmr/ops_associations.txt +++ /dev/null @@ -1,101 +0,0 @@ -C1940473819-POCLOUD -C1940475563-POCLOUD -C2205618215-POCLOUD -C2205618339-POCLOUD -C2205121384-POCLOUD -C2205121394-POCLOUD -C2152044763-POCLOUD -C2158344213-POCLOUD -C2254232941-POCLOUD -C2147947806-POCLOUD -C2247621105-POCLOUD -C2158348170-POCLOUD -C2205121315-POCLOUD -C2152046451-POCLOUD -C2158348264-POCLOUD -C2158350299-POCLOUD -C2152045877-POCLOUD -C1996881636-POCLOUD -C2251465126-POCLOUD -C2075141524-POCLOUD -C1940471193-POCLOUD -C2251464384-POCLOUD -C1940472420-POCLOUD -C1940470304-POCLOUD -C2205121449-POCLOUD -C2596983413-POCLOUD -C2596986276-POCLOUD -C2036882072-POCLOUD -C2075141638-POCLOUD -C2075141605-POCLOUD -C2036882055-POCLOUD -C2036880640-POCLOUD -C2036880717-POCLOUD -C2205618975-POCLOUD -C2075141559-POCLOUD -C2251464495-POCLOUD -C2499940513-POCLOUD -C2075141684-POCLOUD -C1996881752-POCLOUD -C2205121520-POCLOUD -C2628593693-POCLOUD -C2601583089-POCLOUD -C2601585875-POCLOUD -C2628598397-POCLOUD -C2601581863-POCLOUD -C2601584109-POCLOUD -C1996881456-POCLOUD -C2036881016-POCLOUD -C2036882492-POCLOUD -C1996881807-POCLOUD -C2036879048-POCLOUD -C2036882397-POCLOUD -C2036882499-POCLOUD -C2036882482-POCLOUD -C1968980583-POCLOUD -C1968979550-POCLOUD -C1968980609-POCLOUD -C1968979566-POCLOUD -C1968980549-POCLOUD -C1968979997-POCLOUD -C1968980576-POCLOUD -C1968979762-POCLOUD -C1968979561-POCLOUD -C1968979597-POCLOUD -C2036880739-POCLOUD -C2036882048-POCLOUD -C2205121485-POCLOUD -C2036882456-POCLOUD -C2274919541-POCLOUD -C2205620319-POCLOUD -C2183155461-POCLOUD -C2208421887-POCLOUD -C2628595723-POCLOUD -C2746966926-POCLOUD -C2628600898-POCLOUD -C2746966928-POCLOUD -C2746966927-POCLOUD -C2754895884-POCLOUD -C2746966657-POCLOUD -C2628598809-POCLOUD -C2799465529-POCLOUD -C2799465526-POCLOUD -C2799465507-POCLOUD -C2799465497-POCLOUD -C2799465538-POCLOUD -C2799465544-POCLOUD -C2799465542-POCLOUD -C2799465428-POCLOUD -C2799438350-POCLOUD -C2799438351-POCLOUD -C2799438353-POCLOUD -C2296989388-POCLOUD -C2205553958-POCLOUD -C2706513160-POCLOUD -C2147480877-POCLOUD -C2147478146-POCLOUD -C2730520815-POCLOUD -C2799465509-POCLOUD -C2799465518-POCLOUD -C2799465522-POCLOUD -C2068529568-POCLOUD diff --git a/cmr/uat_associations.txt b/cmr/uat_associations.txt deleted file mode 100644 index 4f408724..00000000 --- a/cmr/uat_associations.txt +++ /dev/null @@ -1,119 +0,0 @@ -C1234724470-POCLOUD -C1234724471-POCLOUD -C1240739577-POCLOUD -C1240739709-POCLOUD -C1240739734-POCLOUD -C1240739726-POCLOUD -C1242387592-POCLOUD -C1244459498-POCLOUD -C1244810554-POCLOUD -C1242387601-POCLOUD -C1242387600-POCLOUD -C1242387620-POCLOUD -C1242387621-POCLOUD -C1242387586-POCLOUD -C1242387602-POCLOUD -C1242387624-POCLOUD -C1238658389-POCLOUD -C1256420925-POCLOUD -C1234208436-POCLOUD -C1256507988-POCLOUD -C1256783386-POCLOUD -C1234208437-POCLOUD -C1234208438-POCLOUD -C1256122852-POCLOUD -C1256783391-POCLOUD -C1256507989-POCLOUD -C1256507990-POCLOUD -C1240739606-POCLOUD -C1238621112-POCLOUD -C1240739764-POCLOUD -C1240739611-POCLOUD -C1256420924-POCLOUD -C1238657961-POCLOUD -C1240739654-POCLOUD -C1238657959-POCLOUD -C1238687284-POCLOUD -C1238658088-POCLOUD -C1238621087-POCLOUD -C1238687546-POCLOUD -C1240739688-POCLOUD -C1238658080-POCLOUD -C1238658086-POCLOUD -C1238621092-POCLOUD -C1238658051-POCLOUD -C1238658049-POCLOUD -C1238621172-POCLOUD -C1238657960-POCLOUD -C1238658050-POCLOUD -C1238687534-POCLOUD -C1240817851-POCLOUD -C1240739719-POCLOUD -C1238538233-POCLOUD -C1240739704-POCLOUD -C1238543223-POCLOUD -C1238538241-POCLOUD -C1234071416-POCLOUD -C1238538232-POCLOUD -C1238538231-POCLOUD -C1238538224-POCLOUD -C1238538225-POCLOUD -C1238621176-POCLOUD -C1238621182-POCLOUD -C1238621178-POCLOUD -C1238538230-POCLOUD -C1238538240-POCLOUD -C1238543220-POCLOUD -C1238621111-POCLOUD -C1238687282-POCLOUD -C1238658392-POCLOUD -C1240739768-POCLOUD -C1238621118-POCLOUD -C1238658052-POCLOUD -C1238621219-POCLOUD -C1242735870-POCLOUD -C1245295751-POCLOUD -C1238621088-POCLOUD -C1238621115-POCLOUD -C1256445396-POCLOUD -C1238621102-POCLOUD -C1240739713-POCLOUD -C1243175554-POCLOUD -C1245295750-POCLOUD -C1256783381-POCLOUD -C1259115177-POCLOUD -C1256783388-POCLOUD -C1259115167-POCLOUD -C1259115178-POCLOUD -C1256783382-POCLOUD -C1259115166-POCLOUD -C1261072655-POCLOUD -C1261072658-POCLOUD -C1261072648-POCLOUD -C1261072646-POCLOUD -C1261072656-POCLOUD -C1261072645-POCLOUD -C1261072659-POCLOUD -C1261072654-POCLOUD -C1254854453-LARC_CLOUD -C1254855648-LARC_CLOUD -C1254854962-LARC_CLOUD -C1247485682-LARC_CLOUD -C1247485690-LARC_CLOUD -C1247485685-LARC_CLOUD -C1242274079-POCLOUD -C1240739526-POCLOUD -C1261072651-POCLOUD -C1261072650-POCLOUD -C1242274070-POCLOUD -C1240739691-POCLOUD -C1257081729-POCLOUD -C1261072661-POCLOUD -C1261072652-POCLOUD -C1261072662-POCLOUD -C1261072660-POCLOUD -C1261645986-LARC_CLOUD -C1258237266-POCLOUD -C1259966654-POCLOUD -C1258237267-POCLOUD -C1240739686-POCLOUD diff --git a/remove_association.py b/remove_association.py deleted file mode 100644 index 8166f225..00000000 --- a/remove_association.py +++ /dev/null @@ -1,101 +0,0 @@ -import argparse -import cmr - -from podaac.umms_updater.util import create_assoc -from podaac.umms_updater.util import svc_update - -def parse_args(): - """ - Parses the program arguments - Returns - ------- - args - """ - - parser = argparse.ArgumentParser( - description='Remove collection association from CMR', - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - - parser.add_argument('-e', '--env', - help='CMR environment used to pull results from.', - required=False, - choices=["uat", "ops", "sit"], - metavar='uat or ops') - - parser.add_argument('-i', '--input_file', - help='File of json collections to remove', - required=False, - metavar='') - - parser.add_argument('-t', '--token', - help='Launchpad token string', - default=None, - required=True, - metavar='Launchpad token') - - parser.add_argument('-p', '--provider', - help='A provider ID identifies a provider and is' - 'composed of a combination of upper case' - ' letters, digits, and underscores.' - 'The maximum length is 10 characters.' - 'Concept ID is provided if UMM-S record already' - 'exists and needs to be updated.', - required=False, - default='POCLOUD', - metavar='POCLOUD') - - parser.add_argument('-n', '--umm_name', - help='Name of the umm tool or service', - required=True, - metavar='') - - - args = parser.parse_args() - return args - -def run(): - """ - Run from command line. - - Returns - ------- - """ - - _args = parse_args() - - collection_json = _args.input_file - environment = _args.env - service_name = _args.umm_name - - header = { - 'Content-type': "application/json", - 'Authorization': str(_args.token), - } - url_prefix = svc_update.cmr_environment_url(_args.env) - - mode = cmr.queries.CMR_OPS - if environment == 'uat': - mode = cmr.queries.CMR_UAT - - service_query = cmr.queries.ServiceQuery(mode=mode).provider(_args.provider).name(service_name).get() - service_concept_id = service_query[0].get('concept_id') - - with open(collection_json) as afile: - assoc_concept_ids = afile.readlines() - exception_happen = False - failed_remove = [] - for i, assoc_concept_id in enumerate(assoc_concept_ids): - collection_concept_id = assoc_concept_id.strip('\n') - print(f"Removing {collection_concept_id} from CMR association") - resp = create_assoc.remove_association(url_prefix, service_concept_id, collection_concept_id, header) - print(f'Response: {resp}') - if resp.status_code != 200: - exception_happen = True - failed_remove.append(assoc_concept_ids) - if exception_happen: - print(failed_remove) - raise Exception("Failure to remove a collection from CMR") - -if __name__ == '__main__': - run()