review comments

bin/prepare_submission.py

@@ -19,8 +19,9 @@
 
 from ebi_eva_common_pyutils.logger import logging_config as log_cfg
 
-from eva_submission.submission_config import load_config
+from eva_sub_cli_processing.sub_cli_to_eload_converter.sub_cli_to_eload_converter import SubCLIToEloadConverter
 from eva_submission.eload_preparation import EloadPreparation
+from eva_submission.submission_config import load_config
 
 logger = log_cfg.get_logger(__name__)
 
@@ -32,6 +33,9 @@ def main():
                           help='box number where the data should have been uploaded. Required to copy the data from the FTP')
     argparse.add_argument('--submitter', required=False, type=str,
                           help='the name of the directory for that submitter. Required to copy the data from the FTP')
+    argparse.add_argument('--submission_account_id', required=False, type=str,
+                          help='submission account id of the user which is generally "userId_loginType"')
+    argparse.add_argument('--submission_id', required=False, type=str, help='The Submission Id of the submission')
     argparse.add_argument('--eload', required=True, type=int, help='The ELOAD number for this submission')
     argparse.add_argument('--taxid', required=False, type=str,
                           help='Override and replace the taxonomy id provided in the metadata spreadsheet.')
@@ -49,10 +53,16 @@ def main():
     # Load the config_file from default location
     load_config()
 
-    with EloadPreparation(args.eload) as eload:
-        if args.ftp_box and args.submitter:
-            eload.copy_from_ftp(args.ftp_box, args.submitter)
-        eload.detect_all(args.taxid, args.reference)
+    if args.submission_account_id and args.submission_id:
+        with SubCLIToEloadConverter(args.eload) as sub_cli_eload:
+            sub_cli_eload.retrieve_vcf_files_from_sub_cli_ftp_dir(args.submission_account_id, args.submission_id)
+            sub_cli_eload.download_metadata_json_and_convert_to_xlsx(args.submission_id)
+            sub_cli_eload.detect_all(args.taxid, args.reference)
+    else:
+        with EloadPreparation(args.eload) as eload:
+            if args.ftp_box and args.submitter:
+                eload.copy_from_ftp(args.ftp_box, args.submitter)
+            eload.detect_all(args.taxid, args.reference)
 
 
 if __name__ == "__main__":

eva_sub_cli_processing/sub_cli_to_eload_converter/sub_cli_to_eload_converter.py

@@ -1,11 +1,10 @@
-import json
 import os
 import shutil
 
-import requests
 from ebi_eva_common_pyutils.config import cfg
 
 from eva_sub_cli_processing.sub_cli_to_eload_converter.json_to_xlsx_converter import JsonToXlsxConverter
+from eva_sub_cli_processing.sub_cli_utils import download_metadata_json_file_for_submission_id
 from eva_submission.eload_preparation import EloadPreparation
 from eva_submission.eload_submission import directory_structure
 
@@ -31,28 +30,6 @@ def download_metadata_json_and_convert_to_xlsx(self, submission_id):
         metadata_json_file_path = os.path.join(metadata_dir, "metadata_json.json")
         metadata_xlsx_file_path = os.path.join(metadata_dir, "metadata_xlsx.xlsx")
         # download metadata json
-        self.download_metadata_json(submission_id, metadata_json_file_path)
+        download_metadata_json_file_for_submission_id(submission_id, metadata_json_file_path)
         # convert metadata json to metadata xlsx
         JsonToXlsxConverter().convert_json_to_xlsx(metadata_json_file_path, metadata_xlsx_file_path)
-
-    def download_metadata_json(self, submission_id, metadata_json_file_path):
-        url = f"{cfg['submissions']['webservice']['url']}/admin/submission/{submission_id}"
-        username = cfg['submissions']['webservice']['admin_username']
-        password = cfg['submissions']['webservice']['admin_password']
-        response = requests.get(url, auth=(username, password))
-        if response.status_code == 200:
-            try:
-                response_data = response.json()
-                metadata_json_data = response_data['metadataJson']
-                if metadata_json_data:
-                    with open(metadata_json_file_path, "w", encoding="utf-8") as file:
-                        json.dump(metadata_json_data, file, indent=4)
-
-                    self.info(f"Metadata JSON file downloaded successfully: {metadata_json_file_path}")
-                else:
-                    raise ValueError("Metadata json file download: missing metadata_json field in the response")
-            except Exception as e:
-                raise Exception(f"Metadata json file download: Error processing response from the server: {e}")
-        else:
-            raise Exception(f"Failed to download metadata json file for submission id {submission_id}. "
-                            f"Status code: {response.status_code}")

eva_sub_cli_processing/sub_cli_utils.py

@@ -1,3 +1,5 @@
+import json
+
 import requests
 from ebi_eva_common_pyutils.config import cfg
 from retry import retry
@@ -53,3 +55,19 @@ def put_to_sub_ws(url):
     response = requests.put(url, auth=sub_ws_auth())
     response.raise_for_status()
     return response.json()
+
+
+def get_metadata_json_for_submission_id(submission_id):
+    submission_details_url = sub_ws_url_build("admin", "submission", submission_id)
+    response_data = get_from_sub_ws(submission_details_url)
+    metadata_json_data = response_data.get('metadataJson', {})
+    if metadata_json_data:
+        return metadata_json_data
+    else:
+        raise ValueError("Metadata json retrieval: missing metadata_json field in the response")
+
+
+def download_metadata_json_file_for_submission_id(submission_id, metadata_json_file_path):
+    metadata_json_data = get_metadata_json_for_submission_id(submission_id)
+    with open(metadata_json_file_path, "w", encoding="utf-8") as file:
+        json.dump(metadata_json_data, file, indent=4)

tests/test_sub_cli_processing/test_sub_cli_to_eload_converter.py

@@ -68,31 +68,6 @@ def test_retrieve_vcf_files_from_sub_cli_ftp(self):
         self.cli_to_eload.retrieve_vcf_files_from_sub_cli_ftp_dir('webin123_webin', 'abcdef_ghijkl_mnopqr_stuvwx')
         assert os.listdir(os.path.join(self.cli_to_eload.eload_dir, '10_submitted', 'vcf_files')) == ['data.vcf.gz']
 
-    @patch("requests.get")
-    def test_download_metadata_json(self, mock_requests):
-        mock_response = mock_requests.return_value
-        mock_response.status_code = 200
-        input_json_file = os.path.join(self.resources_folder, 'input_json_for_json_to_xlsx_converter.json')
-        with open(input_json_file) as json_file:
-            input_json_data = json.load(json_file)
-        mock_response.json.return_value = {"metadataJson": input_json_data}
-
-        submission_id = "submission123"
-        metadata_json_file_path = os.path.join(self.cli_to_eload.eload_dir, '10_submitted', 'metadata_file',
-                                               'metadata_json.json')
-        self.cli_to_eload.download_metadata_json(submission_id, metadata_json_file_path)
-
-        # Check if requests.get was called with the correct URL
-        mock_requests.assert_called_once_with(f"{cfg['submissions']['webservice']['url']}/admin/submission/{submission_id}",
-                                              auth=(cfg['submissions']['webservice']['admin_username'],
-                                                    cfg['submissions']['webservice']['admin_password']))
-
-        # Check if file was written correctly
-        assert os.path.exists(metadata_json_file_path)
-        with open(metadata_json_file_path) as json_file:
-            json_data = json.load(json_file)
-        self.assertEquals(input_json_data, json_data)
-
     @patch("requests.get")
     def test_download_metadata_json_and_convert_to_xlsx(self, mock_requests):
         mock_response = mock_requests.return_value
@@ -106,9 +81,10 @@ def test_download_metadata_json_and_convert_to_xlsx(self, mock_requests):
         self.cli_to_eload.download_metadata_json_and_convert_to_xlsx(submission_id)
 
         # Check if requests.get was called with the correct URL
-        mock_requests.assert_called_once_with(f"{cfg['submissions']['webservice']['url']}/admin/submission/{submission_id}",
-                                              auth=(cfg['submissions']['webservice']['admin_username'],
-                                                    cfg['submissions']['webservice']['admin_password']))
+        mock_requests.assert_called_once_with(
+            f"{cfg['submissions']['webservice']['url']}/admin/submission/{submission_id}",
+            auth=(cfg['submissions']['webservice']['admin_username'],
+                  cfg['submissions']['webservice']['admin_password']))
 
         # Check if json file was written correctly and converted to xlsx without any error
         metadata_json_file_path = os.path.join(self.cli_to_eload.eload_dir, '10_submitted', 'metadata_file',

tests/test_sub_cli_processing/test_sub_cli_utils.py

@@ -0,0 +1,71 @@
+import json
+import os
+import shutil
+from unittest import TestCase
+from unittest.mock import patch
+
+from ebi_eva_common_pyutils.config import cfg
+
+from eva_sub_cli_processing.sub_cli_utils import download_metadata_json_file_for_submission_id, \
+    get_metadata_json_for_submission_id
+from eva_submission import ROOT_DIR
+
+
+class TestSubCliToEloadConverter(TestCase):
+    resources_folder = os.path.join(ROOT_DIR, 'tests', 'resources')
+    sub_cli_test_dir = os.path.join(resources_folder, 'sub_cli_test_dir')
+    cfg.load_config_file(os.path.join(resources_folder, 'submission_config.yml'))
+
+    def setUp(self) -> None:
+        if os.path.exists(self.sub_cli_test_dir):
+            shutil.rmtree(self.sub_cli_test_dir)
+        os.mkdir(self.sub_cli_test_dir)
+
+    def tearDown(self) -> None:
+        if os.path.exists(self.sub_cli_test_dir):
+            shutil.rmtree(self.sub_cli_test_dir)
+
+    @patch("requests.get")
+    def test_download_metadata_json_file_for_submission_id(self, mock_requests):
+        mock_response = mock_requests.return_value
+        mock_response.status_code = 200
+        input_json_file = os.path.join(self.resources_folder, 'input_json_for_json_to_xlsx_converter.json')
+        with open(input_json_file) as json_file:
+            input_json_data = json.load(json_file)
+        mock_response.json.return_value = {"metadataJson": input_json_data}
+
+        submission_id = "submission123"
+        metadata_json_file_path = os.path.join(self.sub_cli_test_dir, 'metadata_json.json')
+        download_metadata_json_file_for_submission_id(submission_id, metadata_json_file_path)
+
+        # Check if requests.get was called with the correct URL
+        mock_requests.assert_called_once_with(
+            f"{cfg['submissions']['webservice']['url']}/admin/submission/{submission_id}",
+            auth=(cfg['submissions']['webservice']['admin_username'],
+                  cfg['submissions']['webservice']['admin_password']))
+
+        # Check if file was written correctly
+        assert os.path.exists(metadata_json_file_path)
+        with open(metadata_json_file_path) as json_file:
+            json_data = json.load(json_file)
+        self.assertEquals(input_json_data, json_data)
+
+    @patch("requests.get")
+    def test_get_metadata_json_for_submission_id(self, mock_requests):
+        mock_response = mock_requests.return_value
+        mock_response.status_code = 200
+        input_json_file = os.path.join(self.resources_folder, 'input_json_for_json_to_xlsx_converter.json')
+        with open(input_json_file) as json_file:
+            input_json_data = json.load(json_file)
+        mock_response.json.return_value = {"metadataJson": input_json_data}
+
+        submission_id = "submission123"
+        response_json_data = get_metadata_json_for_submission_id(submission_id)
+
+        # Check if requests.get was called with the correct URL
+        mock_requests.assert_called_once_with(
+            f"{cfg['submissions']['webservice']['url']}/admin/submission/{submission_id}",
+            auth=(cfg['submissions']['webservice']['admin_username'],
+                  cfg['submissions']['webservice']['admin_password']))
+
+        self.assertEquals(input_json_data, response_json_data)