DSEGOG-255 EPAC Simulated Data

.github/ci_config.yml

@@ -13,6 +13,7 @@ images:
   echo_access_key: access_key
   echo_secret_key: secret_key
   image_bucket_name: test-bucket
+  upload_image_threads: 4
   preferred_colour_map_pref_name: PREFERRED_COLOUR_MAP
 mongodb:
   mongodb_url: mongodb://localhost:27017

.github/ci_ingest_echo_config.yml

@@ -0,0 +1,27 @@
+script_options:
+  wipe_database: false
+  delete_images: false
+  launch_api: true
+ssh:
+  enabled: false
+  ssh_connection_url: 127.0.0.1
+database:
+  hostname: localhost
+  port: 27017
+  name: opsgateway
+  remote_experiments_file_path: /tmp/experiments_for_mongoimport.json
+echo:
+  endpoint_url: https://s3.echo.stfc.ac.uk
+  access_key: access_key
+  secret_key: secret_key
+  simulated_data_bucket: og-ci-simulated-data
+  images_bucket: og-bucket
+  page_size: 1
+api:
+  host: 127.0.0.1
+  port: 8000
+  username: backend
+  password: back
+  log_config_path: /home/runner/work/operationsgateway-api/operationsgateway-api/operationsgateway_api/logging.ini
+  gunicorn_num_workers: "1"
+  timeout_seconds: 1000

.github/workflows/ci.yml

@@ -58,7 +58,7 @@ jobs:
           path: ~/.cache/pypoetry/virtualenvs
           key: ${{ runner.os }}-poetry-${{ matrix.python-version }}-${{ hashFiles('poetry.lock') }}
       - name: Install dependencies
-        run: poetry install
+        run: poetry install --without simulated-data
 
       # Configure s4cmd
       - name: Add keys to s4cmd config
@@ -107,15 +107,22 @@ jobs:
         env:
           SSH_KEY_PUBLIC: ${{secrets.SSH_PUBLIC_KEY_FOR_AUTH_OPENSSH}}
 
-      # Clone repo containing test data and use script to ingest the data
-      - name: Checkout OperationsGateway Test Data
-        uses: actions/checkout@v3
-        with:
-          repository: ral-facilities/operationsgateway-test-data
-          path: operationsgateway-test-data
-          ssh-key: ${{ secrets.SSH_PRIV_OG_TEST_DATA_ACTIONS }}
-      - name: Run ingestion script
-        run: poetry run python util/ingest_hdf.py -p operationsgateway-test-data/dev_server -U backend -P back
+      # Setup steps for Echo ingestion script
+      - name: Configure echo access key
+        run: yq -i ".echo.access_key = \"$ECHO_S3_ACCESS_KEY\"" .github/ci_ingest_echo_config.yml
+        env:
+          ECHO_S3_ACCESS_KEY: ${{secrets.ECHO_S3_ACCESS_KEY}}
+      - name: Configure echo secret key
+        run: yq -i ".echo.secret_key = \"$ECHO_S3_SECRET_KEY\"" .github/ci_ingest_echo_config.yml
+        env:
+          ECHO_S3_SECRET_KEY: ${{secrets.ECHO_S3_SECRET_KEY}}
+      - name: Configure bucket name for current run
+        run: yq -i '.echo.images_bucket = "og-actions-${{ github.sha }}-${{ github.run_id }}-${{ matrix.python-version }}"' .github/ci_ingest_echo_config.yml
+      - name: Copy config for Echo Ingest script to correct place
+        run: cp .github/ci_ingest_echo_config.yml util/realistic_data/config.yml
+
+      - name: Run Echo Ingest script
+        run: poetry run python util/realistic_data/ingest_echo_data.py
 
       - name: Load Poetry cache for Nox tests session
         uses: actions/cache@v3

.gitignore

@@ -15,3 +15,5 @@ id_rsa
 id_rsa.pub
 logging.ini*
 !logging.ini.example
+util/realistic_data/data
+util/realistic_data/resources

noxfile.py

@@ -6,7 +6,7 @@
 
 # Separating Black away from the rest of the sessions
 nox.options.sessions = "lint", "safety", "tests"
-code_locations = "operationsgateway_api", "test", "noxfile.py"
+code_locations = "operationsgateway_api", "test", "noxfile.py", "util"
 
 
 def install_with_constraints(session, *args, **kwargs):
@@ -110,5 +110,5 @@ def safety(session):
 @nox.session(python=["3.8", "3.9", "3.10"], reuse_venv=True)
 def tests(session):
     args = session.posargs
-    session.run("poetry", "install", external=True)
+    session.run("poetry", "install", "--without", "simulated-data", external=True)
     session.run("pytest", *args)

operationsgateway_api/config.yml.example

@@ -17,6 +17,7 @@ images:
   echo_access_key: access_key
   echo_secret_key: secret_key
   image_bucket_name: test-bucket
+  upload_image_threads: 4
   preferred_colour_map_pref_name: PREFERRED_COLOUR_MAP
 mongodb:
   mongodb_url: mongodb://localhost:27017

operationsgateway_api/src/config.py

@@ -37,6 +37,7 @@ class ImagesConfig(BaseModel):
     echo_access_key: StrictStr
     echo_secret_key: StrictStr
     image_bucket_name: StrictStr
+    upload_image_threads: StrictInt
     preferred_colour_map_pref_name: StrictStr
 
 

operationsgateway_api/src/main.py

@@ -93,6 +93,7 @@ def setup_logger():
         "s3transfer",
         "matplotlib.font_manager",
         "zeep",
+        "multipart",
     ]
     for name in libraries_info_logging:
         logging.getLogger(name).setLevel(logging.INFO)

operationsgateway_api/src/models.py

@@ -38,16 +38,17 @@ class ImageModel(BaseModel):
 
 class WaveformModel(BaseModel):
     id_: str = Field(alias="_id")
-    x: str
-    y: str
+    x: List[float]
+    y: List[float]
+
+    class Config:
+        arbitrary_types_allowed = True
 
     @field_validator("x", "y", mode="before")
     def encode_values(cls, value):  # noqa: N805
         if isinstance(value, np.ndarray):
-            return str(list(value))
+            return list(value)
         else:
-            # Typically will be a string when putting waveform data into the model from
-            # results of a MongoDB query
             return value
 
 

operationsgateway_api/src/records/echo_interface.py

@@ -38,7 +38,7 @@ def __init__(self) -> None:
             log.error(
                 "%s: %s. This error happened with bucket '%s'",
                 exc.response["Error"]["Code"],
-                exc.response["Error"]["Message"],
+                exc.response["Error"].get("Message"),
                 Config.config.images.image_bucket_name,
             )
             raise EchoS3Error(
@@ -70,7 +70,7 @@ def download_file_object(self, image_path: str) -> BytesIO:
             log.error(
                 "%s: %s",
                 exc.response["Error"]["Code"],
-                exc.response["Error"]["Message"],
+                exc.response["Error"].get("Message"),
             )
             raise EchoS3Error(
                 f"{exc.response['Error']['Code']} when downloading file at"
@@ -96,7 +96,7 @@ def upload_file_object(self, image_object: BytesIO, image_path: str) -> None:
             log.error(
                 "%s: %s",
                 exc.response["Error"]["Code"],
-                exc.response["Error"]["Message"],
+                exc.response["Error"].get("Message"),
             )
             raise EchoS3Error(
                 f"{exc.response['Error']['Code']} when uploading file at"

operationsgateway_api/src/records/hdf_handler.py

@@ -46,18 +46,30 @@ def extract_data(self) -> Tuple[RecordModel, List[WaveformModel], List[ImageMode
         log.debug("Extracting data from HDF files")
 
         metadata_hdf = dict(self.hdf_file.attrs)
+        timestamp_format = "%Y-%m-%dT%H:%M:%S%z"
         try:
             metadata_hdf["timestamp"] = datetime.strptime(
                 metadata_hdf["timestamp"],
-                DATA_DATETIME_FORMAT,
+                timestamp_format,
             )
-            self.record_id = metadata_hdf["timestamp"].strftime(ID_DATETIME_FORMAT)
-        except ValueError as exc:
-            raise HDFDataExtractionError(
-                "Incorrect timestamp format for metadata timestamp. Use"
-                f" {ID_DATETIME_FORMAT} instead",
-            ) from exc
-
+        except ValueError:
+            # Try using alternative timestamp format that might be used for older HDF
+            # files such as old Gemini test data
+            # TODO - when Gemini test data is no longer needed, remove this try/except
+            # block that attempts to convert the timestamp for a second time. Go
+            # straight to raising the exception instead
+            try:
+                metadata_hdf["timestamp"] = datetime.strptime(
+                    metadata_hdf["timestamp"],
+                    DATA_DATETIME_FORMAT,
+                )
+            except ValueError as exc:
+                raise HDFDataExtractionError(
+                    "Incorrect timestamp format for metadata timestamp. Use"
+                    f" {timestamp_format} instead",
+                ) from exc
+
+        self.record_id = metadata_hdf["timestamp"].strftime(ID_DATETIME_FORMAT)
         self.extract_channels()
 
         try:

operationsgateway_api/src/records/waveform.py

@@ -1,6 +1,5 @@
 import base64
 from io import BytesIO
-import json
 import logging
 
 import matplotlib.pyplot as plt
@@ -68,8 +67,8 @@ def _create_plot(self, buffer) -> None:
         plt.xticks([])
         plt.yticks([])
         plt.plot(
-            json.loads(self.waveform.x),
-            json.loads(self.waveform.y),
+            self.waveform.x,
+            self.waveform.y,
             linewidth=0.5,
         )
         plt.axis("off")

operationsgateway_api/src/routes/ingest_data.py

@@ -7,6 +7,7 @@
 
 from operationsgateway_api.src.auth.authorisation import authorise_route
 from operationsgateway_api.src.channels.channel_manifest import ChannelManifest
+from operationsgateway_api.src.config import Config
 from operationsgateway_api.src.error_handling import endpoint_error_handling
 from operationsgateway_api.src.records.hdf_handler import HDFDataHandler
 from operationsgateway_api.src.records.image import Image
@@ -64,10 +65,8 @@ async def submit_hdf(
         image.create_thumbnail()
         record.store_thumbnail(image)
 
-    # Upload images to Echo S3 using a thread pool to be more efficent when uploading
-    # multiple images from the same HDF file
     if len(image_instances) > 0:
-        pool = ThreadPool(processes=len(image_instances))
+        pool = ThreadPool(processes=Config.config.images.upload_image_threads)
         pool.map(Image.upload_image, image_instances)
 
     if stored_record: