318: update tests/data for MTD 3d cluster (#334)

* 318: update tests/data for MTD 3d cluster * Add pyproject.toml and test config * Get consistency with sonarQube and coverage config
dtcenter · Sep 17, 2024 · 2aaedd0 · 2aaedd0
1 parent d714793
commit 2aaedd0
Show file tree

Hide file tree

Showing 8 changed files with 47 additions and 12 deletions.
diff --git a/.coveragerc b/.coveragerc
diff --git a/.github/workflows/sonarqube.yaml b/.github/workflows/sonarqube.yaml
@@ -40,7 +40,16 @@ jobs:
   sonarqube:
     name: SonarQube Scan
     runs-on: ubuntu-latest
-
+
+    services:
+      mariadb:
+        image: mariadb:latest
+        env:
+          MARIADB_ROOT_PASSWORD: root_password
+        ports:
+          - 3306:3306
+        options: --health-cmd="healthcheck.sh --connect --innodb_initialized" --health-interval=10s --health-timeout=5s --health-retries=3
+
     steps:
 
     - name: Check if PR is from a fork
@@ -75,8 +84,7 @@ jobs:
 
     - name: Run Pytests
       run: |
-        coverage run --append -m pytest METreformat
-        coverage run --append -m pytest METreadnc
+        coverage run --append -m pytest
 
     - name: Output coverage report
       run: coverage report -m

diff --git a/.github/workflows/unit_tests.yaml b/.github/workflows/unit_tests.yaml
@@ -76,9 +76,7 @@ jobs:
 
       - name: Test with pytest
         run: |
-          coverage run --append -m pytest METdbLoad/test
-          coverage run --append -m pytest METreformat
-          coverage run --append -m pytest METreadnc
+          coverage run --append -m pytest
           coverage report -m
           echo "Finished unit tests and coverage"
 
diff --git a/METdbLoad/test/data/mtd/mtd_CONV_TIME_20100517_010000V_3d_pair_cluster.txt b/METdbLoad/test/data/mtd/mtd_CONV_TIME_20100517_010000V_3d_pair_cluster.txt
@@ -0,0 +1,5 @@
+VERSION  MODEL  DESC  FCST_LEAD       FCST_VALID  OBS_LEAD        OBS_VALID  T_DELTA  FCST_T_BEG  FCST_T_END  FCST_RAD  FCST_THR  OBS_T_BEG  OBS_T_END  OBS_RAD  OBS_THR  FCST_VAR  FCST_UNITS  FCST_LEV  OBS_VAR  OBS_UNITS  OBS_LEV    OBJECT_ID   OBJECT_CAT  SPACE_CENTROID_DIST  TIME_CENTROID_DELTA  AXIS_DIFF  SPEED_DELTA  DIRECTION_DIFF  VOLUME_RATIO  START_TIME_DELTA  END_TIME_DELTA  INTERSECTION_VOLUME  DURATION_DIFF  INTEREST
+V12.0.0   FCST    NA         NA  20100517_010000        NA  20100517_010000   010000          -2           3         2     >=0.5         -1          1        2    >=0.5   APCP_01      kg/m^2       A01  APCP_01     kg/m^2      A01  CF001_CO001  CF001_CO001                 6.89                -0.01      3.894         0.37           31.96          1.19                 0               0                47061              0    -1.000
+V12.0.0   FCST    NA         NA  20100517_010000        NA  20100517_010000   010000          -2           3         2     >=0.5         -1          1        2    >=0.5   APCP_01      kg/m^2       A01  APCP_01     kg/m^2      A01  CF002_CO002  CF002_CO002                60.32                -1.06      5.214         1.07            3.46          0.75                 0               0                    0              0    -1.000
+V12.0.0   FCST    NA         NA  20100517_010000        NA  20100517_010000   010000          -2           3         2     >=0.5         -1          1        2    >=0.5   APCP_01      kg/m^2       A01  APCP_01     kg/m^2      A01  CF003_CO003  CF003_CO003                65.19                -0.67     62.974        -3.31          177.03          0.98                 0              -1                    7             -1    -1.000
+V12.0.0   FCST    NA         NA  20100517_010000        NA  20100517_010000   010000          -2           3         2     >=0.5         -1          1        2    >=0.5   APCP_01      kg/m^2       A01  APCP_01     kg/m^2      A01  CF004_CO004  CF004_CO004                 4.72                 0.84      2.870        -1.85            1.95          0.90                -2               0                 1151              2    -1.000
diff --git a/METdbLoad/test/test_met_db_load.py b/METdbLoad/test/test_met_db_load.py
@@ -85,6 +85,8 @@ def assert_count_rows(cur, table, expected_count):
             {
                 "mtd_2d_obj": 278,
                 "mtd_3d_obj_single": 8,
+                "mtd_3d_obj_pair": 4,
+                "mtd_header": 24,
             },
         ),
         (
@@ -128,9 +130,14 @@ def test_met_db_table_counts(
     expected_counts,
 ):
 
+    load_flags = {
+        "mode_header_db_check": "true",
+        "mtd_header_db_check": "true",
+    }
+
     test_args = dict_to_args(
         {
-            "xmlfile": str(get_xml_test_file(tmp_path, met_data_dir, met_tool)),
+            "xmlfile": str(get_xml_test_file(tmp_path, met_data_dir, met_tool, load_flags=load_flags)),
             "index": "true",
             "tmpdir": [str(tmp_path)],
             "loglevel": None,

diff --git a/METdbLoad/test/test_read_data_files.py b/METdbLoad/test/test_read_data_files.py
@@ -42,7 +42,7 @@ def test_mtd_loads(tmp_path, get_xml_loadfile):
     )
 
     # number of files
-    assert len(XML_LOADFILE.load_files) == 2
+    assert len(XML_LOADFILE.load_files) == 3
     # number of lines of data
     assert FILE_DATA.mtd_2d_data.shape == (278, 43)
     assert FILE_DATA.mtd_3d_single_data.shape == (8, 48)

diff --git a/internal/scripts/sonarqube/sonar-project.properties b/internal/scripts/sonarqube/sonar-project.properties
@@ -3,8 +3,8 @@ sonar.projectKey=METdataio
 sonar.projectName=METdataio
 sonar.projectVersion=SONAR_PROJECT_VERSION
 sonar.branch.name=SONAR_BRANCH_NAME
-sonar.sources=METdbLoad,METreadnc,METreformat
-sonar.coverage.exclusions=**/test/*,docs/**,setup.py
+sonar.sources=METdbLoad/ush,METreadnc/util,METreformat
+sonar.coverage.exclusions=**/test/*,**/tests/*,docs/**,setup.py
 sonar.python.coverage.reportPaths=coverage.xml
 sonar.sourceEncoding=UTF-8
 

diff --git a/pyproject.toml b/pyproject.toml
@@ -0,0 +1,19 @@
+
+[tool.pytest.ini_options]
+addopts = "-q"
+testpaths = [
+    "METdbLoad/test",
+    "METreadnc/test",
+    "METreformat/test",
+]
+
+[tool.coverage.run]
+relative_files = true
+source = [
+    "METdbLoad/ush",
+    "METreadnc/util",
+    "METreformat/",
+    ]
+omit = [
+    "METreformat/test/*"
+]