Feature #318 test run sql and met db load (#330)

* 318: add test for vsdb and met_db_load

* 318: run_sql tests

* 318: update docs in utils

METdbLoad/test/data/vsdb/gfs_20140802.vsdb

@@ -15,10 +15,12 @@ V01 GFS 45 2014080221 STAGE2 G218/LMV FHO>1.0 APCP/03 SFC =   5789 0.00000 0.000
 V01 GFS 45 2014080221 STAGE2 G218/APL FHO>1.0 APCP/03 SFC =   2814 0.00000 0.00000 0.00000
 V01 GFS 45 2014080221 STAGE2 G218/NEC FHO>1.0 APCP/03 SFC =   3052 0.00000 0.00000 0.00033
 V01 GFS 45 2014080221 STAGE2 G218/SEC FHO>1.0 APCP/03 SFC =   3524 0.00000 0.00000 0.00000
-V01 GFS 45 2014080221 STAGE2 G218/GMC FHO>1.0 APCP/03 SFC =   3524 0.00000 0.00000 0.00000
-V01 GFS 45 2014080221 STAGE4 G218/RFC FHO>1.0 APCP/03 SFC =  60582 0.00000 0.00000 0.00050
+V01 GFS/2 45 2014080221 STAGE2 G218/GMC FHO>1.0 APCP/03 SFC =   3524 0.00000 0.00000 0.00000
+V01 GFS/2 45 2014080221 STAGE4 G218/RFC FHO>1.0 APCP/03 SFC =  60582 0.00000 0.00000 0.00050
 V01 GFS 45 2014080221 STAGE2 G218/NWC SL1L2 APCP/03 SFC=    2556 0.2401E-01 0.6913E-01 0.0000E+00 0.4112E-01 0.1017E+00
 V01 GFS 45 2014080221 STAGE2 G218/SWC SL1L2 APCP/03 SFC=    1422 0.7749E-01 0.2632E+00 0.2126E-01 0.1783E+00 0.8898E+00
 V01 GFS 45 2014080221 STAGE2 G218/NMT SL1L2 APCP/03 SFC=    6251 0.6989E-01 0.3756E-01 0.1510E-02 0.1398E+00 0.6695E-01
 V01 GFS 45 2014080221 STAGE2 G218/GRB SL1L2 APCP/03 SFC=    3400 0.1266E-01 0.4206E-02 0.1994E-04 0.1205E-01 0.3085E-02
-V01 GFS 45 2014080221 STAGE2 G218/SMT SL1L2 APCP/03 SFC=    3802 0.6522E+00 0.2165E+00 0.4879E+00 0.1883E+01 0.7141E+00
+V01 GFS 45 2014080221 STAGE2 G218/SMT SL1L2 APCP/03 SFC=    3802 0.6522E+00 0.2165E+00 0.4879E+00 0.1883E+01 0.7141E+00
+V01 GFS/2 45 2014080221 STAGE2 G218/GMC RELI APCP/03 SFC =   3524 0.00000 0.00000 0.00000
+V01 GFS/2 45 2014080221 STAGE4 G218/RFC RELI APCP/03 SFC =  60582 0.00000 0.00000 0.00050

METdbLoad/test/test_load_specification.xml

@@ -1,14 +1,14 @@
 <load_spec>
     <connection>
-        <management_system>mysql</management_system>
+        <management_system>mariadb</management_system>
         <host>localhost:3306</host>
         <database>mv_test</database>
         <user>root</user>
         <password>root_password</password>
     </connection>
 
     <folder_tmpl>/METdataio/METreformat/test/data/point_stat</folder_tmpl>
-    <verbose>true</verbose>
+    <verbose>false</verbose>
     <insert_size>1</insert_size>
     <stat_header_db_check>true</stat_header_db_check>
     <mode_header_db_check>false</mode_header_db_check>

METdbLoad/test/test_met_db_load.py

@@ -1,6 +1,8 @@
-import argparse
 import pytest
+from unittest.mock import patch
 from METdbLoad.ush.met_db_load import main as load_main
+from METdbLoad.ush.run_sql import RunSql
+from METdbLoad.test.utils import dict_to_args
 
 from METdataio.METdbLoad.test.utils import (
     get_xml_test_file,
@@ -111,16 +113,58 @@ def test_met_db_table_counts(
     met_tool,
     expected_counts,
 ):
-    test_data = {
-        "xmlfile": str(get_xml_test_file(tmp_path, met_data_dir, met_tool)),
-        "index": "true",
-        "tmpdir": [str(tmp_path)],
-    }
-    test_args = argparse.Namespace()
-    for k, v in test_data.items():
-        setattr(test_args, k, v)
+
+    test_args = dict_to_args(
+        {
+            "xmlfile": str(get_xml_test_file(tmp_path, met_data_dir, met_tool)),
+            "index": "true",
+            "tmpdir": [str(tmp_path)],
+        }
+    )
 
     load_main(test_args)
 
     for table, expected_count in expected_counts.items():
         assert_count_rows(testRunSql.cur, table, expected_count)
+
+
+def test_met_db_indexes(
+    emptyDB,
+    testRunSql,
+    tmp_path,
+):
+    # set up to only apply indexes
+    test_args = dict_to_args(
+        {
+            "xmlfile": str(
+                get_xml_test_file(
+                    tmp_path,
+                    POINT_STAT_DATA_DIR,
+                    "point_stat",
+                    {"apply_indexes": "true"},
+                )
+            ),
+            "index": "false",
+            "tmpdir": [str(tmp_path)],
+        }
+    )
+
+    # file_id and stat_header are already indexed
+    idx_cnt = testRunSql.cur.execute("SHOW INDEX FROM line_data_fho")
+    assert idx_cnt == 2
+
+    # sys.exit is called after processing indexes
+    with pytest.raises(SystemExit):
+        load_main(test_args)
+
+    # check extra indicies have been created
+    idx_cnt = testRunSql.cur.execute("SHOW INDEX FROM line_data_fho")
+    assert idx_cnt == 5
+
+    # check no data was loaded
+    assert_count_rows(testRunSql.cur, "line_data_fho", 0)
+
+    # check sys.exit called on error
+    with pytest.raises(SystemExit):
+        with patch.object(RunSql, "apply_indexes", side_effect=KeyError):
+            load_main(test_args)

METdbLoad/test/test_run_sql.py

@@ -0,0 +1,62 @@
+import pytest
+from unittest import mock
+from pymysql import OperationalError
+
+from METdbLoad.ush.met_db_load import main as load_main
+from METdbLoad.ush import constants as CN
+from METdbLoad.test.utils import dict_to_args
+from METdataio.METdbLoad.test.utils import (
+    get_xml_test_file,
+    GRID_STAT_DATA_DIR,
+)
+
+
+def populate_some_data(
+    tmp_path, met_data_dir=GRID_STAT_DATA_DIR, met_tool="grid_stat", load_flags={}
+):
+    test_args = dict_to_args(
+        {
+            "xmlfile": str(
+                get_xml_test_file(tmp_path, met_data_dir, met_tool, load_flags)
+            ),
+            "index": "true",
+            "tmpdir": [str(tmp_path)],
+        }
+    )
+    load_main(test_args)
+
+
+def test_get_file_name(tmp_path, emptyDB, testRunSql):
+    file_name = testRunSql.get_file_name(1, testRunSql.cur)
+    assert file_name == None
+
+    populate_some_data(tmp_path)
+
+    file_name = testRunSql.get_file_name(1, testRunSql.cur)
+    assert file_name == "grid_stat_GTG_latlon_060000L_20130827_180000V.stat"
+
+    # check for a nonexistant id
+    file_name = testRunSql.get_file_name(9999999, testRunSql.cur)
+    assert file_name == None
+
+
+@pytest.mark.parametrize(
+    "drop, cmds",
+    (
+        [True, CN.DROP_INDEXES_QUERIES],
+        [False, CN.CREATE_INDEXES_QUERIES],
+    ),
+)
+def test_apply_indexes(testRunSql, drop, cmds):
+
+    mock_cursor = mock.MagicMock()
+    testRunSql.apply_indexes(drop, mock_cursor)
+
+    # Check the correct commands were executed
+    assert len(mock_cursor.execute.call_args_list) == len(cmds)
+    for cmd in cmds:
+        mock_cursor.execute.assert_any_call(cmd)
+
+    # check no exception is raised on pymysql error
+    mock_cursor.execute.side_effect = OperationalError
+    testRunSql.apply_indexes(drop, mock_cursor)

METdbLoad/test/utils.py

@@ -1,4 +1,5 @@
 from pathlib import Path
+from argparse import Namespace
 
 
 def abs_path(rel_path):
@@ -96,3 +97,19 @@ def get_xml_test_file(tmp_path, met_data_dir, met_tool, load_flags={}):
     with open(xml_path, "w") as text_file:
         text_file.write(populate_xml_load_spec(met_data_dir, met_tool, load_flags))
     return xml_path
+
+
+def dict_to_args(args_dict):
+    """Convert a dcit to an argparse Namespace
+    
+    Args:
+        args_dict (dict): key value pairs to be converted to
+        argparse Namespace.
+    
+    Returns:
+        argparse.Namespace
+    """
+    test_args = Namespace()
+    for k, v in args_dict.items():
+        setattr(test_args, k, v)
+    return test_args