Feature 307 schema total dir (#311)

* Issue #307 TOTAL_DIR for VL1L2, VAL1L2, and VCNT linetypes

* Issue #307 added TOTAL_WIND to VL1L2, VAL1L2, and VCNT

* Issue #307 initial commit of test for VL1L2, VAL1L2, and VCNT for the TOTAL_DIR column

* issue #307 config file for tests

* issue #307 modify setup_db to return the connection object instead of the database settings

* modify setup to return only the database data class

* updated to separate the hostname from the port in the config file and
return the connection object in the setup code

* update to separate the host and port and simplify test setup code

* fix indentation and complete the try-finally block

* add test to check if db is created and fix test for tables test

* added test for presence of the mv_load_test DB

* Added test to check for the total_dir column in the VL1L2, VAL1L2, and VCNT linetype tables

* Clean up extraneous blank lines

* Sample data for testing

---------

Co-authored-by: Minna Win <minnawin@mohawk.rap.ucar.edu>

METdbLoad/sql/mv_mysql.sql

@@ -912,6 +912,7 @@ CREATE TABLE line_data_vl1l2
     uvoobar        DOUBLE,
     f_speed_bar    DOUBLE DEFAULT -9999,
     o_speed_bar    DOUBLE DEFAULT -9999,
+    total_dir      INT UNSIGNED,
     dir_me         DOUBLE,
     dir_mae        DOUBLE,
     dir_mse        DOUBLE,
@@ -953,6 +954,7 @@ CREATE TABLE line_data_val1l2
     uvooabar       DOUBLE,
     fa_speed_bar   DOUBLE DEFAULT -9999,
     oa_speed_bar   DOUBLE DEFAULT -9999,
+    total_dir      INT UNSIGNED,
     dira_me        DOUBLE,
     dira_mae       DOUBLE,
     dira_mse       DOUBLE,
@@ -1590,7 +1592,6 @@ CREATE TABLE line_data_ssvar
     obs_valid_end  DATETIME,
     alpha          DOUBLE,
     total          INT UNSIGNED,
-
     n_bin          INT UNSIGNED,
     bin_i          INT UNSIGNED,
     bin_n          INT UNSIGNED,
@@ -1602,7 +1603,6 @@ CREATE TABLE line_data_ssvar
     fobar          DOUBLE,
     ffbar          DOUBLE,
     oobar          DOUBLE,
-
     fbar_ncl       DOUBLE,
     fbar_ncu       DOUBLE,
     fstdev         DOUBLE,
@@ -1714,6 +1714,7 @@ CREATE TABLE line_data_vcnt
     anom_corr_uncntr     DOUBLE DEFAULT -9999,
     anom_corr_uncntr_bcl DOUBLE DEFAULT -9999,
     anom_corr_uncntr_bcu DOUBLE DEFAULT -9999,
+    total_dir            INT UNSIGNED,
     dir_me               DOUBLE,
     dir_me_bcl           DOUBLE,
     dir_me_bcu           DOUBLE,

METdbLoad/sql/updates/update_for_6_0_beta5.sql

@@ -0,0 +1,13 @@
+DELIMITER |
+
+
+ALTER TABLE line_data_vcnt
+    ADD COLUMN total_dir DOUBLE |
+
+ALTER TABLE line_data_val1l2
+    ADD COLUMN total_dir DOUBLE |
+
+ALTER TABLE line_data_vl1l2
+    ADD COLUMN total_dir DOUBLE |
+
+DELIMITER ;

METdbLoad/tests/update_schema_6.0_beta4/test_loading.py

@@ -29,13 +29,20 @@ def setup_db():
             print(exc)
 
     # Create a dataclass of the database information
-    #TCDiag = make_dataclass("TCDiag", ["total", "index", "diag_src", "diag_val"], frozen=True)
-    #orig = TCDiag(orig_total, orig_index, orig_diag_src, orig_diag_val)
-    DBS = make_dataclass("DBS", ["hostname", "username", "password", "dbname"])
-    db_settings = DBS(parms['hostname'], parms['username'], parms['password'], parms['dbname'])
+    DBS = make_dataclass("DBS", ["hostname", "port", "username", "password", "dbname"])
+    db_settings = DBS(parms['hostname'], parms['port'], parms['username'], parms['password'], parms['dbname'])
 
     # Return the db settings (hostname, username, etc.)
-    yield db_settings
+    conn = pymysql.connect(
+        host=db_settings.hostname,
+        port=db_settings.port,
+        user=db_settings.username,
+        password=db_settings.password,
+        db=db_settings.dbname,
+        charset='utf8mb4'
+    )
+
+    yield conn
 
 
 def test_ecnt_db_created(setup_db):
@@ -45,16 +52,8 @@ def test_ecnt_db_created(setup_db):
     # found.
 
 
-    conn = pymysql.connect(
-        host=setup_db.hostname,
-        user=setup_db.username,
-        password=setup_db.password,
-        db=setup_db.dbname,
-        charset='utf8mb4'
-    )
-
     try:
-        with conn.cursor() as cursor:
+        with setup_db.cursor() as cursor:
             # Check that the mv_load_test database was created
             check_db_exists_query = "show databases;"
             cursor.execute(check_db_exists_query)
@@ -69,23 +68,14 @@ def test_ecnt_db_created(setup_db):
 
 
     finally:
-        conn.close()
+        setup_db.close()
 
 def test_tables_created(setup_db):
 
     # log into the database and verify the ECNT, VCNT, VL1L2, and VAL1L2 tables exist
 
-
-    conn = pymysql.connect(
-        host=setup_db.hostname,
-        user=setup_db.username,
-        password=setup_db.password,
-        db=setup_db.dbname,
-        charset='utf8mb4'
-    )
-
     try:
-        with conn.cursor() as cursor:
+        with setup_db.cursor() as cursor:
             # Check that the line_data_ecnt, line_data_vcnt, line_data_vl1l2, and
             # line_data_val1l2 tables were created
             cursor.execute(CONST_LOAD_DB_CMD)
@@ -102,23 +92,15 @@ def test_tables_created(setup_db):
             assert 'line_data_val1l2' in list_of_rows
 
     finally:
-        conn.close()
+        setup_db.close()
 
 
 def test_ecnt_columns(setup_db):
    # log into the database and verify the ign_conv_oerr and ign_corr_oerr columns are in the
    # list_data_ecnt database table.
 
-    conn = pymysql.connect(
-        host=setup_db.hostname,
-        user=setup_db.username,
-        password=setup_db.password,
-        db=setup_db.dbname,
-        charset='utf8mb4'
-    )
-
     try:
-        with conn.cursor() as cursor:
+        with setup_db.cursor() as cursor:
             # Check that the line_data_ecnt, line_data_vcnt, line_data_vl1l2, and
             # line_data_val1l2 tables were created
             cursor.execute(CONST_LOAD_DB_CMD)
@@ -134,7 +116,7 @@ def test_ecnt_columns(setup_db):
             assert 'ign_corr_oerr' in list_of_rows
 
     finally:
-        conn.close()
+        setup_db.close()
 
 def test_vcnt_columns(setup_db):
    # log into the database and verify the dir_me, dir_me_bcl, dir_me_bcu, ..., etc. columns are in the
@@ -145,16 +127,9 @@ def test_vcnt_columns(setup_db):
                      'dir_mse', 'dir_mse_bcl', 'dir_mse_bcu',
                      'dir_rmse', 'dir_rmse_bcl', 'dir_rmse_bcu'
                      ]
-    conn = pymysql.connect(
-        host=setup_db.hostname,
-        user=setup_db.username,
-        password=setup_db.password,
-        db=setup_db.dbname,
-        charset='utf8mb4'
-    )
 
     try:
-        with conn.cursor() as cursor:
+        with setup_db.cursor() as cursor:
             # Check that the  line_data_vcnt expected columns were created
             cursor.execute(CONST_LOAD_DB_CMD)
 
@@ -169,23 +144,16 @@ def test_vcnt_columns(setup_db):
                 assert expected in list_of_rows
 
     finally:
-        conn.close()
+        setup_db.close()
 
 def test_vl1l2_columns(setup_db):
    # log into the database and verify the dir_me, dir_mae, and dir_mse columns are in the
    # list_data_vl1l2 database table.
 
     expected_cols = ['dir_me', 'dir_mae', 'dir_mse']
-    conn = pymysql.connect(
-        host=setup_db.hostname,
-        user=setup_db.username,
-        password=setup_db.password,
-        db=setup_db.dbname,
-        charset='utf8mb4'
-    )
 
     try:
-        with conn.cursor() as cursor:
+        with setup_db.cursor() as cursor:
             # Check that the  line_data_vl1l2 table has the expected columns
             cursor.execute(CONST_LOAD_DB_CMD)
 
@@ -200,23 +168,16 @@ def test_vl1l2_columns(setup_db):
                 assert expected in list_of_rows
 
     finally:
-        conn.close()
+        setup_db.close()
 
 def test_val1l2_columns(setup_db):
    # log into the database and verify the dira_me, dira_mae, and dira_mse columns are in the
    # list_data_val1l2 database table.
 
     expected_cols = ['dira_me', 'dira_mae', 'dira_mse']
-    conn = pymysql.connect(
-        host=setup_db.hostname,
-        user=setup_db.username,
-        password=setup_db.password,
-        db=setup_db.dbname,
-        charset='utf8mb4'
-    )
 
     try:
-        with conn.cursor() as cursor:
+        with setup_db.cursor() as cursor:
             # Check that the line_data_vl1l2 table has the expected columns
             cursor.execute(CONST_LOAD_DB_CMD)
 
@@ -231,7 +192,7 @@ def test_val1l2_columns(setup_db):
                 assert expected in list_of_rows
 
     finally:
-        conn.close()
+        setup_db.close()
 
 
 def test_ecnt_vals(setup_db):
@@ -242,16 +203,8 @@ def test_ecnt_vals(setup_db):
     ign_conv_oerr = "33.41424"
     ign_corr_oerr = "440.06905"
 
-    conn = pymysql.connect(
-        host=setup_db.hostname,
-        user=setup_db.username,
-        password=setup_db.password,
-        db=setup_db.dbname,
-        charset='utf8mb4'
-    )
-
     try:
-        with conn.cursor() as cursor:
+        with setup_db.cursor() as cursor:
             # Check that the line_data_vl1l2 table has the expected columns
             cursor.execute(CONST_LOAD_DB_CMD)
 
@@ -266,6 +219,6 @@ def test_ecnt_vals(setup_db):
 
 
     finally:
-        conn.close()
+        setup_db.close()
 
 

METdbLoad/tests/update_schema_6.0_beta4/test_loading.yaml

@@ -1,4 +1,5 @@
-hostname: 'localhost:3306'
+hostname: 'localhost'
+port: 3306
 username: 'mvadmin'
 password: 'passwordgoeshere'
 dbname: 'mv_load_test'

METdbLoad/tests/update_schema_6.0_beta5/test_loading.yaml

@@ -0,0 +1,7 @@
+hostname: 'localhost'
+port: 3306
+username: 'mvadmin'
+password: '160GiltVa0D5M'
+dbname: 'mv_load_test'
+output_dir: './output'
+