-
Notifications
You must be signed in to change notification settings - Fork 1
/
8_range_check.py
90 lines (65 loc) · 3.52 KB
/
8_range_check.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
import pandas as pd
import numpy as np
from tqdm import trange
from clim_helpers import vvd_apply_value_flag
import glob
from os.path import basename
def vvd_range_check(vvd, range_df):
vvd['Range_check_flag'] = np.zeros(len(vvd), dtype=int)
for i in trange(len(vvd)): # len(df) 10
# Want to find the last depth in the range_df that the i-th depth is greater than?
# cond = np.where(range_df.loc['Depth_m'] > df.loc[i, 'Depth_m'])[0]
for j in range(len(range_df)):
depth_cond = range_df.loc[j, 'Depth_min'] <= vvd.loc[
i, 'Depth_m'] <= range_df.loc[j, 'Depth_max']
range_cond = range_df.loc[j, 'N_Pacific_min'] <= vvd.loc[
i, 'Value'] <= range_df.loc[j, 'N_Pacific_max']
if depth_cond and not range_cond:
# Flag the df row if value is not within accepted range
vvd.loc[i, 'Range_check_flag'] = 1
return vvd
# ---------------------------STEP 2: Range check----------------------------
# Now do range checks: flag=0 if check passed; flag=1 if check failed
# Use preset ranges from WOD
range_file_T = 'C:\\Users\\HourstonH\\Documents\\NEP_climatology\\literature\\' \
'WOA docs\\wod18_users_manual_tables\\wod18_ranges_TEMP_N_Pac.csv'
range_file_S = 'C:\\Users\\HourstonH\\Documents\\NEP_climatology\\literature\\' \
'WOA docs\\wod18_users_manual_tables\\wod18_ranges_PSAL_N_Pac.csv'
range_files = [range_file_T, range_file_S]
# What is "Coastal Pacific" defined as?
# "depending on the number of one-degree by one-degree
# latitude-longitude grid boxes in the five-degree box which were land areas"
# Use North Pacific not Coastal North Pacific values for now
df_dir = 'C:\\Users\\HourstonH\\Documents\\NEP_climatology\\data\\' \
'value_vs_depth\\7_depth_check\\'
df_outdir = 'C:\\Users\\HourstonH\\Documents\\NEP_climatology\\data\\' \
'value_vs_depth\\8_range_check\\'
for var, range_file in zip(['Temp', 'Sal'], range_files):
vvd_files = glob.glob(df_dir + '*{}*done.csv'.format(var))
print(len(vvd_files))
# df_file = 'Oxy_1991_2020_value_vs_depth_dep_check_done.csv'
# df_file = 'WOD_PFL_Oxy_1991_2020_value_vs_depth_dep_check_done.csv'
df_range = pd.read_csv(range_file)
for df_file in vvd_files:
print(basename(df_file))
df_in = pd.read_csv(df_file)
# Range check function
df_out = vvd_range_check(df_in, df_range)
# Checks
print('number of values out of range:',
len(df_out.loc[df_out.Range_check_flag == 1, 'Range_check_flag']))
indices_rng = np.where(df_out.Range_check_flag == 1)[0]
print('depths and values out of range', df_out.loc[indices_rng, ['Depth_m', 'Value']])
# 24174 rows that were flagged during this check for Oxy: should limits be expanded??
# df_out.drop(columns='Depth_check_flag', inplace=True)
df_outname = df_outdir + basename(df_file).replace('dep_check_done', 'rng_check')
df_out.to_csv(df_outname, index=False)
df_rng_name = df_outname
# df_rng_name = 'C:\\Users\\HourstonH\\Documents\\NEP_climatology\\data\\' \
# 'value_vs_depth\\7_range_check\\' \
# 'WOD_PFL_Oxy_1991_2020_value_vs_depth_rng_check.csv'
df_in_fm = pd.read_csv(df_rng_name)
df_out_fm = vvd_apply_value_flag(df_in_fm, 'Range_check_flag')
df_out_name_fm = df_rng_name.replace('rng_check', 'rng_check_done')
df_out_fm.to_csv(df_out_name_fm, index=False)
print('done')