-
Notifications
You must be signed in to change notification settings - Fork 24
/
pb2nc.cc
3677 lines (3129 loc) · 133 KB
/
pb2nc.cc
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*
// ** Copyright UCAR (c) 1992 - 2024
// ** University Corporation for Atmospheric Research (UCAR)
// ** National Center for Atmospheric Research (NCAR)
// ** Research Applications Lab (RAL)
// ** P.O.Box 3000, Boulder, Colorado, 80307-3000, USA
// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*
////////////////////////////////////////////////////////////////////////
//
// Filename: pb2nc.cc
//
// Description:
// Based on user specified options, this tool filters point
// observations from a PrepBufr input file, derives requested
// observation types, and writes the output to a NetCDF file.
// PrepBufr observations must be reformatted in this way prior to
// using them in the Point-Stat tool.
//
// Mod# Date Name Description
// ---- ---- ---- -----------
// 000 03-27-07 Halley Gotway New
// 001 12-19-07 Halley Gotway Add support for ANYAIR, ANYSFC,
// and ONLYSF message types.
// 002 12-20-07 Halley Gotway Derive PRMSL.
// 003 01-31-08 Halley Gotway Remove unused array elements from
// the hdr_arr and obs_arr and allow it to run on
// multiple PrepBufr files at once.
// 004 08-22-08 Halley Gotway Change the arguments to dumppb
// function for it to work with Fortran90/95
// compilers.
// 005 02-12-09 Halley Gotway Fix npbmsg bug when reading
// multiple PrepBufr files and use of valid_beg and
// valid_end options.
// 006 06-01-10 Halley Gotway Pass flags to dumppb to only dump
// observation for requested message types.
// 007 02-15-11 Halley Gotway Fix event_stack bug for which the
// logic was reversed.
// 008 02-28-11 Halley Gotway Modify relative humidity derivation
// to match the Unified-PostProcessor.
// 009 06-01-11 Halley Gotway Call closepb for input file
// descriptor.
// 010 10/28/11 Holmes Added use of command line class to
// parse the command line arguments.
// 011 11/14/11 Holmes Added code to enable reading of
// multiple config files.
// 012 05/11/12 Halley Gotway Switch to using vx_config library.
// 013 07/12/13 Halley Gotway Use observations of sensible
// temperature instead of virtual temperature.
// 014 07/23/13 Halley Gotway Update sensible temperature fix
// to handle older GDAS PREPBUFR files.
// 06/07/17 Howard Soh Added more options: -vars, -all, and -index.
// 09/15/17 Howard Soh Removed options: -all, and -use_var_id.
// 015 02/10/18 Halley Gotway Add message_type_group_map.
// 016 07/23/18 Halley Gotway Support masks defined by gen_vx_mask.
// 017 07/06/22 Howard Soh METplus-Internal #19 Rename main to met_main
// 018 09/12/22 Prestopnik, J. MET #2227 Remove namespace std and netCDF
// from header files
// 019 07/21/23 Prestopnik, J. MET #2615 Add #include <array> to compile
// successfully using gcc12
// 020 08/26/24 Halley Gotway MET #2938 Silence center time warnings
//
////////////////////////////////////////////////////////////////////////
#include <cstdio>
#include <cstdlib>
#include <ctype.h>
#include <dirent.h>
#include <fstream>
#include <limits>
#include <math.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include <assert.h>
#include <array>
#include <netcdf>
#include "main.h"
#include "pb2nc_conf_info.h"
#include "data_class.h"
#include "data2d_factory.h"
#include "vx_log.h"
#include "vx_nc_util.h"
#include "vx_grid.h"
#include "vx_util.h"
#include "vx_pb_util.h"
#include "vx_analysis_util.h"
#include "vx_cal.h"
#include "vx_math.h"
#include "write_netcdf.h"
#include "vx_summary.h"
#include "nc_obs_util.h"
#include "nc_point_obs_out.h"
#include "nc_summary.h"
using namespace std;
using namespace netCDF;
////////////////////////////////////////////////////////////////////////
struct derive_var_cfg {
int var_index;
ConcatString var_name;
explicit derive_var_cfg(ConcatString _var_name);
derive_var_cfg &operator=(const derive_var_cfg &a) noexcept;
};
////////////////////////////////////////////////////////////////////////
//
// Constants
//
static const char * default_config_filename = "MET_BASE/config/PB2NCConfig_default";
static const char *program_name = "pb2nc";
static const char *not_assigned = "not_assigned";
static const char *tmp_pb2nc_base = "tmp_pb2nc_blk";
constexpr float fill_value = -9999.f;
constexpr int missing_cycle_minute = -1;
// Constants used to interface to Fortran subroutines
// Missing value for BUFR data
constexpr double r8bfms = 1.0E10;
// Maximum number of BUFR parameters
constexpr int mxr8pm = 10;
// Maximum number of BUFR levels
constexpr int mxr8lv_small = 255;
constexpr int mxr8lv = 1023; // was 255;
// Maximum number of BUFR event sequences
constexpr int mxr8vn = 10;
// Maximum number of BUFR variable types
constexpr int mxr8vt = 6;
// Maximum length of BUFR variable name
constexpr int mxr8nm = 8;
// Maximum number of BUFR variable types
constexpr int COUNT_THRESHOLD = 16;
// File unit number for opening the PrepBufr file
constexpr int file_unit = 11;
// 2nd file unit number for opening the PrepBufr file
constexpr int dump_unit = 22;
// Grib codes corresponding to the variable types
const std::array<int, mxr8vt> var_gc = {
pres_grib_code, spfh_grib_code, tmp_grib_code,
hgt_grib_code, ugrd_grib_code, vgrd_grib_code
};
static std::array<int, mxr8vt> bufr_var_code;
// Number of variable types which may be derived
constexpr static int n_derive_gc = 6;
// Listing of grib codes for variable types which may be derived
const std::array<int, n_derive_gc> derive_gc = {
dpt_grib_code, wdir_grib_code,
wind_grib_code, rh_grib_code,
mixr_grib_code, prmsl_grib_code
};
// The fortran code is hard-coded with 200 levels
constexpr int MAX_CAPE_VALUE = 10000;
constexpr int MAX_CAPE_LEVEL = 200;
constexpr int CAPE_INPUT_VARS = 3;
static float cape_data_pres[MAX_CAPE_LEVEL];
static float cape_data_temp[MAX_CAPE_LEVEL];
static float cape_data_spfh[MAX_CAPE_LEVEL];
static float static_dummy_200[MAX_CAPE_LEVEL];
static float static_dummy_201[MAX_CAPE_LEVEL+1];
//#define ROG 287.04
constexpr int MAX_PBL = 10000;
constexpr int MAX_PBL_LEVEL = 256;
constexpr int PBL_DEBUG_LEVEL = 8;
static bool IGNORE_Q_PBL = true;
static bool IGNORE_Z_PBL = true;
static bool USE_LOG_INTERPOLATION = true;
static float pbl_data_pres[MAX_PBL_LEVEL]; // mb
static float pbl_data_temp[MAX_PBL_LEVEL]; // Kelvin
static float pbl_data_spfh[MAX_PBL_LEVEL]; // ? / 1,000,000
static float pbl_data_hgt[MAX_PBL_LEVEL];
static float pbl_data_ugrd[MAX_PBL_LEVEL];
static float pbl_data_vgrd[MAX_PBL_LEVEL];
// PREPBUFR VIRTMP program code
constexpr double virtmp_prog_code = 8.0;
constexpr int MIN_FORTRAN_FILE_ID = 1;
constexpr int MAX_FORTRAN_FILE_ID = 99;
////////////////////////////////////////////////////////////////////////
//
// Variables for command line arguments
//
// StringArray to store PrepBufr file name
static StringArray pbfile;
// Output NetCDF file name
static ConcatString ncfile;
// Input configuration file
static ConcatString config_file;
static PB2NCConfInfo conf_info;
static MetNcPointObsOut nc_point_obs;
// Beginning and ending retention times
static unixtime valid_beg_ut;
static unixtime valid_end_ut;
// Number of PrepBufr messages to process from the command line
static int nmsg = -1;
static int nmsg_percent = -1;
// Dump contents of PrepBufr file to ASCII files
static bool dump_flag = false;
static ConcatString dump_dir = (string)".";
static bool do_all_vars = false;
static bool override_vars = false;
static bool collect_metadata = false;
static StringArray bufr_target_variables;
static ConcatString data_plane_filename;
static int compress_level = -1;
static bool save_summary_only = false;
////////////////////////////////////////////////////////////////////////
// Shared memory defined in the PREPBC common block in readpb.prm
static double hdr[mxr8pm];
static double evns[mxr8vt][mxr8vn][mxr8lv][mxr8pm];
static int nlev;
constexpr int BUFR_NUMBER_START = 13;
constexpr int BUFR_NAME_START = 2;
constexpr int BUFR_NAME_LEN = 8;
constexpr int BUFR_DESCRIPTION_START = 22;
constexpr int BUFR_DESCRIPTION_LEN = 56;
constexpr int BUFR_UNIT_START = 40;
constexpr int BUFR_UNIT_LEN = 25;
constexpr int BUFR_SEQUENCE_START = 13;
constexpr int BUFR_SEQUENCE_LEN = 66;
static const char *prepbufr_p_event = "POB PQM PPC PRC PFC PAN CAT";
static const char *prepbufr_q_event = "QOB QQM QPC QRC QFC QAN CAT";
static const char *prepbufr_t_event = "TOB TQM TPC TRC TFC TAN CAT";
static const char *prepbufr_z_event = "ZOB ZQM ZPC ZRC ZFC ZAN CAT";
static const char *prepbufr_u_event = "UOB WQM WPC WRC UFC UAN CAT";
static const char *prepbufr_v_event = "VOB WQM WPC WRC VFC VAN CAT";
static const char *prepbufr_hdrs_str = "SID XOB YOB DHR ELV TYP T29 ITP";
static const char *default_sid_name = "SID";
static const char *default_lat_name = "YOB";
static const char *default_lon_name = "XOB";
static const char *default_ymd_name = "YEAR MNTH DAYS";
static const char *default_hms_name = "HOUR MINU SECO";
static const char *airnow_aux_vars = "TPHR QCIND";
// Pick the latter one if exists multiuple variables
static const char *bufr_avail_sid_names = "SID SAID RPID";
static const char *bufr_avail_latlon_names = "XOB CLON CLONH YOB CLAT CLATH";
static const char *derived_mlcape = "D_MLCAPE";
static const char *derived_cape = "D_CAPE";
static const char *derived_pbl = "D_PBL";
constexpr float MLCAPE_INTERVAL = 3000.;
static double bufr_obs[mxr8lv][mxr8pm];
static double bufr_obs_extra[mxr8lv][mxr8pm];
static double bufr_pres_lv[mxr8lv]; // Retain the pressure in hPa
static double bufr_msl_lv[mxr8lv]; // Convert from geopotential height to MSL
static StringArray prepbufr_hdrs;
static StringArray prepbufr_vars;
static StringArray prepbufr_event_members;
static StringArray prepbufr_derive_vars;
static StringArray event_names;
static StringArray event_members;
static StringArray ascii_vars;
static StringArray var_names;
static StringArray var_units;
static StringArray tableB_vars;
static StringArray tableB_descs;
static ConcatString bufr_hdrs; // header name list to read header
static StringArray bufr_hdr_name_arr; // available header name list
static StringArray bufr_obs_name_arr; // available obs. name list
static IntArray filtered_times;
static vector<derive_var_cfg> bufr_derive_cfgs;
static map<ConcatString, StringArray> variableTypeMap;
static bool do_summary;
static SummaryObs *summary_obs;
////////////////////////////////////////////////////////////////////////
static int n_total_obs; // Running total of observations
static vector< Observation > observations;
//
// Output NetCDF file, dimensions, and variables
//
static NcFile *f_out = (NcFile *) nullptr;
////////////////////////////////////////////////////////////////////////
extern "C" {
void numpbmsg_(int *, int *);
void numpbmsg_new_(const char *, int *, int *);
void openpb_(const char *, int *);
void closepb_(int *);
void readpb_(int *, int *, int *, double[mxr8pm],
double[mxr8vt][mxr8vn][mxr8lv][mxr8pm], int *);
void readpb_hdr_ (int *,int *,double[mxr8pm]);
void ireadns_ (int *, char *, int *);
void readpbevt_(int *, int *, int *, double[mxr8vn][mxr8lv][mxr8pm],
char[mxr8lv*mxr8pm], int *, int *);
void readpbint_(int *, int *, int *, double[mxr8lv][mxr8pm],
char[mxr8lv*mxr8pm], int *, int *);
void dumppb_(const char *, int *, const char *, int *,
const char *, int *, int *);
void dump_tbl_(const char *, int *, const char *, int *);
int get_tmin_(int *, int *);
void calcape_(const int *, const int *, float [MAX_CAPE_LEVEL], float [MAX_CAPE_LEVEL],
float [MAX_CAPE_LEVEL],float *, float *, float *, float [MAX_CAPE_LEVEL+1],
const int *, const int *, const int *, const int *,
float *, float *, float *, float *, float [MAX_CAPE_LEVEL]);
void calpbl_(float[MAX_PBL_LEVEL], float[MAX_PBL_LEVEL], float[MAX_PBL_LEVEL],
float[MAX_PBL_LEVEL], float[MAX_PBL_LEVEL], float[MAX_PBL_LEVEL],
const int *, float *, int *);
}
// Defined at write_netcdf.cc
extern int obs_data_idx;
////////////////////////////////////////////////////////////////////////
static void initialize();
static void process_command_line(int, char **);
static void open_netcdf();
static void process_pbfile(int);
static void process_pbfile_metadata(int);
static void write_netcdf_hdr_data();
static void clean_up();
static void addObservation(const float *obs_arr, const ConcatString &hdr_typ,
const ConcatString &hdr_sid, const time_t hdr_vld,
const float hdr_lat, const float hdr_lon, const float hdr_elv,
const float quality_mark, const int buf_size);
static int get_event_index(int, int, int);
static int get_event_index_temp(int, int, int);
static void dbl2str(double *, ConcatString&);
static bool keep_message_type(const char *);
static bool keep_station_id(const char *);
static bool keep_valid_time(const unixtime, const unixtime,
const unixtime);
static bool keep_pb_report_type(int);
static bool keep_in_report_type(int);
static bool keep_instrument_type(int);
static bool keep_bufr_obs_index(int);
static bool keep_level_category(int);
static float derive_grib_code(int, float *, float *, double, float&);
static int combine_tqz_and_uv(map<float, float*>, map<float, float*>,
vector<float *> &);
static float compute_pbl(map<float, float*> pqtzuv_map_tq,
map<float, float*> pqtzuv_map_uv);
static void copy_pqtzuv(float *to_pqtzuv, float *from_pqtzuv, bool copy_all=true);
static bool insert_pbl(float *obs_arr, const float pbl_value, const int pbl_code,
const float pbl_p, const float pbl_h, const float pbl_qm,
const float hdr_lat, const float hdr_lon,
const float hdr_elv, const time_t hdr_vld_ut,
const ConcatString &hdr_typ, const ConcatString &hdr_sid);
static int interpolate_by_pressure(int length, float *pres_data, float *var_data);
static void interpolate_pqtzuv(float*, float*, float*);
static bool is_valid_pb_data(float pb_value);
static void check_fortran_file_id(const int unit, const char *method_name);
static void log_merged_tqz_uv(map<float, float*> pqtzuv_map_tq,
map<float, float*> pqtzuv_map_uv,
map<float, float*> &pqtzuv_map_merged,
const char *method_name);
static void log_pbl_input(int pbl_level, const char *method_name);
static void log_tqz_and_uv(map<float, float*> pqtzuv_map_tq,
map<float, float*> pqtzuv_map_uv,
const char *method_name);
static void merge_records(float *first_pqtzuv, map<float, float*> pqtzuv_map_pivot,
map<float, float*> pqtzuv_map_aux,
map<float, float*> &pqtzuv_map_merged);
static void usage();
static void set_pbfile(const StringArray &);
static void set_valid_beg_time(const StringArray &);
static void set_valid_end_time(const StringArray &);
static void set_nmsg(const StringArray &);
static void set_dump_path(const StringArray &);
static void set_collect_metadata(const StringArray &);
static void set_target_variables(const StringArray & a);
static void set_compress(const StringArray &);
static void display_bufr_variables(const StringArray &, const StringArray &,
const StringArray &, const StringArray &);
static void cleanup_hdr_typ(char *hdr_typ, bool is_prepbufr=false);
////////////////////////////////////////////////////////////////////////
void dump_pb_data(
int unit,
const ConcatString &prefix,
const ConcatString &blk_file,
const char *method_name
) {
int msg_typ_ret[n_vld_msg_typ];
int len1 = dump_dir.length();
int len2 = prefix.length();
mlog << Debug(1) << "Dumping to ASCII output directory:\t"
<< dump_dir << "\n";
for(int i=0; i<n_vld_msg_typ; i++) {
msg_typ_ret[i] = keep_message_type(vld_msg_typ_list[i]);
}
check_fortran_file_id(unit, method_name);
dumppb_(blk_file.c_str(), &unit, dump_dir.c_str(), &len1,
prefix.c_str(), &len2, msg_typ_ret);
}
////////////////////////////////////////////////////////////////////////
derive_var_cfg::derive_var_cfg(ConcatString _var_name): var_name{_var_name} {
var_index = bad_data_int;
}
////////////////////////////////////////////////////////////////////////
derive_var_cfg &derive_var_cfg::operator=(const derive_var_cfg &a) noexcept {
if ( this != &a ) {
var_index = a.var_index;
var_name = a.var_name;
}
return *this;
}
////////////////////////////////////////////////////////////////////////
int met_main(int argc, char *argv[]) {
// Initialize static variables
initialize();
// Process the command line arguments
process_command_line(argc, argv);
if (collect_metadata) {
// Process each PrepBufr file
for(int i=0; i<pbfile.n(); i++) {
process_pbfile_metadata(i);
}
display_bufr_variables(tableB_vars, tableB_descs,
bufr_hdr_name_arr, bufr_obs_name_arr);
}
else {
// Open the NetCDF file
open_netcdf();
// Process each PrepBufr file
for(int i=0; i<pbfile.n(); i++) {
process_pbfile_metadata(i);
process_pbfile(i);
}
mlog << Debug(2)
<< "\nTotal Observations retained or derived\t= " << n_total_obs << "\n";
if (do_summary) {
TimeSummaryInfo summaryInfo = conf_info.getSummaryInfo();
summary_obs->summarizeObs(summaryInfo);
summary_obs->setSummaryInfo(summaryInfo);
}
// Write the NetCDF file
write_netcdf_hdr_data();
}
// Deallocate memory and clean up
clean_up();
return 0;
}
////////////////////////////////////////////////////////////////////////
string get_tool_name() {
return program_name;
}
////////////////////////////////////////////////////////////////////////
void initialize() {
n_total_obs = 0;
nc_point_obs.init_buffer();
prepbufr_vars.clear();
prepbufr_hdrs.clear();
prepbufr_event_members.clear();
prepbufr_hdrs.parse_wsss(prepbufr_hdrs_str);
StringArray tmp_hdr_array;
tmp_hdr_array.clear();
tmp_hdr_array.parse_wsss(prepbufr_p_event);
prepbufr_event_members.add(tmp_hdr_array);
if (0 < tmp_hdr_array.n()) prepbufr_vars.add(tmp_hdr_array[0]);
tmp_hdr_array.clear();
tmp_hdr_array.parse_wsss(prepbufr_q_event);
prepbufr_event_members.add(tmp_hdr_array);
if (0 < tmp_hdr_array.n()) prepbufr_vars.add(tmp_hdr_array[0]);
tmp_hdr_array.clear();
tmp_hdr_array.parse_wsss(prepbufr_t_event);
prepbufr_event_members.add(tmp_hdr_array);
if (0 < tmp_hdr_array.n()) prepbufr_vars.add(tmp_hdr_array[0]);
tmp_hdr_array.clear();
tmp_hdr_array.parse_wsss(prepbufr_z_event);
prepbufr_event_members.add(tmp_hdr_array);
if (0 < tmp_hdr_array.n()) prepbufr_vars.add(tmp_hdr_array[0]);
tmp_hdr_array.clear();
tmp_hdr_array.parse_wsss(prepbufr_u_event);
prepbufr_event_members.add(tmp_hdr_array);
if (0 < tmp_hdr_array.n()) prepbufr_vars.add(tmp_hdr_array[0]);
tmp_hdr_array.clear();
tmp_hdr_array.parse_wsss(prepbufr_v_event);
prepbufr_event_members.add(tmp_hdr_array);
if (0 < tmp_hdr_array.n()) prepbufr_vars.add(tmp_hdr_array[0]);
prepbufr_derive_vars.add("D_DPT");
prepbufr_derive_vars.add("D_WDIR");
prepbufr_derive_vars.add("D_WIND");
prepbufr_derive_vars.add("D_RH");
prepbufr_derive_vars.add("D_MIXR");
prepbufr_derive_vars.add("D_PRMSL");
prepbufr_derive_vars.add("D_CAPE");
prepbufr_derive_vars.add("D_MLCAPE");
prepbufr_derive_vars.add("D_PBL");
for (int idx=0; idx<(sizeof(hdr) / sizeof(hdr[0])); idx++) {
hdr[idx] = r8bfms * 10;
}
summary_obs = new SummaryObs();
return;
}
////////////////////////////////////////////////////////////////////////
void process_command_line(int argc, char **argv) {
CommandLine cline;
ConcatString default_config_file;
// Check for zero arguments
if(argc == 1) usage();
// Initialize retention times
valid_beg_ut = valid_end_ut = (unixtime) 0;
// Parse the command line into tokens
cline.set(argc, argv);
// Set the usage function
cline.set_usage(usage);
// Add the options function calls
cline.add(set_pbfile, "-pbfile", 1);
cline.add(set_valid_beg_time, "-valid_beg", 1);
cline.add(set_valid_end_time, "-valid_end", 1);
cline.add(set_nmsg, "-nmsg", 1);
cline.add(set_dump_path, "-dump", 1);
cline.add(set_collect_metadata, "-index", 0);
cline.add(set_target_variables, "-vars", 1);
cline.add(set_compress, "-compress", 1);
// Parse the command line
cline.parse();
// Check for error. There should be three arguments left:
// PrepBufr, output NetCDF, and config filenames
if(cline.n() != 3 && !collect_metadata) usage();
// Store the input file names
pbfile.add(cline[0]);
if(cline.n() > 1) ncfile = cline[1];
if(cline.n() > 2) config_file = cline[2];
// Create the default config file name
default_config_file = replace_path(default_config_filename);
if(cline.n() < 2) config_file = default_config_file;
else if(cline.n() < 3) config_file = cline[1];
// List the config files
mlog << Debug(1)
<< "Default Config File: " << default_config_file << "\n"
<< "User Config File: " << config_file << "\n";
// Read the config files
conf_info.read_config(default_config_file.c_str(), config_file.c_str());
// Process the configuration
conf_info.process_config();
// Check that valid_end_ut >= valid_beg_ut
if(valid_beg_ut != (unixtime) 0 &&
valid_end_ut != (unixtime) 0 &&
valid_beg_ut > valid_end_ut) {
mlog << Error << "\nprocess_command_line() -> "
<< "the ending time (" << unix_to_yyyymmdd_hhmmss(valid_end_ut)
<< ") must be greater than the beginning time ("
<< unix_to_yyyymmdd_hhmmss(valid_beg_ut) << ").\n\n";
exit(1);
}
if (!override_vars) {
bufr_target_variables = conf_info.obs_bufr_var;
do_all_vars = (0 == conf_info.obs_bufr_var.n());
}
do_summary = conf_info.getSummaryInfo().flag;
if (!do_summary) save_summary_only = false;
else save_summary_only = !conf_info.getSummaryInfo().raw_data;
return;
}
////////////////////////////////////////////////////////////////////////
ConcatString save_bufr_table_to_file(const char *blk_file, int _file_id) {
int len;
ConcatString tbl_filename;
ConcatString tbl_prefix;
const char *method_name = "save_bufr_table_to_file() ";
tbl_prefix << conf_info.tmp_dir << "/" << tmp_pb2nc_base;
tbl_filename = make_temp_file_name(tbl_prefix.c_str(), "tbl");
len = tbl_filename.length();
check_fortran_file_id(_file_id, method_name);
openpb_(blk_file, &_file_id);
dump_tbl_(blk_file, &_file_id, tbl_filename.c_str(), &len);
closepb_(&_file_id);
return tbl_filename;
}
////////////////////////////////////////////////////////////////////////
bool is_prepbufr_file(const StringArray *events) {
bool is_prepbufr = events->has("P__EVENT") && events->has("Q__EVENT")
&& events->has("T__EVENT") && events->has("Z__EVENT");
return is_prepbufr;
}
////////////////////////////////////////////////////////////////////////
void get_variable_info(ConcatString blk_file, int unit) {
static const char *method_name = " get_variable_info()";
FILE * fp;
char * line = nullptr;
size_t len = 1024;
event_names.clear();
event_members.clear();
ascii_vars.clear();
var_names.clear();
var_units.clear();
tableB_vars.clear();
tableB_descs.clear();
ConcatString tbl_filename = save_bufr_table_to_file(blk_file.c_str(), unit);
fp = fopen(tbl_filename.c_str(), "r");
if (fp != nullptr) {
char var_name[BUFR_NAME_LEN+1];
char var_desc[max(BUFR_DESCRIPTION_LEN,BUFR_SEQUENCE_LEN)+1];
char var_unit_str[BUFR_UNIT_LEN+1];
bool find_mnemonic = false;
line = (char *)malloc(len * sizeof(char));
if( line == nullptr) {
mlog << Error << "\n" << method_name << " -> "
<< "Unable to allocate buffer\n\n";
exit(1);
}
// Processing section 1
int var_count1 = 0;
while (getline(&line, &len, fp) != -1) {
if (nullptr != strstr(line,"--------")) continue;
if (nullptr != strstr(line,"MNEMONIC")) {
if (find_mnemonic) break;
find_mnemonic = true;
continue;
}
if ('0' != line[BUFR_NUMBER_START]) continue;
m_strncpy(var_name, (line+BUFR_NAME_START), BUFR_NAME_LEN,
method_name, "var_name", true);
var_name[BUFR_NAME_LEN] = '\0';
for (int idx=(BUFR_NAME_LEN-1); idx >=0; idx--) {
if (' ' != var_name[idx] ) break;
var_name[idx] = '\0';
}
if (0 == m_strlen(var_name)) continue;
var_count1++;
m_strncpy(var_desc, (line+BUFR_DESCRIPTION_START), BUFR_DESCRIPTION_LEN,
method_name, "var_desc", true);
var_desc[BUFR_DESCRIPTION_LEN] = '\0';
for (int idx=(BUFR_DESCRIPTION_LEN-1); idx>=0; idx--) {
if (' ' != var_desc[idx] && '|' != var_desc[idx]) {
break;
}
var_desc[idx] = '\0';
}
mlog << Debug(10) << method_name << " sec. 1 var: ["
<< var_name << "] \tdesc: [" << var_desc << "]\n";
tableB_vars.add(var_name);
tableB_descs.add(var_desc);
}
// Skip section 2
while (getline(&line, &len, fp) != -1) {
if (nullptr != strstr(line,"MNEMONIC")) break;
if (nullptr == strstr(line,"EVENT")) continue;
m_strncpy(var_name, (line+BUFR_NAME_START), BUFR_NAME_LEN,
method_name, "var_name2", true);
var_name[BUFR_NAME_LEN] = '\0';
for (int idx=(BUFR_NAME_LEN-1); idx >=0; idx--) {
if (' ' != var_name[idx] ) break;
var_name[idx] = '\0';
}
m_strncpy(var_desc, (line+BUFR_SEQUENCE_START), BUFR_SEQUENCE_LEN,
method_name, "var_desc2", true);
var_desc[BUFR_SEQUENCE_LEN] = '\0';
for (int idx=(BUFR_SEQUENCE_LEN-1); idx>=0; idx--) {
if (' ' != var_desc[idx] && '|' != var_desc[idx]) {
break;
}
var_desc[idx] = '\0';
}
mlog << Debug(10) << method_name << " event: ["
<< var_name << "] \tdesc: [" << var_desc << "]\n";
event_names.add(var_name);
event_members.add(var_desc);
}
getline(&line, &len, fp);
// Processing section 3
while (getline(&line, &len, fp) != -1) {
if (' ' == line[BUFR_NAME_START]) continue;
if ('-' == line[BUFR_NAME_START]) break;
m_strncpy(var_name, (line+BUFR_NAME_START), BUFR_NAME_LEN,
method_name, "var_name3", true);
var_name[BUFR_NAME_LEN] = '\0';
for (int idx=(BUFR_NAME_LEN-1); idx >=0; idx--) {
if (' ' != var_name[idx] ) break;
var_name[idx] = '\0';
}
if (nullptr != strstr(line,"CCITT IA5")) {
ascii_vars.add(var_name);
m_strncpy(var_unit_str, "CCITT IA5", sizeof(var_unit_str),
method_name, "var_unit_str1", true);
}
else {
m_strncpy(var_unit_str, (line+BUFR_UNIT_START), BUFR_UNIT_LEN,
method_name, "var_unit_str2", true);
var_unit_str[BUFR_UNIT_LEN] = '\0';
for (int idx=(BUFR_UNIT_LEN-1); idx>=0; idx--) {
if (' ' != var_unit_str[idx] && '|' != var_unit_str[idx]) {
break;
}
var_unit_str[idx] = '\0';
}
var_names.add(var_name);
var_units.add(var_unit_str);
mlog << Debug(10) << method_name << " sec. 3 var: ["
<< var_name << "] \tdesc: [" << var_desc << "]\n";
}
}
fclose(fp);
if (line) free(line);
}
remove_temp_file(tbl_filename);
return;
}
////////////////////////////////////////////////////////////////////////
void open_netcdf() {
// Create the output netCDF file for writing
mlog << Debug(1) << "Creating NetCDF File:\t\t" << ncfile << "\n";
f_out = open_ncfile(ncfile.c_str(), true);
// Check for a valid file
if(IS_INVALID_NC_P(f_out)) {
mlog << Error << "\nopen_netcdf() -> "
<< "trouble opening output file: " << ncfile << "\n\n";
delete f_out;
f_out = (NcFile *) nullptr;
exit(1);
}
// Define netCDF variables
bool use_var_id = true;
int deflate_level = compress_level;
if (deflate_level < 0) deflate_level = conf_info.conf.nc_compression();
nc_point_obs.set_netcdf(f_out, true);
nc_point_obs.init_obs_vars(use_var_id, deflate_level);
return;
}
////////////////////////////////////////////////////////////////////////
void process_pbfile(int i_pb) {
int npbmsg, npbmsg_total, unit, yr, mon, day, hr, min, sec;
int i, i_msg, n_file_obs, i_ret, i_date, n_hdr_obs;
int rej_typ, rej_sid, rej_vld, rej_grid, rej_poly;
int rej_elv, rej_pb_rpt, rej_in_rpt, rej_itp, rej_nobs;
int lv, ev, ev_temp, kk;
int n_derived_obs;
double x;
double y;
int cycle_minute;
unixtime file_ut;
unixtime adjusted_file_ut;
unixtime msg_ut, beg_ut, end_ut;
unixtime min_msg_ut, max_msg_ut;
file_ut = beg_ut = end_ut = (unixtime) 0;
ConcatString file_name, blk_prefix, blk_file, log_message;
ConcatString prefix;
char time_str[max_str_len];
ConcatString start_time_str, end_time_str;
char min_time_str[max_str_len], max_time_str[max_str_len];
char hdr_typ[max_str_len];
ConcatString hdr_sid;
char modified_hdr_typ[max_str_len];
double hdr_lat = bad_data_double;
double hdr_lon = bad_data_double;
double hdr_elv = bad_data_double;
float pb_report_type, in_report_type, instrument_type;
unixtime hdr_vld_ut = (unixtime) 0;
float quality_mark, dl_category;
float obs_arr[OBS_ARRAY_LEN];
float pqtzuv[mxr8vt], pqtzuv_qty[mxr8vt];
const int debug_level_for_performance = 3;
clock_t start_t, end_t, method_start, method_end;
start_t = end_t = method_start = method_end = clock();
IntArray unique_msg_ut;
StringArray variables_big_nlevels;
static const char *method_name_s = "process_pbfile()";
static const char *method_name = "process_pbfile() -> ";
bool apply_grid_mask = (conf_info.grid_mask.nx() > 0 &&
conf_info.grid_mask.ny() > 0);
bool apply_area_mask = (conf_info.area_mask.nx() > 0 &&
conf_info.area_mask.ny() > 0);
bool apply_poly_mask = (conf_info.poly_mask.n_points() > 0);
// List the PrepBufr file being processed
mlog << Debug(1) << "Processing Bufr File:\t" << pbfile[i_pb]<< "\n";
// Initialize
filtered_times.clear();
min_msg_ut = max_msg_ut = (unixtime) 0;
min_time_str[0] = '\0';
max_time_str[0] = '\0';
// Set the file name for the PrepBufr file
file_name << pbfile[i_pb];
// Build the temporary block file name
blk_prefix << conf_info.tmp_dir << "/" << tmp_pb2nc_base;
blk_file = make_temp_file_name(blk_prefix.c_str(), nullptr);
mlog << Debug(1) << "Blocking Bufr file to:\t" << blk_file << "\n";
// Assume that the input PrepBufr file is unblocked.
// Block the PrepBufr file and open it for reading.
pblock(file_name.c_str(), blk_file.c_str(), Action::block);
// Dump the contents of the PrepBufr file to ASCII files
if(dump_flag) {
// Check for multiple PrepBufr files
if(pbfile.n() > 1) {
mlog << Error << "\n" << method_name
<< "the \"-dump\" and \"-pbfile\" options may not be "
<< "used together. Only one Bufr file may be dump "
<< "to ASCII at a time.\n\n";
exit(1);
}
unit = dump_unit+i_pb;
prefix = get_short_name(pbfile[i_pb].c_str());
dump_pb_data((dump_unit+i_pb), prefix, blk_file, method_name_s);
}
// Open the blocked temp PrepBufr file for reading
unit = file_unit + i_pb;
check_fortran_file_id(unit, method_name_s);
openpb_(blk_file.c_str(), &unit);
// Compute the number of PrepBufr records in the current file.
numpbmsg_(&unit, &npbmsg);
npbmsg_total = npbmsg;
// Use the number of records requested by the user if there
// are enough present.
if(nmsg > 0 && nmsg < npbmsg) {
npbmsg = (nmsg_percent > 0 && nmsg_percent <= 100)
? (npbmsg * nmsg_percent / 100) : nmsg;
}
// Check for zero messages to process
if(npbmsg <= 0 || npbmsg_total <= 0) {
mlog << Warning << "\n" << method_name
<< "No Bufr messages to process in file: "
<< pbfile[i_pb] << "\n\n";
// Delete the temporary blocked file
remove_temp_file(blk_file);
return;
}
// Initialize counts
n_derived_obs = n_file_obs = 0;
i_ret = i_msg = 0;
rej_typ = rej_sid = rej_vld = rej_grid = rej_poly = 0;
rej_elv = rej_pb_rpt = rej_in_rpt = rej_itp = rej_nobs = 0;
//processed_count = 0;
int buf_nlev;
bool showed_progress = false;
bool is_prepbufr = is_prepbufr_file(&event_names);
if(mlog.verbosity_level() >= debug_level_for_performance) {
end_t = clock();
mlog << Debug(debug_level_for_performance) << " PERF: " << method_name_s << " "
<< (end_t-start_t)/double(CLOCKS_PER_SEC)
<< " seconds for preparing\n";
start_t = clock();
}
log_message = (is_prepbufr ? " PrepBufr" : " Bufr");
log_message.add(" messages");
if (npbmsg != npbmsg_total) {
log_message << " (out of " << unixtime_to_string(npbmsg_total) << ")";
}
mlog << Debug(2) << "Processing " << npbmsg << log_message << "...\n";