Skip to content

Commit

Permalink
Per #2574, update the mode engine to store a pointer to the split fie…
Browse files Browse the repository at this point in the history 
…ld rather than the individual object fields. And compute pair intersection, union, and sym diff counts from the split field rather than the object fields. This avoid allocating memory for many, many copies of the input grid, which can be quite large depending on the resolution.
  • Loading branch information
@JohnHalleyGotway
JohnHalleyGotway committed Nov 3, 2023
1 parent 5bdf1bd commit 6547387
Show file tree
Hide file tree
Showing 3 changed files with 75 additions and 110 deletions.
125 changes: 44 additions & 81 deletions src/libcode/vx_shapedata/engine.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1033,32 +1033,22 @@ void ModeFuzzyEngine::do_matching() {

void ModeFuzzyEngine::do_no_match() {
int j, k, n;
ShapeData * fcst_shape = (ShapeData *) 0;
ShapeData * obs_shape = (ShapeData *) 0;
ShapeData cur_shape;

do_fcst_splitting();
do_obs_splitting();

clear_colors();

fcst_shape = new ShapeData [n_fcst];
obs_shape = new ShapeData [n_obs];

if(!fcst_shape || !obs_shape) {

mlog << Error << "\nModeFuzzyEngine::do_no_match() -> "
<< "memory allocation error\n\n";
exit(1);
}

//
// Do the single features
//
fcst_single.set_size(n_fcst);

for(j=0; j<n_fcst; j++) {
fcst_shape[j] = select(*fcst_split, j+1);
fcst_single[j].set(*fcst_raw, *fcst_thresh, fcst_shape[j],
cur_shape = select(*fcst_split, j+1);
fcst_single[j].set(*fcst_raw, *fcst_thresh,
*fcst_split, cur_shape,
conf_info.inten_perc_value,
conf_info.Fcst->var_info->is_precipitation());
fcst_single[j].object_number = j+1;
Expand All @@ -1067,8 +1057,9 @@ void ModeFuzzyEngine::do_no_match() {
obs_single.set_size(n_obs);

for(j=0; j<n_obs; j++) {
obs_shape[j] = select(*obs_split, j+1);
obs_single[j].set(*obs_raw, *obs_thresh, obs_shape[j],
cur_shape = select(*obs_split, j+1);
obs_single[j].set(*obs_raw, *obs_thresh,
*obs_split, cur_shape,
conf_info.inten_perc_value,
conf_info.Obs->var_info->is_precipitation());
obs_single[j].object_number = j+1;
Expand Down Expand Up @@ -1112,9 +1103,6 @@ void ModeFuzzyEngine::do_no_match() {
// Done
//

delete [] fcst_shape; fcst_shape = (ShapeData *) 0;
delete [] obs_shape; obs_shape = (ShapeData *) 0;

return;
}

Expand All @@ -1123,32 +1111,22 @@ void ModeFuzzyEngine::do_no_match() {
void ModeFuzzyEngine::do_match_merge() {
int j, k, n;
InterestInfo junkinfo;
ShapeData * fcst_shape = (ShapeData *) 0;
ShapeData * obs_shape = (ShapeData *) 0;
ShapeData cur_shape;

do_fcst_splitting();
do_obs_splitting();

clear_colors();

fcst_shape = new ShapeData [n_fcst];
obs_shape = new ShapeData [n_obs];

if(!fcst_shape || !obs_shape) {

mlog << Error << "\nModeFuzzyEngine::do_match_merge() -> "
<< "memory allocation error\n\n";
exit(1);
}

//
// Do the single features
//
fcst_single.set_size(n_fcst);

for(j=0; j<n_fcst; j++) {
fcst_shape[j] = select(*fcst_split, j+1);
fcst_single[j].set(*fcst_raw, *fcst_thresh, fcst_shape[j],
cur_shape = select(*fcst_split, j+1);
fcst_single[j].set(*fcst_raw, *fcst_thresh,
*fcst_split, cur_shape,
conf_info.inten_perc_value,
conf_info.Fcst->var_info->is_precipitation());
fcst_single[j].object_number = j+1;
Expand All @@ -1157,8 +1135,9 @@ void ModeFuzzyEngine::do_match_merge() {
obs_single.set_size(n_obs);

for(j=0; j<n_obs; j++) {
obs_shape[j] = select(*obs_split, j+1);
obs_single[j].set(*obs_raw, *obs_thresh, obs_shape[j],
cur_shape = select(*obs_split, j+1);
obs_single[j].set(*obs_raw, *obs_thresh,
*obs_split, cur_shape,
conf_info.inten_perc_value,
conf_info.Obs->var_info->is_precipitation());
obs_single[j].object_number = j+1;
Expand Down Expand Up @@ -1277,9 +1256,6 @@ void ModeFuzzyEngine::do_match_merge() {
// Done
//

delete [] fcst_shape; fcst_shape = (ShapeData *) 0;
delete [] obs_shape; obs_shape = (ShapeData *) 0;

return;
}

Expand All @@ -1296,6 +1272,7 @@ void ModeFuzzyEngine::do_fcst_merge_thresh() {
int n_fcst_merge, intersection;
int count, first_k;
ShapeData fcst_merge_mask, fcst_merge_split;
// TODO: work here
ShapeData * fcst_shape = (ShapeData *) 0;
ShapeData * fcst_merge_shape = (ShapeData *) 0;

Expand Down Expand Up @@ -1419,6 +1396,7 @@ void ModeFuzzyEngine::do_obs_merge_thresh() {
int n_obs_merge, intersection;
int count, first_k;
ShapeData obs_merge_mask, obs_merge_split;
// TODO: work here
ShapeData * obs_shape = (ShapeData *) 0;
ShapeData * obs_merge_shape = (ShapeData *) 0;

Expand Down Expand Up @@ -1528,13 +1506,15 @@ void ModeFuzzyEngine::do_obs_merge_thresh() {
return;
}

///////////////////////////////////////////////////////////////////////

void ModeFuzzyEngine::do_fcst_merge_thresh(const ShapeData &merge_data) {
int j, k, x, y;
int n_fcst_merge, intersection;
int count, first_k;
//ShapeData fcst_merge_mask;
ShapeData fcst_merge_split;
// TODO: work here
ShapeData * fcst_shape = (ShapeData *) 0;
ShapeData * fcst_merge_shape = (ShapeData *) 0;

Expand Down Expand Up @@ -1646,9 +1626,12 @@ void ModeFuzzyEngine::do_fcst_merge_thresh(const ShapeData &merge_data) {
delete [] fcst_merge_shape; fcst_merge_shape = (ShapeData *) 0;

mlog << Debug(4) << " After merging of fcst field: " << fcst_split->sdebug_examine() << "\n";

return;
}

///////////////////////////////////////////////////////////////////////

void ModeFuzzyEngine::do_obs_merge_thresh(const ShapeData &merge_data) {
int j, k, x, y;
int n_obs_merge, intersection;
Expand Down Expand Up @@ -2114,31 +2097,22 @@ void ModeFuzzyEngine::do_obs_merge_engine(const char *default_config,
void ModeFuzzyEngine::do_match_fcst_merge() {
int j, k, n;
InterestInfo junkinfo;
ShapeData * fcst_shape = (ShapeData *) 0;
ShapeData * obs_shape = (ShapeData *) 0;
ShapeData cur_shape;

do_fcst_splitting();
do_obs_splitting();

clear_colors();

fcst_shape = new ShapeData [n_fcst];
obs_shape = new ShapeData [n_obs];

if(!fcst_shape || !obs_shape) {
mlog << Error << "\nModeFuzzyEngine::do_match_fcst_merge() -> "
<< "memory allocation error\n\n";
exit(1);
}

//
// Do the single features
//
fcst_single.set_size(n_fcst);

for(j=0; j<n_fcst; j++) {
fcst_shape[j] = select(*fcst_split, j+1);
fcst_single[j].set(*fcst_raw, *fcst_thresh, fcst_shape[j],
cur_shape = select(*fcst_split, j+1);
fcst_single[j].set(*fcst_raw, *fcst_thresh,
*fcst_split, cur_shape,
conf_info.inten_perc_value,
conf_info.Fcst->var_info->is_precipitation());
fcst_single[j].object_number = j+1;
Expand All @@ -2147,8 +2121,9 @@ void ModeFuzzyEngine::do_match_fcst_merge() {
obs_single.set_size(n_obs);

for(j=0; j<n_obs; j++) {
obs_shape[j] = select(*obs_split, j+1);
obs_single[j].set(*obs_raw, *obs_thresh, obs_shape[j],
cur_shape = select(*obs_split, j+1);
obs_single[j].set(*obs_raw, *obs_thresh,
*obs_split, cur_shape,
conf_info.inten_perc_value,
conf_info.Obs->var_info->is_precipitation());
obs_single[j].object_number = j+1;
Expand Down Expand Up @@ -2275,9 +2250,6 @@ void ModeFuzzyEngine::do_match_fcst_merge() {
// Done
//

delete [] fcst_shape; fcst_shape = (ShapeData *) 0;
delete [] obs_shape; obs_shape = (ShapeData *) 0;

return;
}

Expand All @@ -2291,31 +2263,22 @@ void ModeFuzzyEngine::do_match_fcst_merge() {
void ModeFuzzyEngine::do_match_only() {
int j, k, n;
InterestInfo junkinfo;
ShapeData * fcst_shape = (ShapeData *) 0;
ShapeData * obs_shape = (ShapeData *) 0;
ShapeData cur_shape;

do_fcst_splitting();
do_obs_splitting();

clear_colors();

fcst_shape = new ShapeData [n_fcst];
obs_shape = new ShapeData [n_obs];

if(!fcst_shape || !obs_shape) {
mlog << Error << "\nModeFuzzyEngine::do_match_only() -> "
<< "memory allocation error\n\n";
exit(1);
}

//
// Do the single features
//
fcst_single.set_size(n_fcst);

for(j=0; j<n_fcst; j++) {
fcst_shape[j] = select(*fcst_split, j+1);
fcst_single[j].set(*fcst_raw, *fcst_thresh, fcst_shape[j],
cur_shape = select(*fcst_split, j+1);
fcst_single[j].set(*fcst_raw, *fcst_thresh,
*fcst_split, cur_shape,
conf_info.inten_perc_value,
conf_info.Fcst->var_info->is_precipitation());
fcst_single[j].object_number = j+1;
Expand All @@ -2324,8 +2287,9 @@ void ModeFuzzyEngine::do_match_only() {
obs_single.set_size(n_obs);

for(j=0; j<n_obs; j++) {
obs_shape[j] = select(*obs_split, j+1);
obs_single[j].set(*obs_raw, *obs_thresh, obs_shape[j],
cur_shape = select(*obs_split, j+1);
obs_single[j].set(*obs_raw, *obs_thresh,
*obs_split, cur_shape,
conf_info.inten_perc_value,
conf_info.Obs->var_info->is_precipitation());
obs_single[j].object_number = j+1;
Expand Down Expand Up @@ -2446,9 +2410,6 @@ void ModeFuzzyEngine::do_match_only() {
// Done
//

delete [] fcst_shape; fcst_shape = (ShapeData *) 0;
delete [] obs_shape; obs_shape = (ShapeData *) 0;

return;
}

Expand Down Expand Up @@ -2566,15 +2527,17 @@ void ModeFuzzyEngine::do_cluster_features() {

for(j=0; j<n_clus; j++) {
fcst_clus_shape[j] = select(*fcst_clus_split, j+1);
fcst_cluster[j].set(*fcst_raw, *fcst_thresh, fcst_clus_shape[j],
conf_info.inten_perc_value,
conf_info.Fcst->var_info->is_precipitation());
fcst_cluster[j].set(*fcst_raw, *fcst_thresh,
*fcst_clus_split, fcst_clus_shape[j],
conf_info.inten_perc_value,
conf_info.Fcst->var_info->is_precipitation());
fcst_cluster[j].object_number = j+1;

obs_clus_shape[j] = select(*obs_clus_split, j+1);
obs_cluster[j].set(*obs_raw, *obs_thresh, obs_clus_shape[j],
conf_info.inten_perc_value,
conf_info.Obs->var_info->is_precipitation());
obs_cluster[j].set(*obs_raw, *obs_thresh,
*obs_clus_split, obs_clus_shape[j],
conf_info.inten_perc_value,
conf_info.Obs->var_info->is_precipitation());
obs_cluster[j].object_number = j+1;
}

Expand All @@ -2585,7 +2548,7 @@ void ModeFuzzyEngine::do_cluster_features() {

for(j=0; j<n_clus; j++) {
pair_cluster[j].set(fcst_cluster[j], obs_cluster[j],
conf_info.max_centroid_dist);
conf_info.max_centroid_dist);
pair_cluster[j].pair_number = j+1;
}

Expand Down
Loading

0 comments on commit 6547387

Please sign in to comment.