Skip to content

Commit

Permalink
Retrieving MAs from psconfig is working #76
Browse files Browse the repository at this point in the history
  • Loading branch information
apertome committed Feb 5, 2019
1 parent d7bafe3 commit c7e565a
Show file tree
Hide file tree
Showing 4 changed files with 113 additions and 13 deletions.
119 changes: 109 additions & 10 deletions api/admin/controllers/importer.js
Original file line number Diff line number Diff line change
Expand Up @@ -224,10 +224,14 @@ exports._detect_config_type = function( config ) {

exports._process_imported_config = function ( importedConfig, sub, cb, disable_ensure_hosts) {



var config_format = exports._detect_config_type( importedConfig );

// Main config object
var mainConfig = {};
mainConfig.tests = [];
mainConfig.testspecs = [];
mainConfig.hostgroups = [];

//var out = importedConfig;
//out = JSON.stringify( out, null, "\t" );

Expand All @@ -237,15 +241,23 @@ exports._process_imported_config = function ( importedConfig, sub, cb, disable_e

// config_params holds parameters to pass back to the callback
var config_params = {};
var tests = [];

if ( config_format == "meshconfig" ) {
exports._process_meshconfig( importedConfig, sub, config_params, tests, cb );
exports._process_meshconfig( importedConfig, sub, config_params, mainConfig, cb );
} else if ( config_format == "psconfig" ) {
exports._process_psconfig( importedConfig, sub, config_params, tests, cb );
exports._process_psconfig( importedConfig, sub, config_params, mainConfig, cb );

}

//if ( config_format == "psconfig" ) {

console.log("MAINCONFIG INTERMEDIATE\n");
console.log( JSON.stringify( mainConfig, null, 3 ));

console.log("CONFIG_PARAMS INTERMEDIATE\n");
console.log( JSON.stringify( config_params, null, 3 ));

//}


//now do update (TODO - should I let caller handle this?)
Expand All @@ -269,7 +281,11 @@ exports._process_imported_config = function ( importedConfig, sub, cb, disable_e

};

exports._process_meshconfig = function ( importedConfig, sub, config_params, tests, cb ) {
exports._process_meshconfig = function ( importedConfig, sub, config_params, mainConfig, cb ) {
var tests = mainConfig.tests;
var hostgroups = mainConfig.hostgroups;
var testspecs = mainConfig.testspecs;

var config_desc = importedConfig.description;
if ( config_desc ) {
config_params.description = config_desc;
Expand All @@ -287,10 +303,11 @@ exports._process_meshconfig = function ( importedConfig, sub, config_params, tes

var ma_urls = Object.keys( ma_url_obj );
config_params.archives = ma_urls;
importedConfig.ma_urls = ma_urls;
//importedConfig.ma_urls = ma_urls;
//importedConfig.config_params = config_params;

//console.log("IMPORTER ma_urls", ma_urls);
//console.log("config_params", config_params);


//process hosts
Expand Down Expand Up @@ -334,8 +351,8 @@ exports._process_meshconfig = function ( importedConfig, sub, config_params, tes


//process hostgroups / testspecs
var hostgroups = [];
var testspecs = [];
//var hostgroups = [];
//var testspecs = [];
//var tests = [];


Expand Down Expand Up @@ -400,6 +417,88 @@ exports._process_meshconfig = function ( importedConfig, sub, config_params, tes

};

exports._process_psconfig = function ( importedConfig, sub, cb, disable_ensure_hosts) {
exports._process_psconfig = function ( importedConfig, sub, config_params, mainConfig, cb ) {
var tests = mainConfig.tests;
var hostGroups = mainConfig.hostgroups;
var testspecs = mainConfig.testspecs;

var config_desc = importedConfig._meta["display-name"];
if ( config_desc ) {
config_params.description = config_desc;
}

var archive_obj = exports._extract_psconfig_mas( importedConfig , config_params );

console.log("archive_obj", archive_obj);

config_params.archives = archive_obj.central;

console.log("config_params", config_params);

var hosts_obj = {};
hosts_obj = exports._extract_psconfig_hosts( importedConfig, config_params );
config_params.hosts = hosts_obj.hosts;
config_params.addresses = hosts_obj.addresses;




};

exports._extract_psconfig_hosts = function( importedConfig, config_params ) {
var hosts_obj = {};

// Retrieve host info from importedConfig
var hosts_info = [];



return hosts_info;
};

exports._extract_psconfig_mas = function( importedConfig, config_params ) {
// Use a hash, keyed on archive type, to return the MA info
// i.e. archives.central = []
// archives.host = [] // Local MAs for testpoints
var archives = {};
archives.central = [];
archives.host = [];

// process central MAs
var ma_url_obj = {};
if ( "archives" in importedConfig ) {
console.log("ARCHIVES");
Object.keys( importedConfig.archives ).forEach(function(ma_name) {
var ma = importedConfig.archives[ ma_name ];
if ( ! ( "archives" in config_params ) ) config_params.archives = [];
var ma_url = importedConfig.archives[ ma_name ].data.url;
ma_url_obj[ ma_url ] = 1;

if ( "tasks" in importedConfig ) {
var num_ma_instances = 0;
_.each( importedConfig.tasks, function( task, taskName ) {
if ( _.indexOf( task.archives, ma_name ) > -1 ) {
num_ma_instances++;
}
});
// If the MA appears in all task specs, we consider it a central MA
if ( num_ma_instances == Object.keys( importedConfig.tasks).length) {
archives.central.push( ma_url );
} else {
// Otherwise, it is a host MA
archives.host.push( ma_url );

}
};
//console.log("IMPORTED", JSON.stringify(importedConfig, null, "\t"));
console.log("archives", JSON.stringify( archives, null, "\t"));
console.log("ma_url_obj", JSON.stringify( ma_url_obj, null, "\t"));
var ma_urls = Object.keys( ma_url_obj );

});

}

return archives;

};
2 changes: 1 addition & 1 deletion test/data/latentput-meshconfig.json
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"organizations":[{"sites":[{"hosts":[{"addresses":["ps-latency.hep.pnnl.gov"],"description":"Overridden description","toolkit_url":"http://ps-latency.hep.pnnl.gov/toolkit/"}],"location":{"country":"US","city":"Richland","state":"WA","postal_code":"99352"}},{"hosts":[{"addresses":["perfsonar3.icepp.jp"],"description":"ICEPP, The University of Tokyo","toolkit_url":"auto"}],"location":{"country":"JP","city":"Tokyo","state":"Tokyo","latitude":"35.7137","longitude":"139.7631","postal_code":"113-0033"}},{"hosts":[{"addresses":["perfsonar1.icepp.jp"],"description":"ICEPP, The University of Tokyo","toolkit_url":"auto"}],"location":{"country":"JP","city":"Tokyo","state":"Tokyo","latitude":"35.7137","longitude":"139.7631","postal_code":"113-0033"}},{"hosts":[{"addresses":["perfsonar1.roma1.infn.it"],"description":"INFN-ROMA1","toolkit_url":"auto"}],"location":{"country":"IT","city":"Roma","latitude":"43.1479","longitude":"12.1097"}},{"hosts":[{"addresses":["test-pt1.es.net"],"description":"(OWAMP Server at es.net)","toolkit_url":"auto"}],"location":{}},{"hosts":[{"addresses":["srv.newa.ilight.net"],"measurement_archives":[{"read_url":"https://srv.newa.ilight.net/esmond/perfsonar/archive","write_url":"https://srv.newa.ilight.net/esmond/perfsonar/archive","type":"perfsonarbuoy/owamp"},{"read_url":"asdf","write_url":"asdf","type":"perfsonarbuoy/owamp"},{"read_url":"https://srv.newa.ilight.net/esmond/perfsonar/archive","write_url":"https://srv.newa.ilight.net/esmond/perfsonar/archive","type":"perfsonarbuoy/bwctl"},{"read_url":"asdf","write_url":"asdf","type":"perfsonarbuoy/bwctl"}],"description":"(BWCTL Measurement Point at 2027.srv.newa.ilight.net)","toolkit_url":"auto"}],"location":{}}]}],"tests":[{"members":{"type":"mesh","members":["ps-latency.hep.pnnl.gov","perfsonar3.icepp.jp","perfsonar1.icepp.jp","perfsonar1.roma1.infn.it"]},"parameters":{"packet_padding":"0","ipv4_only":"1","packet_interval":"0.1","sample_count":"600","bucket_width":"0.001","type":"perfsonarbuoy/owamp"},"description":"lat test"},{"members":{"type":"mesh","members":["test-pt1.es.net","srv.newa.ilight.net"]},"parameters":{"random_start_percentage":"25","ipv4_only":"1","tool":"bwctl/iperf3","interval":"14400","duration":"20","protocol":"tcp","type":"perfsonarbuoy/bwctl"},"description":"put test"}],"description":"latentput: One latency and one through","measurement_archives":[{"read_url":"https://latentputma/","write_url":"https://latentputma/","type":"perfsonarbuoy/owamp"},{"read_url":"https://latentputma/","write_url":"https://latentputma/","type":"perfsonarbuoy/bwctl"}],"administrators":[{"name":"Michael Johnson","email":"mj82@globalnoc.iu.edu"}]}
{"organizations":[{"sites":[{"hosts":[{"addresses":["perf-tokyo.sinet.ad.jp"],"measurement_archives":[{"read_url":"https://perf-tokyo.sinet.ad.jp/esmond/perfsonar/archive","write_url":"https://perf-tokyo.sinet.ad.jp/esmond/perfsonar/archive","type":"perfsonarbuoy/owamp"},{"read_url":"https://perf-tokyo.sinet.ad.jp/esmond/perfsonar/archive","write_url":"https://perf-tokyo.sinet.ad.jp/esmond/perfsonar/archive","type":"perfsonarbuoy/bwctl"}],"description":"Science Information Network - SINET","toolkit_url":"auto"}],"location":{"country":"JP","city":"Chiyoda","state":"Tokyo","latitude":"35.693944","longitude":"139.753611"}},{"hosts":[{"addresses":["perfsonar-dev8.grnoc.iu.edu"],"description":"GlobalNOC","toolkit_url":"auto"}],"location":{"country":"US","city":"Bloomington","state":"IN","latitude":"39.2499","longitude":"-86.4555","postal_code":"47408"}},{"hosts":[{"addresses":["em-4.of-proxy.newy32aoa.net.internet2.edu"],"description":"(OWAMP Server at internet2.edu)","toolkit_url":"auto"}],"location":{}},{"hosts":[{"addresses":["test-pt1.es.net"],"description":"(OWAMP Server at es.net)","toolkit_url":"auto"}],"location":{}},{"hosts":[{"addresses":["perfsonar.cbio.uct.ac.za"],"measurement_archives":[{"read_url":"https://perfsonar.cbio.uct.ac.za/esmond/perfsonar/archive","write_url":"https://perfsonar.cbio.uct.ac.za/esmond/perfsonar/archive","type":"perfsonarbuoy/owamp"},{"read_url":"https://perfsonar.cbio.uct.ac.za/esmond/perfsonar/archive","write_url":"https://perfsonar.cbio.uct.ac.za/esmond/perfsonar/archive","type":"perfsonarbuoy/bwctl"}],"description":"cbio host descr!!!! use local ma!!!","toolkit_url":"auto"}],"location":{"country":"ZA","city":"Cape Town","state":"Western Cape","postal_code":"8000"}}]}],"tests":[{"members":{"type":"mesh","members":["perf-tokyo.sinet.ad.jp","perfsonar-dev8.grnoc.iu.edu"]},"parameters":{"packet_interval":0.1,"sample_count":600,"bucket_width":0.001,"ipv4_only":true,"type":"perfsonarbuoy/owamp"},"description":"LatentSeeTestname"},{"members":{"type":"mesh","members":["em-4.of-proxy.newy32aoa.net.internet2.edu","perfsonar-dev8.grnoc.iu.edu","test-pt1.es.net","perfsonar.cbio.uct.ac.za"]},"parameters":{"ipv4_only":true,"omit_interval":5,"random_start_percentage":10,"duration":20,"interval":14400,"protocol":"tcp","tool":"bwctl/iperf3","type":"perfsonarbuoy/bwctl"},"description":"Throughputtestname"}],"description":"Latent See: New Config","measurement_archives":[{"read_url":"http://asdf/","write_url":"http://asdf/","type":"perfsonarbuoy/owamp"},{"read_url":"http://asdf/","write_url":"http://asdf/","type":"perfsonarbuoy/bwctl"}],"administrators":[{"name":"Michael Johnson","email":"mj82@globalnoc.iu.edu"}]}
2 changes: 1 addition & 1 deletion test/data/latentput-psconfig.json
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"archives":{"host-archive0":{"archiver":"esmond","data":{"url":"https://srv.newa.ilight.net/esmond/perfsonar/archive","measurement-agent":"{% scheduled_by_address %}"}},"host-additional-archive0":{"archiver":"esmond","data":{"url":"asdf","measurement-agent":"{% scheduled_by_address %}"}},"test-archive0":{"archiver":"esmond","data":{"url":"https://latentputma/","measurement-agent":"{% scheduled_by_address %}"}}},"addresses":{"ps-latency.hep.pnnl.gov":{"address":"ps-latency.hep.pnnl.gov","host":"ps-latency.hep.pnnl.gov","_meta":{"display-name":"Overridden description","display-url":"http://ps-latency.hep.pnnl.gov/toolkit/"}},"perfsonar3.icepp.jp":{"address":"perfsonar3.icepp.jp","host":"perfsonar3.icepp.jp","_meta":{"display-name":"ICEPP, The University of Tokyo","display-url":"https://perfsonar3.icepp.jp/toolkit/"}},"perfsonar1.icepp.jp":{"address":"perfsonar1.icepp.jp","host":"perfsonar1.icepp.jp","_meta":{"display-name":"ICEPP, The University of Tokyo","display-url":"https://perfsonar1.icepp.jp/toolkit/"}},"perfsonar1.roma1.infn.it":{"address":"perfsonar1.roma1.infn.it","host":"perfsonar1.roma1.infn.it","_meta":{"display-name":"INFN-ROMA1","display-url":"https://perfsonar1.roma1.infn.it/toolkit/"}},"test-pt1.es.net":{"address":"test-pt1.es.net","host":"test-pt1.es.net","_meta":{"display-name":"(OWAMP Server at es.net)","display-url":"https://test-pt1.es.net/toolkit/"}},"srv.newa.ilight.net":{"address":"srv.newa.ilight.net","host":"srv.newa.ilight.net","_meta":{"display-name":"(BWCTL Measurement Point at 2027.srv.newa.ilight.net)","display-url":"https://srv.newa.ilight.net/toolkit/"}}},"groups":{"lat test":{"type":"mesh","addresses":[{"name":"ps-latency.hep.pnnl.gov"},{"name":"perfsonar3.icepp.jp"},{"name":"perfsonar1.icepp.jp"},{"name":"perfsonar1.roma1.infn.it"}]},"put test":{"type":"mesh","addresses":[{"name":"test-pt1.es.net"},{"name":"srv.newa.ilight.net"}]}},"tests":{"lat test":{"type":"latencybg","spec":{"source":"{% address[0] %}","dest":"{% address[1] %}","packet-padding":0,"packet-interval":0.1,"bucket-width":0.001,"packet-count":600,"ip-version":4}},"put test":{"type":"throughput","spec":{"duration":"PT20S","source":"{% address[0] %}","dest":"{% address[1] %}","ip-version":4}}},"schedules":{"repeat-PT4H":{"repeat":"PT4H","sliprand":true,"slip":"PT1H"}},"tasks":{"lat test":{"group":"lat test","test":"lat test","archives":["test-archive0"],"_meta":{"display-name":"lat test"}},"put test":{"group":"put test","test":"put test","archives":["test-archive0"],"_meta":{"display-name":"put test"},"schedule":"repeat-PT4H","tools":["bwctliperf3","iperf3"]}},"_meta":{"display-name":"latentput"},"hosts":{"ps-latency.hep.pnnl.gov":{},"perfsonar3.icepp.jp":{},"perfsonar1.icepp.jp":{},"perfsonar1.roma1.infn.it":{},"test-pt1.es.net":{},"srv.newa.ilight.net":{"archives":["host-archive0","host-additional-archive0"]}}}
{"archives":{"host-archive0":{"archiver":"esmond","data":{"url":"https://perf-tokyo.sinet.ad.jp/esmond/perfsonar/archive","measurement-agent":"{% scheduled_by_address %}"}},"host-archive1":{"archiver":"esmond","data":{"url":"https://perfsonar.cbio.uct.ac.za/esmond/perfsonar/archive","measurement-agent":"{% scheduled_by_address %}"}},"test-archive0":{"archiver":"esmond","data":{"url":"http://asdf/","measurement-agent":"{% scheduled_by_address %}"}}},"addresses":{"perf-tokyo.sinet.ad.jp":{"address":"perf-tokyo.sinet.ad.jp","host":"perf-tokyo.sinet.ad.jp","_meta":{"display-name":"Science Information Network - SINET","display-url":"https://perf-tokyo.sinet.ad.jp/toolkit/"}},"perfsonar-dev8.grnoc.iu.edu":{"address":"perfsonar-dev8.grnoc.iu.edu","host":"perfsonar-dev8.grnoc.iu.edu","_meta":{"display-name":"GlobalNOC","display-url":"https://perfsonar-dev8.grnoc.iu.edu/toolkit/"}},"em-4.of-proxy.newy32aoa.net.internet2.edu":{"address":"em-4.of-proxy.newy32aoa.net.internet2.edu","host":"em-4.of-proxy.newy32aoa.net.internet2.edu","_meta":{"display-name":"(OWAMP Server at internet2.edu)","display-url":"https://em-4.of-proxy.newy32aoa.net.internet2.edu/toolkit/"}},"test-pt1.es.net":{"address":"test-pt1.es.net","host":"test-pt1.es.net","_meta":{"display-name":"(OWAMP Server at es.net)","display-url":"https://test-pt1.es.net/toolkit/"}},"perfsonar.cbio.uct.ac.za":{"address":"perfsonar.cbio.uct.ac.za","host":"perfsonar.cbio.uct.ac.za","_meta":{"display-name":"cbio host descr!!!! use local ma!!!","display-url":"https://perfsonar.cbio.uct.ac.za/toolkit/"}}},"groups":{"LatentSeeTestname":{"type":"mesh","addresses":[{"name":"perf-tokyo.sinet.ad.jp"},{"name":"perfsonar-dev8.grnoc.iu.edu"}]},"Throughputtestname":{"type":"mesh","addresses":[{"name":"em-4.of-proxy.newy32aoa.net.internet2.edu"},{"name":"perfsonar-dev8.grnoc.iu.edu"},{"name":"test-pt1.es.net"},{"name":"perfsonar.cbio.uct.ac.za"}]}},"tests":{"LatentSeeTestname":{"type":"latencybg","spec":{"source":"{% address[0] %}","dest":"{% address[1] %}","packet-interval":0.1,"bucket-width":0.001,"packet-count":600,"ip-version":4}},"Throughputtestname":{"type":"throughput","spec":{"duration":"PT20S","source":"{% address[0] %}","dest":"{% address[1] %}","omit-interval":5,"ip-version":4}}},"schedules":{"repeat-PT4H":{"repeat":"PT4H","sliprand":true,"slip":"PT24M"}},"tasks":{"LatentSeeTestname":{"group":"LatentSeeTestname","test":"LatentSeeTestname","archives":["test-archive0"],"_meta":{"display-name":"LatentSeeTestname"},"tools":["owping"]},"Throughputtestname":{"group":"Throughputtestname","test":"Throughputtestname","archives":["test-archive0"],"_meta":{"display-name":"Throughputtestname"},"schedule":"repeat-PT4H","tools":["bwctliperf3","iperf3"]}},"_meta":{"display-name":"Latent See"},"hosts":{"perf-tokyo.sinet.ad.jp":{"archives":["host-archive0"]},"perfsonar-dev8.grnoc.iu.edu":{},"em-4.of-proxy.newy32aoa.net.internet2.edu":{},"test-pt1.es.net":{},"perfsonar.cbio.uct.ac.za":{"archives":["host-archive1"]}}}
3 changes: 2 additions & 1 deletion test/import.js
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ testfiles.push( 'data/testbed.json' );
testfiles.push( 'data/testbed2-noarchives.json' );
testfiles.push( 'data/testbed3-nodescription.json' );
testfiles.push( 'data/testbed4-no_endpoint_description.json' );
testfiles.push( 'data/latentput-psconfig.json' );

// files for testing psconfig/meshconfig detection
var formatFiles = [];
Expand All @@ -37,7 +38,7 @@ describe('Detect psconfig/meshconfig format', function() {
}
var output = JSON.parse(data);
var format = importer._detect_config_type( output );
assert.equal(format, "meshconfig");
assert.equal(format, "psconfig");
done();

});
Expand Down

0 comments on commit c7e565a

Please sign in to comment.