diff --git a/api/admin/controllers/importer.js b/api/admin/controllers/importer.js index e085d710..84950421 100644 --- a/api/admin/controllers/importer.js +++ b/api/admin/controllers/importer.js @@ -224,10 +224,14 @@ exports._detect_config_type = function( config ) { exports._process_imported_config = function ( importedConfig, sub, cb, disable_ensure_hosts) { - - var config_format = exports._detect_config_type( importedConfig ); + // Main config object + var mainConfig = {}; + mainConfig.tests = []; + mainConfig.testspecs = []; + mainConfig.hostgroups = []; + //var out = importedConfig; //out = JSON.stringify( out, null, "\t" ); @@ -237,15 +241,23 @@ exports._process_imported_config = function ( importedConfig, sub, cb, disable_e // config_params holds parameters to pass back to the callback var config_params = {}; - var tests = []; if ( config_format == "meshconfig" ) { - exports._process_meshconfig( importedConfig, sub, config_params, tests, cb ); + exports._process_meshconfig( importedConfig, sub, config_params, mainConfig, cb ); } else if ( config_format == "psconfig" ) { - exports._process_psconfig( importedConfig, sub, config_params, tests, cb ); + exports._process_psconfig( importedConfig, sub, config_params, mainConfig, cb ); } + //if ( config_format == "psconfig" ) { + + console.log("MAINCONFIG INTERMEDIATE\n"); + console.log( JSON.stringify( mainConfig, null, 3 )); + + console.log("CONFIG_PARAMS INTERMEDIATE\n"); + console.log( JSON.stringify( config_params, null, 3 )); + + //} //now do update (TODO - should I let caller handle this?) @@ -269,7 +281,11 @@ exports._process_imported_config = function ( importedConfig, sub, cb, disable_e }; -exports._process_meshconfig = function ( importedConfig, sub, config_params, tests, cb ) { +exports._process_meshconfig = function ( importedConfig, sub, config_params, mainConfig, cb ) { + var tests = mainConfig.tests; + var hostgroups = mainConfig.hostgroups; + var testspecs = mainConfig.testspecs; + var config_desc = importedConfig.description; if ( config_desc ) { config_params.description = config_desc; @@ -287,10 +303,11 @@ exports._process_meshconfig = function ( importedConfig, sub, config_params, tes var ma_urls = Object.keys( ma_url_obj ); config_params.archives = ma_urls; - importedConfig.ma_urls = ma_urls; + //importedConfig.ma_urls = ma_urls; //importedConfig.config_params = config_params; //console.log("IMPORTER ma_urls", ma_urls); + //console.log("config_params", config_params); //process hosts @@ -334,8 +351,8 @@ exports._process_meshconfig = function ( importedConfig, sub, config_params, tes //process hostgroups / testspecs - var hostgroups = []; - var testspecs = []; + //var hostgroups = []; + //var testspecs = []; //var tests = []; @@ -400,6 +417,88 @@ exports._process_meshconfig = function ( importedConfig, sub, config_params, tes }; -exports._process_psconfig = function ( importedConfig, sub, cb, disable_ensure_hosts) { +exports._process_psconfig = function ( importedConfig, sub, config_params, mainConfig, cb ) { + var tests = mainConfig.tests; + var hostGroups = mainConfig.hostgroups; + var testspecs = mainConfig.testspecs; + + var config_desc = importedConfig._meta["display-name"]; + if ( config_desc ) { + config_params.description = config_desc; + } + + var archive_obj = exports._extract_psconfig_mas( importedConfig , config_params ); + + console.log("archive_obj", archive_obj); + + config_params.archives = archive_obj.central; + + console.log("config_params", config_params); + + var hosts_obj = {}; + hosts_obj = exports._extract_psconfig_hosts( importedConfig, config_params ); + config_params.hosts = hosts_obj.hosts; + config_params.addresses = hosts_obj.addresses; + + + + +}; + +exports._extract_psconfig_hosts = function( importedConfig, config_params ) { + var hosts_obj = {}; + + // Retrieve host info from importedConfig + var hosts_info = []; + + + + return hosts_info; +}; + +exports._extract_psconfig_mas = function( importedConfig, config_params ) { + // Use a hash, keyed on archive type, to return the MA info + // i.e. archives.central = [] + // archives.host = [] // Local MAs for testpoints + var archives = {}; + archives.central = []; + archives.host = []; + + // process central MAs + var ma_url_obj = {}; + if ( "archives" in importedConfig ) { + console.log("ARCHIVES"); + Object.keys( importedConfig.archives ).forEach(function(ma_name) { + var ma = importedConfig.archives[ ma_name ]; + if ( ! ( "archives" in config_params ) ) config_params.archives = []; + var ma_url = importedConfig.archives[ ma_name ].data.url; + ma_url_obj[ ma_url ] = 1; + + if ( "tasks" in importedConfig ) { + var num_ma_instances = 0; + _.each( importedConfig.tasks, function( task, taskName ) { + if ( _.indexOf( task.archives, ma_name ) > -1 ) { + num_ma_instances++; + } + }); + // If the MA appears in all task specs, we consider it a central MA + if ( num_ma_instances == Object.keys( importedConfig.tasks).length) { + archives.central.push( ma_url ); + } else { + // Otherwise, it is a host MA + archives.host.push( ma_url ); + + } + }; + //console.log("IMPORTED", JSON.stringify(importedConfig, null, "\t")); + console.log("archives", JSON.stringify( archives, null, "\t")); + console.log("ma_url_obj", JSON.stringify( ma_url_obj, null, "\t")); + var ma_urls = Object.keys( ma_url_obj ); + + }); + + } + + return archives; }; diff --git a/test/data/latentput-meshconfig.json b/test/data/latentput-meshconfig.json index 3eb213a4..efd189dc 100644 --- a/test/data/latentput-meshconfig.json +++ b/test/data/latentput-meshconfig.json @@ -1 +1 @@ -{"organizations":[{"sites":[{"hosts":[{"addresses":["ps-latency.hep.pnnl.gov"],"description":"Overridden description","toolkit_url":"http://ps-latency.hep.pnnl.gov/toolkit/"}],"location":{"country":"US","city":"Richland","state":"WA","postal_code":"99352"}},{"hosts":[{"addresses":["perfsonar3.icepp.jp"],"description":"ICEPP, The University of Tokyo","toolkit_url":"auto"}],"location":{"country":"JP","city":"Tokyo","state":"Tokyo","latitude":"35.7137","longitude":"139.7631","postal_code":"113-0033"}},{"hosts":[{"addresses":["perfsonar1.icepp.jp"],"description":"ICEPP, The University of Tokyo","toolkit_url":"auto"}],"location":{"country":"JP","city":"Tokyo","state":"Tokyo","latitude":"35.7137","longitude":"139.7631","postal_code":"113-0033"}},{"hosts":[{"addresses":["perfsonar1.roma1.infn.it"],"description":"INFN-ROMA1","toolkit_url":"auto"}],"location":{"country":"IT","city":"Roma","latitude":"43.1479","longitude":"12.1097"}},{"hosts":[{"addresses":["test-pt1.es.net"],"description":"(OWAMP Server at es.net)","toolkit_url":"auto"}],"location":{}},{"hosts":[{"addresses":["srv.newa.ilight.net"],"measurement_archives":[{"read_url":"https://srv.newa.ilight.net/esmond/perfsonar/archive","write_url":"https://srv.newa.ilight.net/esmond/perfsonar/archive","type":"perfsonarbuoy/owamp"},{"read_url":"asdf","write_url":"asdf","type":"perfsonarbuoy/owamp"},{"read_url":"https://srv.newa.ilight.net/esmond/perfsonar/archive","write_url":"https://srv.newa.ilight.net/esmond/perfsonar/archive","type":"perfsonarbuoy/bwctl"},{"read_url":"asdf","write_url":"asdf","type":"perfsonarbuoy/bwctl"}],"description":"(BWCTL Measurement Point at 2027.srv.newa.ilight.net)","toolkit_url":"auto"}],"location":{}}]}],"tests":[{"members":{"type":"mesh","members":["ps-latency.hep.pnnl.gov","perfsonar3.icepp.jp","perfsonar1.icepp.jp","perfsonar1.roma1.infn.it"]},"parameters":{"packet_padding":"0","ipv4_only":"1","packet_interval":"0.1","sample_count":"600","bucket_width":"0.001","type":"perfsonarbuoy/owamp"},"description":"lat test"},{"members":{"type":"mesh","members":["test-pt1.es.net","srv.newa.ilight.net"]},"parameters":{"random_start_percentage":"25","ipv4_only":"1","tool":"bwctl/iperf3","interval":"14400","duration":"20","protocol":"tcp","type":"perfsonarbuoy/bwctl"},"description":"put test"}],"description":"latentput: One latency and one through","measurement_archives":[{"read_url":"https://latentputma/","write_url":"https://latentputma/","type":"perfsonarbuoy/owamp"},{"read_url":"https://latentputma/","write_url":"https://latentputma/","type":"perfsonarbuoy/bwctl"}],"administrators":[{"name":"Michael Johnson","email":"mj82@globalnoc.iu.edu"}]} \ No newline at end of file +{"organizations":[{"sites":[{"hosts":[{"addresses":["perf-tokyo.sinet.ad.jp"],"measurement_archives":[{"read_url":"https://perf-tokyo.sinet.ad.jp/esmond/perfsonar/archive","write_url":"https://perf-tokyo.sinet.ad.jp/esmond/perfsonar/archive","type":"perfsonarbuoy/owamp"},{"read_url":"https://perf-tokyo.sinet.ad.jp/esmond/perfsonar/archive","write_url":"https://perf-tokyo.sinet.ad.jp/esmond/perfsonar/archive","type":"perfsonarbuoy/bwctl"}],"description":"Science Information Network - SINET","toolkit_url":"auto"}],"location":{"country":"JP","city":"Chiyoda","state":"Tokyo","latitude":"35.693944","longitude":"139.753611"}},{"hosts":[{"addresses":["perfsonar-dev8.grnoc.iu.edu"],"description":"GlobalNOC","toolkit_url":"auto"}],"location":{"country":"US","city":"Bloomington","state":"IN","latitude":"39.2499","longitude":"-86.4555","postal_code":"47408"}},{"hosts":[{"addresses":["em-4.of-proxy.newy32aoa.net.internet2.edu"],"description":"(OWAMP Server at internet2.edu)","toolkit_url":"auto"}],"location":{}},{"hosts":[{"addresses":["test-pt1.es.net"],"description":"(OWAMP Server at es.net)","toolkit_url":"auto"}],"location":{}},{"hosts":[{"addresses":["perfsonar.cbio.uct.ac.za"],"measurement_archives":[{"read_url":"https://perfsonar.cbio.uct.ac.za/esmond/perfsonar/archive","write_url":"https://perfsonar.cbio.uct.ac.za/esmond/perfsonar/archive","type":"perfsonarbuoy/owamp"},{"read_url":"https://perfsonar.cbio.uct.ac.za/esmond/perfsonar/archive","write_url":"https://perfsonar.cbio.uct.ac.za/esmond/perfsonar/archive","type":"perfsonarbuoy/bwctl"}],"description":"cbio host descr!!!! use local ma!!!","toolkit_url":"auto"}],"location":{"country":"ZA","city":"Cape Town","state":"Western Cape","postal_code":"8000"}}]}],"tests":[{"members":{"type":"mesh","members":["perf-tokyo.sinet.ad.jp","perfsonar-dev8.grnoc.iu.edu"]},"parameters":{"packet_interval":0.1,"sample_count":600,"bucket_width":0.001,"ipv4_only":true,"type":"perfsonarbuoy/owamp"},"description":"LatentSeeTestname"},{"members":{"type":"mesh","members":["em-4.of-proxy.newy32aoa.net.internet2.edu","perfsonar-dev8.grnoc.iu.edu","test-pt1.es.net","perfsonar.cbio.uct.ac.za"]},"parameters":{"ipv4_only":true,"omit_interval":5,"random_start_percentage":10,"duration":20,"interval":14400,"protocol":"tcp","tool":"bwctl/iperf3","type":"perfsonarbuoy/bwctl"},"description":"Throughputtestname"}],"description":"Latent See: New Config","measurement_archives":[{"read_url":"http://asdf/","write_url":"http://asdf/","type":"perfsonarbuoy/owamp"},{"read_url":"http://asdf/","write_url":"http://asdf/","type":"perfsonarbuoy/bwctl"}],"administrators":[{"name":"Michael Johnson","email":"mj82@globalnoc.iu.edu"}]} \ No newline at end of file diff --git a/test/data/latentput-psconfig.json b/test/data/latentput-psconfig.json index 743b48c9..370d49db 100644 --- a/test/data/latentput-psconfig.json +++ b/test/data/latentput-psconfig.json @@ -1 +1 @@ -{"archives":{"host-archive0":{"archiver":"esmond","data":{"url":"https://srv.newa.ilight.net/esmond/perfsonar/archive","measurement-agent":"{% scheduled_by_address %}"}},"host-additional-archive0":{"archiver":"esmond","data":{"url":"asdf","measurement-agent":"{% scheduled_by_address %}"}},"test-archive0":{"archiver":"esmond","data":{"url":"https://latentputma/","measurement-agent":"{% scheduled_by_address %}"}}},"addresses":{"ps-latency.hep.pnnl.gov":{"address":"ps-latency.hep.pnnl.gov","host":"ps-latency.hep.pnnl.gov","_meta":{"display-name":"Overridden description","display-url":"http://ps-latency.hep.pnnl.gov/toolkit/"}},"perfsonar3.icepp.jp":{"address":"perfsonar3.icepp.jp","host":"perfsonar3.icepp.jp","_meta":{"display-name":"ICEPP, The University of Tokyo","display-url":"https://perfsonar3.icepp.jp/toolkit/"}},"perfsonar1.icepp.jp":{"address":"perfsonar1.icepp.jp","host":"perfsonar1.icepp.jp","_meta":{"display-name":"ICEPP, The University of Tokyo","display-url":"https://perfsonar1.icepp.jp/toolkit/"}},"perfsonar1.roma1.infn.it":{"address":"perfsonar1.roma1.infn.it","host":"perfsonar1.roma1.infn.it","_meta":{"display-name":"INFN-ROMA1","display-url":"https://perfsonar1.roma1.infn.it/toolkit/"}},"test-pt1.es.net":{"address":"test-pt1.es.net","host":"test-pt1.es.net","_meta":{"display-name":"(OWAMP Server at es.net)","display-url":"https://test-pt1.es.net/toolkit/"}},"srv.newa.ilight.net":{"address":"srv.newa.ilight.net","host":"srv.newa.ilight.net","_meta":{"display-name":"(BWCTL Measurement Point at 2027.srv.newa.ilight.net)","display-url":"https://srv.newa.ilight.net/toolkit/"}}},"groups":{"lat test":{"type":"mesh","addresses":[{"name":"ps-latency.hep.pnnl.gov"},{"name":"perfsonar3.icepp.jp"},{"name":"perfsonar1.icepp.jp"},{"name":"perfsonar1.roma1.infn.it"}]},"put test":{"type":"mesh","addresses":[{"name":"test-pt1.es.net"},{"name":"srv.newa.ilight.net"}]}},"tests":{"lat test":{"type":"latencybg","spec":{"source":"{% address[0] %}","dest":"{% address[1] %}","packet-padding":0,"packet-interval":0.1,"bucket-width":0.001,"packet-count":600,"ip-version":4}},"put test":{"type":"throughput","spec":{"duration":"PT20S","source":"{% address[0] %}","dest":"{% address[1] %}","ip-version":4}}},"schedules":{"repeat-PT4H":{"repeat":"PT4H","sliprand":true,"slip":"PT1H"}},"tasks":{"lat test":{"group":"lat test","test":"lat test","archives":["test-archive0"],"_meta":{"display-name":"lat test"}},"put test":{"group":"put test","test":"put test","archives":["test-archive0"],"_meta":{"display-name":"put test"},"schedule":"repeat-PT4H","tools":["bwctliperf3","iperf3"]}},"_meta":{"display-name":"latentput"},"hosts":{"ps-latency.hep.pnnl.gov":{},"perfsonar3.icepp.jp":{},"perfsonar1.icepp.jp":{},"perfsonar1.roma1.infn.it":{},"test-pt1.es.net":{},"srv.newa.ilight.net":{"archives":["host-archive0","host-additional-archive0"]}}} \ No newline at end of file +{"archives":{"host-archive0":{"archiver":"esmond","data":{"url":"https://perf-tokyo.sinet.ad.jp/esmond/perfsonar/archive","measurement-agent":"{% scheduled_by_address %}"}},"host-archive1":{"archiver":"esmond","data":{"url":"https://perfsonar.cbio.uct.ac.za/esmond/perfsonar/archive","measurement-agent":"{% scheduled_by_address %}"}},"test-archive0":{"archiver":"esmond","data":{"url":"http://asdf/","measurement-agent":"{% scheduled_by_address %}"}}},"addresses":{"perf-tokyo.sinet.ad.jp":{"address":"perf-tokyo.sinet.ad.jp","host":"perf-tokyo.sinet.ad.jp","_meta":{"display-name":"Science Information Network - SINET","display-url":"https://perf-tokyo.sinet.ad.jp/toolkit/"}},"perfsonar-dev8.grnoc.iu.edu":{"address":"perfsonar-dev8.grnoc.iu.edu","host":"perfsonar-dev8.grnoc.iu.edu","_meta":{"display-name":"GlobalNOC","display-url":"https://perfsonar-dev8.grnoc.iu.edu/toolkit/"}},"em-4.of-proxy.newy32aoa.net.internet2.edu":{"address":"em-4.of-proxy.newy32aoa.net.internet2.edu","host":"em-4.of-proxy.newy32aoa.net.internet2.edu","_meta":{"display-name":"(OWAMP Server at internet2.edu)","display-url":"https://em-4.of-proxy.newy32aoa.net.internet2.edu/toolkit/"}},"test-pt1.es.net":{"address":"test-pt1.es.net","host":"test-pt1.es.net","_meta":{"display-name":"(OWAMP Server at es.net)","display-url":"https://test-pt1.es.net/toolkit/"}},"perfsonar.cbio.uct.ac.za":{"address":"perfsonar.cbio.uct.ac.za","host":"perfsonar.cbio.uct.ac.za","_meta":{"display-name":"cbio host descr!!!! use local ma!!!","display-url":"https://perfsonar.cbio.uct.ac.za/toolkit/"}}},"groups":{"LatentSeeTestname":{"type":"mesh","addresses":[{"name":"perf-tokyo.sinet.ad.jp"},{"name":"perfsonar-dev8.grnoc.iu.edu"}]},"Throughputtestname":{"type":"mesh","addresses":[{"name":"em-4.of-proxy.newy32aoa.net.internet2.edu"},{"name":"perfsonar-dev8.grnoc.iu.edu"},{"name":"test-pt1.es.net"},{"name":"perfsonar.cbio.uct.ac.za"}]}},"tests":{"LatentSeeTestname":{"type":"latencybg","spec":{"source":"{% address[0] %}","dest":"{% address[1] %}","packet-interval":0.1,"bucket-width":0.001,"packet-count":600,"ip-version":4}},"Throughputtestname":{"type":"throughput","spec":{"duration":"PT20S","source":"{% address[0] %}","dest":"{% address[1] %}","omit-interval":5,"ip-version":4}}},"schedules":{"repeat-PT4H":{"repeat":"PT4H","sliprand":true,"slip":"PT24M"}},"tasks":{"LatentSeeTestname":{"group":"LatentSeeTestname","test":"LatentSeeTestname","archives":["test-archive0"],"_meta":{"display-name":"LatentSeeTestname"},"tools":["owping"]},"Throughputtestname":{"group":"Throughputtestname","test":"Throughputtestname","archives":["test-archive0"],"_meta":{"display-name":"Throughputtestname"},"schedule":"repeat-PT4H","tools":["bwctliperf3","iperf3"]}},"_meta":{"display-name":"Latent See"},"hosts":{"perf-tokyo.sinet.ad.jp":{"archives":["host-archive0"]},"perfsonar-dev8.grnoc.iu.edu":{},"em-4.of-proxy.newy32aoa.net.internet2.edu":{},"test-pt1.es.net":{},"perfsonar.cbio.uct.ac.za":{"archives":["host-archive1"]}}} \ No newline at end of file diff --git a/test/import.js b/test/import.js index 9b297e52..fa759e0d 100644 --- a/test/import.js +++ b/test/import.js @@ -16,6 +16,7 @@ testfiles.push( 'data/testbed.json' ); testfiles.push( 'data/testbed2-noarchives.json' ); testfiles.push( 'data/testbed3-nodescription.json' ); testfiles.push( 'data/testbed4-no_endpoint_description.json' ); +testfiles.push( 'data/latentput-psconfig.json' ); // files for testing psconfig/meshconfig detection var formatFiles = []; @@ -37,7 +38,7 @@ describe('Detect psconfig/meshconfig format', function() { } var output = JSON.parse(data); var format = importer._detect_config_type( output ); - assert.equal(format, "meshconfig"); + assert.equal(format, "psconfig"); done(); });