From 6ad37e711e0d44072813b2d2f0a25654e0cb351e Mon Sep 17 00:00:00 2001 From: schandrika Date: Thu, 2 Sep 2021 13:52:51 -0700 Subject: [PATCH 01/36] minor python2 to python3 fix. --- volttron/platform/control.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volttron/platform/control.py b/volttron/platform/control.py index dd99a394f2..fe2ea7eb0f 100644 --- a/volttron/platform/control.py +++ b/volttron/platform/control.py @@ -4429,7 +4429,7 @@ def add_parser(*args, **kwargs) -> argparse.ArgumentParser: # else we return 0 from here. This has the added effect of # allowing us to cascade short circuit calls. if exc.args[0] != 0: - error = exc.message + error = exc else: return 0 except InstallRuntimeError as exrt: From dd2de218f5859b0bedac59b2d1c9304c6f8be4a0 Mon Sep 17 00:00:00 2001 From: schandrika Date: Thu, 2 Sep 2021 13:58:37 -0700 Subject: [PATCH 02/36] work around for jira-292 and jira-426 - vctl install with --start cause timeout. Waiting for a additional 2 seconds before a start - to wait for all installation steps to be really completed - seems to alleviate the issue --- volttron/platform/install_agents.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volttron/platform/install_agents.py b/volttron/platform/install_agents.py index db7c4998a5..0929b83f47 100644 --- a/volttron/platform/install_agents.py +++ b/volttron/platform/install_agents.py @@ -174,6 +174,7 @@ def _send_and_intialize_agent(opts, publickey, secretkey): try: if opts.start: + gevent.sleep(2) _log.debug(f"Staring agent {agent_uuid}") opts.connection.call('start_agent', agent_uuid) output_dict['starting'] = True From 36d1e19880b835b43b3a25612785840cf9bcdeb7 Mon Sep 17 00:00:00 2001 From: schandrika Date: Thu, 2 Sep 2021 14:00:08 -0700 Subject: [PATCH 03/36] update to document configurations that are common for all sqlhistorian sub classes --- services/core/SQLHistorian/README.md | 38 +++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/services/core/SQLHistorian/README.md b/services/core/SQLHistorian/README.md index 60289d7dd4..66be4ed5c8 100644 --- a/services/core/SQLHistorian/README.md +++ b/services/core/SQLHistorian/README.md @@ -7,7 +7,43 @@ inconsistent network connectivity (automatic re-connection to tcp based databases). All additions to the historian are batched and wrapped within a transaction with commit and rollback functions properly implemented. This allows the maximum throughput of data with the most -protection. +protection + +## Common Configurations +All SQLHistorians support a minimum of two parameters +1. connection - This is a mandatory parameter with type indicating the type of + sql historian (ex. mysql, sqlite, etc.) and params containing the connection + parameters specific to the connecting database type. + +2. tables_def - Optional parameter to provide custom table names for + topics, data, and metadata. This is useful when you want to use more than + one instance of sqlhistorian with the same database + +Example: + +JSON format : + + { + "connection": { + # type should be sqlite + "type": "sqlite", + "params": { + "database": "data/historian.sqlite", + } + } + "tables_def": { + # prefix for data, topics, and (in version < 4.0.0 metadata tables) + # default is "" + "table_prefix": "", + # table name for time series data. default "data" + "data_table": "data", + # table name for list of topics. default "topics" + "topics_table": "topics", + # table name mapping topic to metadata. default "meta" + # In sqlhistorian version >= 4.0.0 metadata is stored in topics table + "meta_table": "meta" + } + } ## MySQL From 14b6a9a1611e1d85bd4d8bbc2efb10063e25f3ee Mon Sep 17 00:00:00 2001 From: schandrika Date: Thu, 2 Sep 2021 14:44:05 -0700 Subject: [PATCH 04/36] jira-521 propagating actual error message when raising RuntimeException --- volttron/platform/agent/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volttron/platform/agent/utils.py b/volttron/platform/agent/utils.py index 911fedf481..285722a486 100644 --- a/volttron/platform/agent/utils.py +++ b/volttron/platform/agent/utils.py @@ -797,7 +797,7 @@ def execute_command(cmds, env=None, cwd=None, logger=None, err_prefix=None) -> s results.stderr) if logger: logger.exception(err_message) - raise RuntimeError() + raise RuntimeError(err_message) else: raise RuntimeError(err_message) From 6e2b3a814bc6aa37f58a5fba39623ed270155904 Mon Sep 17 00:00:00 2001 From: schandrika Date: Thu, 2 Sep 2021 14:46:38 -0700 Subject: [PATCH 05/36] jira-520 base historian should set health status to bad if cache management errors jira-590 base historian should log the actual error message when errors are propagated from child classes/utility methods refactored --- volttron/platform/agent/base_historian.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/volttron/platform/agent/base_historian.py b/volttron/platform/agent/base_historian.py index 4b62dcbbdb..a56d2a7497 100644 --- a/volttron/platform/agent/base_historian.py +++ b/volttron/platform/agent/base_historian.py @@ -312,6 +312,7 @@ def add_timing_data_to_header(headers, agent_id, phase): STATUS_KEY_CACHE_FULL = "cache_full" STATUS_KEY_TIME_ERROR = "records_with_invalid_timestamp" STATUS_KEY_CACHE_ONLY = "cache_only_enabled" +STATUS_KEY_ERROR_MANAGE_DB_SIZE = "error_managing_db_size" class BaseHistorianAgent(Agent): @@ -412,7 +413,9 @@ def __init__(self, STATUS_KEY_BACKLOGGED: False, STATUS_KEY_PUBLISHING: True, STATUS_KEY_CACHE_FULL: False, - STATUS_KEY_CACHE_ONLY: False + STATUS_KEY_CACHE_ONLY: False, + STATUS_KEY_TIME_ERROR: False, + STATUS_KEY_ERROR_MANAGE_DB_SIZE: False } self._all_platforms = bool(all_platforms) self._time_tolerance = float(time_tolerance) if time_tolerance else None @@ -1062,7 +1065,8 @@ def _get_status_from_context(context): if (context.get(STATUS_KEY_BACKLOGGED) or context.get(STATUS_KEY_CACHE_FULL) or not context.get(STATUS_KEY_PUBLISHING) or - context.get(STATUS_KEY_TIME_ERROR)): + context.get(STATUS_KEY_TIME_ERROR) or + context.get(STATUS_KEY_ERROR_MANAGE_DB_SIZE)): status = STATUS_BAD return status @@ -1215,10 +1219,17 @@ def _do_process_loop(self): try: if not cache_only_enabled: self.publish_to_historian(to_publish_list) + except Exception as e: + _log.exception( + f"An unhandled exception occurred while publishing: {e}") + + try: self.manage_db_size(history_limit_timestamp, self._storage_limit_gb) - except: + self._update_status({STATUS_KEY_ERROR_MANAGE_DB_SIZE: False}) + except Exception as e: _log.exception( - "An unhandled exception occurred while publishing.") + f"An unhandled exception occurred while attempting to managing db size: {e}") + self._send_alert({STATUS_KEY_ERROR_MANAGE_DB_SIZE: True}, "error_managing_db_size") # if the success queue is empty then we need not remove # them from the database and we are probably having connection problems. From 0284f41642f483091900b0469374e84d1aba0bac Mon Sep 17 00:00:00 2001 From: schandrika Date: Thu, 2 Sep 2021 14:47:44 -0700 Subject: [PATCH 06/36] refactored creation of dbfuncts class into a separate method so that specific sql historian's override just this method need not override historian methods --- services/core/SQLHistorian/sqlhistorian/historian.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/services/core/SQLHistorian/sqlhistorian/historian.py b/services/core/SQLHistorian/sqlhistorian/historian.py index 4a4db86b6e..bd63e2a23c 100644 --- a/services/core/SQLHistorian/sqlhistorian/historian.py +++ b/services/core/SQLHistorian/sqlhistorian/historian.py @@ -138,8 +138,6 @@ def __init__(self, connection, tables_def=None, **kwargs): self.topic_name_map = {} self.topic_meta = {} self.agg_topic_id_map = {} - database_type = self.connection['type'] - self.db_functs_class = sqlutils.get_dbfuncts_class(database_type) # Create two instance so connection is shared within a single thread. # This is because sqlite only supports sharing of connection within # a single thread. @@ -147,7 +145,7 @@ def __init__(self, connection, tables_def=None, **kwargs): # everything else happens in the MainThread # One utils class instance( hence one db connection) for main thread - self.main_thread_dbutils = self.db_functs_class(self.connection['params'], self.table_names) + self.main_thread_dbutils = self.get_dbfuncts_object() # One utils class instance( hence one db connection) for background thread # this gets initialized in the bg_thread within historian_setup self.bg_thread_dbutils = None @@ -342,7 +340,7 @@ def query_historian(self, topic, start=None, end=None, agg_type=None, agg_period def historian_setup(self): thread_name = threading.currentThread().getName() _log.info("historian_setup on Thread: {}".format(thread_name)) - self.bg_thread_dbutils = self.db_functs_class(self.connection['params'], self.table_names) + self.bg_thread_dbutils = self.get_dbfuncts_object() if not self._readonly: self.bg_thread_dbutils.setup_historian_tables() @@ -356,6 +354,10 @@ def historian_setup(self): _log.debug(f"###DEBUG Loaded topics and metadata on start. Len of topics {len(self.topic_id_map)} " f"Len of metadata: {len(self.topic_meta)}") + def get_dbfuncts_object(self): + db_functs_class = sqlutils.get_dbfuncts_class(self.connection['type']) + return db_functs_class(self.connection['params'], self.table_names) + def main(argv=sys.argv): """ From 706bf0e50b14d05021fec6a3a2417411ee5514df Mon Sep 17 00:00:00 2001 From: Craig <3979063+craig8@users.noreply.github.com> Date: Fri, 3 Sep 2021 14:14:45 -0700 Subject: [PATCH 07/36] Update requirements.py fixed #2779 issues with python3.9 and setup tools. --- requirements.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.py b/requirements.py index 5baf5ffc8c..55b9ebf765 100644 --- a/requirements.py +++ b/requirements.py @@ -85,7 +85,7 @@ 'pytz==2021.1', 'PyYAML==5.4.1', 'pyzmq==22.2.1', - 'setuptools==40.0.0', + 'setuptools>=40.0.0', 'tzlocal==2.1', 'pyOpenSSL==19.0.0', 'cryptography==2.3', From 1c5a8a6412743a4efd30cfbf0615ee15c91d5d29 Mon Sep 17 00:00:00 2001 From: schandrika Date: Sun, 5 Sep 2021 15:26:59 -0700 Subject: [PATCH 08/36] fix for issue #2778. fixed identity used for platform.historian install using vcfg --- volttron/platform/instance_setup.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/volttron/platform/instance_setup.py b/volttron/platform/instance_setup.py index 161b2bbab9..5997039042 100644 --- a/volttron/platform/instance_setup.py +++ b/volttron/platform/instance_setup.py @@ -209,7 +209,7 @@ def _cleanup_on_exit(): _shutdown_platform() -def _install_agent(agent_dir, config, tag): +def _install_agent(agent_dir, config, tag, identity): if not isinstance(config, dict): config_file = config else: @@ -217,7 +217,11 @@ def _install_agent(agent_dir, config, tag): with open(cfg.name, 'w') as fout: fout.write(jsonapi.dumps(config)) config_file = cfg.name - _cmd(['volttron-ctl', 'install', "--agent-config", config_file, "--tag", tag, "--force", agent_dir]) + cmd_array = ['volttron-ctl', 'install', "--agent-config", config_file, "--tag", tag, "--force"] + if identity: + cmd_array.extend(["--vip-identity", identity]) + cmd_array.append(agent_dir) + _cmd(cmd_array) def _is_agent_installed(tag): @@ -238,9 +242,6 @@ def wrap(config_func): global available_agents def func(*args, **kwargs): - if identity is not None: - os.environ['AGENT_VIP_IDENTITY'] = identity - print('Configuring {}.'.format(agent_dir)) config = config_func(*args, **kwargs) _update_config_file() @@ -248,7 +249,7 @@ def func(*args, **kwargs): #TODO: (potentially only starting the platform once per vcfg) _start_platform() - _install_agent(agent_dir, config, tag) + _install_agent(agent_dir, config, tag, identity) if not _is_agent_installed(tag): print(tag + ' not installed correctly!') From a44a9c844c883b7e981df4c8d542dca9da34acd7 Mon Sep 17 00:00:00 2001 From: schandrika Date: Mon, 6 Sep 2021 14:43:57 -0700 Subject: [PATCH 09/36] minor readme change --- services/core/SQLHistorian/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/core/SQLHistorian/README.md b/services/core/SQLHistorian/README.md index 66be4ed5c8..aa96ec0b18 100644 --- a/services/core/SQLHistorian/README.md +++ b/services/core/SQLHistorian/README.md @@ -10,7 +10,7 @@ implemented. This allows the maximum throughput of data with the most protection ## Common Configurations -All SQLHistorians support a minimum of two parameters +All SQLHistorians support two parameters 1. connection - This is a mandatory parameter with type indicating the type of sql historian (ex. mysql, sqlite, etc.) and params containing the connection parameters specific to the connecting database type. From 246e93ef85ff768be1d005051ffc8d6ef78af97f Mon Sep 17 00:00:00 2001 From: schandrika Date: Tue, 7 Sep 2021 14:37:51 -0700 Subject: [PATCH 10/36] removed unnecessary pinned versions in requirements.py --- requirements.py | 107 ++++++++++++++++++++++++++---------------------- 1 file changed, 58 insertions(+), 49 deletions(-) diff --git a/requirements.py b/requirements.py index 55b9ebf765..26deaa0537 100644 --- a/requirements.py +++ b/requirements.py @@ -36,62 +36,71 @@ # under Contract DE-AC05-76RL01830 # }}} +# These need to be importable by bootstrap.py. If we put them in +# setup.py the import may fail if setuptools in not installed +# in the global python3. + +option_requirements = [('pyzmq', ['--zmq=bundled'])] -extras_require = { 'crate': ['crate==0.26.0'], - 'databases': [ 'mysql-connector-python==8.0.26', - 'bson==0.5.7', - 'pymongo==3.7.2', - 'crate==0.26.0', - 'influxdb==5.3.1', - 'psycopg2-binary==2.8.6'], - 'documentation': [ 'mock==4.0.3', - 'Sphinx==4.1.2', - 'sphinx-rtd-theme==0.5.2', - 'sphinx==3.3.0', - 'm2r2==0.3.1'], - 'drivers': [ 'pymodbus==2.5.2', - 'bacpypes==0.16.7', - 'modbus-tk==1.1.2', - 'pyserial==3.5'], - 'influxdb': ['influxdb==5.3.1'], - 'market': ['numpy==1.19.5', 'transitions==0.8.8'], - 'mongo': ['bson==0.5.7pymongo==3.7.2'], - 'mysql': ['mysql-connector-python==8.0.26'], - 'pandas': ['numpy==1.19.5', 'pandas==1.1.5'], - 'postgres': ['psycopg2-binary==2.8.6'], - 'testing': [ 'mock==4.0.3', - 'pytest==6.2.4', - 'pytest-timeout==1.4.2', - 'pytest-rerunfailures==10.1', - 'websocket-client==1.2.1', - 'deepdiff==5.5.0', - 'docker==5.0.0'], - 'weather': ['Pint==0.17'], - 'web': [ 'ws4py==0.5.1', - 'PyJWT==1.7.1', - 'Jinja2==3.0.1', - 'passlib==1.7.4', - 'argon2-cffi==20.1.0', - 'Werkzeug==2.0.1']} -install_requires = [ 'gevent==20.6.1', +install_requires = [ + 'gevent==20.6.1', 'greenlet==0.4.16', - 'grequests==0.6.0', + 'grequests', 'idna<3,>=2.5', 'requests==2.23.0', - 'ply==3.11', - 'psutil==5.8.0', - 'python-dateutil==2.8.2', - 'pytz==2021.1', - 'PyYAML==5.4.1', - 'pyzmq==22.2.1', + 'ply', + 'psutil', + 'python-dateutil', + 'pytz', + 'PyYAML', + 'pyzmq', 'setuptools>=40.0.0', + # tzlocal 3.0 breaks without the backports.tzinfo package on python < 3.9 https://pypi.org/project/tzlocal/3.0/ 'tzlocal==2.1', 'pyOpenSSL==19.0.0', 'cryptography==2.3', - 'watchdog-gevent==0.1.1', - 'wheel==0.30'] - -option_requirements = [('pyzmq==22.2.1', ['--zmq=bundled'])] - + # Cross platform way of handling changes in file/directories. + # https://github.com/Bogdanp/watchdog_gevent + 'watchdog-gevent', + 'wheel==0.30' +] +extras_require = { + 'crate': ['crate'], + 'databases': ['mysql-connector-python', + 'bson==0.5.7', + 'pymongo==3.7.2', + 'crate', + 'influxdb', + 'psycopg2-binary==2.8.6'], + 'documentation': ['mock', + 'Sphinx', + 'sphinx-rtd-theme', + 'm2r2'], + 'drivers': ['pymodbus', + 'bacpypes==0.16.7', + 'modbus-tk', + 'pyserial'], + 'influxdb': ['influxdb'], + 'market': ['numpy', 'transitions'], + 'mongo': ['bson==0.5.7,' + 'pymongo==3.7.2'], + 'mysql': ['mysql-connector-python'], + 'pandas': ['numpy', + 'pandas'], + 'postgres': ['psycopg2-binary==2.8.6'], + 'testing': ['mock', + 'pytest', + 'pytest-timeout', + 'pytest-rerunfailures', + 'websocket-client', + 'deepdiff', + 'docker'], + 'weather': ['Pint'], + 'web': ['ws4py', + 'PyJWT==1.7.1', + 'Jinja2', + 'passlib', + 'argon2-cffi', + 'Werkzeug']} From 29a2411e3da4a71190a7a72b58a22fd5fa82f6a3 Mon Sep 17 00:00:00 2001 From: schandrika Date: Tue, 7 Sep 2021 16:26:40 -0700 Subject: [PATCH 11/36] fixed typo --- requirements.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.py b/requirements.py index 26deaa0537..a6ff695d36 100644 --- a/requirements.py +++ b/requirements.py @@ -84,7 +84,7 @@ 'pyserial'], 'influxdb': ['influxdb'], 'market': ['numpy', 'transitions'], - 'mongo': ['bson==0.5.7,' + 'mongo': ['bson==0.5.7', 'pymongo==3.7.2'], 'mysql': ['mysql-connector-python'], 'pandas': ['numpy', From fbf47fa1cc59d605b902ef4e2006f9a19d3e5bfa Mon Sep 17 00:00:00 2001 From: "C. Allwardt" Date: Thu, 9 Sep 2021 10:47:41 -0700 Subject: [PATCH 12/36] pin versions and ready version number for release. --- docs/source/conf.py | 4 +- requirements.py | 120 +++++++++++++++------------------- volttron/platform/__init__.py | 2 +- 3 files changed, 57 insertions(+), 69 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index ed745b292a..70d3290f4a 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -43,9 +43,9 @@ def __getattr__(cls, name): author = 'The VOLTTRON Community' # The short X.Y version -version = '8.1' +version = '8.1.1' # The full version, including alpha/beta/rc tags -release = '8.1' +release = '8.1.1' # -- General configuration --------------------------------------------------- diff --git a/requirements.py b/requirements.py index a6ff695d36..7b13f730a1 100644 --- a/requirements.py +++ b/requirements.py @@ -36,71 +36,59 @@ # under Contract DE-AC05-76RL01830 # }}} -# These need to be importable by bootstrap.py. If we put them in -# setup.py the import may fail if setuptools in not installed -# in the global python3. -option_requirements = [('pyzmq', ['--zmq=bundled'])] +extras_require = {'crate': ['crate==0.26.0'], + 'databases': ['mysql-connector-python==8.0.26', + 'bson==0.5.7', + 'pymongo==3.7.2', + 'crate==0.26.0', + 'influxdb==5.3.1', + 'psycopg2-binary==2.8.6'], + 'documentation': ['mock==4.0.3', + 'Sphinx==4.1.2', + 'sphinx-rtd-theme==0.5.2', + 'sphinx==3.3.0', + 'm2r2==0.3.1'], + 'drivers': ['pymodbus==2.5.2', + 'bacpypes==0.16.7', + 'modbus-tk==1.1.2', + 'pyserial==3.5'], + 'influxdb': ['influxdb==5.3.1'], + 'market': ['numpy==1.19.5', 'transitions==0.8.8'], + 'mongo': ['bson==0.5.7pymongo==3.7.2'], + 'mysql': ['mysql-connector-python==8.0.26'], + 'pandas': ['numpy==1.19.5', 'pandas==1.1.5'], + 'postgres': ['psycopg2-binary==2.8.6'], + 'testing': ['mock==4.0.3', + 'pytest==6.2.4', + 'pytest-timeout==1.4.2', + 'pytest-rerunfailures==10.1', + 'websocket-client==1.2.1', + 'deepdiff==5.5.0', + 'docker==5.0.0'], + 'weather': ['Pint==0.17'], + 'web': ['ws4py==0.5.1', + 'PyJWT==1.7.1', + 'Jinja2==3.0.1', + 'passlib==1.7.4', + 'argon2-cffi==20.1.0', + 'Werkzeug==2.0.1']} +install_requires = ['gevent==20.6.1', + 'greenlet==0.4.16', + 'grequests==0.6.0', + 'idna<3,>=2.5', + 'requests==2.23.0', + 'ply==3.11', + 'psutil==5.8.0', + 'python-dateutil==2.8.2', + 'pytz==2021.1', + 'PyYAML==5.4.1', + 'pyzmq==22.2.1', + 'setuptools>=40.0.0', + 'tzlocal==2.1', + 'pyOpenSSL==19.0.0', + 'cryptography==2.3', + 'watchdog-gevent==0.1.1', + 'wheel==0.30'] - -install_requires = [ - 'gevent==20.6.1', - 'greenlet==0.4.16', - 'grequests', - 'idna<3,>=2.5', - 'requests==2.23.0', - 'ply', - 'psutil', - 'python-dateutil', - 'pytz', - 'PyYAML', - 'pyzmq', - 'setuptools>=40.0.0', - # tzlocal 3.0 breaks without the backports.tzinfo package on python < 3.9 https://pypi.org/project/tzlocal/3.0/ - 'tzlocal==2.1', - 'pyOpenSSL==19.0.0', - 'cryptography==2.3', - # Cross platform way of handling changes in file/directories. - # https://github.com/Bogdanp/watchdog_gevent - 'watchdog-gevent', - 'wheel==0.30' -] - -extras_require = { - 'crate': ['crate'], - 'databases': ['mysql-connector-python', - 'bson==0.5.7', - 'pymongo==3.7.2', - 'crate', - 'influxdb', - 'psycopg2-binary==2.8.6'], - 'documentation': ['mock', - 'Sphinx', - 'sphinx-rtd-theme', - 'm2r2'], - 'drivers': ['pymodbus', - 'bacpypes==0.16.7', - 'modbus-tk', - 'pyserial'], - 'influxdb': ['influxdb'], - 'market': ['numpy', 'transitions'], - 'mongo': ['bson==0.5.7', - 'pymongo==3.7.2'], - 'mysql': ['mysql-connector-python'], - 'pandas': ['numpy', - 'pandas'], - 'postgres': ['psycopg2-binary==2.8.6'], - 'testing': ['mock', - 'pytest', - 'pytest-timeout', - 'pytest-rerunfailures', - 'websocket-client', - 'deepdiff', - 'docker'], - 'weather': ['Pint'], - 'web': ['ws4py', - 'PyJWT==1.7.1', - 'Jinja2', - 'passlib', - 'argon2-cffi', - 'Werkzeug']} +option_requirements = [('pyzmq==22.2.1', ['--zmq=bundled'])] diff --git a/volttron/platform/__init__.py b/volttron/platform/__init__.py index 57ea86ab2c..2f33379f47 100644 --- a/volttron/platform/__init__.py +++ b/volttron/platform/__init__.py @@ -49,7 +49,7 @@ from urllib.parse import urlparse from ..utils.frozendict import FrozenDict -__version__ = '8.1' +__version__ = '8.1.1' _log = logging.getLogger(__name__) From abd14336da4c51ee4b22f5783127a551d963b0fb Mon Sep 17 00:00:00 2001 From: sgilbride Date: Thu, 9 Sep 2021 11:01:38 -0700 Subject: [PATCH 13/36] Added auth file version upgrade for minor version 2. Allows for proper conversion from pre-1.2 -> 1.3. --- volttron/platform/auth.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volttron/platform/auth.py b/volttron/platform/auth.py index 095783524a..061b815518 100644 --- a/volttron/platform/auth.py +++ b/volttron/platform/auth.py @@ -1866,6 +1866,7 @@ def upgrade_1_2_to_1_3(allow_list): version["minor"] = 1 if version["major"] == 1 and version["minor"] == 1: allow_list = upgrade_1_1_to_1_2(allow_list) + version["minor"] = 2 if version["major"] == 1 and version["minor"] == 2: allow_list = upgrade_1_2_to_1_3(allow_list) From 283be6b3965afcfd85926a1062011776ca8603b8 Mon Sep 17 00:00:00 2001 From: Craig <3979063+craig8@users.noreply.github.com> Date: Thu, 9 Sep 2021 15:41:39 -0700 Subject: [PATCH 14/36] Update instance_setup.py Fix issue caused by not having a default for sqlhistorian during platform.historian install. --- volttron/platform/instance_setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/volttron/platform/instance_setup.py b/volttron/platform/instance_setup.py index 5997039042..2061df46e7 100644 --- a/volttron/platform/instance_setup.py +++ b/volttron/platform/instance_setup.py @@ -268,7 +268,9 @@ def func(*args, **kwargs): _shutdown_platform() if identity is not None: - os.environ.pop('AGENT_VIP_IDENTITY') + # If identity was specified then we don't want the global + # AGENT_VIP_IDENTITY to be set. + os.environ.pop('AGENT_VIP_IDENTITY', None) available_agents[tag] = func return func From 7576174d969bb621f5cde0455abd618f7262382a Mon Sep 17 00:00:00 2001 From: sgilbride Date: Thu, 9 Sep 2021 21:05:53 -0700 Subject: [PATCH 15/36] Added volttron-update-auth to upgrade auth file for existing installation. Allows for the use of the dynamic rpc authorizations. Minor test updates. --- setup.py | 2 + volttron/platform/update_auth_file.py | 92 +++++++++++++++++++ .../platform/auth_tests/test_auth_file.py | 27 ++++-- 3 files changed, 115 insertions(+), 6 deletions(-) create mode 100644 volttron/platform/update_auth_file.py diff --git a/setup.py b/setup.py index 2ec369bd8b..3e0e9b188b 100644 --- a/setup.py +++ b/setup.py @@ -70,6 +70,8 @@ 'vctl = volttron.platform.control:_main', 'vpkg = volttron.platform.packaging:_main', 'vcfg = volttron.platform.config:_main', + # 'volttron-update = ...', + 'volttron-update-auth = volttron.platform.update_auth_file:_main', ] }, zip_safe = False, diff --git a/volttron/platform/update_auth_file.py b/volttron/platform/update_auth_file.py new file mode 100644 index 0000000000..7d3c2da012 --- /dev/null +++ b/volttron/platform/update_auth_file.py @@ -0,0 +1,92 @@ +# -*- coding: utf-8 -*- {{{ +# vim: set fenc=utf-8 ft=python sw=4 ts=4 sts=4 et: +# +# Copyright 2020, Battelle Memorial Institute. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This material was prepared as an account of work sponsored by an agency of +# the United States Government. Neither the United States Government nor the +# United States Department of Energy, nor Battelle, nor any of their +# employees, nor any jurisdiction or organization that has cooperated in the +# development of these materials, makes any warranty, express or +# implied, or assumes any legal liability or responsibility for the accuracy, +# completeness, or usefulness or any information, apparatus, product, +# software, or process disclosed, or represents that its use would not infringe +# privately owned rights. Reference herein to any specific commercial product, +# process, or service by trade name, trademark, manufacturer, or otherwise +# does not necessarily constitute or imply its endorsement, recommendation, or +# favoring by the United States Government or any agency thereof, or +# Battelle Memorial Institute. The views and opinions of authors expressed +# herein do not necessarily state or reflect those of the +# United States Government or any agency thereof. +# +# PACIFIC NORTHWEST NATIONAL LABORATORY operated by +# BATTELLE for the UNITED STATES DEPARTMENT OF ENERGY +# under Contract DE-AC05-76RL01830 +# }}} +import sys + +from volttron.platform import get_home +from volttron.platform.aip import AIPplatform +from volttron.platform.auth import AuthFile +from volttron.platform.instance_setup import fail_if_instance_running + + +def get_identity_credentials(): + """Returns a dictionary containing a mapping from publickey to identity""" + + vhome = get_home() + options = type("Options", (), dict(volttron_home=vhome)) + aip = AIPplatform(options) + agent_map = aip.get_agent_identity_to_uuid_mapping() + agent_credential_map = {} + for agent in agent_map: + agent_credential = aip.get_agent_keystore(agent_map[agent]).public + agent_credential_map[agent_credential] = agent + return agent_credential_map + + +def set_auth_identities(agent_credential_map): + """Updates auth entries' identity field in auth file based on existing agents""" + + auth_file = AuthFile() + entries, deny_entries, groups, roles = auth_file.read() + for entry in entries: + for credential in agent_credential_map: + if entry.credentials == credential: + entry.identity = agent_credential_map[credential] + auth_file._write(entries, deny_entries, groups, roles) + return + + +def main(): + """Upgrade auth file to function with dynamic rpc authorizations""" + + fail_if_instance_running() + identity_map = get_identity_credentials() + set_auth_identities(identity_map) + print("Auth File Update Complete!") + + +def _main(): + """ Wrapper for main function""" + + try: + sys.exit(main()) + except KeyboardInterrupt: + sys.exit(1) + + +if __name__ == "__main__": + _main() diff --git a/volttrontesting/platform/auth_tests/test_auth_file.py b/volttrontesting/platform/auth_tests/test_auth_file.py index ec7724151a..f7d8c65e05 100644 --- a/volttrontesting/platform/auth_tests/test_auth_file.py +++ b/volttrontesting/platform/auth_tests/test_auth_file.py @@ -253,7 +253,7 @@ def test_groups_and_roles(auth_file_platform_tuple): @pytest.mark.auth -def test_upgrade_file_verison_0_to_1_2(tmpdir_factory): +def test_upgrade_file_verison_0_to_latest(tmpdir_factory): mechanism = "CURVE" publickey = "A" * 43 version0 = { @@ -275,7 +275,11 @@ def test_upgrade_file_verison_0_to_1_2(tmpdir_factory): }, "groups": { "admin": ["reader", "writer"] - } + }, + "version": { + "major": 0, + "minor": 0 + }, } filename = str(tmpdir_factory.mktemp('auth_test').join('auth.json')) @@ -293,16 +297,23 @@ def test_upgrade_file_verison_0_to_1_2(tmpdir_factory): expected["mechanism"] = mechanism expected["capabilities"] = {'can_publish_temperature': None, 'edit_config_store': {'identity': entries[0].user_id}} + expected["rpc_method_authorizations"] = {} assert_auth_entries_same(expected, vars(entries[0])) - + # RPC Method Authorizations added with 1.3 + for entry in upgraded.auth_data["allow_list"]: + assert entry["rpc_method_authorizations"] == {} @pytest.mark.auth -def test_upgrade_file_verison_0_to_1_2_minimum_entries(tmpdir_factory): +def test_upgrade_file_verison_0_to_latest_minimum_entries(tmpdir_factory): """The only required field in 'version 0' was credentials""" mechanism = "CURVE" publickey = "A" * 43 version0 = { "allow": [{"credentials": mechanism + ":" + publickey}], + "version": { + "major": 0, + "minor": 0 + }, } filename = str(tmpdir_factory.mktemp('auth_test').join('auth.json')) @@ -323,10 +334,14 @@ def test_upgrade_file_verison_0_to_1_2_minimum_entries(tmpdir_factory): expected["enabled"] = True expected["comments"] = None expected["capabilities"] = {'edit_config_store': {'identity': entries[0].user_id}} + expected["rpc_method_authorizations"] = {} expected["roles"] = [] expected["groups"] = [] assert_auth_entries_same(expected, vars(entries[0])) + # RPC Method Authorizations added with 1.3 + for entry in upgraded.auth_data["allow_list"]: + assert entry["rpc_method_authorizations"] == {} @pytest.mark.auth def test_upgrade_file_version_1_1_to_1_2(tmpdir_factory): @@ -422,7 +437,7 @@ def test_upgrade_file_version_1_1_to_1_2(tmpdir_factory): def test_upgrade_file_version_1_2_to_1_3(tmpdir_factory): """The only required field in 'version 0' was credentials""" - version1_1 = { + version1_2 = { "roles":{ "manager":[ "can_managed_platform" @@ -495,7 +510,7 @@ def test_upgrade_file_version_1_2_to_1_3(tmpdir_factory): filename = str(tmpdir_factory.mktemp('auth_test').join('auth.json')) with open(filename, 'w') as fp: - fp.write(jsonapi.dumps(version1_1, indent=2)) + fp.write(jsonapi.dumps(version1_2, indent=2)) upgraded = AuthFile(filename) entries = upgraded.read()[0] From 16a56a22677d09e060525e47e307920c2c3a5b5d Mon Sep 17 00:00:00 2001 From: sgilbride Date: Thu, 9 Sep 2021 21:24:45 -0700 Subject: [PATCH 16/36] Changed log level for missing auth entry / identity field for updating dynamic rpc authorizations to warning. Added additional context and troubleshooting to warning message. --- volttron/platform/vip/agent/subsystems/auth.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/volttron/platform/vip/agent/subsystems/auth.py b/volttron/platform/vip/agent/subsystems/auth.py index 937241f23b..651224910a 100644 --- a/volttron/platform/vip/agent/subsystems/auth.py +++ b/volttron/platform/vip/agent/subsystems/auth.py @@ -694,9 +694,12 @@ def update_rpc_method_capabilities(self): .get(timeout=4) ) if updated_rpc_authorizations is None: - _log.error( + _log.warning( f"Auth entry not found for {self._core().identity}: " - f"rpc_method_authorizations not updated." + f"rpc_method_authorizations not updated. If this agent " + f"does have an auth entry, verify that the 'identity' field " + f"has been included in the auth entry. This should be set to " + f"the identity of the agent" ) return if rpc_method_authorizations != updated_rpc_authorizations: From 92b5c00ff44227e05f31faa75ec9a1d107a5db0b Mon Sep 17 00:00:00 2001 From: sgilbride Date: Fri, 10 Sep 2021 14:38:01 -0700 Subject: [PATCH 17/36] Updated update_auth_file.py to handle old instances of VOLTTRON. keystore.json will be moved from agent-data to dist-info before update occurs. --- volttron/platform/update_auth_file.py | 65 +++++++++++++++++++++++++-- 1 file changed, 62 insertions(+), 3 deletions(-) diff --git a/volttron/platform/update_auth_file.py b/volttron/platform/update_auth_file.py index 7d3c2da012..0781171998 100644 --- a/volttron/platform/update_auth_file.py +++ b/volttron/platform/update_auth_file.py @@ -35,20 +35,77 @@ # BATTELLE for the UNITED STATES DEPARTMENT OF ENERGY # under Contract DE-AC05-76RL01830 # }}} +import os import sys +from shutil import move +from gevent import monkey as curious_george +curious_george.patch_all(thread=False, select=False) from volttron.platform import get_home from volttron.platform.aip import AIPplatform from volttron.platform.auth import AuthFile from volttron.platform.instance_setup import fail_if_instance_running +from volttron.platform.keystore import KeyStore -def get_identity_credentials(): - """Returns a dictionary containing a mapping from publickey to identity""" +def get_aip(): + """Get AIPplatform to interface with agent directories in vhome""" vhome = get_home() options = type("Options", (), dict(volttron_home=vhome)) aip = AIPplatform(options) + return aip + + +def get_agent_name(agent_uuid, agent_path): + """ + Stand-alone method based off of agent_name method from AIPplatform. + Gets the name of an agent from it's file path location. + """ + + for agent_name in os.listdir(agent_path): + dist_info = os.path.join( + agent_path, agent_name, agent_name + '.dist-info') + if os.path.exists(dist_info): + return agent_name + + raise KeyError(agent_uuid) + + +def upgrade_old_agents(aip): + """ + Moves any keystore.json from agent-data to dist-info. + Only applies to agents in auth file. + """ + + vhome = aip.env.volttron_home + agent_map = aip.get_agent_identity_to_uuid_mapping() + + auth_file = AuthFile() + install_dir = os.path.join(vhome, 'agents') + for agent in agent_map: + agent_path = os.path.join(install_dir, agent_map[agent]) + + agent_name = get_agent_name(agent_map[agent], agent_path) + agent_data = os.path.join(agent_path, agent_name, + agent_name + '.agent-data') + keystore_path = os.path.join(agent_data, 'keystore.json') + dist_info = os.path.join(agent_path, agent_name, + agent_name + '.dist-info') + keystore_dest_path = os.path.join(dist_info, 'keystore.json') + + if os.path.isfile(keystore_path): + agent_keystore = KeyStore(keystore_path) + for entry in auth_file.read()[0]: + if entry.credentials == agent_keystore.public: + move(keystore_path, keystore_dest_path) + return + + + +def get_identity_credentials(aip): + """Returns a dictionary containing a mapping from publickey to identity""" + agent_map = aip.get_agent_identity_to_uuid_mapping() agent_credential_map = {} for agent in agent_map: @@ -74,7 +131,9 @@ def main(): """Upgrade auth file to function with dynamic rpc authorizations""" fail_if_instance_running() - identity_map = get_identity_credentials() + aip = get_aip() + upgrade_old_agents(aip) + identity_map = get_identity_credentials(aip) set_auth_identities(identity_map) print("Auth File Update Complete!") From 91203eb5334e167402fd3bbe6fdeab4a7dfd5698 Mon Sep 17 00:00:00 2001 From: sgilbride Date: Fri, 10 Sep 2021 16:39:38 -0700 Subject: [PATCH 18/36] Implemented pathlib for cleaner directory work. Added warning in casae of connecting new agent to older AuthService. --- volttron/platform/update_auth_file.py | 40 +++++++++---------- .../platform/vip/agent/subsystems/auth.py | 27 ++++++++----- 2 files changed, 37 insertions(+), 30 deletions(-) diff --git a/volttron/platform/update_auth_file.py b/volttron/platform/update_auth_file.py index 0781171998..cefa6ad4f7 100644 --- a/volttron/platform/update_auth_file.py +++ b/volttron/platform/update_auth_file.py @@ -37,7 +37,8 @@ # }}} import os import sys -from shutil import move +import shutil +from pathlib import Path from gevent import monkey as curious_george curious_george.patch_all(thread=False, select=False) @@ -57,19 +58,17 @@ def get_aip(): return aip -def get_agent_name(agent_uuid, agent_path): +def get_agent_name(agent_dir_path): """ Stand-alone method based off of agent_name method from AIPplatform. Gets the name of an agent from it's file path location. """ - for agent_name in os.listdir(agent_path): - dist_info = os.path.join( - agent_path, agent_name, agent_name + '.dist-info') - if os.path.exists(dist_info): - return agent_name + for agent_name in agent_dir_path.iterdir(): + if agent_name.match(".dist-info"): + return agent_name.stem - raise KeyError(agent_uuid) + raise KeyError(agent_dir_path.stem) def upgrade_old_agents(aip): @@ -78,27 +77,28 @@ def upgrade_old_agents(aip): Only applies to agents in auth file. """ - vhome = aip.env.volttron_home + vhome = Path(aip.env.volttron_home) agent_map = aip.get_agent_identity_to_uuid_mapping() auth_file = AuthFile() - install_dir = os.path.join(vhome, 'agents') + install_dir = vhome.joinpath("agents") for agent in agent_map: - agent_path = os.path.join(install_dir, agent_map[agent]) + agent_path = install_dir.joinpath(agent_map[agent]) - agent_name = get_agent_name(agent_map[agent], agent_path) - agent_data = os.path.join(agent_path, agent_name, - agent_name + '.agent-data') - keystore_path = os.path.join(agent_data, 'keystore.json') - dist_info = os.path.join(agent_path, agent_name, - agent_name + '.dist-info') - keystore_dest_path = os.path.join(dist_info, 'keystore.json') + agent_name = get_agent_name(agent_path) + agent_data = agent_path.joinpath(agent_name, + agent_name + '.agent-data') + keystore_path = agent_data.joinpath('keystore.json') + dist_info = agent_path.joinpath(agent_name, + agent_name + '.dist-info') + keystore_dest_path = dist_info.joinpath('keystore.json') - if os.path.isfile(keystore_path): + if keystore_path.exists(): agent_keystore = KeyStore(keystore_path) for entry in auth_file.read()[0]: if entry.credentials == agent_keystore.public: - move(keystore_path, keystore_dest_path) + shutil.move(str(keystore_path), str(keystore_dest_path)) + break return diff --git a/volttron/platform/vip/agent/subsystems/auth.py b/volttron/platform/vip/agent/subsystems/auth.py index 651224910a..26f8fae9a2 100644 --- a/volttron/platform/vip/agent/subsystems/auth.py +++ b/volttron/platform/vip/agent/subsystems/auth.py @@ -61,7 +61,7 @@ get_messagebus, ) from volttron.platform.certs import Certs -from volttron.platform.jsonrpc import RemoteError +from volttron.platform.jsonrpc import RemoteError, MethodNotFound from volttron.utils.rmq_config_params import RMQConfig from volttron.platform.keystore import KeyStore from volttron.platform.vip.agent.subsystems.health import BAD_STATUS, Status @@ -683,16 +683,23 @@ def update_rpc_method_capabilities(self): rpc_method_authorizations[method] = self.get_rpc_authorizations( method ) - updated_rpc_authorizations = ( - self._rpc() - .call( - AUTH, - "update_id_rpc_authorizations", - self._core().identity, - rpc_method_authorizations, + try: + updated_rpc_authorizations = ( + self._rpc() + .call( + AUTH, + "update_id_rpc_authorizations", + self._core().identity, + rpc_method_authorizations, + ) + .get(timeout=4) ) - .get(timeout=4) - ) + except MethodNotFound: + _log.warning("update_id_rpc_authorization method is missing from " + "AuthService! The VOLTTRON Instance you are " + "attempting to connect to is to old to support " + "dynamic RPC authorizations.") + return if updated_rpc_authorizations is None: _log.warning( f"Auth entry not found for {self._core().identity}: " From b53b509d95b4af7bef7f8cfeb85055f9c6b9a967 Mon Sep 17 00:00:00 2001 From: sgilbride Date: Fri, 10 Sep 2021 23:04:14 -0700 Subject: [PATCH 19/36] Fixed get_agent_name to work properly with pathlib. --- volttron/platform/update_auth_file.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/volttron/platform/update_auth_file.py b/volttron/platform/update_auth_file.py index cefa6ad4f7..a13aa0d1ee 100644 --- a/volttron/platform/update_auth_file.py +++ b/volttron/platform/update_auth_file.py @@ -65,8 +65,9 @@ def get_agent_name(agent_dir_path): """ for agent_name in agent_dir_path.iterdir(): - if agent_name.match(".dist-info"): - return agent_name.stem + dist_info = agent_name.joinpath(agent_name.name + ".dist-info") + if dist_info.exists(): + return agent_name.name raise KeyError(agent_dir_path.stem) From 792af7ac43093d6c77f54c028776c7faefe32399 Mon Sep 17 00:00:00 2001 From: sgilbride Date: Mon, 13 Sep 2021 16:04:12 -0700 Subject: [PATCH 20/36] Cleaned up get_agent to reflect naming differences on older releases. Added additional comments. --- volttron/platform/update_auth_file.py | 37 ++++++++++++++++++--------- 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/volttron/platform/update_auth_file.py b/volttron/platform/update_auth_file.py index a13aa0d1ee..f99b51e1ca 100644 --- a/volttron/platform/update_auth_file.py +++ b/volttron/platform/update_auth_file.py @@ -35,7 +35,7 @@ # BATTELLE for the UNITED STATES DEPARTMENT OF ENERGY # under Contract DE-AC05-76RL01830 # }}} -import os + import sys import shutil from pathlib import Path @@ -58,16 +58,21 @@ def get_aip(): return aip -def get_agent_name(agent_dir_path): +def get_agent_path(agent_dir_path, agent_dir_suffix): """ Stand-alone method based off of agent_name method from AIPplatform. - Gets the name of an agent from it's file path location. + Gets the path to the agent file of the specified directory if it exists. """ - for agent_name in agent_dir_path.iterdir(): - dist_info = agent_name.joinpath(agent_name.name + ".dist-info") - if dist_info.exists(): - return agent_name.name + try: + for agent_subdir in agent_name.iterdir(): + agent_dir = agent_name.joinpath( + agent_subdir.stem + f".{agent_dir_suffix}") + if agent_dir.exists(): + return agent_dir + # Ignore files that are not directories + except NotADirectoryError: + pass raise KeyError(agent_dir_path.stem) @@ -85,18 +90,26 @@ def upgrade_old_agents(aip): install_dir = vhome.joinpath("agents") for agent in agent_map: agent_path = install_dir.joinpath(agent_map[agent]) + try: + agent_data = get_agent_path(agent_path, 'agent-data') + # Skip if no agent-data exists + except KeyError as err: + print(f"agent-data not found for {err}") + continue - agent_name = get_agent_name(agent_path) - agent_data = agent_path.joinpath(agent_name, - agent_name + '.agent-data') keystore_path = agent_data.joinpath('keystore.json') - dist_info = agent_path.joinpath(agent_name, - agent_name + '.dist-info') + try: + dist_info = get_agent_path(agent_path, 'dist-info') + # Skip if no dist-info exists + except KeyError as err: + print(f"dist-info not found for {err}") + continue keystore_dest_path = dist_info.joinpath('keystore.json') if keystore_path.exists(): agent_keystore = KeyStore(keystore_path) for entry in auth_file.read()[0]: + # Only move if agent exists in auth file if entry.credentials == agent_keystore.public: shutil.move(str(keystore_path), str(keystore_dest_path)) break From f4f8a4185661d655c53333b8adc0f4faef43e6fb Mon Sep 17 00:00:00 2001 From: sgilbride Date: Tue, 14 Sep 2021 22:36:15 -0700 Subject: [PATCH 21/36] Added documentation to README and upgrading-versions to include requirements for 8.x. Updated test_auth_control to be more robust. --- README.md | 7 ++++ .../change-log/upgrading-versions.rst | 7 ++++ .../platform/auth_tests/test_auth_control.py | 38 +++++++++---------- 3 files changed, 33 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 4c9abb92c2..39078d0ea0 100644 --- a/README.md +++ b/README.md @@ -296,6 +296,13 @@ There are several walkthroughs to explore additional aspects of the platform: - [RabbitMQ setup with Federation and Shovel plugins](https://volttron.readthedocs.io/en/latest/deploying-volttron/multi-platform/multi-platform-rabbitmq-deployment.html) - [Backward compatibility with the RabbitMQ message bus](https://volttron.readthedocs.io/en/latest/deploying-volttron/multi-platform/multi-platform-multi-bus.html) +## Upgrade to VOLTTRON 8.x + +VOLTTRON 8 and above introduces dynamic RPC authorization, which requires a modification to the auth file. +If you have a pre-existing instance of VOLTTRON running on an older version, the auth file will need to be updated. +This can be done by running the ```volttron-update-auth``` in an activated VOLTTRON environment. The platform can not +be running when this update script is ran. + ## Acquiring Third Party Agent Code Third party agents are available under the volttron-applications repository. In diff --git a/docs/source/volttron-topics/change-log/upgrading-versions.rst b/docs/source/volttron-topics/change-log/upgrading-versions.rst index c0cf0fffa0..fe7b27daa1 100644 --- a/docs/source/volttron-topics/change-log/upgrading-versions.rst +++ b/docs/source/volttron-topics/change-log/upgrading-versions.rst @@ -8,6 +8,13 @@ It is often recommended that users upgrade to the latest stable release of VOLTT releases include helpful new features, bug fixes, and other improvements. Please see the guides below for upgrading your existing deployment to the latest version. +VOLTTRON 8 +========== + +VOLTTRON 8 introduces dynamic RPC authorization, which requires a modification to the auth file. +If you have a pre-existing instance of VOLTTRON running on an older version, the auth file will need to be updated. +This can be done by running the ```volttron-update-auth``` in an activated VOLTTRON environment. The platform can not +be running when this update script is ran. VOLTTRON 7 ========== diff --git a/volttrontesting/platform/auth_tests/test_auth_control.py b/volttrontesting/platform/auth_tests/test_auth_control.py index bb58b672d6..8c7e46da78 100644 --- a/volttrontesting/platform/auth_tests/test_auth_control.py +++ b/volttrontesting/platform/auth_tests/test_auth_control.py @@ -193,7 +193,7 @@ def auth_instance(volttron_instance): # Number of tries to check if auth file is updated properly -auth_retry = 30 +auth_retry = 5 @pytest.mark.control @@ -214,8 +214,8 @@ def test_auth_add(auth_instance): print(entries) assert len(entries) > 0 i = 0 - while len(entries) < len_entries and i < auth_retry: - gevent.sleep(1) + while len(entries) <= len_entries and i < auth_retry: + gevent.sleep(i) entries = auth_list_json(platform) i += 1 @@ -234,8 +234,8 @@ def test_auth_add_cmd_line(auth_instance): print(entries) assert len(entries) > 0 i = 0 - while len(entries) < len_entries and i < auth_retry: - gevent.sleep(1) + while len(entries) <= len_entries and i < auth_retry: + gevent.sleep(i) entries = auth_list_json(platform) i += 1 assert_auth_entries_same(entries[-1], _auth_entry2.__dict__) @@ -252,8 +252,8 @@ def test_auth_update(auth_instance): print(entries) assert len(entries) > 0 i = 0 - while len(entries) < len_entries and i < auth_retry: - gevent.sleep(1) + while len(entries) <= len_entries and i < auth_retry: + gevent.sleep(i) entries = auth_list_json(platform) i += 1 auth_update(platform, len(entries) - 1, **_auth_entry4.__dict__) @@ -274,16 +274,16 @@ def test_auth_remove(auth_instance): entries = auth_list_json(platform) assert len(entries) > 0 i = 0 - while len(entries) < len_entries and i < auth_retry: - gevent.sleep(1) + while len(entries) <= len_entries and i < auth_retry: + gevent.sleep(i) entries = auth_list_json(platform) i += 1 auth_add(platform, _auth_entry6) entries = auth_list_json(platform) assert len(entries) > 0 i = 0 - while len(entries) < (len_entries + 1) and i < auth_retry: - gevent.sleep(1) + while len(entries) <= (len_entries + 1) and i < auth_retry: + gevent.sleep(i) entries = auth_list_json(platform) i += 1 print(entries) @@ -296,7 +296,7 @@ def test_auth_remove(auth_instance): assert len(entries) > 0 i = 0 while len(entries) > (len_entries + 1) and i < auth_retry: - gevent.sleep(1) + gevent.sleep(i) entries = auth_list_json(platform) i += 1 assert_auth_entries_same(entries[-1], _auth_entry5.__dict__) @@ -312,8 +312,8 @@ def test_auth_rpc_method_add(auth_instance): entries = auth_list_json(platform) assert len(entries) > 0 i = 0 - while len(entries) < len_entries and i < auth_retry: - gevent.sleep(1) + while len(entries) <= len_entries and i < auth_retry: + gevent.sleep(i) entries = auth_list_json(platform) i += 1 print(entries) @@ -324,7 +324,7 @@ def test_auth_rpc_method_add(auth_instance): i = 0 while entries[-1]['rpc_method_authorizations'] != {'test_method': ["test_auth"]} and i < auth_retry: - gevent.sleep(1) + gevent.sleep(i) entries = auth_list_json(platform) i += 1 @@ -341,8 +341,8 @@ def test_auth_rpc_method_remove(auth_instance): entries = auth_list_json(platform) assert len(entries) > 0 i = 0 - while len(entries) < len_entries and i < auth_retry: - gevent.sleep(1) + while len(entries) <= len_entries and i < auth_retry: + gevent.sleep(i) entries = auth_list_json(platform) i += 1 print(entries) @@ -353,7 +353,7 @@ def test_auth_rpc_method_remove(auth_instance): i = 0 while entries[-1]['rpc_method_authorizations'] != {'test_method': ["test_auth"]} and i < auth_retry: - gevent.sleep(1) + gevent.sleep(i) entries = auth_list_json(platform) i += 1 @@ -365,7 +365,7 @@ def test_auth_rpc_method_remove(auth_instance): i = 0 while entries[-1]['rpc_method_authorizations'] == {'test_method': ["test_auth"]} and i < auth_retry: - gevent.sleep(1) + gevent.sleep(i) entries = auth_list_json(platform) i += 1 From e4d75d4fc2fa16b9407f4a358ea08e11a2fed80d Mon Sep 17 00:00:00 2001 From: sgilbride Date: Wed, 15 Sep 2021 13:25:42 -0700 Subject: [PATCH 22/36] Updated documentation to reflect additional steps required. --- README.md | 14 ++++++++------ .../change-log/upgrading-versions.rst | 12 +++++++++--- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 39078d0ea0..11682604f8 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,14 @@ platform provides services for collecting and storing data from buildings and devices and provides an environment for developing applications which interact with that data. +## Upgrading to VOLTTRON 8.x + +VOLTTRON 8 and above introduces dynamic RPC authorization, which requires a modification to the auth file. +If you have a pre-existing instance of VOLTTRON running on an older version, the auth file will need to be updated. +To begin the upgrade process, activate the volttron environment, and run ```python bootstrap.py --force```. If you +have any additional bootstrap options that you need (rabbitmq, web, drivers, etc.) include these in the above command. +After the bootstrap process is completed, run ```volttron-update-auth``` to update the auth file. + ## Features @@ -296,12 +304,6 @@ There are several walkthroughs to explore additional aspects of the platform: - [RabbitMQ setup with Federation and Shovel plugins](https://volttron.readthedocs.io/en/latest/deploying-volttron/multi-platform/multi-platform-rabbitmq-deployment.html) - [Backward compatibility with the RabbitMQ message bus](https://volttron.readthedocs.io/en/latest/deploying-volttron/multi-platform/multi-platform-multi-bus.html) -## Upgrade to VOLTTRON 8.x - -VOLTTRON 8 and above introduces dynamic RPC authorization, which requires a modification to the auth file. -If you have a pre-existing instance of VOLTTRON running on an older version, the auth file will need to be updated. -This can be done by running the ```volttron-update-auth``` in an activated VOLTTRON environment. The platform can not -be running when this update script is ran. ## Acquiring Third Party Agent Code diff --git a/docs/source/volttron-topics/change-log/upgrading-versions.rst b/docs/source/volttron-topics/change-log/upgrading-versions.rst index fe7b27daa1..ef30037c54 100644 --- a/docs/source/volttron-topics/change-log/upgrading-versions.rst +++ b/docs/source/volttron-topics/change-log/upgrading-versions.rst @@ -11,10 +11,16 @@ your existing deployment to the latest version. VOLTTRON 8 ========== -VOLTTRON 8 introduces dynamic RPC authorization, which requires a modification to the auth file. +VOLTTRON 8 and above introduces dynamic RPC authorization, which requires a modification to the auth file. If you have a pre-existing instance of VOLTTRON running on an older version, the auth file will need to be updated. -This can be done by running the ```volttron-update-auth``` in an activated VOLTTRON environment. The platform can not -be running when this update script is ran. +To begin the upgrade process, activate the volttron environment, and run ```python bootstrap.py --force```. + +.. note:: + + If you have any additional bootstrap options that you need (rabbitmq, web, drivers, etc.) + include these in the above command. + +After the bootstrap process is completed, run ```volttron-update-auth``` to update the auth file. VOLTTRON 7 ========== From cbf4ba9c28efc034b1ff6725509cfc6a8292c912 Mon Sep 17 00:00:00 2001 From: sgilbride Date: Thu, 16 Sep 2021 14:38:40 -0700 Subject: [PATCH 23/36] Updated release history documentation, and added links to GitHub releases. --- docs/source/index.rst | 3 +- .../index.rst | 7 +- .../VOLTTRON-releases/release-history.rst | 123 ++++++++++ .../upgrading-versions.rst | 0 .../scalability/scalability-improvements.rst | 24 -- .../change-log/scalability/scalability.rst | 133 ----------- .../testing-driver-scalability.rst | 214 ------------------ .../change-log/version-history.rst | 164 -------------- 8 files changed, 128 insertions(+), 540 deletions(-) rename docs/source/volttron-topics/{change-log => VOLTTRON-releases}/index.rst (79%) create mode 100644 docs/source/volttron-topics/VOLTTRON-releases/release-history.rst rename docs/source/volttron-topics/{change-log => VOLTTRON-releases}/upgrading-versions.rst (100%) delete mode 100644 docs/source/volttron-topics/change-log/scalability/scalability-improvements.rst delete mode 100644 docs/source/volttron-topics/change-log/scalability/scalability.rst delete mode 100644 docs/source/volttron-topics/change-log/scalability/testing-driver-scalability.rst delete mode 100644 docs/source/volttron-topics/change-log/version-history.rst diff --git a/docs/source/index.rst b/docs/source/index.rst index 82490ca042..2a5a74df9e 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -161,9 +161,10 @@ at our bi-weekly office-hours and on Slack. To be invited to office-hours or sla :titlesonly: :maxdepth: 1 + volttron-topics/VOLTTRON-releases/index volttron-topics/troubleshooting/index volttron-topics/volttron-applications/index - volttron-topics/change-log/index + diff --git a/docs/source/volttron-topics/change-log/index.rst b/docs/source/volttron-topics/VOLTTRON-releases/index.rst similarity index 79% rename from docs/source/volttron-topics/change-log/index.rst rename to docs/source/volttron-topics/VOLTTRON-releases/index.rst index c488332810..68a914e64c 100644 --- a/docs/source/volttron-topics/change-log/index.rst +++ b/docs/source/volttron-topics/VOLTTRON-releases/index.rst @@ -1,7 +1,7 @@ -.. _Change-Log: +.. _VOLTTRON-Releases: ========== -Change Log +VOLTTRON Releases ========== This section includes individual documents describing important changes to platform components, such as the RabbitMQ @@ -10,6 +10,5 @@ message bus implementation. For information on specific changes, please refer to .. toctree:: - scalability/scalability - version-history + release-history upgrading-versions diff --git a/docs/source/volttron-topics/VOLTTRON-releases/release-history.rst b/docs/source/volttron-topics/VOLTTRON-releases/release-history.rst new file mode 100644 index 0000000000..444c8effb3 --- /dev/null +++ b/docs/source/volttron-topics/VOLTTRON-releases/release-history.rst @@ -0,0 +1,123 @@ +.. _Release-History: + +=============== +Release History +=============== + +VOLTTRON Release Documentation for version 5.1.0 and above is found on GitHub. +`https://github.com/VOLTTRON/volttron/releases `_ + +VOLTTRON 8.1.1 Maintenance Release +================================== + +`https://github.com/VOLTTRON/volttron/releases/tag/8.1.1 `_ + + +VOLTTRON 8.1 Release +==================== + +`https://github.com/VOLTTRON/volttron/releases/tag/8.1 `_ + + +VOLTTRON 8.0 Full Release +========================= + +`https://github.com/VOLTTRON/volttron/releases/tag/8.0.0 `_ + + +VOLTTRON 7.0.1 Update +===================== + +`https://github.com/VOLTTRON/volttron/releases/tag/7.0.1 `_ + + +VOLTTRON 8.0 Release Candidate +============================== + +`https://github.com/VOLTTRON/volttron/releases/tag/8.0rc1 `_ + + +VOLTTRON 7.0 Release +==================== + +`https://github.com/VOLTTRON/volttron/releases/tag/7.0 `_ + + +VOLTTRON 7.0 Release Candidate +============================== + +`https://github.com/VOLTTRON/volttron/releases/tag/7.0rc1 `_ + + +VOLTTRON 6.0 Release +==================== + +`https://github.com/VOLTTRON/volttron/releases/tag/6.0 `_ + + +VOLTTRON 6.0 Release Candidate +============================== + +`https://github.com/VOLTTRON/volttron/releases/tag/6.0rc1 `_ + + +VOLTTRON 5.1.0 Release +====================== + +`https://github.com/VOLTTRON/volttron/releases/tag/5.1.0 `_ + + +VOLTTRON 5.0 Release +==================== + +- Tagging service for attaching metadata to topics for simpler retrieval +- Message bus performance improvement +- Multi-platform publish/subscribe for simpler coordination across platforms +- Drivers contributed back for SEP 2.0 and ChargePoint EV + + +VOLTTRON 4.0 Release +==================== + +- Documentation moved to ReadTheDocs +- VOLTTRON Configuration Wizard +- Configuration store to dynamically configure agents +- Aggregator agent for aggregating topics +- More reliable remote install mechanism +- UI for device configuration +- Automatic registration of VOLTTRON instances with management agent + + +VOLTTRON 3.0 Release +==================== + +- Modularize Data Historian +- Modularize Device Drivers +- Secure and accountable communication using the VIP +- Web Console for Monitoring and Administering VOLTTRON Deployments + + +VOLTTRON 2.0 Release +==================== + +- Advanced Security Features +- Guaranteed resource allocation to agents using execution contracts +- Signing and verification of agent packaging +- Agent mobility +- Admin can send agents to another platform +- Agent can request to move +- Enhanced command framework + + +VOLTTRON 1.0 – 1.2 +================== + +- Agent execution platform +- Message bus +- Modbus and BACnet drivers +- Historian +- Data logger +- Device scheduling +- Device actuation +- Multi-node communication +- Weather service diff --git a/docs/source/volttron-topics/change-log/upgrading-versions.rst b/docs/source/volttron-topics/VOLTTRON-releases/upgrading-versions.rst similarity index 100% rename from docs/source/volttron-topics/change-log/upgrading-versions.rst rename to docs/source/volttron-topics/VOLTTRON-releases/upgrading-versions.rst diff --git a/docs/source/volttron-topics/change-log/scalability/scalability-improvements.rst b/docs/source/volttron-topics/change-log/scalability/scalability-improvements.rst deleted file mode 100644 index 1cd4a6eb99..0000000000 --- a/docs/source/volttron-topics/change-log/scalability/scalability-improvements.rst +++ /dev/null @@ -1,24 +0,0 @@ -Improvements Based on Results -============================= - - -Here is the list of scalability improvements so far: - -Reduced the overhead of the base historian by removing unneeded writes -to the backup db. Significantly improved performance on low end devices. - -Added options to reduce the number of publishes per device per scrape. -Common cases where per point publishes and breadth first topics are not -needed the driver can be configured only publish the depth first “all” -or any combination per device the operator needs. This dramatically -decreases the platform hardware requirements while increasing the number -of devices that can be scraped. - -Added support for staggering device scrapes to reduce CPU load during a -scrape. - -Further ideas: - -| Determine if how we use ZMQ is reducing its efficiency. -| Remove an unneeded index in historian backup db. -| Increase backup db page count. diff --git a/docs/source/volttron-topics/change-log/scalability/scalability.rst b/docs/source/volttron-topics/change-log/scalability/scalability.rst deleted file mode 100644 index c2f2e31de8..0000000000 --- a/docs/source/volttron-topics/change-log/scalability/scalability.rst +++ /dev/null @@ -1,133 +0,0 @@ -.. _Scalability: - -Scalability Setup -~~~~~~~~~~~~~~~~~ - -Core Platform -------------- - -- VIP router - how many messages per second can the router pass -- A single agent can connect and send messages to itself as quickly as - possible -- Repeat but with multiple agents -- Maybe just increase the number of connected but inactive agents to - test lookup times -- Inject faults to test impact of error handling - -- Agents -- How many can be started on a single platform? -- How does it affect memory? -- How is CPU affected? - -Socket types ------------- - -- inproc - lockless, copy-free, fast -- ipc - local, reliable, fast -- tcp - remote, less reliable, possibly much slower -- test with different - - - latency - - throughput - - jitter (packet delay variation) - - error rate - -Subsystems ----------- - -- ping - simple protocol which can provide baseline for other - subsystems -- RPC - requests per second -- Pub/Sub - messages per second -- How does it scale as subscribers are added - -Core Services -------------- - -- historian -- How many records can be processed per second? -- drivers -- BACnet drivers use a virtual BACnet device as a proxy to do device - communication. Currently there is no known upper limit to the number - of devices that can be handled at once. The BACnet proxy opens a - single UDP port to do all communication. In theory the upper limit is - the point when UDP packets begin to be lost due to network - congestion. In practice we have communicated with ~190 devices at - once without issue. -- ModBUS opens up a TCP connection for each communication with a device - and then closes it when finished. This has the potential to hit the - limit for open file descriptors available to the platform driver - process. (Before, each driver would run in a separate process, but - that quickly uses up sockets available to the platform.) To protect - from this the platform driver process raises the total allowed open - sockets to the hard limit. The number of concurrently open sockets is - throttled at 80% of the max sockets. On most Linux systems this is - about 3200. Once that limit is hit additional device communications - will have to wait in line for a socket to become available. - -Tweaking tests --------------- - -- Configure message size -- Perform with/without encryption -- Perform with/without authentication - -Hardware profiling ------------------- - -- Perform tests on hardware of varying resources: Raspberry Pi, NUC, - Desktop, etc. - -Scenarios ---------- - -- One platform controlling large numbers of devices -- One platform managing large numbers of platforms -- Peer communication (Hardware demo type setup) - -Impact on Platform ------------------- - -What is the impact of a large number of devices being scraped on a -platform (and how does it scale with the hardware)? - -- Historians -- At what point are historians unable to keep up with the traffic being - generated? -- Is the bottleneck the sqlite cache or the specific implementation - (SQLite, MySQL) -- Do historian queues grow so large we have a memory problem? -- Large number of devices with small number of points vs small number - of devices with large number of points -- How does a large message flow affect the router? -- Examine effects of the watermark (does increasing help) -- Response time for vctl commands (for instance: status) -- Affect on round trip times (Agent A sends message, Agent B replies, - Agent A receives reply) -- Do messages get lost at some point (EAgain error)? -- What impact does security have? Are things significantly faster in - developer-mode? (Option to turn off encryption, no longer available) - -- | Regulation Agent - | Every 10 minutes there is an action the VOLTTRON Central node determines. - Duty cycle cannot be faster than that but is set to 2 seconds for - simulation. - | Some clients miss duty cycle signal - | Mathematically each node solves ODE. - | Model notes accept switch on/off from master. - | Bad to lose connection to clients in the field - -Chaos router to introduce delays and dropped packets. - -MasterNode needs to have :term:`VIP` address of clients. - -Experiment capture historian - not listening to devices, just capturing -results - -- Go straight to db to see how far behind other historians - - -.. toctree:: - - scalability-improvements - testing-driver-scalability diff --git a/docs/source/volttron-topics/change-log/scalability/testing-driver-scalability.rst b/docs/source/volttron-topics/change-log/scalability/testing-driver-scalability.rst deleted file mode 100644 index 74be46aeef..0000000000 --- a/docs/source/volttron-topics/change-log/scalability/testing-driver-scalability.rst +++ /dev/null @@ -1,214 +0,0 @@ -.. _Testing-Driver-Scalability: - -Scalability Planning -==================== - -Goals ------ - -- Determine the limits of the number of devices that can be interacted - with via a single Volttron platform. -- Determine how scaling out affects the rate at which devices are - scraped. i.e. How long from the first device scrape to the last? -- Determine the effects of socket throttling in the platform driver on - the performance of Modbus device scraping. -- Measure total memory consumption of the Platform Driver Agent at scale. -- Measure how well the base history agent and one or more of the - concrete agents handle a large amount of data. -- Determine the volume of messages that can be achieved on the pubsub - before the platform starts rejecting them. - -Test framework --------------- - -Test Devices -~~~~~~~~~~~~ - -Simple, command line configured virtual devices to test against in both -Modbus and BACnet flavors. Devices should create 10 points to read that -generate either random or easily predictable (but not necessarily -constant) data. Process should be completely self contained. - -Test devices will be run on remote hosts from the Volttron test -deployment. - -Launcher Script -~~~~~~~~~~~~~~~ - -- The script will be configurable as to the number and type of devices - to launch. -- The script will be configurable as to the hosts to launch virtual - devices on. -- The script (probably a fabric script) will push out code for and - launch one or more test devices on one or more machines for the - platform to scrape. -- The script will generate all of the platform driver configuration files - to launch the platform driver. -- The script may launch the platform driver. -- The script may launch any other agents used to measure performance. - -Shutdown Script -~~~~~~~~~~~~~~~ - -- The script (probably the same fabric script run with different - options) will shutdown all virtual drivers on the network. -- The script may shutdown the platform driver. -- The script may shutdown any related agents. - -Performance Metrics Agent -~~~~~~~~~~~~~~~~~~~~~~~~~ - -This agent will track the publishes by the different drivers and -generate data in some form to indicate: - -- Total time for all devices to be scraped -- Any devices that were not successfully scraped. -- Performance of the message bus. - -Additional Benefits -~~~~~~~~~~~~~~~~~~~ - -Most parts of a test bed run should be configurable. If a user wanted to -verify that the Platform Driver worked, for instance, they could run the -test bed with only a few virtual device to confirm that the platform is -working correctly. - -Running a simple test -~~~~~~~~~~~~~~~~~~~~~ - -| You will need 2 open terminals to run this test. (3 if you want to run - the platform in it's own terminal) -| Checkout the feature/scalability branch. - -Start the platform. - -Go to the volttron/scripts/scalability-testing directory in two -different terminals. (Both with the environment activated) - -In one terminal run: - -:: - - python config_builder.py --count=1500 --scalability-test --scalability-test-iterations=6 fake fake18.csv localhost - -Change the path to fake.csv as needed. - -(Optional) After it finishes run: - -:: - - ./launch_fake_historian.sh - -to start the null historian. - -In a separate terminal run: - -:: - - ./launch_scalability_drivers.sh - -to start the scalability test. - -This will emulate the scraping of 1500 devices with 18 points each 6 -times, log the timing, and quit. - -Redirecting the driver log output to a file can help improve -performance. Testing should be done with and without the null historian. - -Currently only the depth first all is published by drivers in this -branch. Uncomment the other publishes in driver.py to test out full -publishing. fake.csv has 18 points. - -Optionally you can run two listener agents from the volttron/scripts -directory in two more terminals with the command: - -:: - - ./launch_listener.sh - -and rerun the test to see the how it changes the performance. - -Real Driver Benchmarking ------------------------- - -Scalability testing using actual MODBUS or BACnet drivers can be done -using the virtual device applications in the -scripts/scalability-testing/virtual-drivers/ directory. The -configuration of the platform driver and launching of these virtual -devices on a target machine can be done automatically with fabric. - -Setup -~~~~~ - -This requires two computers to run: One for the VOLTTRON platform to run -the tests on ("the platform") and a target machine to host the virtual -devices ("the target"). - -Target setup -^^^^^^^^^^^^ - -The target machine must have the VOLTTRON source with the -feature/scalability branch checked out and bootstrapped. Make a note of -the directory of the VOLTTRON code. - -Platform setup -^^^^^^^^^^^^^^ - -With the VOLTTRON environment activated install fabric. - -:: - - pip install fabric - -Edit the file scripts/scalability-testing/test\_settings.py as needed. - -- virtual\_device\_host (string) - Login name and IP address of the - target machine. This is used to remotely start and stop virtual - devices via ssh. `"volttron@10.0.0.1 `__" - -- device\_types - map of driver types to tuple of the device count and - registry config to use for the virtual devices. Valid device types - are "bacnet" and "modbus". - -- volttron\_install - location of volttron code on the target. - -To configure the driver on the platform and launch the virtual devices -on the target run - -:: - - fab deploy_virtual_devices - -When prompted enter the password for the target machine. Upon completion -virtual devices will be running on the target and configuration files -written for the platform driver. - -Launch Test -^^^^^^^^^^^ - -If your test includes virtual BACnet devices be sure to configure and -launch the BACnet Proxy before launching the scalability driver test. - -(Optional) - -:: - - ./launch_fake_historian.sh - -to start the null historian. - -In a separate terminal run: - -:: - - ./launch_scalability_drivers.sh - -to start the scalability test. - -To stop the virtual devices run - -:: - - fab stop_virtual_devices - -and enter the user password when prompted. diff --git a/docs/source/volttron-topics/change-log/version-history.rst b/docs/source/volttron-topics/change-log/version-history.rst deleted file mode 100644 index 1671bfc04d..0000000000 --- a/docs/source/volttron-topics/change-log/version-history.rst +++ /dev/null @@ -1,164 +0,0 @@ -.. _Version-History: - -=============== -Version History -=============== - -VOLTTRON 1.0 – 1.2 -================== - -- Agent execution platform -- Message bus -- Modbus and BACnet drivers -- Historian -- Data logger -- Device scheduling -- Device actuation -- Multi-node communication -- Weather service - - -VOLTTRON 2.0 -============ - -- Advanced Security Features -- Guaranteed resource allocation to agents using execution contracts -- Signing and verification of agent packaging -- Agent mobility -- Admin can send agents to another platform -- Agent can request to move -- Enhanced command framework - - -VOLTTRON 3.0 -============ - -- Modularize Data Historian -- Modularize Device Drivers -- Secure and accountable communication using the VIP -- Web Console for Monitoring and Administering VOLTTRON Deployments - - -VOLTTRON 4.0 -============ - -- Documentation moved to ReadTheDocs -- VOLTTRON Configuration Wizard -- Configuration store to dynamically configure agents -- Aggregator agent for aggregating topics -- More reliable remote install mechanism -- UI for device configuration -- Automatic registration of VOLTTRON instances with management agent - - -VOLTTRON 5.0 -============ - -- Tagging service for attaching metadata to topics for simpler retrieval -- Message bus performance improvement -- Multi-platform publish/subscribe for simpler coordination across platforms -- Drivers contributed back for SEP 2.0 and ChargePoint EV - - -VOLTTRON 6.0 -============ - -- Maintained backward compatibility with communication between zmq and rmq deployments. -- Added DarkSky Weather Agent -- Web Based Additions -- Added CSR support for multiplatform communication -- Added SSL support to the platform for secure communication -- Backported SSL support to zmq based deployments. -- Upgraded VC to use the platform login. -- Added docker support to the test environment for easier Rabbitmq testing. -- Updated volttron-config (vcfg) to support both RabbitMQ and ZMQ including https based instances. -- Added test support for RabbitMQ installations of all core agents. -- Added multiplatform (zmq and rmq based platform) testing. -- Integrated RabbitMQ documentation into the core documentation. - - -VOLTTRON 7.0rc1 -=============== - - -Python3 Upgrade ---------------- - -- Update libraries to appropriate and compatible versions -- String handling efficiency -- Encode/Decode of strings has been simplified and centralized -- Added additional test cases for frame serialization in ZMQ -- Syntax updates such difference in handling exceptions, dictionaries, sorting lists, pytest markers etc. -- Made bootstrap process simpler -- Resolved gevent monkey patch issues when using third party libraries - - -RabbitMQ Message Bus --------------------- - -- Client code for integrating non-VOLTTRON applications with the message bus - available at: https://github.com/VOLTTRON/external-clients-for-rabbitmq -- Includes support for MQTT, non-VOLTTRON Python, and Java-based RabbitMQ - clients - - -Config store secured --------------------- - -- Agents can prevent other agents from modifying their configuration store entry - - -Known Issues which will be dealt with for the final release: ------------------------------------------------------------- - -- Python 3.7 has conflicts with some libraries such as gevent -- The VOLTTRON Central agent is not fully integrated into Python3 -- CFFI library has conflicts on the Raspian OS which interferes with bootstrapping - - -VOLTTRON 7.0 Full Release -========================= - -This is a full release of the 7.0 version of VOLTTRON which has been refactored to work with Python3. This release -incorporates community feedback from the release candidate as well as new contributions and features. -Major new features and highlights since the release candidate include: - -* Added secure agent user feature which allows agents to be launched as a user separate from the platform. This - protects the platform against malformed or malicious agents accessing platform level files -* Added a driver to interface with the Ecobee smart thermostat and make data available to agents on the platform -* Updated VOLTTRON Central UI to work with Python3 -* Added web support to authenticate remote VOLTTRON ZMQ message bus-based connections -* Updated ZMQ-based multiplatform RPC with Python 3 -* To reduce installation size and complexity, fewer services are installed by default -* MasterDriver dependencies are not installed by default during bootstrap. To use MasterDriver, please use the - following command: - - .. code-block:: bash - - python3 bootstrap.py --driver - -* Web dependencies are not installed by default during bootstrap. To use the MasterWeb service, please use the - following command: - - .. code-block:: bash - - python3 bootstrap.py --web - -* Added initial version of test cases for `volttron-cfg` (`vcfg`) utility -* On all arm-based systems, `libffi` is now a required dependency, this is reflected in the installation instructions -* On arm-based systems, Raspbian >= 10 or Ubuntu >= 18.04 is required -* Updated examples and several contributed features to Python 3 -* Inclusion of docker in test handling for databases -* A new `/gs` endpoint to access platform services without using Volttron Central through Json-RPC -* A new SCPAgent to transfer files between two remote systems - -Known Issues ------------- - -* Continued documentation updates to ensure correctness -* Rainforest Eagle driver is not yet upgraded to Python3 -* A bug in the Modbus TK library prevents creating connections from 2 different masters to a single slave. -* BACnet Proxy Agent and BACnet auto configuration scripts require the version of BACPypes installed in the virtual - environment of VOLTTRON to be version 0.16.7. We have pinned it to version 0.16.7 since it does not work properly in - later versions of BACPypes. -* VOLTTRON 7.0 code base is not fully tested in Ubuntu 20.04 LTS so issues with this combination have not been addressed From 066444308b9d8c143dea151c5f26e73cfd1f6c09 Mon Sep 17 00:00:00 2001 From: sgilbride Date: Thu, 16 Sep 2021 15:33:07 -0700 Subject: [PATCH 24/36] Added scalability files to deploying VOLTTRON. Minor spacing changes. --- .../scalability/scalability-improvements.rst | 24 ++ .../scalability/scalability.rst | 133 +++++++++++ .../testing-driver-scalability.rst | 214 ++++++++++++++++++ docs/source/index.rst | 6 +- .../VOLTTRON-releases/release-history.rst | 1 + 5 files changed, 376 insertions(+), 2 deletions(-) create mode 100644 docs/source/deploying-volttron/scalability/scalability-improvements.rst create mode 100644 docs/source/deploying-volttron/scalability/scalability.rst create mode 100644 docs/source/deploying-volttron/scalability/testing-driver-scalability.rst diff --git a/docs/source/deploying-volttron/scalability/scalability-improvements.rst b/docs/source/deploying-volttron/scalability/scalability-improvements.rst new file mode 100644 index 0000000000..1cd4a6eb99 --- /dev/null +++ b/docs/source/deploying-volttron/scalability/scalability-improvements.rst @@ -0,0 +1,24 @@ +Improvements Based on Results +============================= + + +Here is the list of scalability improvements so far: + +Reduced the overhead of the base historian by removing unneeded writes +to the backup db. Significantly improved performance on low end devices. + +Added options to reduce the number of publishes per device per scrape. +Common cases where per point publishes and breadth first topics are not +needed the driver can be configured only publish the depth first “all” +or any combination per device the operator needs. This dramatically +decreases the platform hardware requirements while increasing the number +of devices that can be scraped. + +Added support for staggering device scrapes to reduce CPU load during a +scrape. + +Further ideas: + +| Determine if how we use ZMQ is reducing its efficiency. +| Remove an unneeded index in historian backup db. +| Increase backup db page count. diff --git a/docs/source/deploying-volttron/scalability/scalability.rst b/docs/source/deploying-volttron/scalability/scalability.rst new file mode 100644 index 0000000000..c2f2e31de8 --- /dev/null +++ b/docs/source/deploying-volttron/scalability/scalability.rst @@ -0,0 +1,133 @@ +.. _Scalability: + +Scalability Setup +~~~~~~~~~~~~~~~~~ + +Core Platform +------------- + +- VIP router - how many messages per second can the router pass +- A single agent can connect and send messages to itself as quickly as + possible +- Repeat but with multiple agents +- Maybe just increase the number of connected but inactive agents to + test lookup times +- Inject faults to test impact of error handling + +- Agents +- How many can be started on a single platform? +- How does it affect memory? +- How is CPU affected? + +Socket types +------------ + +- inproc - lockless, copy-free, fast +- ipc - local, reliable, fast +- tcp - remote, less reliable, possibly much slower +- test with different + + - latency + - throughput + - jitter (packet delay variation) + - error rate + +Subsystems +---------- + +- ping - simple protocol which can provide baseline for other + subsystems +- RPC - requests per second +- Pub/Sub - messages per second +- How does it scale as subscribers are added + +Core Services +------------- + +- historian +- How many records can be processed per second? +- drivers +- BACnet drivers use a virtual BACnet device as a proxy to do device + communication. Currently there is no known upper limit to the number + of devices that can be handled at once. The BACnet proxy opens a + single UDP port to do all communication. In theory the upper limit is + the point when UDP packets begin to be lost due to network + congestion. In practice we have communicated with ~190 devices at + once without issue. +- ModBUS opens up a TCP connection for each communication with a device + and then closes it when finished. This has the potential to hit the + limit for open file descriptors available to the platform driver + process. (Before, each driver would run in a separate process, but + that quickly uses up sockets available to the platform.) To protect + from this the platform driver process raises the total allowed open + sockets to the hard limit. The number of concurrently open sockets is + throttled at 80% of the max sockets. On most Linux systems this is + about 3200. Once that limit is hit additional device communications + will have to wait in line for a socket to become available. + +Tweaking tests +-------------- + +- Configure message size +- Perform with/without encryption +- Perform with/without authentication + +Hardware profiling +------------------ + +- Perform tests on hardware of varying resources: Raspberry Pi, NUC, + Desktop, etc. + +Scenarios +--------- + +- One platform controlling large numbers of devices +- One platform managing large numbers of platforms +- Peer communication (Hardware demo type setup) + +Impact on Platform +------------------ + +What is the impact of a large number of devices being scraped on a +platform (and how does it scale with the hardware)? + +- Historians +- At what point are historians unable to keep up with the traffic being + generated? +- Is the bottleneck the sqlite cache or the specific implementation + (SQLite, MySQL) +- Do historian queues grow so large we have a memory problem? +- Large number of devices with small number of points vs small number + of devices with large number of points +- How does a large message flow affect the router? +- Examine effects of the watermark (does increasing help) +- Response time for vctl commands (for instance: status) +- Affect on round trip times (Agent A sends message, Agent B replies, + Agent A receives reply) +- Do messages get lost at some point (EAgain error)? +- What impact does security have? Are things significantly faster in + developer-mode? (Option to turn off encryption, no longer available) + +- | Regulation Agent + | Every 10 minutes there is an action the VOLTTRON Central node determines. + Duty cycle cannot be faster than that but is set to 2 seconds for + simulation. + | Some clients miss duty cycle signal + | Mathematically each node solves ODE. + | Model notes accept switch on/off from master. + | Bad to lose connection to clients in the field + +Chaos router to introduce delays and dropped packets. + +MasterNode needs to have :term:`VIP` address of clients. + +Experiment capture historian - not listening to devices, just capturing +results + +- Go straight to db to see how far behind other historians + + +.. toctree:: + + scalability-improvements + testing-driver-scalability diff --git a/docs/source/deploying-volttron/scalability/testing-driver-scalability.rst b/docs/source/deploying-volttron/scalability/testing-driver-scalability.rst new file mode 100644 index 0000000000..74be46aeef --- /dev/null +++ b/docs/source/deploying-volttron/scalability/testing-driver-scalability.rst @@ -0,0 +1,214 @@ +.. _Testing-Driver-Scalability: + +Scalability Planning +==================== + +Goals +----- + +- Determine the limits of the number of devices that can be interacted + with via a single Volttron platform. +- Determine how scaling out affects the rate at which devices are + scraped. i.e. How long from the first device scrape to the last? +- Determine the effects of socket throttling in the platform driver on + the performance of Modbus device scraping. +- Measure total memory consumption of the Platform Driver Agent at scale. +- Measure how well the base history agent and one or more of the + concrete agents handle a large amount of data. +- Determine the volume of messages that can be achieved on the pubsub + before the platform starts rejecting them. + +Test framework +-------------- + +Test Devices +~~~~~~~~~~~~ + +Simple, command line configured virtual devices to test against in both +Modbus and BACnet flavors. Devices should create 10 points to read that +generate either random or easily predictable (but not necessarily +constant) data. Process should be completely self contained. + +Test devices will be run on remote hosts from the Volttron test +deployment. + +Launcher Script +~~~~~~~~~~~~~~~ + +- The script will be configurable as to the number and type of devices + to launch. +- The script will be configurable as to the hosts to launch virtual + devices on. +- The script (probably a fabric script) will push out code for and + launch one or more test devices on one or more machines for the + platform to scrape. +- The script will generate all of the platform driver configuration files + to launch the platform driver. +- The script may launch the platform driver. +- The script may launch any other agents used to measure performance. + +Shutdown Script +~~~~~~~~~~~~~~~ + +- The script (probably the same fabric script run with different + options) will shutdown all virtual drivers on the network. +- The script may shutdown the platform driver. +- The script may shutdown any related agents. + +Performance Metrics Agent +~~~~~~~~~~~~~~~~~~~~~~~~~ + +This agent will track the publishes by the different drivers and +generate data in some form to indicate: + +- Total time for all devices to be scraped +- Any devices that were not successfully scraped. +- Performance of the message bus. + +Additional Benefits +~~~~~~~~~~~~~~~~~~~ + +Most parts of a test bed run should be configurable. If a user wanted to +verify that the Platform Driver worked, for instance, they could run the +test bed with only a few virtual device to confirm that the platform is +working correctly. + +Running a simple test +~~~~~~~~~~~~~~~~~~~~~ + +| You will need 2 open terminals to run this test. (3 if you want to run + the platform in it's own terminal) +| Checkout the feature/scalability branch. + +Start the platform. + +Go to the volttron/scripts/scalability-testing directory in two +different terminals. (Both with the environment activated) + +In one terminal run: + +:: + + python config_builder.py --count=1500 --scalability-test --scalability-test-iterations=6 fake fake18.csv localhost + +Change the path to fake.csv as needed. + +(Optional) After it finishes run: + +:: + + ./launch_fake_historian.sh + +to start the null historian. + +In a separate terminal run: + +:: + + ./launch_scalability_drivers.sh + +to start the scalability test. + +This will emulate the scraping of 1500 devices with 18 points each 6 +times, log the timing, and quit. + +Redirecting the driver log output to a file can help improve +performance. Testing should be done with and without the null historian. + +Currently only the depth first all is published by drivers in this +branch. Uncomment the other publishes in driver.py to test out full +publishing. fake.csv has 18 points. + +Optionally you can run two listener agents from the volttron/scripts +directory in two more terminals with the command: + +:: + + ./launch_listener.sh + +and rerun the test to see the how it changes the performance. + +Real Driver Benchmarking +------------------------ + +Scalability testing using actual MODBUS or BACnet drivers can be done +using the virtual device applications in the +scripts/scalability-testing/virtual-drivers/ directory. The +configuration of the platform driver and launching of these virtual +devices on a target machine can be done automatically with fabric. + +Setup +~~~~~ + +This requires two computers to run: One for the VOLTTRON platform to run +the tests on ("the platform") and a target machine to host the virtual +devices ("the target"). + +Target setup +^^^^^^^^^^^^ + +The target machine must have the VOLTTRON source with the +feature/scalability branch checked out and bootstrapped. Make a note of +the directory of the VOLTTRON code. + +Platform setup +^^^^^^^^^^^^^^ + +With the VOLTTRON environment activated install fabric. + +:: + + pip install fabric + +Edit the file scripts/scalability-testing/test\_settings.py as needed. + +- virtual\_device\_host (string) - Login name and IP address of the + target machine. This is used to remotely start and stop virtual + devices via ssh. `"volttron@10.0.0.1 `__" + +- device\_types - map of driver types to tuple of the device count and + registry config to use for the virtual devices. Valid device types + are "bacnet" and "modbus". + +- volttron\_install - location of volttron code on the target. + +To configure the driver on the platform and launch the virtual devices +on the target run + +:: + + fab deploy_virtual_devices + +When prompted enter the password for the target machine. Upon completion +virtual devices will be running on the target and configuration files +written for the platform driver. + +Launch Test +^^^^^^^^^^^ + +If your test includes virtual BACnet devices be sure to configure and +launch the BACnet Proxy before launching the scalability driver test. + +(Optional) + +:: + + ./launch_fake_historian.sh + +to start the null historian. + +In a separate terminal run: + +:: + + ./launch_scalability_drivers.sh + +to start the scalability test. + +To stop the virtual devices run + +:: + + fab stop_virtual_devices + +and enter the user password when prompted. diff --git a/docs/source/index.rst b/docs/source/index.rst index 2a5a74df9e..a5a921eee0 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -87,6 +87,7 @@ at our bi-weekly office-hours and on Slack. To be invited to office-hours or sla deploying-volttron/secure-deployment-considerations deploying-volttron/linux-system-hardening deploying-volttron/recipe-deployment + deploying-volttron/scalability/scalability .. toctree:: @@ -137,6 +138,7 @@ at our bi-weekly office-hours and on Slack. To be invited to office-hours or sla platform-features/config-store/configuration-store platform-features/security/volttron-security + .. toctree:: :caption: VOLTTRON Core Service Agents :hidden: @@ -146,6 +148,7 @@ at our bi-weekly office-hours and on Slack. To be invited to office-hours or sla volttron-api/services/*/modules + .. toctree:: :caption: VOLTTRON Core Operations Agents :hidden: @@ -155,6 +158,7 @@ at our bi-weekly office-hours and on Slack. To be invited to office-hours or sla volttron-api/ops/*/modules + .. toctree:: :caption: VOLTTRON Topics :hidden: @@ -166,8 +170,6 @@ at our bi-weekly office-hours and on Slack. To be invited to office-hours or sla volttron-topics/volttron-applications/index - - Indices and tables ================== diff --git a/docs/source/volttron-topics/VOLTTRON-releases/release-history.rst b/docs/source/volttron-topics/VOLTTRON-releases/release-history.rst index 444c8effb3..4b7b750090 100644 --- a/docs/source/volttron-topics/VOLTTRON-releases/release-history.rst +++ b/docs/source/volttron-topics/VOLTTRON-releases/release-history.rst @@ -7,6 +7,7 @@ Release History VOLTTRON Release Documentation for version 5.1.0 and above is found on GitHub. `https://github.com/VOLTTRON/volttron/releases `_ + VOLTTRON 8.1.1 Maintenance Release ================================== From 31ed42b977706ffb0354b8dd0b95faa6a36c5f6d Mon Sep 17 00:00:00 2001 From: Craig <3979063+craig8@users.noreply.github.com> Date: Fri, 17 Sep 2021 09:22:02 -0700 Subject: [PATCH 25/36] Update requirements.py Fixed missing comma #2793 --- requirements.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.py b/requirements.py index 7b13f730a1..4ee5f6cbca 100644 --- a/requirements.py +++ b/requirements.py @@ -55,7 +55,7 @@ 'pyserial==3.5'], 'influxdb': ['influxdb==5.3.1'], 'market': ['numpy==1.19.5', 'transitions==0.8.8'], - 'mongo': ['bson==0.5.7pymongo==3.7.2'], + 'mongo': ['bson==0.5.7,pymongo==3.7.2'], 'mysql': ['mysql-connector-python==8.0.26'], 'pandas': ['numpy==1.19.5', 'pandas==1.1.5'], 'postgres': ['psycopg2-binary==2.8.6'], From c9c4b2cf8c500396aa141d3be210dbdfb4c7337b Mon Sep 17 00:00:00 2001 From: Craig <3979063+craig8@users.noreply.github.com> Date: Fri, 17 Sep 2021 09:29:47 -0700 Subject: [PATCH 26/36] Update requirements.py missing string delimiters --- requirements.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.py b/requirements.py index 4ee5f6cbca..b154304461 100644 --- a/requirements.py +++ b/requirements.py @@ -55,7 +55,7 @@ 'pyserial==3.5'], 'influxdb': ['influxdb==5.3.1'], 'market': ['numpy==1.19.5', 'transitions==0.8.8'], - 'mongo': ['bson==0.5.7,pymongo==3.7.2'], + 'mongo': ['bson==0.5.7', 'pymongo==3.7.2'], 'mysql': ['mysql-connector-python==8.0.26'], 'pandas': ['numpy==1.19.5', 'pandas==1.1.5'], 'postgres': ['psycopg2-binary==2.8.6'], From d90101ff9bfff319da488821776c56f3e009b16e Mon Sep 17 00:00:00 2001 From: sgilbride Date: Fri, 17 Sep 2021 10:15:22 -0700 Subject: [PATCH 27/36] Modified Releases Sidebar menu header. Removed Scalability from menu. Needs updating. --- docs/source/index.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/source/index.rst b/docs/source/index.rst index a5a921eee0..4b196a82c0 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -87,7 +87,6 @@ at our bi-weekly office-hours and on Slack. To be invited to office-hours or sla deploying-volttron/secure-deployment-considerations deploying-volttron/linux-system-hardening deploying-volttron/recipe-deployment - deploying-volttron/scalability/scalability .. toctree:: @@ -165,7 +164,7 @@ at our bi-weekly office-hours and on Slack. To be invited to office-hours or sla :titlesonly: :maxdepth: 1 - volttron-topics/VOLTTRON-releases/index + Releases volttron-topics/troubleshooting/index volttron-topics/volttron-applications/index From e339e762c46f3308f33308a808d1d2469f41f896 Mon Sep 17 00:00:00 2001 From: Craig <3979063+craig8@users.noreply.github.com> Date: Fri, 17 Sep 2021 12:54:45 -0700 Subject: [PATCH 28/36] Update index.rst --- docs/source/volttron-topics/VOLTTRON-releases/index.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/source/volttron-topics/VOLTTRON-releases/index.rst b/docs/source/volttron-topics/VOLTTRON-releases/index.rst index 68a914e64c..b762c36d03 100644 --- a/docs/source/volttron-topics/VOLTTRON-releases/index.rst +++ b/docs/source/volttron-topics/VOLTTRON-releases/index.rst @@ -4,8 +4,7 @@ VOLTTRON Releases ========== -This section includes individual documents describing important changes to platform components, such as the RabbitMQ -message bus implementation. For information on specific changes, please refer to the corresponding document. +This section includes individual documents describing important changes to platform components. For information on specific release, please refer to the corresponding document. .. toctree:: From 5d9df2693ce73132b0cae5a3d65f10e17775d004 Mon Sep 17 00:00:00 2001 From: schandrika Date: Mon, 20 Sep 2021 14:12:34 -0700 Subject: [PATCH 29/36] fix for issue#2796 (jira #533) remove duplicate entry in requirements.py. Remove hard coded install of wheel from bootstrap.py --- bootstrap.py | 2 +- requirements.py | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/bootstrap.py b/bootstrap.py index e7e94ead13..836f44c2cf 100644 --- a/bootstrap.py +++ b/bootstrap.py @@ -133,7 +133,7 @@ def update(operation, verbose=None, upgrade=False, offline=False, optional_requi # We must install wheel first to eliminate a bunch of scary looking # errors at first install. # TODO Look towards fixing the packaging so that it works with 0.31 - pip('install', ['wheel==0.30'], verbose, True, offline=offline) + # option_requirements contains wheel as first entry # Build option_requirements separately to pass install options build_option = '--build-option' if wheeling else '--install-option' diff --git a/requirements.py b/requirements.py index 7b13f730a1..c684ea7ec7 100644 --- a/requirements.py +++ b/requirements.py @@ -83,12 +83,10 @@ 'python-dateutil==2.8.2', 'pytz==2021.1', 'PyYAML==5.4.1', - 'pyzmq==22.2.1', 'setuptools>=40.0.0', 'tzlocal==2.1', 'pyOpenSSL==19.0.0', 'cryptography==2.3', - 'watchdog-gevent==0.1.1', - 'wheel==0.30'] + 'watchdog-gevent==0.1.1'] -option_requirements = [('pyzmq==22.2.1', ['--zmq=bundled'])] +option_requirements = [('wheel==0.30', []), ('pyzmq==22.2.1', ['--zmq=bundled'])] From c408c8fda20cc7b811ea75c0e2642fe2bad80695 Mon Sep 17 00:00:00 2001 From: schandrika Date: Tue, 21 Sep 2021 15:55:56 -0700 Subject: [PATCH 30/36] fix for issue #2802 changed path of backup.sqlite --- volttron/platform/agent/base_historian.py | 14 ++++---- .../services/historian/test_base_historian.py | 34 +++++++++++++------ 2 files changed, 30 insertions(+), 18 deletions(-) diff --git a/volttron/platform/agent/base_historian.py b/volttron/platform/agent/base_historian.py index a56d2a7497..b540bc52a9 100644 --- a/volttron/platform/agent/base_historian.py +++ b/volttron/platform/agent/base_historian.py @@ -1745,13 +1745,13 @@ def close(self): def _setupdb(self, check_same_thread): """ Creates a backup database for the historian if doesn't exist.""" - _log.debug("Setting up backup DB.") - if utils.is_secure_mode(): - # we want to create it in the agent-data directory since agent will not have write access to any other - # directory in secure mode - backup_db = os.path.join(os.getcwd(), os.path.basename(os.getcwd()) + ".agent-data", 'backup.sqlite') - else: - backup_db = 'backup.sqlite' + _log.debug(f"Setting up backup DB. {os.getcwd()}") + # we want to create it in the agent-data directory since agent will not have write access to any other + # directory in secure mode + # TODO - revisit logic to get agent-data directory. Refer to aip.get_agent_data() + # Also take into account dynamic agents - especially for testing + backup_db = os.path.join(os.getcwd(), os.path.basename(os.getcwd()) + ".agent-data", 'backup.sqlite') + _log.info(f"Creating backup db at {backup_db}") self._connection = sqlite3.connect( backup_db, diff --git a/volttrontesting/services/historian/test_base_historian.py b/volttrontesting/services/historian/test_base_historian.py index 51a578e09b..4771e257a5 100644 --- a/volttrontesting/services/historian/test_base_historian.py +++ b/volttrontesting/services/historian/test_base_historian.py @@ -60,6 +60,13 @@ class Historian(BaseHistorian): + + def __init__(self, **kwargs): + self.agent_data_dir = os.path.join(os.getcwd(), os.path.basename(os.getcwd()) + ".agent-data") + os.makedirs(self.agent_data_dir, exist_ok=True) + self.backup_sqlite = Path(self.agent_data_dir).joinpath("backup.sqlite") + super(Historian, self).__init__(**kwargs) + def publish_to_historian(self, _): pass @@ -71,9 +78,9 @@ def query_historian(self, **kwargs): def remove_backup_cache_db(self): try: - abspath = Path('backup.sqlite').absolute() - if abspath.exists(): - os.remove(str(abspath)) + print(f"Removing backup cache {self.backup_sqlite}") + os.remove(str(self.backup_sqlite)) + os.remove(str(self.agent_data_dir)) except: print("Don't throw here if os.remove fails...") @@ -89,6 +96,9 @@ def listener(peer, sender, bus, topic, headers, message): class BasicHistorian(BaseHistorian): def __init__(self, **kwargs): + self.agent_data_dir = os.path.join(os.getcwd(), os.path.basename(os.getcwd()) + ".agent-data") + os.makedirs(self.agent_data_dir, exist_ok=True) + self.backup_sqlite = Path(self.agent_data_dir).joinpath("backup.sqlite") super(BasicHistorian, self).__init__(**kwargs) self.publish_fail = False self.publish_sleep = 0 @@ -108,9 +118,9 @@ def reset(self): def remove_backup_cache_db(self): try: - abspath = Path('backup.sqlite').absolute() - if abspath.exists(): - os.remove(str(abspath)) + print(f"Removing backup cache {self.backup_sqlite}") + os.remove(str(self.backup_sqlite)) + os.remove(str(self.agent_data_dir)) except: print("Don't throw here if os.remove fails...") @@ -271,7 +281,6 @@ def test_time_tolerance_check(request, volttron_instance, client_agent): enable_store=True) assert "could not convert string to float: 'invalid'" in str(e.value) print(e) - historian = volttron_instance.build_agent(agent_class=BasicHistorian, identity=identity, submit_size_limit=5, @@ -283,7 +292,7 @@ def test_time_tolerance_check(request, volttron_instance, client_agent): DEVICES_ALL_TOPIC = "devices/Building/LAB/Device/all" gevent.sleep(5) # wait for historian to be fully up historian.publish_sleep = 0 - db_file = Path('backup.sqlite').absolute() + db_file = historian.backup_sqlite assert db_file.exists() db_connection = sqlite3.connect(str(db_file)) c = db_connection.cursor() @@ -524,6 +533,9 @@ def test_health_stuff(request, volttron_instance, client_agent): class FailureHistorian(BaseHistorian): def __init__(self, **kwargs): + self.agent_data_dir = os.path.join(os.getcwd(), os.path.basename(os.getcwd()) + ".agent-data") + os.makedirs(self.agent_data_dir, exist_ok=True) + self.backup_sqlite = Path(self.agent_data_dir).joinpath("backup.sqlite") super(FailureHistorian, self).__init__(**kwargs) self.publish_fail = False self.setup_fail = False @@ -557,9 +569,9 @@ def reset(self): def remove_backup_cache_db(self): try: - abspath = Path('backup.sqlite').absolute() - if abspath.exists(): - os.remove(str(abspath)) + print(f"Removing backup cache {self.backup_sqlite}") + os.remove(str(self.backup_sqlite)) + os.remove(str(self.agent_data_dir)) except: print("Don't throw here if os.remove fails...") From 807cfc7ca30b1509f0276a14cd5923b748016316 Mon Sep 17 00:00:00 2001 From: schandrika Date: Tue, 21 Sep 2021 16:08:27 -0700 Subject: [PATCH 31/36] fix for issue #2803 - Added volttron-upgrade command to update both auth and move backup.sqlite files Added relevant documentation --- README.md | 38 +++++- .../VOLTTRON-releases/upgrading-versions.rst | 32 ++++- setup.py | 3 +- volttron/platform/aip.py | 8 +- volttron/platform/upgrade/__init__.py | 0 .../platform/upgrade/move_sqlite_files.py | 117 ++++++++++++++++++ .../{ => upgrade}/update_auth_file.py | 2 - volttron/platform/upgrade/upgrade_volttron.py | 68 ++++++++++ 8 files changed, 251 insertions(+), 17 deletions(-) create mode 100644 volttron/platform/upgrade/__init__.py create mode 100644 volttron/platform/upgrade/move_sqlite_files.py rename volttron/platform/{ => upgrade}/update_auth_file.py (98%) create mode 100644 volttron/platform/upgrade/upgrade_volttron.py diff --git a/README.md b/README.md index 11682604f8..0f475a3bbc 100644 --- a/README.md +++ b/README.md @@ -10,12 +10,38 @@ with that data. ## Upgrading to VOLTTRON 8.x -VOLTTRON 8 and above introduces dynamic RPC authorization, which requires a modification to the auth file. -If you have a pre-existing instance of VOLTTRON running on an older version, the auth file will need to be updated. -To begin the upgrade process, activate the volttron environment, and run ```python bootstrap.py --force```. If you -have any additional bootstrap options that you need (rabbitmq, web, drivers, etc.) include these in the above command. -After the bootstrap process is completed, run ```volttron-update-auth``` to update the auth file. - +VOLTTRON 8 introduces three changes that require an explict upgrade step when upgrading from a earlier VOLTTRON version + + 1. Dynamic RPC authorization feature - This requires a modification to the auth file. If you have a pre-existing + instance of VOLTTRON running on an older version, the auth file will need to be updated. + 2. Historian agents now store the cache database (backup.sqlite file) in + /agents///.agent-data directory instead of + /agents// directory. In future all core agents will write data only + to the .agent-data subdirectory. This is because vctl install --force backs up and restores + only the contents of this directory. + 3. SQLHistorians (historian version 4.0.0 and above) now use a new database schema where metadata is stored in + topics table instead of separate metadata table. SQLHistorians with version >= 4.0.0 can work with existing + database with older schema however the historian agent code should be upgraded to newer version (>=4.0.0) to run + with VOLTTRON 8 core. + +To begin the upgrade process, activate the volttron environment, and run ```python bootstrap.py --force```. If you have +any additional bootstrap options that you need (rabbitmq, web, drivers, etc.) include these in the above command. + +After the bootstrap process is completed, run ```volttron-upgrade``` to update the auth file and move historian +cache files into agent-data directory. Note that the upgrade script will only move the backup.sqlite file and will not +move sqlite historian's db file if they are within the install directory. If using a SQLite historian, please backup +the database file of sqlite historian before upgrading to the latest historian version. + +Once the volttron-upgrade script is complete, you can do a vctl install --force command to upgrade to the latest +historian version. vctl install --force will backup the cache in .agent-data folder, install the latest +version of the historian and restore the contents of .agent-data folder. + + +### Upgrading aggregate historians + +VOLTTRON 8 also comes with updated SQL aggregate historian schema. However, there is no automated upgrade path for +aggregate historian. To upgrade an existing aggregate historian please refer to the CHANGELOG.md within +SQLAggregateHistorian source directory ## Features diff --git a/docs/source/volttron-topics/VOLTTRON-releases/upgrading-versions.rst b/docs/source/volttron-topics/VOLTTRON-releases/upgrading-versions.rst index ef30037c54..602f2278df 100644 --- a/docs/source/volttron-topics/VOLTTRON-releases/upgrading-versions.rst +++ b/docs/source/volttron-topics/VOLTTRON-releases/upgrading-versions.rst @@ -11,8 +11,20 @@ your existing deployment to the latest version. VOLTTRON 8 ========== -VOLTTRON 8 and above introduces dynamic RPC authorization, which requires a modification to the auth file. -If you have a pre-existing instance of VOLTTRON running on an older version, the auth file will need to be updated. +VOLTTRON 8 introduces three changes that require an explict upgrade step when upgrading from a earlier VOLTTRON version + + 1. Dynamic RPC authorization feature - This requires a modification to the auth file. If you have a pre-existing + instance of VOLTTRON running on an older version, the auth file will need to be updated. + 2. Historian agents now store the cache database (backup.sqlite file) in + /agents///.agent-data directory instead of + /agents// directory. In future all core agents will write data only + to the .agent-data subdirectory. This is because vctl install --force backs up and restores + only the contents of this directory. + 3. SQLHistorians (historian version 4.0.0 and above) now use a new database schema where metadata is stored in + topics table instead of separate metadata table. SQLHistorians with version >= 4.0.0 can work with existing + database with older schema however the historian agent code should be upgraded to newer version (>=4.0.0) to run + with VOLTTRON 8 core. + To begin the upgrade process, activate the volttron environment, and run ```python bootstrap.py --force```. .. note:: @@ -20,7 +32,21 @@ To begin the upgrade process, activate the volttron environment, and run ```pyth If you have any additional bootstrap options that you need (rabbitmq, web, drivers, etc.) include these in the above command. -After the bootstrap process is completed, run ```volttron-update-auth``` to update the auth file. +After the bootstrap process is completed, run ```volttron-upgrade``` to update the auth file and move historian +cache files into agent-data directory. Note that the upgrade script will only move the backup.sqlite file and will not +move sqlite historian's db file if they are within the install directory. If using a SQLite historian, please backup +the database file of sqlite historian before upgrading to the latest historian version. + +Once the volttron-upgrade script is complete, you can do a vctl install --force command to upgrade to the latest +historian version. vctl install --force will backup the cache in .agent-data folder, install the latest +version of the historian and restore the contents of .agent-data folder. + +Upgrading aggregate historians +------------------------------ + +VOLTTRON 8 also comes with updated SQL aggregate historian schema. However, there is no automated upgrade path for +aggregate historian. To upgrade an existing aggregate historian please refer to the CHANGELOG.md within +SQLAggregateHistorian source directory VOLTTRON 7 ========== diff --git a/setup.py b/setup.py index 3e0e9b188b..f7369da700 100644 --- a/setup.py +++ b/setup.py @@ -70,8 +70,7 @@ 'vctl = volttron.platform.control:_main', 'vpkg = volttron.platform.packaging:_main', 'vcfg = volttron.platform.config:_main', - # 'volttron-update = ...', - 'volttron-update-auth = volttron.platform.update_auth_file:_main', + 'volttron-upgrade = volttron.platform.upgrade.upgrade_volttron:_main', ] }, zip_safe = False, diff --git a/volttron/platform/aip.py b/volttron/platform/aip.py index 0a9238ee4f..38f4e31da5 100644 --- a/volttron/platform/aip.py +++ b/volttron/platform/aip.py @@ -343,7 +343,7 @@ def set_agent_user_permissions(self, volttron_agent_user, # except agent-data dir. agent-data dir has rwx self.set_acl_for_path("rx", volttron_agent_user, agent_dir) # creates dir if it doesn't exist - data_dir = self._get_agent_data_dir(agent_path_with_name) + data_dir = self.get_agent_data_dir(agent_path_with_name) for (root, directories, files) in os.walk(agent_dir, topdown=True): for directory in directories: @@ -609,7 +609,7 @@ def _unauthorize_agent_keys(self, agent_uuid): publickey = self.get_agent_keystore(agent_uuid).public AuthFile().remove_by_credentials(publickey) - def _get_agent_data_dir(self, agent_path): + def get_agent_data_dir(self, agent_path): pkg = UnpackedPackage(agent_path) data_dir = os.path.join(os.path.dirname(pkg.distinfo), '{}.agent-data'.format(pkg.package_name)) @@ -618,7 +618,7 @@ def _get_agent_data_dir(self, agent_path): return data_dir def create_agent_data_dir_if_missing(self, agent_uuid): - new_agent_data_dir = self._get_agent_data_dir(self.agent_dir(agent_uuid)) + new_agent_data_dir = self.get_agent_data_dir(self.agent_dir(agent_uuid)) return new_agent_data_dir def _get_data_dir(self, agent_path, agent_name): @@ -953,7 +953,7 @@ def start_agent(self, agent_uuid): resmon = getattr(self.env, 'resmon', None) agent_user = None - data_dir = self._get_agent_data_dir(agent_path_with_name) + data_dir = self.get_agent_data_dir(agent_path_with_name) if self.secure_agent_user: _log.info("Starting agent securely...") diff --git a/volttron/platform/upgrade/__init__.py b/volttron/platform/upgrade/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/volttron/platform/upgrade/move_sqlite_files.py b/volttron/platform/upgrade/move_sqlite_files.py new file mode 100644 index 0000000000..96e654dffc --- /dev/null +++ b/volttron/platform/upgrade/move_sqlite_files.py @@ -0,0 +1,117 @@ +# -*- coding: utf-8 -*- {{{ +# vim: set fenc=utf-8 ft=python sw=4 ts=4 sts=4 et: +# +# Copyright 2020, Battelle Memorial Institute. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This material was prepared as an account of work sponsored by an agency of +# the United States Government. Neither the United States Government nor the +# United States Department of Energy, nor Battelle, nor any of their +# employees, nor any jurisdiction or organization that has cooperated in the +# development of these materials, makes any warranty, express or +# implied, or assumes any legal liability or responsibility for the accuracy, +# completeness, or usefulness or any information, apparatus, product, +# software, or process disclosed, or represents that its use would not infringe +# privately owned rights. Reference herein to any specific commercial product, +# process, or service by trade name, trademark, manufacturer, or otherwise +# does not necessarily constitute or imply its endorsement, recommendation, or +# favoring by the United States Government or any agency thereof, or +# Battelle Memorial Institute. The views and opinions of authors expressed +# herein do not necessarily state or reflect those of the +# United States Government or any agency thereof. +# +# PACIFIC NORTHWEST NATIONAL LABORATORY operated by +# BATTELLE for the UNITED STATES DEPARTMENT OF ENERGY +# under Contract DE-AC05-76RL01830 +# }}} + +import sys +import shutil +from pathlib import Path +import glob +import re +from gevent import monkey as curious_george +curious_george.patch_all(thread=False, select=False) + +from volttron.platform import get_home +from volttron.platform.aip import AIPplatform +from volttron.platform.instance_setup import fail_if_instance_running + + +def get_aip(): + """Get AIPplatform to interface with agent directories in vhome""" + + vhome = get_home() + options = type("Options", (), dict(volttron_home=vhome)) + aip = AIPplatform(options) + return aip + + +def move_historian_cache_files(aip): + """ + Moves any keystore.json from agent-data to dist-info. + Only applies to agents in auth file. + """ + + vhome = Path(aip.env.volttron_home) + install_dir = vhome.joinpath("agents") + # pattern example - (/vhome/agents/uuid/agentname-version/)(backup.sqlite) + # pattern example - (vhome/agents/uuid/agentname-version/)(data/subdir/sqlitehistoriandb.sqlite) + pattern = "(" + str(install_dir) + "/[^/]*/[^/]*/)(.*)" + re_pattern = re.compile(pattern) + # Look for all .sqlite files in installed agents + print(f"Attempting to move backup.sqlite files in {install_dir} into corresponding agent-data directory") + # currently this is only used for backup.sqlite + # In 9.0 we could use the same code for *.sqlite files + # for example ones created by sqlitetagging, topic watcher, weather etc. when we make agent-data folder default + # agent write directory for all core agents + glob_path = str(install_dir.joinpath("**/backup.sqlite")) + for sqlite_file in glob.glob(glob_path, recursive=True): + result = re_pattern.match(sqlite_file).groups() + agent_dir = result[0] # /agents/uuid/ + source_file = result[1].split('/', 1)[0] # file or directory name to be moved to agent-data folder + source_path = str(Path(agent_dir).joinpath(source_file)) + dest_dir = aip.get_agent_data_dir(agent_path=agent_dir) + if source_path != dest_dir: + # if file is not already in agent-data dir + result = shutil.move(source_path, dest_dir) + + # print from uuid dir name so that it is easy to read + print_src = source_path.split(str(install_dir)+"/")[1] + print_dest = result.split(str(install_dir) + "/")[1] + print(f"Moved {print_src} to {print_dest}") + + +def main(): + """Upgrade auth file to function with dynamic rpc authorizations""" + + fail_if_instance_running() + aip = get_aip() + move_historian_cache_files(aip) + print("Moving historian backup files complete. " + "You can now safely upgrade historian agents other than SQLITE Historian with vctl install --force. " + "If using using SQLite historian please back up and restore sqlite historian's db manually") + + +def _main(): + """ Wrapper for main function""" + + try: + sys.exit(main()) + except KeyboardInterrupt: + sys.exit(1) + + +if __name__ == "__main__": + _main() diff --git a/volttron/platform/update_auth_file.py b/volttron/platform/upgrade/update_auth_file.py similarity index 98% rename from volttron/platform/update_auth_file.py rename to volttron/platform/upgrade/update_auth_file.py index f99b51e1ca..fb44b7b5fc 100644 --- a/volttron/platform/update_auth_file.py +++ b/volttron/platform/upgrade/update_auth_file.py @@ -39,8 +39,6 @@ import sys import shutil from pathlib import Path -from gevent import monkey as curious_george -curious_george.patch_all(thread=False, select=False) from volttron.platform import get_home from volttron.platform.aip import AIPplatform diff --git a/volttron/platform/upgrade/upgrade_volttron.py b/volttron/platform/upgrade/upgrade_volttron.py new file mode 100644 index 0000000000..354bf5e71b --- /dev/null +++ b/volttron/platform/upgrade/upgrade_volttron.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- {{{ +# vim: set fenc=utf-8 ft=python sw=4 ts=4 sts=4 et: +# +# Copyright 2020, Battelle Memorial Institute. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This material was prepared as an account of work sponsored by an agency of +# the United States Government. Neither the United States Government nor the +# United States Department of Energy, nor Battelle, nor any of their +# employees, nor any jurisdiction or organization that has cooperated in the +# development of these materials, makes any warranty, express or +# implied, or assumes any legal liability or responsibility for the accuracy, +# completeness, or usefulness or any information, apparatus, product, +# software, or process disclosed, or represents that its use would not infringe +# privately owned rights. Reference herein to any specific commercial product, +# process, or service by trade name, trademark, manufacturer, or otherwise +# does not necessarily constitute or imply its endorsement, recommendation, or +# favoring by the United States Government or any agency thereof, or +# Battelle Memorial Institute. The views and opinions of authors expressed +# herein do not necessarily state or reflect those of the +# United States Government or any agency thereof. +# +# PACIFIC NORTHWEST NATIONAL LABORATORY operated by +# BATTELLE for the UNITED STATES DEPARTMENT OF ENERGY +# under Contract DE-AC05-76RL01830 +# }}} + +import sys +from gevent import monkey as curious_george +curious_george.patch_all(thread=False, select=False) + +from . import update_auth_file +from . import move_sqlite_files + + +def main(): + # Upgrade auth file to function with dynamic rpc authorizations + update_auth_file.main() + + # Moves backup cache of historian (backup.sqlite files) into corresponding agent-data directory so that + # historian agents other than sqlitehistorian, can be upgraded to latest version using + # vctl install --force without losing cache data. vctl install --force will backup and restore + # contents of //.agent-data directory + # If using sqlite historian manually backup and restore sqlite historian's db before upgrading to historian version + # 4.0.0 or later + move_sqlite_files.main() + + +def _main(): + """ Wrapper for main function""" + try: + sys.exit(main()) + except KeyboardInterrupt: + sys.exit(1) + +if __name__ == "__main__": + _main() From 6cf38738f9c1104ba1f0b36960be4b70007fe002 Mon Sep 17 00:00:00 2001 From: schandrika Date: Tue, 21 Sep 2021 16:26:48 -0700 Subject: [PATCH 32/36] fix for issue #2802 updated test cases to handle the new location of backup.sqlite --- .../SQLHistorian/tests/test_sqlitehistorian_unit.py | 7 ++++++- .../platform/dbutils/test_backup_database.py | 13 +++++++++---- .../testutils/test_base_historian_unit.py | 9 ++++++++- 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/services/core/SQLHistorian/tests/test_sqlitehistorian_unit.py b/services/core/SQLHistorian/tests/test_sqlitehistorian_unit.py index ac2bfe6251..1ce2982b9e 100644 --- a/services/core/SQLHistorian/tests/test_sqlitehistorian_unit.py +++ b/services/core/SQLHistorian/tests/test_sqlitehistorian_unit.py @@ -1,13 +1,16 @@ import os from shutil import rmtree import subprocess +from pathlib import Path import pytest from gevent import sleep from datetime import timedelta from services.core.SQLHistorian.sqlhistorian import historian -CACHE_NAME = "backup.sqlite" +agent_data_dir = os.path.join(os.getcwd(), os.path.basename(os.getcwd()) + ".agent-data") +os.makedirs(agent_data_dir, exist_ok=True) +CACHE_NAME = str(Path(agent_data_dir).joinpath("backup.sqlite")) HISTORIAN_DB = "./data/historian.sqlite" @@ -76,6 +79,8 @@ def sql_historian(): rmtree("./data") if os.path.exists(CACHE_NAME): os.remove(CACHE_NAME) + if os.path.exists(agent_data_dir): + os.rmdir(agent_data_dir) def query_db(query, db): diff --git a/volttrontesting/platform/dbutils/test_backup_database.py b/volttrontesting/platform/dbutils/test_backup_database.py index a87ce62084..d3ecdc82bb 100644 --- a/volttrontesting/platform/dbutils/test_backup_database.py +++ b/volttrontesting/platform/dbutils/test_backup_database.py @@ -1,6 +1,6 @@ import os import pytest - +from pathlib import Path from gevent import subprocess from datetime import datetime from pytz import UTC @@ -9,6 +9,8 @@ SIZE_LIMIT = 1000 # the default submit_size_limit for BaseHistorianAgents +agent_data_dir = os.path.join(os.getcwd(), os.path.basename(os.getcwd()) + ".agent-data") +cache_db = str(Path(agent_data_dir).joinpath("backup.sqlite")) def test_get_outstanding_to_publish_should_return_records( backup_database, new_publish_list_unique @@ -245,13 +247,16 @@ def new_publish_list_dupes(): @pytest.fixture() def backup_database(): + os.makedirs(agent_data_dir, exist_ok=True) yield BackupDatabase(BaseHistorian(), None, 0.9) # Teardown # the backup database is an sqlite database with the name "backup.sqlite". # the db is created if it doesn't exist; see the method: BackupDatabase._setupdb(check_same_thread) for details - if os.path.exists("./backup.sqlite"): - os.remove("./backup.sqlite") + if os.path.exists(cache_db): + os.remove(cache_db) + if os.path.exists(agent_data_dir): + os.rmdir(agent_data_dir) def get_all_data(table): @@ -262,7 +267,7 @@ def get_all_data(table): def query_db(query): output = subprocess.run( - ["sqlite3", "backup.sqlite", query], text=True, capture_output=True + ["sqlite3", cache_db, query], text=True, capture_output=True ) # check_returncode() will raise a CalledProcessError if the query fails # see https://docs.python.org/3/library/subprocess.html#subprocess.CompletedProcess.returncode diff --git a/volttrontesting/testutils/test_base_historian_unit.py b/volttrontesting/testutils/test_base_historian_unit.py index 52640b13ba..6534ad6d71 100644 --- a/volttrontesting/testutils/test_base_historian_unit.py +++ b/volttrontesting/testutils/test_base_historian_unit.py @@ -3,6 +3,7 @@ import os from shutil import rmtree from time import sleep +from pathlib import Path import pytest from pytz import UTC @@ -10,7 +11,11 @@ from volttrontesting.utils.utils import AgentMock from volttron.platform.agent.base_historian import BaseHistorianAgent, Agent -CACHE_NAME = "backup.sqlite" + +agent_data_dir = os.path.join(os.getcwd(), os.path.basename(os.getcwd()) + ".agent-data") +os.makedirs(agent_data_dir, exist_ok=True) +CACHE_NAME = str(Path(agent_data_dir).joinpath("backup.sqlite")) + HISTORIAN_DB = "./data/historian.sqlite" @@ -113,3 +118,5 @@ def base_historian_agent(): rmtree("./data") if os.path.exists(CACHE_NAME): os.remove(CACHE_NAME) + if os.path.exists(agent_data_dir): + os.rmdir(agent_data_dir) From a12bea45f7b3193d9ecb01557f6722df4429c5e6 Mon Sep 17 00:00:00 2001 From: schandrika Date: Tue, 21 Sep 2021 17:13:12 -0700 Subject: [PATCH 33/36] minor changes for output readability --- volttron/platform/upgrade/move_sqlite_files.py | 1 + volttron/platform/upgrade/upgrade_volttron.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/volttron/platform/upgrade/move_sqlite_files.py b/volttron/platform/upgrade/move_sqlite_files.py index 96e654dffc..9b41e62b8b 100644 --- a/volttron/platform/upgrade/move_sqlite_files.py +++ b/volttron/platform/upgrade/move_sqlite_files.py @@ -99,6 +99,7 @@ def main(): fail_if_instance_running() aip = get_aip() move_historian_cache_files(aip) + print("") print("Moving historian backup files complete. " "You can now safely upgrade historian agents other than SQLITE Historian with vctl install --force. " "If using using SQLite historian please back up and restore sqlite historian's db manually") diff --git a/volttron/platform/upgrade/upgrade_volttron.py b/volttron/platform/upgrade/upgrade_volttron.py index 354bf5e71b..24cc837258 100644 --- a/volttron/platform/upgrade/upgrade_volttron.py +++ b/volttron/platform/upgrade/upgrade_volttron.py @@ -47,7 +47,7 @@ def main(): # Upgrade auth file to function with dynamic rpc authorizations update_auth_file.main() - + print("") # Moves backup cache of historian (backup.sqlite files) into corresponding agent-data directory so that # historian agents other than sqlitehistorian, can be upgraded to latest version using # vctl install --force without losing cache data. vctl install --force will backup and restore From cd945cce403a8b5b41824154ab9f4260450b79eb Mon Sep 17 00:00:00 2001 From: schandrika Date: Wed, 22 Sep 2021 11:09:32 -0700 Subject: [PATCH 34/36] added details 8x upgrade about historians that get auto started --- README.md | 29 ++++++++++------- .../VOLTTRON-releases/upgrading-versions.rst | 31 ++++++++++--------- 2 files changed, 34 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 0f475a3bbc..498efd242e 100644 --- a/README.md +++ b/README.md @@ -24,18 +24,23 @@ VOLTTRON 8 introduces three changes that require an explict upgrade step when up database with older schema however the historian agent code should be upgraded to newer version (>=4.0.0) to run with VOLTTRON 8 core. -To begin the upgrade process, activate the volttron environment, and run ```python bootstrap.py --force```. If you have -any additional bootstrap options that you need (rabbitmq, web, drivers, etc.) include these in the above command. - -After the bootstrap process is completed, run ```volttron-upgrade``` to update the auth file and move historian -cache files into agent-data directory. Note that the upgrade script will only move the backup.sqlite file and will not -move sqlite historian's db file if they are within the install directory. If using a SQLite historian, please backup -the database file of sqlite historian before upgrading to the latest historian version. - -Once the volttron-upgrade script is complete, you can do a vctl install --force command to upgrade to the latest -historian version. vctl install --force will backup the cache in .agent-data folder, install the latest -version of the historian and restore the contents of .agent-data folder. - +To upgrade: + + 1. If upgrading historian, make sure historians are not in auto start mode. To remove any historian from auto start + mode use the command 'vctl disable . This is necessary so that old + sqlhistorian does not automatically start after step 5. + 2. Update volttron source code version to VOLTTRON 8 + 3. activate the volttron environment, and run ```python bootstrap.py --force```. If you have + any additional bootstrap options that you need (rabbitmq, web, drivers, etc.) include these in the above command. + 4. Run ```volttron-upgrade``` to update the auth file and move historian cache files into agent-data directory. + Note that the upgrade script will only move the backup.sqlite file and will not move sqlite historian's db file + if they are within the install directory. If using a SQLite historian, please backup the database file of + sqlite historian before upgrading to the latest historian version. + 5. Start VOLTTRON + 6. Run ```vctl install --force --vip-identity --agent-config ``` to upgrade + to the latest historian version. vctl install --force will backup the cache in .agent-data + folder, installs the latest version of the historian and restore the contents of + .agent-data folder. ### Upgrading aggregate historians diff --git a/docs/source/volttron-topics/VOLTTRON-releases/upgrading-versions.rst b/docs/source/volttron-topics/VOLTTRON-releases/upgrading-versions.rst index 602f2278df..9914c778d8 100644 --- a/docs/source/volttron-topics/VOLTTRON-releases/upgrading-versions.rst +++ b/docs/source/volttron-topics/VOLTTRON-releases/upgrading-versions.rst @@ -25,21 +25,24 @@ VOLTTRON 8 introduces three changes that require an explict upgrade step when up database with older schema however the historian agent code should be upgraded to newer version (>=4.0.0) to run with VOLTTRON 8 core. -To begin the upgrade process, activate the volttron environment, and run ```python bootstrap.py --force```. -.. note:: - - If you have any additional bootstrap options that you need (rabbitmq, web, drivers, etc.) - include these in the above command. - -After the bootstrap process is completed, run ```volttron-upgrade``` to update the auth file and move historian -cache files into agent-data directory. Note that the upgrade script will only move the backup.sqlite file and will not -move sqlite historian's db file if they are within the install directory. If using a SQLite historian, please backup -the database file of sqlite historian before upgrading to the latest historian version. - -Once the volttron-upgrade script is complete, you can do a vctl install --force command to upgrade to the latest -historian version. vctl install --force will backup the cache in .agent-data folder, install the latest -version of the historian and restore the contents of .agent-data folder. +To upgrade: + + 1. If upgrading historian, make sure historians are not in auto start mode. To remove any historian from auto start + mode use the command 'vctl disable . This is necessary so that old + sqlhistorian does not automatically start after step 5. + 2. Update volttron source code version to VOLTTRON 8 + 3. activate the volttron environment, and run ```python bootstrap.py --force```. If you have + any additional bootstrap options that you need (rabbitmq, web, drivers, etc.) include these in the above command. + 4. Run ```volttron-upgrade``` to update the auth file and move historian cache files into agent-data directory. + Note that the upgrade script will only move the backup.sqlite file and will not move sqlite historian's db file + if they are within the install directory. If using a SQLite historian, please backup the database file of + sqlite historian before upgrading to the latest historian version. + 5. Start VOLTTRON + 6. Run ```vctl install --force --vip-identity --agent-config ``` to upgrade + to the latest historian version. vctl install --force will backup the cache in .agent-data + folder, installs the latest version of the historian and restore the contents of + .agent-data folder. Upgrading aggregate historians ------------------------------ From 0d89565d9cd35df991e90f4484be147089688d72 Mon Sep 17 00:00:00 2001 From: Craig <3979063+craig8@users.noreply.github.com> Date: Wed, 22 Sep 2021 11:13:12 -0700 Subject: [PATCH 35/36] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 498efd242e..398d7cf8dc 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ VOLTTRON 8 introduces three changes that require an explict upgrade step when up To upgrade: 1. If upgrading historian, make sure historians are not in auto start mode. To remove any historian from auto start - mode use the command 'vctl disable . This is necessary so that old + mode use the command 'vctl disable . This is necessary so that the old sqlhistorian does not automatically start after step 5. 2. Update volttron source code version to VOLTTRON 8 3. activate the volttron environment, and run ```python bootstrap.py --force```. If you have From 95eca761b4249b9e74528496260366c7fb5c65fc Mon Sep 17 00:00:00 2001 From: Craig <3979063+craig8@users.noreply.github.com> Date: Wed, 22 Sep 2021 11:13:44 -0700 Subject: [PATCH 36/36] Update upgrading-versions.rst --- .../volttron-topics/VOLTTRON-releases/upgrading-versions.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/volttron-topics/VOLTTRON-releases/upgrading-versions.rst b/docs/source/volttron-topics/VOLTTRON-releases/upgrading-versions.rst index 9914c778d8..5160e01d5c 100644 --- a/docs/source/volttron-topics/VOLTTRON-releases/upgrading-versions.rst +++ b/docs/source/volttron-topics/VOLTTRON-releases/upgrading-versions.rst @@ -29,7 +29,7 @@ VOLTTRON 8 introduces three changes that require an explict upgrade step when up To upgrade: 1. If upgrading historian, make sure historians are not in auto start mode. To remove any historian from auto start - mode use the command 'vctl disable . This is necessary so that old + mode use the command 'vctl disable . This is necessary so that the old sqlhistorian does not automatically start after step 5. 2. Update volttron source code version to VOLTTRON 8 3. activate the volttron environment, and run ```python bootstrap.py --force```. If you have