From b922cb6ff032b2f8f2270088ad04d9dd66777132 Mon Sep 17 00:00:00 2001 From: Matthew Taylor Date: Wed, 10 May 2017 15:33:10 -0700 Subject: [PATCH] Moved opf_helpers module to helpers --- examples/opf/experiments/params/EnsembleOnline.py | 4 ++-- examples/opf/experiments/params/test_all.py | 4 ++-- src/nupic/frameworks/opf/experiment_runner.py | 6 +++--- .../frameworks/opf/{opf_helpers.py => helpers.py} | 0 src/nupic/swarming/DummyModelRunner.py | 6 +++--- src/nupic/swarming/HypersearchV2.py | 4 ++-- src/nupic/swarming/ModelRunner.py | 6 +++--- src/nupic/swarming/api.py | 11 ++++++----- .../opf_description_template_test.py | 2 +- 9 files changed, 22 insertions(+), 21 deletions(-) rename src/nupic/frameworks/opf/{opf_helpers.py => helpers.py} (100%) diff --git a/examples/opf/experiments/params/EnsembleOnline.py b/examples/opf/experiments/params/EnsembleOnline.py index 3fc8fef616..d02034779a 100644 --- a/examples/opf/experiments/params/EnsembleOnline.py +++ b/examples/opf/experiments/params/EnsembleOnline.py @@ -1,7 +1,7 @@ import random import multiprocessing import numpy as np -from nupic.frameworks.opf import opf_helpers +from nupic.frameworks.opf import helpers from nupic.frameworks.opf.client import Client from random import shuffle from random import randrange, uniform @@ -168,7 +168,7 @@ def getFieldPermutations(config, predictedField): def getModelDescriptionLists(numProcesses, experiment): - config, control = opf_helpers.loadExperiment(experiment) + config, control = helpers.loadExperiment(experiment) encodersList=getFieldPermutations(config, 'pounds') ns=range(50, 140, 120) clAlphas=np.arange(0.01, 0.16, 0.104) diff --git a/examples/opf/experiments/params/test_all.py b/examples/opf/experiments/params/test_all.py index 071bca94c3..4ccdf1e294 100644 --- a/examples/opf/experiments/params/test_all.py +++ b/examples/opf/experiments/params/test_all.py @@ -25,7 +25,7 @@ from pprint import pprint import time -from nupic.frameworks.opf import opf_helpers +from nupic.frameworks.opf import helpers from nupic.frameworks.opf.client import Client # Experiment directories relative to "trunk/examples/opf/experiments." @@ -38,7 +38,7 @@ def testAll(experiments): for experiment in experiments: experimentBase = os.path.join(os.getcwd(), experimentsDir, experiment) - config, control = opf_helpers.loadExperiment(experimentBase) + config, control = helpers.loadExperiment(experimentBase) if control['environment'] == 'opfExperiment': experimentTasks = control['tasks'] diff --git a/src/nupic/frameworks/opf/experiment_runner.py b/src/nupic/frameworks/opf/experiment_runner.py index c752d5858f..ad41c91480 100644 --- a/src/nupic/frameworks/opf/experiment_runner.py +++ b/src/nupic/frameworks/opf/experiment_runner.py @@ -36,7 +36,7 @@ import numpy from nupic.data import jsonhelpers -from nupic.frameworks.opf import opf_basic_environment, opf_helpers +from nupic.frameworks.opf import opf_basic_environment, helpers from nupic.frameworks.opf.exp_description_api import OpfEnvironment from nupic.frameworks.opf.model_factory import ModelFactory from nupic.frameworks.opf.opf_task_driver import OPFTaskDriver @@ -378,9 +378,9 @@ def _runExperimentImpl(options, model=None): # Load the experiment's description.py module experimentDir = options.experimentDir - descriptionPyModule = opf_helpers.loadExperimentDescriptionScriptFromDir( + descriptionPyModule = helpers.loadExperimentDescriptionScriptFromDir( experimentDir) - expIface = opf_helpers.getExperimentDescriptionInterfaceFromModule( + expIface = helpers.getExperimentDescriptionInterfaceFromModule( descriptionPyModule) # Handle "list checkpoints" request diff --git a/src/nupic/frameworks/opf/opf_helpers.py b/src/nupic/frameworks/opf/helpers.py similarity index 100% rename from src/nupic/frameworks/opf/opf_helpers.py rename to src/nupic/frameworks/opf/helpers.py diff --git a/src/nupic/swarming/DummyModelRunner.py b/src/nupic/swarming/DummyModelRunner.py index f1231a9b46..dbd8f5f020 100644 --- a/src/nupic/swarming/DummyModelRunner.py +++ b/src/nupic/swarming/DummyModelRunner.py @@ -29,7 +29,7 @@ import time from nupic.frameworks.opf.model_factory import ModelFactory -from nupic.frameworks.opf import opf_helpers +from nupic.frameworks.opf import helpers from nupic.frameworks.opf.opf_utils import ModelResult from nupic.swarming import utils from nupic.swarming.ModelRunner import OPFModelRunner @@ -560,9 +560,9 @@ def __computeWaitTime(self): def __createModel(self, expDir): # ----------------------------------------------------------------------- # Load the experiment's description.py module - descriptionPyModule = opf_helpers.loadExperimentDescriptionScriptFromDir( + descriptionPyModule = helpers.loadExperimentDescriptionScriptFromDir( expDir) - expIface = opf_helpers.getExperimentDescriptionInterfaceFromModule( + expIface = helpers.getExperimentDescriptionInterfaceFromModule( descriptionPyModule) diff --git a/src/nupic/swarming/HypersearchV2.py b/src/nupic/swarming/HypersearchV2.py index 98ef8c73d8..44eed6aaf7 100644 --- a/src/nupic/swarming/HypersearchV2.py +++ b/src/nupic/swarming/HypersearchV2.py @@ -38,7 +38,7 @@ from nupic.swarming.hypersearch.SwarmTerminator import SwarmTerminator from nupic.swarming.hypersearch.HsState import HsState, HsSearchType -from nupic.frameworks.opf import opf_helpers +from nupic.frameworks.opf import helpers from nupic.swarming.experimentutils import InferenceType from nupic.swarming.utils import sortedJSONDumpS, rApply, rCopy from nupic.swarming.utils import clippedObj @@ -1018,7 +1018,7 @@ def __init__(self, searchParams, workerID=None, cjDAO=None, jobID=None, self._baseDescriptionHash = hashlib.md5(self._baseDescription).digest() # Read the model config to figure out the inference type - modelDescription, _ = opf_helpers.loadExperiment(self._basePath) + modelDescription, _ = helpers.loadExperiment(self._basePath) # Read info from permutations file. This sets up the following member # variables: diff --git a/src/nupic/swarming/ModelRunner.py b/src/nupic/swarming/ModelRunner.py index 9c3488f33a..04bc9b5577 100644 --- a/src/nupic/swarming/ModelRunner.py +++ b/src/nupic/swarming/ModelRunner.py @@ -34,7 +34,7 @@ from nupic.swarming.hypersearch.errorcodes import ErrorCodes from nupic.database.ClientJobsDAO import ClientJobsDAO -from nupic.frameworks.opf import opf_helpers +from nupic.frameworks.opf import helpers from nupic.frameworks.opf.model_factory import ModelFactory from nupic.frameworks.opf.opf_basic_environment import BasicPredictionLogger from nupic.frameworks.opf.opf_utils import matchPatterns @@ -217,9 +217,9 @@ def run(self): """ # ----------------------------------------------------------------------- # Load the experiment's description.py module - descriptionPyModule = opf_helpers.loadExperimentDescriptionScriptFromDir( + descriptionPyModule = helpers.loadExperimentDescriptionScriptFromDir( self._experimentDir) - expIface = opf_helpers.getExperimentDescriptionInterfaceFromModule( + expIface = helpers.getExperimentDescriptionInterfaceFromModule( descriptionPyModule) expIface.normalizeStreamSources() diff --git a/src/nupic/swarming/api.py b/src/nupic/swarming/api.py index 40451fccc2..0353d729f2 100644 --- a/src/nupic/swarming/api.py +++ b/src/nupic/swarming/api.py @@ -26,7 +26,7 @@ import shutil import tempfile -from nupic.frameworks.opf import opf_helpers +from nupic.frameworks.opf import helpers from nupic.database.ClientJobsDAO import ClientJobsDAO from nupic.support.configuration import Configuration @@ -80,7 +80,7 @@ def getSwarmModelParams(modelID): JSON-encoded string containing Model Params """ - # TODO: the use of opf_helpers.loadExperimentDescriptionScriptFromDir when + # TODO: the use of nupic.frameworks.opf.helpers.loadExperimentDescriptionScriptFromDir when # retrieving module params results in a leakage of pf_base_descriptionNN and # pf_descriptionNN module imports for every call to getSwarmModelParams, so # the leakage is unlimited when getSwarmModelParams is called by a @@ -96,7 +96,8 @@ def getSwarmModelParams(modelID): (baseDescription,) = cjDAO.jobGetFields(jobID, ["genBaseDescription"]) # Construct a directory with base.py and description.py for loading model - # params, and use opf_helpers to extract model params from those files + # params, and use nupic.frameworks.opf.helpers to extract model params from + # those files descriptionDirectory = tempfile.mkdtemp() try: baseDescriptionFilePath = os.path.join(descriptionDirectory, "base.py") @@ -107,8 +108,8 @@ def getSwarmModelParams(modelID): with open(descriptionFilePath, mode="wb") as f: f.write(description) - expIface = opf_helpers.getExperimentDescriptionInterfaceFromModule( - opf_helpers.loadExperimentDescriptionScriptFromDir(descriptionDirectory)) + expIface = helpers.getExperimentDescriptionInterfaceFromModule( + helpers.loadExperimentDescriptionScriptFromDir(descriptionDirectory)) return json.dumps( dict( diff --git a/tests/integration/nupic/opf/opf_description_template_test/opf_description_template_test.py b/tests/integration/nupic/opf/opf_description_template_test/opf_description_template_test.py index 6b3d4b5f66..683468ae28 100755 --- a/tests/integration/nupic/opf/opf_description_template_test/opf_description_template_test.py +++ b/tests/integration/nupic/opf/opf_description_template_test/opf_description_template_test.py @@ -28,7 +28,7 @@ from pkg_resources import resource_filename -from nupic.frameworks.opf.opf_helpers import ( +from nupic.frameworks.opf.helpers import ( loadExperimentDescriptionScriptFromDir, getExperimentDescriptionInterfaceFromModule )