Package updates (#109)

* ActivitySim 0.9.2; Pandas 1.0

* Freeze ortools package below 7.5

example_calm/run_populationsim.py

@@ -12,7 +12,7 @@
 from activitysim.core.config import handle_standard_args
 from activitysim.core.tracing import print_elapsed_time
 
-from populationsim.util import setting
+from activitysim.core.config import setting
 from populationsim import lp
 from populationsim import multi_integerizer
 

example_calm_repop/run_populationsim.py

@@ -12,7 +12,7 @@
 from activitysim.core.config import handle_standard_args
 from activitysim.core.tracing import print_elapsed_time
 
-from populationsim.util import setting
+from activitysim.core.config import setting
 from populationsim import lp
 from populationsim import multi_integerizer
 

example_survey_weighting/run_populationsim.py

@@ -12,7 +12,7 @@
 from activitysim.core.config import handle_standard_args
 from activitysim.core.tracing import print_elapsed_time
 
-from populationsim.util import setting
+from activitysim.core.config import setting
 from populationsim import lp
 from populationsim import multi_integerizer
 

example_test/run_populationsim.py

@@ -11,7 +11,7 @@
 from activitysim.core.config import handle_standard_args
 
 from populationsim import steps
-from populationsim.util import setting
+from activitysim.core.config import setting
 from populationsim import lp
 from populationsim import multi_integerizer
 

populationsim/balancer.py

@@ -10,7 +10,7 @@
 
 import pandas as pd
 
-from .util import setting
+from activitysim.core.config import setting
 
 
 logger = logging.getLogger(__name__)

populationsim/integerizer.py

@@ -9,7 +9,7 @@
 
 import numpy as np
 import pandas as pd
-from .util import setting
+from activitysim.core.config import setting
 
 from .lp import get_single_integerizer
 from .lp import STATUS_SUCCESS

populationsim/lp.py

@@ -4,7 +4,7 @@
 
 import logging
 
-from .util import setting
+from activitysim.core.config import setting
 from . import lp_cvx
 from . import lp_ortools
 

populationsim/lp_cvx.py

@@ -5,7 +5,7 @@
 import logging
 
 import numpy as np
-from .util import setting
+from activitysim.core.config import setting
 
 logger = logging.getLogger(__name__)
 

populationsim/multi_integerizer.py

@@ -12,7 +12,7 @@
 import pandas as pd
 
 
-from .util import setting
+from activitysim.core.config import setting
 
 from .lp import get_simul_integerizer
 from .lp import STATUS_SUCCESS

populationsim/simul_balancer.py

@@ -11,7 +11,7 @@
 
 import pandas as pd
 
-from .util import setting
+from activitysim.core.config import setting
 
 logger = logging.getLogger(__name__)
 

populationsim/steps/__init__.py

@@ -1,7 +1,10 @@
-from __future__ import absolute_import
 # PopulationSim
 # See full license in LICENSE.txt.
 
+from __future__ import absolute_import
+
+from activitysim.core import inject as _inject
+
 from . import input_pre_processor
 from . import setup_data_structures
 from . import initial_seed_balancing
@@ -11,7 +14,14 @@
 from . import sub_balancing
 from . import expand_households
 from . import summarize
-from . import write_tables
 from . import write_synthetic_population
-
 from . import repop_balancing
+
+from activitysim.core.steps.output import write_data_dictionary
+from activitysim.core.steps.output import write_tables
+
+
+@_inject.injectable(cache=True)
+def preload_injectables():
+    _inject.add_step('write_data_dictionary', write_data_dictionary)
+    _inject.add_step('write_tables', write_tables)

populationsim/steps/expand_households.py

@@ -11,7 +11,7 @@
 from activitysim.core import pipeline
 from activitysim.core import inject
 
-from populationsim.util import setting
+from activitysim.core.config import setting
 from .helper import get_control_table
 from .helper import get_weight_table
 

populationsim/steps/final_seed_balancing.py

@@ -8,7 +8,7 @@
 
 from activitysim.core import inject
 
-from populationsim.util import setting
+from activitysim.core.config import setting
 
 from ..balancer import do_balancing
 from .helper import get_control_table

populationsim/steps/initial_seed_balancing.py

@@ -8,7 +8,7 @@
 from activitysim.core import inject
 from activitysim.core import pipeline
 
-from populationsim.util import setting
+from activitysim.core.config import setting
 
 from ..balancer import do_balancing
 

populationsim/steps/input_pre_processor.py

@@ -7,12 +7,11 @@
 import pandas as pd
 import numpy as np
 
-from activitysim.core import inject
-from activitysim.core import pipeline
-
-from populationsim.util import data_dir_from_settings
-from populationsim.util import setting
-
+from activitysim.core import (
+    inject,
+    config,
+    input
+)
 
 logger = logging.getLogger(__name__)
 
@@ -30,16 +29,16 @@ def input_pre_processor():
     unless an alternate table_list name is specified as a model step argument 'table_list'.
     (This allows alternate/additional input files to be read for repop)
 
-    In the case of repop, this step is being run after an initial populationsim run has
-    completed, in which case the input_table_list may specify replacement tables.
+    In the case of repop, this step is being run after an initial run has completed,
+    in which case the input_table_list may specify replacement tables.
     (e.g. lowest geography controls that will replace the previous low controls dataframe.)
 
     See input_table_list in settings.yaml in the example folder for a working example
 
     +--------------+----------------------------------------------------------+
     | key          | description                                              |
     +==============+=========================================+================+
-    | tablename    |  ame of pipeline table in which to store dataframe       |
+    | tablename    | name of pipeline table in which to store dataframe       |
     +--------------+----------------------------------------------------------+
     | filename     | name of csv file to read (in data_dir)                   |
     +--------------+----------------------------------------------------------+
@@ -54,85 +53,18 @@ def input_pre_processor():
 
     # alternate table list name may have been provided as a model argument
     table_list_name = inject.get_step_arg('table_list', default='input_table_list')
-    table_list = setting(table_list_name)
-    assert table_list is not None, "table list '%s' not in settings." % table_list_name
+    table_list = config.setting(table_list_name)
 
-    data_dir = data_dir_from_settings()
+    assert table_list is not None, "no table list '%s' found in settings." % table_list_name
+
+    logger.info('Using table list: %s' % table_list)
 
     for table_info in table_list:
 
-        tablename = table_info['tablename']
-
-        logger.info("input_pre_processor processing %s" % tablename)
-
-        # read the csv file
-        data_filename = table_info.get('filename', None)
-        data_file_path = os.path.join(data_dir, data_filename)
-        if not os.path.exists(data_file_path):
-            raise RuntimeError("input_pre_processor %s - input file not found: %s"
-                               % (tablename, data_file_path, ))
-
-        logger.info("Reading csv file %s" % data_file_path)
-        df = read_csv_with_fallback_encoding(data_file_path)
-
-        logger.info("input file columns: %s" % df.columns.values)
-
-        drop_columns = table_info.get('drop_columns', None)
-        if drop_columns:
-            for c in drop_columns:
-                logger.info("dropping column '%s'" % c)
-                del df[c]
-
-        # rename columns
-        column_map = table_info.get('column_map', None)
-        if column_map:
-            df.rename(columns=column_map, inplace=True)
-
-        # set index
-        index_col = table_info.get('index_col', None)
-        if index_col is not None:
-            if index_col in df.columns:
-                assert not df.duplicated(index_col).any()
-                df.set_index(index_col, inplace=True)
-            else:
-                df.index.names = [index_col]
-
-        # read expression file
-        # expression_filename = table_info.get('expression_filename', None)
-        # if expression_filename:
-        #     assert False
-        #     expression_file_path = os.path.join(configs_dir, expression_filename)
-        #     if not os.path.exists(expression_file_path):
-        #         raise RuntimeError("input_pre_processor %s - expression file not found: %s"
-        #                            % (table, expression_file_path, ))
-        #     spec = assign.read_assignment_spec(expression_file_path)
-        #
-        #     df_alias = table_info.get('df_alias', table)
-        #
-        #     locals_d = {}
-        #
-        #     results, trace_results, trace_assigned_locals \
-        #         = assign.assign_variables(spec, df, locals_d, df_alias=df_alias)
-        #     # for column in results.columns:
-        #     #     orca.add_column(table, column, results[column])
-        #
-        #     df = pd.concat([df, results], axis=1)
-
-        logger.info("adding table %s" % tablename)
+        tablename = table_info.get('tablename')
+        df = input.read_from_table_info(table_info)
+        logger.info('registering table %s' % tablename)
 
         # add (or replace) pipeline table
         repop = inject.get_step_arg('repop', default=False)
         inject.add_table(tablename, df, replace=repop)
-
-
-def read_csv_with_fallback_encoding(filepath):
-    """read a CSV to a pandas DataFrame using default utf-8 encoding,
-    but try alternate Windows-compatible cp1252 if unicode fails
-
-    """
-    try:
-        return pd.read_csv(filepath, comment='#')
-    except UnicodeDecodeError:
-        logger.warning(
-            "Reading %s with default utf-8 encoding failed, trying cp1252 instead", filepath)
-        return pd.read_csv(filepath, comment='#', encoding='cp1252')

populationsim/steps/integerize_final_seed_weights.py

@@ -13,7 +13,7 @@
 from .helper import get_control_table
 from .helper import weight_table_name
 from .helper import get_weight_table
-from populationsim.util import setting
+from activitysim.core.config import setting
 
 logger = logging.getLogger(__name__)
 

populationsim/steps/repop_balancing.py

@@ -7,7 +7,7 @@
 
 from activitysim.core import inject
 
-from populationsim.util import setting
+from activitysim.core.config import setting
 
 from .helper import get_control_table
 from .helper import weight_table_name

populationsim/steps/setup_data_structures.py

@@ -18,7 +18,7 @@
 from .helper import get_control_table
 from .helper import get_control_data_table
 
-from populationsim.util import setting
+from activitysim.core.config import setting
 
 logger = logging.getLogger(__name__)
 

populationsim/steps/sub_balancing.py

@@ -12,7 +12,7 @@
 from activitysim.core import inject
 from activitysim.core import pipeline
 
-from populationsim.util import setting
+from activitysim.core.config import setting
 
 from .helper import get_control_table
 from .helper import weight_table_name

populationsim/steps/summarize.py

@@ -12,7 +12,7 @@
 
 from .helper import get_control_table
 from .helper import get_weight_table
-from populationsim.util import setting
+from activitysim.core.config import setting
 
 logger = logging.getLogger(__name__)
 

populationsim/steps/write_synthetic_population.py

@@ -8,7 +8,7 @@
 from activitysim.core import pipeline
 from activitysim.core import inject
 
-from populationsim.util import setting
+from activitysim.core.config import setting
 
 logger = logging.getLogger(__name__)