Merge pull request #105 from UDST/col-from-expression

[0.2.dev5] Template for column from expression

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## 0.2 (not yet released)
 
+#### 0.2.dev5 (2019-03-29)
+
+- adds new template: `urbansim_templates.data.ColumnFromExpression`
+
 #### 0.2.dev4 (2019-03-26)
 
 - adds new data management utilities: `utils.validate_table()`, `utils.validate_all_tables()`, `utils.merge_tables()`

docs/source/data-templates.rst

@@ -1,8 +1,8 @@
 Data template APIs
 ==================
 
-Discussion
-----------
+Usage
+-----
 
 Data templates help you load tables into `Orca <https://udst.github.io/orca>`__ or save tables or subsets of tables to disk. 
 
@@ -75,15 +75,22 @@ From Orca's perspective, tables set up using the :mod:`~urbansim_templates.data.
 Unlike the templates, Orca relies on user-specified "`broadcast <http://udst.github.io/orca/core.html#merge-api>`__" relationships to perform automatic merging of tables. :mod:`~urbansim_templates.data.LoadTable` does not register any broadcasts, because they're not needed if tables follow the schema rules above. So if you use these tables in non-template model steps, you may need to add broadcasts separately.
 
 
-LoadTable API
--------------
+LoadTable()
+-----------
 
 .. autoclass:: urbansim_templates.data.LoadTable
    :members:
 
 
-SaveTable API
--------------
+SaveTable()
+-----------
 
 .. autoclass:: urbansim_templates.data.SaveTable
    :members:
+
+
+ColumnFromExpression()
+----------------------
+
+.. autoclass:: urbansim_templates.data.ColumnFromExpression
+   :members:

docs/source/index.rst

@@ -10,7 +10,7 @@ UrbanSim Templates provides building blocks for Orca-based simulation models. It
 
 The library contains templates for common types of model steps, plus a tool called ModelManager that runs as an extension to the `Orca <https://udst.github.io/orca>`__ task orchestrator. ModelManager can register template-based model steps with the orchestrator, save them to disk, and automatically reload them for future sessions.
 
-v0.2.dev4, released March 26, 2019
+v0.2.dev5, released March 29, 2019
 
 
 Contents

setup.py

@@ -6,7 +6,7 @@
 
 setup(
     name='urbansim_templates',
-    version='0.2.dev4',
+    version='0.2.dev5',
     description='UrbanSim extension for managing model steps',
     author='UrbanSim Inc.',
     author_email='[email protected]',

tests/test_column_expression.py

@@ -0,0 +1,191 @@
+import numpy as np
+import pandas as pd
+import pytest
+
+import orca
+
+from urbansim_templates import modelmanager
+from urbansim_templates.data import ColumnFromExpression
+from urbansim_templates.utils import validate_template
+
+
+@pytest.fixture
+def orca_session():
+    """
+    Set up a clean Orca and ModelManager session, with a data table.
+    
+    """
+    orca.clear_all()
+    modelmanager.initialize()
+
+    d1 = {'id': np.arange(5), 
+          'a': np.random.random(5),
+          'b': np.random.choice(np.arange(20), size=5)}
+
+    df = pd.DataFrame(d1).set_index('id')
+    orca.add_table('obs', df)
+
+
+def test_template_validity():
+    """
+    Check template conforms to basic spec.
+    
+    """
+    assert validate_template(ColumnFromExpression)
+
+
+def test_missing_colname(orca_session):
+    """
+    Missing column_name should raise a ValueError.
+    
+    """
+    c = ColumnFromExpression()
+    c.table = 'tab'
+    c.expression = 'a'
+
+    try:
+        c.run()
+    except ValueError as e:
+        print(e)
+        return
+
+    pytest.fail()
+
+
+def test_missing_table(orca_session):
+    """
+    Missing table should raise a ValueError.
+    
+    """
+    c = ColumnFromExpression()
+    c.column_name = 'col'
+    c.expression = 'a'
+
+    try:
+        c.run()
+    except ValueError as e:
+        print(e)
+        return
+
+    pytest.fail()
+
+
+def test_missing_expression(orca_session):
+    """
+    Missing expression should raise a ValueError.
+    
+    """
+    c = ColumnFromExpression()
+    c.column_name = 'col'
+    c.table = 'tab'
+
+    try:
+        c.run()
+    except ValueError as e:
+        print(e)
+        return
+
+    pytest.fail()
+
+
+def test_expression(orca_session):
+    """
+    Check that column is created and expression evaluated correctly.
+    
+    """
+    c = ColumnFromExpression()
+    c.column_name = 'c'
+    c.table = 'obs'
+    c.expression = 'a * 5 + sqrt(b)'
+
+    c.run()
+
+    val1 = orca.get_table('obs').get_column('c')
+    df = orca.get_table('obs').to_frame()
+    val2 = df.a * 5 + np.sqrt(df.b)
+    assert(val1.equals(val2))
+
+
+def test_data_type(orca_session):
+    """
+    Check that casting data type works.
+    
+    """
+    orca.add_table('tab', pd.DataFrame({'a': [0.1, 1.33, 2.4]}))
+
+    c = ColumnFromExpression()
+    c.column_name = 'b'
+    c.table = 'tab'
+    c.expression = 'a'
+    c.run()
+
+    v1 = orca.get_table('tab').get_column('b').values
+    np.testing.assert_equal(v1, [0.1, 1.33, 2.4])
+
+    c.data_type = 'int'
+    c.run()
+
+    v1 = orca.get_table('tab').get_column('b').values
+    np.testing.assert_equal(v1, [0, 1, 2])
+
+
+def test_missing_values(orca_session):
+    """
+    Check that filling in missing values works.
+    
+    """
+    orca.add_table('tab', pd.DataFrame({'a': [0.1, np.nan, 2.4]}))
+
+    c = ColumnFromExpression()
+    c.column_name = 'b'
+    c.table = 'tab'
+    c.expression = 'a'
+    c.run()
+
+    v1 = orca.get_table('tab').get_column('b').values
+    np.testing.assert_equal(v1, [0.1, np.nan, 2.4])
+
+    c.missing_values = 5
+    c.run()
+
+    v1 = orca.get_table('tab').get_column('b').values
+    np.testing.assert_equal(v1, [0.1, 5.0, 2.4])
+
+
+def test_modelmanager_registration(orca_session):
+    """
+    Check that modelmanager registration and auto-run work as expected.
+    
+    """
+    c = ColumnFromExpression()
+    c.column_name = 'c'
+    c.table = 'obs'
+    c.expression = 'a + b'
+
+    modelmanager.register(c)
+    modelmanager.remove_step(c.name)
+    assert('c' in orca.get_table('obs').columns)
+
+
+def test_expression_with_standalone_columns(orca_session):
+    """
+    Check that expression can assemble data from stand-alone columns that are not part 
+    of the core DataFrame wrapped by a table.
+    
+    """
+    c = ColumnFromExpression()
+    c.column_name = 'c'
+    c.table = 'obs'
+    c.expression = 'a + b'
+
+    modelmanager.register(c)
+    modelmanager.remove_step(c.name)
+
+    d = ColumnFromExpression()
+    d.column_name = 'd'
+    d.table = 'obs'
+    d.expression = 'a + c'
+
+    d.run()
+    assert('d' in orca.get_table('obs').columns)
+

urbansim_templates/__init__.py

@@ -1 +1 @@
-version = __version__ = '0.2.dev4'
+version = __version__ = '0.2.dev5'

urbansim_templates/data/__init__.py

@@ -1,2 +1,3 @@
+from .column_from_expression import ColumnFromExpression
 from .load_table import LoadTable
 from .save_table import SaveTable