From 84f9de07c23e5c0d7c22728f8baf5b7612667286 Mon Sep 17 00:00:00 2001
From: Sam Maurer <maurer@berkeley.edu>
Date: Mon, 11 Mar 2019 21:30:58 -0700
Subject: [PATCH 01/11] Sketch of column from expression

---
 .../data/column_from_expression.py            | 156 ++++++++++++++++++
 1 file changed, 156 insertions(+)
 create mode 100644 urbansim_templates/data/column_from_expression.py

diff --git a/urbansim_templates/data/column_from_expression.py b/urbansim_templates/data/column_from_expression.py
new file mode 100644
index 0000000..099b2ce
--- /dev/null
+++ b/urbansim_templates/data/column_from_expression.py
@@ -0,0 +1,156 @@
+from __future__ import print_function
+
+import orca
+import pandas as pd
+
+from urbansim_templates import modelmanager, __version__
+
+
+@modelmanager.template
+class ColumnFromExpression():
+    """
+    Template to register a column of derived data with Orca, based on an expression. The 
+    column will be associated with an existing table. Values will be calculated lazily, 
+    only when the column is requested for a specific operation. 
+    
+    The expression will be passed to ``pd.eval()`` and can refer to other columns in the 
+    table. See the Pandas documentation for further details.
+    
+    All the parameters can also be set as properties after creating the template 
+    instance.
+    
+    Parameters
+    ----------
+    column_name : str, optional
+        Name of the Orca column to be registered. Must be provided to run the template.
+    
+    table : str, optional
+        Name of the Orca table the column will be associated with. Must be provided to 
+        run the template.
+    
+    expression : str, optional
+        Expression for calculating values of the column. Must be provided to run the  
+        template.
+    
+    cache : bool, default False
+        Passed to ``orca.column()``.
+    
+    cache_scope : 'step', 'iteration', or 'forever', default 'forever'
+        Passed to ``orca.table()``.
+    
+    name : str, optional
+        Name of the template instance and associated model step. 
+    
+    tags : list of str, optional
+        Tags, passed to ModelManager.
+    
+    autorun : bool, default True
+        Automatically run the template whenever it's registered with ModelManager.
+    
+    """
+    def __init__(self, 
+            column_name = None, 
+            table = None, 
+            expression = None, 
+            cache = False, 
+            cache_scope = 'forever', 
+            name = None,
+            tags = [], 
+            autorun = True):
+        
+        # Template-specific params
+        self.column_name = column_name
+        self.table = table
+        self.expression = expression
+        self.cache = cache
+        self.cache_scope = cache_scope
+        
+        # Standard params
+        self.name = name
+        self.tags = tags
+        self.autorun = autorun
+        
+        # Automatic params
+        self.template = self.__class__.__name__
+        self.template_version = __version__
+    
+    
+    @classmethod
+    def from_dict(cls, d):
+        """
+        Create an object instance from a saved dictionary representation.
+        
+        Parameters
+        ----------
+        d : dict
+        
+        Returns
+        -------
+        Table
+        
+        """
+        obj = cls(
+            column_name = d['column_name'],
+            table = d['table'],
+            expression = d['expression'],
+            cache = d['cache'],
+            cache_scope = d['cache_scope'],
+            name = d['name'],
+            tags = d['tags'],
+            autorun = d['autorun']
+        )
+        return obj
+    
+    
+    def to_dict(self):
+        """
+        Create a dictionary representation of the object.
+        
+        Returns
+        -------
+        dict
+        
+        """
+        d = {
+            'template': self.template,
+            'template_version': self.template_version,
+            'name': self.name,
+            'tags': self.tags,
+            'autorun': self.autorun,
+            'column_name': self.column_name,
+            'table': self.table,
+            'expression': self.expression,
+            'cache': self.cache,
+            'cache_scope': self.cache_scope,
+        }
+        return d
+    
+    
+    def run(self):
+        """
+        Run the template, registering a column of derived data with Orca.
+        
+        Requires values to be set for ``column_name``, ``table``, and ``expression``.
+        
+        Returns
+        -------
+        None
+        
+        """
+        if self.column_name is None:
+            raise ValueError("Please provide a column name")
+        
+        if self.table not in ['csv', 'hdf']:
+            raise ValueError("Please provide a table")
+        
+        if self.expression is None:
+            raise ValueError("Please provide an expression")
+        
+        @orca.column(table_name = self.table, 
+                     column_name = self.column_name, 
+                     cache = self.cache, 
+                     cache_scope = self.cache_scope)
+        def orca_column():
+            pass
+        
+    
\ No newline at end of file

From ee67fe2b11361fc29573b4d7dd708ded8c75940f Mon Sep 17 00:00:00 2001
From: Sam Maurer <maurer@berkeley.edu>
Date: Mon, 11 Mar 2019 21:41:29 -0700
Subject: [PATCH 02/11] Better docstrings

---
 .../data/column_from_expression.py            | 20 +++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/urbansim_templates/data/column_from_expression.py b/urbansim_templates/data/column_from_expression.py
index 099b2ce..9bbbd3a 100644
--- a/urbansim_templates/data/column_from_expression.py
+++ b/urbansim_templates/data/column_from_expression.py
@@ -14,7 +14,7 @@ class ColumnFromExpression():
     only when the column is requested for a specific operation. 
     
     The expression will be passed to ``pd.eval()`` and can refer to other columns in the 
-    table. See the Pandas documentation for further details.
+    same table. See the Pandas documentation for further details.
     
     All the parameters can also be set as properties after creating the template 
     instance.
@@ -22,30 +22,30 @@ class ColumnFromExpression():
     Parameters
     ----------
     column_name : str, optional
-        Name of the Orca column to be registered. Must be provided to run the template.
+        Name of the Orca column to be registered. Required before running.
     
     table : str, optional
-        Name of the Orca table the column will be associated with. Must be provided to 
-        run the template.
+        Name of the Orca table the column will be associated with. Required before 
+        running.
     
     expression : str, optional
-        Expression for calculating values of the column. Must be provided to run the  
-        template.
+        Expression for calculating values of the column. Required before running.
     
     cache : bool, default False
-        Passed to ``orca.column()``.
+        Whether to cache column values after they are calculated.
     
     cache_scope : 'step', 'iteration', or 'forever', default 'forever'
-        Passed to ``orca.table()``.
+        How long to cache column values for (ignored if ``cache`` is False).
     
     name : str, optional
         Name of the template instance and associated model step. 
     
     tags : list of str, optional
-        Tags, passed to ModelManager.
+        Tags to associate with the template instance.
     
     autorun : bool, default True
-        Automatically run the template whenever it's registered with ModelManager.
+        Whether to run automatically when the template instance is registered with 
+        ModelManager.
     
     """
     def __init__(self, 

From 1d6b22a6fb6342707f4445e5a47d831bd5ca1c72 Mon Sep 17 00:00:00 2001
From: Sam Maurer <maurer@berkeley.edu>
Date: Tue, 12 Mar 2019 12:02:31 -0700
Subject: [PATCH 03/11] Implementing column generation logic

---
 .../data/column_from_expression.py            | 40 +++++++++++++++++--
 1 file changed, 37 insertions(+), 3 deletions(-)

diff --git a/urbansim_templates/data/column_from_expression.py b/urbansim_templates/data/column_from_expression.py
index 9bbbd3a..f137d8e 100644
--- a/urbansim_templates/data/column_from_expression.py
+++ b/urbansim_templates/data/column_from_expression.py
@@ -1,5 +1,7 @@
 from __future__ import print_function
 
+import re
+
 import orca
 import pandas as pd
 
@@ -13,7 +15,7 @@ class ColumnFromExpression():
     column will be associated with an existing table. Values will be calculated lazily, 
     only when the column is requested for a specific operation. 
     
-    The expression will be passed to ``pd.eval()`` and can refer to other columns in the 
+    The expression will be passed to ``df.eval()`` and can refer to other columns in the 
     same table. See the Pandas documentation for further details.
     
     All the parameters can also be set as properties after creating the template 
@@ -29,7 +31,16 @@ class ColumnFromExpression():
         running.
     
     expression : str, optional
-        Expression for calculating values of the column. Required before running.
+        String describing operations on existing columns of the table, for example 
+        "a/log(b+c)". Required before running. Supports arithmetic and math functions 
+        including sqrt, abs, log, log1p, exp, and expm1 -- see Pandas ``df.eval()`` 
+        documentation for further details.
+    
+    data_type : str, optional
+        Python type or ``numpy.dtype`` to cast the column's values into.
+    
+    missing_values : str or numeric, optional
+        Value to use for rows that would otherwise be missing.
     
     cache : bool, default False
         Whether to cache column values after they are calculated.
@@ -52,6 +63,8 @@ def __init__(self,
             column_name = None, 
             table = None, 
             expression = None, 
+            data_type = None, 
+            missing_values = None, 
             cache = False, 
             cache_scope = 'forever', 
             name = None,
@@ -62,6 +75,8 @@ def __init__(self,
         self.column_name = column_name
         self.table = table
         self.expression = expression
+        self.data_type = data_type
+        self.missing_values = missing_values
         self.cache = cache
         self.cache_scope = cache_scope
         
@@ -93,6 +108,8 @@ def from_dict(cls, d):
             column_name = d['column_name'],
             table = d['table'],
             expression = d['expression'],
+            data_type = d['data_type'],
+            missing_values = d['missing_values'],
             cache = d['cache'],
             cache_scope = d['cache_scope'],
             name = d['name'],
@@ -120,6 +137,8 @@ def to_dict(self):
             'column_name': self.column_name,
             'table': self.table,
             'expression': self.expression,
+            'data_type': self.data_type,
+            'missing_values': self.missing_values,
             'cache': self.cache,
             'cache_scope': self.cache_scope,
         }
@@ -146,11 +165,26 @@ def run(self):
         if self.expression is None:
             raise ValueError("Please provide an expression")
         
+        # Some column names in the expression may not be part of the core DataFrame, so 
+        # we'll need to request them from Orca explicitly. Identify tokens that begin 
+        # with a letter and contain any number of alphanumerics or underscores, but do 
+        # not end with an opening parenthesis.
+        cols = re.findall('[a-zA-Z_][a-zA-Z0-9_]*(?!\()', self.expression)
+        
         @orca.column(table_name = self.table, 
                      column_name = self.column_name, 
                      cache = self.cache, 
                      cache_scope = self.cache_scope)
         def orca_column():
-            pass
+            df = orca.get_table(self.table).to_frame(columns=cols)            
+            series = df.eval(self.expression)
+            
+            if self.missing_values is not None:
+                series = series.fillna(self.missing_values)
+            
+            if self.data_type is not None:
+                series = series.astype(self.data_type)
+            
+            return series
         
     
\ No newline at end of file

From 2d31bb8575bdddc1293d0fdc151210e8afd04aaa Mon Sep 17 00:00:00 2001
From: Sam Maurer <maurer@berkeley.edu>
Date: Tue, 12 Mar 2019 12:38:14 -0700
Subject: [PATCH 04/11] Adding to do

---
 urbansim_templates/data/column_from_expression.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/urbansim_templates/data/column_from_expression.py b/urbansim_templates/data/column_from_expression.py
index f137d8e..f4c5f55 100644
--- a/urbansim_templates/data/column_from_expression.py
+++ b/urbansim_templates/data/column_from_expression.py
@@ -171,6 +171,10 @@ def run(self):
         # not end with an opening parenthesis.
         cols = re.findall('[a-zA-Z_][a-zA-Z0-9_]*(?!\()', self.expression)
         
+        # TO DO - make sure requesting indexes by name doesn't raise an error from Orca
+        # - probably should just check which of the elements in the list Orca thinks are 
+        #   valid columns, and only request those
+        
         @orca.column(table_name = self.table, 
                      column_name = self.column_name, 
                      cache = self.cache, 

From 94616458d084653889ea9b28c9b337716fdf8760 Mon Sep 17 00:00:00 2001
From: Sam Maurer <maurer@berkeley.edu>
Date: Tue, 12 Mar 2019 14:14:24 -0700
Subject: [PATCH 05/11] Initial tests

---
 tests/test_column_expression.py     | 65 +++++++++++++++++++++++++++++
 urbansim_templates/data/__init__.py |  1 +
 2 files changed, 66 insertions(+)
 create mode 100644 tests/test_column_expression.py

diff --git a/tests/test_column_expression.py b/tests/test_column_expression.py
new file mode 100644
index 0000000..aad7da9
--- /dev/null
+++ b/tests/test_column_expression.py
@@ -0,0 +1,65 @@
+import numpy as np
+import pandas as pd
+import pytest
+
+import orca
+
+from urbansim_templates import modelmanager
+from urbansim_templates.data import ColumnFromExpression
+from urbansim_templates.utils import validate_template
+
+
+@pytest.fixture
+def orca_session():
+    """
+    Set up a clean Orca and ModelManager session, with a data table.
+    
+    """
+    orca.clear_all()
+    modelmanager.initialize()
+
+    d1 = {'id': np.arange(10), 
+          'a': np.random.random(10),
+          'b': np.random.choice(np.arange(20), size=10)}
+
+    df = pd.DataFrame(d1).set_index('id')
+    orca.add_table('obs', df)
+
+
+def test_template_validity():
+    """
+    Check template conforms to basic spec.
+    
+    """
+    assert validate_template(ColumnFromExpression)
+
+
+def test_run_requirements(orca_session):
+    """
+    
+    
+    """
+    pass
+
+
+def test_expression(orca_session):
+    """
+    Check that column is created correctly.
+    
+    """
+    c = ColumnFromExpression()
+    c.column_name = 'c'
+    c.table = 'obs'
+    c.expression = 'a + sqrt(b)'
+    
+    c.run()
+    series = orca.get_column('obs', 'c')
+    print(series)
+    
+
+def test_modelmanager_registration(orca_session):
+    """
+    Check that modelmanager registration and auto-run work as expected.
+    
+    """
+    pass
\ No newline at end of file
diff --git a/urbansim_templates/data/__init__.py b/urbansim_templates/data/__init__.py
index ccdcdf3..90dc264 100644
--- a/urbansim_templates/data/__init__.py
+++ b/urbansim_templates/data/__init__.py
@@ -1,2 +1,3 @@
+from .column_from_expression import ColumnFromExpression
 from .load_table import LoadTable
 from .save_table import SaveTable

From 2ea8ac74b01c0d9440e4654b8a246151bbe2ce22 Mon Sep 17 00:00:00 2001
From: Sam Maurer <maurer@berkeley.edu>
Date: Tue, 12 Mar 2019 14:48:24 -0700
Subject: [PATCH 06/11] Fixing failures

---
 tests/test_column_expression.py               | 19 ++++++++++++++++---
 .../data/column_from_expression.py            |  2 +-
 2 files changed, 17 insertions(+), 4 deletions(-)

diff --git a/tests/test_column_expression.py b/tests/test_column_expression.py
index aad7da9..f523935 100644
--- a/tests/test_column_expression.py
+++ b/tests/test_column_expression.py
@@ -50,16 +50,29 @@ def test_expression(orca_session):
     c = ColumnFromExpression()
     c.column_name = 'c'
     c.table = 'obs'
-    c.expression = 'a + sqrt(b)'
+    c.expression = 'a * 5 + sqrt(b)'
     
     c.run()
-    series = orca.get_column('obs', 'c')
+    series = orca.get_table('obs').get_column('c')
     print(series)
     
 
+def test_data_type_and_missing_values(orca_session):
+    """
+    """
+    pass
+
+
 def test_modelmanager_registration(orca_session):
     """
     Check that modelmanager registration and auto-run work as expected.
     
     """
-    pass
\ No newline at end of file
+    pass
+
+
+def test_expression_with_standalone_columns(orca_session):
+    """
+    """
+    pass
+
diff --git a/urbansim_templates/data/column_from_expression.py b/urbansim_templates/data/column_from_expression.py
index f4c5f55..4e384af 100644
--- a/urbansim_templates/data/column_from_expression.py
+++ b/urbansim_templates/data/column_from_expression.py
@@ -159,7 +159,7 @@ def run(self):
         if self.column_name is None:
             raise ValueError("Please provide a column name")
         
-        if self.table not in ['csv', 'hdf']:
+        if self.table is None:
             raise ValueError("Please provide a table")
         
         if self.expression is None:

From 952c1f3d03b90da92bafdaba4e5814361c865787 Mon Sep 17 00:00:00 2001
From: Sam Maurer <maurer@berkeley.edu>
Date: Wed, 13 Mar 2019 12:03:11 -0700
Subject: [PATCH 07/11] More tests

---
 tests/test_column_expression.py | 109 +++++++++++++++++++++++++++++---
 1 file changed, 99 insertions(+), 10 deletions(-)

diff --git a/tests/test_column_expression.py b/tests/test_column_expression.py
index f523935..edf6e24 100644
--- a/tests/test_column_expression.py
+++ b/tests/test_column_expression.py
@@ -18,9 +18,9 @@ def orca_session():
     orca.clear_all()
     modelmanager.initialize()
 
-    d1 = {'id': np.arange(10), 
-          'a': np.random.random(10),
-          'b': np.random.choice(np.arange(20), size=10)}
+    d1 = {'id': np.arange(5), 
+          'a': np.random.random(5),
+          'b': np.random.choice(np.arange(20), size=5)}
 
     df = pd.DataFrame(d1).set_index('id')
     orca.add_table('obs', df)
@@ -34,17 +34,63 @@ def test_template_validity():
     assert validate_template(ColumnFromExpression)
 
 
-def test_run_requirements(orca_session):
+def test_missing_colname(orca_session):
     """
+    Missing column_name should raise a ValueError.
     
+    """
+    c = ColumnFromExpression()
+    c.table = 'tab'
+    c.expression = 'a'
+    
+    try:
+        c.run()
+    except ValueError as e:
+        print(e)
+        return
     
+    pytest.fail()
+
+
+def test_missing_table(orca_session):
     """
-    pass
+    Missing table should raise a ValueError.
+    
+    """
+    c = ColumnFromExpression()
+    c.column_name = 'col'
+    c.expression = 'a'
+    
+    try:
+        c.run()
+    except ValueError as e:
+        print(e)
+        return
+    
+    pytest.fail()
+
+
+def test_missing_expression(orca_session):
+    """
+    Missing expression should raise a ValueError.
+    
+    """
+    c = ColumnFromExpression()
+    c.column_name = 'col'
+    c.table = 'tab'
+    
+    try:
+        c.run()
+    except ValueError as e:
+        print(e)
+        return
+    
+    pytest.fail()
 
 
 def test_expression(orca_session):
     """
-    Check that column is created correctly.
+    Check that column is created and expression evaluated correctly.
     
     """
     c = ColumnFromExpression()
@@ -53,14 +99,57 @@ def test_expression(orca_session):
     c.expression = 'a * 5 + sqrt(b)'
     
     c.run()
-    series = orca.get_table('obs').get_column('c')
-    print(series)
+    
+    val1 = orca.get_table('obs').get_column('c')
+    df = orca.get_table('obs').to_frame()
+    val2 = df.a * 5 + np.sqrt(df.b)
+    assert(val1.equals(val2))
     
 
-def test_data_type_and_missing_values(orca_session):
+def test_data_type(orca_session):
     """
+    Check that casting data type works.
+    
     """
-    pass
+    orca.add_table('tab', pd.DataFrame({'a': [0.1, 1.33, 2.4]}))
+    
+    c = ColumnFromExpression()
+    c.column_name = 'b'
+    c.table = 'tab'
+    c.expression = 'a'
+    c.run()
+    
+    v1 = orca.get_table('tab').get_column('b').values
+    np.testing.assert_equal(v1, [0.1, 1.33, 2.4])
+    
+    c.data_type = 'int'
+    c.run()
+
+    v1 = orca.get_table('tab').get_column('b').values
+    np.testing.assert_equal(v1, [0, 1, 2])
+
+
+def test_missing_values(orca_session):
+    """
+    Check that filling in missing values works.
+    
+    """
+    orca.add_table('tab', pd.DataFrame({'a': [0.1, np.nan, 2.4]}))
+    
+    c = ColumnFromExpression()
+    c.column_name = 'b'
+    c.table = 'tab'
+    c.expression = 'a'
+    c.run()
+    
+    v1 = orca.get_table('tab').get_column('b').values
+    np.testing.assert_equal(v1, [0.1, np.nan, 2.4])
+    
+    c.missing_values = 5
+    c.run()
+
+    v1 = orca.get_table('tab').get_column('b').values
+    np.testing.assert_equal(v1, [0.1, 5.0, 2.4])
 
 
 def test_modelmanager_registration(orca_session):

From a4edb1b2ddd61412205602d45ca7396563629c21 Mon Sep 17 00:00:00 2001
From: Sam Maurer <maurer@berkeley.edu>
Date: Thu, 28 Mar 2019 20:52:05 -0700
Subject: [PATCH 08/11] Use new get_df utility

---
 urbansim_templates/data/column_from_expression.py | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/urbansim_templates/data/column_from_expression.py b/urbansim_templates/data/column_from_expression.py
index 4e384af..a844c27 100644
--- a/urbansim_templates/data/column_from_expression.py
+++ b/urbansim_templates/data/column_from_expression.py
@@ -6,6 +6,7 @@
 import pandas as pd
 
 from urbansim_templates import modelmanager, __version__
+from urbansim_templates.utils import get_df
 
 
 @modelmanager.template
@@ -166,21 +167,18 @@ def run(self):
             raise ValueError("Please provide an expression")
         
         # Some column names in the expression may not be part of the core DataFrame, so 
-        # we'll need to request them from Orca explicitly. Identify tokens that begin 
-        # with a letter and contain any number of alphanumerics or underscores, but do 
-        # not end with an opening parenthesis.
+        # we'll need to request them from Orca explicitly. This regex pulls out column 
+        # names into a list, by identifying tokens in the expression that begin with a 
+        # letter and contain any number of alphanumerics or underscores, but do not end 
+        # with an opening parenthesis.
         cols = re.findall('[a-zA-Z_][a-zA-Z0-9_]*(?!\()', self.expression)
         
-        # TO DO - make sure requesting indexes by name doesn't raise an error from Orca
-        # - probably should just check which of the elements in the list Orca thinks are 
-        #   valid columns, and only request those
-        
         @orca.column(table_name = self.table, 
                      column_name = self.column_name, 
                      cache = self.cache, 
                      cache_scope = self.cache_scope)
         def orca_column():
-            df = orca.get_table(self.table).to_frame(columns=cols)            
+            df = get_df(self.table, columns=cols)
             series = df.eval(self.expression)
             
             if self.missing_values is not None:

From d98f023c62137302e429a831fed97de6d1919139 Mon Sep 17 00:00:00 2001
From: Sam Maurer <maurer@berkeley.edu>
Date: Thu, 28 Mar 2019 21:06:39 -0700
Subject: [PATCH 09/11] Final tests

---
 tests/test_column_expression.py               | 28 +++++++++++++++++--
 .../data/column_from_expression.py            |  3 +-
 2 files changed, 28 insertions(+), 3 deletions(-)

diff --git a/tests/test_column_expression.py b/tests/test_column_expression.py
index edf6e24..4af02df 100644
--- a/tests/test_column_expression.py
+++ b/tests/test_column_expression.py
@@ -157,11 +157,35 @@ def test_modelmanager_registration(orca_session):
     Check that modelmanager registration and auto-run work as expected.
     
     """
-    pass
+    c = ColumnFromExpression()
+    c.column_name = 'c'
+    c.table = 'obs'
+    c.expression = 'a + b'
+    
+    modelmanager.register(c)
+    modelmanager.remove_step(c.name)
+    assert('c' in orca.get_table('obs').columns)
 
 
 def test_expression_with_standalone_columns(orca_session):
     """
+    Check that expression can assemble data from stand-alone columns that are not part 
+    of the core DataFrame wrapped by a table.
+    
     """
-    pass
+    c = ColumnFromExpression()
+    c.column_name = 'c'
+    c.table = 'obs'
+    c.expression = 'a + b'
+    
+    modelmanager.register(c)
+    modelmanager.remove_step(c.name)
+
+    d = ColumnFromExpression()
+    d.column_name = 'd'
+    d.table = 'obs'
+    d.expression = 'a + c'
+    
+    d.run()
+    assert('d' in orca.get_table('obs').columns)
 
diff --git a/urbansim_templates/data/column_from_expression.py b/urbansim_templates/data/column_from_expression.py
index a844c27..8c6c40d 100644
--- a/urbansim_templates/data/column_from_expression.py
+++ b/urbansim_templates/data/column_from_expression.py
@@ -170,7 +170,8 @@ def run(self):
         # we'll need to request them from Orca explicitly. This regex pulls out column 
         # names into a list, by identifying tokens in the expression that begin with a 
         # letter and contain any number of alphanumerics or underscores, but do not end 
-        # with an opening parenthesis.
+        # with an opening parenthesis. This will also pick up constants, like "pi", but  
+        # invalid column names will be ignored when we request them from get_df().
         cols = re.findall('[a-zA-Z_][a-zA-Z0-9_]*(?!\()', self.expression)
         
         @orca.column(table_name = self.table, 

From 45ecb47750d5be0dac807644b9018b151baa84cf Mon Sep 17 00:00:00 2001
From: Sam Maurer <maurer@berkeley.edu>
Date: Thu, 28 Mar 2019 21:08:01 -0700
Subject: [PATCH 10/11] Updating version

---
 setup.py                       | 2 +-
 urbansim_templates/__init__.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/setup.py b/setup.py
index 635e8e4..b77543a 100644
--- a/setup.py
+++ b/setup.py
@@ -6,7 +6,7 @@
 
 setup(
     name='urbansim_templates',
-    version='0.2.dev4',
+    version='0.2.dev5',
     description='UrbanSim extension for managing model steps',
     author='UrbanSim Inc.',
     author_email='info@urbansim.com',
diff --git a/urbansim_templates/__init__.py b/urbansim_templates/__init__.py
index c8a0bc7..aba1d24 100644
--- a/urbansim_templates/__init__.py
+++ b/urbansim_templates/__init__.py
@@ -1 +1 @@
-version = __version__ = '0.2.dev4'
+version = __version__ = '0.2.dev5'

From b5843587aed84e54fefdbeec0bd2342bfb4a2531 Mon Sep 17 00:00:00 2001
From: Sam Maurer <maurer@berkeley.edu>
Date: Fri, 29 Mar 2019 11:46:35 -0700
Subject: [PATCH 11/11] Docs and changelog

---
 CHANGELOG.md                                  |  4 ++++
 docs/source/data-templates.rst                | 19 +++++++++++++------
 docs/source/index.rst                         |  2 +-
 .../data/column_from_expression.py            |  4 ++--
 4 files changed, 20 insertions(+), 9 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 68650e3..c196666 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,10 @@
 
 ## 0.2 (not yet released)
 
+#### 0.2.dev5 (2019-03-29)
+
+- adds new template: `urbansim_templates.data.ColumnFromExpression`
+
 #### 0.2.dev4 (2019-03-26)
 
 - adds new data management utilities: `utils.validate_table()`, `utils.validate_all_tables()`, `utils.merge_tables()`
diff --git a/docs/source/data-templates.rst b/docs/source/data-templates.rst
index fc0d3e1..9317a3e 100644
--- a/docs/source/data-templates.rst
+++ b/docs/source/data-templates.rst
@@ -1,8 +1,8 @@
 Data template APIs
 ==================
 
-Discussion
-----------
+Usage
+-----
 
 Data templates help you load tables into `Orca <https://udst.github.io/orca>`__ or save tables or subsets of tables to disk. 
 
@@ -75,15 +75,22 @@ From Orca's perspective, tables set up using the :mod:`~urbansim_templates.data.
 Unlike the templates, Orca relies on user-specified "`broadcast <http://udst.github.io/orca/core.html#merge-api>`__" relationships to perform automatic merging of tables. :mod:`~urbansim_templates.data.LoadTable` does not register any broadcasts, because they're not needed if tables follow the schema rules above. So if you use these tables in non-template model steps, you may need to add broadcasts separately.
 
 
-LoadTable API
--------------
+LoadTable()
+-----------
 
 .. autoclass:: urbansim_templates.data.LoadTable
    :members:
 
 
-SaveTable API
--------------
+SaveTable()
+-----------
 
 .. autoclass:: urbansim_templates.data.SaveTable
    :members:
+
+
+ColumnFromExpression()
+----------------------
+
+.. autoclass:: urbansim_templates.data.ColumnFromExpression
+   :members:
diff --git a/docs/source/index.rst b/docs/source/index.rst
index 0b8a812..1e5ce05 100644
--- a/docs/source/index.rst
+++ b/docs/source/index.rst
@@ -10,7 +10,7 @@ UrbanSim Templates provides building blocks for Orca-based simulation models. It
 
 The library contains templates for common types of model steps, plus a tool called ModelManager that runs as an extension to the `Orca <https://udst.github.io/orca>`__ task orchestrator. ModelManager can register template-based model steps with the orchestrator, save them to disk, and automatically reload them for future sessions.
 
-v0.2.dev4, released March 26, 2019
+v0.2.dev5, released March 29, 2019
 
 
 Contents
diff --git a/urbansim_templates/data/column_from_expression.py b/urbansim_templates/data/column_from_expression.py
index 8c6c40d..a7ce796 100644
--- a/urbansim_templates/data/column_from_expression.py
+++ b/urbansim_templates/data/column_from_expression.py
@@ -14,9 +14,9 @@ class ColumnFromExpression():
     """
     Template to register a column of derived data with Orca, based on an expression. The 
     column will be associated with an existing table. Values will be calculated lazily, 
-    only when the column is requested for a specific operation. 
+    only when the column is needed for a specific operation. 
     
-    The expression will be passed to ``df.eval()`` and can refer to other columns in the 
+    The expression will be passed to ``df.eval()`` and can refer to any columns in the 
     same table. See the Pandas documentation for further details.
     
     All the parameters can also be set as properties after creating the template