Merge pull request #65 from kaklise/master

Updates to the mask, no longer assigns Null values to False
sandialabs · Jul 29, 2020 · dde783f · dde783f
2 parents 66f9120 + 1e32a7b
commit dde783f
Show file tree

Hide file tree

Showing 4 changed files with 16 additions and 3 deletions.
diff --git a/documentation/whatsnew.rst b/documentation/whatsnew.rst
@@ -1,6 +1,8 @@
 Release Notes
 ================
 
+.. include:: whatsnew/v0.1.8.1.rst
+
 .. include:: whatsnew/v0.1.8.rst
 
 .. include:: whatsnew/v0.1.7.rst

diff --git a/documentation/whatsnew/v0.1.8.1.rst b/documentation/whatsnew/v0.1.8.1.rst
@@ -0,0 +1,9 @@
+.. _whatsnew_0181:
+
+v0.1.8.1 (master)
+--------------------------
+
+* Bug fix in the way masks are generated.  Data points that have Null values were always assigned to False, indicating 
+  that a quality control test failed.  Null values are now assumed to be True, unless a specific test fails (e.g. check_missing).
+
+
diff --git a/pecos/__init__.py b/pecos/__init__.py
@@ -6,7 +6,7 @@
 from pecos import utils
 from pecos import pv
 
-__version__ = '0.1.8'
+__version__ = '0.1.8.1'
 
 __copyright__ = """Copyright 2016-2020 National Technology & Engineering 
     Solutions of Sandia, LLC (NTESS). Under the terms of Contract 

diff --git a/pecos/monitoring.py b/pecos/monitoring.py
@@ -68,7 +68,7 @@ def mask(self):
             logger.info("Empty database")
             return
 
-        mask = ~pd.isnull(self.df) # False if NaN
+        mask = pd.DataFrame(True, index=self.df.index, columns=self.df.columns)
         for i in self.test_results.index:
             variable = self.test_results.loc[i, 'Variable Name']
             start_date = self.test_results.loc[i, 'Start Time']
@@ -78,7 +78,9 @@ def mask(self):
                     mask.loc[start_date:end_date,variable] = False
                 except:
                     pass
-
+            elif self.test_results.loc[i, 'Error Flag'] == 'Missing timestamp':
+                mask.loc[start_date:end_date,:] = False
+
         return mask
 
     @property