diff --git a/README.md b/README.md index 3fad313..7c575df 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ ![Pecos](documentation/figures/logo.png) -======================================= +========================================= [![TravisCI](https://travis-ci.org/sandialabs/pecos.svg?branch=master)](https://travis-ci.org/sandialabs/pecos) [![Coverage Status](https://coveralls.io/repos/github/sandialabs/pecos/badge.svg?branch=master)](https://coveralls.io/github/sandialabs/pecos?branch=master) diff --git a/ci/requirements-py36.yml b/ci/requirements-py36.yml index 595faa2..8f1e10a 100644 --- a/ci/requirements-py36.yml +++ b/ci/requirements-py36.yml @@ -12,7 +12,6 @@ dependencies: - pvlib - plotly - ephem - - xlrd - sqlalchemy - pip: - coveralls \ No newline at end of file diff --git a/ci/requirements-py37.yml b/ci/requirements-py37.yml index 34d804a..d3262df 100644 --- a/ci/requirements-py37.yml +++ b/ci/requirements-py37.yml @@ -12,7 +12,6 @@ dependencies: - pvlib - plotly - ephem - - xlrd - sqlalchemy - pip: - coveralls \ No newline at end of file diff --git a/ci/requirements-py38.yml b/ci/requirements-py38.yml index 6859532..39a8c03 100644 --- a/ci/requirements-py38.yml +++ b/ci/requirements-py38.yml @@ -12,7 +12,6 @@ dependencies: - pvlib - plotly - ephem - - xlrd - sqlalchemy - pip: - coveralls \ No newline at end of file diff --git a/documentation/example.rst b/documentation/example.rst index 2afc5fa..ff413d6 100644 --- a/documentation/example.rst +++ b/documentation/example.rst @@ -4,7 +4,7 @@ Simple example ================ A simple example is included in the `examples/simple `_ directory. -This example uses data from an excel file, **simple.xlsx**, which +This example uses data from a CSV file, **simple.csv**, which contains 4 columns of data (A through D). * A = elapsed time in days @@ -41,7 +41,7 @@ of expected range, data that doesn't change, and data that changes abruptly, as The script, **simple_example.py** (shown below), is used to run quality control analysis using Pecos. The script performs the following steps: -* Load time series data from an excel file +* Load time series data from a CSV file * Run quality control tests diff --git a/documentation/whatsnew.rst b/documentation/whatsnew.rst index 8fea40d..e5bf5ed 100644 --- a/documentation/whatsnew.rst +++ b/documentation/whatsnew.rst @@ -1,6 +1,8 @@ Release Notes ================ +.. include:: whatsnew/v0.2.0.rst + .. include:: whatsnew/v0.1.9.rst .. include:: whatsnew/v0.1.8.rst diff --git a/documentation/whatsnew/v0.2.0.rst b/documentation/whatsnew/v0.2.0.rst new file mode 100644 index 0000000..d86544f --- /dev/null +++ b/documentation/whatsnew/v0.2.0.rst @@ -0,0 +1,12 @@ +.. _whatsnew_020: + +v0.2.0 (master) +-------------------------- + +* Replaced the use of Excel files in examples/tests with CSV files. The Excel files were causing test failures. +* Added `min_failures` to the streaming outlier test +* Replaced mutable default arguments with None +* Removed pecos logo from monitoring reports +* Added timestamp to logger +* Updated documentation and tests + diff --git a/examples/metrics/System1_2015_performance_metrics.csv b/examples/metrics/System1_2015_performance_metrics.csv new file mode 100644 index 0000000..507fa11 --- /dev/null +++ b/examples/metrics/System1_2015_performance_metrics.csv @@ -0,0 +1,399 @@ +,Quality Control Index,Performance Ratio +1/1/2015,0.955603448,0.88984202 +1/2/2015,0.947868585,0.889105932 +1/3/2015,0.954879423,0.934635912 +1/4/2015,0.957177539,0.927932798 +1/5/2015,0.951231527,0.938031582 +1/6/2015,0.953568876,0.912235587 +1/7/2015,0.944343228,0.857848913 +1/8/2015,0.956276039,0.907509256 +1/9/2015,0.958430836,0.910073157 +1/10/2015,0.959373761,0.898035104 +1/11/2015,0.953569075,0.868802475 +1/12/2015,0.961491336,0.857900582 +1/13/2015,0.958336971,0.852716219 +1/14/2015,0.95321752,0.849064364 +1/15/2015,0.957264772,0.875677399 +1/16/2015,0.961909083,0.856211057 +1/17/2015,0.96112069,0.838436043 +1/18/2015,0.960701694,0.830289753 +1/19/2015,0.959134718,0.860527002 +1/20/2015,0.948788617,0.845570524 +1/21/2015,0.938365799,0.869293438 +1/22/2015,0.951443514,0.869310363 +1/23/2015,0.953005047,0.890299422 +1/24/2015,0.959530358,0.874913574 +1/25/2015,0.959355084,0.893494607 +1/26/2015,0.9481285,0.89083644 +1/27/2015,0.958881095,0.848866169 +1/28/2015,0.954770185,0.853872819 +1/29/2015,0.951745037,0.854456812 +1/30/2015,0.952528875,0.876158873 +1/31/2015,0.957052535,0.865836294 +2/1/2015,0.953939998,0.888252343 +2/2/2015,0.959321121,0.875873501 +2/3/2015,0.95257587,0.840309753 +2/4/2015,0.946027883,0.918899538 +2/5/2015,0.952004285,0.837994215 +2/6/2015,0.954009289,0.828956141 +2/7/2015,0.949321415,0.827040204 +2/8/2015,0.955348723,0.83750544 +2/9/2015,0.948316767,0.853874125 +2/10/2015,0.94684589,0.906763648 +2/11/2015,0.945626478,0.933681628 +2/12/2015,0.95375805,0.918462844 +2/13/2015,0.958075797,0.869106525 +2/14/2015,0.960379351,0.848367625 +2/15/2015,0.953752535,0.877266361 +2/16/2015,0.962052757,0.889239746 +2/17/2015,0.960559333,0.874806435 +2/18/2015,0.942710863,0.845447313 +2/19/2015,0.955091752,0.84972823 +2/20/2015,0.958419036,0.846411316 +2/21/2015,0.942253464,0.871887895 +2/22/2015,0.956050596,0.842993061 +2/23/2015,0.957429687,0.854355761 +2/24/2015,0.942240373,0.878581623 +2/25/2015,0.951977628,0.845808362 +2/26/2015,0.958664898,0.85785912 +2/27/2015,0.955590387,0.85620158 +2/28/2015,0.953801945,0.880221098 +3/1/2015,0.959435591,0.851972294 +3/2/2015,0.945743911,0.827832066 +3/3/2015,0.956735793,0.835745151 +3/4/2015,0.942890443,0.837258788 +3/5/2015,0.957439113,0.848375424 +3/6/2015,0.956544482,0.853414197 +3/7/2015,0.960286485,0.849863398 +3/8/2015,0.959059268,0.880561063 +3/9/2015,0.946442958,0.880396984 +3/10/2015,0.950128908,0.893196744 +3/11/2015,0.956261343,0.837964024 +3/12/2015,0.939241783,0.81373208 +3/13/2015,0.95752241,0.845145289 +3/14/2015,0.95948074,0.840341192 +3/15/2015,0.94726523,0.918257259 +3/16/2015,0.960276024,0.845167859 +3/17/2015,0.961789376,0.832255072 +3/18/2015,0.956511156,0.835863326 +3/19/2015,0.951545429,0.866489756 +3/20/2015,0.953114834,0.89731324 +3/21/2015,0.960238852,0.837875719 +3/22/2015,0.958078585,0.821832048 +3/23/2015,0.959457254,0.823272711 +3/24/2015,0.96150857,0.82358776 +3/25/2015,0.956168765,0.830886558 +3/26/2015,0.950102627,0.854740679 +3/27/2015,0.960646911,0.855322004 +3/28/2015,0.956032425,0.846845848 +3/29/2015,0.950523841,0.837161648 +3/30/2015,0.951438849,0.831086286 +3/31/2015,0.957311008,0.817254529 +4/1/2015,0.950848583,0.838854148 +4/2/2015,0.955534296,0.833447251 +4/3/2015,0.952232217,0.839440063 +4/4/2015,0.948818569,0.846374807 +4/5/2015,0.952753599,0.825059841 +4/6/2015,0.953246917,0.820025768 +4/7/2015,0.953784693,0.815590918 +4/8/2015,0.95866706,0.813633092 +4/9/2015,0.960023665,0.809461313 +4/10/2015,0.95043633,0.836969679 +4/11/2015,0.952756073,0.809628793 +4/12/2015,0.94916986,0.831333275 +4/13/2015,0.958326221,0.831298752 +4/14/2015,0.958278004,0.82841677 +4/15/2015,0.948919998,0.822344237 +4/16/2015,0.951590684,0.838209497 +4/17/2015,0.947045876,0.836494749 +4/18/2015,0.958737654,0.836102212 +4/19/2015,0.956592548,0.870002128 +4/20/2015,0.958367845,0.828813402 +4/21/2015,0.9546648,0.83210756 +4/22/2015,0.942243573,0.86816181 +4/23/2015,0.953971675,0.832449983 +4/24/2015,0.950704225,0.836848936 +4/25/2015,0.952149086,0.843259953 +4/26/2015,0.958429613,0.842260217 +4/27/2015,0.960304731,0.833541751 +4/28/2015,0.961961626,0.825185328 +4/29/2015,0.962714676,0.818851296 +4/30/2015,0.960655443,0.829846172 +5/1/2015,0.956467774,0.822117477 +5/2/2015,0.948003987,0.822152667 +5/3/2015,0.960051018,0.825478598 +5/4/2015,0.960287737,0.829936802 +5/5/2015,0.95368672,0.827291085 +5/6/2015,0.953356322,0.839831041 +5/7/2015,0.959677792,0.839137326 +5/8/2015,0.953582684,0.851424984 +5/9/2015,0.961073107,0.832867065 +5/10/2015,0.957118193,0.845547135 +5/11/2015,0.956498313,0.81971196 +5/12/2015,0.948934469,0.825129913 +5/13/2015,0.959872461,0.823849271 +5/14/2015,0.959232557,0.82587655 +5/15/2015,0.949799962,0.838992696 +5/16/2015,0.963269406,0.830256502 +5/17/2015,0.962711621,0.827884042 +5/18/2015,0.962988827,0.824407916 +5/19/2015,0.959544421,0.821815418 +5/20/2015,0.944754735,0.835170071 +5/21/2015,0.953917163,0.821046009 +5/22/2015,0.948521116,0.88221502 +5/23/2015,0.958261597,0.84099738 +5/24/2015,0.957835194,0.855898814 +5/25/2015,0.944182585,0.851467438 +5/26/2015,0.951166035,0.825037337 +5/27/2015,0.963440268,0.845874206 +5/28/2015,0.963604329,0.838743999 +5/29/2015,0.964101527,0.83379854 +5/30/2015,0.963083164,0.834163825 +5/31/2015,0.951204604,0.846379321 +6/1/2015,0.95231761,0.871698771 +6/2/2015,0.960146976,0.877402289 +6/3/2015,0.958331179,0.862665835 +6/4/2015,0.963595762,0.857098556 +6/5/2015,0.962387188,0.875841131 +6/6/2015,0.959621633,0.881512653 +6/7/2015,0.964569624,0.876603853 +6/8/2015,0.964327269,0.878138435 +6/9/2015,0.955103342,0.860741859 +6/10/2015,0.946674496,0.820869331 +6/11/2015,0.93135625,0.778601657 +6/12/2015,0.954903017,0.900309103 +6/13/2015,0.960827369,0.946532662 +6/14/2015,0.961077518,0.90345114 +6/15/2015,0.963711002,0.892141599 +6/16/2015,0.957747639,0.890970632 +6/17/2015,0.949381079,0.893352116 +6/18/2015,0.954689322,0.859875538 +6/19/2015,0.962085705,0.899060683 +6/20/2015,0.929237336,0.807726458 +6/21/2015,0,NaN +6/22/2015,0.956612976,0.901117064 +6/23/2015,0.9619324,0.887168348 +6/24/2015,0.952371406,0.867343995 +6/25/2015,0.950204364,0.885327631 +6/26/2015,0.957664445,0.909412447 +6/27/2015,0.942038366,0.823843668 +6/28/2015,0.962855622,0.955219464 +6/29/2015,0.944292509,0.881780069 +6/30/2015,0.951742067,0.888348996 +7/1/2015,0.942508711,0.869778902 +7/2/2015,0.951572898,0.886598035 +7/3/2015,0.956623361,0.928004193 +7/4/2015,0.942905096,0.825223034 +7/5/2015,0.957710984,0.922591297 +7/6/2015,0.96351243,0.946035735 +7/7/2015,0.942311277,0.815860017 +7/8/2015,0.943380932,0.855876257 +7/9/2015,0.949469254,0.879717142 +7/10/2015,0.959181744,0.938020728 +7/11/2015,0.936771045,0.809124131 +7/12/2015,0.955906769,0.929097681 +7/13/2015,0.954813485,0.887826939 +7/14/2015,0.963809217,0.920728441 +7/15/2015,0.953039927,0.871147349 +7/16/2015,0.932110032,0.757576923 +7/17/2015,0.953028714,0.912931487 +7/18/2015,0.959220676,0.938242935 +7/19/2015,0.964623734,0.939823279 +7/20/2015,0.959180951,0.91100229 +7/21/2015,0.95587546,0.896097195 +7/22/2015,0.935883621,0.753288446 +7/23/2015,0.928093306,0.659013358 +7/24/2015,0.941950622,0.765564899 +7/25/2015,0.956280788,0.918438204 +7/26/2015,0.961490811,0.948445815 +7/27/2015,0.964066063,0.948361605 +7/28/2015,0.964927496,0.949420224 +7/29/2015,0.956582438,0.803374759 +7/30/2015,0.961206897,0.918846802 +7/31/2015,0.960298992,0.987915933 +8/1/2015,0.95043546,0.926331718 +8/2/2015,0.960113758,0.962618331 +8/3/2015,0.952166774,0.89620742 +8/4/2015,0.953663793,0.995555958 +8/5/2015,0.957548224,0.936421414 +8/6/2015,0.051724138,NaN +8/7/2015,0.96368066,1.790901545 +8/8/2015,0.963338592,1.246571592 +8/9/2015,0.931185062,0.79259661 +8/10/2015,0.873651249,NaN +8/11/2015,0.922035693,1.14946554 +8/12/2015,0.870178098,NaN +8/13/2015,0.916907185,0.357029371 +8/14/2015,0.957790972,0.961172415 +8/15/2015,0.95113671,0.876426168 +8/16/2015,0.957524549,0.852764764 +8/17/2015,0.944766555,0.829971978 +8/18/2015,0.919068736,0.693887664 +8/19/2015,0.956686291,0.894970487 +8/20/2015,0.964523282,1.078456497 +8/21/2015,0.959118774,1.042923074 +8/22/2015,0.961312027,0.934670534 +8/23/2015,0.935889594,0.596605169 +8/24/2015,0.939483141,0.756717504 +8/25/2015,0.948314007,0.816187316 +8/26/2015,0.947587736,0.944956782 +8/27/2015,0.950869698,0.811201577 +8/28/2015,0.949722159,0.948279566 +8/29/2015,0.948921464,0.955899414 +8/30/2015,0.965365669,0.995223905 +8/31/2015,0.94646098,0.857457365 +9/1/2015,0.957669962,0.935662681 +9/2/2015,0.946130101,0.942754689 +9/3/2015,0.953797611,0.902298551 +9/4/2015,0.930472264,0.735346276 +9/5/2015,0.953399656,0.843335678 +9/6/2015,0.960927004,0.944816178 +9/7/2015,0.95903776,0.941271561 +9/8/2015,0.933444568,0.748459519 +9/9/2015,0.953492674,0.942365966 +9/10/2015,0.957466363,0.973430488 +9/11/2015,0.957483492,0.881394541 +9/12/2015,0.923054396,0.351520795 +9/13/2015,0.952358387,0.134664704 +9/14/2015,0.95972182,0.548744927 +9/15/2015,0.963709969,1.027038642 +9/16/2015,0.945504283,0.933758228 +9/17/2015,0.963692946,0.989419348 +9/18/2015,0.957199971,0.956405149 +9/19/2015,0.895806726,0.029412549 +9/20/2015,0.962266817,0.496872431 +9/21/2015,0.963234778,0.977178041 +9/22/2015,0.962754424,1.012148677 +9/23/2015,0.955158509,0.977806554 +9/24/2015,0.95880072,0.959182926 +9/25/2015,0.958565627,0.902039315 +9/26/2015,0.956673751,1.077956267 +9/27/2015,0.949332152,0.79704649 +9/28/2015,0.961410535,0.95123902 +9/29/2015,0.962041973,1.03271728 +9/30/2015,0.906432749,0.027289554 +10/1/2015,0.940939105,0.078001945 +10/2/2015,0.943259584,0.100009742 +10/3/2015,0.878286431,NaN +10/4/2015,0.850947618,0.006183093 +10/5/2015,0.878689916,0.015788212 +10/6/2015,0.913175016,0.019561578 +10/7/2015,0.8681827,0.015609266 +10/8/2015,0.882913021,NaN +10/9/2015,0.868018681,NaN +10/10/2015,0.863055096,NaN +10/11/2015,0.876906042,NaN +10/12/2015,0.84913008,NaN +10/13/2015,0.876275111,NaN +10/14/2015,0.893414104,0.010878626 +10/15/2015,0.876617347,NaN +10/16/2015,0.882844674,0.010752095 +10/17/2015,0.889092688,0.012951565 +10/18/2015,0.875722192,0.013121776 +10/19/2015,0.917332123,0.242147672 +10/20/2015,0.88608052,NaN +10/21/2015,0.923611111,0.134691187 +10/22/2015,0.909624203,0.388376744 +10/23/2015,0.943159638,0.795382465 +10/24/2015,0.962687887,1.021638248 +10/25/2015,0.939347813,1.022803555 +10/26/2015,0.963099953,0.966686865 +10/27/2015,0.958794552,1.033182024 +10/28/2015,0.953004267,0.946817532 +10/29/2015,0.954415055,0.964541381 +10/30/2015,0.961527501,0.983285409 +10/31/2015,0.95094147,0.847368935 +11/1/2015,0.95908347,0.978718905 +11/2/2015,0.952403684,0.917656354 +11/3/2015,0.961605097,1.028164717 +11/4/2015,0.953689822,0.968474494 +11/5/2015,0.962698939,0.970447138 +11/6/2015,0.962707943,0.980753488 +11/7/2015,0.948850575,0.91053878 +11/8/2015,0.959633927,0.94711413 +11/9/2015,0.950010844,0.928350379 +11/10/2015,0.963011853,0.927522398 +11/11/2015,0.963533008,0.95817757 +11/12/2015,0.952632986,0.964626163 +11/13/2015,0.924127305,0.239211432 +11/14/2015,0.962343981,0.900127033 +11/15/2015,0.941458278,0.906060625 +11/16/2015,0.961069541,0.927992375 +11/17/2015,0.965074487,0.945310115 +11/18/2015,0.961628208,0.97152058 +11/19/2015,0.955668717,0.965980518 +11/20/2015,0.963153297,0.947733417 +11/21/2015,0.964800982,0.964709762 +11/22/2015,0.964778689,0.935542698 +11/23/2015,0.956110548,0.925049774 +11/24/2015,0.949889091,0.883263898 +11/25/2015,0.947347287,0.903458901 +11/26/2015,0.960437416,0.985231834 +11/27/2015,0.963897748,0.915753449 +11/28/2015,0.494091632,0.205613714 +11/29/2015,0.963431591,1.06253759 +11/30/2015,0.94712411,1.017543074 +12/1/2015,0.842545123,0.021095452 +12/2/2015,0.828397151,0.057383673 +12/3/2015,0.078347377,0 +12/4/2015,0.051724138,NaN +12/5/2015,0.939815806,0.89536963 +12/6/2015,0.955890805,0.928310429 +12/7/2015,0.919744626,0.623476678 +12/8/2015,0.875869313,NaN +12/9/2015,0.878182744,0.009115166 +12/10/2015,0.934760374,0.293583868 +12/11/2015,0.951973058,0.918009848 +12/12/2015,0.938570693,0.878956985 +12/13/2015,0.963671269,0.96725543 +12/14/2015,0.94097145,0.831732802 +12/15/2015,0.953663793,0.995555958 +12/16/2015,0.957548224,0.936421414 +12/17/2015,0.051724138,NaN +12/18/2015,0.96368066,1.790901545 +12/19/2015,0.963338592,1.246571592 +12/20/2015,0.931185062,0.79259661 +12/21/2015,0.873651249,NaN +12/22/2015,0.922035693,1.14946554 +12/23/2015,0.947559043,0.909905891 +12/24/2015,0.955339511,0.920903984 +12/25/2015,0.947362412,0.780118994 +12/26/2015,0.935192249,0.569652082 +12/27/2015,0.953452082,0.807538305 +12/28/2015,0.925236497,0.634790292 +12/29/2015,0.930805107,0.596915374 +12/30/2015,0.951678263,0.929611012 +12/31/2015,0.946746693,0.645779399 +1/1/2016,0.94091954,0.724529181 +,, +,, +,, +,, +,, +,, +,, +,, +,, +,, +,, +,, +,, +,, +,, +,, +,, +,, +,, +,, +,, +,, +,, +,, +,, +,, +,, +,, +,, +,, +,, +,, diff --git a/examples/metrics/System1_2015_performance_metrics.xlsx b/examples/metrics/System1_2015_performance_metrics.xlsx deleted file mode 100644 index 576db9f..0000000 Binary files a/examples/metrics/System1_2015_performance_metrics.xlsx and /dev/null differ diff --git a/examples/metrics/metrics_example.py b/examples/metrics/metrics_example.py index 64aecc4..2b5e03d 100644 --- a/examples/metrics/metrics_example.py +++ b/examples/metrics/metrics_example.py @@ -1,11 +1,11 @@ """ In this example, performance metrics from a pv system are analyzed to determine long term system health -* Daily performance metrics for 2015 are loaded from a csv file +* Daily performance metrics for 2015 are loaded from a CSV file * The files contain performance ratio and system availability. * The metrics are loaded into a pecos PerformanceMonitoring object and a series of quality control tests are run -* The results are printed to csv and html reports +* The results are printed to CSV and HTML reports """ import pandas as pd import matplotlib.pyplot as plt @@ -18,8 +18,8 @@ pm = pecos.monitoring.PerformanceMonitoring() # Populate the object with a dataframe and translation dictionary -data_file = 'System1_2015_performance_metrics.xlsx' -df = pd.read_excel(data_file, index_col=0) +data_file = 'System1_2015_performance_metrics.csv' +df = pd.read_csv(data_file, index_col=0, parse_dates=True) pm.add_dataframe(df) # Check timestamp diff --git a/examples/pv/pv_example.py b/examples/pv/pv_example.py index 173d7a4..8e27d33 100644 --- a/examples/pv/pv_example.py +++ b/examples/pv/pv_example.py @@ -12,7 +12,7 @@ * A performance model is computed using pvlib, additional quality control tests are run to compare observed to predicted power output * PV performance metrics are computed -* The results are written to an html report +* The results are written to an HTML report """ import yaml import pandas as pd diff --git a/examples/simple/simple.csv b/examples/simple/simple.csv new file mode 100644 index 0000000..7552ea1 --- /dev/null +++ b/examples/simple/simple.csv @@ -0,0 +1,97 @@ +,A,B,C,D +1/1/2015 0:00,0.00,0.429793393,0,-0.035103303 +1/1/2015 0:15,0.01,0.510237161,0.103978389,0.109096969 +1/1/2015 0:30,0.02,0.460397817,0.20682956,0.187028468 +1/1/2015 0:45,0.03,0.62928264,0.307438515,0.372079834 +1/1/2015 1:00,0.04,0.111771863,0.404714563,0.210600495 +1/1/2015 1:15,0.05,0.647281193,0.497603148,0.571243744 +1/1/2015 1:30,0.06,0.424915498,0.585097273,0.547555022 +1/1/2015 1:45,0.07,0.1707199,0.666248425,0.501608375 +1/1/2015 2:00,0.08,0.810439948,0.740176853,0.895396827 +1/1/2015 2:15,0.09,0.912303974,0.806081108,1.012233095 +1/1/2015 2:30,0.10,0.87598201,0.863246729,1.051237735 +1/1/2015 2:45,0.11,0.161385063,0.911053991,0.741746522 +1/1/2015 3:00,0.12,0.672271076,0.948984619,1.035120157 +1/1/2015 3:15,0.14,0.772940816,0.976627413,1.113097821 +1/1/2015 3:30,0.15,0.687847121,0.9936827,1.08760626 +1/1/2015 3:45,0.16,0.38357856,0.999965586,0.941754866 +1/1/2015 4:00,0.17,0.740154063,0.995407958,1.11548499 +1/1/2015 4:15,0.18,0.451618353,0.980059225,0.955868402 +1/1/2015 4:30,0.19,0.949575356,0.954085782,1.17887346 +1/1/2015 4:45,0.20,0.687189909,0.917769203,1.011364157 +1/1/2015 5:15,0.22,0.601406805,0.815789314,0.866492716 +1/1/2015 5:30,0.23,0.680165564,0.751231556,0.841314338 +1/1/2015 5:45,0.24,0.522002065,0.678529782,0.689530815 +1/1/2015 6:00,0.25,0.015504149,0.598472145,0.356224219 +1/1/2015 6:15,0.26,0.772375249,0.511926538,0.648114163 +1/1/2015 6:30,0.27,-0.001,0.419831194,0.171936862 +1/1/2015 6:45,0.28,0.559056037,0.323184508,0.352712526 +1/1/2015 7:00,0.29,0.847987577,0.223034215,0.397028003 +1/1/2015 7:15,0.30,0.621564606,0.120466033,0.181248336 +1/1/2015 7:30,0.31,0.7372716,-999,0.135227693 +1/1/2015 7:45,0.32,0.316489709,-999,-0.179217263 +1/1/2015 8:00,0.33,0.785302493,-999,-0.047916716 +1/1/2015 8:15,0.34,0.579974315,-999,-0.251620723 +1/1/2015 8:30,0.35,0.345476747,-999,-0.466748138 +1/1/2015 8:45,0.36,0.350357633,-999,-0.557963946 +1/1/2015 9:00,0.37,0.559362238,-999,-0.541880199 +1/1/2015 9:15,0.39,0.205868519,-999,-0.800849383 +1/1/2015 9:30,0.40,0.793399286,-999,-0.582218729 +1/1/2015 9:45,0.41,0.581206564,-0.796150981,-0.755547699 +1/1/2015 10:00,0.42,0.88546213,-0.854752607,-0.662021542 +1/1/2015 10:15,0.43,0.862051747,-0.904087957,-0.723062084 +1/1/2015 10:30,0.44,0.69936246,-0.943622192,-0.843940962 +1/1/2015 10:45,0.45,0.736740858,-0.972926726,-0.854556297 +1/1/2015 11:00,0.46,0.506848318,-0.991683872,-0.988259713 +1/1/2015 11:15,0.47,0.403798294,-0.999690285,-1.047791138 +1/1/2015 11:30,0.48,0.535661908,-0.99685917,-0.979028216 +1/1/2015 11:45,0.49,0.862132565,-0.983221217,-0.802154935 +1/1/2015 12:00,0.50,0.883275665,-0.958924275,-0.767286442 +1/1/2015 12:15,0.50,0.786830248,-0.924231744,-0.780816619 +1/1/2015 12:30,0.50,0.427837005,-0.879519721,-0.915601218 +1/1/2015 12:45,0.50,0.021097359,-0.825272924,-1.064724245 +1/1/2015 13:00,0.50,0.073028713,-0.112079438,-0.975565081 +1/1/2015 13:15,0.50,0.328392757,-0.090624334,-0.776427956 +1/1/2015 13:30,0.50,0.373087924,-0.061682251,-0.675138289 +1/1/2015 13:45,0.50,0.618797188,-0.02610899,-0.466710396 +1/1/2015 14:00,0.50,0.570540234,0.015167759,-0.399562124 +1/1/2015 14:15,0.50,0.565068395,0.061158475,-0.306307327 +1/1/2015 14:30,0.50,0.981633604,0.110822534,0.001639335 +1/1/2015 14:45,0.61,0.906677808,0.163079488,0.066418392 +1/1/2015 15:00,0.62,0.956760766,0.216820782,0.195201165 +1/1/2015 15:15,0.64,0.015060399,0.270921768,-0.171548032 +1/1/2015 15:30,0.65,1.001,0.324253899,0.421727402 +1/1/2015 15:45,0.66,0.2512004,0.375696964,0.151297164 +1/1/2015 16:00,0.67,0.643948326,0.424151229,0.446125392 +1/1/2015 16:15,0.68,0.84826817,0.468549363,0.642683448 +1/1/2015 16:30,0.69,0.908221965,0.557868007,0.761978989 +1/1/2015 16:45,0.70,0.048439506,0.641138867,0.41535862 +1/1/2015 17:00,0.71,0.181077653,0.717459213,0.557998039 +1/1/2015 17:00,0.71,0.771620858,0.717459213,0.853269642 +1/1/2015 17:15,0.72,0.088979337,0.786001665,0.580491334 +1/1/2015 17:30,0.73,0.363817745,0.846023162,0.777932035 +1/1/2015 17:45,0.74,0.648708326,0.896873019, +1/1/2015 18:00,0.75,0.832214314,0.937999976, +1/1/2015 18:15,0.76,0.694004841,0.968958183, +1/1/2015 18:30,0.77,0.568827238,0.989412023,1.023825642 +1/1/2015 18:45,0.78,0.753781269,0.99913976,1.126030395 +1/1/2015 19:00,0.79,0.178798582,0.998035937,0.837435228 +1/1/2015 19:15,0.80,0.541904069,0.986112519,1.007064553 +1/1/2015 19:45,0.82,0.722501127,0.930439834,1.041690398 +1/1/2015 19:30,0.81,0.678701932,0.963498767,1.052849733 +1/1/2015 20:00,0.83,0.102360557,0.887294109,0.688474387 +1/1/2015 20:15,0.84,0.140058519,0.834529329,0.654558588 +1/1/2015 20:30,0.85,0.23017013,0.772717511,0.637802576 +1/1/2015 20:45,0.86,0.085699622,0.70252875,0.495378561 +1/1/2015 21:00,0.87,0.742430465,0.624723955,0.745939188 +1/1/2015 21:15,0.89,0.897746216,0.540146599,0.739019707 +1/1/2015 21:30,0.90,0.062808732,0.449713574,0.23111794 +1/1/2015 21:45,0.91,0.153075676,0.354405256,0.180943093 +1/1/2015 22:00,0.92,0.576533359,0.25525487,0.293521549 +1/1/2015 22:15,0.93,0.003205705,0.153337295,-0.095059852 +1/1/2015 22:30,0.94,0.71136348,0.049757408,0.155439148 +1/1/2015 22:45,0.95,0.951160038,-0.054361893,0.171218126 +1/1/2015 23:00,0.96,0.028180644,-0.157891863,-0.393801541 +1/1/2015 23:15,0.97,0.154533482,-0.259710145,-0.432443404 +1/1/2015 23:30,0.98,0.543068484,-0.35871294,-0.337178698 +1/1/2015 23:45,0.99,0.585191477,-0.453826968,-0.411231229 diff --git a/examples/simple/simple.xlsx b/examples/simple/simple.xlsx deleted file mode 100644 index c5fa164..0000000 Binary files a/examples/simple/simple.xlsx and /dev/null differ diff --git a/examples/simple/simple_example.py b/examples/simple/simple_example.py index 13183ae..c959ebf 100644 --- a/examples/simple/simple_example.py +++ b/examples/simple/simple_example.py @@ -1,14 +1,14 @@ """ In this example, simple time series data is used to demonstrate basic functions in pecos. -* Data is loaded from an excel file which contains four columns of values that +* Data is loaded from a CSV file which contains four columns of values that are expected to follow linear, random, and sine models. * A translation dictionary is defined to map and group the raw data into common names for analysis * A time filter is established to screen out data between 3 AM and 9 PM * The data is loaded into a pecos PerformanceMonitoring object and a series of quality control tests are run, including range tests and increment tests -* The results are printed to csv and html reports +* The results are printed to CSV and HTML reports """ import pandas as pd import matplotlib.pyplot as plt @@ -22,8 +22,8 @@ pm = pecos.monitoring.PerformanceMonitoring() # Populate the object with a DataFrame and translation dictionary -data_file = 'simple.xlsx' -df = pd.read_excel(data_file, index_col=0) +data_file = 'simple.csv' +df = pd.read_csv(data_file, index_col=0, parse_dates=True) pm.add_dataframe(df) pm.add_translation_dictionary({'Wave': ['C','D']}) # group C and D diff --git a/examples/simple/simple_example_using_config.py b/examples/simple/simple_example_using_config.py index 2cc50e6..0a806ea 100644 --- a/examples/simple/simple_example_using_config.py +++ b/examples/simple/simple_example_using_config.py @@ -20,8 +20,8 @@ pm = pecos.monitoring.PerformanceMonitoring() # Populate the object with a DataFrame and translation dictionary -data_file = 'simple.xlsx' -df = pd.read_excel(data_file, index_col=0) +data_file = 'simple.csv' +df = pd.read_csv(data_file, index_col=0, parse_dates=True) pm.add_dataframe(df) pm.add_translation_dictionary(config['Translation']) diff --git a/pecos/__init__.py b/pecos/__init__.py index cc7687e..e617273 100644 --- a/pecos/__init__.py +++ b/pecos/__init__.py @@ -6,7 +6,7 @@ from pecos import utils from pecos import pv -__version__ = '0.1.9' +__version__ = '0.2.0' __copyright__ = """Copyright 2016 National Technology & Engineering Solutions of Sandia, LLC (NTESS). Under the terms of Contract diff --git a/pecos/graphics.py b/pecos/graphics.py index 39ff745..76a0bf1 100644 --- a/pecos/graphics.py +++ b/pecos/graphics.py @@ -305,9 +305,8 @@ def plot_interactive_timeseries(data, xaxis_min=None, xaxis_max=None, yaxis_min= else: plotly.offline.plot(fig, auto_open=auto_open) -def plot_heatmap(data, colors=[(0.75, 0.15, 0.15), (1, 0.75, 0.15), (0.15, 0.75, 0.15)], - nColors=12, cmap=None, vmin=None, vmax=None, - show_axis=False, title=None, figsize=(5.0, 5.0)): +def plot_heatmap(data, colors=None, nColors=12, cmap=None, vmin=None, vmax=None, + show_axis=False, title=None, figsize=(5.0, 5.0)): """ Create a heatmap. Default color scheme is red to yellow to green with 12 colors. This function can be used to generate dashboards with simple color @@ -319,10 +318,10 @@ def plot_heatmap(data, colors=[(0.75, 0.15, 0.15), (1, 0.75, 0.15), (0.15, 0.75, data : pandas DataFrame, pandas Series, or numpy array Data - colors : list, optional + colors : list or None, optional List of colors, colors can be specified in any way understandable by matplotlib.colors.ColorConverter.to_rgb(). - Default is red to yellow to green. + If None, colors transitions from red to yellow to green. num_colors : int, optional Number of colors in the colormap, default = 12 @@ -342,8 +341,11 @@ def plot_heatmap(data, colors=[(0.75, 0.15, 0.15), (1, 0.75, 0.15), (0.15, 0.75, figsize : tuple, optional Figure size, default = (5.0, 5.0) """ + if colors is None: + colors = [(0.75, 0.15, 0.15), (1, 0.75, 0.15), (0.15, 0.75, 0.15)] + if isinstance(data, (pd.DataFrame, pd.Series)): - data = data.values + data = data.values if len(data.shape) == 1: data = np.expand_dims(data, axis=0) diff --git a/pecos/io.py b/pecos/io.py index beb18d4..888a569 100644 --- a/pecos/io.py +++ b/pecos/io.py @@ -217,9 +217,9 @@ def write_test_results(test_results, filename='test_results.csv'): return full_filename -def write_monitoring_report(data, test_results, test_results_graphics=[], - custom_graphics=[], metrics=None, - title='Pecos Monitoring Report', config={}, logo=False, +def write_monitoring_report(data, test_results, test_results_graphics=None, + custom_graphics=None, metrics=None, + title='Pecos Monitoring Report', config=None, logo=False, im_width_test_results=1, im_width_custom=1, im_width_logo=0.1, encode=False, file_format='html', filename='monitoring_report.html'): @@ -236,12 +236,14 @@ def write_monitoring_report(data, test_results, test_results_graphics=[], test_results : pandas DataFrame Summary of the quality control test results (pm.test_results) - test_results_graphics : list of strings, optional + test_results_graphics : list of strings or None, optional Graphics files, with full path. These graphics highlight data points - that failed a quality control test, created using pecos.graphics.plot_test_results() + that failed a quality control test, created using pecos.graphics.plot_test_results(). + If None, test results graphics are not included in the report. - custom_graphics : list of strings, optional + custom_graphics : list of strings or None, optional Custom files, with full path. Created by the user. + If None, custom graphics are not included in the report. metrics : pandas Series or DataFrame, optional Performance metrics to add as a table to the monitoring report @@ -249,8 +251,9 @@ def write_monitoring_report(data, test_results, test_results_graphics=[], title : string, optional Monitoring report title, default = 'Pecos Monitoring Report' - config : dictionary, optional - Configuration options, to be printed at the end of the report + config : dictionary or None, optional + Configuration options, to be printed at the end of the report. + If None, configuration options are not included in the report. logo : string, optional Graphic to be added to the report header @@ -278,6 +281,13 @@ def write_monitoring_report(data, test_results, test_results_graphics=[], """ logger.info("Writing HTML report") + if test_results_graphics is None: + test_results_graphics = [] + if custom_graphics is None: + custom_graphics = [] + if config is None: + config = {} + if data.empty: logger.warning("Empty database") start_time = 'NaN' @@ -308,8 +318,6 @@ def write_monitoring_report(data, test_results, test_results_graphics=[], if metrics is None: metrics = pd.DataFrame() - pecos_logo = join(dirname(pecos.__file__), '..', 'documentation', 'figures', 'logo.png') - content = {'start_time': str(start_time), 'end_time': str(end_time), 'num_notes': str(notes_df.shape[0]), @@ -323,7 +331,6 @@ def write_monitoring_report(data, test_results, test_results_graphics=[], if file_format == 'html': content['test_results_graphics'] = test_results_graphics content['custom_graphics'] = custom_graphics - content['pecos_logo'] = pecos_logo if isinstance(metrics, pd.Series): metrics_html = metrics.to_frame().to_html(header=False) @@ -343,11 +350,9 @@ def write_monitoring_report(data, test_results, test_results_graphics=[], else: test_results_graphics = [g.replace('\\', '/') for g in test_results_graphics] custom_graphics = [g.replace('\\', '/') for g in custom_graphics] - pecos_logo = pecos_logo.replace('\\', '/') content['test_results_graphics'] = test_results_graphics content['custom_graphics'] = custom_graphics - content['pecos_logo'] = pecos_logo content['metrics'] = metrics.to_latex(longtable=True) content['test_results'] = test_results.to_latex(longtable=True) @@ -477,9 +482,6 @@ def _html_template_monitoring_report(content, title, logo, im_width_test_results for im in content['test_results_graphics']: img_encode = base64.b64encode(open(im, "rb").read()).decode("utf-8") img_dic[im] = img_encode - im = content['pecos_logo'] - img_encode = base64.b64encode(open(im, "rb").read()).decode("utf-8") - img_dic[im] = img_encode template = env.get_template('monitoring_report.html') diff --git a/pecos/logger.py b/pecos/logger.py index f7544cd..f689c82 100644 --- a/pecos/logger.py +++ b/pecos/logger.py @@ -21,11 +21,9 @@ def initialize(): ch = logging.StreamHandler() ch.setLevel(logging.INFO) - formatter = logging.Formatter('%(message)s') + formatter = logging.Formatter("%(asctime)s : %(message)s", "%Y-%m-%d %H:%M:%S") fh.setFormatter(formatter) ch.setFormatter(formatter) if not len(pecos_logger.handlers): pecos_logger.addHandler(fh) pecos_logger.addHandler(ch) - - diff --git a/pecos/monitoring.py b/pecos/monitoring.py index 90e4056..70f3bb3 100644 --- a/pecos/monitoring.py +++ b/pecos/monitoring.py @@ -666,7 +666,7 @@ def outlier(data_pt, history): error_prefix = 'Outlier' if streaming: - metadata = self.check_custom_streaming(outlier, window, rebase=0.5, error_message=error_prefix) + metadata = self.check_custom_streaming(outlier, window, rebase=0.5, min_failures=min_failures, error_message=error_prefix) else: # Compute normalized data if window is not None: diff --git a/pecos/templates/base.html b/pecos/templates/base.html index a156d3e..3eed23d 100644 --- a/pecos/templates/base.html +++ b/pecos/templates/base.html @@ -7,7 +7,7 @@ {% block body %} -
Report generated by Pecos
+
Report generated by Pecos
Version {{ version }}, Date {{ datestr }} {% endblock %} diff --git a/pecos/templates/monitoring_report.tex b/pecos/templates/monitoring_report.tex index 7d765c3..a4e1cc1 100644 --- a/pecos/templates/monitoring_report.tex +++ b/pecos/templates/monitoring_report.tex @@ -45,7 +45,11 @@ {% if content['num_metrics']|int > 0 %} \textbf{\Large{Performance Metrics}} + + \vspace{0.1in} + \small{\centering{ {{ content['metrics'] }} + }} {% endif %} \textbf{\Large{Test Results}} @@ -77,7 +81,7 @@ \underline{\hspace{\linewidth}} \footnotesize{ -Report generated by \href{https://pecos.readthedocs.io/}{\includegraphics[height=0.26cm]{% raw -%}{{%- endraw %}{{content['pecos_logo']}}{% raw -%}}{%- endraw %} } +Report generated by \href{https://github.com/sandialabs/pecos}{Pecos} Version {{ version }}, Date {{ datestr }} } diff --git a/pecos/tests/test_monitoring.py b/pecos/tests/test_monitoring.py index 599d43e..784e141 100644 --- a/pecos/tests/test_monitoring.py +++ b/pecos/tests/test_monitoring.py @@ -77,9 +77,9 @@ def setUp(self): 'Random': ['B'], 'Wave': ['C','D']} - file_name = join(simpleexampledir,'simple.xlsx') + file_name = join(simpleexampledir,'simple.csv') - self.raw_data = pd.read_excel(file_name, index_col=0) + self.raw_data = pd.read_csv(file_name, index_col=0, parse_dates=True) self.pm = pecos.monitoring.PerformanceMonitoring() self.pm.add_dataframe(self.raw_data) self.pm.add_translation_dictionary(trans) @@ -264,8 +264,8 @@ def test_composite_signal(self): assert_frame_equal(temp, expected, check_dtype=False) def test_full_example(self): - data_file = join(simpleexampledir,'simple.xlsx') - df = pd.read_excel(data_file, index_col=0) + data_file = join(simpleexampledir,'simple.csv') + df = pd.read_csv(data_file, index_col=0, parse_dates=True) QCI = simple_example_run_analysis(df) @@ -284,8 +284,8 @@ def test_full_example(self): assert_frame_equal(actual, expected, check_dtype=False) def test_millisecond_timestamp(self): - data_file = join(simpleexampledir,'simple.xlsx') - df = pd.read_excel(data_file, index_col=0) + data_file = join(simpleexampledir,'simple.csv') + df = pd.read_csv(data_file, index_col=0, parse_dates=True) index = pecos.utils.datetime_to_elapsedtime(df.index) df.index = index/1e5 # millisecond resolution @@ -308,8 +308,8 @@ def test_millisecond_timestamp(self): assert_frame_equal(pm.test_results, expected, check_dtype=False) def test_full_example_with_timezone(self): - data_file = join(simpleexampledir,'simple.xlsx') - df = pd.read_excel(data_file, index_col=0) + data_file = join(simpleexampledir,'simple.csv') + df = pd.read_csv(data_file, index_col=0, parse_dates=True) df.index = df.index.tz_localize('MST') QCI = simple_example_run_analysis(df)