From d76e299eb92f41205438943def62a6741b910fa3 Mon Sep 17 00:00:00 2001 From: romfabbro Date: Fri, 13 Apr 2018 12:10:27 +0200 Subject: [PATCH] verify sensor identifier in file --- .../modules/import/importRFID.py | 416 ++++++++++++------ .../modules/importFile/rfid/lyt-step1-rfid.js | 175 ++++---- 2 files changed, 368 insertions(+), 223 deletions(-) diff --git a/Back/ecoreleve_server/modules/import/importRFID.py b/Back/ecoreleve_server/modules/import/importRFID.py index ad5162ae0..49bc4b335 100644 --- a/Back/ecoreleve_server/modules/import/importRFID.py +++ b/Back/ecoreleve_server/modules/import/importRFID.py @@ -1,156 +1,305 @@ import re import itertools import pandas as pd +import time +import codecs +import shutil +import os from datetime import datetime from traceback import print_exc from pyramid import threadlocal -from sqlalchemy import select, and_ +from sqlalchemy import select, and_, join from sqlalchemy.exc import IntegrityError - from ecoreleve_server.core import dbConfig from ..sensors.sensor_data import Rfid +from ..sensors import Sensor from .import_model import Import -def uploadFileRFID(request): - session = request.dbsession +def convertAntennaID(val): + return 'DU-' + format(int(val), '03d') + + +def parse_date(str_date, str_time): + + # default datetime format + format_dt = '%d/%m/%Y %H:%M:%S' + dt = str_date + ' ' + re.sub('\s', '', str_time) + + # english format + if re.search('PM|AM', str_time): + format_dt = '%m/%d/%Y %I:%M:%S%p' + format_dtBis = '%d/%m/%Y %I:%M:%S%p' + try: + dt = datetime.strptime(dt, format_dt) + except Exception as e: + dt = datetime.strptime(dt, format_dtBis) + + return dt + + +def getDataFrameFromFile(input_file, creator): + + # multiple header for RFID files + fieldtype1 = {'NB': 'no', 'TYPE': 'type', + '"PUCE "': 'chip_code', 'DATE': 'no', 'TIME': 'no'} + fieldtype2 = {'#': 'no', 'Transponder Type:': 'type', + 'Transponder Code:': 'chip_code', 'Date:': 'no', 'Time:': 'no', + 'Event:': 'Event', 'Unit #:': 'Unit', 'Antenna #:': 'Antenna', + 'Memo:': 'Memo', 'Custom:': 'Custom', '': ''} + fieldtype3 = {'Transponder Type:': 'type', 'Transponder Code:': 'chip_code', + 'Date:': 'no', 'Time:': 'no', + 'Event:': 'Event', 'Unit #:': 'Unit', + 'Antenna #:': 'Antenna', 'Memo:': 'Memo', + 'Custom:': 'Custom'} + + fieldDict = {'TYPE': 'type', 'Transponder Type:': 'type', + 'Transponder Code:': 'chip_code', '"PUCE "': 'chip_code', + 'Date:': 'Date', 'Time:': 'Time', + 'DATE': 'Date', 'TIME': 'Time', + 'Unit #:': 'Unit', + 'Antenna #:': 'Antenna' + } + + # blob = request.POST['file'] + # input_file = blob.file + + try: + df = pd.read_csv(input_file, + sep=';', + header=0, + ) + if len(df.columns) <= 1: + df = pd.read_csv(input_file, + sep='\t', + header=0, + ) + except: + # weird behaviour in files using tab separation, some hack below + input_file.seek(0) + temp_filename = 'temp_rfid' + str(int(time.time())) + '.txt' + with open(temp_filename, 'wb') as output_file: + shutil.copyfileobj(input_file, output_file) + doc = codecs.open(temp_filename, 'rU', 'UTF-8') + df = pd.read_csv(doc, + sep='\t', + encoding='utf8', + header=0, + ) + os.remove(temp_filename) + df = df.rename(columns=fieldDict) + df['date_'] = df.apply( + lambda row: parse_date(row.Date, row.Time), axis=1) + df['identifier'] = df['Unit'].apply( + lambda row: convertAntennaID(row)) + + return df + + +def getDataFrame(input_file, creator): + import time + import shutil data = [] message = "" field_label = [] isHead = False - try: - creator = int(request.authenticated_userid['iss']) - content = request.POST['data'] - idModule = int(request.POST['FK_Sensor']) - startEquip = request.POST['StartDate'] - endEquip = request.POST['EndDate'] - filename = request.POST['fileName'] - now = datetime.now() - - if re.compile('\r\n').search(content): - data = content.split('\r\n') - elif re.compile('\n').search(content): - data = content.split('\n') - elif re.compile('\r').search(content): - data = content.split('\r') - - fieldtype1 = {'NB': 'no', 'TYPE': 'type', - '"PUCE "': 'code', 'DATE': 'no', 'TIME': 'no'} - fieldtype2 = {'#': 'no', 'Transponder Type:': 'type', - 'Transponder Code:': 'code', 'Date:': 'no', 'Time:': 'no', - 'Event:': 'Event', 'Unit #:': 'Unit', 'Antenna #:': 'Antenna', - 'Memo:': 'Memo', 'Custom:': 'Custom', '': ''} - fieldtype3 = {'Transponder Type:': 'type', 'Transponder Code:': 'code', - 'Date:': 'no', 'Time:': 'no', - 'Event:': 'Event', 'Unit #:': 'Unit', - 'Antenna #:': 'Antenna', 'Memo:': 'Memo', - 'Custom:': 'Custom'} - - entete = data[0] - - if re.compile('\t').search(entete): - separateur = '\t' - - elif re.compile(';').search(entete): - separateur = ';' - entete = entete.split(separateur) - - # file with head - if (sorted(entete) == sorted(fieldtype1.keys())): - field_label = ["no", "Type", "Code", "date", "time"] - isHead = True - - elif (sorted(entete) == sorted(fieldtype2.keys())): - field_label = ["no", "Type", "Code", "date", - "time", "no", "no", "no", "no", "no"] - isHead = True + now = datetime.now() + + input_file.seek(0) + temp_fielname = 'temp_rfid' + str(int(time.time())) + '.txt' + with open(temp_fielname, 'wb') as output_file: + shutil.copyfileobj(input_file, output_file) + + content = open(temp_fielname).read() + + # NEED a big Refact !! + if re.compile('\r\n').search(content): + data = content.split('\r\n') + elif re.compile('\n').search(content): + data = content.split('\n') + elif re.compile('\r').search(content): + data = content.split('\r') + + fieldtype1 = {'NB': 'no', 'TYPE': 'type', + '"PUCE "': 'code', 'DATE': 'no', 'TIME': 'no'} + fieldtype2 = {'#': 'no', 'Transponder Type:': 'type', + 'Transponder Code:': 'code', 'Date:': 'no', 'Time:': 'no', + 'Event:': 'Event', 'Unit #:': 'Unit', 'Antenna #:': 'Antenna', + 'Memo:': 'Memo', 'Custom:': 'Custom', '': ''} + fieldtype3 = {'Transponder Type:': 'type', 'Transponder Code:': 'code', + 'Date:': 'no', 'Time:': 'no', + 'Event:': 'Event', 'Unit #:': 'Unit', + 'Antenna #:': 'Antenna', 'Memo:': 'Memo', + 'Custom:': 'Custom'} + + entete = data[0] + + if re.compile('\t').search(entete): + separateur = '\t' + + elif re.compile(';').search(entete): + separateur = ';' + entete = entete.split(separateur) - elif (sorted(entete) == sorted(fieldtype3.keys())): - field_label = ["Type", "Code", "date", + # file with head + if (sorted(entete) == sorted(fieldtype1.keys())): + field_label = ["no", "Type", "Code", "date", "time"] + isHead = True + + elif (sorted(entete) == sorted(fieldtype2.keys())): + field_label = ["no", "Type", "Code", "date", + "time", "no", "no", "no", "no", "no"] + isHead = True + + elif (sorted(entete) == sorted(fieldtype3.keys())): + field_label = ["Type", "Code", "date", + "time", "no", "no", "no", "no", "no"] + isHead = True + else: # without head + isHead = False + if separateur == ';': + field_label = ["no", "Type", "Code", "date", "time", "no", "no", "no", "no", "no"] - isHead = True - else: # without head - isHead = False - if separateur == ';': - field_label = ["no", "Type", "Code", "date", + else: + if len(entete) > 5: + field_label = ["Type", "Code", "date", "time", "no", "no", "no", "no", "no"] + if entete[0] == 'Transponder Type:': + isHead = True + elif entete[1] == 'Transponder Type:': + isHead = True + field_label = ["no", "Type", "Code", "date", "time"] else: - if len(entete) > 5: - field_label = ["Type", "Code", "date", - "time", "no", "no", "no", "no", "no"] - if entete[0] == 'Transponder Type:': - isHead = True - elif entete[1] == 'Transponder Type:': - isHead = True - field_label = ["no", "Type", "Code", "date", "time"] - else: - field_label = ["no", "Type", "Code", "date", "time"] - - j = 0 - code = "" - date = "" - dt = "" - list_RFID = [] - - if (isHead): - j = 1 - # Parsing data - allDate = [] - while j < len(data): - i = 0 - if data[j] != "": - line = data[j].replace('"', '').split(separateur) - while i < len(field_label): - if field_label[i] == 'Code': - code = line[i] - if field_label[i] == 'date': - date = line[i] - if field_label[i] == 'time': - time = re.sub('\s', '', line[i]) - format_dt = '%d/%m/%Y %H:%M:%S' - if re.search('PM|AM', time): - format_dt = '%m/%d/%Y %I:%M:%S%p' - format_dtBis = '%d/%m/%Y %I:%M:%S%p' - dt = date + ' ' + time - try: - dt = datetime.strptime(dt, format_dt) - except Exception as e: - dt = datetime.strptime(dt, format_dtBis) - allDate.append(dt) - i = i + 1 - - row = {'id_': j, - 'FK_Sensor': idModule, - 'date_': dt, - 'chip_code': code, - 'creator': creator, - 'creation_date': now, - 'validated': 0, - 'checked': 0} - list_RFID.append(row) - j = j + 1 - data_to_check = pd.DataFrame.from_records( - list_RFID, - columns=['id_', 'FK_Sensor', 'date_', - 'chip_code', 'creator', 'creation_date', - 'validated', 'checked']) + field_label = ["no", "Type", "Code", "date", "time"] + + j = 0 + code = "" + date = "" + dt = "" + list_RFID = [] + + if (isHead): + j = 1 + # Parsing data + allDate = [] + while j < len(data): + i = 0 + if data[j] != "": + line = data[j].replace('"', '').split(separateur) + while i < len(field_label): + if field_label[i] == 'Code': + code = line[i] + if field_label[i] == 'Antenna': + identifier = convertAntennaID(line[i]) + if field_label[i] == 'date': + date = line[i] + if field_label[i] == 'time': + time = re.sub('\s', '', line[i]) + format_dt = '%d/%m/%Y %H:%M:%S' + if re.search('PM|AM', time): + format_dt = '%m/%d/%Y %I:%M:%S%p' + format_dtBis = '%d/%m/%Y %I:%M:%S%p' + dt = date + ' ' + time + try: + dt = datetime.strptime(dt, format_dt) + except Exception as e: + dt = datetime.strptime(dt, format_dtBis) + allDate.append(dt) + i = i + 1 + + row = {'id_': j, + # 'FK_Sensor': idModule, + 'identifier': identifier, + 'date_': dt, + 'chip_code': code, + 'creator': creator, + 'creation_date': now, + 'validated': 0, + 'checked': 0} + list_RFID.append(row) + j = j + 1 + df = pd.DataFrame.from_records( + list_RFID, + columns=['id_', 'identifier', 'date_', + 'chip_code', 'creator', 'creation_date', + 'validated', 'checked']) + return df + + +def findEquipmentSession(session, identifier, maxDate, minDate): + table = Base.metadata.tables['SensorEquipment'] + joinTable = join(table, Sensor, table.c['FK_Sensor'] == Sensor.ID) + + query = select([table.c['StartDate'], + table.c['EndDate'], + Sensor.ID] + ).select_from(joinTable + ).where(Sensor.UnicIdentifier == identifier + ).where(table.c['StartDate'] <= minDate + ).where(table.c['EndDate'] >= maxDate + ) + equipSession = session.execute(query).fetchone() + if equipSession: + equipSession = dict(equipSession) + return equipSession + + +def uploadFileRFID(request): + session = request.dbsession + data = [] + message = "" + field_label = [] + isHead = False + now = datetime.now() + creator = int(request.authenticated_userid['iss']) + # content = request.POST['data'] + # blob = request.POST['file'] + blob = request.POST['file'] + input_file = blob.file + filename = blob.filename + # filename = request.POST['fileName'] + + idModule = int(request.POST['FK_Sensor']) + startEquip = request.POST['StartDate'] + endEquip = request.POST['EndDate'] + + try: + data_to_check = getDataFrameFromFile(input_file, creator) + sensorIdentifier = data_to_check.iloc[0].identifier + + sensor_from_request = session.query(Sensor).get(idModule) + + allDate = list(data_to_check['date_']) minDateEquip = datetime.strptime(startEquip, '%Y-%m-%d %H:%M:%S') try: maxDateEquip = datetime.strptime(endEquip, '%Y-%m-%d %H:%M:%S') except: maxDateEquip = None + # print('date from file : ', min(allDate), max(allDate)) + # print('date Equipment : ', minDateEquip, maxDateEquip) + + # print('sensor Identifier (file|selected): ',sensorIdentifier, sensor_from_request.UnicIdentifier) # check if Date corresponds with pose remove module if (min(allDate) >= minDateEquip and - (maxDateEquip is None or max(allDate) <= maxDateEquip)): + (maxDateEquip is None or max(allDate) <= maxDateEquip) and + sensorIdentifier == sensor_from_request.UnicIdentifier + ): + data_to_check.loc[:, ('FK_Sensor')] = list( + itertools.repeat(idModule, len(data_to_check.index))) + data_to_check.loc[:, ('id_')] = data_to_check.index data_to_insert = checkDuplicatedRFID( data_to_check, min(allDate), max(allDate), idModule) - data_to_insert = data_to_insert.drop(['id_'], 1) + # data_to_insert = data_to_insert.drop(['id_'], 1) data_to_insert = data_to_insert.drop_duplicates() + data_to_insert['validated'] = 0 + data_to_insert['checked'] = 0 - importObj = Import(ImportFileName=filename, FK_User=creator, ImportDate=now) + + importObj = Import(ImportFileName=filename, + FK_User=creator, ImportDate=now) importObj.ImportType = 'RFID' importObj.maxDate = max(allDate) importObj.minDate = min(allDate) @@ -160,9 +309,11 @@ def uploadFileRFID(request): session.add(importObj) session.flush() - data_to_insert.loc[:, ('FK_Import')] = list(itertools.repeat(importObj.ID, len(data_to_insert.index))) - if data_to_insert.shape[0] == 0: - raise(IntegrityError) + data_to_insert.loc[:, ('FK_Import')] = list( + itertools.repeat(importObj.ID, len(data_to_insert.index))) + + # if data_to_insert.shape[0] == 0: + # raise(IntegrityError) data_to_insert.to_sql( Rfid.__table__.name, @@ -175,13 +326,18 @@ def uploadFileRFID(request): return message else: session.rollback() - request.response.status_code = 510 - message = 'File dates (first date : ' + str(allDate[0])+ ', last date : ' + str(allDate[-1])+ ') do not correspond with the deploy/remove dates of the selected module' + request.response.status_code = 409 + + if sensorIdentifier != sensor_from_request.UnicIdentifier: + message = 'Identifier in file ({identifierFile}) does not correspond with the selected module ({identifierSelect})'.format(identifierFile=sensorIdentifier, identifierSelect=sensor_from_request.UnicIdentifier) + else: + message = 'File dates (first date : ' + str(allDate[0]) + ', last date : ' + str( + allDate[-1]) + ') do not correspond with the deploy/remove dates of the selected module' return message except IntegrityError as e: print_exc() - request.response.status_code = 520 + request.response.status_code = 409 message = 'Data already exist.' except Exception as e: print_exc() @@ -192,18 +348,15 @@ def uploadFileRFID(request): def checkDuplicatedRFID(data_to_check, startEquip, endEquip, fk_sensor): session1 = threadlocal.get_current_registry().dbmaker() + query = select([Rfid] ).where( and_(Rfid.date_ >= startEquip, and_( Rfid.date_ <= endEquip, Rfid.FK_Sensor == fk_sensor)) ) - result = session1.execute(query).fetchall() + # result = session1.execute(query).fetchall() - existingData = pd.DataFrame.from_records( - result, - columns=['ID', 'FK_Sensor', 'date_', - 'chip_code', 'creator', 'creation_date', - 'validated', 'checked', 'frequency']) + existingData = pd.read_sql(query, session1.get_bind()) existingData.rename(columns={'ID': '$ID', 'FK_Sensor': '$FK_Sensor', 'date_': '$date_', 'chip_code': '$chip_code', 'creator': '$creator', 'creation_date': '$creation_date', @@ -218,4 +371,5 @@ def checkDuplicatedRFID(data_to_check, startEquip, endEquip, fk_sensor): DFToInsert = data_to_check[~data_to_check['id_'].isin(merge['id_'])] + DFToInsert = DFToInsert[['FK_Sensor','date_', 'chip_code']] return DFToInsert diff --git a/Front/app/modules/importFile/rfid/lyt-step1-rfid.js b/Front/app/modules/importFile/rfid/lyt-step1-rfid.js index 558148129..9f597b49f 100644 --- a/Front/app/modules/importFile/rfid/lyt-step1-rfid.js +++ b/Front/app/modules/importFile/rfid/lyt-step1-rfid.js @@ -37,10 +37,11 @@ define([ initialize: function(options) { this.model = new Backbone.Model(); this.importedFiles = options.model.attributes.files; + this.parent = options.parent; }, check: function() { - if (this.ui.requirement.val()) { + if (this.row) { return true; } else { return false; @@ -73,6 +74,7 @@ define([ }, handleRfidSelection: function(e){ + this.row = null; var id = $(e.currentTarget).val(); this.displayGrid(id); }, @@ -114,7 +116,8 @@ define([ onRowSelected: function(row){ _this.model.set('row', row.node.data); _this.row = row.node.data; - _this.ui.requirement.val('check').change(); + _this.parent.checkNextBtn(); + // _this.ui.requirement.val('check').change(); } } })); @@ -124,101 +127,89 @@ define([ var _this = this; var data = new FormData(); this.reader = new FileReader(); + // this.reader.readAsText(file); + + var row = this.gridView.gridOptions.api.getSelectedRows()[0]; + data.append('file',file, file.name) + data.append('FK_Sensor',_this.model.get('sensorId')); + data.append('StartDate', row.StartDate); + data.append('EndDate', row.EndDate); this.reader.onload = function(e) { data.append('data', e.target.result); data.append('fileName', file.name) data.append('FK_Sensor',_this.model.get('sensorId')); - data.append('StartDate', _this.row.StartDate); - data.append('EndDate', _this.row.EndDate); - - $.ajax({ - type: 'POST', - url: 'sensors/rfid/datas', - data: data, - processData: false, - contentType: false - }).done(function(data) { - $('.cancel').removeAttr('disabled'); - - //self.ui.progressBar.css({'background-color': 'green'}) - // swal( - // { - // title: 'Succes', - // text: 'importing RFID file', - // type: 'success', - // showCancelButton: true, - // confirmButtonColor: 'green', - // confirmButtonText: 'Go to Validate', - // cancelButtonText: 'Import new RFID', - // closeOnConfirm: true, - - // }, - // function(isConfirm) { - // //self.ui.progress.hide(); - // if (isConfirm) { - // Backbone.history.navigate('validate/rfid',{trigger: true}); - // } else { - // //Backbone.history.navigate('importFile',{trigger: true}); - // _this.options.parent.currentStepIndex = 1; - // var index = _this.options.parent.currentStepIndex; - // _this.options.parent.displayStep(index); - // } - // } - // ); - swal( - { - title: 'Succes', - text: 'importing RFID file', - type: 'success', - showCancelButton: true, - confirmButtonColor: 'green', - confirmButtonText: 'Go to Validate', - cancelButtonText: 'Import new RFID', - - }) - .then( (result) => { - if( 'value' in result) { - Backbone.history.navigate('validate/rfid',{trigger: true}); - } - else { - //Backbone.history.navigate('importFile',{trigger: true}); - _this.options.parent.currentStepIndex = 1; - var index = _this.options.parent.currentStepIndex; - _this.options.parent.displayStep(index); - } - }); - - }).fail(function(data) { - $('#btnNext').attr('disabled'); - if (data.status == 520 || data.status == 510) { - var type = 'warning'; - var title = 'Warning !' - //self.ui.progressBar.css({'background-color': 'rgb(218, 146, 15)'}) - var color = 'rgb(218, 146, 15)'; + data.append('StartDate', row.StartDate); + data.append('EndDate', row.EndDate); + }; + + $.ajax({ + type: 'POST', + url: 'sensors/rfid/datas', + data: data, + processData: false, + contentType: false + }).done(function(data) { + $('.cancel').removeAttr('disabled'); + + //self.ui.progressBar.css({'background-color': 'green'}) + swal( + { + title: 'Succes', + text: 'importing RFID file', + type: 'success', + showCancelButton: true, + confirmButtonColor: 'green', + confirmButtonText: 'Go to Validate', + cancelButtonText: 'Import new RFID', + closeOnConfirm: true, + + }, + function(isConfirm) { + //self.ui.progress.hide(); + if (isConfirm) { + Backbone.history.navigate('validate/rfid',{trigger: true}); } else { - var type = 'error'; - var title = 'Error !' - //self.ui.progressBar.css({'background-color': 'rgb(147, 14, 14)'}) - var color = 'rgb(147, 14, 14)'; - //_this.clearFile(); - + //Backbone.history.navigate('importFile',{trigger: true}); + _this.options.parent.currentStepIndex = 1; + var index = _this.options.parent.currentStepIndex; + _this.options.parent.displayStep(index); } - if (data.responseText.length > 250) { - data.responseText = 'An error occured, please contact an admninstrator'; - } - swal( - { - title: title, - text: data.responseText, - type: type, - showCancelButton: false, - confirmButtonColor: color, - confirmButtonText: 'OK' - } - ); - }); - }; - this.reader.readAsText(file); + } + ); + + }).fail(function(data) { + $('#btnNext').attr('disabled'); + _this.row = null; + if (data.status == 520 || data.status == 510) { + var type = 'warning'; + var title = 'Warning !' + //self.ui.progressBar.css({'background-color': 'rgb(218, 146, 15)'}) + var color = 'rgb(218, 146, 15)'; + } else { + var type = 'error'; + var title = 'Error !' + //self.ui.progressBar.css({'background-color': 'rgb(147, 14, 14)'}) + var color = 'rgb(147, 14, 14)'; + //_this.clearFile(); + + } + if (data.responseText.length > 250) { + data.responseText = 'An error occured, please contact an admninstrator'; + } + swal( + { + title: title, + text: data.responseText, + type: type, + showCancelButton: false, + confirmButtonColor: color, + confirmButtonText: 'OK', + closeOnConfirm: true, + }, + function(isConfirm) { + } + ); + }); }, validate: function() { @@ -227,4 +218,4 @@ define([ }, }); -}); +}); \ No newline at end of file