From 707993f8c9d2384222c5157655fe0a905a6678d7 Mon Sep 17 00:00:00 2001
From: Ted Summer <ted.summer2@gmail.com>
Date: Thu, 9 Apr 2020 15:19:37 -0600
Subject: [PATCH] feat(bricr): translate bricr to 2.0

chore(bricr): remove schema file
---
 .gitignore                                  |   2 +
 {v2.0pr1-to-v2.0 => BRICR-to-v2.0}/.gitkeep |   0
 BRICR-to-v2.0/README.md                     |  68 ++++++
 BRICR-to-v2.0/main.py                       | 256 ++++++++++++++++++++
 BRICR-to-v2.0/parse_errors.py               |  92 +++++++
 BRICR-to-v2.0/requirements.txt              |   2 +
 BRICR-to-v2.0/utils.py                      | 104 ++++++++
 BRICR-to-v2.0/validate.sh                   |  16 ++
 8 files changed, 540 insertions(+)
 create mode 100644 .gitignore
 rename {v2.0pr1-to-v2.0 => BRICR-to-v2.0}/.gitkeep (100%)
 create mode 100644 BRICR-to-v2.0/README.md
 create mode 100644 BRICR-to-v2.0/main.py
 create mode 100644 BRICR-to-v2.0/parse_errors.py
 create mode 100644 BRICR-to-v2.0/requirements.txt
 create mode 100644 BRICR-to-v2.0/utils.py
 create mode 100755 BRICR-to-v2.0/validate.sh

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..96bb5cf
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,2 @@
+__pycache__
+**/*.xsd
\ No newline at end of file
diff --git a/v2.0pr1-to-v2.0/.gitkeep b/BRICR-to-v2.0/.gitkeep
similarity index 100%
rename from v2.0pr1-to-v2.0/.gitkeep
rename to BRICR-to-v2.0/.gitkeep
diff --git a/BRICR-to-v2.0/README.md b/BRICR-to-v2.0/README.md
new file mode 100644
index 0000000..6a77ff2
--- /dev/null
+++ b/BRICR-to-v2.0/README.md
@@ -0,0 +1,68 @@
+## Setup
+```
+python3 -m pip install -r requirements.txt
+```
+For validation, might need to install [xmllint](https://linux.die.net/man/1/xmllint) if not on a Mac or Ubuntu machine.
+Download the v2.0 schema locally
+```bash
+curl -o schema_2_0.xsd https://raw.githubusercontent.com/BuildingSync/schema/v2.0/BuildingSync.xsd
+```
+## Steps
+Overview:
+- generate validation errors for original files
+- using those errors, fix the files
+- validate the fixed files
+- replace the original files with the fixed files
+
+First step is to generate validation errors for the files.
+```bash
+# stdin: line separated paths to files to validate
+# arg1: label for this validation run - e.g. initial_validation
+# arg2: path to the local v2.0 xsd
+# output:
+#   the names of files that failed validation are put in failed_<label>.txt
+#   errors for failed files are found in <label>_errors directory
+for i in <path to files>/backup/media/buildingsync_files/*.xml; do echo $i; done \
+  | ./validate.sh initial_validation schema_2_0.xsd
+```
+
+Optionally you can generate a json file summarizing the errors
+```bash
+python3 parse_errors.py initial_validation_errors parsed_errors.json
+```
+
+Fix the files by running the python script:
+```bash
+# Fix files - change the path to the files
+# The script will put the files at the same path, but within a new dir with the
+# same name but with '_fixed' appended. e.g. ./foo/bar fixed files go to ./foo/bar_fixed
+python3 main.py <path to files>/backup/media/buildingsync_files initial_validation_errors
+```
+
+Verify the files were fixed by running xmllint on them.
+```bash
+# !! make sure you point the path to the _fixed directory !!
+for i in <path to files>/backup/media/buildingsync_files_fixed/*.xml; do echo $i; done \
+  | ./validate.sh validate1 schema_2_0.xsd
+```
+Check the output file, failed_validate1.txt to see what files failed. Only the files mentioned below should fail, and you should either delete or manually edit them accordingly.
+
+Lastly, replace the original directory with the '_fixed' directory.
+
+## Exceptional files
+These files are special cases that should be manually edited or removed
+
+### Files to manually edit
+These files contain additional Address and Lat/Long that are in the wrong sequence order. Just manually remove auc:Address, auc:Latitude, and auc:Longitude elements at `/auc:BuildingSync/auc:Facilities/auc:Facility/auc:Sites/auc:Site` (the data can still be found in Buildings/Building)
+- buildingsync_property_6463_HVkGvAY.xml
+- buildingsync_property_6463.xml
+
+### Files to delete
+These files are old, v0.3 and can be removed
+- ID0023002_505_Beach_Street_0_report_Gu6y955.xml
+- ID0023002_505_Beach_Street_0_report_X19eD1L.xml
+- ID0023002_505_Beach_Street_0_report.xml
+- 12.xml
+
+This file is just a one-off bad file which uses non-existing elements (auc:AnnualElectricity)
+- building_151_results.xml
diff --git a/BRICR-to-v2.0/main.py b/BRICR-to-v2.0/main.py
new file mode 100644
index 0000000..e560ce4
--- /dev/null
+++ b/BRICR-to-v2.0/main.py
@@ -0,0 +1,256 @@
+import os
+import shutil
+import json
+import glob
+
+from lxml import etree
+import xmlschema
+
+from parse_errors import summarize_errors
+from utils import sort_element, remove_dupes, fix_namespaces, BUILDINGSYNC_URI, NAMESPACES
+
+
+def fix_timestamp(file, directory):
+    """Turn TimeStamp into Timestamp"""
+    filepath = os.path.join(directory, file)
+    with open(filepath, 'r') as f:
+        filedata = f.read()
+
+    # Replace the target string
+    filedata = filedata.replace('TimeStamp', 'Timestamp')
+
+    # Write the file out again
+    with open(filepath, 'w') as f:
+        f.write(filedata)
+
+
+package_of_measures_xpath = '/'.join([
+    'auc:Facilities',
+    'auc:Facility',
+    'auc:Reports',
+    'auc:Report',
+    'auc:Scenarios',
+    'auc:Scenario',
+    'auc:ScenarioType',
+    'auc:PackageOfMeasures'
+])
+
+scenarios_xpath = '/'.join([
+    'auc:Facilities',
+    'auc:Facility',
+    'auc:Reports',
+    'auc:Report',
+    'auc:Scenarios',
+    'auc:Scenario',
+])
+schema_2_0_instance = xmlschema.XMLSchema('./schema_2_0.xsd')
+
+
+def fix_calculationmethod(file, directory):
+    """Does a couple of fixes
+    - sort elements at PackageOfMeasures
+    - remove duplicated elements in PackageOfMeasures
+    - sort elements at Scenario
+    - remove duplicated elements in Scenario
+    """
+
+    filepath = os.path.join(directory, file)
+    # get the calculation method parent, PackageOfMeasures, and sort its children
+    tree = etree.parse(filepath)
+    elements = tree.xpath(package_of_measures_xpath, namespaces=NAMESPACES)
+    for element in elements:
+        sort_element(schema_2_0_instance, tree, element)
+
+        # remove duplicate children in PackageOfMeasures
+        remove_dupes(element)
+
+    # sort the scenario children (ResourceUses in wrong position)
+    elements = tree.xpath(scenarios_xpath, namespaces=NAMESPACES)
+    for element in elements:
+        sort_element(schema_2_0_instance, tree, element)
+
+        # remove duplicates
+        remove_dupes(element)
+
+    filedata = etree.tostring(tree).decode()
+
+    # Write the file out again
+    with open(filepath, 'w') as f:
+        f.write(filedata)
+
+
+def fix_subsections(file, directory):
+    """Replaces Subsection(s) with Section(s)"""
+    filepath = os.path.join(directory, file)
+    with open(filepath, 'r') as f:
+        filedata = f.read()
+
+    # Replace the target string
+    filedata = filedata.replace('auc:Subsection', 'auc:Section')
+
+    # Write the file out again
+    with open(filepath, 'w') as f:
+        f.write(filedata)
+
+
+report_xpath = 'auc:Facilities/auc:Facility/auc:Report'
+
+
+def fix_report(file, directory):
+    """nest Report in Reports"""
+    filepath = os.path.join(directory, file)
+    tree = etree.parse(filepath)
+
+    report = tree.xpath(report_xpath, namespaces=NAMESPACES)
+    assert len(report) == 1
+    report = report[0]
+
+    facility = report.getparent()
+    facility.remove(report)
+    reports = etree.SubElement(facility, f'{{{BUILDINGSYNC_URI}}}Reports')
+    reports.append(report)
+
+    filedata = etree.tostring(tree).decode()
+
+    # Write the file out again
+    with open(filepath, 'w') as f:
+        f.write(filedata)
+
+
+light_xpath = 'auc:Facilities/auc:Facility/auc:Systems/auc:LightingSystems/auc:LightingSystem/auc:PrimaryLightingSystemType'
+
+
+def fix_primarylightingsystemtype(file, directory):
+    """turns PrimaryLightingSystemType into a user defined field"""
+    filepath = os.path.join(directory, file)
+
+    tree = etree.parse(filepath)
+    elements = tree.xpath(light_xpath, namespaces=NAMESPACES)
+    assert len(elements) > 0
+    for element in elements:
+        lighting_type = element.text
+
+        parent_element = element.getparent()
+        parent_element.remove(element)
+
+        udfs = etree.SubElement(parent_element, f'{{{BUILDINGSYNC_URI}}}UserDefinedFields')
+        udf = etree.SubElement(udfs, f'{{{BUILDINGSYNC_URI}}}UserDefinedField')
+        etree.SubElement(udf, f'{{{BUILDINGSYNC_URI}}}FieldName').text = 'PrimaryLightingSystemType'
+        etree.SubElement(udf, f'{{{BUILDINGSYNC_URI}}}FieldValue').text = lighting_type
+
+    filedata = etree.tostring(tree).decode()
+
+    # Write the file out again
+    with open(filepath, 'w') as f:
+        f.write(filedata)
+
+
+def fix_occupancyclassification(file, directory):
+    """Replace Hotel with Lodging"""
+    filepath = os.path.join(directory, file)
+    with open(filepath, 'r') as f:
+        filedata = f.read()
+
+    # Replace the target string
+    filedata = filedata.replace('Hotel', 'Lodging')
+
+    # Write the file out again
+    with open(filepath, 'w') as f:
+        f.write(filedata)
+
+
+def fix_schemalocation(file, directory):
+    """Makes sure that the schemaLocation is properly set"""
+    filepath = os.path.join(directory, file)
+    tree = etree.parse(filepath)
+
+    # fix the namespaces of the tree if necessary
+    root_nsmap = tree.getroot().nsmap
+    if root_nsmap.get('auc') is None or root_nsmap.get('xsi') is None:
+        tree = fix_namespaces(tree)
+    
+    # make sure schemalocation is set
+    root = tree.getroot()
+    root.set('{http://www.w3.org/2001/XMLSchema-instance}schemaLocation', 'http://buildingsync.net/schemas/bedes-auc/2019 https://raw.githubusercontent.com/BuildingSync/schema/v2.0/BuildingSync.xsd')
+
+    # save the finished tree
+    filedata = etree.tostring(tree).decode()
+
+    # Write the file out again
+    with open(filepath, 'w') as f:
+        f.write(filedata)
+
+error_fixes_map = {
+    'starttimestamp': fix_timestamp,
+    'calculationmethod': fix_calculationmethod,
+    'subsections': fix_subsections,
+    'report': fix_report,
+    'primarylightingsystemtype': fix_primarylightingsystemtype,
+    'occupancyclassification': fix_occupancyclassification
+}
+
+data_dir = os.sys.argv[1]
+
+# copy over the source files
+fixed_data_dir = os.path.join(
+    os.path.dirname(data_dir),
+    os.path.basename(data_dir) + '_fixed')
+print(f'Copying source files into {fixed_data_dir}')
+if not os.path.isdir(fixed_data_dir):
+    shutil.copytree(data_dir, fixed_data_dir)
+else:
+    raise Exception(f'Remove the _fixed data directory before running this script to fix files: {fixed_data_dir}')
+
+# get the summary of validation errors
+validation_errors_dir = os.sys.argv[2]
+print(f'Summarizing validation errors from {validation_errors_dir}')
+parsed_errors = summarize_errors(validation_errors_dir)
+
+# only care about the schema validity errors
+# we are just going to delete the file which has a parsing error
+element_errors = parsed_errors['Schemas validity error ']
+
+# elements we won't try to fix (either fixed manually or deleted) - see README
+skipped_elements = [
+    'Address',
+    'Audits',
+    'ResourceUses'  # handled by fixes to CalculationMethod (same files)
+]
+
+# iterate through the errors and fix the files
+for element, specific_errors in element_errors.items():
+    print(f'\nFixing {element}')
+    # get the fixer function
+    element_tag = element.split(' ')[1]
+    fixer = error_fixes_map.get(element_tag.lower())
+    if not fixer:
+        if element_tag not in skipped_elements:
+            raise Exception(f'Failed to find fixer for {element_tag}, which is not supposed to be skipped')
+        # skipping this element
+        continue
+
+    # collect all files that had errors with this element (fixes are grouped by elements, not specific errors)
+    files_to_fix = []
+    for error_description, error_data in specific_errors.items():
+        # get _only_ the basename after removing the ':<linenumber>' part of the filename
+        files_to_fix += [os.path.basename(filename.split(':')[0]) for filename in error_data['files']]
+
+    # only grab unique files
+    files_to_fix = list(set(files_to_fix))
+
+    # fix to the file
+    for source_file in files_to_fix:
+        fixer(source_file, fixed_data_dir)
+        print('.', end='', flush=True)
+
+# lastly fix the namespaces and schemaLocation for ALL files
+print('\nFixing namespaces and schemalocation')
+for file_path in glob.glob(os.path.join(fixed_data_dir, '*.xml')):
+    try:
+        fix_schemalocation(os.path.basename(file_path), fixed_data_dir)
+    except Exception as e:
+        print(f'\nSkipping file {file_path} due to exception: {str(e)}')
+    print('.', end='', flush=True)
+
+print('\n\n========  DONE  ========')
+print('Fixed files saved to ', fixed_data_dir)
diff --git a/BRICR-to-v2.0/parse_errors.py b/BRICR-to-v2.0/parse_errors.py
new file mode 100644
index 0000000..f76f402
--- /dev/null
+++ b/BRICR-to-v2.0/parse_errors.py
@@ -0,0 +1,92 @@
+import glob
+import os
+import json
+
+def get_errors(file):
+    with open(file, 'r') as f:
+        lines = f.readlines()
+        # don't look at last line, it's just 
+        processed_lines = []
+        for line in lines:
+            split_line = line.split(': ')
+            
+            # skip lines that aren't describing an error
+            if len(split_line) <= 1:
+                continue
+            
+            processed_lines.append(split_line)
+        return processed_lines
+
+NOT_ELEMENT = 'NOT_ELEMENT'
+def update_errors(errors, error):
+    """adds an error to the errors dict"""
+    if len(error) == 4:
+        filename, short_error, error_details, _ = error
+        element_tag = NOT_ELEMENT
+    elif len(error) == 5:
+        filename, element_tag, short_error, full_element_name, error_details = error
+    else:
+        print(error)
+        raise Exception('Had lines ' + str(len(error)))
+
+    error_details = error_details.strip()
+    if short_error not in errors:
+        # build the error description
+        errors[short_error] = {}
+        errors[short_error][element_tag] = {}
+        errors[short_error][element_tag][error_details] = {
+            'files': [filename]
+        }
+        return
+    
+    if element_tag not in errors[short_error]:
+        errors[short_error][element_tag] = {}
+        errors[short_error][element_tag][error_details] = {
+            'files': [filename]
+        }
+        return
+    
+    if error_details not in errors[short_error][element_tag]:
+        errors[short_error][element_tag][error_details] = {
+            'files': [filename]
+        }
+        return
+    
+    errors[short_error][element_tag][error_details]['files'].append(filename)
+
+
+def summarize_errors(directory, filesubset=None, errors=None):
+    """Returns a parsed dict of errors from a directory of error files
+    
+    e.g.
+    {
+      <short_error>: {
+        <element_tag>: {
+          <error_details>: {
+              'files': [...] # files that had this error
+          }
+        }
+      }
+    }
+    """
+    if errors is None:
+        errors = {}
+
+    filespath = os.path.join(directory, '*.xml')
+    files = glob.glob(filespath)
+    if filesubset is not None:
+        files = [file for file in files if os.path.basename(file) in filesubset]
+
+    for file in files:
+        for error in get_errors(file):
+            update_errors(errors, error)
+    
+    return errors
+
+if __name__ == '__main__':
+    # parse the errors and save as json
+    # usage: parse_errors.py <errors_dir> <json_filename>
+    errors = summarize_errors(os.sys.argv[1])
+    errors_json = json.dumps(errors, indent=2)
+    with open(os.sys.argv[2], 'w') as f:
+        f.write(errors_json)
diff --git a/BRICR-to-v2.0/requirements.txt b/BRICR-to-v2.0/requirements.txt
new file mode 100644
index 0000000..54f10ab
--- /dev/null
+++ b/BRICR-to-v2.0/requirements.txt
@@ -0,0 +1,2 @@
+lxml==4.5.0
+xmlschema==1.1.2
\ No newline at end of file
diff --git a/BRICR-to-v2.0/utils.py b/BRICR-to-v2.0/utils.py
new file mode 100644
index 0000000..fb5cb9d
--- /dev/null
+++ b/BRICR-to-v2.0/utils.py
@@ -0,0 +1,104 @@
+from io import StringIO
+
+from lxml import etree
+
+BUILDINGSYNC_URI = 'http://buildingsync.net/schemas/bedes-auc/2019'
+NAMESPACES = {
+    'auc': BUILDINGSYNC_URI
+}
+
+def children_sorter_factory(schema, tree, element):
+    """returns a function for getting a key value for sorting elements
+
+    Used to sort an elements children after inserting a new element to ensure
+    it meets the schema sequence specification
+
+    :param schema: xmlschema.XmlSchema, the schema to follow
+    :param tree: ElementTree, the tree from which the element came from
+    :param element: Element, the element whose children are to be sorted
+    """
+    # get the element from the schema
+    element_path = tree.getpath(element)
+    schema_element = schema.find(element_path)
+    if schema_element is None:
+        raise Exception(f'Unable to find path in schema: "{element_path}"')
+
+    ordered_children = [child.name for child in schema_element.iterchildren()]
+
+    # construct a function for sorting an element's children by returning the index of the
+    # child according to the ordering of the schema
+    def _getkey(element):
+        if isinstance(element, etree._Comment):
+            # put comments at the end
+            return 100
+        elif not isinstance(element, etree._Element):
+            raise Exception(f'Unknown type while sorting: "{type(element)}"')
+
+        if element.tag not in ordered_children:
+            # a more helpful exception that the one raised by .index()
+            raise Exception(f'Failed to find "{element.tag}" in {ordered_children}')
+
+        return ordered_children.index(element.tag)
+
+    return _getkey
+
+
+def sort_element(schema, tree, element):
+    """Sorts an element's children in place"""
+    getter = children_sorter_factory(schema, tree, element)
+    element[:] = sorted(element, key=getter)
+
+
+def remove_dupes(element):
+    """Removes duplicate children
+    !! assumes that element's children are already in sorted order
+    be careful which elements this is run on as it might find false positives that it removes
+    """
+    prev_child_text = ''
+    prev_child_tag = ''
+    prev_child = None
+    for child in element:
+        if child.text == prev_child_text and child.tag == prev_child_tag:
+            element.remove(child)
+        elif child.tag == prev_child_tag and 'AnnualSavingsSourceEnergy' in child.tag:
+            # keep the one with a smaller value (being conservative)
+            if float(prev_child_text) < float(child.text):
+                element.remove(child)
+            else:
+                element.remove(prev_child)
+
+        prev_child_text = child.text
+        prev_child_tag = child.tag
+        prev_child = child
+
+
+def fix_namespaces(tree):
+    """This method should be called when then namespace map is not correct.
+    It will clone the tree, ensuring all nodes have the proper namespace prefixes
+    """
+    original_tree = tree
+
+    etree.register_namespace('auc', BUILDINGSYNC_URI)
+    etree.register_namespace('xsi', 'http://www.w3.org/2001/XMLSchema-instance')
+
+    new_tree = etree.parse(StringIO('''<?xml version="1.0"?>
+        <auc:BuildingSync xmlns:auc="http://buildingsync.net/schemas/bedes-auc/2019" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://buildingsync.net/schemas/bedes-auc/2019 https://raw.githubusercontent.com/BuildingSync/schema/v2.0/BuildingSync.xsd">
+        </auc:BuildingSync>'''))
+
+    new_root = new_tree.getroot()
+    original_root = original_tree.getroot()
+
+    def clone_subtree(original, new):
+        for child in original.iterchildren():
+            new_child = etree.Element(child.tag)
+            # update text
+            new_child.text = child.text
+            # update attributes
+            for attr, val in child.items():
+                new_child.set(attr, val)
+            new.append(new_child)
+            clone_subtree(child, new_child)
+
+    clone_subtree(original_root, new_root)
+
+    return new_tree
diff --git a/BRICR-to-v2.0/validate.sh b/BRICR-to-v2.0/validate.sh
new file mode 100755
index 0000000..28a6937
--- /dev/null
+++ b/BRICR-to-v2.0/validate.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+# arg: name
+# arg: schema
+# stdin: lines of paths to files to process
+failure_file="failed_${1}.txt"
+errors_dir="${1}_errors"
+mkdir $errors_dir
+while read -r f; do
+  xmllint $f --schema $2 --noout &> ignore.txt
+  if [[ $? != 0 ]]; then
+    echo $f >> $failure_file
+    cp ignore.txt ./${errors_dir}/$(basename -- $f)
+  fi
+done <&0;
+
+rm ignore.txt