diff --git a/.idea/workspace.xml b/.idea/workspace.xml index 630d263..9ab6d06 100644 --- a/.idea/workspace.xml +++ b/.idea/workspace.xml @@ -1,14 +1,20 @@ + + + + + + - - - - - - + + + + + + \ No newline at end of file diff --git a/setup.py b/setup.py index f0b0889..b8c66d0 100644 --- a/setup.py +++ b/setup.py @@ -1,12 +1,12 @@ from setuptools import setup setup( - name='UmiTemplateDB', - version='', - packages=['tests', 'umitemplatedb'], - url='', - license='', - author='Samuel Letellier-Duchesne', - author_email='samueld@mit.edu', - description='Functions to import and query the UmiTemplate DataBase' + name="UmiTemplateDB", + version="", + packages=["tests", "umitemplatedb"], + url="", + license="", + author="Samuel Letellier-Duchesne", + author_email="samueld@mit.edu", + description="Functions to import and query the UmiTemplate DataBase", ) diff --git a/tests/test_core.py b/tests/test_core.py deleted file mode 100644 index bfd473d..0000000 --- a/tests/test_core.py +++ /dev/null @@ -1,147 +0,0 @@ -import datetime -import json -from tempfile import TemporaryFile - -import pytest -from mockupdb import go, MockupDB -from pymongo import MongoClient -from pymongo.errors import DuplicateKeyError - - -class MockupDBDatabaseTest: - def setUp(self): - self.server = MockupDB(auto_ismaster={"maxWireVersion": 3}) - self.server.run() - - # Replace pymongo connection url to use MockupDB - self.database = MongoDatabase(self.server.uri) - - def tearDown(self): - self.server.stop() - - def list_documents(self, DOCUMENT): - document = DOCUMENT - - # adding datetime object to document - document["request_time"] = datetime.datetime.now() - - # The database list method converts any datetime object returned - # from the database to string. See below. - document_query = go(self.database.list, {}) - - request = self.server.receives() - - # returns to pymongo a list containing only 1 the document - request.reply([DOCUMENT]) - - assert isinstance(document_query()[0]["request_time"], str) - assert isinstance( - document["request_time"].strftime("%Y-%m-%dT%H:%M:%S"), - document_query()[0]["request_time"], - ) - - -class MongoDatabase(object): - def __init__(self, uri): - self.client = MongoClient(uri) - self.db = self.client.db - self.collection = self.db.collection - - def list(self, query): - data_list = list() - for data in self.collection.find(query): - data["id"] = str(data.pop("_id")) - data = self._convert_datetime_to_str(data) - data_list.append(data) - return data_list - - def _convert_datetime_to_str(self, data): - for key in data.keys(): - if isinstance(data[key], datetime.datetime): - data[key] = data[key].strftime("%Y-%m-%dT%H:%M:%S") - return data - - -class TestImports: - @pytest.fixture() - def DOCUMENT(self): - umi_template = "test_templates\BostonTemplateLibrary.json" - with open(umi_template) as file: - yield json.load(file) - - # @pytest.fixture(scope="module") - # def db(self): - # db = MockupDBDatabaseTest() - # db.setUp() - # yield db - # db.tearDown() - - @pytest.fixture(scope="module") - def db(self): - client = MongoClient("mongodb://localhost:27017/") - db = client["test"] - yield db - client.close() - - def test_import_umitemplate(self, db, DOCUMENT): - - assert db.list_documents(DOCUMENT) - - def test_import_library(self, db): - import archetypal as ar - - def dict_generator(indict, pre=None): - pre = pre[:] if pre else [] - if isinstance(indict, dict): - for key, value in indict.items(): - if isinstance(value, dict): - for d in dict_generator(value, pre + [key]): - yield d - elif isinstance(value, (list, tuple)): - for v in value: - for d in dict_generator(v, pre + [key]): - yield d - else: - yield pre + [key, value] - else: - yield pre + [indict] - - def recursive_ref(obj): - if isinstance(obj, (dict)): - for key, value in obj.items(): - if isinstance(value, dict): - if "$ref" in value.keys(): - ref = obj.get(key).get("$ref") - db_dict[f"{key}_id"] = ref - elif isinstance(value, list): - recursive_ref(value) - else: - db_dict[key] = value - elif isinstance(obj, list): - for value in obj: - recursive_ref(value) - - ut = ar.UmiTemplate.read_file( - "test_templates\Ireland_UK_Tabula_Templates_Res.json" - ) - with TemporaryFile() as buf: - library_json = ut.to_dict() - - for coll, l in library_json.items(): - if isinstance(l, list): - for document in l: - db_dict = {} - collection = db[coll] - try: - document["_id"] = document.pop("$id") - recursive_ref(document) - post_id = collection.insert(db_dict) - print(f"inserted {post_id}") - except DuplicateKeyError: - collection.replace_one( - {"_id": db_dict["_id"]}, db_dict, upsert=True - ) - except AttributeError as e: - raise e - except Exception as e: - raise e diff --git a/tests/test_mongodb_schema.py b/tests/test_mongodb_schema.py index 01ad4cb..fcdd3f4 100644 --- a/tests/test_mongodb_schema.py +++ b/tests/test_mongodb_schema.py @@ -3,23 +3,20 @@ import geojson import pytest import shapely.geometry +from archetypal import UmiTemplateLibrary from umitemplatedb.core import import_umitemplate, serialize from umitemplatedb.mongodb_schema import * -@pytest.fixture +@pytest.fixture(scope="session") def db(): - # connect("templatelibrary", host="mongomock://localhost") - connect("templatelibrary") + connect("templatelibrary", host="mongomock://localhost") + # connect("templatelibrary") yield disconnect() -def test_retreive(db): - assert BuildingTemplate.objects() - - def test_save_and_retrieve_building(bldg, window, struct, core): # To filter by an attribute of MetaData, use double underscore """ @@ -29,7 +26,7 @@ def test_save_and_retrieve_building(bldg, window, struct, core): struct: core: """ - a_bldg = BuildingTemplate.objects(MetaData__Country="FR").first() + a_bldg = BuildingTemplate.objects(Country="FR").first() assert a_bldg.Name == bldg.Name @@ -46,8 +43,7 @@ def test_filter_by_geo(bldg): We would create the geoquery this way: First create a geojson-like dict using :meth:`shapely.geometry.mapping`. Then pass this pt to the - `MetaData__Polygon__geo_intersects` attribute. MetaData is the first - Attribute. Polygon is the attribute of MetaData and finally, + `Polygon__geo_intersects` attribute. Polygon is the attribute of MetaData and finally, `geo_intersects` is the embedded MongoDB function for the `intersects` predicate. See `MongoEngine geo-queries`_ for more details. @@ -62,7 +58,7 @@ def test_filter_by_geo(bldg): # First, a sanity check. We build a pt and use # the :meth:`intersects` method. pt = Point(42.370145, -71.112077) - polygon = json.dumps(bldg.MetaData.Polygon) + polygon = json.dumps(bldg.Polygon) # Convert to geojson.geometry.Polygon g1 = geojson.loads(polygon) g2 = shapely.geometry.shape(g1) @@ -71,11 +67,11 @@ def test_filter_by_geo(bldg): # Second, the actual filter with point pt ptj = shapely.geometry.mapping(pt) - a_bldg = BuildingTemplate.objects(MetaData__Polygon__geo_intersects=ptj).first() + a_bldg = BuildingTemplate.objects(Polygon__geo_intersects=ptj).first() assert a_bldg -@pytest.fixture() +@pytest.fixture(scope="session") def imported(db): path = "tests/test_templates/BostonTemplateLibrary.json" import_umitemplate(path, Author="Carlos Cerezo", Country="US") @@ -84,20 +80,25 @@ def imported(db): def test_import_library(db, imported): """Try using recursive""" for bldg in BuildingTemplate.objects(): - print(f"downloaded {bldg}") + print(f"downloaded {bldg.Name}") assert bldg -def test_db_to_json(imported): - serialize() +def test_serialize_templatelist(bldg, window, struct, core): + """From a list of :class:~`umitemplatedb.mongodb_schema.BuildingTemplate` + create an :class:~`archetypal.umi_template.UmiTemplateLibrary`""" + bldgs = [bldg] + templates = [] + for bldg in bldgs: + templates.append(bldg.to_template()) -def test_to_json(bldg): - """ - Args: - bldg: - """ - print(bldg.to_json()) + lib = UmiTemplateLibrary(BuildingTemplates=templates) + lib.to_json() + + +def test_db_to_json(imported): + serialize() @pytest.fixture() @@ -115,12 +116,13 @@ def bldg(db, core, struct, window): Perimeter=core, Structure=struct, Windows=window, - MetaData=MetaData(Author="Samuel Letellier-Duchesne", Country="FR"), + Author="Samuel Letellier-Duchesne", + Country="FR", ).save() @pytest.fixture() -def days(): +def day(): return DaySchedule( Values=[ 1.0, @@ -147,17 +149,18 @@ def days(): 1.0, 1.0, 1.0, - ] + ], + Name="DaySch", ).save() @pytest.fixture() -def weekschd(days): +def weekschd(day): """ Args: days: """ - return WeekSchedule(Days=[days]).save() + return WeekSchedule(Days=[day] * 7, Name="WeekSch").save() @pytest.fixture() @@ -167,7 +170,7 @@ def ys_part(weekschd): weekschd: """ return YearSchedulePart( - FromDay=0, FromMonth=0, ToDay=31, ToMonth=12, Schedule=weekschd + FromDay=1, FromMonth=1, ToDay=31, ToMonth=12, Schedule=weekschd ) @@ -191,18 +194,19 @@ def cond(alwaysOn): "CoolingSchedule": alwaysOn, "HeatingSchedule": alwaysOn, "MechVentSchedule": alwaysOn, - } + }, + Name="Zone Conditioning", ).save() @pytest.fixture() def opaquematerial(): - return OpaqueMaterial().save() + return OpaqueMaterial(Name="OpaqueMaterial").save() @pytest.fixture() def glazingmaterial(): - return GlazingMaterial().save() + return GlazingMaterial(Name="GlazingMaterial").save() @pytest.fixture() @@ -264,7 +268,9 @@ def dhw(alwaysOn): Args: alwaysOn: """ - return DomesticHotWaterSetting(WaterSchedule=alwaysOn).save() + return DomesticHotWaterSetting( + WaterSchedule=alwaysOn, Name="DomesticHotWaterSetting" + ).save() @pytest.fixture() @@ -280,13 +286,14 @@ def conset(construction): "Partition": construction, "Roof": construction, "Slab": construction, - } + }, + Name="ZoneConstructionSet", ).save() @pytest.fixture() def intmass(): - return OpaqueConstruction().save() + return OpaqueConstruction(Name="OpaqueConstruction").save() @pytest.fixture() @@ -300,7 +307,8 @@ def loads(alwaysOn): "EquipmentAvailabilitySchedule": alwaysOn, "LightsAvailabilitySchedule": alwaysOn, "OccupancySchedule": alwaysOn, - } + }, + Name="ZoneLoad", ).save() @@ -311,7 +319,8 @@ def vent(alwaysOn): alwaysOn: """ return VentilationSetting( - **{"NatVentSchedule": alwaysOn, "ScheduledVentilationSchedule": alwaysOn} + **{"NatVentSchedule": alwaysOn, "ScheduledVentilationSchedule": alwaysOn}, + Name="VentilationSetting", ).save() @@ -354,7 +363,9 @@ def struct(massratio): Args: massratio: """ - return StructureInformation(MassRatio=[massratio]).save() + return StructureInformation( + MassRatios=[massratio], Name="StructureInformation" + ).save() @pytest.fixture() @@ -370,14 +381,6 @@ def window(alwaysOn, windowconstruction): "Construction": windowconstruction, "ShadingSystemAvailabilitySchedule": alwaysOn, "ZoneMixingAvailabilitySchedule": alwaysOn, - } + }, + Name="WindowSetting", ).save() - - -def test_building_template(db): - - """ - Args: - db: - """ - assert BuildingTemplate.objects diff --git a/umitemplatedb/__init__.py b/umitemplatedb/__init__.py index e69de29..c213408 100644 --- a/umitemplatedb/__init__.py +++ b/umitemplatedb/__init__.py @@ -0,0 +1,2 @@ +from .core import serialize, import_umitemplate +from .mongodb_schema import * diff --git a/umitemplatedb/core.py b/umitemplatedb/core.py index 13a1e94..fe641c8 100644 --- a/umitemplatedb/core.py +++ b/umitemplatedb/core.py @@ -5,9 +5,9 @@ import numpy as np from mongoengine import * +from umitemplatedb import mongodb_schema from umitemplatedb.mongodb_schema import ( BuildingTemplate, - MetaData, GasMaterial, GlazingMaterial, OpaqueMaterial, @@ -64,7 +64,7 @@ def recursive(umibase, **metaattributes): """recursively create db objects from UmiBase objects. Start with BuildingTemplates.""" instance_attr = {} - class_ = getattr(umitemplatedb.mongodb_schema, type(umibase).__name__) + class_ = getattr(mongodb_schema, type(umibase).__name__) for key, value in umibase.mapping().items(): if isinstance( value, @@ -103,10 +103,19 @@ def recursive(umibase, **metaattributes): ) ) if isinstance(class_instance, BuildingTemplate): - class_instance["MetaData"] = MetaData(**metaattributes) + for key, value in metaattributes.items(): + class_instance[key] = value class_instance.save() return class_instance + if not Polygon: + Polygon = { + "type": "Polygon", + "coordinates": [ + [[-180, -90], [180, -90], [180, 90], [-180, 90], [-180, -90]] + ], + } + # loop starts here bldg = recursive( bldgtemplate, @@ -172,8 +181,7 @@ def serialize(): "Zones": [obj.to_json(indent=3) for obj in ZoneDefinition.objects()], "WindowSettings": [obj.to_json(indent=3) for obj in WindowSetting.objects()], "BuildingTemplates": [ - obj.to_json(indent=3) - for obj in BuildingTemplate.objects().exclude("MetaData", "_cls") + obj.to_json(indent=3) for obj in BuildingTemplate.objects().exclude("_cls") ], } diff --git a/umitemplatedb/mongodb_schema.py b/umitemplatedb/mongodb_schema.py index 15e5580..e72bc05 100644 --- a/umitemplatedb/mongodb_schema.py +++ b/umitemplatedb/mongodb_schema.py @@ -1,10 +1,10 @@ import hashlib from datetime import datetime +import archetypal.template import pycountry from bson import DBRef as BaseDBRef from mongoengine import * -from mongoengine import signals class ReferenceField(ReferenceField): @@ -100,10 +100,12 @@ class UmiBase(Document): """ - key = PrimaryKey(primary_key=True) + Name = StringField(required=True) + key = PrimaryKey( + primary_key=True, default=dict(_name=str(Name), _class=type(Document).__name__) + ) Comments = StringField(null=True) DataSource = StringField(null=True) - Name = StringField(required=True) Category = StringField(default="Uncategorized") meta = {"allow_inheritance": True} @@ -226,16 +228,22 @@ class DaySchedule(UmiBase): ) +def min_length(x): + """WeekSchedule.Days should have lenght == 7""" + if len(x) != 7: + raise ValidationError + + class WeekSchedule(UmiBase): - Days = ListField(ReferenceField(DaySchedule), required=True) + Days = ListField(ReferenceField(DaySchedule), validation=min_length, required=True) Type = StringField(default="Fraction") class YearSchedulePart(EmbeddedDocument): - FromDay = IntField() - FromMonth = IntField() - ToDay = IntField() - ToMonth = IntField() + FromDay = IntField(min_value=1, max_value=31) + FromMonth = IntField(min_value=1, max_value=12) + ToDay = IntField(min_value=1, max_value=31) + ToMonth = IntField(min_value=1, max_value=12) Schedule = ReferenceField(WeekSchedule, required=True) # key = EmbeddedDocumentField(PrimaryKey, primary_key=True) @@ -367,45 +375,62 @@ def update_modified(sender, document): document.DateModified = datetime.utcnow() -class MetaData(EmbeddedDocument): - """archetype template attributes - - Attributes: - Author: (StringField): - DateCreated (DateTimeField): - DateModified (DateTimeField): - Image (ImageField): - Country (StringField): - Description (StringField): +class BuildingTemplate(UmiBase): + """Top most object in Umi Template Structure""" - """ + Core = ReferenceField(ZoneDefinition, required=True) + Lifespan = IntField(default=60) + PartitionRatio = FloatField(default=0) + Perimeter = ReferenceField(ZoneDefinition, required=True) + Structure = ReferenceField(StructureInformation, required=True) + Windows = ReferenceField(WindowSetting, required=True) + DefaultWindowToWallRatio = FloatField(default=0.4, min_value=0, max_value=1) + # MetaData (Not in UMI) Author = StringField(required=True) DateCreated = DateTimeField(default=datetime.utcnow, required=True) DateModified = DateTimeField(default=datetime.utcnow) # Image = ImageField() - - Country = StringField(choices=[country.alpha_2 for country in pycountry.countries]) + Country = StringField( + choices=tuple((a.alpha_2, a.name) for a in list(pycountry.countries)) + ) YearFrom = StringField( help_text="Starting year for the range this template applies to" ) YearTo = StringField(help_text="End year ") + ClimateZone = StringField() Polygon = PolygonField(default=world_poly) Description = StringField(help_text="") - meta = {"allow_inheritance": True, "strict": False} - - signals.pre_save.connect(update_modified) - - -class BuildingTemplate(UmiBase): - """Top most object in Umi Template Structure""" - - Core = ReferenceField(ZoneDefinition, required=True) - Lifespan = IntField(default=60) - PartitionRatio = FloatField(default=0) - Perimeter = ReferenceField(ZoneDefinition, required=True) - Structure = ReferenceField(StructureInformation, required=True) - Windows = ReferenceField(WindowSetting, required=True) - DefaultWindowToWallRatio = FloatField(default=0.4, min_value=0, max_value=1) - MetaData = EmbeddedDocumentField(MetaData, required=True) + def to_template(self, idf=None): + """Converts to an :class:~`archetypal.template.building_template + .BuildingTemplate` object""" + + def recursive(document, idf): + """recursively create UmiBase objects from Document objects. Start with + BuildingTemplates.""" + instance_attr = {} + class_ = getattr(archetypal.template, type(document).__name__) + for key in document: + if isinstance(document[key], (UmiBase, YearSchedulePart)): + instance_attr[key] = recursive(document[key], idf) + elif isinstance(document[key], list): + instance_attr[key] = [] + for value in document[key]: + if isinstance( + value, (UmiBase, YearSchedulePart, MaterialLayer, MassRatio) + ): + instance_attr[key].append(recursive(value, idf)) + else: + instance_attr[key].append(value) + elif isinstance(document[key], (str, int, float)): + instance_attr[key] = document[key] + class_instance = class_(**instance_attr, idf=idf) + return class_instance + + if idf is None: + from archetypal import IDF + + idf = IDF() + + return recursive(self, idf=idf)