Merge pull request #1041 from OpenEnergyPlatform/release/v0.10.2

Release v0.10.2

.eslintrc.json

@@ -0,0 +1,29 @@
+{
+  "env": {
+    "browser": true,
+    "es6": true,
+    "commonjs": true,
+    "es2021": true
+  },
+  "extends": ["google"],
+  "plugins": ["jsdoc"],
+  "parserOptions": {
+    "ecmaVersion": "latest",
+    "sourceType": "module"
+  },
+  "rules": {
+    "camelcase": 0,
+    "prefer-const": 0,
+    "valid-jsdoc": [2, { "prefer": { "return": "returns" } }],
+    "quotes": [0, "single", "double"],
+    "space-before-function-paren": [
+      "error",
+      {
+        "anonymous": "never",
+        "named": "never",
+        "asyncArrow": "never"
+      }
+    ],
+    "linebreak-style": 0
+  }
+}

.gitignore

@@ -6,9 +6,10 @@ __pycache__/
 *$py.class
 *~
 
-media/
-local/
-static/
+media
+local
+# only on root dir
+/static
 
 # C extensions
 *.so

.pre-commit-config.yaml

@@ -0,0 +1,36 @@
+exclude: ^$
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.2.0
+    hooks:
+    -   id: check-yaml
+    -   id: end-of-file-fixer
+    -   id: trailing-whitespace
+-   repo: https://github.com/psf/black
+    rev: 22.3.0
+    hooks:
+    -   id: black
+-   repo: https://github.com/pycqa/isort
+    rev: 5.10.1
+    hooks:
+    -   id: isort
+        args: ["--profile", "black", "--filter-files"]
+-   repo: https://github.com/pycqa/flake8
+    rev: 4.0.1
+    hooks:
+    -   id: flake8
+        args: ["--max-line-length=88", "--ignore=E20,W503"] # black compatible
+-   repo: https://github.com/pre-commit/mirrors-eslint
+    rev: v8.14.0
+    hooks:
+    -   id: eslint
+        additional_dependencies:
+        - [email protected]
+        - [email protected]
+        - [email protected]
+        args: ["--fix"]
+-   repo: https://github.com/pre-commit/mirrors-prettier
+    rev: v2.6.2
+    hooks:
+    -   id: prettier
+        files: \.(css|less|md|json|sql)$

README.md

@@ -12,7 +12,7 @@ This repository is licensed under [GNU Affero General Public License v3.0 (AGPL-
 
 ## Installation
 
-In the meantime, we also offer the possibility to use [docker](https://www.docker.com/) in addition to the manual installation. For this purpose, 2 [docker container images](https://docs.docker.com/get-started/#what-is-a-container-image) (OEP-website and OEP-database) are published with each release, which can be pulled from [GitHub packages](https://github.com/OpenEnergyPlatform/oeplatform/pkgs/container/oeplatform).
+Below we describe the complete manual installation of the OEP website and database. We also offer the possibility to use [docker](https://www.docker.com/), if you are a developer you could manually install the OEP alongside the dockercontainer to run the database or run everything in docker. For this purpose, 2 [docker container images](https://docs.docker.com/get-started/#what-is-a-container-image) (OEP-website and OEP-database) are published with each release, which can be pulled from [GitHub packages](https://github.com/OpenEnergyPlatform/oeplatform/pkgs/container/oeplatform).
 
 [Here you can find instructions on how to install the docker images.](https://github.com/OpenEnergyPlatform/oeplatform/blob/develop/docker/USAGE.md)
 
@@ -44,6 +44,10 @@ If you are a windows user, we recommand you use conda because of the dependency
 4)      conda install shapely
 5)      pip install –r requirements.txt
 
+You can also use Python to create the environment
+
+    python -m venv env
+
 
 If you don't want to use conda, [here](https://packaging.python.org/guides/installing-using-pip-and-virtual-environments/) you can find instructions for setting up virtual environment
 

RELEASE_PROCEDURE.md

@@ -28,18 +28,27 @@ The project can be linked in an issue or pull request via the menu "Projects"
 ## Basic-Steps for deploy and release (publishing a new release)
 Before see How to [Contribute](https://github.com/OpenEnergyPlatform/oeplatform/blob/develop/CONTRIBUTING.md)
 
+![git branching model](https://nvie.com/img/[email protected])
+
+1. Merge all feature and hotfix branches into `develop`
 1. Starting out in the `develop` branch, make a release candidate branch (e.g., `release/vx.x.x`)
-   and pull request it into `master` with the following updates:
-   1. Update the oeplatform/versions/changelogs/ [`current.md`](https://github.com/OpenEnergyPlatform/oeplatform/blob/develop/versions/changelogs/current.md) (see the examples of previous releases)
-	  - Change filename to release version (x_x_x.md)
-  1. Confirm that the PR passes all tests and checks
-  1. Once successful, delete the tag, and merge the candidate PR into `master` on Github
-1. Switch to the now-updated master branch: `git checkout master` and `git pull upstream master`
+1. Update the oeplatform/versions/changelogs/ [`current.md`](https://github.com/OpenEnergyPlatform/oeplatform/blob/develop/versions/changelogs/current.md) (see the examples of previous releases)
+   - Change filename to release version (x_x_x.md)
+   - Copy template to `current.md`
+   - Update `VERSION` with lastest version number
+1. Deploy release branch on TEOP.
+   - Test the changes 
+   - Create a hotfix and merge changes into the release branch
+1. merge release branch into `master` and `develop`
+   - make sure to pull before merge
+1. Deploy the master branch on production OEP
 1. Tag the release number: `git tag v<release version>`, e.g., `git tag v1.2.0`
    - `versioneer` automatically updates the version number based on the tag
    - this is now the official tagged commit
-1. Push the tag upstream: `git push upstream --tags`
+   - Push the tag upstream: `git push upstream --tags`
+   - Alternatievely: tag on github platform while creating release
 1. Make a new release on Github
+   - https://github.com/OpenEnergyPlatform/oeplatform/releases/new
    - make sure that you choose the tag name defined above
    - copy the release summary from changelog into the description box
 1. Announce it on our mailing list: [email protected]

VERSION

@@ -1 +1 @@
-0.10.1
+0.10.2

api/actions.py

@@ -749,12 +749,12 @@ def assert_valid_table_name(table):
                        "and start with a letter.")
 
 
-def table_create(schema, table, columns, constraints_definitions, cursor, table_metadata=None):
+def table_create(schema, table, column_definitions, constraints_definitions, cursor, table_metadata=None):
     """
     Creates a new table.
     :param schema: schema
     :param table: table
-    :param columns: Description of columns
+    :param column_definitions: Description of columns
     :param constraints: Description of constraints
     :return: Dictionary with results
     """
@@ -796,7 +796,22 @@ def table_create(schema, table, columns, constraints_definitions, cursor, table_
 
     metadata = MetaData()
 
-    columns = [get_column_definition_query(c) for c in columns]
+    primary_key_col_names = None
+    columns_by_name = {}    
+
+    columns = []
+    for cdef in column_definitions:
+        col = get_column_definition_query(cdef)
+        columns.append(col)
+
+        # check for duplicate column names
+        if col.name in columns_by_name:
+            raise APIError("Duplicate column name: %s" % col.name)
+        columns_by_name[col.name] = col
+        if col.primary_key:
+            if primary_key_col_names:
+                raise APIError("Multiple definitions of primary key")
+            primary_key_col_names = [col.name]
 
     constraints = []
 
@@ -814,8 +829,14 @@ def table_create(schema, table, columns, constraints_definitions, cursor, table_
             if "columns" in constraint:
                 ccolumns = constraint["columns"]
             else:
-                ccolumns = [constraint["constraint_parameter"]]
-            constraints.append(sa.schema.PrimaryKeyConstraint(*ccolumns, **kwargs))
+                ccolumns = [constraint["constraint_parameter"]]                        
+
+            if primary_key_col_names:
+                raise APIError("Multiple definitions of primary key")
+            primary_key_col_names = ccolumns
+
+            const = sa.schema.PrimaryKeyConstraint(*ccolumns, **kwargs)            
+            constraints.append(const)
         elif constraint_type == "unique":
             kwargs = {}
             cname = constraint.get("name")
@@ -828,6 +849,28 @@ def table_create(schema, table, columns, constraints_definitions, cursor, table_
             constraints.append(sa.schema.UniqueConstraint(*ccolumns, **kwargs))
 
     assert_valid_table_name(table)
+
+    # autogenerate id column if missing
+    if 'id' not in columns_by_name:        
+        columns_by_name["id"] = sa.Column("id", sa.BigInteger, autoincrement=True)
+        columns.insert(0, columns_by_name["id"])
+
+    # check id column type
+    id_col_type = str(columns_by_name["id"].type).upper()
+    if not "INT" in id_col_type or "SERIAL" in id_col_type:
+        raise APIError("Id column must be of int type")
+
+    # autogenerate primary key
+    if not primary_key_col_names:
+        constraints.append(sa.schema.PrimaryKeyConstraint("id"))
+        primary_key_col_names = ["id"]
+
+    # check pk == id
+    if tuple(primary_key_col_names) != ("id",):
+        raise APIError("Primary key must be column id")
+
+
+
     t = Table(table, metadata, *(columns + constraints), schema=schema, comment=comment_on_table)
     t.create(_get_engine())
 
@@ -1184,7 +1227,7 @@ def data_insert_check(schema, table, values, context):
                             {
                                 "operands": [
                                     {"type": "column", "column": c},
-                                    {"type": "value", "value": _load_value(row[c])}
+                                    {"type": "value", "value": row[c]}
                                     if c in row
                                     else {"type": "value"},
                                 ],
@@ -1233,13 +1276,6 @@ def data_insert_check(schema, table, values, context):
                         )
 
 
-def _load_value(v):
-    if isinstance(v, str):
-        if v.isdigit():
-            return int(v)
-    return v
-
-
 def data_insert(request, context=None):
     cursor = load_cursor_from_context(context)
     # If the insert request is not for a meta table, change the request to do so
@@ -2139,7 +2175,7 @@ def apply_deletion(session, table, rows, rids):
 
 def update_meta_search(table, schema):
     schema_obj, _ = DBSchema.objects.get_or_create(name=schema if schema is not None else DEFAULT_SCHEMA)
-    t, _ = DBTable.objects.get_or_create(name=table, schema=schema_obj)
+    t = DBTable.objects.get(name=table, schema=schema_obj)
     comment = str(dataedit.metadata.load_metadata_from_db(schema, table))
     session = sessionmaker()(bind=_get_engine())
     tags = session.query(OEDBTag.name).filter(OEDBTableTags.schema_name==schema, OEDBTableTags.table_name==table, OEDBTableTags.tag==OEDBTag.id)

api/connection.py

@@ -1,11 +1,19 @@
+"""Contains functions to interact with the postgres oedb"""
+
 import sqlalchemy as sqla
+from sqlalchemy.orm import sessionmaker
+
+from api import DEFAULT_SCHEMA
+from dataedit.models import Schema, Table
 
 try:
     import oeplatform.securitysettings as sec
-except:
+except Exception:
     import logging
+
     logging.error("No securitysettings found. Triggerd in api/connection.py")
 
+
 def get_connection_string():
     return "postgresql://{0}:{1}@{2}:{3}/{4}".format(
         sec.dbuser, sec.dbpasswd, sec.dbhost, sec.dbport, sec.dbname
@@ -19,3 +27,51 @@ def get_connection_string():
 
 def _get_engine():
     return __ENGINE
+
+
+def table_exists_in_oedb(table, schema=None):
+    """check if table exists in oedb
+
+    Args:
+        table (str): table name
+        schema (str, optional): table schema name
+
+    Returns:
+        bool
+    """
+    schema = schema or DEFAULT_SCHEMA
+    engine = _get_engine()
+    conn = engine.connect()
+    try:
+        result = engine.dialect.has_table(conn, table, schema=schema)
+    finally:
+        conn.close()
+    return result
+
+
+def table_exists_in_django(table, schema=None):
+    """check if table exists in django
+
+    Args:
+        table (str): table name
+        schema (str, optional): table schema name
+
+    Returns:
+        bool
+    """
+    schema = schema or DEFAULT_SCHEMA
+    schema_obj = Schema.objects.get_or_create(name=schema)[0]
+    try:
+        Table.objects.get(name=table, schema=schema_obj)
+        return True
+    except Table.DoesNotExist:
+        return False
+
+
+
+def create_oedb_session():
+    """Return a sqlalchemy session to the oedb
+    
+    Should only be created once per user request.
+    """
+    return sessionmaker(bind=_get_engine())()

api/tests/__init__.py

@@ -36,18 +36,19 @@ def setUpClass(cls):
 
         super(APITestCase, cls).setUpClass()
         cls.user, _ = myuser.objects.get_or_create(
-            name="MrTest", email="[email protected]"
+            name="MrTest", email="[email protected]", did_agree=True, is_mail_verified=True
         )
         cls.user.save()
         cls.token = Token.objects.get(user=cls.user)
 
         cls.other_user, _ = myuser.objects.get_or_create(
-            name="NotMrTest", email="[email protected]"
+            name="NotMrTest", email="[email protected]", did_agree=True, is_mail_verified=True
         )
         cls.other_user.save()
         cls.other_token = Token.objects.get(user=cls.other_user)
 
         cls.client = Client()
+
 
     def assertDictEqualKeywise(self, d1, d2, excluded=None):
         if not excluded:
@@ -168,3 +169,4 @@ def drop_table(self, schema=None, table=None):
         self.assertEqual(
             resp.status_code, 200, resp.json().get("reason", "No reason returned")
         )
+

api/tests/test_meta.py

@@ -75,7 +75,7 @@ def tearDown(self):
                 )
             )
 
-    def metadata_roundtrip(self, meta):
+    def metadata_roundtrip(self, meta):        
         response = self.__class__.client.post(
             "/api/v0/schema/{schema}/tables/{table}/meta/".format(
                 schema=self.test_schema, table=self.test_table
@@ -101,8 +101,16 @@ def metadata_roundtrip(self, meta):
         d_14 = OEP_V_1_4_Dialect() # not tested anymore as OEM version is currently v1.5.1
         d_15 = OEP_V_1_5_Dialect()
         omi_meta = json.loads(json.dumps((d_15.compile(d_15.parse(json.dumps(meta))))))
-
-        self.assertDictEqualKeywise(response.json(), omi_meta)
+
+        omi_meta_return = response.json() 
+
+        # ignore difference in keywords (by setting resulting keywords == input keywords)
+        # REASON: the test re-uses the same test table, but does not delete the table tags in between
+        # if we want to synchronize tagsand keywords, the roundtrip would otherwise fail        
+        omi_meta["keywords"] = omi_meta.get("keywords", [])
+        omi_meta_return["keywords"] = omi_meta["keywords"]
+
+        self.assertDictEqualKeywise(omi_meta_return, omi_meta)
 
     def test_nonexistent_key(self):
         meta = {"id":"id", "nonexistent_key": ""}