initial commit

.gitignore

@@ -0,0 +1 @@
+credentials.json

credentials.json.example

@@ -0,0 +1,12 @@
+{
+    "type": "service_account",
+    "project_id": "my_gcp_project",
+    "private_key_id": "XXX",
+    "private_key": "-----BEGIN PRIVATE KEY-----\nXXX==\n-----END PRIVATE KEY-----\n",
+    "client_email": "my_great_service_account.iam.gserviceaccount.com",
+    "client_id": "12345678910",
+    "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+    "token_uri": "https://oauth2.googleapis.com/token",
+    "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+    "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/my_great_service_account.iam.gserviceaccount.com"
+  }

example.py

@@ -0,0 +1,79 @@
+import sqlalchemy
+import sqlalchemy_bigquery
+import lookml
+import sqlalchemy as SA
+from sqlalchemy.engine import create_engine
+from sqlalchemy.schema import *
+import json
+from typing import Union
+
+tables_in_database, tables_in_project = [], []
+
+#Create connection to BQ
+engine = create_engine(
+            'bigquery://my_great_bg_project', 
+            credentials_info=json.load(open('credentials.json','r'))
+            )
+# Initialize the inspector
+inspector = SA.inspect(engine)
+
+#loop over tables in DB
+for t in inspector.get_table_names('vision'):
+    tables_in_database.append(t)
+
+#Initialize pyLookML project (see lookml docs, this is local files for simplicity.
+# Also can be done over github https or ssh. See https://pylookml.readthedocs.io/en/latest/)
+myProject = lookml.Project(path='example_project')
+
+# Loop over views in LookML project
+for vf in myProject.view_files():
+    for viewObject in vf.views:
+        tables_in_project.append(viewObject.sql_table_name.value.strip())
+
+# Obtain New tables to target generation
+new_tables = set(tables_in_database) - set(tables_in_project)
+
+#Function to convert sqlalchemy to lookml types
+def sa_to_lookml_type(
+        satype: Union[
+            sqlalchemy.Integer
+            ,sqlalchemy.String
+            ,sqlalchemy.TIMESTAMP
+            ,sqlalchemy.Float
+        ]
+        ) -> str:
+    typeMap = {
+         sqlalchemy.Integer:'number'
+        ,sqlalchemy.String:'string'
+        ,sqlalchemy.TIMESTAMP: 'date'
+        ,sqlalchemy.DATE: 'date'
+        ,sqlalchemy.Float: 'number'
+    }
+    return typeMap[type(satype)]
+
+#functions for pretty casing
+def dimName(col:dict)->str:
+    if col['name'] == 'name':
+        return 'name_'
+    else:
+        return col['name'].lower().replace(' ','')
+label = lambda c: c['name'].replace('_',' ').title()
+
+for table in new_tables:
+    schemaName = table.split('.')[0] 
+    tableName = table.split('.')[1]
+    newView = lookml.View(f'{tableName}')
+    newView + f'sql_table_name: {schemaName}.{tableName} ;;'
+    columns = inspector.get_columns(tableName,schemaName)
+    for column in columns:
+        newDim = f'''
+            dimension: {dimName(column)} {{
+                label: "{label(column)}"
+                type: {sa_to_lookml_type(column['type'])}
+                sql: ${{TABLE}}.{column['name']} ;;
+            }}
+        '''
+        newView + newDim
+    newFile = myProject.new_file(f'{tableName}.view.lkml')
+    newFile + newView
+    newFile.write()

example_project/view1.view.lkml

@@ -0,0 +1,5 @@
+view: view2 {
+    sql_table_name: view2 ;;
+    dimension: id {}
+    dimension: cool_field {}
+}

example_project/view2.view.lkml

@@ -0,0 +1,5 @@
+view: view2 {
+    sql_table_name: view2 ;;
+    dimension: id {}
+    dimension: fk {}
+}

requirements.txt

@@ -0,0 +1,45 @@
+attrs==22.1.0
+cachetools==5.2.0
+cattrs==22.1.0
+certifi==2022.6.15
+cffi==1.15.1
+charset-normalizer==2.1.0
+click==7.1.2
+Deprecated==1.2.13
+exceptiongroup==1.0.0rc8
+future==0.18.2
+google-api-core==2.8.2
+google-auth==2.10.0
+google-cloud-bigquery==3.3.1
+google-cloud-bigquery-storage==2.14.1
+google-cloud-core==2.3.2
+google-crc32c==1.3.0
+google-resumable-media==2.3.3
+googleapis-common-protos==1.56.4
+greenlet==1.1.2
+grpcio==1.47.0
+grpcio-status==1.47.0
+idna==3.3
+looker-sdk==0.1.3b20
+lookml==3.0.3
+numpy==1.23.1
+packaging==21.3
+proto-plus==1.22.0
+protobuf==3.20.1
+pyarrow==6.0.1
+pyasn1==0.4.8
+pyasn1-modules==0.2.8
+pycparser==2.21
+PyGithub==1.55
+PyJWT==2.4.0
+PyNaCl==1.5.0
+pyparsing==3.0.9
+python-dateutil==2.8.2
+PyYAML==5.4.1
+requests==2.28.1
+rsa==4.9
+six==1.16.0
+SQLAlchemy==1.4.27
+sqlalchemy-bigquery==1.4.4
+urllib3==1.26.11
+wrapt==1.14.1