-
Notifications
You must be signed in to change notification settings - Fork 230
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #1231 from swirlai/revert-1224-DS-1723
Revert "Bump cryptography and Django, remove snowflake."
- Loading branch information
Showing
10 changed files
with
302 additions
and
99 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,30 @@ | ||
| { | ||
| "name": "Free Company Records - Snowflake", | ||
| "active": false, | ||
| "default": false, | ||
| "authenticator": "", | ||
| "connector": "Snowflake", | ||
| "url": "<snowflake-instance-address>", | ||
| "query_template": "SELECT {fields} FROM {table} WHERE {field1} ILIKE '%{query_string}%' AND NULLIF(TRIM(founded), '') IS NOT NULL ORDER BY TRY_TO_NUMBER(REGEXP_REPLACE(SPLIT_PART(size, '-', 1), '[^0-9]', '')) DESC;", | ||
| "post_query_template": {}, | ||
| "http_request_headers": {}, | ||
| "page_fetch_config_json": {}, | ||
| "query_processors": [ | ||
| "AdaptiveQueryProcessor" | ||
| ], | ||
| "query_mappings": "fields=*,sort_by_date=founded,table=FREECOMPANYDATASET,field1=name", | ||
| "result_grouping_field": "", | ||
| "result_processors": [ | ||
| "MappingResultProcessor", | ||
| "CosineRelevancyResultProcessor" | ||
| ], | ||
| "response_mappings": "", | ||
| "result_mappings": "title='{name} ({founded})',body='{name} was founded in {founded} in {country}. It has {size} employees and operates in the {industry} industry.',url='https://{linkedin_url}',date_published=founded,NO_PAYLOAD", | ||
| "results_per_query": 10, | ||
| "credentials": "<username>:<password>:FREE_COMPANY_DATASET:COMPUTE_WH", | ||
| "eval_credentials": "", | ||
| "tags": [ | ||
| "Company", | ||
| "Snowflake" | ||
| ] | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Large diffs are not rendered by default.
Oops, something went wrong.
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,92 @@ | ||
| ''' | ||
| @author: Sid Probstein | ||
| @contact: [email protected] | ||
| ''' | ||
|
|
||
| from sys import path | ||
| from os import environ | ||
|
|
||
| import snowflake.connector | ||
| from snowflake.connector import ProgrammingError | ||
|
|
||
| import json | ||
|
|
||
| import django | ||
|
|
||
| from swirl.utils import swirl_setdir | ||
| path.append(swirl_setdir()) # path to settings.py file | ||
| environ.setdefault('DJANGO_SETTINGS_MODULE', 'swirl_server.settings') | ||
| django.setup() | ||
|
|
||
| from celery.utils.log import get_task_logger | ||
| from logging import DEBUG | ||
| logger = get_task_logger(__name__) | ||
| # logger.setLevel(DEBUG) | ||
|
|
||
| from swirl.connectors.db_connector import DBConnector | ||
| from swirl.connectors.utils import bind_query_mappings | ||
|
|
||
| ######################################## | ||
| ######################################## | ||
|
|
||
| class Snowflake(DBConnector): | ||
|
|
||
| type = "Snowflake" | ||
|
|
||
| ######################################## | ||
|
|
||
| def execute_search(self, session=None): | ||
|
|
||
| logger.debug(f"{self}: execute_search()") | ||
|
|
||
| if self.provider.credentials: | ||
| if ':' in self.provider.credentials: | ||
| credlist = self.provider.credentials.split(':') | ||
| if len(credlist) == 4: | ||
| username = credlist[0] | ||
| password = credlist[1] | ||
| database = credlist[2] | ||
| warehouse = credlist[3] | ||
| else: | ||
| self.warning("Invalid credentials, should be: username:password:database:warehouse") | ||
| else: | ||
| self.warning("No credentials!") | ||
| account = self.provider.url | ||
|
|
||
| try: | ||
| # Create a new connection | ||
| conn = snowflake.connector.connect(user=username, password=password, account=account) | ||
| cursor = conn.cursor() | ||
| cursor.execute(f"USE WAREHOUSE {warehouse}") | ||
| cursor.execute(f"USE DATABASE {database}") | ||
|
|
||
| cursor.execute(self.count_query) | ||
| count_result = cursor.fetchone() | ||
| found = count_result[0] if count_result else 0 | ||
| if found == 0: | ||
| self.message(f"Retrieved 0 of 0 results from: {self.provider.name}") | ||
| self.status = 'READY' | ||
| self.found = 0 | ||
| self.retrieved = 0 | ||
| return | ||
|
|
||
| cursor.execute(self.query_to_provider) | ||
| self.column_names = [col[0].lower() for col in cursor.description] | ||
| results = cursor.fetchall() | ||
|
|
||
| except ProgrammingError as err: | ||
| self.error(f"{err} querying {self.type}") | ||
| self.status = 'ERR' | ||
| cursor.close() | ||
| conn.close() | ||
| return | ||
|
|
||
| self.response = list(results) | ||
|
|
||
| cursor.close() | ||
| conn.close() | ||
|
|
||
| self.found = found | ||
| self.retrieved = self.provider.results_per_query | ||
| return | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters