Skip to content

Commit

Permalink
Commit after formatters applied (black, isort)
Browse files Browse the repository at this point in the history
  • Loading branch information
jgrethe committed Nov 30, 2023
1 parent bd20f0a commit 1cfc66a
Show file tree
Hide file tree
Showing 8 changed files with 48 additions and 52 deletions.
1 change: 1 addition & 0 deletions docs/conf.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

import os
import sys

_HERE = os.path.dirname(__file__)
sys.path.insert(0, os.path.abspath(os.path.join(_HERE, '../src')))

Expand Down
1 change: 1 addition & 0 deletions docs/examples_metadata.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
# Instructions for getting an API key can be found at: https://fdilab.gitbook.io/api-handbook/sparc-k-core-api-overview/getting-started-with-sparc-apis

import json

from sparc.client import SparcClient

client = SparcClient(connect=False, config_file='../config/config.ini')
Expand Down
81 changes: 39 additions & 42 deletions src/sparc/client/services/metadata.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
import logging
import json
import logging
from configparser import SectionProxy
from typing import List, Optional, Union

import requests
from requests.adapters import HTTPAdapter, Retry

from configparser import SectionProxy
from typing import List, Optional, Union
from ._default import ServiceBase


Expand Down Expand Up @@ -75,7 +75,7 @@ def __init__(
self.connect()

def connect(self) -> str:
""" Not needed as metadata services are REST service calls """
"""Not needed as metadata services are REST service calls"""
logging.info("Metadata REST services available...")

self.host_api = "https://scicrunch.org/api/1/elastic"
Expand Down Expand Up @@ -115,21 +115,20 @@ def close(self) -> None:
"""Not needed as metadata services are REST service calls"""
return self.host_api

#####################################################################
# Supporting Functions
#####################################################################
# Supporting Functions

#####################################################################
# Function to GET content from URL with retries
#####################################################################
# Function to GET content from URL with retries
def getURL(self, url, headers="NONE"):

result = "[ERROR]"
url_session = requests.Session()

retries = Retry(total=6,
backoff_factor=1,
status_forcelist=[403, 404, 413, 429, 500, 502, 503, 504])
retries = Retry(
total=6, backoff_factor=1, status_forcelist=[403, 404, 413, 429, 500, 502, 503, 504]
)

url_session.mount('https://', HTTPAdapter(max_retries=retries))
url_session.mount("https://", HTTPAdapter(max_retries=retries))

success = 1

Expand Down Expand Up @@ -166,35 +165,33 @@ def getURL(self, url, headers="NONE"):

return result.json()


#####################################################################
# Function to retrieve content via POST from URL with retries
#####################################################################
# Function to retrieve content via POST from URL with retries
def postURL(self, url, body, headers="NONE"):

result = "[ERROR]"
url_session = requests.Session()

retries = Retry(total=6,
backoff_factor=1,
status_forcelist=[403, 404, 413, 429, 500, 502, 503, 504])
retries = Retry(
total=6, backoff_factor=1, status_forcelist=[403, 404, 413, 429, 500, 502, 503, 504]
)

url_session.mount('https://', HTTPAdapter(max_retries=retries))
url_session.mount("https://", HTTPAdapter(max_retries=retries))

try:
if type(body) is dict:
body_json = body
else:
body_json = json.loads(body)
except:
logging.error("Elasticsearch query body can not be read")
logging.error("Elasticsearch query body can not be read")

success = 1

try:
if headers == "NONE":
url_result = url_session.post(url, json = body_json)
url_result = url_session.post(url, json=body_json)
else:
url_result = url_session.post(url, json = body_json, headers=headers)
url_result = url_session.post(url, json=body_json, headers=headers)

if url_result.status_code == 410:
logging.warning("Retrieval Status 410 - URL Unpublished:" + url)
Expand Down Expand Up @@ -222,16 +219,11 @@ def postURL(self, url, body, headers="NONE"):
result = {}

return result.json()


#####################################################################
# Metadata Search Functions
#####################################################################
# Metadata Search Functions

def list_datasets(
self,
limit: int = 10,
offset: int = 0
) -> list:
def list_datasets(self, limit: int = 10, offset: int = 0) -> list:
"""Lists datasets and associated metadata.
Parameters:
Expand All @@ -248,19 +240,24 @@ def list_datasets(
"""
self.host_api = "https://scicrunch.org/api/1/elastic/SPARC_Algolia_pr/_search"

list_url = self.host_api + "?" + "from=" + str(offset) + "&size=" + str(limit) + "&key=" + self.scicrunch_api_key
list_url = (
self.host_api
+ "?"
+ "from="
+ str(offset)
+ "&size="
+ str(limit)
+ "&key="
+ self.scicrunch_api_key
)

list_results = self.getURL(list_url, headers=self.default_headers)
return list_results

def search_datasets(self, query: str = '{"query": { "match_all": {}}}') -> list:
"""Gets datasets matching specified query.
def search_datasets(
self,
query: str = "{\"query\": { \"match_all\": {}}}"
) -> list:
"""Gets datasets matching specified query.
This function provides
This function provides
Parameters:
-----------
Expand All @@ -277,5 +274,5 @@ def search_datasets(

list_url = self.host_api + "?" + "key=" + self.scicrunch_api_key

list_results = self.postURL(list_url, body = query, headers=self.default_headers)
list_results = self.postURL(list_url, body=query, headers=self.default_headers)
return list_results
5 changes: 3 additions & 2 deletions src/sparc/client/services/pennsieve.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
import logging
import os
from configparser import SectionProxy
from typing import List, Optional, Union

import requests
from pennsieve2 import Pennsieve
from configparser import SectionProxy
from typing import List, Optional, Union

from ._default import ServiceBase


Expand Down
2 changes: 1 addition & 1 deletion src/sparc/client/zinchelper.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@
import os
import re

from cmlibs.exporter.vtk import ArgonSceneExporter as VTKExporter
from cmlibs.exporter.stl import ArgonSceneExporter as STLExporter
from cmlibs.exporter.vtk import ArgonSceneExporter as VTKExporter
from cmlibs.utils.zinc.field import get_group_list
from cmlibs.zinc.context import Context
from cmlibs.zinc.result import RESULT_OK
Expand Down
1 change: 1 addition & 0 deletions tests/test_pennsieve.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import json

from pennsieve2 import Pennsieve

from sparc.client.services.pennsieve import PennsieveService
Expand Down
1 change: 0 additions & 1 deletion tests/test_pennsieve_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@

@responses.activate
def test_register_responses():

responses.add(
method=responses.GET,
url="http://api.pennsieve.io/datasets/",
Expand Down
8 changes: 2 additions & 6 deletions tests/test_zinc.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,7 @@ def zinc():

def test_export_scaffold_into_vtk_format(zinc):
# create a temporary output file
output_location = os.path.abspath(
os.path.join(os.path.dirname(__file__), "resources/")
)
output_location = os.path.abspath(os.path.join(os.path.dirname(__file__), "resources/"))

# ensure the function returns None if the dataset has no Scaffold_Creator-settings.json file
invalid_dataset_id = 1000000
Expand Down Expand Up @@ -42,9 +40,7 @@ def test_export_scaffold_into_vtk_format(zinc):

def test_export_scaffold_into_stl_format(zinc):
# create a temporary output file
output_location = os.path.abspath(
os.path.join(os.path.dirname(__file__), "resources/")
)
output_location = os.path.abspath(os.path.join(os.path.dirname(__file__), "resources/"))

# ensure the function returns None if the dataset has no Scaffold_Creator-settings.json file
invalid_dataset_id = 1000000
Expand Down

0 comments on commit 1cfc66a

Please sign in to comment.