Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: ims-service accepts new profiles #5

Merged
merged 25 commits into from
Dec 30, 2024
Merged
Changes from 1 commit
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
34f8560
add proxy_web_service, event_store to ims_service
mackenzie-grimes-noaa Dec 18, 2024
2d6e4ff
bug fixes with filenaming, writing json, health endpoint
mackenzie-grimes-noaa Dec 18, 2024
424833f
rename EventStore to ProfileStore
mackenzie-grimes-noaa Dec 18, 2024
fad680e
delete unnecessary private methods
mackenzie-grimes-noaa Dec 18, 2024
a644bfb
move ims_service dir to nwsc_dummy_service
mackenzie-grimes-noaa Dec 18, 2024
f39cd5e
load IMS responses as individual support profiles on startup, NamedTu…
mackenzie-grimes-noaa Dec 18, 2024
c72c42a
add .id attribute to CachedProfile
mackenzie-grimes-noaa Dec 18, 2024
4b73ec5
drop ims_service from setup.py, update docker/local/Dockerfile
mackenzie-grimes-noaa Dec 18, 2024
caa722c
add test_profile_store unit tests
mackenzie-grimes-noaa Dec 18, 2024
7fd21ac
rename nwsc_dummy_web_service to ncd_web_service
mackenzie-grimes-noaa Dec 18, 2024
7049977
Add unit test GitHub Action, add dummy service to linter
mackenzie-grimes-noaa Dec 18, 2024
e79aa31
improve test coverage, fix unit test import
mackenzie-grimes-noaa Dec 18, 2024
c2470fe
ignore pylint warning about "id" variable
mackenzie-grimes-noaa Dec 18, 2024
f9a9def
increase test_ncd_web_service.py unit test coverage
mackenzie-grimes-noaa Dec 19, 2024
e97d3f0
add flask to run-tests.yml, linter ignore pytest syntax
mackenzie-grimes-noaa Dec 19, 2024
b23926f
store GSL_KEY in app.config
mackenzie-grimes-noaa Dec 19, 2024
019330e
minor code cleanup
mackenzie-grimes-noaa Dec 19, 2024
889e9e4
allow dataSource 'ANY'
mackenzie-grimes-noaa Dec 19, 2024
0b12e6b
fix Dockerfile
mackenzie-grimes-noaa Dec 19, 2024
175ae71
move docker/mockims to docker/dummyservice
mackenzie-grimes-noaa Dec 19, 2024
a80f35f
update readme.md
mackenzie-grimes-noaa Dec 19, 2024
8ab95f6
hide /test and __init__.py from test coverage
mackenzie-grimes-noaa Dec 19, 2024
1587839
rename service to to nwsc_proxy
mackenzie-grimes-noaa Dec 19, 2024
16b9fa4
rename a couple more "ims" references
mackenzie-grimes-noaa Dec 19, 2024
ecf31a0
Merge branch 'main' into proxy-accepts-new-profiles
mackenzie-grimes-noaa Dec 20, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
delete unnecessary private methods
  • Loading branch information
mackenzie-grimes-noaa committed Dec 18, 2024
commit fad680e2179748c195ad993fd6e7835f0e547cb2
114 changes: 36 additions & 78 deletions python/idsse/testing/ims_service/src/profile_store.py
Original file line number Diff line number Diff line change
@@ -33,10 +33,7 @@ def __init__(self, base_dir: str):
os.mkdir(_dir)

# cache of JSON data of all Support Profiles, divided into new vs. existing Profiles
self.new_profiles = []
self.existing_profiles = []

logger.info('Loading existing Support Profiles from path: %s', self._existing_dir)
self.new_profiles: list[dict] = []
self.existing_profiles = self._read_existing_profiles()

def get_all(self, filter_new_profiles = False) -> list[dict]:
@@ -60,8 +57,17 @@ def save(self, profile: dict) -> str | None:
str | None: UUID of saved Support Profile on success, otherwise None
"""
logger.debug('Now saving new profile: %s', profile)
# save to JSON file and add to in-memory cache
return self._save_profile(profile, is_new=True)
profile_id = profile.get('id')

# determine the right filepath where JSON data will be written
filepath = os.path.join(self._new_dir , profile_id + '.json')

logger.info('Now saving profile to path: %s', filepath)
with open(filepath, 'w', encoding='utf-8') as file:
json.dump(profile, file) # save JSON to filesystem

self.new_profiles.append(profile) # add profile to in-memory cache
return profile_id

def move_to_existing(self, profile_id: str) -> bool:
"""Mark a formerly "new" Support Profile as "existing", a.k.a. has been returned in
@@ -90,10 +96,13 @@ def move_to_existing(self, profile_id: str) -> bool:
profile_id)

# unexpectedly, profile is not in new_profiles cache;
# recover from this by re-reading the JSON from file, because file did exist
profile_data = self._read_existing_profile(profile_id)
# recover from this by re-reading the JSON from file, because file must exist
filename = os.path.join(self._existing_dir, profile_id + '.json')
logger.debug('Attempting to read existing profile from path: %s', filename)
with open(filename, 'r', encoding='utf-8') as file:
profile_data = json.load(file)

# add Profile/Profile to existing_profiles cache, and scrub from new_profiles cache
# add Profile to existing_profiles cache, and scrub from new_profiles cache
self.existing_profiles.append(profile_data)
self.new_profiles = [profile for profile in self.existing_profiles
if profile['id'] != profile_id]
@@ -106,49 +115,32 @@ def delete(self, profile_id: str) -> bool:
Returns:
bool: True on success
"""
logger.debug('Attempting to delete profile_id %s', profile_id)
is_deleted = self._delete_profile(profile_id)
if not is_deleted:
return False

# drop profile from new or existing cache (could be in either)
self.new_profiles = [profile for profile in self.new_profiles
if profile['id'] != profile_id]
self.existing_profiles = [profile for profile in self.existing_profiles
if profile['id'] != profile_id]

return True

# private methods that do the actual disk read/write operations
def _save_profile(self, profile: dict, is_new: bool) -> str:
"""Writes profile JSON to disk and adds the JSON to appropriate in-memory cache:
`self.new_profiles` if `is_new`, otherwise `self.existing_profiles`.
logger.info('Deleting profile_id %s', profile_id)
filepath = os.path.join(self._existing_dir, profile_id + '.json')
profile_cache = self.existing_profiles # assume profile is in "existing" cache

Args:
profile (dict): full JSON data to be saved
is_new (bool): if True, profile will be saved in subdirectory and in-memory cache for
"new" profiles. Otherwise will be stored in "existing" profiles subdir/cache.
Returns:
str: the profile_id, if save was successful
"""
profile_id = profile.get('id')
if not os.path.exists(filepath):
# profile does not in exist in "existing" subdirectory, maybe its in "new"
filepath = os.path.join(self._existing_dir, profile_id + '.json')
profile_cache = self.new_profiles # now assume profile is in "new" cache

# determine the right filepath where JSON data will be written
dir_path = self._new_dir if is_new else self._existing_dir
filepath = os.path.join(dir_path, profile_id + '.json')
if not os.path.exists(filepath):
logger.warning('Cannot delete profile %s; JSON file not found in %s or %s',
profile_id, self._existing_dir, self._new_dir)
return False

# will be saved to the appropriate in-memory list of profile data
profile_cache = self.new_profiles if is_new else self.existing_profiles
logger.debug('Attempting to delete profile at path: %s', filepath)
os.remove(filepath)

logger.info('Now saving profile to path: %s', filepath)
with open(filepath, 'w', encoding='utf-8') as file:
json.dump(profile, file) # save JSON to filesystem
# drop profile from new/existing cache (whichever subdirectory it was found in)
profile_cache = [profile for profile in profile_cache if profile['id'] != profile_id]

profile_cache.append(profile)
return profile_id
return True

def _read_existing_profiles(self) -> list[dict]:
"""Read all JSON files from this ProfileStore's `existing` subdirectory"""
logger.info('Loading existing Support Profiles from path: %s', self._existing_dir)

profile_list: list[dict] = []
for filename in glob('*.json', root_dir=self._existing_dir):
with open(os.path.join(self._existing_dir, filename), 'r', encoding='utf-8') as file:
@@ -162,37 +154,3 @@ def _read_existing_profiles(self) -> list[dict]:
profile_list.append(json_data)

return profile_list

def _read_existing_profile(self, profile_id: str) -> dict:
"""Get an existing profile from disk based on the profile ID.
Reads from NEW subdirectory should almost never be necessary.
"""
filename = os.path.join(self._existing_dir, profile_id + '.json')
logger.debug('Attempting to read existing profile from path: %s', filename)
with open(filename, 'r', encoding='utf-8') as file:
return json.load(file)

def _write_new_profile(self, data: dict):
"""Write a new Support Profile Profile to disk using `self._new_dir`"""
profile_id = data['id']
with open(os.path.join(self._new_dir, profile_id + '.json'), 'w', encoding='utf-8') as file:
os.write(file, data)

def _delete_profile(self, profile_id: str) -> bool:
"""Delete profile from disk by profile_id.
Returns:
bool: True on success, False if JSON file not found
"""
filepath = os.path.join(self._existing_dir, profile_id + '.json')
if not os.path.exists(filepath):
# profile does not in exist in existing subdirectory, maybe its in the new one
filepath = os.path.join(self._existing_dir, profile_id + '.json')

if not os.path.exists(filepath):
logger.warning('Cannot delete profile %s; JSON file not found in %s or %s',
profile_id, self._existing_dir, self._new_dir)
return False

logger.debug('Attempting to delete profile at path: %s', filepath)
os.remove(filepath)
return True