Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

task/WI-239: Add audit logs for datafiles/projects endpoints #1047

Open
wants to merge 10 commits into
base: main
Choose a base branch
from
Open

task/WI-239: Add audit logs for datafiles/projects endpoints #1047

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
1876548
add audit logs for datafiles/projects endpoints
jarosenb Jan 30, 2025
b54cce9
unit test fixes
jarosenb Jan 30, 2025
ffdfc38
linting fixes
jarosenb Jan 30, 2025
d4226af
more linting fixes
jarosenb Jan 30, 2025
ea30ea6
unit test fixes
jarosenb Jan 30, 2025
dda21b1
Merge branch 'main' into task/WI-239--audit-logs-datafiles
jarosenb Jan 30, 2025
3dc2329
Merge branch 'main' into task/WI-239--audit-logs-datafiles
rstijerina Feb 5, 2025
1213d12
pass tracking id to shared workspace mkdir
jarosenb Feb 7, 2025
099ebcf
WI-240: auth, apps, jobs audit logs (#1052)
rstijerina Feb 12, 2025
8a1d97d
Merge branch 'main' into task/WI-239--audit-logs-datafiles
rstijerina Feb 12, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 12 additions & 1 deletion server/portal/apps/auth/views.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
)
from portal.apps.search.tasks import index_allocations
from portal.apps.users.utils import check_user_groups
from portal.utils import get_client_ip

logger = logging.getLogger(__name__)
METRICS = logging.getLogger(f'metrics.{__name__}')
Expand Down Expand Up @@ -138,8 +139,18 @@ def tapis_oauth_callback(request):
TapisOAuthToken.objects.update_or_create(user=user, defaults={**token_data})

login(request, user)
METRICS.debug(f"user:{user.username} successful oauth login")
launch_setup_checks(user)
METRICS.info(
"Auth",
extra={
"user": user.username,
"sessionId": getattr(request.session, "session_key", ""),
"operation": "LOGIN",
"agent": request.META.get("HTTP_USER_AGENT"),
"ip": get_client_ip(request),
"info": {},
},
)
else:
messages.error(
request,
Expand Down
12 changes: 6 additions & 6 deletions server/portal/apps/datafiles/handlers/tapis_handlers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,27 +15,27 @@
}


def tapis_get_handler(client, scheme, system, path, operation, **kwargs):
def tapis_get_handler(client, scheme, system, path, operation, tapis_tracking_id=None, **kwargs):
if operation not in allowed_actions[scheme]:
raise PermissionDenied
op = getattr(operations, operation)
return op(client, system, path, **kwargs)
return op(client, system, path, tapis_tracking_id=tapis_tracking_id, **kwargs)


def tapis_post_handler(client, scheme, system,
path, operation, body=None):
path, operation, body=None, tapis_tracking_id=None):
if operation not in allowed_actions[scheme]:
raise PermissionDenied("")

op = getattr(operations, operation)
return op(client, system, path, **body)
return op(client, system, path, tapis_tracking_id=tapis_tracking_id, **body)


def tapis_put_handler(client, scheme, system,
path, operation, body=None):
path, operation, body=None, tapis_tracking_id=None):
if operation not in allowed_actions[scheme]:
raise PermissionDenied

op = getattr(operations, operation)

return op(client, system, path, **body)
return op(client, system, path, tapis_tracking_id=tapis_tracking_id, **body)
150 changes: 124 additions & 26 deletions server/portal/apps/datafiles/views.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
from requests.exceptions import HTTPError
from tapipy.errors import InternalServerError, UnauthorizedError
from portal.views.base import BaseApiView
from portal.utils import get_client_ip
from portal.libs.agave.utils import service_account
from portal.apps.datafiles.handlers.tapis_handlers import (tapis_get_handler,
tapis_put_handler,
Expand All @@ -25,7 +26,7 @@
import dateutil.parser

logger = logging.getLogger(__name__)
METRICS = logging.getLogger('metrics.{}'.format(__name__))
METRICS = logging.getLogger(f"metrics.{__name__}")


class SystemListingView(BaseApiView):
Expand Down Expand Up @@ -97,15 +98,21 @@ def get(self, request, operation=None, scheme=None, system=None, path='/'):
{'message': 'This data requires authentication to view.'},
status=403)
try:
METRICS.info("user:{} op:{} api:tapis scheme:{} "
"system:{} path:{} filesize:{}".format(request.user.username,
operation,
scheme,
system,
path,
request.GET.get('length')))
METRICS.info('Data Files',
extra={
'user': request.user.username,
'sessionId': getattr(request.session, 'session_key', ''),
'operation': operation,
'agent': request.META.get('HTTP_USER_AGENT'),
'ip': get_client_ip(request),
'info': {
'api': 'tapis',
'systemId': system,
'filePath': path,
'query': request.GET.dict()}
})
response = tapis_get_handler(
client, scheme, system, path, operation, **request.GET.dict())
client, scheme, system, path, operation, tapis_tracking_id=f"portals.{request.session.session_key}", **request.GET.dict())

operation in NOTIFY_ACTIONS and \
notify(request.user.username, operation, 'success', {'response': response})
Expand Down Expand Up @@ -140,14 +147,22 @@ def put(self, request, operation=None, scheme=None,
return HttpResponseForbidden("This data requires authentication to view.")

try:
METRICS.info("user:{} op:{} api:tapis scheme:{} "
"system:{} path:{} body:{}".format(request.user.username,
operation,
scheme,
system,
path,
body))
response = tapis_put_handler(client, scheme, system, path, operation, body=body)
METRICS.info('Data Depot',
extra={
'user': request.user.username,
'sessionId': getattr(request.session, 'session_key', ''),
'operation': operation,
'agent': request.META.get('HTTP_USER_AGENT'),
'ip': get_client_ip(request),
'info': {
'api': 'tapis',
'scheme': scheme,
'system': system,
'path': path,
'body': body,
}
})
response = tapis_put_handler(client, scheme, system, path, operation, tapis_tracking_id=f"portals.{request.session.session_key}", body=body)
except Exception as exc:
operation in NOTIFY_ACTIONS and notify(request.user.username, operation, 'error', {})
raise exc
Expand All @@ -163,15 +178,22 @@ def post(self, request, operation=None, scheme=None,
return HttpResponseForbidden("This data requires authentication to upload.")

try:
METRICS.info("user:{} op:{} api:tapis scheme:{} "
"system:{} path:{} filename:{}".format(request.user.username,
operation,
scheme,
system,
path,
body['uploaded_file'].name))

response = tapis_post_handler(client, scheme, system, path, operation, body=body)
METRICS.info('Data Files',
extra={
'user': request.user.username,
'sessionId': getattr(request.session, 'session_key', ''),
'operation': operation,
'agent': request.META.get('HTTP_USER_AGENT'),
'ip': get_client_ip(request),
'info': {
'api': 'tapis',
'scheme': scheme,
'system': system,
'path': path,
'body': request.POST.dict()
}})

response = tapis_post_handler(client, scheme, system, path, operation, tapis_tracking_id=f"portals.{request.session.session_key}", body=body)
except Exception as exc:
operation in NOTIFY_ACTIONS and notify(request.user.username, operation, 'error', {})
raise exc
Expand All @@ -183,6 +205,19 @@ class GoogleDriveFilesView(BaseApiView):
def get(self, request, operation=None, scheme=None, system=None,
path='root'):
try:
METRICS.info('Data Files',
extra={
'user': request.user.username,
'sessionId': getattr(request.session, 'session_key', ''),
'operation': operation,
'agent': request.META.get('HTTP_USER_AGENT'),
'ip': get_client_ip(request),
'info': {
'api': 'googledrive',
'systemId': system,
'filePath': path,
'query': request.GET.dict()}
})
client = request.user.googledrive_user_token.client
except AttributeError:
raise ApiException("Login Required", status=400)
Expand Down Expand Up @@ -230,6 +265,17 @@ def put(self, request, filetype):

src_client = get_client(request.user, body['src_api'])
dest_client = get_client(request.user, body['dest_api'])
METRICS.info('Data Files',
extra={
'user': request.user.username,
'sessionId': getattr(request.session, 'session_key', ''),
'operation': 'transfer',
'agent': request.META.get('HTTP_USER_AGENT'),
'ip': get_client_ip(request),
'info': {
'body': body
}
})

try:
if filetype == 'dir':
Expand Down Expand Up @@ -275,6 +321,19 @@ def get(self, request, scheme, system, path):
"""Given a file, returns a link for a file
"""
try:
METRICS.info('Data Files',
extra={
'user': request.user.username,
'sessionId': getattr(request.session, 'session_key', ''),
'operation': 'retrieve-postit',
'agent': request.META.get('HTTP_USER_AGENT'),
'ip': get_client_ip(request),
'info': {
'api': 'tapis',
'systemId': system,
'filePath': path,
'query': request.GET.dict()}
})
link = Link.objects.get(tapis_uri=f"{system}/{path}")
except Link.DoesNotExist:
return JsonResponse({"data": None, "expiration": None})
Expand All @@ -285,6 +344,19 @@ def delete(self, request, scheme, system, path):
"""Delete an existing link for a file
"""
try:
METRICS.info('Data Files',
extra={
'user': request.user.username,
'sessionId': getattr(request.session, 'session_key', ''),
'operation': 'delete-postit',
'agent': request.META.get('HTTP_USER_AGENT'),
'ip': get_client_ip(request),
'info': {
'api': 'tapis',
'systemId': system,
'filePath': path,
'query': request.GET.dict()}
})
link = Link.objects.get(tapis_uri=f"{system}/{path}")
except Link.DoesNotExist:
raise ApiException("Post-it does not exist")
Expand All @@ -297,6 +369,19 @@ def post(self, request, scheme, system, path):
try:
Link.objects.get(tapis_uri=f"{system}/{path}")
except Link.DoesNotExist:
METRICS.info('Data Files',
extra={
'user': request.user.username,
'sessionId': getattr(request.session, 'session_key', ''),
'operation': 'create-postit',
'agent': request.META.get('HTTP_USER_AGENT'),
'ip': get_client_ip(request),
'info': {
'api': 'tapis',
'systemId': system,
'filePath': path,
'query': request.GET.dict()}
})
# Link doesn't exist - proceed with creating one
postit = self.create_postit(request, scheme, system, path)
return JsonResponse({"data": postit['data'], "expiration": postit['expiration']})
Expand All @@ -307,6 +392,19 @@ def put(self, request, scheme, system, path):
"""Replace an existing link for a file
"""
try:
METRICS.info('Data Files',
extra={
'user': request.user.username,
'sessionId': getattr(request.session, 'session_key', ''),
'operation': 'replace-postit',
'agent': request.META.get('HTTP_USER_AGENT'),
'ip': get_client_ip(request),
'info': {
'api': 'tapis',
'systemId': system,
'filePath': path,
'query': request.GET.dict()}
})
link = Link.objects.get(tapis_uri=f"{system}/{path}")
self.delete_link(request, link)
except Link.DoesNotExist:
Expand Down
12 changes: 3 additions & 9 deletions server/portal/apps/datafiles/views_unit_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -229,9 +229,7 @@ def test_tapis_file_view_get_is_logged_for_metrics(mock_indexer, client, authent
}

# Ensure metric-related logging is being performed
logging_metric_mock.assert_called_with(
"user:{} op:listing api:tapis scheme:private system:frontera.home.username path:test.txt filesize:1234".format(
authenticated_user.username))
logging_metric_mock.assert_called()


@patch('portal.libs.agave.operations.tapis_indexer')
Expand Down Expand Up @@ -264,9 +262,7 @@ def test_tapis_file_view_put_is_logged_for_metrics(mock_indexer, client, authent
assert response.status_code == 200

# Ensure metric-related logging is being performed
logging_metric_mock.assert_called_with(
"user:{} op:move api:tapis scheme:private "
"system:frontera.home.username path:test.txt body:{}".format(authenticated_user.username, body))
logging_metric_mock.assert_called()


@patch('portal.libs.agave.operations.tapis_indexer')
Expand Down Expand Up @@ -309,9 +305,7 @@ def test_tapis_file_view_post_is_logged_for_metrics(mock_indexer, client, authen
assert response.json() == {"data": tapis_file_mock}

# Ensure metric-related logging is being performed
logging_metric_mock.assert_called_with(
"user:{} op:upload api:tapis scheme:private "
"system:frontera.home.username path:/ filename:text_file.txt".format(authenticated_user.username))
logging_metric_mock.assert_called()


@patch('portal.libs.agave.operations.tapis_indexer')
Expand Down
2 changes: 1 addition & 1 deletion server/portal/apps/projects/unit_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,7 @@ def create_shared_workspace(
mock_create_workspace_dir.assert_called()
mock_service_account.assert_called()
mock_service_account().files.mkdir.assert_called_with(
systemId="projects.system.name", path=f"test.project-{workspace_num}"
systemId="projects.system.name", path=f"test.project-{workspace_num}", headers={'X-Tapis-Tracking-ID': ''}
)
# Set Workspace ACLS
# Authenticated_user is whoever the mock_owner or creator of the project is
Expand Down
Loading