diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..c51bb8e --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,39 @@ +name: Run tests + +on: + push: + branches: + - main + - develop + pull_request: + branches: + - main + - develop + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.8", "3.9"] + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install \ + -r requirements.txt \ + flake8 \ + pytest \ + . + - name: Lint + run: | + flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Test + run: | + pytest --capture=sys --ignore=test/test_docker.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7a86eb0..6399338 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,11 +1,12 @@ name: Publish on: + release: + types: + - published push: - tags: - - '*' branches: - - main + - main # publish pre-release packages on every push to main jobs: package: @@ -20,7 +21,7 @@ jobs: # probably be removed. fetch-depth: 0 fetch-tags: true - - name: Set up Python ${{ matrix.python-version }} + - name: Set up Python uses: actions/setup-python@v5 with: python-version: "3.10" @@ -36,22 +37,24 @@ jobs: name: python-package-distributions path: dist/ -# pypi-publish: -# name: Upload release to PyPI -# runs-on: ubuntu-latest -# environment: -# name: release -# url: https://pypi.org/p/aardvark -# permissions: -# id-token: write -# steps: -# - name: Download all the dists -# uses: actions/download-artifact@v4 -# with: -# name: python-package-distributions -# path: dist/ -# - name: Publish package distributions to PyPI -# uses: pypa/gh-action-pypi-publish@release/v1 + pypi-publish: + name: Upload release to PyPI + runs-on: ubuntu-latest + needs: + - package + environment: + name: release + url: https://pypi.org/p/aardvark + permissions: + id-token: write + steps: + - name: Download all the dists + uses: actions/download-artifact@v4 + with: + name: python-package-distributions + path: dist/ + - name: Publish package distributions to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 pypi-test-publish: name: Upload release to TestPyPI diff --git a/README.md b/README.md index 5d52f6a..48de534 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -Aardvark +Aardvark - Multi-Account AWS IAM Access Advisor API ======== [![NetflixOSS Lifecycle](https://img.shields.io/osslifecycle/Netflix/osstracker.svg)]() [![Discord chat](https://img.shields.io/discord/754080763070382130?logo=discord)](https://discord.gg/9kwMWa6) diff --git a/aardvark/__init__.py b/aardvark/__init__.py index 0fd02d8..e69de29 100644 --- a/aardvark/__init__.py +++ b/aardvark/__init__.py @@ -1,88 +0,0 @@ -import logging -import os.path -from logging.config import dictConfig - -from dynaconf.contrib import FlaskDynaconf -from flasgger import Swagger -from flask import Flask - -from aardvark.advisors import advisor_bp -from aardvark.persistence.sqlalchemy import SQLAlchemyPersistence - -BLUEPRINTS = [advisor_bp] - -API_VERSION = "1" - -log = logging.getLogger("aardvark") - - -def create_app(**kwargs): - init_logging() - app = Flask(__name__, static_url_path="/static") - Swagger(app) - persistence = SQLAlchemyPersistence() - - FlaskDynaconf(app, **kwargs) - - # For ELB and/or Eureka - @app.route("/healthcheck") - def healthcheck(): - """Healthcheck - Simple healthcheck that indicates the services is up - --- - responses: - 200: - description: service is up - """ - return "ok" - - # Blueprints - for bp in BLUEPRINTS: - app.register_blueprint(bp, url_prefix=f"/api/{API_VERSION}") - - # Extensions: - persistence.init_db() - - return app - - -def init_logging(): - log_cfg = { - "version": 1, - "disable_existing_loggers": False, - "formatters": { - "standard": {"format": "%(asctime)s %(levelname)s: %(message)s " "[in %(pathname)s:%(lineno)d]"} - }, - "handlers": { - "file": { - "class": "logging.handlers.RotatingFileHandler", - "level": "DEBUG", - "formatter": "standard", - "filename": "aardvark.log", - "maxBytes": 10485760, - "backupCount": 100, - "encoding": "utf8", - }, - "console": { - "class": "logging.StreamHandler", - "level": "DEBUG", - "formatter": "standard", - "stream": "ext://sys.stdout", - }, - }, - "loggers": {"aardvark": {"handlers": ["file", "console"], "level": "DEBUG"}}, - } - dictConfig(log_cfg) - - -def _find_config(): - """Search for config.py in order of preference and return path if it exists, else None""" - config_paths = [ - os.path.join(os.getcwd(), "config.py"), - "/etc/aardvark/config.py", - "/apps/aardvark/config.py", - ] - for path in config_paths: - if os.path.exists(path): - return path - return None diff --git a/aardvark/app.py b/aardvark/app.py new file mode 100644 index 0000000..29b8e60 --- /dev/null +++ b/aardvark/app.py @@ -0,0 +1,89 @@ +from __future__ import annotations + +import os.path +import logging +import sys +from logging import DEBUG, Formatter, StreamHandler +from logging.config import dictConfig +from typing import TYPE_CHECKING + +from flask_sqlalchemy import SQLAlchemy +from flask import Flask +from flasgger import Swagger + +if TYPE_CHECKING: + from flask import Config + +db = SQLAlchemy() + +from aardvark.view import mod as advisor_bp # noqa + +BLUEPRINTS = [ + advisor_bp +] + +API_VERSION = "1" + + +def create_app(config_override: Config = None): + app = Flask(__name__, static_url_path="/static") + Swagger(app) + + if config_override: + app.config.from_mapping(config_override) + else: + path = _find_config() + if not path: + print("No config") + app.config.from_pyfile("_config.py") + else: + app.config.from_pyfile(path) + + # For ELB and/or Eureka + @app.route("/healthcheck") + def healthcheck(): + """Healthcheck + Simple healthcheck that indicates the services is up + --- + responses: + 200: + description: service is up + """ + return "ok" + + # Blueprints + for bp in BLUEPRINTS: + app.register_blueprint(bp, url_prefix=f"/api/{API_VERSION}") + + # Extensions: + db.init_app(app) + setup_logging(app) + + return app + + +def _find_config(): + """Search for config.py in order of preference and return path if it exists, else None""" + config_paths = [ + os.path.join(os.getcwd(), "config.py"), + "/etc/aardvark/config.py", + "/apps/aardvark/config.py", + ] + for path in config_paths: + if os.path.exists(path): + return path + return None + + +def setup_logging(app): + if not app.debug: + if app.config.get("LOG_CFG"): + # initialize the Flask logger (removes all handlers) + dictConfig(app.config.get("LOG_CFG")) + app.logger = logging.getLogger(__name__) + else: + handler = StreamHandler(stream=sys.stderr) + + handler.setFormatter(Formatter("%(asctime)s %(levelname)s: %(message)s [in %(pathname)s:%(lineno)d]")) + app.logger.setLevel(app.config.get("LOG_LEVEL", DEBUG)) + app.logger.addHandler(handler) diff --git a/aardvark/model.py b/aardvark/model.py new file mode 100644 index 0000000..e69de29 diff --git a/aardvark/updater/__init__.py b/aardvark/updater/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aardvark/utils/sqla_regex.py b/aardvark/utils/sqla_regex.py index 939088b..f98acb2 100644 --- a/aardvark/utils/sqla_regex.py +++ b/aardvark/utils/sqla_regex.py @@ -4,6 +4,7 @@ """ # courtesy of Xion: http://xion.io/post/code/sqlalchemy-regex-filters.html + import re import sqlite3 diff --git a/aardvark/view.py b/aardvark/view.py new file mode 100644 index 0000000..e69de29 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..e69de29 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..e69de29 diff --git a/test/conftest.py b/test/conftest.py new file mode 100644 index 0000000..a7a58b4 --- /dev/null +++ b/test/conftest.py @@ -0,0 +1,34 @@ +import pytest +from flask_sqlalchemy import SQLAlchemy + + +@pytest.fixture(scope="function") +def app_config(): + """Return a Flask configuration object for testing. The returned configuration is intended to be a good base for + testing and can be customized for specific testing needs. + """ + from flask import Config + c = Config('.') + c.from_mapping({ + "SQLALCHEMY_DATABASE_URI": "sqlite:///:memory:", + "SQLALCHEMY_TRACK_MODIFICATIONS": False, + "DEBUG": False, + "LOG_CFG": {'version': 1, 'handlers': []}, # silence logging + }) + return c + + +@pytest.fixture(scope="function") +def mock_database(app_config): + """Yield an instance of flask_sqlalchemy.SQLAlchemy associated with the base model class used in aardvark.model. + This is almost certainly not safe for parallel/multi-threaded use. + """ + from aardvark.app import db + mock_db = SQLAlchemy(model_class=db.Model) + + from aardvark.app import create_app + app = create_app(config_override=app_config) + with app.app_context(): + mock_db.create_all() + yield mock_db + mock_db.drop_all() diff --git a/test/test_model.py b/test/test_model.py new file mode 100644 index 0000000..3780acf --- /dev/null +++ b/test/test_model.py @@ -0,0 +1,146 @@ +def test_advisor_data_create(mock_database): + from aardvark.model import AdvisorData, db + AdvisorData.create_or_update( + 9999, + 1111, + "Swifties United", + "swift", + "taylor", + 1, + ) + db.session.commit() + + record: AdvisorData = AdvisorData.query.filter( + AdvisorData.item_id == 9999, + AdvisorData.serviceNamespace == "swift", + ).scalar() + assert record + assert record.id + assert record.item_id == 9999 + assert record.lastAuthenticated == 1111 + assert record.lastAuthenticatedEntity == "taylor" + assert record.serviceName == "Swifties United" + assert record.serviceNamespace == "swift" + assert record.totalAuthenticatedEntities == 1 + + +def test_advisor_data_update(mock_database): + from aardvark.model import AdvisorData, db + AdvisorData.create_or_update( + 9999, + 0, + "Pink Pony Club", + "roan", + None, + 0, + ) + db.session.commit() + + record: AdvisorData = db.session.query(AdvisorData).filter( + AdvisorData.id == 1, + ).scalar() + assert record + assert record.lastAuthenticated == 0 + + AdvisorData.create_or_update( + 9999, + 1111, + "Pink Pony Club", + "roan", + "chappell", + 1, + ) + db.session.commit() + + record: AdvisorData = db.session.query(AdvisorData).filter( + AdvisorData.id == 1, + ).scalar() + assert record + assert record.item_id == 9999 + assert record.lastAuthenticated == 1111 + assert record.lastAuthenticatedEntity == "chappell" + assert record.serviceName == "Pink Pony Club" + assert record.serviceNamespace == "roan" + assert record.totalAuthenticatedEntities == 1 + + +def test_advisor_data_update_older_last_authenticated(mock_database): + from aardvark.model import AdvisorData, db + AdvisorData.create_or_update( + 9999, + 1111, + "Pink Pony Club", + "roan", + "chappell", + 1, + ) + db.session.commit() + + record: AdvisorData = db.session.query(AdvisorData).filter( + AdvisorData.id == 1, + ).scalar() + assert record + assert record.lastAuthenticated == 1111 + + # Calling create_or_update with a lower lastAuthenticated value should NOT update lastAuthenticated in the DB + AdvisorData.create_or_update( + 9999, + 1000, + "Pink Pony Club", + "roan", + "chappell", + 1, + ) + db.session.commit() + + record: AdvisorData = db.session.query(AdvisorData).filter( + AdvisorData.id == 1, + ).scalar() + assert record + assert record.item_id == 9999 + assert record.lastAuthenticated == 1111 + assert record.lastAuthenticatedEntity == "chappell" + assert record.serviceName == "Pink Pony Club" + assert record.serviceNamespace == "roan" + assert record.totalAuthenticatedEntities == 1 + + +def test_advisor_data_update_zero_last_authenticated(mock_database): + from aardvark.model import AdvisorData, db + AdvisorData.create_or_update( + 9999, + 1111, + "Pink Pony Club", + "roan", + "chappell", + 1, + ) + db.session.commit() + + record: AdvisorData = db.session.query(AdvisorData).filter( + AdvisorData.id == 1, + ).scalar() + assert record + assert record.lastAuthenticated == 1111 + + # Calling create_or_update with a zero lastAuthenticated value SHOULD update lastAuthenticated in the DB + AdvisorData.create_or_update( + 9999, + 0, + "Pink Pony Club", + "roan", + "", + 0, + ) + db.session.commit() + + record: AdvisorData = db.session.query(AdvisorData).filter( + AdvisorData.id == 1, + ).scalar() + assert record + assert record.item_id == 9999 + assert record.lastAuthenticated == 0 + assert record.lastAuthenticatedEntity == "" + assert record.serviceName == "Pink Pony Club" + assert record.serviceNamespace == "roan" + assert record.totalAuthenticatedEntities == 0