diff --git a/.build_properties b/.build_properties new file mode 100644 index 00000000..50769aa3 --- /dev/null +++ b/.build_properties @@ -0,0 +1,3 @@ +MODULE_NAME=brewtils +PACKAGE_NAME=brewtils +BUILD_IMAGE=beer-garden/build diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 00000000..f5a4da89 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,4 @@ + +[report] +show_missing = True + diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..526d3639 --- /dev/null +++ b/.gitignore @@ -0,0 +1,75 @@ + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +env/ +build/ +develop-eggs/ +dist/ +downloads/ +lib/ +lib64/ +parts/ +sdist/ +var/ +eggs/ +.eggs/ +*.egg-info/ +*.egg +.installed.cfg +.Python + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/otehr infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +output/ +htmlcov/ +coverage/ +cover/ +.tox/ +.hypothesis/ +.cache +.coverage +.coverage.* +nosetests.xml +coverage.xml +*.cover + +# Translations +*.mo +*.pot + +# Swap files +*.swp + +# Django stuff: +*.log + +# Documentation +docs/_build/ +docs/generated_docs/ + +# PyBuilder +target/ + +# Ipython Notebook +.ipynb_checkpoints + +# IDE stuff +.idea/ +.editorconfig + diff --git a/.venv b/.venv new file mode 100644 index 00000000..4122e7bf --- /dev/null +++ b/.venv @@ -0,0 +1 @@ +brewtils diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..f08d2097 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,76 @@ +# CHANGELOG + +## [2.3.0] +Date: 1/26/18 +#### Added Features +- Added methods for interacting with the Queue API to RestClient and EasyClient (#329) +- Clients and Plugins can now be configured to skip server certificate verification when making HTTPS requests (#326) +- Timestamps now have true millisecond precision on platforms that support it (#325) +- Added `form_input_type` to Parameter model (#294) +- Plugins can now be stopped correctly by calling their `_stop` method (#263) +- Added Event model (#21) + +#### Bug Fixes +- Plugins now additionally look for `ca_cert` and `client_cert` in `BG_CA_CERT` and `BG_CLIENT_CERT` (#326) + +#### Other Changes +- Better data integrity by only allowing certain Request status transitions (#214) + +## [2.2.1] +1/11/18 +#### Bug Fixes +- Nested requests that reference a different BEERGARDEN no longer fail (#313) + +## [2.2.0] +10/23/17 +#### Added Features +- Command descriptions can now be changed without updating the System version (#225) +- Standardized Remote Plugin logging configuration (#168) +- Added domain-specific language for dynamic choices configuration (#130) +- Added `metadata` field to Instance model + +#### Bug Fixes +- Removed some default values from model `__init__` functions (#237) +- System descriptors (description, display name, icon name, metadata) now always updated during startup (#213, #228) +- Requests with output type 'JSON' will now have JSON error messages (#92) + +#### Other changes +- Added license file + +## [2.1.1] +8/25/17 +#### Added Features +- Added `updated_at` field to `Request` model (#182) +- `SystemClient` now allows specifying a `client_cert` (#178) +- `RestClient` now reuses the same session for subsequent connections (#174) +- `SystemClient` can now make non-blocking requests (#121) +- `RestClient` and `EasyClient` now support PATCHing a `System` + +#### Deprecations / Removals +- `multithreaded` argument to `PluginBase` has been superseded by `max_concurrent` +- These decorators are now deprecated (#164): + - `@command_registrar`, instead use `@system` + - `@plugin_param`, instead use `@parameter` + - `@register`, instead use `@command` +- These classes are now deprecated (#165): + - `BrewmasterSchemaParser`, instead use `SchemaParser` + - `BrewmasterRestClient`, instead use `RestClient` + - `BrewmasterEasyClient`, instead use `EasyClient` + - `BrewmasterSystemClient`, instead use `SystemClient` + +#### Bug fixes +- Reworked message processing to remove the possibility of a failed request being stuck in 'IN_PROGRESS' (#183, #210) +- Correctly handle custom form definitions with a top-level array (#177) +- Smarter reconnect logic when the RabbitMQ connection fails (#83) + +#### Other changes +- Removed dependency on `pyopenssl` so there's need to compile any Python extensions (#196) +- Request processing now occurs inside of a `ThreadPoolExecutor` thread (#183) +- Better serialization handling for epoch fields (#167) + + +[unreleased]: https://github.com/beer-garden/bindings/compare/master...develop +[2.3.0]: https://github.com/beer-garden/bindings/compare/2.2.1...2.3.0 +[2.2.1]: https://github.com/beer-garden/bindings/compare/2.2.0...2.2.1 +[2.2.0]: https://github.com/beer-garden/bindings/compare/2.1.1...2.2.0 +[2.1.1]: https://github.com/beer-garden/bindings/compare/2.1.0...2.1.1 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..83484d9e --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,17 @@ +Contributing to the Bindings +========= + +Since many people have many different uses for Beergarden, it makes sense that you may need functionality that's not currently supported. If that's the case, and you think you can add the functionality yourself, feel free! Here's the easiest way to make changes: + +1. Clone or fork the project (`git clone git@github.com:beer-garden/brewtils.git`) +2. Create a new branch (`git checkout -b my_amazing_feature`) +3. Commit your changes (`git commit -m "It's done!"`) +4. Push to the branch (`git push origin my_amazing_feature`) +5. Create a new Merge Request in GitHub (https://github.com/beer-garden/brewtils/merge_requests/new) + +We want to do everything we can to make sure we're delivering robust software. So we just ask that before submitting a merge request you: + +1. Make sure all the existing tests work (`nosetests` for the Python bindings) +2. Create new tests for the functionality you've created. These should work too :) + +Finally, __thank you for your contribution!__ Your help is very much appreciated by the Beergarden developers and users. diff --git a/Dockerfile.test b/Dockerfile.test new file mode 100644 index 00000000..10b0e52e --- /dev/null +++ b/Dockerfile.test @@ -0,0 +1 @@ +FROM beer-garden/tox:latest diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 00000000..07240ba6 --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,19 @@ +The MIT License + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO TEH WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 00000000..b7baed70 --- /dev/null +++ b/README.md @@ -0,0 +1,148 @@ +Brewtils +========= + +Brewtils is the Python library for interfacing with Beergarden systems. It provides simple ways to query information, access systems, generate requests, and more. + +## Installation +Brewtils lives on PyPI. You can install it via: + + pip install brewtils + +If you already have a requirements definition file you can add brewtils like this: + + brewtils + +And then install with pip as normal. + +## Usage + +There are three major ways to use `brewtils`. A brief summary of each is given below so you can determine which best fits your use-case. + + +### Remote Plugin Decorators + +The `decorators` module helps you create your own remote plugin. Suppose you have the following class: + +```python +Class MyClient(object): + + def do_something(self): + print("Hello, World!") + return "Hello, World!" + + def echo_message(self, message): + print(message) + return message +``` + +There are two steps for making this class into a Beergarden plugin. First, you'll need to decorate your class and methods: + +```python +from brewtils.decorators import system, parameter, command + +@system +Class MyClient(object): + + @command + def do_something(self): + print("Hello, World!") + return "Hello, World!" + + @parameter(key="message", description="The message to echo", type="String") + def echo_message(self, message): + print(message) + return message +``` + +The `@system` tells us that the `MyClient` class is going to be a Beergarden plugin. + +The `@command` tells us that `do_something` is going to be a command. + +The `@parameter` tells us information about the parameters to the `echo_message` method. + +Now that we've decorated the client definition we just need to point the remote plugin at a Beergarden and start it. We can do that like this: + +```python +from brewtils.plugin import RemotePlugin + +#...MyClient definition... + +def main(): + client = MyClient() + plugin = RemotePlugin(client, name="My Client", version="0.0.1", bg_host='127.0.0.1', bg_port=2337) + plugin.run() + +if __name__ == "__main__": + main() +``` + +Assuming you have a Beergarden running on port 2337 on localhost, running this will register and start your plugin. + + +### System Client + +The `SystemClient` is designed to help you interact with registered Systems as if they were native Python objects. Suppose the following System has been registered with Beergarden: + + System: + Name: foo + Version: 0.0.1 + Commands: + do_something1 + Params: + key1 + key2 + + do_something2 + Params: + key3 + key4 + +That is, we have a System named "foo" with two possible commands: `do_something1` and `do_something2`, each of which takes 2 parameters (key1-4). + +Now suppose we want to exercise `do_something1` and inspect the result. The `SystemClient` makes this trivial: + +```python +from brewtils.rest.system_client import SystemClient + +foo_client = SystemClient('localhost', 2337, 'foo') + +request = foo_client.do_something1(key1="k1", key2="k2") + +print(request.status) # 'SUCCESS' +print(request.output) # do_something1 output +``` +When you call `do_something1` on the `SystemClient` object it will make a REST call to Beergarden to generate a Request. It will then block until that request has completed, at which point the returned object can be queried the same way as any other Request object. + +If the system you are using has multiple instances, you can specify the default instance to use: + +```python +foo_client = SystemClient('localhost', 2337, 'foo', default_instance="01") + +foo_client.do_something1(key1="k1", key2="k2") # Will set instance_name to '01' +``` + +If you want to operate on multiple instances of the same system, you can specify the instance name each time: + +```python +foo_client = SystemClient('localhost', 2337, 'foo') +request = foo_client.do_something1(key1="k1", key2="k2", _instance_name="01") # Will set instance_name to '01' +``` + +Notice the leading `_` in the `_instance_name` keyword argument. This is necessary to distinguish command arguments (things to pass to `do_something1`) from Beergarden arguments. In general, you should try to avoid naming parameters with a leading underscore to avoid name collisions. + +### Easy Client + +The `EasyClient` is intended to make it easy to directly query Beergarden. Suppose the same `foo` System as above has been registered in Beergarden. We can use the `EasyClient` to gather information: + +```python +from brewtils.rest.easy_client import EasyClient + +client = EasyClient('localhost', 2337) + +foo_system = client.find_unique_system(name='foo', version='0.0.1') + +for command in foo_system.commands: + print(command.name) +``` + +The `EasyClient` is full of helpful commands so feel free to explore all the methods that are exposed. diff --git a/bin/bump_and_tag_version.sh b/bin/bump_and_tag_version.sh new file mode 100755 index 00000000..5159a18f --- /dev/null +++ b/bin/bump_and_tag_version.sh @@ -0,0 +1,82 @@ +#!/bin/bash + +# works with a file called _version.py in the top module directory, +# the contents of which should be a semantic version number +# such as '__version__ = "1.2.3"' + +# This script will display the current version, automatically +# suggest a "patch" version update, and ask for input to use +# the suggestion, or a newly entered value. + +# once the new version number is determined, the script will +# create a GIT tag. + +BINDIR=$(dirname $0) +BASEDIR=$(dirname $BINDIR) +MODULE_NAME="brewtils" +VERSION="$BASEDIR/$MODULE_NAME/_version.py" + +if [ -f $VERSION ]; then + BASE_STRING=$(cat $VERSION | cut -d'"' -f2) + BASE_LIST=(`echo $BASE_STRING | tr '.' ' '`) + V_MAJOR=${BASE_LIST[0]} + V_MINOR=${BASE_LIST[1]} + V_PATCH=${BASE_LIST[2]} + echo "Current Version: $BASE_STRING" + + V_PATCH=$((V_PATCH + 1)) + SUGGESTED_VERSION="$V_MAJOR.$V_MINOR.$V_PATCH" + read -p "Enter a version number [$SUGGESTED_VERSION]: " INPUT_STRING + if [ "$INPUT_STRING" = "" ]; then + INPUT_STRING=$SUGGESTED_VERSION + fi + if [[ ! "$INPUT_STRING" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "Bad Version $INPUT_STRING. Needs to match X.X.X where X is a number." + exit 1 + fi + + # Before we mess with git make sure the local branch is up-to-date + echo "Fetching upstream changes..." + git fetch + + LOCAL=$(git rev-parse @{0}) + REMOTE=$(git rev-parse @{u}) + BASE=$(git merge-base @{0} @{u}) + if [ $LOCAL = $REMOTE ]; then + echo "Local and remote branches are up-to-date" + elif [ $LOCAL = $BASE ]; then + echo ""; echo "Local branch is behind upstream. Have you run 'git pull'?" + exit 1 + elif [ $REMOTE = $BASE ]; then + echo "Branch push will be a fast-forward" + else + echo ""; echo "Local branch has diverged from the upstream, please merge" + exit 1 + fi + + echo "Will set new version to be $INPUT_STRING" + echo "__version__ = \"$INPUT_STRING\"" > $VERSION + git add $VERSION + + # There is a jenkins job based on this commit message. Please be sure you update it if you + # choose to change the commit message string from something other than "Version bump to" + git commit -m "Version bump to $INPUT_STRING" + git tag -a -m "Tagging version $INPUT_STRING" "$INPUT_STRING" + git push --tags origin HEAD +else + echo "Could not find a version file." + read -p "Do you want to create a version file and start from scratch? [y]" RESPONSE + if [ "$RESPONSE" = "" ]; then RESPONSE="y"; fi + if [ "$RESPONSE" = "Y" ]; then RESPONSE="y"; fi + if [ "$RESPONSE" = "Yes" ]; then RESPONSE="y"; fi + if [ "$RESPONSE" = "yes" ]; then RESPONSE="y"; fi + if [ "$RESPONSE" = "YES" ]; then RESPONSE="y"; fi + if [ "$RESPONSE" = "y" ]; then + echo "__version__ = \"0.0.1\"" > $VERSION + git add $VERSION + git commit -m "Added $VERSION file, Version bump to v0.0.1" + git tag -a -m "Tagging version 0.0.1" "0.0.1" + git push origin + git push origin --tags + fi +fi diff --git a/bin/generate_coverage.sh b/bin/generate_coverage.sh new file mode 100755 index 00000000..5a4884a5 --- /dev/null +++ b/bin/generate_coverage.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env sh + +mkdir -p output/python + +nosetests --with-xcoverage --cover-package=brewtils --cover-tests --cover-erase \ + --xcoverage-file=output/python/cobertura.xml \ + --with-xunit --xunit-file=output/python/test-report.xml \ + --cover-html --cover-html-dir=output/python/html \ + --rednose test + diff --git a/bin/make_docs.sh b/bin/make_docs.sh new file mode 100755 index 00000000..209247c6 --- /dev/null +++ b/bin/make_docs.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +# This script uses sphinx utilities to generate documenation +# from Python docstrings + +BASEDIR=$(dirname $(dirname $0)) +make -C "$BASEDIR/docs" diff --git a/bin/run-pylint.py b/bin/run-pylint.py new file mode 100755 index 00000000..8c4babb1 --- /dev/null +++ b/bin/run-pylint.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python + + +import os +import sys +import subprocess +import re + +if len(sys.argv) > 2: + PYLINT = sys.argv[2] + '/pylint' +else: + PYLINT = 'pylint' + +BASE_PATH = sys.argv[1] +OUTPUT_FILE = None +EXTRA_LIBS = [] +DISABLED_SETTINGS = [] +IGNORE_PATTERNS = [] +ADDITIONAL_PARAMETERS = ["--max-args=11", "--max-attributes=15", "--max-line-length=120", "--disable=bad-builtin,locally-disabled"] +CODE_RATING = re.compile(r'Your code has been rated at ([-0-9].*)/10 \(previous run: ([-0-9.]*)/10') +FILE_NAME = re.compile(r'[-a-zA-Z0-9_/]*\.py') + + + + + +def setup_paths(): + old_pythonpath = None + old_path = os.environ['PATH'] + for path in EXTRA_LIBS: + os.environ['PATH'] += os.pathsep + path + if not os.environ.get("PYTHONPATH"): + os.environ['PYTHONPATH'] = '' + else: + old_pythonpath = os.environ['PYTHONPATH'] + for path in EXTRA_LIBS: + os.environ["PYTHONPATH"] += os.pathsep + path + return old_path, old_pythonpath + + +def reset_paths(old_path, old_pythonpath=None): + os.environ['PATH'] = old_path + if old_pythonpath: + os.environ['PYTHONPATH'] = old_pythonpath + else: + del os.environ['PYTHONPATH'] + + +def construct_command(): + command = [PYLINT, BASE_PATH, '-f', 'parseable'] + if DISABLED_SETTINGS: + command.append('--disable=%s' % ','.join(DISABLED_SETTINGS)) + if IGNORE_PATTERNS: + command.append('--ignore=%s' % ','.join(IGNORE_PATTERNS)) + if ADDITIONAL_PARAMETERS: + command.extend(ADDITIONAL_PARAMETERS) + return command + + +def run_pylint(): + os.chdir(BASE_PATH) + command = construct_command() + try: + output = subprocess.check_output(command, stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as e: + output = e.output + match = CODE_RATING.search(output) + if not match or float(match.group(1)) < float(match.group(2)): + exitcode = 1 + else: + exitcode = 0 + if OUTPUT_FILE: + with open(OUTPUT_FILE, 'w') as fd: + fd.write(output) + return exitcode, output + + +def add_file_paths(input): + output = '' + for line in input.split('\n'): + if FILE_NAME.match(line): + output += '%s/%s' % (BASE_PATH, line) + else: + output += line + output += '\n' + return output + + +def main(): + old_path, old_pythonpath = setup_paths() + exitcode, output = run_pylint() + output = add_file_paths(output) + reset_paths(old_path, old_pythonpath) + print output + sys.exit(exitcode) + +if __name__ == '__main__': + main() diff --git a/bin/run-tox.sh b/bin/run-tox.sh new file mode 100755 index 00000000..ea88ae7a --- /dev/null +++ b/bin/run-tox.sh @@ -0,0 +1,20 @@ +set -ex + +SCRIPT_PATH=$( cd $(dirname $0) ; pwd -P) +SRC_PATH=$( cd $(dirname $(dirname $0 )) ; pwd -P ) + +if [ -d $SRC_PATH/output ]; then + rm -rf $SRC_PATH/output +fi +if [ -d $SRC_PATH/docs/_build ]; then + rm -rf $SRC_PATH/docs/_build +fi + +mkdir -p $SRC_PATH/docs/_build +chmod 777 $SRC_PATH/docs/_build + +mkdir -p $SRC_PATH/output +chmod 777 $SRC_PATH/output + + +docker run --rm -v $SRC_PATH:/src brewtils:test diff --git a/brewtils/__init__.py b/brewtils/__init__.py new file mode 100644 index 00000000..8e62ee84 --- /dev/null +++ b/brewtils/__init__.py @@ -0,0 +1,72 @@ +import os + +import six + +from brewtils.rest import normalize_url_prefix +from ._version import __version__ as generated_version + +__version__ = generated_version + + +def get_easy_client(**kwargs): + """Initialize an EasyClient using Environment variables as default values + + :param kwargs: Options for configuring the EasyClient + :return: An EasyClient + """ + from brewtils.rest.easy_client import EasyClient + + parser = kwargs.pop('parser', None) + logger = kwargs.pop('logger', None) + + return EasyClient(logger=logger, parser=parser, **get_bg_connection_parameters(**kwargs)) + + +def get_bg_connection_parameters(**kwargs): + """Parse the keyword arguments, search in the arguments, and environment for the values + + :param kwargs: + :return: + """ + from brewtils.rest.client import RestClient + from brewtils.errors import BrewmasterValidationError + + host = kwargs.pop('host', None) or os.environ.get('BG_WEB_HOST') + if not host: + raise BrewmasterValidationError('Unable to create a plugin without a BEERGARDEN host. Please specify one ' + 'with bg_host= or by setting the BG_WEB_HOST environment variable.') + + port = kwargs.pop('port', None) or os.environ.get('BG_WEB_PORT', '2337') + + url_prefix = kwargs.pop('url_prefix', None) or os.environ.get('BG_URL_PREFIX', None) + url_prefix = normalize_url_prefix(url_prefix) + + # Default to true + ssl_enabled = kwargs.pop('ssl_enabled', None) + if ssl_enabled is not None: + ssl_enabled = ssl_enabled.lower() != "false" if isinstance(ssl_enabled, six.string_types) else bool(ssl_enabled) + else: + ssl_enabled = os.environ.get('BG_SSL_ENABLED', 'true').lower() != 'false' + + # Default to true + ca_verify = kwargs.pop('ca_verify', None) + if ca_verify is not None: + ca_verify = ca_verify.lower() != "false" if isinstance(ca_verify, six.string_types) else bool(ca_verify) + else: + ca_verify = os.environ.get('BG_CA_VERIFY', 'true').lower() != 'false' + + api_version = kwargs.pop('api_version', RestClient.LATEST_VERSION) + ca_cert = kwargs.pop('ca_cert', None) or os.environ.get('BG_CA_CERT') or os.environ.get('BG_SSL_CA_CERT') + client_cert = kwargs.pop('client_cert', None) or os.environ.get('BG_CLIENT_CERT') or \ + os.environ.get('BG_SSL_CLIENT_CERT') + + return { + 'host': host, + 'port': port, + 'ssl_enabled': ssl_enabled, + 'api_version': api_version, + 'ca_cert': ca_cert, + 'client_cert': client_cert, + 'url_prefix': url_prefix, + 'ca_verify': ca_verify + } diff --git a/brewtils/_version.py b/brewtils/_version.py new file mode 100644 index 00000000..55e47090 --- /dev/null +++ b/brewtils/_version.py @@ -0,0 +1 @@ +__version__ = "2.3.0" diff --git a/brewtils/choices.py b/brewtils/choices.py new file mode 100644 index 00000000..3c9c000b --- /dev/null +++ b/brewtils/choices.py @@ -0,0 +1,79 @@ +from lark import Lark, Transformer +from lark.common import ParseError + + +choices_grammar = """ + func: CNAME [func_args] + url: ADDRESS [url_args] + reference: ref + + func_args: "(" [arg_pair ("," arg_pair)*] ")" + url_args: "?" arg_pair ("&" arg_pair)* + + arg_pair: CNAME "=" ref + ?ref: "${" CNAME "}" + + ADDRESS: /^http[^\?]*/ + + %import common.CNAME + %import common.WS + %ignore WS +""" + +parsers = { + 'func': Lark(choices_grammar, start='func'), + 'url': Lark(choices_grammar, start='url'), + 'reference': Lark(choices_grammar, start='reference') +} + + +class FunctionTransformer(Transformer): + + @staticmethod + def func(s): + return { + 'name': str(s[0]), + 'args': s[1] if len(s) > 1 else [] + } + + @staticmethod + def url(s): + return { + 'address': str(s[0]), + 'args': s[1] if len(s) > 1 else [] + } + + @staticmethod + def reference(s): + return str(s[0]) + + @staticmethod + def arg_pair(s): + return str(s[0]), str(s[1]) + + func_args = list + url_args = list + + +def parse(input_string, parse_as=None): + """Attempt to parse a string into a choices dictionary. + + :param input_string: The string to parse + :param parse_as: String specifying how to parse `input_string`. Valid values are 'func' or 'url'. Will try all valid + values if None. + :return: A dictionary containing the results of the parse + :raise lark.common.ParseError: The parser was not able to find a valid parsing of `input_string` + """ + def _parse(_input_string, _parser): + return FunctionTransformer().transform(_parser.parse(_input_string)) + + if parse_as is not None: + return _parse(input_string, parsers[parse_as]) + else: + for parser in parsers.values(): + try: + return _parse(input_string, parser) + except ParseError: + continue + + raise ParseError('Unable to successfully parse input "%s"' % input_string) diff --git a/brewtils/decorators.py b/brewtils/decorators.py new file mode 100644 index 00000000..ee213462 --- /dev/null +++ b/brewtils/decorators.py @@ -0,0 +1,406 @@ +import functools +import inspect +import json +import os + +import requests +import six +import types +import wrapt +from lark.common import ParseError + +from brewtils.choices import parse +from brewtils.errors import PluginParamError +from brewtils.models import Command, Parameter, Choices + +__all__ = ['system', 'parameter', 'command', 'command_registrar', 'plugin_param', 'register'] + + +# The wrapt module has a cool feature where you can disable wrapping a decorated function, instead just using the +# original function. This is pretty much exactly what we want - we aren't using decorators for their 'real' purpose of +# wrapping a function, we just want to add some metadata to the function object. So we'll disable the wrapping normally, +# but we need to test that enabling the wrapping would work. +_wrap_functions = False + + +def system(cls): + """Class decorator that marks a class as a BEERGARDEN System + + Creates a _commands property on the class that holds all registered commands. + + :param cls: The class to decorated + :return: The decorated class + """ + commands = [] + for method_name in dir(cls): + method = getattr(cls, method_name) + method_command = getattr(method, '_command', None) + if method_command: + commands.append(method_command) + + cls._commands = commands + + return cls + + +def command(_wrapped=None, command_type='ACTION', output_type='STRING', schema=None, form=None, template=None, + icon_name=None): + """Decorator that marks a function as a BEERGARDEN command + + For example: + + @command(output_type='JSON') + def echo_json(self, message): + return message + + :param _wrapped: The function to decorate. This is handled as a positional argument and shouldn't be explicitly set. + :param command_type: The command type. Valid options are Command.COMMAND_TYPES. + :param output_type: The output type. Valid options are Command.OUTPUT_TYPES. + :param schema: A custom schema definition. + :param form: A custom form definition. + :param template: A custom template definition. + :param icon_name: The icon name. Should be either a FontAwesome or a Glyphicon name (e.g. fa-heart). + :return: The decorated function. + """ + if _wrapped is None: + return functools.partial(command, command_type=command_type, output_type=output_type, schema=schema, form=form, + template=template, icon_name=icon_name) + + generated_command = _generate_command_from_function(_wrapped) + generated_command.command_type = command_type + generated_command.output_type = output_type + generated_command.icon_name = icon_name + + resolved_mod = _resolve_display_modifiers(_wrapped, generated_command.name, schema=schema, form=form, + template=template) + generated_command.schema = resolved_mod['schema'] + generated_command.form = resolved_mod['form'] + generated_command.template = resolved_mod['template'] + + func_command = getattr(_wrapped, '_command', None) + if func_command: + _update_func_command(func_command, generated_command) + else: + _wrapped._command = generated_command + + @wrapt.decorator(enabled=_wrap_functions) + def wrapper(_double_wrapped, _, _args, _kwargs): + return _double_wrapped(*_args, **_kwargs) + + return wrapper(_wrapped) + + +def parameter(_wrapped=None, key=None, type=None, multi=None, display_name=None, optional=None, default=None, + description=None, choices=None, nullable=None, maximum=None, minimum=None, regex=None, is_kwarg=None, + model=None, form_input_type=None): + """Decorator that enables Parameter specifications for a BEERGARDEN Command + + This decorator is intended to be used when more specification is desired for a specific Parameter. + + For example: + + @parameter(key="message", description="Message to echo", optional=True, type="String", default="Hello, World!") + def echo(self, message): + return message + + :param _wrapped: The function to decorate. This is handled as a positional argument and shouldn't be explicitly set. + :param key: String specifying the parameter identifier. Must match an argument name of the decorated function. + :param type: String indicating the type to use for this parameter. + :param multi: Boolean indicating if this parameter is a multi. See documentation for discussion of what this means. + :param display_name: String that will be displayed as a label in the user interface. + :param optional: Boolean indicating if this parameter must be specified. + :param default: The value this parameter will be assigned if not overridden when creating a request. + :param description: An additional string that will be displayed in the user interface. + :param choices: List or dictionary specifying allowed values. See documentation for more information. + :param nullable: Boolean indicating if this parameter is allowed to be null. + :param maximum: Integer indicating the maximum value of the parameter. + :param minimum: Integer indicating the minimum value of the parameter. + :param regex: String describing a regular expression constraint on the parameter. + :param is_kwarg: Boolean indicating if this parameter is meant to be part of the decorated function's kwargs. + :param model: Class to be used as a model for this parameter. Must be a Python type object, not an instance. + :param form_input_type: Only used for string fields. Changes the form input field (e.g. textarea, email, password) + :return: The decorated function. + """ + if _wrapped is None: + return functools.partial(parameter, key=key, type=type, multi=multi, display_name=display_name, + optional=optional, default=default, description=description, choices=choices, + nullable=nullable, maximum=maximum, minimum=minimum, regex=regex, is_kwarg=is_kwarg, + model=model, form_input_type=form_input_type) + + # First see if this method already has a command object associated. If not, create one. + cmd = getattr(_wrapped, '_command', None) + if not cmd: + cmd = _generate_command_from_function(_wrapped) + _wrapped._command = cmd + + # Every parameter needs a key, so stop that right here + if key is None: + raise PluginParamError("Found a parameter definition without a key for command '%s'" % cmd.name) + + # If the command doesn't already have a parameter with this key then the method doesn't have an explicit keyword + # argument with as the name. That's only OK if this parameter is meant to be part of the **kwargs. + param = cmd.get_parameter_by_key(key) + if param is None: + if is_kwarg: + param = Parameter(key=key, optional=False) + cmd.parameters.append(param) + else: + raise PluginParamError(("Parameter '%s' was not an explicit keyword argument for command '%s' and was " + + "not marked as part of kwargs (should is_kwarg be True?)") % (key, cmd.name)) + + # Update parameter definition with the plugin_param arguments + param.type = param.type if type is None else type + param.multi = param.multi if multi is None else multi + param.display_name = param.display_name if display_name is None else display_name + param.optional = param.optional if optional is None else optional + param.default = param.default if default is None else default + param.description = param.description if description is None else description + param.choices = param.choices if choices is None else choices + param.nullable = param.nullable if nullable is None else nullable + param.maximum = param.maximum if maximum is None else maximum + param.minimum = param.minimum if minimum is None else minimum + param.regex = param.regex if regex is None else regex + param.form_input_type = param.form_input_type if form_input_type is None else form_input_type + + param.choices = _format_choices(param.choices) + + # Model is another special case - it requires its own handling + if model is not None: + param.type = 'Dictionary' + param.parameters = _generate_nested_params(model) + + # If the model is not nullable and does not have a default defined we will try to generate a default using + # the defaults defined on the model parameters + if not param.nullable and not param.default: + param.default = {} + for nested_param in param.parameters: + if nested_param.default: + param.default[nested_param.key] = nested_param.default + + @wrapt.decorator(enabled=_wrap_functions) + def wrapper(_double_wrapped, _, _args, _kwargs): + return _double_wrapped(*_args, **_kwargs) + + return wrapper(_wrapped) + + +def _update_func_command(func_command, generated_command): + """Updates the current function's command with info, (will not override plugin_params)""" + func_command.name = generated_command.name + func_command.description = generated_command.description + func_command.command_type = generated_command.command_type + func_command.output_type = generated_command.output_type + func_command.schema = generated_command.schema + func_command.form = generated_command.form + func_command.template = generated_command.template + func_command.icon_name = generated_command.icon_name + + +def _generate_command_from_function(func): + """Generates a Command from a function. Uses first line of pydoc as the description.""" + # Required for Python 2/3 compatibility + if hasattr(func, "func_name"): + command_name = func.func_name + else: + command_name = func.__name__ + + # Required for Python 2/3 compatibility + if hasattr(func, "func_doc"): + docstring = func.func_doc + else: + docstring = func.__doc__ + + return Command(name=command_name, description=docstring.split('\n')[0] if docstring else None, + parameters=_generate_params_from_function(func)) + + +def _generate_params_from_function(func): + """Generate Parameters from function arguments. Will set the Parameter key, default value, and optional value.""" + parameters_to_return = [] + + code = six.get_function_code(func) + function_arguments = list(code.co_varnames or [])[:code.co_argcount] + function_defaults = list(six.get_function_defaults(func) or []) + + while len(function_defaults) != len(function_arguments): + function_defaults.insert(0, None) + + for index, param_name in enumerate(function_arguments): + # Skip Self or Class reference + if index == 0 and isinstance(func, types.FunctionType): + continue + + default = function_defaults[index] + optional = False if default is None else True + + parameters_to_return.append(Parameter(key=param_name, default=default, optional=optional)) + + return parameters_to_return + + +def _generate_nested_params(model_class): + """Generates Nested Parameters from a Model Class""" + parameters_to_return = [] + for parameter_definition in model_class.parameters: + key = parameter_definition.key + parameter_type = parameter_definition.type + multi = parameter_definition.multi + display_name = parameter_definition.display_name + optional = parameter_definition.optional + default = parameter_definition.default + description = parameter_definition.description + nullable = parameter_definition.nullable + maximum = parameter_definition.maximum + minimum = parameter_definition.minimum + regex = parameter_definition.regex + + choices = _format_choices(parameter_definition.choices) + + nested_parameters = [] + if parameter_definition.parameters: + parameter_type = 'Dictionary' + for nested_class in parameter_definition.parameters: + nested_parameters = _generate_nested_params(nested_class) + + parameters_to_return.append(Parameter(key=key, type=parameter_type, multi=multi, display_name=display_name, + optional=optional, default=default, description=description, + choices=choices, parameters=nested_parameters, nullable=nullable, + maximum=maximum, minimum=minimum, regex=regex)) + return parameters_to_return + + +def _resolve_display_modifiers(wrapped, command_name, schema=None, form=None, template=None): + + def _load_from_url(url): + return json.loads(requests.get(url).text) + + def _load_from_path(path): + current_dir = os.path.dirname(inspect.getfile(wrapped)) + file_path = os.path.abspath(os.path.join(current_dir, path)) + + with open(file_path, 'r') as definition_file: + return definition_file.read() + + resolved = {} + + for key, value in {'schema': schema, 'form': form, 'template': template}.items(): + + if isinstance(value, six.string_types): + try: + if value.startswith('http'): + resolved[key] = _load_from_url(value) + + elif value.startswith('/') or value.startswith('.'): + loaded_value = _load_from_path(value) + resolved[key] = loaded_value if key == 'template' else json.loads(loaded_value) + + elif key == 'template': + resolved[key] = value + + else: + raise PluginParamError("%s specified for command '%s' was not a definition, file path, or URL" % + (key, command_name)) + except Exception as ex: + raise PluginParamError("Error reading %s definition from '%s' for command '%s': %s" % + (key, value, command_name, ex)) + + elif value is None or (key in ['schema', 'form'] and isinstance(value, dict)): + resolved[key] = value + + elif key == 'form' and isinstance(value, list): + resolved[key] = {'type': 'fieldset', 'items': value} + + else: + raise PluginParamError("%s specified for command '%s' was not a definition, file path, or URL" % + (key, command_name)) + + return resolved + + +def _format_choices(choices): + + def determine_display(display_value): + if isinstance(display_value, six.string_types): + return 'typeahead' + + return 'select' if len(display_value) <= 50 else 'typeahead' + + def determine_type(type_value): + if isinstance(type_value, (list, dict)): + return 'static' + elif type_value.startswith('http'): + return 'url' + else: + return 'command' + + if not choices: + return None + + if not isinstance(choices, (list, six.string_types, dict)): + raise PluginParamError("Invalid 'choices' provided. Must be a list, dictionary or string.") + + elif isinstance(choices, dict): + if not choices.get('value'): + raise PluginParamError("No 'value' provided for choices. You must at least provide valid values.") + + value = choices.get('value') + display = choices.get('display', determine_display(value)) + choice_type = choices.get('type') + strict = choices.get('strict', True) + + if choice_type is None: + choice_type = determine_type(value) + elif choice_type not in Choices.TYPES: + raise PluginParamError("Invalid choices type '%s' - Valid type options are %s" % + (choice_type, Choices.TYPES)) + else: + if (choice_type == 'command' and not isinstance(value, (six.string_types, dict))) \ + or (choice_type == 'url' and not isinstance(value, six.string_types)) \ + or (choice_type == 'static' and not isinstance(value, (list, dict))): + allowed_types = {'command': "('string', 'dictionary')", 'url': "('string')", + 'static': "('list', 'dictionary)"} + raise PluginParamError("Invalid choices value type '%s' - Valid value types for choice type '%s' are %s" + % (type(value), choice_type, allowed_types[choice_type])) + + if display not in Choices.DISPLAYS: + raise PluginParamError("Invalid choices display '%s' - Valid display options are %s" % + (display, Choices.DISPLAYS)) + else: + value = choices + display = determine_display(value) + choice_type = determine_type(value) + strict = True + + # Now parse out type-specific aspects + unparsed_value = '' + try: + if choice_type == 'command': + if isinstance(value, six.string_types): + unparsed_value = value + else: + unparsed_value = value['command'] + + details = parse(unparsed_value, parse_as='func') + elif choice_type == 'url': + unparsed_value = value + details = parse(unparsed_value, parse_as='url') + else: + if isinstance(value, dict): + unparsed_value = choices.get('key_reference') + if unparsed_value is None: + raise PluginParamError('Specifying a static choices dictionary requires a "key_reference" field ' + 'with a reference to another parameter ("key_reference": "${param_key}")') + + details = {'key_reference': parse(unparsed_value, parse_as='reference')} + else: + details = {} + except ParseError: + raise PluginParamError("Invalid choices definition - Unable to parse '%s'" % unparsed_value) + + return Choices(type=choice_type, display=display, value=value, strict=strict, details=details) + + +# Alias the old names for compatibility +command_registrar = system +plugin_param = parameter +register = command diff --git a/brewtils/errors.py b/brewtils/errors.py new file mode 100644 index 00000000..517a3561 --- /dev/null +++ b/brewtils/errors.py @@ -0,0 +1,113 @@ +"""Module containing all of the BREWMASTER error definitions""" + + +# Models +class BrewmasterModelError(Exception): + """Wrapper Error for All BrewmasterModelErrors""" + pass + + +class BrewmasterModelValidationError(BrewmasterModelError): + """Error to indicate an invalid Brewmaster Model""" + pass + + +class RequestStatusTransitionError(BrewmasterModelValidationError): + """Error to indicate an updated status was not a valid transition""" + pass + + +# Plugins +class PluginError(Exception): + """Generic error class""" + pass + + +class PluginValidationError(PluginError): + """Plugin could not be validated successfully""" + pass + + +class PluginParamError(PluginError): + """Error used when plugins have illegal parameters""" + pass + + +# Requests +class AckAndContinueException(Exception): + pass + + +class NoAckAndDieException(Exception): + pass + + +class AckAndDieException(Exception): + pass + + +class DiscardMessageException(Exception): + """Raising an instance will result in a message not being requeued""" + pass + + +class RepublishRequestException(Exception): + """Republish to the end of the message queue + + :param request: The Request to republish + :param headers: A dictionary of headers to be used by `brewtils.request_consumer.RequestConsumer` + :type request: :py:class:`brewtils.models.Request` + """ + def __init__(self, request, headers): + self.request = request + self.headers = headers + + +class RequestProcessingError(AckAndContinueException): + pass + + +# Rest / Client errors +class BrewmasterRestError(Exception): + """Wrapper Error to Wrap more specific BREWMASTER Rest Errors""" + pass + + +class BrewmasterConnectionError(BrewmasterRestError): + """Error indicating a connection error while performing a request""" + pass + + +class BrewmasterTimeoutError(BrewmasterRestError): + """Error Indicating a Timeout was reached while performing a request""" + pass + + +class BrewmasterFetchError(BrewmasterRestError): + """Error Indicating a server Error occurred performing a GET""" + pass + + +class BrewmasterValidationError(BrewmasterRestError): + """Error Indicating a client (400) Error occurred performing a POST/PUT""" + pass + + +class BrewmasterSaveError(BrewmasterRestError): + """Error Indicating a server Error occurred performing a POST/PUT""" + pass + + +class BrewmasterDeleteError(BrewmasterRestError): + """Error Indicating a server Error occurred performing a DELETE""" + pass + + +class BGConflictError(BrewmasterRestError): + """Error indicating a 409 was raised on the server""" + pass + + +class BGNotFoundError(BrewmasterRestError): + """Error Indicating a 404 was raised on the server""" + pass \ No newline at end of file diff --git a/brewtils/log/__init__.py b/brewtils/log/__init__.py new file mode 100644 index 00000000..a000c2dc --- /dev/null +++ b/brewtils/log/__init__.py @@ -0,0 +1,141 @@ +"""Brewtils Logging Utilities + +This module is for setting up your plugins logging correctly. + +Example: + In order to use this, you should simply call ``setup_logger`` in the same file where you initialize your + plugin sometime before you initialize your Plugin object. + + host = 'localhost' + port = 2337 + ssl_enabled = False + system_name = 'my_system' + + setup_logger(bg_host=host, bg_port=port, system_name=system_name, ssl_enabled=ssl_enabled) + plugin = Plugin(my_client, bg_host=host, bg_port=port, ssl_enabled=ssl_enabled, + name=system_name, version="0.0.1") + plugin.run() +""" + +import logging.config +import copy +import brewtils + +# Loggers to always use. These are things that generally, people do not want to see and/or are too verbose. +DEFAULT_LOGGERS = { + "pika": { + "level": "ERROR" + }, + "requests.packages.urllib3.connectionpool": { + "level": "WARN" + } +} + +# A simple default format/formatter. Generally speaking, the API should return formatters, but since +# users can configure their logging, it's better if the formatter has a logical backup. +DEFAULT_FORMAT = "%(asctime)s - %(name)s - %(levelname)s - %(message)s" +DEFAULT_FORMATTERS = { + "default": { + "format": DEFAULT_FORMAT + } +} + +# A simple default handler. Generally speaking, the API should return handlers, but since +# users can configure their logging, it's better if the handler has a logical backup. +DEFAULT_HANDLERS = { + "default": { + "class": "logging.StreamHandler", + "formatter": "default", + "stream": "ext://sys.stdout" + } +} + +# The template that plugins will use to log +DEFAULT_PLUGIN_LOGGING_TEMPLATE = { + "version": 1, + "disable_existing_loggers": False, + "formatters": {}, + "handlers": {}, + "loggers": DEFAULT_LOGGERS, +} + +# If no logging was configured, this will be used as the logging configuration +DEFAULT_LOGGING_CONFIG = { + "version": 1, + "disable_existing_loggers": False, + "formatters": DEFAULT_FORMATTERS, + "handlers": DEFAULT_HANDLERS, + "loggers": DEFAULT_LOGGERS, + "root": { + "level": "INFO", + "handlers": ["default"] + } +} + + +def setup_logger(bg_host, bg_port, system_name, ca_cert=None, client_cert=None, ssl_enabled=None): + """Configures python logging module to use logging specified in BEERGARDEN API. + + This method will overwrite your current logging configuration, so only call it if you want + BEERGARDEN's logging configuration. + + :param str bg_host: Hostname of a BEERGARDEN + :param int bg_port: Port BEERGARDEN is listening on + :param str system_name: The system + :param ca_cert: Certificate that issued the server certificate used by the BEERGARDEN server + :param client_cert: Certificate used by the server making the connection to BEERGARDEN + :param bool ssl_enabled: Whether to use SSL for BEERGARDEN communication + :return: + """ + config = get_python_logging_config(bg_host=bg_host, bg_port=bg_port, system_name=system_name, ca_cert=ca_cert, + client_cert=client_cert, ssl_enabled=ssl_enabled) + logging.config.dictConfig(config) + + +def get_python_logging_config(bg_host, bg_port, system_name, ca_cert=None, client_cert=None, ssl_enabled=None): + """Returns a dictionary for the python logging configuration + + :param str bg_host: Hostname of a BEERGARDEN + :param int bg_port: Port BEERGARDEN is listening on + :param str system_name: The system + :param ca_cert: Certificate that issued the server certificate used by the BEERGARDEN server + :param client_cert: Certificate used by the server making the connection to BEERGARDEN + :param bool ssl_enabled: Whether to use SSL for BEERGARDEN communication + :return: Python logging configuration + """ + client = brewtils.get_easy_client(host=bg_host, port=bg_port, ssl_enabled=ssl_enabled, + ca_cert=ca_cert, client_cert=client_cert) + + logging_config = client.get_logging_config(system_name=system_name) + return convert_logging_config(logging_config) + + +def convert_logging_config(logging_config): + """Converts a LoggingConfig object into a python logging configuration + + The python logging configuration that is returned can be passed to `logging.config.dictConfig` + + :param logging_config: + :return: Python logging configuration + """ + config_to_return = copy.deepcopy(DEFAULT_PLUGIN_LOGGING_TEMPLATE) + + if logging_config.handlers: + handlers = logging_config.handlers + else: + handlers = copy.deepcopy(DEFAULT_HANDLERS) + config_to_return['handlers'] = handlers + + if logging_config.formatters: + formatters = logging_config.formatters + else: + formatters = copy.deepcopy(DEFAULT_FORMATTERS) + config_to_return['formatters'] = formatters + + config_to_return['root'] = { + "level": logging_config.level, + "handlers": list(config_to_return['handlers'].keys()) + } + return config_to_return + + diff --git a/brewtils/models.py b/brewtils/models.py new file mode 100644 index 00000000..3c7b2ea0 --- /dev/null +++ b/brewtils/models.py @@ -0,0 +1,482 @@ +from enum import Enum + +import six +from brewtils.errors import RequestStatusTransitionError + + +class Events(Enum): + BREWVIEW_STARTED = 1 + BREWVIEW_STOPPED = 2 + BARTENDER_STARTED = 3 + BARTENDER_STOPPED = 4 + REQUEST_CREATED = 5 + REQUEST_STARTED = 6 + REQUEST_COMPLETED = 7 + INSTANCE_INITIALIZED = 8 + INSTANCE_STARTED = 9 + INSTANCE_STOPPED = 10 + SYSTEM_CREATED = 11 + SYSTEM_UPDATED = 12 + SYSTEM_REMOVED = 13 + QUEUE_CLEARED = 14 + ALL_QUEUES_CLEARED = 15 + + +class Command(object): + + schema = 'CommandSchema' + + COMMAND_TYPES = ('ACTION', 'INFO', 'EPHEMERAL') + OUTPUT_TYPES = ('STRING', 'JSON', 'XML', 'HTML') + + def __init__(self, name, description=None, id=None, parameters=None, command_type=None, output_type=None, + schema=None, form=None, template=None, icon_name=None, system=None): + self.name = name + self.description = description + self.id = id + self.parameters = parameters or [] + self.command_type = command_type + self.output_type = output_type + self.schema = schema + self.form = form + self.template = template + self.icon_name = icon_name + self.system = system + + def __str__(self): + return self.name + + def __repr__(self): + return '' % self.name + + def parameter_keys(self): + """Convenience Method for returning all the keys of this command's parameters. + + :return list_of_parameters: + """ + return [p.key for p in self.parameters] + + def get_parameter_by_key(self, key): + """Given a Key, it will return the parameter (or None) with that key + + :param key: + :return parameter: + """ + for parameter in self.parameters: + if parameter.key == key: + return parameter + + return None + + def has_different_parameters(self, parameters): + """Given a set of parameters, determines if the parameters provided differ from the + parameters already defined on this command. + + :param parameters: + :return boolean: + """ + if len(parameters) != len(self.parameters): + return True + + for parameter in parameters: + if parameter.key not in self.parameter_keys(): + return True + + current_param = self.get_parameter_by_key(parameter.key) + if current_param.is_different(parameter): + return True + + return False + + +class Instance(object): + + schema = 'InstanceSchema' + + INSTANCE_STATUSES = {'INITIALIZING', 'RUNNING', 'PAUSED', 'STOPPED', 'DEAD', 'UNRESPONSIVE', 'STARTING', 'STOPPING', + 'UNKNOWN'} + + def __init__(self, name=None, description=None, id=None, status=None, status_info=None, queue_type=None, + queue_info=None, icon_name=None, metadata=None): + + self.name = name + self.description = description + self.id = id + self.status = status.upper() if status else None + self.status_info = status_info or {} + self.queue_type = queue_type + self.queue_info = queue_info or {} + self.icon_name = icon_name + self.metadata = metadata or {} + + def __str__(self): + return self.name + + def __repr__(self): + return '' % (self.name, self.status) + + +class Choices(object): + + schema = 'ChoicesSchema' + + TYPES = ('static', 'url', 'command') + DISPLAYS = ('select', 'typeahead') + + def __init__(self, type=None, display=None, value=None, strict=None, details=None): + self.type = type + self.strict = strict + self.value = value + self.display = display + self.details = details or {} + + def __str__(self): + return self.value.__str__() + + def __repr__(self): + return '' % (self.type, self.display, self.value) + + +class Parameter(object): + + schema = 'ParameterSchema' + + TYPES = ("String", "Integer", "Float", "Boolean", "Any", "Dictionary", "Date", "DateTime") + FORM_INPUT_TYPES = ("textarea",) + + def __init__(self, key, type=None, multi=None, display_name=None, optional=None, default=None, description=None, + choices=None, parameters=None, nullable=None, maximum=None, minimum=None, regex=None, + form_input_type=None): + + self.key = key + self.type = type + self.multi = multi + self.display_name = display_name + self.optional = optional + self.default = default + self.description = description + self.choices = choices + self.parameters = parameters or [] + self.nullable = nullable + self.maximum = maximum + self.minimum = minimum + self.regex = regex + self.form_input_type = form_input_type + + def __str__(self): + return self.key + + def __repr__(self): + return '' % (self.key, self.type, self.description) + + def is_different(self, other): + if not type(other) is type(self): + return True + + fields_to_compare = ['key', 'type', 'multi', 'optional', 'default', 'nullable', 'maximum', 'minimum', 'regex'] + for field in fields_to_compare: + if getattr(self, field) != getattr(other, field): + return True + + if len(self.parameters) != len(other.parameters): + return True + + parameter_keys = [p.key for p in self.parameters] + for parameter in other.parameters: + if parameter.key not in parameter_keys: + return True + + current_param = list(filter((lambda p: p.key == parameter.key), self.parameters))[0] + if current_param.is_different(parameter): + return True + + return False + + +class Request(object): + + schema = 'RequestSchema' + + STATUS_LIST = ('CREATED', 'RECEIVED', 'IN_PROGRESS', 'CANCELED', 'SUCCESS', 'ERROR') + COMPLETED_STATUSES = ('CANCELED', 'SUCCESS', 'ERROR') + COMMAND_TYPES = ('ACTION', 'INFO', 'EPHEMERAL') + OUTPUT_TYPES = ('STRING', 'JSON', 'XML', 'HTML') + + def __init__(self, system=None, system_version=None, instance_name=None, command=None, id=None, parent=None, + children=None, parameters=None, comment=None, output=None, output_type=None, status=None, + command_type=None, created_at=None, error_class=None, metadata=None, updated_at=None): + + self.system = system + self.system_version = system_version + self.instance_name = instance_name + self.command = command + self.id = id + self.parent = parent + self.children = children + self.parameters = parameters + self.comment = comment + self.output = output + self.output_type = output_type + self._status = status + self.command_type = command_type + self.created_at = created_at + self.updated_at = updated_at + self.error_class = error_class + self.metadata = metadata or {} + + def __str__(self): + return self.command + + def __repr__(self): + return '' %\ + (self.command, self.status, self.system, self.system_version, self.instance_name) + + @property + def status(self): + return self._status + + @property + def is_ephemeral(self): + return self.command_type and self.command_type.upper() == 'EPHEMERAL' + + @status.setter + def status(self, value): + if self._status in self.COMPLETED_STATUSES: + raise RequestStatusTransitionError("Status for a request cannot be updated once it has been completed. " + "Current status: {0} Requested status: {1}".format(self.status, value)) + + elif self._status == 'IN_PROGRESS' and value not in self.COMPLETED_STATUSES + ('IN_PROGRESS', ): + raise RequestStatusTransitionError("A request cannot go from IN_PROGRESS to a non-completed status. " + "Completed statuses are {0}. You requested: {1}" + .format(self.COMPLETED_STATUSES, value)) + self._status = value + + +class System(object): + + schema = 'SystemSchema' + + def __init__(self, name=None, description=None, version=None, id=None, max_instances=None, instances=None, + commands=None, icon_name=None, display_name=None, metadata=None): + + self.name = name + self.description = description + self.version = version + self.id = id + self.max_instances = max_instances + self.instances = instances or [] + self.commands = commands or [] + self.icon_name = icon_name + self.display_name = display_name + self.metadata = metadata or {} + + def __str__(self): + return '%s-%s' % (self.name, self.version) + + def __repr__(self): + return '' % (self.name, self.version) + + @property + def instance_names(self): + return [i.name for i in self.instances] + + def has_instance(self, name): + """Determine if an instance currently exists in the system + + :param name: The name of the instance to search + :return: True if an instance with the given name exists for this system, False otherwise. + """ + return True if self.get_instance(name) else False + + def get_instance(self, name): + """Get an instance that currently exists in the system + + :param name: The name of the instance to search + :return: The instance with the given name exists for this system, None otherwise + """ + for instance in self.instances: + if instance.name == name: + return instance + return None + + def get_command_by_name(self, command_name): + """Retrieve a particular command from the system + + :param command_name: Name of the command to retrieve + :return: The command object. None if the given command name does not exist in this system. + """ + for command in self.commands: + if command.name == command_name: + return command + + return None + + def has_different_commands(self, commands): + """Check if a set of commands is different than the current commands + + :param commands: The set commands to compare against the current set + :return: True if the sets are different, False if the sets are the same + """ + if len(commands) != len(self.commands): + return True + + for command in commands: + if command.name not in [c.name for c in self.commands]: + return True + + current_command = self.get_command_by_name(command.name) + + if current_command.has_different_parameters(command.parameters): + return True + + return False + + +class PatchOperation(object): + + schema = 'PatchSchema' + + def __init__(self, operation=None, path=None, value=None): + self.operation = operation + self.path = path + self.value = value + + def __str__(self): + return '%s, %s, %s' % (self.operation, self.path, self.value) + + def __repr__(self): + return '' % (self.operation, self.path, self.value) + + +class LoggingConfig(object): + + schema = 'LoggingConfigSchema' + + LEVELS = ("DEBUG", "INFO", "WARN", "ERROR") + SUPPORTED_HANDLERS = ("stdout", "file", "logstash") + + DEFAULT_FORMAT = "%(asctime)s - %(name)s - %(levelname)s - %(message)s" + DEFAULT_HANDLER = {"class": "logging.StreamHandler", "stream": "ext::/sys.stdout", "formatter": "default"} + + def __init__(self, level=None, handlers=None, formatters=None, loggers=None): + self.level = level + self.handlers = handlers + self.formatters = formatters + self._loggers = loggers or {} + + @property + def handler_names(self): + if self.handlers: + return self.handlers.keys() + else: + return None + + @property + def formatter_names(self): + if self.formatters: + return self.formatters.keys() + else: + return None + + def get_plugin_log_config(self, **kwargs): + """Get a specific plugin logging configuration. + + It is possible for different systems to have different logging configurations. This method + will create the correct plugin logging configuration and return it. If a specific logger + is not found for a system, then the current logging configuration will be returned. + + :param kwargs: Identifying information for a system (i.e. system_name) + :return: + """ + system_name = kwargs.pop("system_name", None) + specific_logger = self._loggers.get(system_name, {}) + + # If there is no specific logger, then we simply return this object + # otherwise, we need to construct a new LoggingConfig object with + # the overrides given in the logger. + if not specific_logger: + return self + + level = specific_logger.get("level", self.level) + handlers = self._generate_handlers(specific_logger.get("handlers")) + formatters = self._generate_formatters(specific_logger.get("formatters", {})) + + return LoggingConfig(level=level, handlers=handlers, formatters=formatters) + + def _generate_handlers(self, specific_handlers): + + # If we are not given an override for handlers, then we will just + # assume that we want to use all the handlers given in the current + # configuration. + if not specific_handlers: + return self.handlers + + if isinstance(specific_handlers, list): + handlers = {} + for handler_name in specific_handlers: + handlers[handler_name] = self.handlers[handler_name] + else: + return specific_handlers + + return handlers + + def _generate_formatters(self, specific_formatters): + + # If we are not given an override for formatters, then we will just + # assume that we want to use the formatters given in the current + # configuration + if not specific_formatters: + return self.formatters + + # In case no formatter is provided, we always want a default. + formatters = {'default': {'format': self.DEFAULT_FORMAT}} + for formatter_name, format_str in six.iteritems(specific_formatters): + formatters[formatter_name] = {'format': format_str} + + return formatters + + def __str__(self): + return '%s, %s, %s' % (self.level, self.handler_names, self.formatter_names) + + def __repr__(self): + return '' % \ + (self.name, self.error, self.payload, self.metadata) + + +class Queue(object): + + schema = 'QueueSchema' + + def __init__(self, name=None, system=None, version=None, instance=None, system_id=None, display=None, size=None): + self.name = name + self.system = system + self.version = version + self.instance = instance + self.system_id = system_id + self.display = display + self.size = size + + def __str__(self): + return '%s: %s' % (self.name, self.size) + + def __repr__(self): + return '' % (self.name, self.size) diff --git a/brewtils/plugin.py b/brewtils/plugin.py new file mode 100644 index 00000000..d7ea82cc --- /dev/null +++ b/brewtils/plugin.py @@ -0,0 +1,598 @@ +import json +import logging +import logging.config +import os +import sys +import threading +import warnings +from concurrent.futures import ThreadPoolExecutor + +import six +from requests import ConnectionError + +import brewtils +from brewtils.errors import BrewmasterValidationError, RequestProcessingError, DiscardMessageException, \ + RepublishRequestException, BrewmasterConnectionError +from brewtils.models import Instance, Request, System +from brewtils.request_consumer import RequestConsumer +from brewtils.rest.easy_client import EasyClient +from brewtils.schema_parser import SchemaParser +from brewtils.log import DEFAULT_LOGGING_CONFIG + +request_context = threading.local() + + +class PluginBase(object): + """A BEERGARDEN Plugin. + + This class represents a BEERGARDEN Plugin - a continuously-running process that can receive and process Requests. + + To work, a Plugin needs a Client instance - an instance of a class defining which Requests this plugin can accept + and process. The easiest way to define a `Client` is by annotating a class with the @system decorator. + + When creating a Plugin you can pass certain keyword arguments to let the Plugin know how to communicate with the + BEERGARDEN instance. These are: + + - bg_host + - bg_port + - ssl_enabled + - ca_cert + - client_cert + - bg_url_prefix + + A Plugin also needs some identifying data. You can either pass parameters to the Plugin or pass a fully defined + System object (but not both). Note that some fields are optional: + + PluginBase(name="Test", version="1.0.0", instance_name="default, description="A Test") + + or + + the_system = System(name="Test", version="1.0.0", instance_name="default, description="A Test") + PluginBase(system=the_system) + + If passing parameters directly note that these fields are required: + - name (Environment variable BG_NAME will be used if not specified) + - version (Environment variable BG_VERSION will be used if not specified) + - instance_name (Environment variable BG_INSTANCE_NAME will be used if not specified. 'default' will be used + if not specified and loading from environment variable is unsuccessful) + + And these fields are optional: + - description (Will use docstring summary line from Client if not specified) + - icon_name + - metadata + - display_name + + Plugins service requests using a :py:class:`concurrent.futures.ThreadPoolExecutor`. The maximum number of threads + available is controlled by the max_concurrent argument (the 'multithreaded' argument has been deprecated). + + **WARNING**: The default value for max_concurrent is 1. This means that a Plugin that invokes a Command on itself in + the course of processing a Request will deadlock! If you intend to do this, please set max_concurrent to a value + that makes sense and be aware that Requests are processed in separate thread contexts! + + :param client: Instance of a class annotated with @system + :param str bg_host: Hostname of a BEERGARDEN + :param int bg_port: Port BEERGARDEN is listening on + :param bool ssl_enabled: Whether to use SSL for BEERGARDEN communication + :param ca_cert: Certificate that issued the server certificate used by the BEERGARDEN server + :param client_cert: Certificate used by the server making the connection to BEERGARDEN + :param system: The system definition + :param name: The system name + :param description: The system description + :param version: The system version + :param icon_name: The system icon name + :param str instance_name: The name of the instance + :param logger: A logger that will be used by the Plugin + :type logger: :py:class:`logging.Logger` + :param parser: The parser to use when communicating with BEERGARDEN + :type parser: :py:class:`brewtils.schema_parser.SchemaParser` + :param bool multithreaded: DEPRECATED Flag specifying whether each message should be processed in a separate thread + :param int worker_shutdown_timeout: Amount of time to wait during shutdown for threads to finish processing + :param dict metadata: Metadata specific to this plugin + :param int max_concurrent: Maximum number of requests to process concurrently + :param str bg_url_prefix: URL Prefix BEERGARDEN is on + :param str display_name: The display name to use for the system + :param int max_attempts: Number of times to attempt updating the request before giving up (default -1 aka never) + :param int max_timeout: Maximum amount of time to wait before retrying to update a request + :param int starting_timeout: Initial time to wait before the first retry + :param int max_instances: Maximum number of instances allowed for the system + :param bool ca_verify: Flag indicating whether to verify server certificate when making a request. + """ + + def __init__(self, client, bg_host=None, bg_port=None, ssl_enabled=None, ca_cert=None, client_cert=None, + system=None, name=None, description=None, version=None, icon_name=None, instance_name=None, + logger=None, parser=None, multithreaded=None, metadata=None, max_concurrent=None, + bg_url_prefix=None, **kwargs): + # If a logger is specified or the logging module already has additional handlers + # then we assume that logging has already been configured + if logger or len(logging.getLogger(__name__).root.handlers) > 0: + self.logger = logger or logging.getLogger(__name__) + self._custom_logger = True + else: + logging.config.dictConfig(DEFAULT_LOGGING_CONFIG) + self.logger = logging.getLogger(__name__) + self._custom_logger = False + + connection_parameters = brewtils.get_bg_connection_parameters( + host=bg_host, + port=bg_port, + ssl_enabled=ssl_enabled, + ca_cert=ca_cert, + client_cert=client_cert, + url_prefix=bg_url_prefix, + ca_verify=kwargs.get('ca_verify', None) + ) + self.bg_host = connection_parameters['host'] + self.bg_port = connection_parameters['port'] + self.ssl_enabled = connection_parameters['ssl_enabled'] + self.ca_cert = connection_parameters['ca_cert'] + self.client_cert = connection_parameters['client_cert'] + self.bg_url_prefix = connection_parameters['url_prefix'] + self.ca_verify = connection_parameters['ca_verify'] + + self.max_attempts = kwargs.get('max_attempts', -1) + self.max_timeout = kwargs.get('max_timeout', 30) + self.starting_timeout = kwargs.get('starting_timeout', 5) + + self.max_concurrent = self._setup_max_concurrent(multithreaded, max_concurrent) + self.instance_name = instance_name or os.environ.get('BG_INSTANCE_NAME', 'default') + self.metadata = metadata or {} + + self.instance = None + self.admin_consumer = None + self.request_consumer = None + self.connection_poll_thread = None + self.client = client + self.shutdown_event = threading.Event() + self.parser = parser or SchemaParser() + self.system = self._setup_system(client, self.instance_name, system, name, description, + version, icon_name, self.metadata, kwargs.pop("display_name", None), + kwargs.get('max_instances', None)) + self.unique_name = '%s[%s]-%s' % (self.system.name, self.instance_name, self.system.version) + + # We need to tightly manage when we're in an 'error' state, aka Brew-view is down + self.brew_view_error_condition = threading.Condition() + self.brew_view_down = False + + self.pool = ThreadPoolExecutor(max_workers=self.max_concurrent) + self.admin_pool = ThreadPoolExecutor(max_workers=1) + + self.bm_client = EasyClient(logger=self.logger, parser=self.parser, **connection_parameters) + + def run(self): + # Let Beergarden know about our system and instance + self._initialize() + + self.logger.debug("Creating and starting admin queue consumer") + self.admin_consumer = self._create_admin_consumer() + self.admin_consumer.start() + + self.logger.debug("Creating and starting request queue consumer") + self.request_consumer = self._create_standard_consumer() + self.request_consumer.start() + + self.logger.debug("Creating and starting connection poll thread") + self.connection_poll_thread = self._create_connection_poll_thread() + self.connection_poll_thread.start() + + self.logger.info("Plugin %s has started", self.unique_name) + + try: + while not self.shutdown_event.wait(0.1): + if not self.admin_consumer.isAlive() and not self.admin_consumer.shutdown_event.is_set(): + self.logger.warning("Looks like admin consumer has died - attempting to restart") + self.admin_consumer = self._create_admin_consumer() + self.admin_consumer.start() + + if not self.request_consumer.isAlive() and not self.request_consumer.shutdown_event.is_set(): + self.logger.warning("Looks like request consumer has died - attempting to restart") + self.request_consumer = self._create_standard_consumer() + self.request_consumer.start() + + if not self.connection_poll_thread.isAlive(): + self.logger.warning("Looks like connection poll thread has died - attempting to restart") + self.connection_poll_thread = self._create_connection_poll_thread() + self.connection_poll_thread.start() + + if self.request_consumer.shutdown_event.is_set() and self.admin_consumer.shutdown_event.is_set(): + self.shutdown_event.set() + + except KeyboardInterrupt: + self.logger.debug("Received KeyboardInterrupt - shutting down") + except Exception as ex: + self.logger.error("Event loop terminated unexpectedly - shutting down") + self.logger.exception(ex) + + self.logger.debug("About to shut down plugin %s", self.unique_name) + self._shutdown() + + self.logger.info("Plugin %s has terminated", self.unique_name) + + def process_message(self, target, request, headers): + """Process a message. Intended to be run on an Executor. + + :param target: The object to invoke received commands on. (self or self.client) + :param request: The parsed Request object + :param headers: Dictionary of headers from the `brewtils.request_consumer.RequestConsumer` + :return: None + """ + request.status = 'IN_PROGRESS' + self._update_request(request, headers) + + try: + # Set request context so this request will be the parent of any generated requests and update status + # We also need the host/port of the current plugin. We currently don't support parent/child requests + # across different servers. + request_context.current_request = request + request_context.bg_host = self.bg_host + request_context.bg_port = self.bg_port + + output = self._invoke_command(target, request) + except Exception as ex: + self.logger.exception('Plugin %s raised an exception while processing request %s: %s', self.unique_name, + str(request), ex) + request.status = 'ERROR' + request.output = self._format_error_output(request, ex) + request.error_class = type(ex).__name__ + else: + request.status = 'SUCCESS' + request.output = self._format_output(output) + + self._update_request(request, headers) + + def process_request_message(self, message, headers): + """Processes a message from a RequestConsumer + + :param message: A valid string-representation of a `brewtils.models.Request` + :param headers: A dictionary of headers from the `brewtils.request_consumer.RequestConsumer` + :return: A `concurrent.futures.Future` + """ + + request = self._pre_process(message) + + # This message has already been processed, all it needs to do is update + if request.status in Request.COMPLETED_STATUSES: + return self.pool.submit(self._update_request, request, headers) + else: + return self.pool.submit(self.process_message, self.client, request, headers) + + def process_admin_message(self, message, headers): + + # Admin requests won't have a system, so don't verify it + request = self._pre_process(message, verify_system=False) + + return self.admin_pool.submit(self.process_message, self, request, headers) + + def _pre_process(self, message, verify_system=True): + + if self.shutdown_event.is_set(): + raise RequestProcessingError('Unable to process message - currently shutting down') + + try: + request = self.parser.parse_request(message, from_string=True) + except Exception as ex: + self.logger.exception("Unable to parse message body: {0}. Exception: {1}".format(message, ex)) + raise DiscardMessageException('Error parsing message body') + + if verify_system and request.command_type and request.command_type.upper() != 'EPHEMERAL' and \ + request.system.upper() != self.system.name.upper(): + raise DiscardMessageException("Received message for a different system {0}".format(request.system.upper())) + + return request + + def _initialize(self): + self.logger.debug("Initializing plugin %s", self.unique_name) + + # TODO: We should use self.bm_client.upsert_system once it is supported (issue/217) + existing_system = self.bm_client.find_unique_system(name=self.system.name, version=self.system.version) + if existing_system: + if existing_system.has_different_commands(self.system.commands): + new_commands = self.system.commands + else: + new_commands = None + + # We always update in case the metadata has changed. + self.system = self.bm_client.update_system(existing_system.id, + new_commands=new_commands, + metadata=self.system.metadata, + description=self.system.description, + display_name=self.system.display_name, + icon_name=self.system.icon_name) + else: + self.system = self.bm_client.create_system(self.system) + + instance_id = next(instance.id for instance in self.system.instances if instance.name == self.instance_name) + self.instance = self.bm_client.initialize_instance(instance_id) + + self.logger.debug("Plugin %s is initialized", self.unique_name) + + def _shutdown(self): + self.shutdown_event.set() + + self.logger.debug('About to stop message consuming') + self.request_consumer.stop_consuming() + self.admin_consumer.stop_consuming() + + self.logger.debug('About to wake up all waiting request processing threads') + with self.brew_view_error_condition: + self.brew_view_error_condition.notify_all() + + self.logger.debug('Shutting down request processing pool') + self.pool.shutdown(wait=True) + self.logger.debug('Shutting down admin processing pool') + self.admin_pool.shutdown(wait=True) + + self.logger.debug('Attempting to stop request queue consumer') + self.request_consumer.stop() + self.request_consumer.join() + + self.logger.debug('Attempting to stop admin queue consumer') + self.admin_consumer.stop() + self.admin_consumer.join() + + self.logger.debug("Successfully shutdown plugin {0}".format(self.unique_name)) + + def _create_standard_consumer(self): + return RequestConsumer(amqp_url=self.instance.queue_info['url'], + queue_name=self.instance.queue_info['request']['name'], + on_message_callback=self.process_request_message, + panic_event=self.shutdown_event, thread_name='Request Consumer', + max_concurrent=self.max_concurrent) + + def _create_admin_consumer(self): + return RequestConsumer(amqp_url=self.instance.queue_info['url'], + queue_name=self.instance.queue_info['admin']['name'], + on_message_callback=self.process_admin_message, + panic_event=self.shutdown_event, thread_name='Admin Consumer', + max_concurrent=1, logger=logging.getLogger('brewtils.admin_consumer')) + + def _create_connection_poll_thread(self): + connection_poll_thread = threading.Thread(target=self._connection_poll) + connection_poll_thread.daemon = True + return connection_poll_thread + + def _invoke_command(self, target, request): + """Invoke the function named in request.command. + + :param target: The object to search for the function implementation. Will be self or self.client. + :param request: The request to process + :raise RequestProcessingError: The specified target does not define a callable implementation of request.command + :return: The output of the function invocation + """ + if not callable(getattr(target, request.command, None)): + raise RequestProcessingError("Could not find an implementation of command '%s'" % request.command) + + # It's kinda weird that we need to add the object arg only if we're trying to call a function on self + # In both cases the function object is bound... think it has something to do with our decorators + args = [self] if target is self else [] + return getattr(target, request.command)(*args, **request.parameters) + + def _update_request(self, request, headers): + """Sends a Request update to BEERGARDEN + + Ephemeral requests do not get updated, so we simply skip them. + + If brew-view appears to be down, it will wait for brew-view to come back up before updating. + + If this is the final attempt to update, we will attempt a known, good request to give some + information to the user. If this attempt fails, then we simply discard the message + + :param request: The request to update + :param headers: A dictionary of headers from the `brewtils.request_consumer.RequestConsumer` + :raise RepublishMessageException: If the Request update failed for any reason + :return: None + """ + + if request.is_ephemeral: + sys.stdout.flush() + return + + with self.brew_view_error_condition: + + self._wait_for_brew_view_if_down(request) + + try: + if not self._should_be_final_attempt(headers): + self._wait_if_not_first_attempt(headers) + self.bm_client.update_request(request.id, status=request.status, output=request.output, + error_class=request.error_class) + else: + self.bm_client.update_request(request.id, status='ERROR', + output='We tried to update the request, but ' + 'it failed too many times. Please check ' + 'the plugin logs to figure out why the request ' + 'update failed. It is possible for this request to have ' + 'succeeded, but we cannot update beer-garden with that ' + 'information.', + error_class='BGGivesUpError') + except Exception as ex: + self._handle_request_update_failure(request, headers, ex) + finally: + sys.stdout.flush() + + def _wait_if_not_first_attempt(self, headers): + if headers.get('retry_attempt', 0) > 0: + time_to_sleep = min(headers.get('time_to_wait', self.starting_timeout), self.max_timeout) + self.shutdown_event.wait(time_to_sleep) + + def _handle_request_update_failure(self, request, headers, exc): + + # If brew-view is down, we always want to try again (yes even if it is the 'final_attempt') + if isinstance(exc, (ConnectionError, BrewmasterConnectionError)): + self.brew_view_down = True + self.logger.error('Error updating request status: {0} exception: {1}'.format(request.id, exc)) + raise RepublishRequestException(request, headers) + + # Time to discard the message because we've given up + elif self._should_be_final_attempt(headers): + message = 'Could not update request {0} even with a known good status, output and error_class. ' \ + 'We have reached the final attempt and will now discard the message. Attempted to ' \ + 'process this message {1} times'.format(request.id, headers['retry_attempt']) + self.logger.error(message) + raise DiscardMessageException(message) + + else: + self._update_retry_attempt_information(headers) + self.logger.exception('Error updating request (Attempt #{0}: request: {1} exception: {2}' + .format(headers.get('retry_attempt', 0), request.id, exc)) + raise RepublishRequestException(request, headers) + + def _update_retry_attempt_information(self, headers): + headers['retry_attempt'] = headers.get('retry_attempt', 0) + 1 + headers['time_to_wait'] = min( + headers.get('time_to_wait', self.starting_timeout / 2) * 2, + self.max_timeout + ) + + def _should_be_final_attempt(self, headers): + if self.max_attempts <= 0: + return False + + return self.max_attempts <= headers.get('retry_attempt', 0) + + def _wait_for_brew_view_if_down(self, request): + if self.brew_view_down and not self.shutdown_event.is_set(): + self.logger.warning('Currently unable to communicate with Brew-view, about to wait until ' + 'connection is reestablished to update request %s', request.id) + self.brew_view_error_condition.wait() + + def _start(self, request): + """Handle start message by marking this instance as running. + + :param request: The start message + :return: Success output message + """ + self.instance = self.bm_client.update_instance_status(self.instance.id, 'RUNNING') + + return "Successfully started plugin" + + def _stop(self, request): + """Handle stop message by marking this instance as stopped. + + :param request: The stop message + :return: Success output message + """ + self.shutdown_event.set() + self.instance = self.bm_client.update_instance_status(self.instance.id, 'STOPPED') + + return "Successfully stopped plugin" + + def _status(self, request): + """Handle status message by sending a heartbeat. + + :param request: The status message + :return: None + """ + with self.brew_view_error_condition: + if not self.brew_view_down: + try: + self.bm_client.instance_heartbeat(self.instance.id) + except (ConnectionError, BrewmasterConnectionError): + self.brew_view_down = True + raise + + def _setup_max_concurrent(self, multithreaded, max_concurrent): + """Determine correct max_concurrent value. Will be unnecessary when multithreaded flag is removed.""" + if multithreaded is not None: + warnings.warn("Keyword argument 'multithreaded' is deprecated and will be removed in version 3.0, please " + "use 'max_concurrent' instead.", DeprecationWarning, stacklevel=2) + + # Both multithreaded and max_concurrent kwargs explicitly set ... check for mutually exclusive settings + if max_concurrent is not None: + if multithreaded is True and max_concurrent == 1: + self.logger.warning("Plugin created with multithreaded=True and max_concurrent=1, ignoring " + "'multithreaded' argument") + elif multithreaded is False and max_concurrent > 1: + self.logger.warning("Plugin created with multithreaded=False and max_concurrent>1, ignoring " + "'multithreaded' argument") + + return max_concurrent + else: + return 5 if multithreaded else 1 + else: + return max_concurrent or 1 + + def _setup_system(self, client, inst_name, system, name, description, version, icon_name, metadata, display_name, + max_instances): + if system: + if name or description or version or icon_name or display_name or max_instances: + raise BrewmasterValidationError("Sorry, you can't specify a system as well as system creation helper " + "keywords (name, description, version, max_instances, display_name, " + "and icon_name)") + + if not system.instances: + raise BrewmasterValidationError("Explicit system definition requires explicit instance definition " + "(use instances=[Instance(name='default')] for default behavior)") + + if not system.max_instances: + system.max_instances = len(system.instances) + + else: + name = name or os.environ.get('BG_NAME', None) + version = version or os.environ.get('BG_VERSION', None) + + if client.__doc__ and not description: + description = self.client.__doc__.split("\n")[0] + + system = System(name=name, description=description, version=version, icon_name=icon_name, + commands=client._commands, max_instances=max_instances or 1, + instances=[Instance(name=inst_name)], + metadata=metadata, display_name=display_name) + + return system + + def _connection_poll(self): + """Periodically attempt to re-connect to BEERGARDEN""" + + while not self.shutdown_event.wait(5): + with self.brew_view_error_condition: + if self.brew_view_down: + try: + self.bm_client.get_version() + except Exception: + self.logger.debug('Attempt to reconnect to Brew-view failed') + else: + self.logger.info('Brew-view connection reestablished, about to notify any waiting requests') + self.brew_view_down = False + self.brew_view_error_condition.notify_all() + + def _format_error_output(self, request, exc): + """Formats error output appropriately. + + If the request's output type is JSON, then we format it appropriately. Otherwise, we simply return a + string version of the Exception. If the JSON formatting fails, we will simply return a string version of + the __dict__ object of the exception. + + :param request: + :param exc: + :return: + """ + + message = str(exc) + + if not request.output_type or request.output_type.upper() != "JSON": + return message + + # Process a JSON request type + output = {"message": message, "attributes": exc.__dict__} + try: + return json.dumps(output) + except Exception: + self.logger.debug("Could not convert attributes of exception to JSON. Just stringify dict.") + output['attributes'] = str(exc.__dict__) + return json.dumps(output) + + @staticmethod + def _format_output(output): + """Formats output from Plugins so that no validation errors accidentally occur""" + + if isinstance(output, six.string_types): + return output + + try: + return json.dumps(output) + except (TypeError, ValueError): + return str(output) + + +class RemotePlugin(PluginBase): + pass diff --git a/brewtils/request_consumer.py b/brewtils/request_consumer.py new file mode 100644 index 00000000..2ab1d931 --- /dev/null +++ b/brewtils/request_consumer.py @@ -0,0 +1,289 @@ +import logging +import threading +from functools import partial + +import pika +from pika.exceptions import AMQPConnectionError + +from brewtils.errors import DiscardMessageException, RepublishRequestException +from brewtils.schema_parser import SchemaParser + + +class RequestConsumer(threading.Thread): + """Consumer that will handle unexpected interactions with RabbitMQ such as channel and connection closures. + + If RabbitMQ closes the connection, it will reopen it. You should look at the output, as there are limited reasons + why the connection may be closed, which usually are tied to permission related issues or socket timeouts. + + If the channel is closed, it will indicate a problem with one of the commands that were issued and that should + surface in the output as well. + + :param str amqp_url: The AMQP url to connection with + :param str queue_name: The name of the queue to connect to + :param func on_message_callback: The function called to invoke message processing. Must return a Future. + :param event panic_event: An event to be set in the event of a catastrophic failure + :type event: :py:class:`threading.Event` + :param logger: A configured logger + :type logger: :py:class:`logging.Logger` + :param str thread_name: The name to use for this thread + :param int max_connect_retries: Number of connection retry attempts before failure. Default -1 (retry forever). + :param int max_connect_backoff: Maximum amount of time to wait between connection retry attempts. Default 30. + :param int max_concurrent: Maximum number of requests to process concurrently + """ + + def __init__(self, amqp_url, queue_name, on_message_callback, panic_event, + logger=None, thread_name=None, **kwargs): + + self._connection = None + self._channel = None + self._consumer_tag = None + + self._url = amqp_url + self._queue_name = queue_name + self._on_message_callback = on_message_callback + self._panic_event = panic_event + self._max_connect_retries = kwargs.pop("max_connect_retries", -1) + self._max_connect_backoff = kwargs.pop("max_connect_backoff", 30) + self._max_concurrent = kwargs.pop("max_concurrent", 1) + self.logger = logger or logging.getLogger(__name__) + self.shutdown_event = threading.Event() + + super(RequestConsumer, self).__init__(name=thread_name) + + def run(self): + """Run the example consumer by connection to RabbitMQ and then starting the IOLoop to block and allow the + SelectConnection to operate. + """ + self._connection = self.open_connection() + + # It is possible to return from open_connection without acquiring a connection. + # This usually happens if no max_connect_retries was set and we are constantly trying to connect to a queue that + # does not exist. For those cases, there is no reason to start an ioloop. + if self._connection: + self._connection.ioloop.start() + + def stop(self): + """Cleanly shutdown the connection to RabbitMQ by closing the channel. The stop_consuming method should already + have been called. When Pika acknowledges the channel closure we will close the connection, which will end the + RequestConsumer. + """ + self.logger.debug('Stopping request consumer') + self.shutdown_event.set() + self.close_channel() + + def on_message(self, channel, basic_deliver, properties, body): + """Invoked by pika when a message is delivered from RabbitMQ. The channel is passed for your convenience. The + basic_deliver object that is passed in carries the exchange, routing key, delivery tag and a redelivered flag + for the message. the properties passed in is an instance of BasicProperties with the message properties and the + body is the message that was sent. + + :param pika.channel.Channel channel: The channel object + :param pika.Spec.Basic.Deliver basic_deliver: basic_deliver method + :param pika.Spec.BasicProperties properties: properties + :param str|unicode body: The message body + """ + self.logger.debug("Received message #%s from %s on channel %s: %s", basic_deliver.delivery_tag, + properties.app_id, channel.channel_number, body) + + try: + future = self._on_message_callback(body, properties.headers) + future.add_done_callback(partial(self.on_message_callback_complete, basic_deliver)) + except DiscardMessageException: + self.logger.debug('Nacking message %s, not attempting to requeue', basic_deliver.delivery_tag) + self._channel.basic_nack(basic_deliver.delivery_tag, requeue=False) + except Exception as ex: + self.logger.exception('Exception while trying to schedule message %s, about to nack and requeue: %s', + basic_deliver.delivery_tag, ex) + self._channel.basic_nack(basic_deliver.delivery_tag, requeue=True) + + def on_message_callback_complete(self, basic_deliver, future): + """Invoked when the future returned by _on_message_callback completes. + + :param pika.Spec.Basic.Deliver basic_deliver: basic_deliver method + :param concurrent.futures.Future future: Completed future + :return: None + """ + if not future.exception(): + try: + self.logger.debug('Acking message %s', basic_deliver.delivery_tag) + self._channel.basic_ack(basic_deliver.delivery_tag) + except Exception as ex: + self.logger.exception('Error acking message %s, about to shut down: %s', basic_deliver.delivery_tag, ex) + self._panic_event.set() + else: + future_ex = future.exception() + + if isinstance(future_ex, RepublishRequestException): + try: + with pika.BlockingConnection(pika.URLParameters(self._url)) as conn: + headers = future_ex.headers + headers.update({'request_id': future_ex.request.id}) + props = pika.BasicProperties(app_id='beer-garden', content_type='text/plain', + headers=headers, priority=1) + conn.channel().basic_publish(exchange=basic_deliver.exchange, properties=props, + routing_key=basic_deliver.routing_key, + body=SchemaParser.serialize_request(future_ex.request)) + + self._channel.basic_ack(basic_deliver.delivery_tag) + except Exception as ex: + self.logger.exception('Error republishing message %s, about to shut down: %s', + basic_deliver.delivery_tag, ex) + self._panic_event.set() + elif isinstance(future_ex, DiscardMessageException): + self.logger.info('Nacking message %s, not attempting to requeue', basic_deliver.delivery_tag) + self._channel.basic_nack(basic_deliver.delivery_tag, requeue=False) + else: + # If request processing throws anything else we are in a seriously bad state, we should just die + self.logger.exception('Unexpected exception during request %s processing, about to shut down: %s', + basic_deliver.delivery_tag, future_ex, exc_info=False) + self._panic_event.set() + + def open_connection(self): + """This method connects to RabbitMQ, returning the connection handle. When the connection is established, the + on_connection_open method will be invoked by pika. + + :rtype: pika.SelectConnection + """ + self.logger.debug('Connecting to %s' % self._url) + time_to_wait = 0.1 + retries = 0 + while not self.shutdown_event.is_set(): + try: + return pika.SelectConnection(pika.URLParameters(self._url), self.on_connection_open, + stop_ioloop_on_close=False) + except AMQPConnectionError as ex: + if 0 <= self._max_connect_retries <= retries: + raise ex + self.logger.warning("Error attempting to connect to %s" % self._url) + self.logger.warning("Waiting %s seconds and attempting again" % time_to_wait) + self.shutdown_event.wait(time_to_wait) + time_to_wait = min(time_to_wait * 2, self._max_connect_backoff) + retries += 1 + + def on_connection_open(self, unused_connection): + """This method is called by pika once the connection to RabbitMQ has been established. It passes the handle to + the connection object in case we need it, but in this case, we'll just mark it unused. + + :type unused_connection: pika.SelectConnection + """ + self.logger.debug("Connection opened: %s", unused_connection) + self._connection.add_on_close_callback(self.on_connection_closed) + self.open_channel() + + def close_connection(self): + """This method closes the connection to RabbitMQ.""" + self.logger.debug("Closing connection") + self._connection.close() + + def on_connection_closed(self, connection, reply_code, reply_text): + """This method is invoked by pika when the connection to RabbitMQ is closed unexpectedly. Since it is + unexpected, we will reconnect to RabbitMQ if it disconnects. + + :param pika.connection.Connection connection: the closed connection object + :param int reply_code: The server provided reply_code if given + :param basestring reply_text: The server provided reply_text if given + """ + self.logger.debug('Connection "%s" closed: (%s) %s' % (connection, reply_code, reply_text)) + self._channel = None + + # A 320 is the server forcing the connection to close + if reply_code == 320: + self.shutdown_event.set() + + if self.shutdown_event.is_set(): + self._connection.ioloop.stop() + else: + self.logger.warning('Connection unexpectedly closed: (%s) %s' % (reply_code, reply_text)) + self.logger.warning('Attempting to reopen connection in 5 seconds') + self._connection.add_timeout(5, self.reconnect) + + def reconnect(self): + """Will be invoked by the IOLoop timer if the connection is closed. See the on_connection_closed method.""" + + # This is the old connection IOLoop instance, stop its ioloop + self._connection.ioloop.stop() + + if not self.shutdown_event.is_set(): + # Creates a new connection + self._connection = self.open_connection() + + # There is now a new connection, needs a new ioloop to run + if self._connection: + self._connection.ioloop.start() + + def open_channel(self): + """Open a new channel with RabbitMQ by issuing the Channel.Open RPC command. When RabbitMQ responds that the + channel is open, the on_channel_open callback will be invoked by pika. + """ + self.logger.debug('Opening a new channel') + self._connection.channel(on_open_callback=self.on_channel_open) + + def on_channel_open(self, channel): + """This method is invoked by pika when the channel has been opened. + The channel object is passed in so we can make use of it. + + The exchange / queue binding should have already been set up so + just start consuming. + + :param pika.channel.Channel channel: The channel object + """ + self.logger.debug('Channel opened: %s', channel) + self._channel = channel + self._channel.add_on_close_callback(self.on_channel_closed) + self.start_consuming() + + def close_channel(self): + """Call to close the channel with RabbitMQ cleanly by issuing the Channel.Close RPC command.""" + self.logger.debug('Closing the channel') + self._channel.close() + + def on_channel_closed(self, channel, reply_code, reply_text): + """Invoked by pika when RabbitMQ unexpectedly closes the channel. Channels are usually closed if you attempt to + do something that violates the protocol, such as re-declare an exchange or queue with different parameters. In + this case, we'll close the connection to shutdown the object. + + :param pika.channel.Channel channel: The closed channel + :param int reply_code: The numeric reason the channel was closed + :param str reply_text: The text reason the channel was closed + """ + self.logger.debug('Channel %i was closed: (%s) %s' % (channel, reply_code, reply_text)) + self._connection.close() + + def start_consuming(self): + """This method sets up the consumer by first calling add_on_cancel_callback so that the object is notified if + RabbitMQ cancels the consumer. It then issues the Basic.Consume RPC command which returns the consumer tag that + is used to uniquely identify the consumer with RabbitMQ. We keep the value to use it when we want to cancel + consuming. The on_message method is passed in as a callback pika will invoke when a message is fully received. + """ + self.logger.debug('Issuing consumer related RPC commands') + + # Prefetch of 1 to prevent RabbitMQ from sending us multiple messages at once + self._channel.basic_qos(prefetch_count=self._max_concurrent) + self._channel.add_on_cancel_callback(self.on_consumer_cancelled) + self._consumer_tag = self._channel.basic_consume(self.on_message, queue=self._queue_name) + + def stop_consuming(self): + """Tell RabbitMQ that you would like to stop consuming by sending the Basic.Cancel RPC command.""" + self.logger.debug("Stopping consuming on channel %s", self._channel) + if self._channel: + self.logger.debug('Sending a Basic.Cancel RPC command to RabbitMQ') + self._channel.basic_cancel(self.on_cancelok, self._consumer_tag) + + def on_consumer_cancelled(self, method_frame): + """Invoked by pika when RabbitMQ sends a Basic.Cancel for a consumer receiving messages. + + :param pika.frame.Method method_frame: The Basic.Cancel frame + """ + self.logger.debug('Consumer was cancelled remotely, shutting down: %r' % method_frame) + if self._channel: + self.close_channel() + + def on_cancelok(self, unused_frame): + """This method is invoked by pika when RabbitMQ acknowledges the cancellation of a consumer. At this point we + will close the channel. This will invoke the on_channel_closed method once the channel has been closed, which + will in-turn close the connection. + + :param pika.frame.Method unused_frame: The Basic.CancelOK frame + """ + self.logger.debug(unused_frame) + self.logger.debug('RabbitMQ acknowledged the cancellation of the consumer') diff --git a/brewtils/rest/__init__.py b/brewtils/rest/__init__.py new file mode 100644 index 00000000..d7e7acc2 --- /dev/null +++ b/brewtils/rest/__init__.py @@ -0,0 +1,30 @@ + +def normalize_url_prefix(url_prefix): + + # Regex to find everything between the / / in the url + # Should cover all cases + # url_prefix -------- base_url + # None http://localhost:2337/ + # '' http://localhost:2337/ + # '/' http://localhost:2337/ + # 'example' http://localhost:2337/example/ + # '/example' http://localhost:2337/example/ + # 'example/' http://localhost:2337/example/ + # '/example/' http://localhost:2337/example/ + + if url_prefix in (None, '/', ''): + return '/' + + new_url_prefix = "" + + # Make string begin with / + if not url_prefix.startswith("/"): + new_url_prefix += '/' + + new_url_prefix += url_prefix + + # Make string end with / + if not url_prefix.endswith("/"): + new_url_prefix += '/' + + return new_url_prefix diff --git a/brewtils/rest/client.py b/brewtils/rest/client.py new file mode 100644 index 00000000..58fed6c6 --- /dev/null +++ b/brewtils/rest/client.py @@ -0,0 +1,213 @@ +import logging +import warnings + +import urllib3 +from requests import Session + +from brewtils.rest import normalize_url_prefix + + +class RestClient(object): + """Simple Rest Client for communicating to with BEERGARDEN. + + The is the low-level client responsible for making the actual REST calls. Other clients + (e.g. :py:class:`brewtils.rest.easy_client.EasyClient`) build on this by providing useful abstractions. + + :param host: BEERGARDEN REST API hostname. + :param port: BEERGARDEN REST API port. + :param ssl_enabled: Flag indicating whether to use HTTPS when communicating with BEERGARDEN. + :param api_version: The BEERGARDEN REST API version. Will default to the latest version. + :param logger: The logger to use. If None one will be created. + :param ca_cert: BEERGARDEN REST API server CA certificate. + :param client_cert: The client certificate to use when making requests. + :param url_prefix: BEERGARDEN REST API Url Prefix. + :param ca_verify: Flag indicating whether to verify server certificate when making a request. + """ + + # The Latest Version Currently released + LATEST_VERSION = 1 + + JSON_HEADERS = {'Content-type': 'application/json', 'Accept': 'text/plain'} + + def __init__(self, host, port, ssl_enabled=False, api_version=None, logger=None, ca_cert=None, client_cert=None, + url_prefix=None, ca_verify=True): + self.logger = logger or logging.getLogger(__name__) + + # Configure the session to use when making requests + self.session = Session() + + if not ca_verify: + urllib3.disable_warnings() + self.session.verify = False + elif ca_cert: + self.session.verify = ca_cert + + if client_cert: + self.session.cert = client_cert + + # Configure the BEERGARDEN URLs + base_url = 'http%s://%s:%s%s' % ('s' if ssl_enabled else '', host, port, normalize_url_prefix(url_prefix)) + self.version_url = base_url + 'version' + self.config_url = base_url + 'config' + + api_version = api_version or self.LATEST_VERSION + if api_version == 1: + self.system_url = base_url + 'api/v1/systems/' + self.instance_url = base_url + 'api/v1/instances/' + self.command_url = base_url + 'api/v1/commands/' + self.request_url = base_url + 'api/v1/requests/' + self.queue_url = base_url + 'api/v1/queues/' + self.logging_config_url = base_url + 'api/v1/config/logging/' + self.event_url = base_url + 'api/vbeta/events/' + else: + raise ValueError("Invalid BEERGARDEN API version: %s" % api_version) + + def get_version(self, **kwargs): + """Perform a GET to the version URL + + :param kwargs: Parameters to be used in the GET request + :return: The request response + """ + return self.session.get(self.version_url, params=kwargs) + + def get_logging_config(self, **kwargs): + """Perform a GET to the logging config URL + + :param kwargs: Parameters to be used in the GET request + :return: The request response + """ + return self.session.get(self.logging_config_url, params=kwargs) + + def get_systems(self, **kwargs): + """Perform a GET on the System collection URL + + :param kwargs: Parameters to be used in the GET request + :return: The request response + """ + return self.session.get(self.system_url, params=kwargs) + + def get_system(self, system_id, **kwargs): + """Performs a GET on the System URL + + :param system_id: ID of system + :param kwargs: Parameters to be used in the GET request + :return: Response to the request + """ + return self.session.get(self.system_url + system_id, params=kwargs) + + def post_systems(self, payload): + """Performs a POST on the System URL + + :param payload: New request definition + :return: Response to the request + """ + return self.session.post(self.system_url, data=payload, headers=self.JSON_HEADERS) + + def patch_system(self, system_id, payload): + """Performs a PATCH on a System URL + + :param system_id: ID of system + :param payload: The update specification + :return: Response + """ + return self.session.patch(self.system_url + str(system_id), data=payload, headers=self.JSON_HEADERS) + + def delete_system(self, system_id): + """Performs a DELETE on a System URL + + :param system_id: The ID of the system to remove + :return: Response to the request + """ + return self.session.delete(self.system_url + system_id) + + def patch_instance(self, instance_id, payload): + """Performs a PATCH on the instance URL + + :param instance_id: ID of instance + :param payload: The update specification + :return: Response + """ + return self.session.patch(self.instance_url + str(instance_id), data=payload, headers=self.JSON_HEADERS) + + def get_commands(self): + """Performs a GET on the Commands URL""" + return self.session.get(self.command_url) + + def get_command(self, command_id): + """Performs a GET on the Command URL + + :param command_id: ID of command + :return: Response to the request + """ + return self.session.get(self.command_url + command_id) + + def get_requests(self, **kwargs): + """Performs a GET on the Requests URL + + :param kwargs: Parameters to be used in the GET request + :return: Response to the request + """ + return self.session.get(self.request_url, params=kwargs) + + def get_request(self, request_id): + """Performs a GET on the Request URL + + :param request_id: ID of request + :return: Response to the request + """ + return self.session.get(self.request_url + request_id) + + def post_requests(self, payload): + """Performs a POST on the Request URL + + :param payload: New request definition + :return: Response to the request + """ + return self.session.post(self.request_url, data=payload, headers=self.JSON_HEADERS) + + def patch_request(self, request_id, payload): + """Performs a PATCH on the Request URL + + :param request_id: ID of request + :param payload: New request definition + :return: Response to the request + """ + return self.session.patch(self.request_url + str(request_id), data=payload, headers=self.JSON_HEADERS) + + def post_event(self, payload, publishers=None): + """Performs a POST on the event URL + + :param payload: New event definition + :param publishers: Array of publishers to use + :return: Response to the request + """ + return self.session.post(self.event_url, data=payload, headers=self.JSON_HEADERS, + params={'publisher': publishers} if publishers else None) + + def get_queues(self): + """Performs a GET on the Queues URL + + :return: Response to the request + """ + return self.session.get(self.queue_url) + + def delete_queues(self): + """Performs a DELETE on the Queues URL + + :return: Response to the request + """ + return self.session.delete(self.queue_url) + + def delete_queue(self, queue_name): + """Performs a DELETE on a specific Queue URL + + :return: Response to the request + """ + return self.session.delete(self.queue_url + queue_name) + + +class BrewmasterRestClient(RestClient): + def __init__(self, *args, **kwargs): + warnings.warn("Call made to 'BrewmasterRestClient'. This name will be removed in version 3.0, please use " + "'RestClient' instead.", DeprecationWarning, stacklevel=2) + super(BrewmasterRestClient, self).__init__(*args, **kwargs) diff --git a/brewtils/rest/easy_client.py b/brewtils/rest/easy_client.py new file mode 100644 index 00000000..b46d67f3 --- /dev/null +++ b/brewtils/rest/easy_client.py @@ -0,0 +1,369 @@ +import logging +import warnings + +from brewtils.errors import BrewmasterFetchError, BrewmasterValidationError, BrewmasterSaveError, \ + BrewmasterDeleteError, BrewmasterConnectionError, BGNotFoundError, BGConflictError, BrewmasterRestError +from brewtils.models import Event, PatchOperation +from brewtils.rest.client import RestClient +from brewtils.schema_parser import SchemaParser + + +class EasyClient(object): + """Client for communicating with BEERGARDEN. + + This class provides nice wrappers around the functionality provided by a :py:class:`brewtils.rest.client.RestClient` + + :param host: BEERGARDEN REST API hostname. + :param port: BEERGARDEN REST API port. + :param ssl_enabled: Flag indicating whether to use HTTPS when communicating with BEERGARDEN. + :param api_version: The BEERGARDEN REST API version. Will default to the latest version. + :param ca_cert: BEERGARDEN REST API server CA certificate. + :param client_cert: The client certificate to use when making requests. + :param parser: The parser to use. If None will default to an instance of BrewmasterSchemaParser. + :param logger: The logger to use. If None one will be created. + :param url_prefix: BEERGARDEN REST API URL Prefix. + :param ca_verify: Flag indicating whether to verify server certificate when making a request. + """ + + def __init__(self, host, port, ssl_enabled=False, api_version=None, ca_cert=None, client_cert=None, + parser=None, logger=None, url_prefix=None, ca_verify=True): + self.logger = logger or logging.getLogger(__name__) + self.parser = parser or SchemaParser() + self.client = RestClient(host=host, port=port, ssl_enabled=ssl_enabled, api_version=api_version, + ca_cert=ca_cert, client_cert=client_cert, url_prefix=url_prefix, ca_verify=ca_verify) + + def get_version(self, **kwargs): + response = self.client.get_version(**kwargs) + if response.ok: + return response + else: + self._handle_response_failure(response, default_exc=BrewmasterFetchError) + + def find_unique_system(self, **kwargs): + """Find a unique system using keyword arguments as search parameters. + + :param kwargs: Search parameters + :return: One system instance + """ + if 'id' in kwargs: + return self._find_system_by_id(kwargs.pop('id'), **kwargs) + else: + systems = self.find_systems(**kwargs) + + if not systems: + return None + + if len(systems) > 1: + raise BrewmasterFetchError("More than one system found that specifies the given constraints") + + return systems[0] + + def find_systems(self, **kwargs): + """Find systems using keyword arguments as search parameters. + + :param kwargs: Search parameters + :return: A list of system instances satisfying the given search parameters + """ + response = self.client.get_systems(**kwargs) + + if response.ok: + return self.parser.parse_system(response.json(), many=True) + else: + self._handle_response_failure(response, default_exc=BrewmasterFetchError) + + def _find_system_by_id(self, system_id, **kwargs): + """Finds a system by id, convert JSON to a system object and return it.""" + response = self.client.get_system(system_id, **kwargs) + + if response.ok: + return self.parser.parse_system(response.json()) + else: + self._handle_response_failure(response, default_exc=BrewmasterFetchError, raise_404=False) + + def create_system(self, system): + """Create a new system by POSTing to a BREWMASTER server. + + :param system: The system to create + :return: The system creation response + """ + json_system = self.parser.serialize_system(system) + response = self.client.post_systems(json_system) + + if response.ok: + return self.parser.parse_system(response.json()) + else: + self._handle_response_failure(response, default_exc=BrewmasterSaveError) + + def update_system(self, system_id, new_commands=None, **kwargs): + """Update a system with a PATCH + + :param system_id: The ID of the system to update + :param new_commands: The new commands + + :Keyword Arguments: + * *metadata* (``dict``) The updated metadata for the system + * *description* (``str``) The updated description for the system + * *display_name* (``str``) The updated display_name for the system + * *icon_name* (``str``) The updated icon_name for the system + + :return: The response + """ + operations = [] + metadata = kwargs.pop("metadata", {}) + + if new_commands: + operations.append(PatchOperation('replace', '/commands', + self.parser.serialize_command(new_commands, to_string=False, many=True))) + + if metadata: + operations.append(PatchOperation('update', '/metadata', metadata)) + + for attr, value in kwargs.items(): + if value is not None: + operations.append(PatchOperation('replace', '/%s' % attr, value)) + + response = self.client.patch_system(system_id, self.parser.serialize_patch(operations, many=True)) + + if response.ok: + return self.parser.parse_system(response.json()) + else: + self._handle_response_failure(response, default_exc=BrewmasterSaveError) + + def remove_system(self, **kwargs): + """Remove a specific system using keyword arguments as search parameters. + + :param kwargs: Search parameters + :return: The response + """ + system = self.find_unique_system(**kwargs) + + if system is None: + raise BrewmasterFetchError("Could not find system matching the given search parameters") + + return self._remove_system_by_id(system.id) + + def _remove_system_by_id(self, system_id): + """Removes a system with the specified ID from Brewmaster system, raises BrewmasterDeleteError if an error + occurs while deleting""" + + if system_id is None: + raise BrewmasterDeleteError("Cannot delete a system without an id") + + response = self.client.delete_system(system_id) + if response.ok: + return True + else: + self._handle_response_failure(response, default_exc=BrewmasterDeleteError) + + def initialize_instance(self, instance_id): + """Start an instance by PATCHing to a BREWMASTER server. + + :param instance_id: The ID of the instance to start + :return: The start response + """ + response = self.client.patch_instance(instance_id, self.parser.serialize_patch(PatchOperation('initialize'))) + + if response.ok: + return self.parser.parse_instance(response.json()) + else: + self._handle_response_failure(response, default_exc=BrewmasterSaveError) + + def update_instance_status(self, instance_id, new_status): + """Update an instance by PATCHing to a BREWMASTER server. + + :param instance_id: The ID of the instance to start + :param new_status: The updated status + :return: The start response + """ + payload = PatchOperation('replace', '/status', new_status) + response = self.client.patch_instance(instance_id, self.parser.serialize_patch(payload)) + + if response.ok: + return self.parser.parse_instance(response.json()) + else: + self._handle_response_failure(response, default_exc=BrewmasterSaveError) + + def instance_heartbeat(self, instance_id): + """Send heartbeat to BREWMASTER for health and status purposes + + :param instance_id: The ID of the instance + :return: The response + """ + payload = PatchOperation('heartbeat') + response = self.client.patch_instance(instance_id, self.parser.serialize_patch(payload)) + + if response.ok: + return True + else: + self._handle_response_failure(response, default_exc=BrewmasterSaveError) + + def find_unique_request(self, **kwargs): + """Find a unique request using keyword arguments as search parameters. + + *NOTE: If 'id' is present in kwargs then all other parameters will be ignored. + + :param kwargs: Search parameters + :return: One request instance + """ + if 'id' in kwargs: + return self._find_request_by_id(kwargs.pop('id')) + else: + requests = self.find_requests(**kwargs) + + if not requests: + return None + + if len(requests) > 1: + raise BrewmasterFetchError("More than one request found that specifies the given constraints") + + return requests[0] + + def find_requests(self, **kwargs): + """Find requests using keyword arguments as search parameters. + + :param kwargs: Search parameters + :return: A list of request instances satisfying the given search parameters + """ + response = self.client.get_requests(**kwargs) + + if response.ok: + return self.parser.parse_request(response.json(), many=True) + else: + self._handle_response_failure(response, default_exc=BrewmasterFetchError) + + def _find_request_by_id(self, request_id): + """Finds a request by id, convert JSON to a request object and return it.""" + response = self.client.get_request(request_id) + + if response.ok: + return self.parser.parse_request(response.json()) + else: + self._handle_response_failure(response, default_exc=BrewmasterFetchError, raise_404=False) + + def create_request(self, request): + """Create a new request. + + :param request: The request to create + :return: The response + """ + json_request = self.parser.serialize_request(request) + response = self.client.post_requests(json_request) + + if response.ok: + return self.parser.parse_request(response.json()) + else: + self._handle_response_failure(response, default_exc=BrewmasterSaveError) + + def update_request(self, request_id, status=None, output=None, error_class=None): + """Set various fields on a request with a PATCH + + :param request_id: The ID of the request to update + :param status: The new status + :param output: The new output + :param error_class: The new error class + :return: The response + """ + operations = [] + + if status: + operations.append(PatchOperation('replace', '/status', status)) + if output: + operations.append(PatchOperation('replace', '/output', output)) + if error_class: + operations.append(PatchOperation('replace', '/error_class', error_class)) + + response = self.client.patch_request(request_id, self.parser.serialize_patch(operations, many=True)) + + if response.ok: + return self.parser.parse_request(response.json()) + else: + self._handle_response_failure(response, default_exc=BrewmasterSaveError) + + def get_logging_config(self, system_name): + """Get the logging configuration for a particular system. + + :param system_name: Name of system + :return: LoggingConfig object + """ + response = self.client.get_logging_config(system_name=system_name) + if response.ok: + return self.parser.parse_logging_config(response.json()) + else: + self._handle_response_failure(response, default_exc=BrewmasterConnectionError) + + def publish_event(self, *args, **kwargs): + """Publish a new event. + + :param args: The Event to create + :param _publishers: Optional list of specific publishers. If None all publishers will be used. + :param kwargs: If no Event is given in the *args, on will be constructed from the kwargs + :return: The response + """ + publishers = kwargs.pop('_publishers', None) + json_event = self.parser.serialize_event(args[0] if args else Event(**kwargs)) + + response = self.client.post_event(json_event, publishers=publishers) + + if response.ok: + return True + else: + self._handle_response_failure(response) + + def get_queues(self): + """Retrieve all queue information + + :return: The response + """ + response = self.client.get_queues() + + if response.ok: + return self.parser.parse_queue(response.json(), many=True) + else: + self._handle_response_failure(response) + + def clear_queue(self, queue_name): + """Cancel and clear all messages from a queue + + :return: The response + """ + response = self.client.delete_queue(queue_name) + + if response.ok: + return True + else: + self._handle_response_failure(response) + + def clear_all_queues(self): + """Cancel and clear all messages from all queues + + :return: The response + """ + response = self.client.delete_queues() + + if response.ok: + return True + else: + self._handle_response_failure(response) + + @staticmethod + def _handle_response_failure(response, default_exc=BrewmasterRestError, raise_404=True): + if response.status_code == 404: + if raise_404: + raise BGNotFoundError(response.json()) + else: + return None + elif response.status_code == 409: + raise BGConflictError(response.json()) + elif 400 <= response.status_code < 500: + raise BrewmasterValidationError(response.json()) + elif response.status_code == 503: + raise BrewmasterConnectionError(response.json()) + else: + raise default_exc(response.json()) + + +class BrewmasterEasyClient(EasyClient): + def __init__(self, *args, **kwargs): + warnings.warn("Call made to 'BrewmasterEasyClient'. This name will be removed in version 3.0, please use " + "'EasyClient' instead.", DeprecationWarning, stacklevel=2) + super(BrewmasterEasyClient, self).__init__(*args, **kwargs) diff --git a/brewtils/rest/system_client.py b/brewtils/rest/system_client.py new file mode 100644 index 00000000..5e331871 --- /dev/null +++ b/brewtils/rest/system_client.py @@ -0,0 +1,289 @@ +import logging +import time +import warnings +from concurrent.futures import ThreadPoolExecutor +from functools import partial + +from brewtils.errors import BrewmasterTimeoutError, BrewmasterFetchError, BrewmasterValidationError +from brewtils.models import Request +from brewtils.plugin import request_context +from brewtils.rest.easy_client import EasyClient + + +class SystemClient(object): + """High-level client for generating requests for a BEERGARDEN System. + + SystemClient creation: + This class is intended to be the main way to create BEERGARDEN requests. Create an instance with BEERGARDEN + connection information (optionally including a url_prefix) and a system name: + + client = SystemClient(host, port, 'example_system', ssl_enabled=True, url_prefix=None) + + Pass additional keyword arguments for more granularity: + + version_constraint: + Allows specifying a particular system version. Can be a version literal ('1.0.0') or the special value + 'latest.' Using 'latest' will allow the the SystemClient to retry a request if it fails due to a missing + system (see Creating Requests). + + default_instance: + The instance name to use when creating a request if no other instance name is specified. Since each + request must be addressed to a specific instance this is a convenience to prevent needing to specify the + 'default' instance for each request. + + always_update: + Always attempt to reload the system definition before making a request. This is useful to ensure + Requests are always made against the latest version of the system. If not set the System definition will + be loaded once (upon making the first request) and then only reloaded if a Request fails. + + Loading the System: + The System definition is lazily loaded, so nothing happens until the first attempt to send a Request. At that + point the SystemClient will query BEERGARDEN to get a system definition that matches the system_name and + version_constraint. If no matching system can be found a BrewmasterFetchError will be raised. If always_update + was set to True this will happen before making each request, not just the first. + + Making a Request: + The standard way to create and send requests is by calling object attributes: + + request = client.example_command(param_1='example_param') + + In the normal case this will block until the request completes. Request completion is determined by periodically + polling BEERGARDEN to check the Request status. The time between polling requests starts at 0.5s and doubles + each time the request has still not completed, up to max_delay. If a timeout was specified and the Request has + not completed within that time a BrewmasterTimeoutError will be raised. + + It is also possible to create the SystemClient in non-blocking mode by specifying blocking=False. In this case + the request creation will immediately return a Future and will spawn a separate thread to poll for Request + completion. The max_concurrent parameter is used to control the maximum threads available for polling. + + # Create a SystemClient with blocking=False + client = SystemClient(host, port, 'example_system', ssl_enabled=True, blocking=False) + + # Create and send 5 requests without waiting for request completion + futures = [client.example_command(param_1=number) for number in range(5)] + + # Now wait on all requests to complete + concurrent.futures.wait(futures) + + If the request creation process fails (e.g. the command failed validation) and version_constraint is 'latest' + then the SystemClient will check to see if a different version is available, and if so it will attempt to make + the request on that version. This is so users of the SystemClient that don't necessarily care about the target + system version don't need to be restarted if the target system is updated. + + Tweaking BEERGARDEN Request Parameters: + There are several parameters that control how BEERGARDEN routes / processes a request. To denote these as + intended for BEERGARDEN itself (rather than a parameter to be passed to the Plugin) prepend a leading underscore + to the argument name. + + Sending to another instance: + request = client.example_command(_instance_name='instance_2', param_1='example_param') + + Request with a comment: + request = client.example_command(_comment='I'm a BEERGARDEN comment!', param_1='example_param') + + Without the leading underscore the arguments would be treated the same as param_1 - another parameter to be + passed to the plugin. + + :param host: BEERGARDEN REST API hostname. + :param port: BEERGARDEN REST API port. + :param system_name: The name of the system to use. + :param version_constraint: The system version to use. Can be specific or 'latest'. + :param default_instance: The instance to use if not specified when creating a request. + :param always_update: Specify if SystemClient should check for a newer System version before each request. + :param timeout: Length of time to wait for a request to complete. 'None' means wait forever. + :param max_delay: Maximum time to wait between checking the status of a created request. + :param api_version: BEERGARDEN API version. + :param ssl_enabled: Flag indicating whether to use HTTPS when communicating with BEERGARDEN. + :param ca_cert: BEERGARDEN REST API server CA certificate. + :param blocking: Flag indicating whether to block after request creation until the request completes. + :param max_concurrent: Maximum number of concurrent requests allowed. + :param client_cert: The client certificate to use when making requests. + :param url_prefix: BEERGARDEN REST API URL Prefix. + :param ca_verify: Flag indicating whether to verify server certificate when making a request. + """ + + def __init__(self, host, port, system_name, version_constraint='latest', default_instance='default', + always_update=False, timeout=None, max_delay=30, api_version=None, ssl_enabled=False, ca_cert=None, + blocking=True, max_concurrent=None, client_cert=None, url_prefix=None, ca_verify=True): + self._system_name = system_name + self._version_constraint = version_constraint + self._default_instance = default_instance + self._always_update = always_update + self._timeout = timeout + self._max_delay = max_delay + self._blocking = blocking + self._host = host + self._port = port + self.logger = logging.getLogger(__name__) + + self._loaded = False + self._system = None + self._commands = None + + self._thread_pool = ThreadPoolExecutor(max_workers=max_concurrent) + self._easy_client = EasyClient(host, port, ssl_enabled=ssl_enabled, api_version=api_version, ca_cert=ca_cert, + client_cert=client_cert, url_prefix=url_prefix, ca_verify=ca_verify) + + def __getattr__(self, item): + """Standard way to create and send BEERGARDEN requests""" + return self.create_bg_request(item) + + def create_bg_request(self, command_name, **kwargs): + """Create a callable that will execute a BEERGARDEN request when called. + + Normally you interact with the SystemClient by accessing attributes, but there could be certain cases where you + want to create a request without sending it. + + Example: + client = SystemClient(host, port, 'system', blocking=False) + requests = [] + + requests.append(client.create_bg_request('command_1')) # No arguments + requests.append(client.create_bg_request('command_2', arg_1='Hi!')) # arg_1 will be passed as a parameter + + futures = [request() for request in requests] # Calling creates and sends the request + concurrent.futures.wait(futures) # Wait for all the futures to complete + + :param command_name: The name of the command that will be sent. + :param kwargs: Additional arguments to pass to send_bg_request. + :raise AttributeError: The system does not have a command with the given command_name. + :return: A partial that will create and execute a BEERGARDEN request when called. + """ + + if not self._loaded or self._always_update: + self.load_bg_system() + + if command_name in self._commands: + return partial(self.send_bg_request, _command=command_name, _system_name=self._system.name, + _system_version=self._system.version, _system_display=self._system.display_name, + _output_type=self._commands[command_name].output_type, _instance_name=self._default_instance, + **kwargs) + else: + raise AttributeError("System '%s' version '%s' has no command named '%s'" % + (self._system.name, self._system.version, command_name)) + + def send_bg_request(self, **kwargs): + """Actually create a Request and send it to BEERGARDEN + + NOTE: This method is intended for advanced use only, mainly cases where you're using the SystemClient without a + predefined System. It assumes that everything needed to construct the request is being passed in kwargs. If + this doesn't sound like what you want you should check out create_bg_request. + + :param kwargs: All necessary request parameters, including BEERGARDEN internal parameters + :raise BrewmasterValidationError: If the Request creation failed validation on the server + :return: If the SystemClient was created with blocking=True a completed request object, otherwise a Future that + will return the Request when it completes. + """ + + # If the request fails validation and the version constraint allows, check for a new version and retry + try: + request = self._easy_client.create_request(self._construct_bg_request(**kwargs)) + except BrewmasterValidationError: + if self._system and self._version_constraint == 'latest': + old_version = self._system.version + + self.load_bg_system() + + if old_version != self._system.version: + kwargs['_system_version'] = self._system.version + return self.send_bg_request(**kwargs) + raise + + if self._blocking: + return self._wait_for_request(request) + else: + return self._thread_pool.submit(self._wait_for_request, request) + + def load_bg_system(self): + """Query BEERGARDEN for a System definition + + This method will make the query to BEERGARDEN for a System matching the name and version constraints specified + during SystemClient instance creation. + + If this method completes successfully the SystemClient will be ready to create and send Requests. + + :raise BrewmasterFetchError: If unable to find a matching System + :return: None + """ + + if self._version_constraint == 'latest': + systems = self._easy_client.find_systems(name=self._system_name) + self._system = sorted(systems, key=lambda x: x.version, reverse=True)[0] if systems else None + else: + self._system = self._easy_client.find_unique_system(name=self._system_name, + version=self._version_constraint) + + if self._system is None: + raise BrewmasterFetchError("BEERGARDEN has no system named '%s' with a version matching '%s'" % + (self._system_name, self._version_constraint)) + + self._commands = {command.name: command for command in self._system.commands} + self._loaded = True + + def _wait_for_request(self, request): + """Poll the server until the request is completed or errors""" + + delay_time = 0.5 + total_wait_time = 0 + while request.status not in Request.COMPLETED_STATUSES: + + if self._timeout and total_wait_time > self._timeout: + raise BrewmasterTimeoutError("Timeout reached waiting for request '%s' to complete" % str(request)) + + time.sleep(delay_time) + total_wait_time += delay_time + delay_time = min(delay_time * 2, self._max_delay) + + request = self._easy_client.find_unique_request(id=request.id) + + return request + + def _get_parent_for_request(self): + parent = getattr(request_context, 'current_request', None) + if parent is None: + return None + + if request_context.bg_host.upper() != self._host.upper() or request_context.bg_port != self._port: + self.logger.warning("A parent request was found, but the destination BEERGARDEN appears " + "to be different than the BEERGARDEN to which this plugin is assigned. " + "Cross-server parent/child requests are not supported at this time. " + "Removing the parent context so the request doesn't fail.") + return None + + return Request(id=str(parent.id)) + + def _construct_bg_request(self, **kwargs): + """Create a request that can be used with EasyClient.create_request""" + + command = kwargs.pop('_command', None) + system_name = kwargs.pop('_system_name', None) + system_version = kwargs.pop('_system_version', None) + system_display = kwargs.pop('_system_display', None) + instance_name = kwargs.pop('_instance_name', None) + comment = kwargs.pop('_comment', None) + output_type = kwargs.pop('_output_type', None) + metadata = kwargs.pop('_metadata', {}) + + parent = self._get_parent_for_request() + + if system_display: + metadata['system_display_name'] = system_display + + if command is None: + raise BrewmasterValidationError('Unable to send a request with no command') + if system_name is None: + raise BrewmasterValidationError('Unable to send a request with no system name') + if system_version is None: + raise BrewmasterValidationError('Unable to send a request with no system version') + if instance_name is None: + raise BrewmasterValidationError('Unable to send a request with no instance name') + + return Request(command=command, system=system_name, system_version=system_version, instance_name=instance_name, + comment=comment, output_type=output_type, parent=parent, metadata=metadata, parameters=kwargs) + + +class BrewmasterSystemClient(SystemClient): + def __init__(self, *args, **kwargs): + warnings.warn("Call made to 'BrewmasterSystemClient'. This name will be removed in version 3.0, please use " + "'SystemClient' instead.", DeprecationWarning, stacklevel=2) + super(BrewmasterSystemClient, self).__init__(*args, **kwargs) diff --git a/brewtils/schema_parser.py b/brewtils/schema_parser.py new file mode 100644 index 00000000..f163dc9c --- /dev/null +++ b/brewtils/schema_parser.py @@ -0,0 +1,263 @@ +import logging +import warnings + +from brewtils.models import System, Instance, Command, Parameter, Request, PatchOperation, Choices, LoggingConfig,\ + Event, Queue +from brewtils.schemas import SystemSchema, InstanceSchema, CommandSchema, ParameterSchema, RequestSchema, \ + PatchSchema, LoggingConfigSchema, EventSchema, QueueSchema + + +class SchemaParser(object): + """Serialize and deserialize Brewtils models""" + + _models = { + 'SystemSchema': System, + 'InstanceSchema': Instance, + 'CommandSchema': Command, + 'ParameterSchema': Parameter, + 'RequestSchema': Request, + 'PatchSchema': PatchOperation, + 'ChoicesSchema': Choices, + 'LoggingConfigSchema': LoggingConfig, + 'EventSchema': Event, + 'QueueSchema': Queue + } + + logger = logging.getLogger(__name__) + + # Deserialization methods + @classmethod + def parse_system(cls, system, from_string=False, **kwargs): + """Convert raw JSON string or dictionary to a system model object + + :param system: The raw input + :param from_string: True if 'system' is a JSON string, False if a dictionary + :param kwargs: Additional parameters to be passed to the Schema (e.g. many=True) + :return: A System object + """ + return cls._do_parse(system, SystemSchema(**kwargs), from_string=from_string) + + @classmethod + def parse_instance(cls, instance, from_string=False, **kwargs): + """Convert raw JSON string or dictionary to an instance model object + + :param instance: The raw input + :param from_string: True if 'instance' is a JSON string, False if a dictionary + :param kwargs: Additional parameters to be passed to the Schema (e.g. many=True) + :return: An Instance object + """ + return cls._do_parse(instance, InstanceSchema(**kwargs), from_string=from_string) + + @classmethod + def parse_command(cls, command, from_string=False, **kwargs): + """Convert raw JSON string or dictionary to a command model object + + :param command: The raw input + :param from_string: True if 'command' is a JSON string, False if a dictionary + :param kwargs: Additional parameters to be passed to the Schema (e.g. many=True) + :return: A Command object + """ + return cls._do_parse(command, CommandSchema(**kwargs), from_string=from_string) + + @classmethod + def parse_parameter(cls, parameter, from_string=False, **kwargs): + """Convert raw JSON string or dictionary to a parameter model object + + :param parameter: The raw input + :param from_string: True if 'parameter' is a JSON string, False if a dictionary + :param kwargs: Additional parameters to be passed to the Schema (e.g. many=True) + :return: An Parameter object + """ + return cls._do_parse(parameter, ParameterSchema(**kwargs), from_string=from_string) + + @classmethod + def parse_request(cls, request, from_string=False, **kwargs): + """Convert raw JSON string or dictionary to a request model object + + :param request: The raw input + :param from_string: True if 'request' is a JSON string, False if a dictionary + :param kwargs: Additional parameters to be passed to the Schema (e.g. many=True) + :return: A Request object + """ + return cls._do_parse(request, RequestSchema(**kwargs), from_string=from_string) + + @classmethod + def parse_patch(cls, patch, from_string=False, **kwargs): + """Convert raw JSON string or dictionary to a patch model object + + Note: for our patches, many is _always_ set to True. We will always return a list + from this method. + + :param patch: The raw input + :param from_string: True if 'patch' is a JSON string, False if a dictionary + :param kwargs: Additional parameters to be passed to the Schema (e.g. many=True) + :return: A PatchOperation object + """ + if not kwargs.pop('many', True): + cls.logger.warning("A patch object should always be wrapped as a list of objects. Thus, parsing will " + "always return a list. You specified many as False, this is being ignored and a list " + "will be returned anyway.") + return cls._do_parse(patch, PatchSchema(many=True, **kwargs), from_string=from_string) + + @classmethod + def parse_logging_config(cls, logging_config, from_string=False, **kwargs): + """Convert raw JSON string or dictionary to a logging config model object + + Note: for our logging_config, many is _always_ set to False. We will always return a dict + from this method. + + :param logging_config: The raw input + :param from_string: True if 'logging_config' is a JSON string, False if a dictionary + :param kwargs: Additional parameters to be passed to the Schema (e.g. many=True) + :return: A LoggingConfig object + """ + if kwargs.pop('many', False): + cls.logger.warning("A logging config object should never be wrapped as a list of objects. Thus, parsing " + "will always return a dict. You specified many as True, this is being ignored and a " + "dict will be returned anyway.") + return cls._do_parse(logging_config, LoggingConfigSchema(many=False, **kwargs), from_string=from_string) + + @classmethod + def parse_event(cls, event, from_string=False, **kwargs): + """Convert raw JSON string or dictionary to an event model object + + :param event: The raw input + :param from_string: True if 'event' is a JSON string, False if a dictionary + :param kwargs: Additional parameters to be passed to the Schema (e.g. many=True) + :return: An Event object + """ + return cls._do_parse(event, EventSchema(**kwargs), from_string=from_string) + + @classmethod + def parse_queue(cls, queue, from_string=False, **kwargs): + """Convert raw JSON string or dictionary to a queue model object + + :param queue: The raw input + :param from_string: True if 'event' is a JSON string, False if a dictionary + :param kwargs: Additional parameters to be passed to the Schema (e.g. many=True) + :return: A Queue object + """ + return cls._do_parse(queue, QueueSchema(**kwargs), from_string=from_string) + + @classmethod + def _do_parse(cls, data, schema, from_string=False): + schema.context['models'] = cls._models + return schema.loads(data).data if from_string else schema.load(data).data + + # Serialization methods + @classmethod + def serialize_system(cls, system, to_string=True, include_commands=True, **kwargs): + """Convert a system model into serialized form + + :param system: The system object(s) to be serialized + :param to_string: True to generate a JSON-formatted string, False to generate a dictionary + :param include_commands: True if the system's command list should be included + :param kwargs: Additional parameters to be passed to the Schema (e.g. many=True) + :return: Serialized representation of system + """ + if not include_commands: + if 'exclude' in kwargs: + kwargs['exclude'] += ('commands', ) + else: + kwargs['exclude'] = ('commands', ) + + return cls._do_serialize(SystemSchema(**kwargs), system, to_string) + + @classmethod + def serialize_instance(cls, instance, to_string=True, **kwargs): + """Convert an instance model into serialized form + + :param instance: The instance object(s) to be serialized + :param to_string: True to generate a JSON-formatted string, False to generate a dictionary + :param kwargs: Additional parameters to be passed to the Schema (e.g. many=True) + :return: Serialized representation of instance + """ + return cls._do_serialize(InstanceSchema(**kwargs), instance, to_string) + + @classmethod + def serialize_command(cls, command, to_string=True, **kwargs): + """Convert a command model into serialized form + + :param command: The command object(s) to be serialized + :param to_string: True to generate a JSON-formatted string, False to generate a dictionary + :param kwargs: Additional parameters to be passed to the Schema (e.g. many=True) + :return: Serialized representation of command + """ + return cls._do_serialize(CommandSchema(**kwargs), command, to_string) + + @classmethod + def serialize_parameter(cls, parameter, to_string=True, **kwargs): + """Convert a parameter model into serialized form + + :param parameter: The parameter object(s) to be serialized + :param to_string: True to generate a JSON-formatted string, False to generate a dictionary + :param kwargs: Additional parameters to be passed to the Schema (e.g. many=True) + :return: Serialized representation of parameter + """ + return cls._do_serialize(ParameterSchema(**kwargs), parameter, to_string) + + @classmethod + def serialize_request(cls, request, to_string=True, **kwargs): + """Convert a request model into serialized form + + :param request: The request object(s) to be serialized + :param to_string: True to generate a JSON-formatted string, False to generate a dictionary + :param kwargs: Additional parameters to be passed to the Schema (e.g. many=True) + :return: Serialized representation of request + """ + return cls._do_serialize(RequestSchema(**kwargs), request, to_string) + + @classmethod + def serialize_patch(cls, patch, to_string=True, **kwargs): + """Convert a patch model into serialized form + + :param patch: The patch object(s) to be serialized + :param to_string: True to generate a JSON-formatted string, False to generate a dictionary + :param kwargs: Additional parameters to be passed to the Schema (e.g. many=True) + :return: Serialized representation of patch + """ + return cls._do_serialize(PatchSchema(**kwargs), patch, to_string) + + @classmethod + def serialize_logging_config(cls, logging_config, to_string=True, **kwargs): + """Convert a logging config model into serialize form + + :param logging_config: The logging config object(s) to be serialized + :param to_string: True to generate a JSON-formatted string, False to generate a dictionary + :param kwargs: Additional parameters to be passed to the Schema (e.g. many=True) + :return: Serialized representation of logging config + """ + return cls._do_serialize(LoggingConfigSchema(**kwargs), logging_config, to_string) + + @classmethod + def serialize_event(cls, event, to_string=True, **kwargs): + """Convert a logging config model into serialized form + + :param event: The event object(s) to be serialized + :param to_string: True to generate a JSON-formatted string, False to generate a dictionary + :param kwargs: Additional parameters to be passed to the Schema (e.g. many=True) + :return: Serialized representation of event + """ + return cls._do_serialize(EventSchema(**kwargs), event, to_string) + + @classmethod + def serialize_queue(cls, queue, to_string=True, **kwargs): + """Convert a queue model into serialized form + + :param queue: The queue object(s) to be serialized + :param to_string: True to generate a JSON-formatted string, False to generate a dictionary + :param kwargs: Additional parameters to be passed to the Schema (e.g. many=True) + :return: Serialized representation of queue + """ + return cls._do_serialize(QueueSchema(**kwargs), queue, to_string) + + @staticmethod + def _do_serialize(schema, data, to_string): + return schema.dumps(data).data if to_string else schema.dump(data).data + + +class BrewmasterSchemaParser(SchemaParser): + def __init__(self): + warnings.warn("Reference made to 'BrewmasterSchemaParser'. This name will be removed in version 3.0, please " + "use 'SchemaParser' instead.", DeprecationWarning, stacklevel=2) + super(BrewmasterSchemaParser, self).__init__() diff --git a/brewtils/schemas.py b/brewtils/schemas.py new file mode 100644 index 00000000..2c737e9b --- /dev/null +++ b/brewtils/schemas.py @@ -0,0 +1,192 @@ +import calendar +import datetime + +from marshmallow import Schema, post_dump, post_load, pre_load, fields +from marshmallow.utils import UTC + + +class DateTime(fields.DateTime): + """Class that adds methods for (de)serializing DateTime fields as an epoch""" + + def __init__(self, format='epoch', **kwargs): + self.DATEFORMAT_SERIALIZATION_FUNCS['epoch'] = self.to_epoch + self.DATEFORMAT_DESERIALIZATION_FUNCS['epoch'] = self.from_epoch + super(DateTime, self).__init__(format=format, **kwargs) + + @staticmethod + def to_epoch(dt, localtime=False): + if localtime and dt.tzinfo is not None: + localized = dt + else: + if dt.tzinfo is None: + localized = UTC.localize(dt) + else: + localized = dt.astimezone(UTC) + return (calendar.timegm(localized.timetuple()) * 1000) + int(localized.microsecond / 1000) + + @staticmethod + def from_epoch(epoch): + # utcfromtimestamp will correctly parse milliseconds in Python 3, but in Python 2 we need to help it + seconds, millis = divmod(epoch, 1000) + return datetime.datetime.utcfromtimestamp(seconds).replace(microsecond=millis*1000) + + +class BaseSchema(Schema): + + def __init__(self, strict=True, **kwargs): + super(BaseSchema, self).__init__(strict=strict, **kwargs) + + @post_load + def make_object(self, data): + try: + model_class = self.context['models'][self.__class__.__name__] + except KeyError: + return data + + return model_class(**data) + + @classmethod + def get_attribute_names(cls): + return [key for key, value in cls._declared_fields.items() if isinstance(value, fields.FieldABC)] + + +class ChoicesSchema(BaseSchema): + + type = fields.Str() + display = fields.Str() + value = fields.Raw(many=True) + strict = fields.Bool(default=False) + details = fields.Dict() + + +class ParameterSchema(BaseSchema): + + key = fields.Str() + type = fields.Str(allow_none=True) + multi = fields.Bool(allow_none=True) + display_name = fields.Str(allow_none=True) + optional = fields.Bool(allow_none=True) + default = fields.Raw(allow_none=True) + description = fields.Str(allow_none=True) + choices = fields.Nested('ChoicesSchema', allow_none=True, many=False) + parameters = fields.Nested('self', many=True, allow_none=True) + nullable = fields.Bool(allow_none=True) + maximum = fields.Int(allow_none=True) + minimum = fields.Int(allow_none=True) + regex = fields.Str(allow_none=True) + form_input_type = fields.Str(allow_none=True) + + +class CommandSchema(BaseSchema): + + id = fields.Str(allow_none=True) + name = fields.Str() + description = fields.Str(allow_none=True) + parameters = fields.Nested('ParameterSchema', many=True) + command_type = fields.Str(allow_none=True) + output_type = fields.Str(allow_none=True) + schema = fields.Dict(allow_none=True) + form = fields.Dict(allow_none=True) + template = fields.Str(allow_none=True) + icon_name = fields.Str(allow_none=True) + system = fields.Nested('SystemSchema', only=('id', ), allow_none=True) + + +class InstanceSchema(BaseSchema): + + id = fields.Str(allow_none=True) + name = fields.Str() + description = fields.Str(allow_none=True) + status = fields.Str(allow_none=True) + status_info = fields.Nested('StatusInfoSchema', allow_none=True) + queue_type = fields.Str(allow_none=True) + queue_info = fields.Dict(allow_none=True) + icon_name = fields.Str(allow_none=True) + metadata = fields.Dict(allow_none=True) + + +class SystemSchema(BaseSchema): + + id = fields.Str(allow_none=True) + name = fields.Str() + description = fields.Str(allow_none=True) + version = fields.Str() + max_instances = fields.Integer(allow_none=True) + icon_name = fields.Str(allow_none=True) + instances = fields.Nested('InstanceSchema', many=True, allow_none=True) + commands = fields.Nested('CommandSchema', many=True) + display_name = fields.Str(allow_none=True) + metadata = fields.Dict(allow_none=True) + + +class RequestSchema(BaseSchema): + + id = fields.Str(allow_none=True) + system = fields.Str(allow_none=True) + system_version = fields.Str(allow_none=True) + instance_name = fields.Str(allow_none=True) + command = fields.Str(allow_none=True) + parent = fields.Nested('self', exclude=('children', ), allow_none=True) + children = fields.Nested('self', exclude=('parent', 'children'), many=True, default=None, allow_none=True) + parameters = fields.Dict(allow_none=True) + comment = fields.Str(allow_none=True) + output = fields.Str(allow_none=True) + output_type = fields.Str(allow_none=True) + status = fields.Str(allow_none=True) + command_type = fields.Str(allow_none=True) + error_class = fields.Str(allow_none=True) + created_at = DateTime(allow_none=True, format='epoch', example='1500065932000') + updated_at = DateTime(allow_none=True, format='epoch', example='1500065932000') + metadata = fields.Dict(allow_none=True) + + +class StatusInfoSchema(BaseSchema): + + heartbeat = DateTime(allow_none=True, format='epoch', example='1500065932000') + + +class PatchSchema(BaseSchema): + + operation = fields.Str() + path = fields.Str(allow_none=True) + value = fields.Raw(allow_none=True) + + @pre_load(pass_many=True) + def unwrap_envelope(self, data, many): + if isinstance(data, list): + return data + elif 'operations' in data: + return data['operations'] + else: + return [data] + + @post_dump(pass_many=True) + def wrap_envelope(self, data, many): + return {u'operations': data if many else [data]} + + +class LoggingConfigSchema(BaseSchema): + + level = fields.Str(allow_none=True) + formatters = fields.Dict(allow_none=True) + handlers = fields.Dict(allow_none=True) + + +class EventSchema(BaseSchema): + + name = fields.Str(allow_none=True) + payload = fields.Dict(allow_none=True) + error = fields.Bool(allow_none=True) + metadata = fields.Dict(allow_none=True) + timestamp = DateTime(allow_none=True, format='epoch', example='1500065932000') + + +class QueueSchema(BaseSchema): + + name = fields.Str(allow_none=True) + system = fields.Str(allow_none=True) + version = fields.Str(allow_none=True) + instance = fields.Str(allow_none=True) + system_id = fields.Str(allow_none=True) + display = fields.Str(allow_none=True) + size = fields.Integer(allow_none=True) diff --git a/brewtils/stoppable_thread.py b/brewtils/stoppable_thread.py new file mode 100644 index 00000000..f035da7c --- /dev/null +++ b/brewtils/stoppable_thread.py @@ -0,0 +1,25 @@ +import logging +from threading import Event, Thread + + +class StoppableThread(Thread): + """Thread class with a stop() method. The thread itself has to check regularly for the stopped() condition.""" + + def __init__(self, **kwargs): + self.logger = kwargs.pop('logger', logging.getLogger(__name__)) + self._stop_event = Event() + + Thread.__init__(self, **kwargs) + + def stop(self): + """Sets the stop event""" + self.logger.debug("Stopping thread: %s", self.name) + self._stop_event.set() + + def stopped(self): + """Determines if stop has been called yet.""" + return self._stop_event.isSet() + + def wait(self, timeout=None): + """Delegate wait call to threading.Event""" + return self._stop_event.wait(timeout) diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 00000000..49e95faa --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,238 @@ +# Makefile for Sphinx documentation +# + +# Module name +MODULE_NAME = brewtils + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = _build +APIBUILD = sphinx-apidoc +GENERATEDDIR = generated_docs + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . +# the i18n builder cannot share the environment and doctrees with the others +I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . + +.PHONY: default +default: clean apidocs html + +.PHONY: help +help: + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " singlehtml to make a single large HTML file" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " applehelp to make an Apple Help Book" + @echo " devhelp to make HTML files and a Devhelp project" + @echo " epub to make an epub" + @echo " epub3 to make an epub3" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " latexpdf to make LaTeX files and run them through pdflatex" + @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" + @echo " text to make text files" + @echo " man to make manual pages" + @echo " texinfo to make Texinfo files" + @echo " info to make Texinfo files and run them through makeinfo" + @echo " gettext to make PO message catalogs" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " xml to make Docutils-native XML files" + @echo " pseudoxml to make pseudoxml-XML files for display purposes" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + @echo " coverage to run coverage check of the documentation (if enabled)" + @echo " dummy to check syntax errors of document sources" + +.PHONY: clean +clean: + rm -rf $(BUILDDIR)/* + rm -rf $(GENERATEDDIR)/* + +.PHONY: apidocs +apidocs: + $(APIBUILD) -o $(GENERATEDDIR) ../$(MODULE_NAME) + +.PHONY: html +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +.PHONY: dirhtml +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +.PHONY: singlehtml +singlehtml: + $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml + @echo + @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." + +.PHONY: pickle +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +.PHONY: json +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +.PHONY: htmlhelp +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +.PHONY: qthelp +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Brewtils.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Brewtils.qhc" + +.PHONY: applehelp +applehelp: + $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp + @echo + @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." + @echo "N.B. You won't be able to view it unless you put it in" \ + "~/Library/Documentation/Help or install it in your application" \ + "bundle." + +.PHONY: devhelp +devhelp: + $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp + @echo + @echo "Build finished." + @echo "To view the help file:" + @echo "# mkdir -p $$HOME/.local/share/devhelp/Brewtils" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Brewtils" + @echo "# devhelp" + +.PHONY: epub +epub: + $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub + @echo + @echo "Build finished. The epub file is in $(BUILDDIR)/epub." + +.PHONY: epub3 +epub3: + $(SPHINXBUILD) -b epub3 $(ALLSPHINXOPTS) $(BUILDDIR)/epub3 + @echo + @echo "Build finished. The epub3 file is in $(BUILDDIR)/epub3." + +.PHONY: latex +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make' in that directory to run these through (pdf)latex" \ + "(use \`make latexpdf' here to do that automatically)." + +.PHONY: latexpdf +latexpdf: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through pdflatex..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +.PHONY: latexpdfja +latexpdfja: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through platex and dvipdfmx..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +.PHONY: text +text: + $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text + @echo + @echo "Build finished. The text files are in $(BUILDDIR)/text." + +.PHONY: man +man: + $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man." + +.PHONY: texinfo +texinfo: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo + @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." + @echo "Run \`make' in that directory to run these through makeinfo" \ + "(use \`make info' here to do that automatically)." + +.PHONY: info +info: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo "Running Texinfo files through makeinfo..." + make -C $(BUILDDIR)/texinfo info + @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." + +.PHONY: gettext +gettext: + $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale + @echo + @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." + +.PHONY: changes +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +.PHONY: linkcheck +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +.PHONY: doctest +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." + +.PHONY: coverage +coverage: + $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage + @echo "Testing of coverage in the sources finished, look at the " \ + "results in $(BUILDDIR)/coverage/python.txt." + +.PHONY: xml +xml: + $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml + @echo + @echo "Build finished. The XML files are in $(BUILDDIR)/xml." + +.PHONY: pseudoxml +pseudoxml: + $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml + @echo + @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." + +.PHONY: dummy +dummy: + $(SPHINXBUILD) -b dummy $(ALLSPHINXOPTS) $(BUILDDIR)/dummy + @echo + @echo "Build finished. Dummy builder generates no files." diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 00000000..97cf28f3 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,334 @@ +# -*- coding: utf-8 -*- +# +# Brewtils documentation build configuration file, created by +# sphinx-quickstart on Wed Sep 7 17:04:53 2016. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys +sys.path.insert(0, os.path.abspath('../')) + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autodoc', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The encoding of source files. +# +# source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'Brewtils' +copyright = u'2016, Logan Asher Jones' +author = u'Logan Asher Jones' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +from brewtils import _version +release = _version.__version__ + +# The short X.Y version. +version = '.'.join(release.split('.')[0:2]) + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +# +# today = '' +# +# Else, today_fmt is used as the format for a strftime call. +# +# today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +# The reST default role (used for this markup: `text`) to use for all +# documents. +# +# default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +# +# add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +# +# add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +# +# show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +# modindex_common_prefix = [] + +# If true, keep warnings as "system message" paragraphs in the built documents. +# keep_warnings = False + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'alabaster' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +# html_theme_path = [] + +# The name for this set of Sphinx documents. +# " v documentation" by default. +# +# html_title = u'Brewtils v1.0.0' + +# A shorter title for the navigation bar. Default is the same as html_title. +# +# html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +# +# html_logo = None + +# The name of an image file (relative to this directory) to use as a favicon of +# the docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +# +# html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Add any extra paths that contain custom files (such as robots.txt or +# .htaccess) here, relative to this directory. These files are copied +# directly to the root of the documentation. +# +# html_extra_path = [] + +# If not None, a 'Last updated on:' timestamp is inserted at every page +# bottom, using the given strftime format. +# The empty string is equivalent to '%b %d, %Y'. +# +# html_last_updated_fmt = None + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +# +# html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +# +# html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +# +# html_additional_pages = {} + +# If false, no module index is generated. +# +# html_domain_indices = True + +# If false, no index is generated. +# +# html_use_index = True + +# If true, the index is split into individual pages for each letter. +# +# html_split_index = False + +# If true, links to the reST sources are added to the pages. +# +# html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +# +# html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +# +# html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +# +# html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +# html_file_suffix = None + +# Language to be used for generating the HTML full-text search index. +# Sphinx supports the following languages: +# 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' +# 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr', 'zh' +# +# html_search_language = 'en' + +# A dictionary with options for the search language support, empty by default. +# 'ja' uses this config value. +# 'zh' user can custom change `jieba` dictionary path. +# +# html_search_options = {'type': 'default'} + +# The name of a javascript file (relative to the configuration directory) that +# implements a search results scorer. If empty, the default will be used. +# +# html_search_scorer = 'scorer.js' + +# Output file base name for HTML help builder. +htmlhelp_basename = 'Brewtilsdoc' + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'Brewtils.tex', u'Brewtils Documentation', + u'Logan Jones', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +# +# latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +# +# latex_use_parts = False + +# If true, show page references after internal links. +# +# latex_show_pagerefs = False + +# If true, show URL addresses after external links. +# +# latex_show_urls = False + +# Documents to append as an appendix to all manuals. +# +# latex_appendices = [] + +# If false, no module index is generated. +# +# latex_domain_indices = True + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'brewtils', u'Brewtils Documentation', + [author], 1) +] + +# If true, show URL addresses after external links. +# +# man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'Brewtils', u'Brewtils Documentation', + author, 'Brewtils', 'One line description of project.', + 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +# +# texinfo_appendices = [] + +# If false, no module index is generated. +# +# texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +# +# texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +# +# texinfo_no_detailmenu = False diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 00000000..6021ab31 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,22 @@ +.. Brewtils documentation master file, created by + sphinx-quickstart on Wed Sep 7 17:04:53 2016. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to Brewtils's documentation! +==================================== + +Contents: + +.. toctree:: + :maxdepth: 2 + + generated_docs/modules + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + diff --git a/requirements.in b/requirements.in new file mode 100644 index 00000000..7fc682cf --- /dev/null +++ b/requirements.in @@ -0,0 +1,24 @@ + +# Dependencies +six +requests +marshmallow +pika +wrapt +lark-parser +enum34 +futures ; python_version < "3.0" + +# Documentation Dependencies +sphinx + +# Build / Test Dependencies +mock +nose +rednose +coverage +nosexcover +pylint +tox +pytz + diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..49220049 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,52 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile --output-file requirements.txt requirements.in +# + +alabaster==0.7.10 # via sphinx +astroid==1.5.3 # via pylint +babel==2.5.1 # via sphinx +backports.functools-lru-cache==1.4 # via astroid, pylint +certifi==2017.7.27.1 # via requests +chardet==3.0.4 # via requests +colorama==0.3.9 # via rednose +configparser==3.5.0 # via pylint +coverage==4.4.1 +docutils==0.14 # via sphinx +enum34==1.1.6 +funcsigs==1.0.2 # via mock +futures==3.1.1 ; python_version < "3.0" +idna==2.6 # via requests +imagesize==0.7.1 # via sphinx +isort==4.2.15 # via pylint +jinja2==2.9.6 # via sphinx +lark-parser==0.3.7 +lazy-object-proxy==1.3.1 # via astroid +markupsafe==1.0 # via jinja2 +marshmallow==2.13.6 +mccabe==0.6.1 # via pylint +mock==2.0.0 +nose==1.3.7 +nosexcover==1.0.11 +pbr==3.1.1 # via mock +pika==0.11.0 +pluggy==0.5.2 # via tox +py==1.4.34 # via tox +pygments==2.2.0 # via sphinx +pylint==1.7.4 +pytz==2017.2 +rednose==1.2.2 +requests==2.18.4 +singledispatch==3.4.0.3 # via astroid, pylint +six==1.11.0 +snowballstemmer==1.2.1 # via sphinx +sphinx==1.6.4 +sphinxcontrib-websupport==1.0.1 # via sphinx +termstyle==0.1.11 # via rednose +tox==2.9.1 +typing==3.6.2 # via sphinx +urllib3==1.22 # via requests +virtualenv==15.1.0 # via tox +wrapt==1.10.11 diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..66c821ac --- /dev/null +++ b/setup.py @@ -0,0 +1,47 @@ +import re + +from setuptools import setup, find_packages + + +def find_version(): + version_file = "brewtils/_version.py" + version_line = open(version_file, "rt").read() + match_object = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", version_line, re.M) + + if not match_object: + raise RuntimeError("Unable to find version string in %s" % version_file) + + return match_object.group(1) + + +setup( + name='brewtils', + version=find_version(), + description='Utilities for building and running BEERGARDEN Systems', + url=' ', + author='The BEERGARDEN Team', + author_email=' ', + license='MIT', + packages=find_packages(exclude=['test', 'test.*']), + package_data={'': ['README.md']}, + install_requires=[ + 'marshmallow', + 'pika', + 'requests', + 'six', + 'wrapt', + 'lark-parser', + 'enum34', + 'futures;python_version<"3.0"' + ], + classifiers=[ + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Topic :: Software Development :: Libraries :: Python Modules", + ] +) diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/brewtils_test.py b/test/brewtils_test.py new file mode 100644 index 00000000..2710b1a5 --- /dev/null +++ b/test/brewtils_test.py @@ -0,0 +1,27 @@ +import unittest + +import brewtils +import brewtils.rest +from brewtils.rest.easy_client import EasyClient + + +class BrewtilsTest(unittest.TestCase): + + def setUp(self): + self.params = { + 'host': 'bg_host', + 'port': 1234, + 'system_name': 'system_name', + 'ca_cert': 'ca_cert', + 'client_cert': 'client_cert', + 'ssl_enabled': False, + "parser": "parser", + "logger": "logger" + } + + def test_get_easy_client(self): + client = brewtils.get_easy_client(**self.params) + self.assertIsInstance(client, EasyClient) + +if __name__ == '__main__': + unittest.main() diff --git a/test/choices_test.py b/test/choices_test.py new file mode 100644 index 00000000..00df878f --- /dev/null +++ b/test/choices_test.py @@ -0,0 +1,73 @@ +import unittest + +from lark.common import ParseError +from brewtils.choices import parse + + +class ChoicesTest(unittest.TestCase): + + def test_func_parse_success(self): + parse('function_name', parse_as='func') + parse('function_name()', parse_as='func') + parse('function_name(single=${arg})', parse_as='func') + parse('function_name(single=${arg}, another=${arg})', parse_as='func') + parse('function_name(single_param=${arg_param}, another=${arg})', parse_as='func') + + def test_func_parse_error(self): + self.assertRaises(ParseError, parse, '', parse_as='func') + self.assertRaises(ParseError, parse, 'function_name(', parse_as='func') + self.assertRaises(ParseError, parse, 'function_name(single)', parse_as='func') + self.assertRaises(ParseError, parse, 'function_name(single=)', parse_as='func') + self.assertRaises(ParseError, parse, 'function_name(single=arg)', parse_as='func') + self.assertRaises(ParseError, parse, 'function_name(single=$arg)', parse_as='func') + self.assertRaises(ParseError, parse, 'function_name(single=${arg)', parse_as='func') + self.assertRaises(ParseError, parse, 'function_name(single=$arg})', parse_as='func') + self.assertRaises(ParseError, parse, 'function_name(single=${arg},)', parse_as='func') + self.assertRaises(ParseError, parse, 'function_name(single=$arg, another=$arg)', parse_as='func') + self.assertRaises(ParseError, parse, 'function_name(single=${arg}, another=$arg)', parse_as='func') + self.assertRaises(ParseError, parse, 'function_name(single=${arg}, another=${arg}', parse_as='func') + + def test_url_parse_success(self): + parse('http://address', parse_as='url') + parse('https://address', parse_as='url') + parse('http://address:1234', parse_as='url') + parse('https://address:1234', parse_as='url') + parse('https://address:1234?param1=${arg}', parse_as='url') + parse('https://address:1234?param_1=${arg}¶m_2=${arg2}', parse_as='url') + + def test_url_parse_error(self): + self.assertRaises(ParseError, parse, '', parse_as='url') + self.assertRaises(ParseError, parse, 'htp://address', parse_as='url') + self.assertRaises(ParseError, parse, 'http://address?', parse_as='url') + self.assertRaises(ParseError, parse, 'http://address?param', parse_as='url') + self.assertRaises(ParseError, parse, 'http://address?param=', parse_as='url') + self.assertRaises(ParseError, parse, 'http://address?param=literal', parse_as='url') + self.assertRaises(ParseError, parse, 'http://address?param=$arg', parse_as='url') + self.assertRaises(ParseError, parse, 'http://address?param=${arg', parse_as='url') + self.assertRaises(ParseError, parse, 'http://address?param=${arg}&', parse_as='url') + self.assertRaises(ParseError, parse, 'http://address?param=${arg}¶m_2', parse_as='url') + self.assertRaises(ParseError, parse, 'http://address?param=${arg}¶m_2=', parse_as='url') + self.assertRaises(ParseError, parse, 'http://address?param=${arg}¶m_2=arg2', parse_as='url') + self.assertRaises(ParseError, parse, 'http://address?param=${arg}¶m_2=$arg2', parse_as='url') + self.assertRaises(ParseError, parse, 'http://address?param=${arg}¶m_2=${arg2', parse_as='url') + + def test_reference_parse_success(self): + self.assertEqual('index', parse('${index}', parse_as='reference')) + + def test_reference_parse_error(self): + self.assertRaises(ParseError, parse, '', parse_as='reference') + self.assertRaises(ParseError, parse, '$', parse_as='reference') + self.assertRaises(ParseError, parse, '${', parse_as='reference') + self.assertRaises(ParseError, parse, '$}', parse_as='reference') + self.assertRaises(ParseError, parse, '${}', parse_as='reference') + self.assertRaises(ParseError, parse, '{index}', parse_as='reference') + self.assertRaises(ParseError, parse, '$index}', parse_as='reference') + self.assertRaises(ParseError, parse, '${index', parse_as='reference') + self.assertRaises(ParseError, parse, 'a${index}', parse_as='reference') + self.assertRaises(ParseError, parse, '${index}a', parse_as='reference') + self.assertRaises(ParseError, parse, '${index} ${index2}', parse_as='reference') + + def test_parse_unknown_parse_as(self): + self.assertIn('address', parse('http://address')) + self.assertIn('name', parse('function')) + self.assertRaises(ParseError, parse, '') diff --git a/test/decorators_test.py b/test/decorators_test.py new file mode 100644 index 00000000..18122daa --- /dev/null +++ b/test/decorators_test.py @@ -0,0 +1,734 @@ +import sys +import unittest + +from mock import Mock, PropertyMock, call, patch + +import brewtils.decorators +from brewtils.decorators import system, command, parameter, _resolve_display_modifiers +from brewtils.errors import PluginParamError +from brewtils.models import Command, Parameter + + +class SystemTest(unittest.TestCase): + + def test_system(self): + @system + class A(object): + @command + def foo(self): + pass + + a = A() + self.assertEqual(len(a._commands), 1) + self.assertEqual(a._commands[0].name, 'foo') + + +class ParameterTest(unittest.TestCase): + + @patch('brewtils.decorators._generate_command_from_function') + def test_parameter_no_command_call_generate(self, mock_generate): + mock_name = PropertyMock(return_value='command1') + mock_command = Mock() + type(mock_command).name = mock_name + mock_param = Mock(choices=None) + mock_command.get_parameter_by_key = Mock(return_value=mock_param) + mock_generate.return_value = mock_command + + @parameter(key='foo') + def foo(self, foo): + pass + + mock_generate.assert_called_once() + + @patch('brewtils.decorators._generate_command_from_function') + def test_parameter_no_key(self, mock_generate): + mock_name = PropertyMock(return_value='command1') + mock_command = Mock() + type(mock_command).name = mock_name + mock_param = Mock() + mock_command.get_parameter_by_key = Mock(return_value=mock_param) + mock_generate.return_value = mock_command + + with self.assertRaises(PluginParamError): + @parameter() + def foo(self, x): + pass + + @patch('brewtils.decorators._generate_command_from_function') + def test_parameter_bad_key(self, mock_generate): + mock_name = PropertyMock(return_value='command1') + mock_command = Mock() + type(mock_command).name = mock_name + mock_command.get_parameter_by_key = Mock(return_value=None) + mock_generate.return_value = mock_command + + with self.assertRaises(PluginParamError): + @parameter(key='y') + def foo(self, x): + pass + + @patch('brewtils.decorators._generate_command_from_function') + def test_parameter_set_param_values(self, mock_generate): + mock_param = Mock(key='x') + mock_generate.return_value = Command('command1', parameters=[mock_param]) + + @parameter(key='x', type='Integer', multi=True, display_name='Professor X', optional=True, default='Charles', + description='Mutant', choices={'type': 'static', 'value': ['1', '2', '3']}) + def foo(self, x): + pass + + self.assertEqual(mock_param.key, 'x') + self.assertEqual(mock_param.multi, True) + self.assertEqual(mock_param.display_name, 'Professor X') + self.assertEqual(mock_param.optional, True) + self.assertEqual(mock_param.default, 'Charles') + self.assertEqual(mock_param.description, 'Mutant') + self.assertEqual(mock_param.choices.type, 'static') + self.assertEqual(mock_param.choices.value, ['1', '2', '3']) + self.assertEqual(mock_param.choices.display, 'select') + self.assertEqual(mock_param.choices.strict, True) + + def test_parameter_set_choices_list_small(self): + + @parameter(key='x', choices=['1', '2', '3']) + def foo(self, x): + pass + + self.assertEqual(hasattr(foo, '_command'), True) + c = foo._command + self.assertEqual(len(c.parameters), 1) + p = c.parameters[0] + self.assertEqual(p.choices.type, 'static') + self.assertEqual(p.choices.value, ['1', '2', '3']) + self.assertEqual(p.choices.display, 'select') + self.assertEqual(p.choices.strict, True) + + def test_parameter_set_choices_display_based_on_size(self): + + big_choices = [i for i in range(1000)] + + @parameter(key='x', choices=big_choices) + def foo(self, x): + pass + + self.assertEqual(hasattr(foo, '_command'), True) + c = foo._command + self.assertEqual(len(c.parameters), 1) + p = c.parameters[0] + self.assertEqual(p.choices.type, 'static') + self.assertEqual(p.choices.value, big_choices) + self.assertEqual(p.choices.display, 'typeahead') + self.assertEqual(p.choices.strict, True) + + def test_parameter_choices_is_dictionary_no_value_provided(self): + with self.assertRaises(PluginParamError): + @parameter(key='x', choices={'type': 'static', 'display': 'select'}) + def foo(self, x): + pass + + def test_parameter_choices_invalid_type(self): + with self.assertRaises(PluginParamError): + @parameter(key='x', choices={'type': 'INVALID_TYPE', 'value': [1, 2, 3], 'display': 'select'}) + def foo(self, x): + pass + + def test_parameter_choices_invalid_display(self): + with self.assertRaises(PluginParamError): + @parameter(key='x', choices={'type': 'static', 'value': [1, 2, 3], 'display': 'INVALID_DISPLAY'}) + def foo(self, x): + pass + + def test_parameter_choices_with_just_value_list(self): + + @parameter(key='x', choices={'value': [1, 2, 3]}) + def foo(self, x): + pass + + self.assertEqual(hasattr(foo, '_command'), True) + c = foo._command + self.assertEqual(len(c.parameters), 1) + p = c.parameters[0] + self.assertEqual(p.choices.type, 'static') + self.assertEqual(p.choices.value, [1, 2, 3]) + self.assertEqual(p.choices.display, 'select') + self.assertEqual(p.choices.strict, True) + + def test_parameter_choices_with_value_dict(self): + + @parameter(key='y') + @parameter(key='x', choices={'value': {'a': [1, 2, 3], 'b': [4, 5, 6]}, 'key_reference': '${y}'}) + def foo(self, x, y): + pass + + self.assertEqual(hasattr(foo, '_command'), True) + c = foo._command + self.assertEqual(len(c.parameters), 2) + p = c.parameters[0] + self.assertEqual(p.choices.type, 'static') + self.assertEqual(p.choices.value, {'a': [1, 2, 3], 'b': [4, 5, 6]}) + self.assertEqual(p.choices.display, 'select') + self.assertEqual(p.choices.strict, True) + self.assertEqual(p.choices.details['key_reference'], 'y') + + def test_parameter_choices_value_as_url_string(self): + + @parameter(key='x', choices='http://myhost:1234') + def foo(self, x): + pass + + self.assertEqual(hasattr(foo, '_command'), True) + c = foo._command + self.assertEqual(len(c.parameters), 1) + p = c.parameters[0] + self.assertEqual(p.choices.type, 'url') + self.assertEqual(p.choices.value, 'http://myhost:1234') + self.assertEqual(p.choices.display, 'typeahead') + self.assertEqual(p.choices.strict, True) + + def test_parameter_choices_value_as_command_string(self): + + @parameter(key='x', choices='my_command') + def foo(self, x): + pass + + self.assertEqual(hasattr(foo, '_command'), True) + c = foo._command + self.assertEqual(len(c.parameters), 1) + p = c.parameters[0] + self.assertEqual(p.choices.type, 'command') + self.assertEqual(p.choices.value, 'my_command') + self.assertEqual(p.choices.display, 'typeahead') + self.assertEqual(p.choices.strict, True) + + def test_parameter_choices_value_as_command_dict(self): + + @parameter(key='x', choices={'type': 'command', 'value': {'command': 'my_command'}}) + def foo(self, x): + pass + + self.assertEqual(hasattr(foo, '_command'), True) + c = foo._command + self.assertEqual(len(c.parameters), 1) + p = c.parameters[0] + self.assertEqual(p.choices.type, 'command') + self.assertEqual(p.choices.display, 'select') + self.assertEqual(p.choices.strict, True) + self.assertEqual(p.choices.details['name'], 'my_command') + + def test_parameter_choices_set_display_static_greater_than_50(self): + + @parameter(key='x', choices={'type': 'static', 'value': list(range(51))}) + def foo(self, x): + pass + + self.assertEqual(foo._command.parameters[0].choices.display, 'typeahead') + + def test_parameter_choices_set_display_static_less_than_50(self): + + @parameter(key='x', choices={'type': 'static', 'value': list(range(49))}) + def foo(self, x): + pass + + self.assertEqual(foo._command.parameters[0].choices.display, 'select') + + def test_parameter_invalid_choices(self): + + with self.assertRaises(PluginParamError): + @parameter(key='x', choices=1) + def foo(self, x): + pass + + def test_parameter_choices_url_value_mismatch(self): + + with self.assertRaises(PluginParamError): + @parameter(key='x', choices={'type': 'command', 'value': [1, 2, 3]}) + def foo(self, x): + pass + + def test_parameter_choices_static_value_mismatch(self): + + with self.assertRaises(PluginParamError): + @parameter(key='x', choices={'type': 'static', 'value': 'THIS_SHOULD_NOT_BE_A_STRING'}) + def foo(self, x): + pass + + def test_parameter_choices_dictionary_no_key_reference(self): + + with self.assertRaises(PluginParamError): + @parameter(key='x', choices={'type': 'static', 'value': {"a": [1, 2, 3]}}) + def foo(self, x): + pass + + def test_parameter_choices_parse_error(self): + + with self.assertRaises(PluginParamError): + @parameter(key='x', choices={'type': 'command', 'value': 'bad_def(x='}) + def foo(self, x): + pass + + +class CommandTest(unittest.TestCase): + + @patch('brewtils.decorators._generate_command_from_function', Mock()) + def test_command_no_wrapper(self): + flag = False + + @command + def foo(x): + return x + + self.assertEqual(flag, foo(flag)) + + @patch('brewtils.decorators._generate_command_from_function', Mock()) + def test_command_wrapper(self): + flag = False + brewtils.decorators._wrap_functions = True + + @command + def foo(x): + return x + + self.assertEqual(flag, foo(flag)) + + @patch('brewtils.decorators._generate_command_from_function') + def test_command_no_command_yet(self, mock_generate): + command_mock = Mock() + mock_generate.return_value = command_mock + + @command + def foo(self): + pass + + mock_generate.assert_called_once() + self.assertEqual(foo._command, command_mock) + + @patch('brewtils.decorators._generate_command_from_function') + @patch('brewtils.decorators._update_func_command') + def test_command_update_command(self, mock_update, mock_generate): + command1 = Mock() + command2 = Mock() + mock_generate.side_effect = [command1, command2] + + @command + @command + def foo(self): + pass + + mock_update.assert_called_with(command1, command2) + + @patch('brewtils.decorators._generate_params_from_function') + def test_command_generate_command_from_function(self, mock_generate): + mock_generate.return_value = [] + + @command + def foo(self): + """This is a doc""" + pass + + mock_generate.assert_called_once() + self.assertEqual(hasattr(foo, '_command'), True) + c = foo._command + self.assertEqual(c.name, 'foo') + self.assertEqual(c.description, 'This is a doc') + self.assertEqual(c.parameters, []) + + def test_command_generate_command_from_function_py2_compatibility(self): + py2_code_mock = Mock(co_varnames=["var1"], co_argcount=1, spec=["co_varnames", "co_argcount"]) + py2_method_mock = Mock(func_name="func_name", func_doc="func_doc", + __name__="__name__", __doc__="__doc__", + func_code=py2_code_mock, func_defaults=["default1"], + __code__=py2_code_mock, __defaults__=["default1"], + spec=["func_name", "func_doc", "func_code", "func_defaults"]) + command(py2_method_mock) + c = py2_method_mock._command + self.assertEqual(c.name, "func_name") + self.assertEqual(c.description, "func_doc") + + def test_command_generate_command_from_function_py3_compatibility(self): + py3_code_mock = Mock(co_varnames=["var1"], co_argcount=1, spec=["co_varnames", "co_argcount"]) + py3_method_mock = Mock(__name__="__name__", __doc__="__doc__", + func_code=py3_code_mock, func_defaults=["default1"], + __code__=py3_code_mock, __defaults__=["default1"], + spec=["__name__", "__doc__", "__code__", "__defaults__"]) + command(py3_method_mock) + c = py3_method_mock._command + self.assertEqual(c.name, "__name__") + self.assertEqual(c.description, "__doc__") + + def test_command_generate_params_from_function_with_extra_variables(self): + + @command + def foo(self, x, y='some_default'): + pass + + self.assertEqual(hasattr(foo, '_command'), True) + c = foo._command + self.assertEqual(len(c.parameters), 2) + + def test_command_generate_params_from_function(self): + @command + def foo(self, x, y='some_default'): + pass + + self.assertEqual(hasattr(foo, '_command'), True) + c = foo._command + self.assertEqual(len(c.parameters), 2) + x = c.get_parameter_by_key('x') + y = c.get_parameter_by_key('y') + self.assertIsNotNone(x) + self.assertIsNotNone(y) + + self.assertEqual(x.key, 'x') + self.assertEqual(x.default, None) + self.assertEqual(x.optional, False) + + self.assertEqual(y.key, 'y') + self.assertEqual(y.default, 'some_default') + self.assertEqual(y.optional, True) + + @patch('brewtils.decorators._generate_command_from_function') + def test_command_update_func_replace_command_attrs(self, mock_generate): + c1 = Command(name='command1', description='command1 desc', parameters=[]) + c2 = Command(name='command2', description='command2 desc', parameters=[]) + + mock_generate.side_effect = [c1, c2] + + @command + @command + def foo(self): + pass + + self.assertEqual(hasattr(foo, '_command'), True) + c = foo._command + self.assertEqual(c.name, 'command2') + self.assertEqual(c.description, 'command2 desc') + self.assertEqual(c.command_type, 'ACTION') + self.assertEqual(c.output_type, 'STRING') + + @patch('brewtils.decorators._generate_command_from_function') + def test_command_update_func_command_type(self, mock_generate): + c1 = Command(name='command1', description='command1 desc', parameters=[]) + c2 = Command(name='command2', description='command2 desc', parameters=[]) + + mock_generate.side_effect = [c1, c2] + + @command(command_type='INFO', output_type='JSON') + @command(command_type='ACTION', output_type='XML') + def foo(self): + pass + + self.assertEqual(hasattr(foo, '_command'), True) + c = foo._command + self.assertEqual(c.name, 'command2') + self.assertEqual(c.description, 'command2 desc') + self.assertEqual(c.command_type, 'INFO') + self.assertEqual(c.output_type, 'JSON') + + def test_parameter_no_wrapper(self): + flag = False + + @parameter(key='x') + def foo(self, x): + return x + + self.assertEqual(flag, foo(self, flag)) + + def test_parameter_wrapper(self): + flag = False + brewtils.decorators._wrap_functions = True + + @parameter(key='x') + def foo(self, x): + return x + + self.assertEqual(flag, foo(self, flag)) + + def test_parameter_default_empty_list(self): + + @parameter(key='x', default=[]) + def foo(self, x): + return x + + self.assertEqual(hasattr(foo, '_command'), True) + c = foo._command + self.assertEqual(len(c.parameters), 1) + p = c.parameters[0] + self.assertEqual(p.default, []) + + def test_parameter_is_kwarg(self): + @parameter(key='x', type='Integer', display_name="Professor X", optional=True, default="Charles", + description="cool psychic guy.", multi=False, is_kwarg=True) + def foo(self, **kwargs): + pass + + self.assertEqual(hasattr(foo, '_command'), True) + c = foo._command + self.assertEqual(len(c.parameters), 1) + p = c.parameters[0] + self.assertEqual(p.key, 'x') + self.assertEqual(p.type, 'Integer') + self.assertEqual(p.multi, False) + self.assertEqual(p.display_name, 'Professor X') + self.assertEqual(p.optional, True) + self.assertEqual(p.default, 'Charles') + self.assertEqual(p.description, 'cool psychic guy.') + + def test_command_do_not_override_parameter(self): + @parameter(key='x', type='Integer', multi=True, display_name='Professor X', optional=True, default='Charles', + description='I dont know') + @command + def foo(self, x): + pass + + self.assertEqual(hasattr(foo, '_command'), True) + c = foo._command + self.assertEqual(len(c.parameters), 1) + p = c.parameters[0] + self.assertEqual(p.key, 'x') + self.assertEqual(p.type, 'Integer') + self.assertEqual(p.multi, True) + self.assertEqual(p.display_name, 'Professor X') + self.assertEqual(p.optional, True) + self.assertEqual(p.default, 'Charles') + self.assertEqual(p.description, 'I dont know') + + def test_command_after_parameter_check_command_type(self): + @command(command_type='INFO') + @parameter(key='x', type='Integer', multi=True, display_name='Professor X', optional=True, default='Charles', + description='I dont know') + def foo(self, x): + pass + + self.assertEqual(hasattr(foo, '_command'), True) + c = foo._command + self.assertEqual(c.command_type, 'INFO') + + def test_command_before_parameter_check_command_type(self): + @parameter(key='x', type='Integer', multi=True, display_name='Professor X', optional=True, default='Charles', + description='I dont know') + @command(command_type='INFO') + def foo(self, x): + pass + + self.assertEqual(hasattr(foo, '_command'), True) + c = foo._command + self.assertEqual(c.command_type, 'INFO') + + def test_command_after_parameter_check_output_type(self): + @command(output_type='JSON') + @parameter(key='x', type='Integer', multi=True, display_name='Professor X', optional=True, default='Charles', + description='I dont know') + def foo(self, x): + pass + + self.assertEqual(hasattr(foo, '_command'), True) + c = foo._command + self.assertEqual(c.output_type, 'JSON') + + def test_command_before_parameter_check_output_type(self): + @parameter(key='x', type='Integer', multi=True, display_name='Professor X', optional=True, default='Charles', + description='I dont know') + @command(output_type='JSON') + def foo(self, x): + pass + + self.assertEqual(hasattr(foo, '_command'), True) + c = foo._command + self.assertEqual(c.output_type, 'JSON') + + def test_parameters_with_nested_model(self): + + class MyModel: + parameters = [ + Parameter(key='key1', type='Integer', multi=False, display_name='x', optional=True, default=1, + description='key1', choices={'type': 'static', 'value': [1, 2]}), + Parameter(key='key2', type='String', multi=False, display_name='y', optional=False, default='100', + description='key2', choices=['a', 'b', 'c']) + ] + + @parameter(key='complex', model=MyModel) + def foo(self, complex): + pass + + self.assertEqual(hasattr(foo, '_command'), True) + c = foo._command + self.assertEqual(len(c.parameters), 1) + p = c.parameters[0] + self.assertEqual(p.key, 'complex') + self.assertEqual(p.type, 'Dictionary') + self.assertEqual(p.default, {'key1': 1, 'key2': '100'}) + self.assertEqual(len(p.parameters), 2) + + np1 = p.parameters[0] + self.assertEqual(np1.key, 'key1') + self.assertEqual(np1.type, 'Integer') + self.assertEqual(np1.multi, False) + self.assertEqual(np1.display_name, 'x') + self.assertEqual(np1.optional, True) + self.assertEqual(np1.default, 1) + self.assertEqual(np1.description, 'key1') + self.assertEqual(np1.choices.type, 'static') + self.assertEqual(np1.choices.value, [1, 2]) + self.assertEqual(np1.choices.display, 'select') + self.assertEqual(np1.choices.strict, True) + + np2 = p.parameters[1] + self.assertEqual(np2.key, 'key2') + self.assertEqual(np2.type, 'String') + self.assertEqual(np2.multi, False) + self.assertEqual(np2.display_name, 'y') + self.assertEqual(np2.optional, False) + self.assertEqual(np2.default, '100') + self.assertEqual(np2.description, 'key2') + self.assertEqual(np2.choices.type, 'static') + self.assertEqual(np2.choices.value, ['a', 'b', 'c']) + self.assertEqual(np2.choices.display, 'select') + self.assertEqual(np2.choices.strict, True) + + def test_parameters_with_nested_model_with_default(self): + class MyModel: + parameters = [ + Parameter(key='key1', type='Integer', multi=False, display_name='x', optional=True, default=1, + description='key1'), + Parameter(key='key2', type='String', multi=False, display_name='y', optional=False, default='100', + description='key2') + ] + + @parameter(key='complex', model=MyModel, default={'key1': 123}) + def foo(self, complex): + pass + + p = foo._command.parameters[0] + self.assertEqual(p.key, 'complex') + self.assertEqual(p.type, 'Dictionary') + self.assertEqual(p.default, {'key1': 123}) + + def test_parameters_deep_nesting(self): + + class MyNestedModel: + parameters = [ + Parameter(key='key2', type='String', multi=False, display_name='y', optional=False, default='100', + description='key2') + ] + + class MyModel: + parameters = [ + Parameter(key='key1', multi=False, display_name='x', optional=True, description='key1', + parameters=[MyNestedModel], default="xval") + ] + + @parameter(key='nested_complex', model=MyModel) + def foo(self, nested_complex): + pass + + self.assertEqual(hasattr(foo, '_command'), True) + c = foo._command + self.assertEqual(len(c.parameters), 1) + p = c.parameters[0] + self.assertEqual(p.key, 'nested_complex') + self.assertEqual(p.type, 'Dictionary') + self.assertEqual(len(p.parameters), 1) + np1 = p.parameters[0] + self.assertEqual(np1.key, 'key1') + self.assertEqual(np1.type, 'Dictionary') + self.assertEqual(np1.multi, False) + self.assertEqual(np1.display_name, 'x') + self.assertEqual(np1.optional, True) + self.assertEqual(np1.description, 'key1') + self.assertEqual(len(np1.parameters), 1) + np2 = np1.parameters[0] + self.assertEqual(np2.key, 'key2') + self.assertEqual(np2.type, 'String') + self.assertEqual(np2.multi, False) + self.assertEqual(np2.display_name, 'y') + self.assertEqual(np2.optional, False) + self.assertEqual(np2.default, '100') + self.assertEqual(np2.description, 'key2') + + +class ResolveModfiersTester(unittest.TestCase): + + def setUp(self): + json_patcher = patch('brewtils.decorators.json') + self.addCleanup(json_patcher.stop) + self.json_patch = json_patcher.start() + + requests_patcher = patch('brewtils.decorators.requests') + self.addCleanup(requests_patcher.stop) + self.requests_patch = requests_patcher.start() + + def test_none(self): + args = {'schema': None, 'form': None, 'template': None} + self.assertEqual(args, _resolve_display_modifiers(Mock, Mock(), **args)) + + def test_dicts(self): + args = {'schema': {}, 'form': {}, 'template': None} + self.assertEqual(args, _resolve_display_modifiers(Mock, Mock(), **args)) + + def test_form_list(self): + self.assertEqual({'type': 'fieldset', 'items': []}, _resolve_display_modifiers(Mock, Mock(), form=[])['form']) + + def test_raw_template_string(self): + self.assertEqual('', _resolve_display_modifiers(Mock(), Mock(), template='').get('template')) + + def test_load_url(self): + args = {'schema': 'http://test/schema', 'form': 'http://test/form', 'template': 'http://test/template'} + + _resolve_display_modifiers(Mock(), Mock(), **args) + self.requests_patch.get.assert_has_calls([call(args['schema']), call(args['form']), call(args['template'])], + any_order=True) + + @patch('brewtils.decorators.inspect') + def test_absolute_path(self, inspect_mock): + args = {'schema': '/abs/path/schema.json', 'form': '/abs/path/form.json', 'template': '/abs/path/template.html'} + inspect_mock.getfile.return_value = '/abs/test/dir/client.py' + + with patch('builtins.open' if sys.version_info >= (3,) else '__builtin__.open') as open_mock: + _resolve_display_modifiers(Mock(), Mock(), **args) + open_mock.assert_has_calls([call(args['schema'], 'r'), call(args['form'], 'r'), + call(args['template'], 'r')], any_order=True) + + @patch('brewtils.decorators.inspect') + def test_relative_path(self, inspect_mock): + args = {'schema': '../rel/schema.json', 'form': '../rel/form.json', 'template': '../rel/template.html'} + inspect_mock.getfile.return_value = '/abs/test/dir/client.py' + + # DON'T PUT BREAKPOINTS INSIDE THIS CONTEXT MANAGER! PYCHARM WILL SCREW THINGS UP! + with patch('builtins.open' if sys.version_info >= (3,) else '__builtin__.open') as open_mock: + _resolve_display_modifiers(Mock(), Mock(), **args) + open_mock.assert_has_calls([call('/abs/test/rel/schema.json', 'r'), + call('/abs/test/rel/form.json', 'r'), + call('/abs/test/rel/template.html', 'r')], any_order=True) + + @patch('brewtils.decorators.inspect') + def test_json_parsing(self, inspect_mock): + inspect_mock.getfile.return_value = '/abs/test/dir/client.py' + + with patch('builtins.open' if sys.version_info >= (3,) else '__builtin__.open'): + _resolve_display_modifiers(Mock(), Mock(), template='/abs/template.html') + self.assertFalse(self.json_patch.loads.called) + + _resolve_display_modifiers(Mock(), Mock(), schema='/abs/schema', form='/abs/form', template='/abs/template') + self.assertEqual(2, self.json_patch.loads.call_count) + + def test_resolve_errors(self): + self.requests_patch.get.side_effect = Exception + self.assertRaises(PluginParamError, _resolve_display_modifiers, Mock(), Mock(), schema='http://test') + self.assertRaises(PluginParamError, _resolve_display_modifiers, Mock(), Mock(), form='http://test') + self.assertRaises(PluginParamError, _resolve_display_modifiers, Mock(), Mock(), template='http://test') + + with patch('builtins.open' if sys.version_info >= (3,) else '__builtin__.open') as open_mock: + open_mock.side_effect = Exception + self.assertRaises(PluginParamError, _resolve_display_modifiers, Mock(), Mock(), schema='./test') + self.assertRaises(PluginParamError, _resolve_display_modifiers, Mock(), Mock(), form='./test') + self.assertRaises(PluginParamError, _resolve_display_modifiers, Mock(), Mock(), template='./test') + + def test_type_errors(self): + self.assertRaises(PluginParamError, _resolve_display_modifiers, Mock, Mock(), template={}) + + self.assertRaises(PluginParamError, _resolve_display_modifiers, Mock(), Mock(), schema='') + self.assertRaises(PluginParamError, _resolve_display_modifiers, Mock(), Mock(), form='') + + self.assertRaises(PluginParamError, _resolve_display_modifiers, Mock(), Mock(), schema=123) + self.assertRaises(PluginParamError, _resolve_display_modifiers, Mock(), Mock(), form=123) + self.assertRaises(PluginParamError, _resolve_display_modifiers, Mock(), Mock(), template=123) diff --git a/test/log/__init__.py b/test/log/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/log/log_test.py b/test/log/log_test.py new file mode 100644 index 00000000..19f2bc08 --- /dev/null +++ b/test/log/log_test.py @@ -0,0 +1,55 @@ +import unittest +from mock import Mock, patch +from brewtils.log import * +from brewtils.models import LoggingConfig + + +class LogTest(unittest.TestCase): + + def setUp(self): + self.params = { + 'bg_host': 'bg_host', + 'bg_port': 1234, + 'system_name': 'system_name', + 'ca_cert': 'ca_cert', + 'client_cert': 'client_cert', + 'ssl_enabled': False + } + + @patch('brewtils.log.get_python_logging_config', Mock(return_value={})) + @patch('brewtils.log.logging.config.dictConfig') + def test_setup_logger(self, dict_config_mock): + setup_logger(**self.params) + dict_config_mock.assert_called_with({}) + + @patch('brewtils.get_easy_client') + @patch('brewtils.log.convert_logging_config', Mock(return_value="python_logging_config")) + def test_get_python_logging_config(self, get_client_mock): + mock_client = Mock(get_logging_config=Mock(return_value="logging_config")) + get_client_mock.return_value = mock_client + self.assertEqual("python_logging_config", get_python_logging_config(**self.params)) + + def test_convert_logging_config_all_overrides(self): + handlers = {"handler1": {}, "handler2": {}} + formatters = {"formatter1": {}, "formatter2": {}} + logging_config = LoggingConfig(level="level", handlers=handlers, + formatters=formatters) + python_config = convert_logging_config(logging_config) + self.assertEqual(python_config['handlers'], handlers) + self.assertEqual(python_config['formatters'], formatters) + self.assertTrue('root' in python_config) + root_logger = python_config['root'] + self.assertTrue('level' in root_logger) + self.assertTrue('handlers' in root_logger) + self.assertEqual(root_logger['level'], 'level') + self.assertEqual(sorted(root_logger['handlers']), ['handler1', 'handler2']) + + def test_convert_logging_config_no_overrides(self): + logging_config = LoggingConfig(level="level", handlers={}, formatters={}) + python_config = convert_logging_config(logging_config) + self.assertEqual(python_config['handlers'], DEFAULT_HANDLERS) + self.assertEqual(python_config['formatters'], DEFAULT_FORMATTERS) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/models_test.py b/test/models_test.py new file mode 100644 index 00000000..4e838b76 --- /dev/null +++ b/test/models_test.py @@ -0,0 +1,479 @@ +import unittest + +from mock import Mock, PropertyMock + +from brewtils.errors import BrewmasterModelValidationError, RequestStatusTransitionError +from brewtils.models import Command, Instance, Parameter, PatchOperation, Request, System, Choices, LoggingConfig,\ + Event, Queue + + +class CommandTest(unittest.TestCase): + + def test_parameter_keys(self): + param1 = Parameter(key='key1', optional=False) + param2 = Parameter(key='key2', optional=False) + c = Command(name='foo', description='bar', parameters=[param1, param2]) + keys = c.parameter_keys() + self.assertEqual(len(keys), 2) + self.assertIn('key1', keys) + self.assertIn('key2', keys) + + def test_get_parameter_by_key_none(self): + c = Command(name='foo', description='bar') + self.assertIsNone(c.get_parameter_by_key('some_key')) + + def test_get_parameter_by_key_true(self): + param1 = Parameter(key='key1', optional=False) + c = Command(name='foo', description='bar', parameters=[param1]) + self.assertEqual(c.get_parameter_by_key('key1'), param1) + + def test_get_parameter_by_key_False(self): + param1 = Parameter(key='key1', optional=False) + c = Command(name='foo', description='bar', parameters=[param1]) + self.assertIsNone(c.get_parameter_by_key('key2')) + + def test_has_different_parameters_different_length(self): + c = Command(name='foo', description='bar') + self.assertTrue(c.has_different_parameters([Parameter(key='key1', optional=False)])) + + def test_has_different_parameters_different_keys(self): + param1 = Parameter(key='key1', type='String', multi=True, display_name='Key 1', optional=True, default='key1', + description='this is key1') + c = Command(name='foo', description='bar', parameters=[param1]) + param2 = Parameter(key='key2', type='String', multi=True, display_name='Key 1', optional=True, default='key1', + description='this is key1') + self.assertTrue(c.has_different_parameters([param2])) + + def test_has_different_parameters_different_type(self): + param1 = Parameter(key='key1', type='String', multi=True, display_name='Key 1', optional=True, default='key1', + description='this is key1') + c = Command(name='foo', description='bar', parameters=[param1]) + param2 = Parameter(key='key1', type='Integer', multi=True, display_name='Key 1', optional=True, default='key1', + description='this is key1') + self.assertTrue(c.has_different_parameters([param2])) + + def test_has_different_parameters_different_multi(self): + param1 = Parameter(key='key1', type='String', multi=True, display_name='Key 1', optional=True, default='key1', + description='this is key1') + c = Command(name='foo', description='bar', parameters=[param1]) + param2 = Parameter(key='key1', type='String', multi=False, display_name='Key 1', optional=True, default='key1', + description='this is key1') + self.assertTrue(c.has_different_parameters([param2])) + + def test_has_different_parameters_different_optional(self): + param1 = Parameter(key='key1', type='String', multi=True, display_name='Key 1', optional=True, default='key1', + description='this is key1') + c = Command(name='foo', description='bar', parameters=[param1]) + param2 = Parameter(key='key1', type='String', multi=True, display_name='Key 1', optional=False, default='key1', + description='this is key1') + self.assertTrue(c.has_different_parameters([param2])) + + def test_has_different_parameters_different_default(self): + param1 = Parameter(key='key1', type='String', multi=True, display_name='Key 1', optional=True, default='key1', + description='this is key1') + c = Command(name='foo', description='bar', parameters=[param1]) + param2 = Parameter(key='key1', type='String', multi=True, display_name='Key 1', optional=True, default='key2', + description='this is key1') + self.assertTrue(c.has_different_parameters([param2])) + + def test_has_different_parameters_different_maximum(self): + param1 = Parameter(key='key1', type='String', multi=True, display_name='Key 1', optional=True, default='key1', + maximum=10) + c = Command(name='foo', description='bar', parameters=[param1]) + param2 = Parameter(key='key1', type='String', multi=True, display_name='Key 1', optional=True, default='key1', + maximum=20) + self.assertTrue(c.has_different_parameters([param2])) + + def test_has_different_parameters_different_minimum(self): + param1 = Parameter(key='key1', type='String', multi=True, display_name='Key 1', optional=True, default='key1', + minimum=10) + c = Command(name='foo', description='bar', parameters=[param1]) + param2 = Parameter(key='key1', type='String', multi=True, display_name='Key 1', optional=True, default='key1', + minimum=20) + self.assertTrue(c.has_different_parameters([param2])) + + def test_has_different_parameters_different_regex(self): + param1 = Parameter(key='key1', type='String', multi=False, display_name='Key 1', optional=True, default='key1', + regex=r'.') + c = Command(name='foo', description='bar', parameters=[param1]) + param2 = Parameter(key='key1', type='String', multi=False, display_name='Key 1', optional=True, default='key1', + regex=r'.*') + self.assertTrue(c.has_different_parameters([param2])) + + def test_has_different_parameters_different_order(self): + param1 = Parameter(key='key1', type='String', multi=True, display_name='Key 1', optional=True, default='key1', + description='this is key1') + param2 = Parameter(key='key2', type='String', multi=True, display_name='Key 2', optional=True, default='key2', + description='this is key2') + + c = Command(name='foo', description='bar', parameters=[param1, param2]) + self.assertFalse(c.has_different_parameters([param2, param1])) + + def test_has_different_parameters_false(self): + param1 = Parameter(key='key1', type='String', multi=True, display_name='Key 1', optional=True, default='key1', + description='this is key1') + c = Command(name='foo', description='bar', parameters=[param1]) + param2 = Parameter(key='key1', type='String', multi=True, display_name='Key 1', optional=True, default='key1', + description='this is key1') + self.assertFalse(c.has_different_parameters([param2])) + + def test_str(self): + c = Command(name='foo', description='bar', parameters=[]) + self.assertEqual('foo', str(c)) + + def test_repr(self): + c = Command(name='foo', description='bar', parameters=[]) + self.assertEqual('', repr(c)) + + +class InstanceTest(unittest.TestCase): + + def test_str(self): + self.assertEqual('name', str(Instance(name='name'))) + + def test_repr(self): + instance = Instance(name='name', status='RUNNING') + self.assertNotEqual(-1, repr(instance).find('name')) + self.assertNotEqual(-1, repr(instance).find('RUNNING')) + + +class ChoicesTest(unittest.TestCase): + + def test_str(self): + self.assertEqual('value', str(Choices(value='value'))) + + def test_repr(self): + choices = Choices(type='static', display='select', value=[1], strict=True) + self.assertNotEqual(-1, repr(choices).find('static')) + self.assertNotEqual(-1, repr(choices).find('select')) + self.assertNotEqual(-1, repr(choices).find('[1]')) + + +class ParameterTest(unittest.TestCase): + + def test_status_fields(self): + self.assertIn("String", Parameter.TYPES) + self.assertIn("Integer", Parameter.TYPES) + self.assertIn("Float", Parameter.TYPES) + self.assertIn("Boolean", Parameter.TYPES) + self.assertIn("Any", Parameter.TYPES) + self.assertIn("Dictionary", Parameter.TYPES) + self.assertIn("Date", Parameter.TYPES) + self.assertIn("DateTime", Parameter.TYPES) + + def test_str(self): + p = Parameter(key='foo', description='bar', type='Boolean', optional=False) + self.assertEqual('foo', str(p)) + + def test_repr(self): + p = Parameter(key='foo', description='bar', type='Boolean', optional=False) + self.assertEqual('', repr(p)) + + def test_is_different_mismatched_type(self): + p = Parameter(key='foo', description='bar', type='Boolean', optional=False) + self.assertTrue(p.is_different("NOT_A_PARAMETER")) + + def test_is_different_mismatched_required_field(self): + p1 = Parameter(key='foo', description='bar', type='Boolean', optional=False) + p2 = Parameter(key='bar', description='bar', type='Boolean', optional=False) + self.assertTrue(p1.is_different(p2)) + + def test_is_different_mismatched_number_of_parameters(self): + p1 = Parameter(key='foo', description='bar', type='Boolean', optional=False, parameters=[]) + p2 = Parameter(key='foo', description='bar', type='Boolean', optional=False, parameters=[p1]) + self.assertTrue(p1.is_different(p2)) + + def test_is_different_nested_parameter_different_missing_key(self): + nested_parameter1 = Parameter(key='foo', description='bar', type='Boolean', optional=False, parameters=[]) + nested_parameter2 = Parameter(key='bar', description='bar', type='Boolean', optional=False, parameters=[]) + p1 = Parameter(key='foo', description='bar', type='Boolean', optional=False, parameters=[nested_parameter1]) + p2 = Parameter(key='foo', description='bar', type='Boolean', optional=False, parameters=[nested_parameter2]) + self.assertTrue(p1.is_different(p2)) + + def test_is_different_nested_parameter_different(self): + nested_parameter1 = Parameter(key='foo', description='bar', type='Boolean', optional=False, parameters=[]) + nested_parameter2 = Parameter(key='foo', description='bar', type='String', optional=False, parameters=[]) + p1 = Parameter(key='foo', description='bar', type='Boolean', optional=False, parameters=[nested_parameter1]) + p2 = Parameter(key='foo', description='bar', type='Boolean', optional=False, parameters=[nested_parameter2]) + self.assertTrue(p1.is_different(p2)) + + def test_is_different_false_no_nested(self): + p1 = Parameter(key='foo', description='bar', type='Boolean', optional=False) + self.assertFalse(p1.is_different(p1)) + + def test_is_different_false_nested(self): + nested_parameter1 = Parameter(key='foo', description='bar', type='Boolean', optional=False, parameters=[]) + p1 = Parameter(key='foo', description='bar', type='Boolean', optional=False, parameters=[nested_parameter1]) + self.assertFalse(p1.is_different(p1)) + + +class RequestTest(unittest.TestCase): + + def test_command_type_fields(self): + self.assertEqual(Request.COMMAND_TYPES, Command.COMMAND_TYPES) + + def test_init_none_command_type(self): + try: + Request(system='foo', command='bar', command_type=None) + except BrewmasterModelValidationError: + self.fail("Request should be allowed to initialize a None Command Type.") + + def test_str(self): + self.assertEqual('command', str(Request(command='command'))) + + def test_repr(self): + request = Request(command='command', status='CREATED') + self.assertNotEqual(-1, repr(request).find('name')) + self.assertNotEqual(-1, repr(request).find('CREATED')) + + def test_set_valid_status(self): + request = Request(status='CREATED') + request.status = 'RECEIVED' + request.status = 'IN_PROGRESS' + request.status = 'SUCCESS' + + def test_invalid_status_transitions(self): + states = [ + ('SUCCESS', 'IN_PROGRESS'), + ('SUCCESS', 'ERROR'), + ('IN_PROGRESS', 'CREATED') + ] + for begin_status, end_status in states: + request = Request(status=begin_status) + try: + request.status = end_status + self.fail("Request should not be able to go from status {0} to {1}".format(begin_status, end_status)) + except RequestStatusTransitionError: + pass + + def test_is_ephemral(self): + request = Request(command_type=None) + self.assertFalse(request.is_ephemeral) + request.command_type = 'EPHEMERAL' + self.assertTrue(request.is_ephemeral) + + +class SystemTest(unittest.TestCase): + + def setUp(self): + self.default_system = System(name='foo', version='1.0.0') + + def tearDown(self): + self.default_system = None + + def test_get_command_by_name_found(self): + mock_name = PropertyMock(return_value='name') + command = Mock() + type(command).name = mock_name + self.default_system.commands.append(command) + self.assertEqual(self.default_system.get_command_by_name('name'), command) + + def test_get_command_by_name_none(self): + mock_name = PropertyMock(return_value='foo') + command = Mock() + type(command).name = mock_name + self.default_system.commands.append(command) + self.assertIsNone(self.default_system.get_command_by_name('name')) + + def test_has_instance_true(self): + self.default_system.instances = [Instance(name='foo')] + self.assertTrue(self.default_system.has_instance('foo')) + + def test_has_instance_false(self): + self.default_system.instances = [Instance(name='foo')] + self.assertFalse(self.default_system.has_instance('bar')) + + def test_instance_names(self): + self.default_system.instances = [Instance(name='foo'), Instance(name='bar')] + self.assertEqual(self.default_system.instance_names, ['foo', 'bar']) + + def test_get_instance_true(self): + instance = Instance(name='bar') + self.default_system.instances = [Instance(name='foo'), instance] + self.assertEqual(instance, self.default_system.get_instance('bar')) + + def test_get_instance_false(self): + self.default_system.instances = [Instance(name='foo')] + self.assertIsNone(self.default_system.get_instance('bar')) + + def test_has_different_commands_different_length(self): + self.assertEqual(self.default_system.has_different_commands([1]), True) + + def test_has_different_commands_different_name(self): + mock_name1 = PropertyMock(return_value='name') + mock_name2 = PropertyMock(return_value='name2') + command = Mock(description='description') + type(command).name = mock_name1 + + self.default_system.commands.append(command) + + new_command = Mock(description='description') + type(new_command).name = mock_name2 + command.has_different_parameters = Mock(return_value=False) + self.assertTrue(self.default_system.has_different_commands([new_command])) + + def test_has_different_commands_different_description(self): + mock_name = PropertyMock(return_value='name') + command = Mock(description='description') + type(command).name = mock_name + self.default_system.commands.append(command) + new_command = Mock(description='description2') + type(new_command).name = mock_name + command.has_different_parameters = Mock(return_value=False) + self.assertFalse(self.default_system.has_different_commands([new_command])) + + def test_has_different_commands_different_parameters_true(self): + mock_name = PropertyMock(return_value='name') + command = Mock(description='description') + type(command).name = mock_name + self.default_system.commands.append(command) + new_command = Mock(description='description') + type(new_command).name = mock_name + command.has_different_parameters = Mock(return_value=True) + self.assertTrue(self.default_system.has_different_commands([new_command])) + + def test_has_different_commands_the_same(self): + mock_name = PropertyMock(return_value='name') + command = Mock() + type(command).name = mock_name + command.description = 'description' + + self.default_system.commands.append(command) + + new_command = Mock() + type(new_command).name = mock_name + new_command.description = 'description' + command.has_different_parameters = Mock(return_value=False) + + self.assertFalse(self.default_system.has_different_commands([new_command])) + + def test_str(self): + self.assertEqual('foo-1.0.0', str(self.default_system)) + + def test_repr(self): + self.assertNotEqual(-1, repr(self.default_system).find('foo')) + self.assertNotEqual(-1, repr(self.default_system).find('1.0.0')) + + +class PatchOperationTest(unittest.TestCase): + + def test_str(self): + p = PatchOperation(operation='op', path='path', value='value') + self.assertEqual('op, path, value', str(p)) + + def test_str_only_operation(self): + p = PatchOperation(operation='op') + self.assertEqual('op, None, None', str(p)) + + def test_repr(self): + p = PatchOperation(operation='op', path='path', value='value') + self.assertEqual('', repr(p)) + + def test_repr_only_operation(self): + p = PatchOperation(operation='op') + self.assertEqual('', repr(p)) + + +class LoggingConfigTest(unittest.TestCase): + + def test_str(self): + c = LoggingConfig(level="INFO", + handlers={"logstash": {}, "stdout": {}, "file": {}}, + formatters={"default": {"format": LoggingConfig.DEFAULT_FORMAT}}, + loggers=None) + self.assertEqual('INFO, %s, %s' % (c.handler_names, c.formatter_names), str(c)) + + def test_repr(self): + c = LoggingConfig(level="INFO", + handlers={"logstash": {}, "stdout": {}, "file": {}}, + formatters={"default": {"format": LoggingConfig.DEFAULT_FORMAT}}, + loggers=None) + self.assertEqual('' % + (event.name, event.error, event.payload, event.metadata), repr(event)) + + +class QueueTest(unittest.TestCase): + + def test_str(self): + queue = Queue(name='echo.1-0-0.default', system='echo', version='1.0.0', instance='default', system_id='1234', + display='foo.1-0-0.default', size=3) + self.assertEqual('%s: %s' % (queue.name, queue.size), str(queue)) + + def test_repr(self): + queue = Queue(name='echo.1-0-0.default', system='echo', version='1.0.0', instance='default', system_id='1234', + display='foo.1-0-0.default', size=3) + self.assertEqual('', repr(queue)) diff --git a/test/plugin_test.py b/test/plugin_test.py new file mode 100644 index 00000000..4d394ca9 --- /dev/null +++ b/test/plugin_test.py @@ -0,0 +1,680 @@ +import json +import os +import threading +import unittest + +from mock import MagicMock, Mock, patch +from requests import ConnectionError + +from brewtils.errors import BrewmasterValidationError, RequestProcessingError, DiscardMessageException, \ + RepublishRequestException +from brewtils.models import Instance, Request, System, Command +from brewtils.plugin import PluginBase + + +class PluginBaseTest(unittest.TestCase): + + def setUp(self): + self.safe_copy = os.environ.copy() + + consumer_patcher = patch('brewtils.plugin.RequestConsumer') + self.addCleanup(consumer_patcher.stop) + self.consumer_patch = consumer_patcher.start() + + self.instance = Instance(id='id', name='default', queue_type='rabbitmq', + queue_info={'url': 'url', 'admin': {'name': 'admin_queue'}, + 'request': {'name': 'request_queue'}}) + self.system = System(name='test_system', version='1.0.0', instances=[self.instance], metadata={'foo': 'bar'}) + self.client = MagicMock(name='client', spec='command', _commands=['command']) + self.plugin = PluginBase(self.client, bg_host='localhost', system=self.system, metadata={'foo': 'bar'}) + self.plugin.instance = self.instance + + self.bm_client_mock = Mock(create_system=Mock(return_value=self.system), + initialize_instance=Mock(return_value=self.instance)) + self.plugin.bm_client = self.bm_client_mock + + self.parser_mock = Mock() + self.plugin.parser = self.parser_mock + + def tearDown(self): + os.environ = self.safe_copy + + def test_init_no_bg_host(self): + self.assertRaises(BrewmasterValidationError, PluginBase, self.client) + + def test_init_no_instance_name_unique_name_check(self): + self.assertEqual(self.plugin.unique_name, 'test_system[default]-1.0.0') + + def test_init_with_instance_name_unique_name_check(self): + plugin = PluginBase(self.client, bg_host='localhost', system=self.system, instance_name='unique') + self.assertEqual(plugin.unique_name, 'test_system[unique]-1.0.0') + + @patch('brewtils.plugin.logging.getLogger', Mock(return_value=Mock(root=Mock(handlers=[])))) + @patch('brewtils.plugin.logging.config.dictConfig', Mock()) + def test_init_defaults(self): + plugin = PluginBase(self.client, bg_host='localhost', system=self.system, metadata={'foo': 'bar'}) + self.assertEqual(plugin.instance_name, 'default') + self.assertEqual(plugin.bg_host, 'localhost') + self.assertEqual(plugin.bg_port, '2337') + self.assertEqual(plugin.ssl_enabled, True) + self.assertFalse(plugin._custom_logger) + self.assertEqual(plugin.ca_verify, True) + + def test_init_ssl_bad_env_variable(self): + os.environ['BG_SSL_ENABLED'] = 'BAD VALUE HERE' + plugin = PluginBase(self.client, bg_host='localhost', system=self.system) + self.assertEqual(plugin.ssl_enabled, True) + + def test_init_not_ssl(self): + plugin = PluginBase(self.client, bg_host='localhost', system=self.system, ssl_enabled=False) + self.assertEqual(plugin.ssl_enabled, False) + + def test_init_ssl_true_env(self): + os.environ['BG_SSL_ENABLED'] = 'True' + plugin = PluginBase(self.client, bg_host='localhost', system=self.system) + self.assertEqual(plugin.ssl_enabled, True) + + def test_init_ssl_false_env(self): + os.environ['BG_SSL_ENABLED'] = 'False' + plugin = PluginBase(self.client, bg_host='localhost', system=self.system) + self.assertEqual(plugin.ssl_enabled, False) + + def test_init_ca_verify_bad_env_variable(self): + os.environ['BG_CA_VERIFY'] = 'BAD VALUE HERE' + plugin = PluginBase(self.client, bg_host='localhost', system=self.system) + self.assertEqual(plugin.ca_verify, True) + + def test_init_not_ca_verify(self): + plugin = PluginBase(self.client, bg_host='localhost', system=self.system, ca_verify=False) + self.assertEqual(plugin.ca_verify, False) + + def test_init_ca_verify_true_env(self): + os.environ['BG_CA_VERIFY'] = 'True' + plugin = PluginBase(self.client, bg_host='localhost', system=self.system) + self.assertEqual(plugin.ca_verify, True) + + def test_init_ca_verify_false_env(self): + os.environ['BG_CA_VERIFY'] = 'False' + plugin = PluginBase(self.client, bg_host='localhost', system=self.system) + self.assertEqual(plugin.ca_verify, False) + + def test_init_bg_host_env(self): + os.environ['BG_WEB_HOST'] = 'envhost' + plugin = PluginBase(self.client, system=self.system) + self.assertEqual(plugin.bg_host, 'envhost') + + def test_init_bg_host_both(self): + os.environ['BG_WEB_HOST'] = 'envhost' + plugin = PluginBase(self.client, bg_host='localhost', system=self.system) + self.assertEqual(plugin.bg_host, 'localhost') + + def test_init_bg_port_env(self): + os.environ['BG_WEB_PORT'] = '4000' + plugin = PluginBase(self.client, bg_host='localhost', system=self.system) + self.assertEqual(plugin.bg_port, '4000') + + def test_init_bg_port_both(self): + os.environ['BG_WEB_PORT'] = '4000' + plugin = PluginBase(self.client, bg_host='localhost', bg_port=3000, system=self.system) + self.assertEqual(plugin.bg_port, 3000) + + def test_init_custom_logger(self): + plugin = PluginBase(self.client, bg_host='localhost', bg_port=3000, system=self.system, logger=Mock()) + self.assertTrue(plugin._custom_logger) + + @patch('brewtils.plugin.PluginBase._create_connection_poll_thread') + @patch('brewtils.plugin.PluginBase._create_standard_consumer') + @patch('brewtils.plugin.PluginBase._create_admin_consumer') + @patch('brewtils.plugin.PluginBase._initialize', Mock()) + def test_run(self, admin_create_mock, request_create_mock, poll_create_mock): + self.plugin.shutdown_event = Mock(wait=Mock(return_value=True)) + self.plugin.run() + self.assertTrue(admin_create_mock.called) + self.assertTrue(admin_create_mock.return_value.start.called) + self.assertTrue(admin_create_mock.return_value.stop.called) + self.assertTrue(request_create_mock.called) + self.assertTrue(request_create_mock.return_value.start.called) + self.assertTrue(request_create_mock.return_value.stop.called) + self.assertTrue(poll_create_mock.called) + self.assertTrue(poll_create_mock.return_value.start.called) + + @patch('brewtils.plugin.PluginBase._initialize', Mock()) + def test_run_things_died_unexpected(self): + self.plugin.shutdown_event = Mock(wait=Mock(side_effect=[False, True])) + admin_mock = Mock(isAlive=Mock(return_value=False), shutdown_event=Mock(is_set=Mock(return_value=False))) + request_mock = Mock(isAlive=Mock(return_value=False), shutdown_event=Mock(is_set=Mock(return_value=False))) + poll_mock = Mock(isAlive=Mock(return_value=False)) + self.plugin._create_admin_consumer = Mock(return_value=admin_mock) + self.plugin._create_standard_consumer = Mock(return_value=request_mock) + self.plugin._create_connection_poll_thread = Mock(return_value=poll_mock) + + self.plugin.run() + self.assertEqual(2, admin_mock.start.call_count) + self.assertEqual(2, request_mock.start.call_count) + self.assertEqual(2, poll_mock.start.call_count) + + @patch('brewtils.plugin.PluginBase._initialize', Mock()) + def test_run_consumers_closed_by_server(self): + self.plugin.shutdown_event = Mock(wait=Mock(side_effect=[False, True])) + admin_mock = Mock(isAlive=Mock(return_value=False), shutdown_event=Mock(is_set=Mock(return_value=True))) + request_mock = Mock(isAlive=Mock(return_value=False), shutdown_event=Mock(is_set=Mock(return_value=True))) + poll_mock = Mock(isAlive=Mock(return_value=True)) + self.plugin._create_admin_consumer = Mock(return_value=admin_mock) + self.plugin._create_standard_consumer = Mock(return_value=request_mock) + self.plugin._create_connection_poll_thread = Mock(return_value=poll_mock) + + self.plugin.run() + self.assertTrue(self.plugin.shutdown_event.set.called) + self.assertEqual(1, admin_mock.start.call_count) + self.assertEqual(1, request_mock.start.call_count) + + @patch('brewtils.plugin.PluginBase._create_standard_consumer') + @patch('brewtils.plugin.PluginBase._create_admin_consumer') + @patch('brewtils.plugin.PluginBase._create_connection_poll_thread', Mock()) + @patch('brewtils.plugin.PluginBase._initialize', Mock()) + def test_run_consumers_keyboard_interrupt(self, admin_create_mock, request_create_mock): + self.plugin.shutdown_event = Mock(wait=Mock(side_effect=KeyboardInterrupt)) + + self.plugin.run() + self.assertTrue(admin_create_mock.called) + self.assertTrue(admin_create_mock.return_value.start.called) + self.assertTrue(admin_create_mock.return_value.stop.called) + self.assertTrue(request_create_mock.called) + self.assertTrue(request_create_mock.return_value.start.called) + self.assertTrue(request_create_mock.return_value.stop.called) + + @patch('brewtils.plugin.PluginBase._create_standard_consumer') + @patch('brewtils.plugin.PluginBase._create_admin_consumer') + @patch('brewtils.plugin.PluginBase._create_connection_poll_thread', Mock()) + @patch('brewtils.plugin.PluginBase._initialize', Mock()) + def test_run_consumers_exception(self, admin_create_mock, request_create_mock): + self.plugin.shutdown_event = Mock(wait=Mock(side_effect=Exception)) + + self.plugin.run() + self.assertTrue(admin_create_mock.called) + self.assertTrue(admin_create_mock.return_value.start.called) + self.assertTrue(admin_create_mock.return_value.stop.called) + self.assertTrue(request_create_mock.called) + self.assertTrue(request_create_mock.return_value.start.called) + self.assertTrue(request_create_mock.return_value.stop.called) + + @patch('brewtils.plugin.PluginBase._format_output') + @patch('brewtils.plugin.PluginBase._invoke_command') + @patch('brewtils.plugin.PluginBase._update_request') + def test_process_message(self, update_mock, invoke_mock, format_mock): + target_mock = Mock() + request_mock = Mock(is_ephemeral=False) + + self.plugin.process_message(target_mock, request_mock, {}) + invoke_mock.assert_called_once_with(target_mock, request_mock) + self.assertEqual(2, update_mock.call_count) + self.assertEqual('SUCCESS', request_mock.status) + self.assertEqual(format_mock.return_value, request_mock.output) + format_mock.assert_called_once_with(invoke_mock.return_value) + + @patch('brewtils.plugin.PluginBase._invoke_command') + @patch('brewtils.plugin.PluginBase._update_request') + def test_process_message_invoke_exception(self, update_mock, invoke_mock): + target_mock = Mock() + request_mock = Mock(is_ephemeral=False) + invoke_mock.side_effect = ValueError('I am an error') + + self.plugin.process_message(target_mock, request_mock, {}) + invoke_mock.assert_called_once_with(target_mock, request_mock) + self.assertEqual(2, update_mock.call_count) + self.assertEqual('ERROR', request_mock.status) + self.assertEqual('I am an error', request_mock.output) + self.assertEqual('ValueError', request_mock.error_class) + + @patch('brewtils.plugin.PluginBase._invoke_command') + @patch('brewtils.plugin.PluginBase._update_request') + def test_process_message_invoke_exception_json_output_good_json(self, update_mock, invoke_mock): + target_mock = Mock() + request_mock = Mock(output_type="JSON") + invoke_mock.side_effect = ValueError('I am an error, but not JSON') + + self.plugin.process_message(target_mock, request_mock, {}) + invoke_mock.assert_called_once_with(target_mock, request_mock) + self.assertEqual(2, update_mock.call_count) + self.assertEqual('ERROR', request_mock.status) + self.assertEqual(json.dumps({"message": "I am an error, but not JSON", "attributes": {}}), request_mock.output) + self.assertEqual('ValueError', request_mock.error_class) + + @patch('brewtils.plugin.PluginBase._invoke_command') + @patch('brewtils.plugin.PluginBase._update_request') + def test_process_message_invoke_exception_json_output_exception_with_attributes(self, update_mock, invoke_mock): + class MyError(Exception): + def __init__(self, foo): + self.foo = foo + + target_mock = Mock() + request_mock = Mock(output_type="JSON") + exc = MyError("bar") + invoke_mock.side_effect = exc + + self.plugin.process_message(target_mock, request_mock, {}) + invoke_mock.assert_called_once_with(target_mock, request_mock) + self.assertEqual(2, update_mock.call_count) + self.assertEqual('ERROR', request_mock.status) + self.assertEqual(json.dumps({"message": str(exc), "attributes": {"foo": "bar"}}), request_mock.output) + self.assertEqual('MyError', request_mock.error_class) + + @patch('brewtils.plugin.PluginBase._invoke_command') + @patch('brewtils.plugin.PluginBase._update_request') + def test_process_message_invoke_exception_json_output_exception_with_bad_attributes(self, update_mock, invoke_mock): + class MyError(Exception): + def __init__(self, foo): + self.foo = foo + + target_mock = Mock() + request_mock = Mock(output_type="JSON") + message = MyError("another object") + thing = MyError(message) + invoke_mock.side_effect = thing + + self.plugin.process_message(target_mock, request_mock, {}) + invoke_mock.assert_called_once_with(target_mock, request_mock) + self.assertEqual(2, update_mock.call_count) + self.assertEqual('ERROR', request_mock.status) + self.assertEqual(json.dumps({"message": str(thing), "attributes": str(thing.__dict__)}), request_mock.output) + self.assertEqual('MyError', request_mock.error_class) + + @patch('brewtils.plugin.PluginBase._pre_process') + def test_process_new_request_message(self, pre_process_mock): + message_mock = Mock() + pool_mock = Mock() + self.plugin.pool = pool_mock + + self.plugin.process_request_message(message_mock, {}) + pre_process_mock.assert_called_once_with(message_mock) + pool_mock.submit.assert_called_once_with(self.plugin.process_message, self.plugin.client, + pre_process_mock.return_value, {}) + + @patch('brewtils.plugin.PluginBase._pre_process') + def test_process_completed_request_message(self, pre_process_mock): + message_mock = Mock() + pool_mock = Mock() + pre_process_mock.return_value.status = 'SUCCESS' + self.plugin.pool = pool_mock + + self.plugin.process_request_message(message_mock, {}) + pre_process_mock.assert_called_once_with(message_mock) + pool_mock.submit.assert_called_once_with(self.plugin._update_request, pre_process_mock.return_value, {}) + + @patch('brewtils.plugin.PluginBase._pre_process') + def test_process_admin_message(self, pre_process_mock): + message_mock = Mock() + pool_mock = Mock() + self.plugin.admin_pool = pool_mock + + self.plugin.process_admin_message(message_mock, {}) + pre_process_mock.assert_called_once_with(message_mock, verify_system=False) + pool_mock.submit.assert_called_once_with(self.plugin.process_message, self.plugin, + pre_process_mock.return_value, {}) + + def test_pre_process_request(self): + request = Request(id='id', system='test_system', system_version='1.0.0', command_type='ACTION') + self.parser_mock.parse_request.return_value = request + self.assertEqual(request, self.plugin._pre_process(Mock())) + + def test_pre_process_request_no_command_type(self): + request = Request(id='id', system='test_system', system_version='1.0.0') + self.parser_mock.parse_request.return_value = request + self.assertEqual(request, self.plugin._pre_process(Mock())) + + def test_pre_process_request_ephemeral(self): + request = Request(id='id', system='test_system', system_version='1.0.0', command_type='EPHEMERAL') + self.parser_mock.parse_request.return_value = request + self.assertEqual(request, self.plugin._pre_process(Mock())) + + def test_pre_process_shutting_down(self): + self.plugin.shutdown_event.set() + self.assertRaises(RequestProcessingError, self.plugin._pre_process, Mock()) + + def test_pre_process_request_wrong_system(self): + request = Request(system='foo', system_version='1.0.0', command_type='ACTION') + self.parser_mock.parse_request.return_value = request + self.assertRaises(DiscardMessageException, self.plugin._pre_process, Mock()) + + def test_pre_process_parse_error(self): + self.parser_mock.parse_request.side_effect = Exception + self.assertRaises(DiscardMessageException, self.plugin._pre_process, Mock()) + + def test_initialize_system_nonexistent(self): + self.bm_client_mock.find_unique_system.return_value = None + + self.plugin._initialize() + self.bm_client_mock.create_system.assert_called_once_with(self.system) + self.bm_client_mock.initialize_instance.assert_called_once_with(self.instance.id) + self.assertEqual(self.plugin.system, self.bm_client_mock.create_system.return_value) + self.assertEqual(self.plugin.instance, self.bm_client_mock.initialize_instance.return_value) + + def test_initialize_system_exists_same_commands(self): + self.bm_client_mock.update_system.return_value = self.system + self.bm_client_mock.find_unique_system.return_value = self.system + + self.plugin._initialize() + self.assertFalse(self.bm_client_mock.create_system.called) + self.bm_client_mock.initialize_instance.assert_called_once_with(self.instance.id) + self.assertEqual(self.plugin.system, self.bm_client_mock.create_system.return_value) + self.assertEqual(self.plugin.instance, self.bm_client_mock.initialize_instance.return_value) + + def test_initialize_system_exists_different_commands(self): + self.system.commands = [Command('test')] + self.bm_client_mock.update_system.return_value = self.system + + existing_system = System(id='id', name='test_system', version='1.0.0', instances=[self.instance], + metadata={'foo': 'bar'}) + self.bm_client_mock.find_unique_system.return_value = existing_system + + self.plugin._initialize() + self.assertFalse(self.bm_client_mock.create_system.called) + self.bm_client_mock.update_system.assert_called_once_with(self.instance.id, new_commands=self.system.commands, + metadata={"foo": "bar"}, + description=self.system.description, + icon_name=self.system.icon_name, + display_name=self.system.display_name) + self.bm_client_mock.initialize_instance.assert_called_once_with(self.instance.id) + self.assertEqual(self.plugin.system, self.bm_client_mock.create_system.return_value) + self.assertEqual(self.plugin.instance, self.bm_client_mock.initialize_instance.return_value) + + def test_initialize_system_update_metadata(self): + self.system.commands = [Command('test')] + self.bm_client_mock.update_system.return_value = self.system + + existing_system = System(id='id', name='test_system', version='1.0.0', instances=[self.instance], + metadata={}) + self.bm_client_mock.find_unique_system.return_value = existing_system + + self.plugin._initialize() + self.assertFalse(self.bm_client_mock.create_system.called) + self.bm_client_mock.update_system.assert_called_once_with(self.instance.id, new_commands=self.system.commands, + description=None, + display_name=None, + icon_name=None, + metadata={"foo": "bar"}) + self.bm_client_mock.initialize_instance.assert_called_once_with(self.instance.id) + self.assertEqual(self.plugin.system, self.bm_client_mock.create_system.return_value) + self.assertEqual(self.plugin.instance, self.bm_client_mock.initialize_instance.return_value) + + def test_shutdown(self): + self.plugin.request_consumer = Mock() + self.plugin.admin_consumer = Mock() + + self.plugin._shutdown() + self.assertTrue(self.plugin.request_consumer.stop.called) + self.assertTrue(self.plugin.request_consumer.join.called) + self.assertTrue(self.plugin.admin_consumer.stop.called) + self.assertTrue(self.plugin.admin_consumer.join.called) + + def test_create_request_consumer(self): + self.plugin._create_standard_consumer() + self.assertTrue(self.consumer_patch.called) + + def test_create_admin_consumer(self): + self.plugin._create_admin_consumer() + self.assertTrue(self.consumer_patch.called) + + def test_create_connection_poll_thread(self): + connection_poll_thread = self.plugin._create_connection_poll_thread() + self.assertIsInstance(connection_poll_thread, threading.Thread) + self.assertTrue(connection_poll_thread.daemon) + + @patch('brewtils.plugin.PluginBase._start') + def test_invoke_command_admin(self, start_mock): + params = {'p1': 'param'} + request = Request(system='test_system', system_version='1.0.0', command='_start', parameters=params) + + self.plugin._invoke_command(self.plugin, request) + start_mock.assert_called_once_with(self.plugin, **params) + + def test_invoke_command_request(self): + params = {'p1': 'param'} + request = Request(system='test_system', system_version='1.0.0', command='command', parameters=params) + self.client.command = Mock() + + self.plugin._invoke_command(self.client, request) + self.client.command.assert_called_once_with(**params) + + def test_invoke_command_no_attribute(self): + params = {'p1': 'param'} + request = Request(system='test_system', system_version='1.0.0', command='foo', parameters=params) + self.assertRaises(RequestProcessingError, self.plugin._invoke_command, self.plugin.client, request) + + def test_invoke_command_non_callable_attribute(self): + params = {'p1': 'param'} + request = Request(system='test_system', system_version='1.0.0', command='command', parameters=params) + self.assertRaises(RequestProcessingError, self.plugin._invoke_command, self.plugin.client, request) + + def test_update_request(self): + request_mock = Mock(is_ephemeral=False) + self.plugin._update_request(request_mock, {}) + self.assertTrue(self.bm_client_mock.update_request.called) + + def test_update_request_wait_during_error(self): + request_mock = Mock(is_ephemeral=False) + error_condition_mock = MagicMock() + self.plugin.brew_view_down = True + self.plugin.brew_view_error_condition = error_condition_mock + + self.plugin._update_request(request_mock, {}) + self.assertTrue(error_condition_mock.wait.called) + self.assertTrue(self.bm_client_mock.update_request.called) + + def test_update_request_connection_error(self): + request_mock = Mock(is_ephemeral=False) + self.bm_client_mock.update_request.side_effect = ConnectionError + + self.assertRaises(RepublishRequestException, self.plugin._update_request, request_mock, {}) + self.assertTrue(self.bm_client_mock.update_request.called) + self.assertTrue(self.plugin.brew_view_down) + + def test_update_request_different_error(self): + request_mock = Mock(is_ephemeral=False) + self.bm_client_mock.update_request.side_effect = ValueError + + self.assertRaises(RepublishRequestException, self.plugin._update_request, request_mock, {}) + self.assertTrue(self.bm_client_mock.update_request.called) + self.assertFalse(self.plugin.brew_view_down) + + def test_update_request_ephemeral(self): + request_mock = Mock(is_ephemeral=True) + self.plugin._update_request(request_mock, {}) + self.assertFalse(self.bm_client_mock.update_request.called) + + def test_update_request_final_attempt_succeeds(self): + request_mock = Mock(is_ephemeral=False, status='SUCCESS', output='Some output', error_class=None) + self.plugin.max_attempts = 1 + self.plugin._update_request(request_mock, {'retry_attempt': 1, 'time_to_wait': 5}) + self.bm_client_mock.update_request.assert_called_with( + request_mock.id, status='ERROR', + output='We tried to update the request, but ' + 'it failed too many times. Please check ' + 'the plugin logs to figure out why the request ' + 'update failed. It is possible for this request to have ' + 'succeeded, but we cannot update beer-garden with that ' + 'information.', error_class='BGGivesUpError') + + def test_wait_if_in_headers(self): + request_mock = Mock(is_ephemeral=False) + self.plugin.shutdown_event = Mock(wait=Mock(return_value=True)) + self.plugin._update_request(request_mock, {'retry_attempt': 1, 'time_to_wait': 1}) + self.assertTrue(self.plugin.shutdown_event.wait.called) + + def test_update_request_headers(self): + request_mock = Mock(is_ephemeral=False, status='SUCCESS', output='Some output', error_class=None) + self.plugin.shutdown_event = Mock(wait=Mock(return_value=True)) + self.bm_client_mock.update_request.side_effect = ValueError + with self.assertRaises(RepublishRequestException) as ex: + self.plugin._update_request(request_mock, {'retry_attempt': 1, 'time_to_wait': 5}) + print(dir(ex)) + print(ex.exception) + self.assertEqual(ex.exception.headers['retry_attempt'], 2) + self.assertEqual(ex.exception.headers['time_to_wait'], 10) + + def test_update_request_final_attempt_fails(self): + request_mock = Mock(is_ephemeral=False, status='SUCCESS', output='Some output', error_class=None) + self.plugin.max_attempts = 1 + self.bm_client_mock.update_request.side_effect = ValueError + self.assertRaises(DiscardMessageException, self.plugin._update_request, request_mock, {'retry_attempt': 1}) + + def test_start(self): + new_instance = Mock() + self.plugin.instance = self.instance + self.bm_client_mock.update_instance_status.return_value = new_instance + + self.assertTrue(self.plugin._start(Mock())) + self.bm_client_mock.update_instance_status.assert_called_once_with(self.instance.id, 'RUNNING') + self.assertEqual(self.plugin.instance, new_instance) + + def test_stop(self): + new_instance = Mock() + self.plugin.instance = self.instance + self.bm_client_mock.update_instance_status.return_value = new_instance + + self.assertTrue(self.plugin._stop(Mock())) + self.bm_client_mock.update_instance_status.assert_called_once_with(self.instance.id, 'STOPPED') + self.assertEqual(self.plugin.instance, new_instance) + + def test_status(self): + self.plugin._status(Mock()) + self.bm_client_mock.instance_heartbeat.assert_called_once_with(self.instance.id) + + def test_status_connection_error(self): + self.bm_client_mock.instance_heartbeat.side_effect = ConnectionError + self.assertRaises(ConnectionError, self.plugin._status, Mock()) + self.assertTrue(self.plugin.brew_view_down) + + def test_status_other_error(self): + self.bm_client_mock.instance_heartbeat.side_effect = ValueError + self.assertRaises(ValueError, self.plugin._status, Mock()) + self.assertFalse(self.plugin.brew_view_down) + + def test_status_brew_view_down(self): + self.plugin.brew_view_down = True + self.plugin._status(Mock()) + self.assertFalse(self.bm_client_mock.instance_heartbeat.called) + + def test_setup_max_concurrent(self): + self.assertEqual(1, self.plugin._setup_max_concurrent(None, None)) + self.assertEqual(5, self.plugin._setup_max_concurrent(True, None)) + self.assertEqual(1, self.plugin._setup_max_concurrent(False, None)) + self.assertEqual(4, self.plugin._setup_max_concurrent(None, 4)) + self.assertEqual(1, self.plugin._setup_max_concurrent(True, 1)) + self.assertEqual(1, self.plugin._setup_max_concurrent(False, 1)) + self.assertEqual(4, self.plugin._setup_max_concurrent(True, 4)) + self.assertEqual(4, self.plugin._setup_max_concurrent(False, 4)) + + def test_setup_system_system_and_extra_params(self): + self.assertRaises(BrewmasterValidationError, self.plugin._setup_system, self.client, 'default', self.system, + 'name', '', '', '', {}, None, None) + self.assertRaises(BrewmasterValidationError, self.plugin._setup_system, self.client, 'default', self.system, + '', 'description', '', '', {}, None, None) + self.assertRaises(BrewmasterValidationError, self.plugin._setup_system, self.client, 'default', self.system, + '', '', 'version', '', {}, None, None) + self.assertRaises(BrewmasterValidationError, self.plugin._setup_system, self.client, 'default', self.system, + '', '', '', 'icon name', {}, None, None) + self.assertRaises(BrewmasterValidationError, self.plugin._setup_system, self.client, 'default', self.system, + '', '', '', '', {}, "display_name", None) + + def test_setup_system_no_instances(self): + system = System(name='test_system', version='1.0.0') + self.assertRaises(BrewmasterValidationError, self.plugin._setup_system, self.client, 'default', system, + '', '', '', '', {}, None, None) + + def test_setup_system_no_max_instances(self): + system = System(name='test_system', version='1.0.0', instances=[Instance(name='1'), Instance(name='2')]) + new_system = self.plugin._setup_system(self.client, 'default', system, '', '', '', '', {}, None, None) + self.assertEqual(2, new_system.max_instances) + + def test_setup_system_construct(self): + new_system = self.plugin._setup_system(self.client, 'default', None, 'name', 'desc', '1.0.0', 'icon', + {'foo': 'bar'}, "display_name", None) + self.assertEqual('name', new_system.name) + self.assertEqual('desc', new_system.description) + self.assertEqual('1.0.0', new_system.version) + self.assertEqual('icon', new_system.icon_name) + self.assertEqual({'foo': 'bar'}, new_system.metadata) + self.assertEqual("display_name", new_system.display_name) + + def test_setup_system_construct_no_description(self): + self.client.__doc__ = 'Description\nSome more stuff' + new_system = self.plugin._setup_system(self.client, 'default', None, 'name', '', '1.0.0', 'icon', {}, None, None) + self.assertEqual('name', new_system.name) + self.assertEqual('Description', new_system.description) + self.assertEqual('1.0.0', new_system.version) + self.assertEqual('icon', new_system.icon_name) + self.assertIsNone(new_system.display_name) + + def test_setup_system_construct_name_version_from_env(self): + os.environ['BG_NAME'] = 'name' + os.environ['BG_VERSION'] = '1.0.0' + + new_system = self.plugin._setup_system(self.client, 'default', None, None, 'desc', None, 'icon', {'foo': 'bar'}, + "display_name", None) + self.assertEqual('name', new_system.name) + self.assertEqual('desc', new_system.description) + self.assertEqual('1.0.0', new_system.version) + self.assertEqual('icon', new_system.icon_name) + self.assertEqual({'foo': 'bar'}, new_system.metadata) + self.assertEqual("display_name", new_system.display_name) + + def test_connection_poll_already_shut_down(self): + self.plugin.shutdown_event.set() + self.plugin._connection_poll() + self.assertFalse(self.bm_client_mock.get_version.called) + + def test_connection_poll_brew_view_ok(self): + self.plugin.shutdown_event = Mock(wait=Mock(side_effect=[False, True])) + self.plugin._connection_poll() + self.assertFalse(self.bm_client_mock.get_version.called) + + def test_connection_poll_brew_view_down(self): + self.plugin.shutdown_event = Mock(wait=Mock(side_effect=[False, True])) + self.plugin.brew_view_down = True + self.bm_client_mock.get_version.side_effect = ValueError + + self.plugin._connection_poll() + self.assertTrue(self.bm_client_mock.get_version.called) + self.assertTrue(self.plugin.brew_view_down) + + def test_connection_poll_brew_view_back(self): + self.plugin.shutdown_event = Mock(wait=Mock(side_effect=[False, True])) + self.plugin.brew_view_down = True + + self.plugin._connection_poll() + self.assertTrue(self.bm_client_mock.get_version.called) + self.assertFalse(self.plugin.brew_view_down) + + def test_format_output_string(self): + output = 'output' + self.assertEqual(output, self.plugin._format_output(output)) + + def test_format_output_unicode_string(self): + output = u'output' + self.assertEqual(output, self.plugin._format_output(output)) + + def test_format_output_object(self): + output = MagicMock() + self.assertEqual(str(output), self.plugin._format_output(output)) + + @patch('brewtils.plugin.json') + def test_format_output_dict(self, json_mock): + output = {'dict': 'output'} + json_mock.dumps = Mock() + self.assertEqual(json_mock.dumps.return_value, self.plugin._format_output(output)) + json_mock.dumps.assert_called_once_with(output) + + @patch('brewtils.plugin.json') + def test_format_output_list(self, json_mock): + output = ['list', 'output'] + json_mock.dumps = Mock() + self.assertEqual(json_mock.dumps.return_value, self.plugin._format_output(output)) + json_mock.dumps.assert_called_once_with(output) + + @patch('brewtils.plugin.json') + def test_format_output_json_error(self, json_mock): + output = ['list', 'output'] + json_mock.dumps = Mock(side_effect=ValueError) + self.assertEqual(str(output), self.plugin._format_output(output)) + json_mock.dumps.assert_called_once_with(output) diff --git a/test/request_consumer_test.py b/test/request_consumer_test.py new file mode 100644 index 00000000..bf7ffd95 --- /dev/null +++ b/test/request_consumer_test.py @@ -0,0 +1,255 @@ +import unittest +from concurrent.futures import Future + +from mock import Mock, patch +from pika.exceptions import AMQPConnectionError + +from brewtils.errors import DiscardMessageException, RepublishRequestException +from brewtils.request_consumer import RequestConsumer + + +class RequestConsumerTest(unittest.TestCase): + + def setUp(self): + pika_patcher = patch('brewtils.request_consumer.pika') + self.addCleanup(pika_patcher.stop) + self.pika_patch = pika_patcher.start() + + self.callback_future = Future() + self.callback = Mock(return_value=self.callback_future) + self.panic_event = Mock() + self.consumer = RequestConsumer('url', 'queue', self.callback, self.panic_event) + + @patch('brewtils.request_consumer.RequestConsumer.open_connection') + def test_run(self, open_mock): + fake_connection = Mock() + open_mock.return_value = fake_connection + + self.consumer.run() + self.assertEqual(self.consumer._connection, fake_connection) + open_mock.assert_called_once_with() + fake_connection.ioloop.start.assert_called_once_with() + + @patch('brewtils.request_consumer.RequestConsumer.close_channel') + def test_stop(self, close_mock): + self.consumer.stop() + self.assertTrue(self.consumer.shutdown_event.is_set()) + self.assertTrue(close_mock.called) + + @patch('brewtils.request_consumer.RequestConsumer.on_message_callback_complete') + def test_on_message(self, callback_complete_mock): + fake_message = Mock() + + props = Mock(headers='headers') + self.consumer.on_message(Mock(), Mock(delivery_tag='tag'), props, fake_message) + self.callback.assert_called_with(fake_message, 'headers') + + self.callback_future.set_result(None) + self.assertTrue(callback_complete_mock.called) + + def test_on_message_discard(self): + channel_mock = Mock() + basic_deliver_mock = Mock() + self.consumer._channel = channel_mock + self.callback.side_effect = DiscardMessageException + + self.consumer.on_message(Mock(), basic_deliver_mock, Mock(), Mock()) + channel_mock.basic_nack.assert_called_once_with(basic_deliver_mock.delivery_tag, requeue=False) + + def test_on_message_unknown_exception(self): + channel_mock = Mock() + basic_deliver_mock = Mock() + self.consumer._channel = channel_mock + self.callback.side_effect = ValueError + + self.consumer.on_message(Mock(), basic_deliver_mock, Mock(), Mock()) + channel_mock.basic_nack.assert_called_once_with(basic_deliver_mock.delivery_tag, requeue=True) + + def test_on_message_callback_complete(self): + basic_deliver_mock = Mock() + channel_mock = Mock() + self.consumer._channel = channel_mock + + self.callback_future.set_result(None) + self.consumer.on_message_callback_complete(basic_deliver_mock, self.callback_future) + channel_mock.basic_ack.assert_called_once_with(basic_deliver_mock.delivery_tag) + + def test_on_message_callback_complete_error_on_ack(self): + basic_deliver_mock = Mock() + channel_mock = Mock(basic_ack=Mock(side_effect=ValueError)) + self.consumer._channel = channel_mock + + self.callback_future.set_result(None) + self.consumer.on_message_callback_complete(basic_deliver_mock, self.callback_future) + channel_mock.basic_ack.assert_called_once_with(basic_deliver_mock.delivery_tag) + self.assertTrue(self.panic_event.set.called) + + @patch('brewtils.request_consumer.SchemaParser') + def test_on_message_callback_complete_exception_republish(self, parser_mock): + basic_deliver_mock = Mock() + request_mock = Mock() + future_exception = RepublishRequestException(request_mock, {}) + + channel_mock = Mock() + self.consumer._channel = channel_mock + publish_channel_mock = Mock() + publish_connection_mock = Mock() + publish_connection_mock.channel.return_value = publish_channel_mock + self.pika_patch.BlockingConnection.return_value.__enter__.return_value = publish_connection_mock + + self.callback_future.set_exception(future_exception) + self.consumer.on_message_callback_complete(basic_deliver_mock, self.callback_future) + publish_channel_mock.basic_publish.assert_called_once_with( + exchange=basic_deliver_mock.exchange, properties=self.pika_patch.BasicProperties.return_value, + routing_key=basic_deliver_mock.routing_key, body=parser_mock.serialize_request.return_value) + parser_mock.serialize_request.assert_called_once_with(request_mock) + channel_mock.basic_ack.assert_called_once_with(basic_deliver_mock.delivery_tag) + + def test_on_message_callback_complete_exception_republish_failure(self): + self.pika_patch.BlockingConnection.side_effect = ValueError + + self.callback_future.set_exception(RepublishRequestException(Mock(), {})) + self.consumer.on_message_callback_complete(Mock(), self.callback_future) + self.assertTrue(self.panic_event.set.called) + + def test_on_message_callback_complete_exception_discard_message(self): + channel_mock = Mock() + self.consumer._channel = channel_mock + self.pika_patch.BlockingConnection.side_effect = ValueError + + self.callback_future.set_exception(DiscardMessageException()) + self.consumer.on_message_callback_complete(Mock(), self.callback_future) + self.assertFalse(self.panic_event.set.called) + self.assertTrue(channel_mock.basic_nack.called) + + def test_on_message_callback_complete_unknown_exception(self): + self.callback_future.set_exception(ValueError()) + self.consumer.on_message_callback_complete(Mock(), self.callback_future) + self.assertTrue(self.panic_event.set.called) + + def test_open_connection(self): + ret_val = self.consumer.open_connection() + self.assertEqual(self.pika_patch.SelectConnection.return_value, ret_val) + self.pika_patch.URLParameters.assert_called_with('url') + self.pika_patch.SelectConnection.assert_called_with(self.pika_patch.URLParameters.return_value, + self.consumer.on_connection_open, + stop_ioloop_on_close=False) + + def test_open_connection_shutdown_is_set(self): + self.consumer.shutdown_event.set() + self.assertIsNone(self.consumer.open_connection()) + self.assertFalse(self.pika_patch.SelectConnection.called) + + def test_open_connection_error_raised_no_retries(self): + self.pika_patch.SelectConnection.side_effect = AMQPConnectionError + self.consumer._max_connect_retries = 0 + + self.assertRaises(AMQPConnectionError, self.consumer.open_connection) + + def test_open_connection_retry(self): + self.pika_patch.SelectConnection.side_effect = [AMQPConnectionError, 'connection'] + self.assertEqual('connection', self.consumer.open_connection()) + + @patch('brewtils.request_consumer.RequestConsumer.open_channel') + def test_on_connection_open(self, open_channel_mock): + self.consumer._connection = Mock() + self.consumer.on_connection_open(Mock()) + self.consumer._connection.add_on_close_callback.assert_called_once_with(self.consumer.on_connection_closed) + open_channel_mock.assert_called_once_with() + + def test_close_connection(self): + self.consumer._connection = Mock() + self.consumer.close_connection() + self.consumer._connection.close.assert_called_with() + + def test_on_connection_closed_shutdown_set(self): + self.consumer._connection = Mock() + self.consumer._channel = "not none" + self.consumer.shutdown_event.set() + self.consumer.on_connection_closed(Mock(), 200, 'text') + self.consumer._connection.ioloop.stop.assert_called_with() + + def test_on_connection_closed_shutdown_unset(self): + self.consumer._connection = Mock() + self.consumer._channel = "not none" + self.consumer.on_connection_closed(Mock(), 200, 'text') + self.consumer._connection.add_timeout.assert_called_with(5, self.consumer.reconnect) + + def test_on_connection_closed_by_server(self): + self.consumer._connection = Mock() + self.consumer._channel = "not none" + self.consumer.on_connection_closed(Mock(), 320, 'text') + self.consumer._connection.ioloop.stop.assert_called_with() + + @patch('brewtils.request_consumer.RequestConsumer.open_connection') + def test_reconnect_shutting_down(self, open_mock): + self.consumer._connection = Mock() + self.consumer.shutdown_event.set() + self.consumer.reconnect() + self.consumer._connection.ioloop.stop.assert_called_with() + self.assertFalse(self.consumer._connection.ioloop.start.called) + self.assertFalse(open_mock.called) + + @patch('brewtils.request_consumer.RequestConsumer.open_connection') + def test_reconnect_not_shutting_down(self, open_mock): + old_connection = Mock() + self.consumer._connection = old_connection + new_connection = Mock() + open_mock.return_value = new_connection + + self.consumer.reconnect() + old_connection.ioloop.stop.assert_called_once_with() + open_mock.assert_called_once_with() + new_connection.ioloop.start.assert_called_once_with() + + def test_open_channel(self): + self.consumer._connection = Mock() + self.consumer.open_channel() + self.consumer._connection.channel.assert_called_with(on_open_callback=self.consumer.on_channel_open) + + @patch('brewtils.request_consumer.RequestConsumer.start_consuming') + def test_on_channel_open(self, start_consuming_mock): + self.consumer.add_on_channel_close_callback = Mock() + fake_channel = Mock() + + self.consumer.on_channel_open(fake_channel) + self.assertEqual(self.consumer._channel, fake_channel) + fake_channel.add_on_close_callback.assert_called_with(self.consumer.on_channel_closed) + start_consuming_mock.assert_called_once_with() + + def test_close_channel(self): + self.consumer._channel = Mock() + self.consumer.close_channel() + self.consumer._channel.close.assert_called_with() + + def test_on_channel_closed(self): + self.consumer._connection = Mock() + self.consumer.on_channel_closed(1, 200, 'text') + self.consumer._connection.close.assert_called_with() + + def test_start_consuming(self): + self.consumer._channel = Mock() + self.consumer._channel.basic_consume = Mock(return_value='consumer_tag') + + self.consumer.start_consuming() + self.consumer._channel.add_on_cancel_callback.assert_called_with(self.consumer.on_consumer_cancelled) + self.consumer._channel.basic_qos.assert_called_with(prefetch_count=1) + self.consumer._channel.basic_consume.assert_called_with(self.consumer.on_message, + queue=self.consumer._queue_name) + self.assertEqual(self.consumer._consumer_tag, 'consumer_tag') + + def test_stop_consuming(self): + self.consumer._channel = Mock() + self.consumer.stop_consuming() + self.consumer._channel.basic_cancel.assert_called_with(self.consumer.on_cancelok, self.consumer._consumer_tag) + + @patch('brewtils.request_consumer.RequestConsumer.close_channel') + def test_on_consumer_cancelled(self, close_channel_mock): + self.consumer._channel = Mock() + self.consumer.on_consumer_cancelled('frame') + self.assertTrue(close_channel_mock.called) + + @patch('brewtils.request_consumer.RequestConsumer.close_channel') + def test_on_cancelok(self, close_channel_mock): + self.consumer.on_cancelok('frame') + self.assertFalse(close_channel_mock.called) diff --git a/test/rest/__init__.py b/test/rest/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/rest/client_test.py b/test/rest/client_test.py new file mode 100644 index 00000000..51732e79 --- /dev/null +++ b/test/rest/client_test.py @@ -0,0 +1,167 @@ +import unittest +import warnings + +from mock import patch, Mock + +from brewtils.rest.client import RestClient, BrewmasterRestClient +import brewtils.rest + + +class RestClientTest(unittest.TestCase): + + def setUp(self): + self.session_mock = Mock() + + self.url_prefix = "beer" + self.url_prefix = brewtils.rest.normalize_url_prefix(self.url_prefix) + + self.client_version_1 = RestClient('host', 80, api_version=1, url_prefix=self.url_prefix) + self.client_version_1.session = self.session_mock + + def test_non_versioned_uris(self): + client = RestClient('host', 80, url_prefix=self.url_prefix) + self.assertEqual(client.version_url, 'http://host:80' + self.url_prefix + 'version') + self.assertEqual(client.config_url, 'http://host:80' + self.url_prefix + 'config') + + def test_version_1_uris(self): + ssl = RestClient('host', 80, ssl_enabled=True, api_version=1, url_prefix=self.url_prefix) + non_ssl = RestClient('host', 80, ssl_enabled=False, api_version=1, url_prefix=self.url_prefix) + + self.assertEqual(ssl.system_url, 'https://host:80' + self.url_prefix + 'api/v1/systems/') + self.assertEqual(ssl.instance_url, 'https://host:80' + self.url_prefix + 'api/v1/instances/') + self.assertEqual(ssl.command_url, 'https://host:80' + self.url_prefix + 'api/v1/commands/') + self.assertEqual(ssl.request_url, 'https://host:80' + self.url_prefix + 'api/v1/requests/') + self.assertEqual(ssl.queue_url, 'https://host:80' + self.url_prefix + 'api/v1/queues/') + self.assertEqual(ssl.logging_config_url, 'https://host:80' + self.url_prefix + 'api/v1/config/logging/') + self.assertEqual(non_ssl.system_url, 'http://host:80' + self.url_prefix + 'api/v1/systems/') + self.assertEqual(non_ssl.instance_url, 'http://host:80' + self.url_prefix + 'api/v1/instances/') + self.assertEqual(non_ssl.command_url, 'http://host:80' + self.url_prefix + 'api/v1/commands/') + self.assertEqual(non_ssl.request_url, 'http://host:80' + self.url_prefix + 'api/v1/requests/') + self.assertEqual(non_ssl.queue_url, 'http://host:80' + self.url_prefix + 'api/v1/queues/') + self.assertEqual(ssl.logging_config_url, 'https://host:80' + self.url_prefix + 'api/v1/config/logging/') + + def test_init_invalid_api_version(self): + self.assertRaises(ValueError, RestClient, 'host', 80, api_version=-1) + + def test_get_version_1(self): + self.client_version_1.get_version(key='value') + self.session_mock.get.assert_called_with(self.client_version_1.version_url, params={'key': 'value'}) + + def test_get_logging_config_1(self): + self.client_version_1.get_logging_config(system_name="system_name") + self.session_mock.get.assert_called_with(self.client_version_1.logging_config_url, + params={"system_name": "system_name"}) + + def test_get_systems_1(self): + self.client_version_1.get_systems(key='value') + self.session_mock.get.assert_called_with(self.client_version_1.system_url, params={'key': 'value'}) + + def test_get_system_1(self): + self.client_version_1.get_system('id') + self.session_mock.get.assert_called_with(self.client_version_1.system_url + 'id', params={}) + + def test_get_system_2(self): + self.client_version_1.get_system('id', key="value") + self.session_mock.get.assert_called_with(self.client_version_1.system_url + 'id', params={"key": "value"}) + + def test_post_systems_1(self): + self.client_version_1.post_systems(payload='payload') + self.session_mock.post.assert_called_with(self.client_version_1.system_url, data='payload', + headers=self.client_version_1.JSON_HEADERS) + + def test_patch_system(self): + self.client_version_1.patch_system('id', payload='payload') + self.session_mock.patch.assert_called_with(self.client_version_1.system_url + 'id', data='payload', + headers=self.client_version_1.JSON_HEADERS) + + def test_delete_system_1(self): + self.client_version_1.delete_system('id') + self.session_mock.delete.assert_called_with(self.client_version_1.system_url + 'id') + + def test_patch_instance_1(self): + self.client_version_1.patch_instance('id', payload='payload') + self.session_mock.patch.assert_called_with(self.client_version_1.instance_url + 'id', data='payload', + headers=self.client_version_1.JSON_HEADERS) + + def test_get_commands_1(self): + self.client_version_1.get_commands() + self.session_mock.get.assert_called_with(self.client_version_1.command_url) + + def test_get_command_1(self): + self.client_version_1.get_command(command_id='id') + self.session_mock.get.assert_called_with(self.client_version_1.command_url + 'id') + + def test_get_requests(self,): + self.client_version_1.get_requests(key='value') + self.session_mock.get.assert_called_with(self.client_version_1.request_url, params={'key': 'value'}) + + def test_get_request(self): + self.client_version_1.get_request(request_id='id') + self.session_mock.get.assert_called_with(self.client_version_1.request_url + 'id') + + def test_post_requests(self): + self.client_version_1.post_requests(payload='payload') + self.session_mock.post.assert_called_with(self.client_version_1.request_url, data='payload', + headers=self.client_version_1.JSON_HEADERS) + + def test_patch_request(self): + self.client_version_1.patch_request('id', payload='payload') + self.session_mock.patch.assert_called_with(self.client_version_1.request_url + 'id', data='payload', + headers=self.client_version_1.JSON_HEADERS) + + def test_post_event(self): + self.client_version_1.post_event(payload='payload') + self.session_mock.post.assert_called_with(self.client_version_1.event_url, data='payload', + headers=self.client_version_1.JSON_HEADERS, params=None) + + def test_post_event_specific_publisher(self): + self.client_version_1.post_event(payload='payload', publishers=['pika']) + self.session_mock.post.assert_called_with(self.client_version_1.event_url, data='payload', + headers=self.client_version_1.JSON_HEADERS, + params={'publisher': ['pika']}) + + def test_get_queues(self): + self.client_version_1.get_queues() + self.session_mock.get.assert_called_with(self.client_version_1.queue_url) + + def test_delete_queues(self): + self.client_version_1.delete_queues() + self.session_mock.delete.assert_called_with(self.client_version_1.queue_url) + + def test_delete_queue(self): + self.client_version_1.delete_queue('queue_name') + self.session_mock.delete.assert_called_with(self.client_version_1.queue_url + 'queue_name') + + def test_session_client_cert(self): + self.client_version_1 = RestClient('host', 80, api_version=1, client_cert='/path/to/cert') + self.assertEqual(self.client_version_1.session.cert, '/path/to/cert') + + def test_session_ca_cert(self): + self.client_version_1 = RestClient('host', 80, api_version=1, ca_cert='/path/to/ca/cert') + self.assertEqual(self.client_version_1.session.verify, '/path/to/ca/cert') + + def test_session_no_ca_cert(self): + self.client_version_1 = RestClient('host', 80, api_version=1) + self.assertTrue(self.client_version_1.session.verify) + + @patch('brewtils.rest.client.urllib3') + def test_session_no_ca_verify(self, urllib_mock): + self.client_version_1 = RestClient('host', 80, api_version=1, ca_verify=False) + self.assertFalse(self.client_version_1.session.verify) + self.assertTrue(urllib_mock.disable_warnings.called) + + +class BrewmasterRestClientTest(unittest.TestCase): + + def test_deprecation(self): + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter('always') + + BrewmasterRestClient('host', 'port') + self.assertEqual(1, len(w)) + + warning = w[0] + self.assertEqual(warning.category, DeprecationWarning) + self.assertIn("'BrewmasterRestClient'", str(warning)) + self.assertIn("'RestClient'", str(warning)) + self.assertIn('3.0', str(warning)) diff --git a/test/rest/easy_client_test.py b/test/rest/easy_client_test.py new file mode 100644 index 00000000..4842d0be --- /dev/null +++ b/test/rest/easy_client_test.py @@ -0,0 +1,565 @@ +import unittest +import warnings + +from mock import ANY, Mock, patch + +from brewtils.errors import BrewmasterFetchError, BrewmasterValidationError, BrewmasterSaveError, \ + BrewmasterDeleteError, BrewmasterConnectionError, BGNotFoundError, BGConflictError, BrewmasterRestError +from brewtils.models import System +from brewtils.rest.easy_client import EasyClient, BrewmasterEasyClient + + +class EasyClientTest(unittest.TestCase): + + def setUp(self): + self.parser = Mock() + self.client = EasyClient(host='localhost', port='3000', api_version=1, parser=self.parser) + self.fake_success_response = Mock(ok=True, status_code=200, json=Mock(return_value='payload')) + self.fake_server_error_response = Mock(ok=False, status_code=500, json=Mock(return_value='payload')) + self.fake_connection_error_response = Mock(ok=False, status_code=503, json=Mock(return_value='payload')) + self.fake_client_error_response = Mock(ok=False, status_code=400, json=Mock(return_value='payload')) + self.fake_not_found_error_response = Mock(ok=False, status_code=404, json=Mock(return_value='payload')) + self.fake_conflict_error_response = Mock(ok=False, status_code=409, json=Mock(return_value='payload')) + + @patch('brewtils.rest.client.RestClient.get_version') + def test_get_version(self, mock_get): + mock_get.return_value = self.fake_success_response + + self.assertEqual(self.fake_success_response, self.client.get_version()) + mock_get.assert_called() + + @patch('brewtils.rest.client.RestClient.get_version') + def test_get_version_error(self, mock_get): + mock_get.return_value = self.fake_server_error_response + + self.assertRaises(BrewmasterFetchError, self.client.get_version) + mock_get.assert_called() + + @patch('brewtils.rest.client.RestClient.get_version') + def test_get_version_connection_error(self, request_mock): + request_mock.return_value = self.fake_connection_error_response + self.assertRaises(BrewmasterConnectionError, self.client.get_version) + + @patch('brewtils.rest.client.RestClient.get_logging_config') + def test_get_logging_config(self, mock_get): + mock_get.return_value = self.fake_success_response + self.parser.parse_logging_config = Mock(return_value='logging_config') + + self.assertEqual('logging_config', self.client.get_logging_config('system_name')) + self.parser.parse_logging_config.assert_called_with('payload') + mock_get.assert_called() + + @patch('brewtils.rest.client.RestClient.get_logging_config') + def test_get_logging_config_connection_error(self, request_mock): + request_mock.return_value = self.fake_connection_error_response + self.assertRaises(BrewmasterConnectionError, self.client.get_logging_config, 'system_name') + + # Find systems + @patch('brewtils.rest.client.RestClient.get_systems') + def test_find_systems_call_get_systems(self, mock_get): + mock_get.return_value = self.fake_success_response + self.client.find_systems() + mock_get.assert_called() + + @patch('brewtils.rest.client.RestClient.get_systems') + def test_find_systems_with_params_get_systems(self, mock_get): + mock_get.return_value = self.fake_success_response + self.client.find_systems(name='foo') + mock_get.assert_called_with(name='foo') + + @patch('brewtils.rest.client.RestClient.get_systems') + def test_find_systems_server_error(self, mock_get): + mock_get.return_value = self.fake_server_error_response + self.assertRaises(BrewmasterFetchError, self.client.find_systems) + + @patch('brewtils.rest.client.RestClient.get_systems') + def test_find_systems_connection_error(self, request_mock): + request_mock.return_value = self.fake_connection_error_response + self.assertRaises(BrewmasterConnectionError, self.client.find_systems) + + @patch('brewtils.rest.client.RestClient.get_systems') + def test_find_systems_call_parser(self, mock_get): + mock_get.return_value = self.fake_success_response + self.client.find_systems() + self.parser.parse_system.assert_called_with('payload', many=True) + + @patch('brewtils.rest.easy_client.EasyClient._find_system_by_id') + def test_find_unique_system_by_id(self, find_mock): + system_mock = Mock() + find_mock.return_value = system_mock + + self.assertEqual(system_mock, self.client.find_unique_system(id='id')) + find_mock.assert_called_with('id') + + def test_find_unique_system_none(self): + self.client.find_systems = Mock(return_value=None) + self.assertIsNone(self.client.find_unique_system()) + + def test_find_unique_system_one(self): + self.client.find_systems = Mock(return_value=['system1']) + self.assertEqual('system1', self.client.find_unique_system()) + + def test_find_unique_system_multiple(self): + self.client.find_systems = Mock(return_value=['system1', 'system2']) + self.assertRaises(BrewmasterFetchError, self.client.find_unique_system) + + @patch('brewtils.rest.client.RestClient.get_system') + def test_find_system_by_id(self, mock_get): + mock_get.return_value = self.fake_success_response + self.parser.parse_system = Mock(return_value='system') + + self.assertEqual(self.client._find_system_by_id('id', foo='bar'), 'system') + self.parser.parse_system.assert_called_with('payload') + mock_get.assert_called_with('id', foo='bar') + + @patch('brewtils.rest.client.RestClient.get_system') + def test_find_system_by_id_404(self, mock_get): + mock_get.return_value = self.fake_not_found_error_response + + self.assertIsNone(self.client._find_system_by_id('id', foo='bar')) + mock_get.assert_called_with('id', foo='bar') + + @patch('brewtils.rest.client.RestClient.get_system') + def test_find_system_by_id_server_error(self, mock_get): + mock_get.return_value = self.fake_server_error_response + + self.assertRaises(BrewmasterFetchError, self.client._find_system_by_id, 'id') + mock_get.assert_called_with('id') + + @patch('brewtils.rest.client.RestClient.get_system') + def test_find_system_by_id_connection_error(self, request_mock): + request_mock.return_value = self.fake_connection_error_response + self.assertRaises(BrewmasterConnectionError, self.client._find_system_by_id, 'id') + + # Create system + @patch('brewtils.rest.client.RestClient.post_systems') + def test_create_system(self, mock_post): + mock_post.return_value = self.fake_success_response + self.parser.serialize_system = Mock(return_value='json_system') + self.parser.parse_system = Mock(return_value='system_response') + + self.assertEqual('system_response', self.client.create_system('system')) + self.parser.serialize_system.assert_called_with('system') + self.parser.parse_system.assert_called_with('payload') + + @patch('brewtils.rest.client.RestClient.post_systems') + def test_create_system_client_error(self, mock_post): + mock_post.return_value = self.fake_client_error_response + self.assertRaises(BrewmasterValidationError, self.client.create_system, 'system') + + @patch('brewtils.rest.client.RestClient.post_systems') + def test_create_system_server_error(self, mock_post): + mock_post.return_value = self.fake_server_error_response + self.assertRaises(BrewmasterSaveError, self.client.create_system, 'system') + + @patch('brewtils.rest.client.RestClient.post_systems') + def test_create_system_connection_error(self, request_mock): + request_mock.return_value = self.fake_connection_error_response + self.assertRaises(BrewmasterConnectionError, self.client.create_system, 'system') + + # Update request + @patch('brewtils.rest.client.RestClient.patch_system') + def test_update_system(self, mock_patch): + mock_patch.return_value = self.fake_success_response + self.parser.serialize_command = Mock(return_value='new_commands') + + self.client.update_system('id', new_commands='new_commands') + self.parser.parse_system.assert_called_with('payload') + self.assertEqual(1, mock_patch.call_count) + payload = mock_patch.call_args[0][1] + self.assertNotEqual(-1, payload.find('new_commands')) + + @patch('brewtils.rest.easy_client.PatchOperation') + @patch('brewtils.rest.client.RestClient.patch_system') + def test_update_system_metadata(self, mock_patch, MockPatch): + MockPatch.return_value = "patch" + mock_patch.return_value = self.fake_success_response + metadata = {"foo": "bar"} + + self.client.update_system('id', new_commands=None, metadata=metadata) + MockPatch.assert_called_with('update', '/metadata', {"foo": "bar"}) + self.parser.serialize_patch.assert_called_with(["patch"], many=True) + self.parser.parse_system.assert_called_with('payload') + + @patch('brewtils.rest.easy_client.PatchOperation') + @patch('brewtils.rest.client.RestClient.patch_system') + def test_update_system_kwargs(self, mock_patch, MockPatch): + MockPatch.return_value = "patch" + mock_patch.return_value = self.fake_success_response + + self.client.update_system('id', new_commands=None, display_name="foo") + MockPatch.assert_called_with('replace', '/display_name', "foo") + self.parser.serialize_patch.assert_called_with(["patch"], many=True) + self.parser.parse_system.assert_called_with('payload') + + @patch('brewtils.rest.client.RestClient.patch_system') + def test_update_system_client_error(self, mock_patch): + mock_patch.return_value = self.fake_client_error_response + + self.assertRaises(BrewmasterValidationError, self.client.update_system, 'id') + mock_patch.assert_called_once_with('id', ANY) + + @patch('brewtils.rest.client.RestClient.patch_system') + def test_update_system_invalid_id(self, mock_patch): + mock_patch.return_value = self.fake_not_found_error_response + + self.assertRaises(BGNotFoundError, self.client.update_system, 'id') + mock_patch.assert_called_once_with('id', ANY) + + @patch('brewtils.rest.client.RestClient.patch_system') + def test_update_system_conflict(self, mock_patch): + mock_patch.return_value = self.fake_conflict_error_response + + self.assertRaises(BGConflictError, self.client.update_system, 'id') + mock_patch.assert_called_once_with('id', ANY) + + @patch('brewtils.rest.client.RestClient.patch_system') + def test_update_system_server_error(self, mock_patch): + mock_patch.return_value = self.fake_server_error_response + + self.assertRaises(BrewmasterSaveError, self.client.update_system, 'id') + mock_patch.assert_called_once_with('id', ANY) + + @patch('brewtils.rest.client.RestClient.patch_system') + def test_update_system_connection_error(self, request_mock): + request_mock.return_value = self.fake_connection_error_response + self.assertRaises(BrewmasterConnectionError, self.client.update_system, 'system') + + # Remove system + @patch('brewtils.rest.easy_client.EasyClient._remove_system_by_id') + @patch('brewtils.rest.easy_client.EasyClient.find_unique_system') + def test_remove_system(self, find_mock, remove_mock): + find_mock.return_value = System(id='id') + remove_mock.return_value = 'delete_response' + + self.assertEqual('delete_response', self.client.remove_system(search='search params')) + find_mock.assert_called_once_with(search='search params') + remove_mock.assert_called_once_with('id') + + @patch('brewtils.rest.easy_client.EasyClient._remove_system_by_id') + @patch('brewtils.rest.easy_client.EasyClient.find_unique_system') + def test_remove_system_none_found(self, find_mock, remove_mock): + find_mock.return_value = None + + self.assertRaises(BrewmasterFetchError, self.client.remove_system, search='search params') + self.assertFalse(remove_mock.called) + find_mock.assert_called_once_with(search='search params') + + @patch('brewtils.rest.client.RestClient.delete_system') + def test_remove_system_by_id(self, mock_delete): + mock_delete.return_value = self.fake_success_response + + self.assertTrue(self.client._remove_system_by_id('foo')) + mock_delete.assert_called_with('foo') + + @patch('brewtils.rest.client.RestClient.delete_system') + def test_remove_system_by_id_client_error(self, mock_remove): + mock_remove.return_value = self.fake_client_error_response + self.assertRaises(BrewmasterValidationError, self.client._remove_system_by_id, 'foo') + + @patch('brewtils.rest.client.RestClient.delete_system') + def test_remove_system_by_id_server_error(self, mock_remove): + mock_remove.return_value = self.fake_server_error_response + self.assertRaises(BrewmasterDeleteError, self.client._remove_system_by_id, 'foo') + + @patch('brewtils.rest.client.RestClient.delete_system') + def test_remove_system_by_id_connection_error(self, request_mock): + request_mock.return_value = self.fake_connection_error_response + self.assertRaises(BrewmasterConnectionError, self.client._remove_system_by_id, 'foo') + + def test_remove_system_by_id_none(self): + self.assertRaises(BrewmasterDeleteError, self.client._remove_system_by_id, None) + + # Initialize instance + @patch('brewtils.rest.client.RestClient.patch_instance') + def test_initialize_instance(self, request_mock): + request_mock.return_value = self.fake_success_response + + self.client.initialize_instance('id') + self.assertTrue(self.parser.parse_instance.called) + request_mock.assert_called_once_with('id', ANY) + + @patch('brewtils.rest.client.RestClient.patch_instance') + def test_initialize_instance_client_error(self, request_mock): + request_mock.return_value = self.fake_client_error_response + + self.assertRaises(BrewmasterValidationError, self.client.initialize_instance, 'id') + self.assertFalse(self.parser.parse_instance.called) + request_mock.assert_called_once_with('id', ANY) + + @patch('brewtils.rest.client.RestClient.patch_instance') + def test_initialize_instance_server_error(self, request_mock): + request_mock.return_value = self.fake_server_error_response + + self.assertRaises(BrewmasterSaveError, self.client.initialize_instance, 'id') + self.assertFalse(self.parser.parse_instance.called) + request_mock.assert_called_once_with('id', ANY) + + @patch('brewtils.rest.client.RestClient.patch_instance') + def test_initialize_instance_connection_error(self, request_mock): + request_mock.return_value = self.fake_connection_error_response + self.assertRaises(BrewmasterConnectionError, self.client.initialize_instance, 'id') + + @patch('brewtils.rest.client.RestClient.patch_instance') + def test_update_instance_status(self, request_mock): + request_mock.return_value = self.fake_success_response + + self.client.update_instance_status('id', 'status') + self.assertTrue(self.parser.parse_instance.called) + request_mock.assert_called_once_with('id', ANY) + + @patch('brewtils.rest.client.RestClient.patch_instance') + def test_update_instance_status_client_error(self, request_mock): + request_mock.return_value = self.fake_client_error_response + + self.assertRaises(BrewmasterValidationError, self.client.update_instance_status, 'id', 'status') + self.assertFalse(self.parser.parse_instance.called) + request_mock.assert_called_once_with('id', ANY) + + @patch('brewtils.rest.client.RestClient.patch_instance') + def test_update_instance_status_server_error(self, request_mock): + request_mock.return_value = self.fake_server_error_response + + self.assertRaises(BrewmasterSaveError, self.client.update_instance_status, 'id', 'status') + self.assertFalse(self.parser.parse_instance.called) + request_mock.assert_called_once_with('id', ANY) + + @patch('brewtils.rest.client.RestClient.patch_instance') + def test_update_instance_connection_error(self, request_mock): + request_mock.return_value = self.fake_connection_error_response + self.assertRaises(BrewmasterConnectionError, self.client.update_instance_status, 'id', 'status') + + # Instance heartbeat + @patch('brewtils.rest.client.RestClient.patch_instance') + def test_instance_heartbeat(self, request_mock): + request_mock.return_value = self.fake_success_response + + self.assertTrue(self.client.instance_heartbeat('id')) + request_mock.assert_called_once_with('id', ANY) + + @patch('brewtils.rest.client.RestClient.patch_instance') + def test_instance_heartbeat_client_error(self, request_mock): + request_mock.return_value = self.fake_client_error_response + + self.assertRaises(BrewmasterValidationError, self.client.instance_heartbeat, 'id') + request_mock.assert_called_once_with('id', ANY) + + @patch('brewtils.rest.client.RestClient.patch_instance') + def test_instance_heartbeat_server_error(self, request_mock): + request_mock.return_value = self.fake_server_error_response + + self.assertRaises(BrewmasterSaveError, self.client.instance_heartbeat, 'id') + request_mock.assert_called_once_with('id', ANY) + + @patch('brewtils.rest.client.RestClient.patch_instance') + def test_instance_heartbeat_connection_error(self, request_mock): + request_mock.return_value = self.fake_connection_error_response + self.assertRaises(BrewmasterConnectionError, self.client.instance_heartbeat, 'id') + + # Find requests + @patch('brewtils.rest.easy_client.EasyClient._find_request_by_id') + def test_find_unique_request_by_id(self, find_mock): + self.client.find_unique_request(id='id') + find_mock.assert_called_with('id') + + def test_find_unique_request_none(self): + self.client.find_requests = Mock(return_value=None) + self.assertIsNone(self.client.find_unique_request()) + + def test_find_unique_request_one(self): + self.client.find_requests = Mock(return_value=['request1']) + self.assertEqual('request1', self.client.find_unique_request()) + + def test_find_unique_request_multiple(self): + self.client.find_requests = Mock(return_value=['request1', 'request2']) + self.assertRaises(BrewmasterFetchError, self.client.find_unique_request) + + @patch('brewtils.rest.client.RestClient.get_requests') + def test_find_requests(self, mock_get): + mock_get.return_value = self.fake_success_response + self.parser.parse_request = Mock(return_value='request') + + self.assertEqual('request', self.client.find_requests(search='params')) + self.parser.parse_request.assert_called_with('payload', many=True) + mock_get.assert_called_with(search='params') + + @patch('brewtils.rest.client.RestClient.get_requests') + def test_find_requests_error(self, mock_get): + mock_get.return_value = self.fake_server_error_response + + self.assertRaises(BrewmasterFetchError, self.client.find_requests, search='params') + mock_get.assert_called_with(search='params') + + @patch('brewtils.rest.client.RestClient.get_requests') + def test_find_requests_connection_error(self, request_mock): + request_mock.return_value = self.fake_connection_error_response + self.assertRaises(BrewmasterConnectionError, self.client.find_requests, search='params') + + @patch('brewtils.rest.client.RestClient.get_request') + def test_find_request_by_id(self, mock_get): + mock_get.return_value = self.fake_success_response + self.parser.parse_request = Mock(return_value='request') + + self.assertEqual(self.client._find_request_by_id('id'), 'request') + self.parser.parse_request.assert_called_with('payload') + mock_get.assert_called_with('id') + + @patch('brewtils.rest.client.RestClient.get_request') + def test_find_request_by_id_404(self, mock_get): + mock_get.return_value = self.fake_not_found_error_response + + self.assertIsNone(self.client._find_request_by_id('id')) + mock_get.assert_called_with('id') + + @patch('brewtils.rest.client.RestClient.get_request') + def test_find_request_by_id_server_error(self, mock_get): + mock_get.return_value = self.fake_server_error_response + + self.assertRaises(BrewmasterFetchError, self.client._find_request_by_id, 'id') + mock_get.assert_called_with('id') + + @patch('brewtils.rest.client.RestClient.get_request') + def test_find_request_by_id_connection_error(self, request_mock): + request_mock.return_value = self.fake_connection_error_response + self.assertRaises(BrewmasterConnectionError, self.client._find_request_by_id, 'id') + + # Create request + @patch('brewtils.rest.client.RestClient.post_requests') + def test_create_request(self, mock_post): + mock_post.return_value = self.fake_success_response + self.parser.serialize_request = Mock(return_value='json_request') + self.parser.parse_request = Mock(return_value='request_response') + + self.assertEqual('request_response', self.client.create_request('request')) + self.parser.serialize_request.assert_called_with('request') + self.parser.parse_request.assert_called_with('payload') + + @patch('brewtils.rest.client.RestClient.post_requests') + def test_create_request_client_error(self, mock_post): + mock_post.return_value = self.fake_client_error_response + self.assertRaises(BrewmasterValidationError, self.client.create_request, 'request') + + @patch('brewtils.rest.client.RestClient.post_requests') + def test_create_request_server_error(self, mock_post): + mock_post.return_value = self.fake_server_error_response + self.assertRaises(BrewmasterSaveError, self.client.create_request, 'request') + + @patch('brewtils.rest.client.RestClient.post_requests') + def test_create_request_connection_error(self, request_mock): + request_mock.return_value = self.fake_connection_error_response + self.assertRaises(BrewmasterConnectionError, self.client.create_request, 'request') + + # Update request + @patch('brewtils.rest.client.RestClient.patch_request') + def test_update_request(self, request_mock): + request_mock.return_value = self.fake_success_response + + self.client.update_request('id', status='new_status', output='new_output', error_class='ValueError') + self.parser.parse_request.assert_called_with('payload') + self.assertEqual(1, request_mock.call_count) + payload = request_mock.call_args[0][1] + self.assertNotEqual(-1, payload.find('new_status')) + self.assertNotEqual(-1, payload.find('new_output')) + self.assertNotEqual(-1, payload.find('ValueError')) + + @patch('brewtils.rest.client.RestClient.patch_request') + def test_update_request_client_error(self, request_mock): + request_mock.return_value = self.fake_client_error_response + + self.assertRaises(BrewmasterValidationError, self.client.update_request, 'id') + request_mock.assert_called_once_with('id', ANY) + + @patch('brewtils.rest.client.RestClient.patch_request') + def test_update_request_server_error(self, request_mock): + request_mock.return_value = self.fake_server_error_response + + self.assertRaises(BrewmasterSaveError, self.client.update_request, 'id') + request_mock.assert_called_once_with('id', ANY) + + @patch('brewtils.rest.client.RestClient.patch_request') + def test_update_request_connection_error(self, request_mock): + request_mock.return_value = self.fake_connection_error_response + self.assertRaises(BrewmasterConnectionError, self.client.update_request, 'id') + + # Publish Event + @patch('brewtils.rest.client.RestClient.post_event') + def test_publish_event(self, mock_post): + mock_post.return_value = self.fake_success_response + self.assertTrue(self.client.publish_event(Mock())) + + @patch('brewtils.rest.client.RestClient.post_event') + def test_publish_event_errors(self, mock_post): + mock_post.return_value = self.fake_client_error_response + self.assertRaises(BrewmasterValidationError, self.client.publish_event, 'system') + + mock_post.return_value = self.fake_server_error_response + self.assertRaises(BrewmasterRestError, self.client.publish_event, 'system') + + mock_post.return_value = self.fake_connection_error_response + self.assertRaises(BrewmasterConnectionError, self.client.publish_event, 'system') + + # Queues + @patch('brewtils.rest.client.RestClient.get_queues') + def test_get_queues(self, mock_get): + mock_get.return_value = self.fake_success_response + self.client.get_queues() + self.assertTrue(self.parser.parse_queue.called) + + @patch('brewtils.rest.client.RestClient.get_queues') + def test_get_queues_errors(self, mock_get): + mock_get.return_value = self.fake_client_error_response + self.assertRaises(BrewmasterValidationError, self.client.get_queues) + + mock_get.return_value = self.fake_server_error_response + self.assertRaises(BrewmasterRestError, self.client.get_queues) + + mock_get.return_value = self.fake_connection_error_response + self.assertRaises(BrewmasterConnectionError, self.client.get_queues) + + @patch('brewtils.rest.client.RestClient.delete_queue') + def test_clear_queue(self, mock_delete): + mock_delete.return_value = self.fake_success_response + self.assertTrue(self.client.clear_queue('queue')) + + @patch('brewtils.rest.client.RestClient.delete_queue') + def test_clear_queue_errors(self, mock_delete): + mock_delete.return_value = self.fake_client_error_response + self.assertRaises(BrewmasterValidationError, self.client.clear_queue, 'queue') + + mock_delete.return_value = self.fake_server_error_response + self.assertRaises(BrewmasterRestError, self.client.clear_queue, 'queue') + + mock_delete.return_value = self.fake_connection_error_response + self.assertRaises(BrewmasterConnectionError, self.client.clear_queue, 'queue') + + @patch('brewtils.rest.client.RestClient.delete_queues') + def test_clear_all_queues(self, mock_delete): + mock_delete.return_value = self.fake_success_response + self.assertTrue(self.client.clear_all_queues()) + + @patch('brewtils.rest.client.RestClient.delete_queues') + def test_clear_all_queues_errors(self, mock_delete): + mock_delete.return_value = self.fake_client_error_response + self.assertRaises(BrewmasterValidationError, self.client.clear_all_queues) + + mock_delete.return_value = self.fake_server_error_response + self.assertRaises(BrewmasterRestError, self.client.clear_all_queues) + + mock_delete.return_value = self.fake_connection_error_response + self.assertRaises(BrewmasterConnectionError, self.client.clear_all_queues) + + +class BrewmasterEasyClientTest(unittest.TestCase): + + def test_deprecation(self): + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter('always') + + BrewmasterEasyClient('host', 'port') + self.assertEqual(1, len(w)) + + warning = w[0] + self.assertEqual(warning.category, DeprecationWarning) + self.assertIn("'BrewmasterEasyClient'", str(warning)) + self.assertIn("'EasyClient'", str(warning)) + self.assertIn('3.0', str(warning)) diff --git a/test/rest/rest_test.py b/test/rest/rest_test.py new file mode 100644 index 00000000..33b950ac --- /dev/null +++ b/test/rest/rest_test.py @@ -0,0 +1,24 @@ +import unittest + +from brewtils.rest import normalize_url_prefix + + +class RestTest(unittest.TestCase): + + def test_normalize_url_prefix(self): + simple_prefixes = [None, '', '/'] + example_prefixes = ['example', '/example', 'example/', '/example/'] + example_prefixes_2 = ['beer/garden', '/beer/garden', '/beer/garden/'] + example_prefix_chars = ['+-?.,', '/+-?.,', '+-?.,/', '/+-?.,/'] + + for i in simple_prefixes: + self.assertEquals("/", normalize_url_prefix(i)) + + for i in example_prefixes: + self.assertEquals("/example/", normalize_url_prefix(i)) + + for i in example_prefixes_2: + self.assertEquals('/beer/garden/', normalize_url_prefix(i)) + + for i in example_prefix_chars: + self.assertEquals('/+-?.,/', normalize_url_prefix(i)) \ No newline at end of file diff --git a/test/rest/system_client_test.py b/test/rest/system_client_test.py new file mode 100644 index 00000000..88ecc456 --- /dev/null +++ b/test/rest/system_client_test.py @@ -0,0 +1,259 @@ +import unittest +import warnings +from concurrent.futures import wait + +from mock import call, patch, Mock, PropertyMock + +from brewtils.errors import BrewmasterTimeoutError, BrewmasterFetchError, BrewmasterValidationError +from brewtils.rest.system_client import BrewmasterSystemClient, SystemClient + + +class SystemClientTest(unittest.TestCase): + + def setUp(self): + self.fake_command_1 = Mock() + self.fake_command_2 = Mock() + type(self.fake_command_1).name = PropertyMock(return_value='command_1') + type(self.fake_command_2).name = PropertyMock(return_value='command_2') + + self.fake_system = Mock(version='1.0.0', commands=[self.fake_command_1, self.fake_command_2], + instance_names=[u'default']) + type(self.fake_system).name = PropertyMock(return_value='system') + + self.mock_in_progress = Mock(status='IN PROGRESS', output='output') + self.mock_success = Mock(status='SUCCESS', output='output') + + easy_client_patcher = patch('brewtils.rest.system_client.EasyClient') + self.addCleanup(easy_client_patcher.stop) + easy_client_patcher_mock = easy_client_patcher.start() + + self.easy_client_mock = Mock(name='easy_client') + easy_client_patcher_mock.return_value = self.easy_client_mock + self.easy_client_mock.find_unique_system.return_value = self.fake_system + self.easy_client_mock.find_systems.return_value = [self.fake_system] + + self.client = SystemClient('localhost', 3000, 'system') + + def test_lazy_system_loading(self): + self.assertFalse(self.client._loaded) + self.assertIsNone(self.client._system) + + send_mock = Mock() + self.client.send_bg_request = send_mock + + self.client.command_1() + self.assertTrue(self.client._loaded) + self.assertIsNotNone(self.client._system) + self.assertIsNotNone(self.client._commands) + self.assertTrue(send_mock.called) + + def test_no_attribute(self): + with self.assertRaises(AttributeError): + self.client.command_3() + + def test_load_bg_system_with_version_constraint(self): + self.client._version_constraint = '1.0.0' + self.client.load_bg_system() + self.assertTrue(self.client._loaded) + + def test_load_bg_system_without_version_constraint(self): + self.client.load_bg_system() + self.assertTrue(self.client._loaded) + + def test_load_bg_system_no_system_with_version_constraint(self): + self.client._version_constraint = '1.0.0' + self.easy_client_mock.find_unique_system.return_value = None + self.assertRaises(BrewmasterFetchError, self.client.load_bg_system) + + def test_load_bg_system_no_system_without_version_constraint(self): + self.easy_client_mock.find_systems.return_value = [] + self.assertRaises(BrewmasterFetchError, self.client.load_bg_system) + + def test_load_bg_system_latest_version(self): + fake_system_2 = Mock(version='2.0.0', commands=[self.fake_command_1, self.fake_command_2], + instance_names=[u'default']) + type(fake_system_2).name = PropertyMock(return_value='system') + self.easy_client_mock.find_systems.return_value = [self.fake_system, fake_system_2] + + self.client.load_bg_system() + self.assertEqual(self.client._system, fake_system_2) + + def test_create_request_no_context(self): + self.easy_client_mock.create_request.return_value = self.mock_success + + self.client.command_1() + self.assertIsNone(self.easy_client_mock.create_request.call_args[0][0].parent) + + @patch('brewtils.rest.system_client.request_context', None) + def test_create_request_none_context(self): + self.easy_client_mock.create_request.return_value = self.mock_success + + self.client.command_1() + self.assertIsNone(self.easy_client_mock.create_request.call_args[0][0].parent) + + @patch('brewtils.rest.system_client.request_context', Mock(current_request=None)) + def test_create_request_empty_context(self): + self.easy_client_mock.create_request.return_value = self.mock_success + + self.client.command_1() + self.assertIsNone(self.easy_client_mock.create_request.call_args[0][0].parent) + + @patch('brewtils.rest.system_client.request_context', Mock(current_request=Mock(id='1234'), + bg_host="localhost", + bg_port=3000)) + def test_create_request_valid_context(self): + self.easy_client_mock.create_request.return_value = self.mock_success + + self.client.command_1() + self.assertEqual('1234', self.easy_client_mock.create_request.call_args[0][0].parent.id) + + @patch('brewtils.rest.system_client.request_context', Mock(current_request=Mock(id='1234'), + bg_host="NOT_THE_SAME_BG", + bg_port=3000)) + def test_create_request_different_bg(self): + self.easy_client_mock.create_request.return_value = self.mock_success + + self.client.command_1() + self.assertIsNone(self.easy_client_mock.create_request.call_args[0][0].parent) + + def test_create_request_missing_fields(self): + + self.assertRaises(BrewmasterValidationError, self.client._construct_bg_request, _system_name='', + _system_version='', _instance_name='') + self.assertRaises(BrewmasterValidationError, self.client._construct_bg_request, _command='', _system_version='', + _instance_name='') + self.assertRaises(BrewmasterValidationError, self.client._construct_bg_request, _command='', _system_name='', + _instance_name='') + self.assertRaises(BrewmasterValidationError, self.client._construct_bg_request, _command='', _system_name='', + _system_version='') + + @patch('brewtils.rest.system_client.time.sleep', Mock()) + def test_execute_command_1(self): + self.easy_client_mock.find_unique_request.return_value = self.mock_success + self.easy_client_mock.create_request.return_value = self.mock_in_progress + + request = self.client.command_1() + + self.easy_client_mock.find_unique_request.assert_called_with(id=self.mock_in_progress.id) + self.assertEqual(request.status, self.mock_success.status) + self.assertEqual(request.output, self.mock_success.output) + + @patch('brewtils.rest.system_client.time.sleep') + def test_execute_command_with_delays(self, sleep_mock): + self.easy_client_mock.create_request.return_value = self.mock_in_progress + self.easy_client_mock.find_unique_request.side_effect = [self.mock_in_progress, self.mock_in_progress, + self.mock_success] + + self.client.command_1() + + sleep_mock.assert_has_calls([call(0.5), call(1.0), call(2.0)]) + self.easy_client_mock.find_unique_request.assert_called_with(id=self.mock_in_progress.id) + + @patch('brewtils.rest.system_client.time.sleep') + def test_execute_with_max_delay(self, sleep_mock): + self.easy_client_mock.create_request.return_value = self.mock_in_progress + self.easy_client_mock.find_unique_request.side_effect = [self.mock_in_progress, self.mock_in_progress, + self.mock_success] + + self.client._max_delay = 1 + self.client.command_1() + + sleep_mock.assert_has_calls([call(0.5), call(1.0), call(1.0)]) + self.easy_client_mock.find_unique_request.assert_called_with(id=self.mock_in_progress.id) + + @patch('brewtils.rest.system_client.time.sleep', Mock()) + def test_execute_with_timeout(self): + self.easy_client_mock.create_request.return_value = self.mock_in_progress + self.easy_client_mock.find_unique_request.return_value = self.mock_in_progress + + self.client._timeout = 1 + + self.assertRaises(BrewmasterTimeoutError, self.client.command_1) + self.easy_client_mock.find_unique_request.assert_called_with(id=self.mock_in_progress.id) + + @patch('brewtils.rest.system_client.time.sleep', Mock()) + def test_execute_non_blocking_command_1(self): + self.easy_client_mock.find_unique_request.return_value = self.mock_success + self.easy_client_mock.create_request.return_value = self.mock_in_progress + + self.client._blocking = False + request = self.client.command_1().result() + + self.easy_client_mock.find_unique_request.assert_called_with(id=self.mock_in_progress.id) + self.assertEqual(request.status, self.mock_success.status) + self.assertEqual(request.output, self.mock_success.output) + + @patch('brewtils.rest.system_client.time.sleep', Mock()) + def test_execute_non_blocking_multiple_commands(self): + self.easy_client_mock.find_unique_request.return_value = self.mock_success + self.easy_client_mock.create_request.return_value = self.mock_in_progress + + self.client._blocking = False + futures = [self.client.command_1() for _ in range(3)] + wait(futures) + + self.easy_client_mock.find_unique_request.assert_called_with(id=self.mock_in_progress.id) + for future in futures: + self.assertEqual(future.result().status, self.mock_success.status) + self.assertEqual(future.result().output, self.mock_success.output) + + @patch('brewtils.rest.system_client.time.sleep', Mock()) + def test_execute_non_blocking_multiple_commands_with_timeout(self): + self.easy_client_mock.find_unique_request.return_value = self.mock_in_progress + self.easy_client_mock.create_request.return_value = self.mock_in_progress + + self.client._timeout = 1 + self.client._blocking = False + futures = [self.client.command_1() for _ in range(3)] + wait(futures) + + self.easy_client_mock.find_unique_request.assert_called_with(id=self.mock_in_progress.id) + for future in futures: + self.assertRaises(BrewmasterTimeoutError, future.result) + + def test_always_update(self): + self.client._always_update = True + self.client.load_bg_system() + self.easy_client_mock.create_request.return_value = self.mock_success + + load_mock = Mock() + self.client.load_bg_system = load_mock + + self.client.command_1() + self.assertTrue(load_mock.called) + + def test_retry_send_no_different_version(self): + self.easy_client_mock.create_request.side_effect = BrewmasterValidationError + + self.assertRaises(BrewmasterValidationError, self.client.command_1) + self.assertEqual(1, self.easy_client_mock.create_request.call_count) + + def test_retry_send_different_version(self): + self.client.load_bg_system() + + fake_system_2 = Mock(version='2.0.0', commands=[self.fake_command_1, self.fake_command_2], + instance_names=[u'default']) + type(fake_system_2).name = PropertyMock(return_value='system') + self.easy_client_mock.find_systems.return_value = [fake_system_2] + + self.easy_client_mock.create_request.side_effect = [BrewmasterValidationError, self.mock_success] + + self.client.command_1() + self.assertEqual('2.0.0', self.client._system.version) + self.assertEqual(2, self.easy_client_mock.create_request.call_count) + + +class BrewmasterSystemClientTest(unittest.TestCase): + + def test_deprecation(self): + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter('always') + + BrewmasterSystemClient('host', 'port', 'system') + self.assertEqual(1, len(w)) + + warning = w[0] + self.assertEqual(warning.category, DeprecationWarning) + self.assertIn("'BrewmasterSystemClient'", str(warning)) + self.assertIn("'SystemClient'", str(warning)) + self.assertIn('3.0', str(warning)) diff --git a/test/schema_parser_test.py b/test/schema_parser_test.py new file mode 100644 index 00000000..db26129e --- /dev/null +++ b/test/schema_parser_test.py @@ -0,0 +1,370 @@ +from __future__ import unicode_literals +# Doing this because marshmallow uses unicode when it serializes things to dictionaries + +# Detailed explanation: +# When everything works (the dictionary comparisons are good) it's fine, but if there's a problem (like a new field +# was added to the data model and not to the test object here) then the diff output between the two dictionaries will +# be enormous (because the serialized keys are unicode and the literals in this file aren't). It's kind of weird that +# there's a 'difference' that doesn't fail the comparison but does show up in the diff if the comparison fails, but +# that's the way it is :/ + +import copy +import unittest +import warnings +from datetime import datetime + +from marshmallow.exceptions import MarshmallowError + +from brewtils.models import Command, Instance, Parameter, Request, System, PatchOperation, Choices, LoggingConfig,\ + Event, Queue +from brewtils.schema_parser import SchemaParser, BrewmasterSchemaParser +from test.utils.comparable import assert_parameter_equal, assert_command_equal, assert_system_equal, \ + assert_instance_equal, assert_request_equal, assert_patch_equal, assert_logging_config_equal, assert_event_equal, \ + assert_queue_equal + + +class SchemaParserTest(unittest.TestCase): + + def setUp(self): + self.maxDiff = None + self.parser = SchemaParser() + + nested_parameter_dict = { + 'key': 'nested', 'type': None, 'multi': None, 'display_name': None, 'optional': None, + 'default': None, 'description': None, 'choices': None, 'parameters': [], + 'nullable': None, 'maximum': None, 'minimum': None, 'regex': None, + 'form_input_type': None, + } + + self.parameter_dict = { + 'key': 'key', + 'type': 'Any', + 'multi': False, + 'display_name': 'display', + 'optional': True, + 'default': 'default', + 'description': 'desc', + 'choices': {'display': 'select', 'strict': True, 'type': 'static', 'value': ['choiceA', 'choiceB'], + 'details': {}}, + 'parameters': [nested_parameter_dict], + 'nullable': False, + 'maximum': 10, + 'minimum': 1, + 'regex': '.*', + 'form_input_type': None + } + self.parameter = Parameter('key', type='Any', multi=False, display_name='display', optional=True, + default='default', description='desc', nullable=False, regex='.*', + parameters=[Parameter('nested')], maximum=10, minimum=1, + choices=Choices(type='static', value=['choiceA', 'choiceB'], strict=True, + display='select', details={}), + form_input_type=None) + + # Need to set the system after we declare the system... + self.command_dict = { + 'name': 'name', + 'description': 'desc', + 'id': '123f11af55a38e64799f1234', + 'parameters': [self.parameter_dict], + 'command_type': 'ACTION', + 'output_type': 'STRING', + 'schema': {}, + 'form': {}, + 'template': '', + 'icon_name': 'icon!', + 'system': None # Set at the bottom of __init__ + } + self.command = Command('name', description='desc', id='123f11af55a38e64799f1234', parameters=[self.parameter], + command_type='ACTION', output_type='STRING', schema={}, form={}, + template='', icon_name='icon!', system=None) + + self.instance_dict = { + 'id': '584f11af55a38e64799fd1d4', + 'name': 'default', + 'description': 'desc', + 'status': 'RUNNING', + 'icon_name': 'icon!', + 'queue_type': 'rabbitmq', + 'queue_info': {'queue': 'abc[default]-0.0.1', 'url': 'amqp://guest:guest@localhost:5672'}, + 'status_info': {'heartbeat': 1451606400000}, + 'metadata': {} + } + self.instance = Instance(id='584f11af55a38e64799fd1d4', name='default', description='desc', status='RUNNING', + icon_name='icon!', status_info={'heartbeat': datetime(2016, 1, 1)}, + metadata={}, queue_type='rabbitmq', + queue_info={'queue': 'abc[default]-0.0.1', 'url': 'amqp://guest:guest@localhost:5672'}) + + self.system_dict = { + 'name': 'name', + 'description': 'desc', + 'version': '1.0.0', + 'id': '584f11af55a38e64799f1234', + 'max_instances': 1, + 'instances': [self.instance_dict], + 'commands': [self.command_dict], + 'icon_name': 'fa-beer', + 'display_name': 'non-offensive', + 'metadata': {'some': 'stuff'} + } + self.system = System(name='name', description='desc', version='1.0.0', id='584f11af55a38e64799f1234', + max_instances=1, instances=[self.instance], commands=[self.command], icon_name='fa-beer', + display_name='non-offensive', metadata={'some': 'stuff'}) + + self.child_request_dict = { + 'system': 'child_system', + 'system_version': '1.0.0', + 'instance_name': 'default', + 'command': 'say', + 'id': '58542eb571afd47ead90d25f', + 'parameters': {}, + 'comment': 'bye!', + 'output': 'nested output', + 'output_type': 'STRING', + 'status': 'CREATED', + 'command_type': 'ACTION', + 'created_at': 1451606400000, + 'updated_at': 1451606400000, + 'error_class': None, + 'metadata': {'child': 'stuff'} + } + self.child_request =\ + Request(system='child_system', system_version='1.0.0', instance_name='default', command='say', + id='58542eb571afd47ead90d25f', parent=None, children=None, parameters={}, comment='bye!', + output='nested output', output_type='STRING', status='CREATED', command_type='ACTION', + created_at=datetime(2016, 1, 1), error_class=None, metadata={'child': 'stuff'}, + updated_at=datetime(2016, 1, 1)) + + self.parent_request_dict = { + 'system': 'parent_system', + 'system_version': '1.0.0', + 'instance_name': 'default', + 'command': 'say', + 'id': '58542eb571afd47ead90d25f', + 'parent': None, + 'parameters': {}, + 'comment': 'bye!', + 'output': 'nested output', + 'output_type': 'STRING', + 'status': 'CREATED', + 'command_type': 'ACTION', + 'created_at': 1451606400000, + 'updated_at': 1451606400000, + 'error_class': None, + 'metadata': {'parent': 'stuff'} + } + self.parent_request =\ + Request(system='parent_system', system_version='1.0.0', instance_name='default', command='say', + id='58542eb571afd47ead90d25f', parent=None, children=None, parameters={}, comment='bye!', + output='nested output', output_type='STRING', status='CREATED', command_type='ACTION', + created_at=datetime(2016, 1, 1), error_class=None, metadata={'parent': 'stuff'}, + updated_at=datetime(2016, 1, 1)) + + self.request_dict = { + 'system': 'system', + 'system_version': '1.0.0', + 'instance_name': 'default', + 'command': 'speak', + 'id': '58542eb571afd47ead90d25e', + 'parent': self.parent_request_dict, + 'children': [self.child_request_dict], + 'parameters': {'message': 'hey!'}, + 'comment': 'hi!', + 'output': 'output', + 'output_type': 'STRING', + 'status': 'CREATED', + 'command_type': 'ACTION', + 'created_at': 1451606400000, + 'updated_at': 1451606400000, + 'error_class': 'ValueError', + 'metadata': {'request': 'stuff'} + } + self.request =\ + Request(system='system', system_version='1.0.0', instance_name='default', command='speak', + id='58542eb571afd47ead90d25e', parent=self.parent_request, children=[self.child_request], + parameters={'message': 'hey!'}, comment='hi!', output='output', output_type='STRING', + status='CREATED', command_type='ACTION', created_at=datetime(2016, 1, 1), error_class='ValueError', + metadata={'request': 'stuff'}, updated_at=datetime(2016, 1, 1)) + + self.patch_dict = {'operations': [{'operation': 'replace', 'path': '/status', 'value': 'RUNNING'}]} + self.patch_many_dict = {'operations': [ + {'operation': 'replace', 'path': '/status', 'value': 'RUNNING'}, + {'operation': 'replace2', 'path': '/status2', 'value': 'RUNNING2'} + ]} + self.patch_no_envelope_dict = {'operation': 'replace', 'path': '/status', 'value': 'RUNNING'} + self.patch1 = PatchOperation(operation='replace', path='/status', value='RUNNING') + self.patch2 = PatchOperation(operation='replace2', path='/status2', value='RUNNING2') + + self.logging_config_dict = { + "level": "INFO", + "handlers": {"stdout": {"foo": "bar"}}, + "formatters": {"default": {"format": LoggingConfig.DEFAULT_FORMAT}} + } + self.logging_config = LoggingConfig(level="INFO", + handlers={"stdout": {"foo": "bar"}}, + formatters={"default": {"format": LoggingConfig.DEFAULT_FORMAT}}) + + self.event_dict = { + 'name': 'REQUEST_CREATED', + 'error': False, + 'payload': {'id': '58542eb571afd47ead90d25e'}, + 'metadata': {'extra': 'info'}, + 'timestamp': 1451606400000 + } + self.event = Event(name='REQUEST_CREATED', error=False, payload={'id': '58542eb571afd47ead90d25e'}, + metadata={'extra': 'info'}, timestamp=datetime(2016, 1, 1)) + + self.queue_dict = { + 'name': 'echo.1-0-0.default', + 'system': 'echo', + 'version': '1.0.0', + 'instance': 'default', + 'system_id': '1234', + 'display': 'foo.1-0-0.default', + 'size': 3 + } + self.queue = Queue(name='echo.1-0-0.default', system='echo', version='1.0.0', instance='default', + system_id='1234', display='foo.1-0-0.default', size=3) + + # Finish setting up our circular system <-> command dependency + self.command.system = self.system + self.command_dict['system'] = {'id': self.system.id} + + # Finish setting up our circular request parent <-> dependencies + self.child_request.parent = self.request + self.parent_request.children = [self.request] + + def test_parse_none(self): + self.assertRaises(TypeError, self.parser.parse_system, None, from_string=True) + self.assertRaises(TypeError, self.parser.parse_system, None, from_string=False) + + def test_parse_empty(self): + self.parser.parse_system({}, from_string=False) + self.parser.parse_system('{}', from_string=True) + + def test_parse_error(self): + self.assertRaises(ValueError, self.parser.parse_system, '', from_string=True) + self.assertRaises(ValueError, self.parser.parse_system, 'bad bad bad', from_string=True) + + def test_parse_bad_input_type(self): + self.assertRaises(TypeError, self.parser.parse_system, ['list', 'is', 'bad'], from_string=True) + self.assertRaises(TypeError, self.parser.parse_system, {'bad': 'bad bad'}, from_string=True) + + def test_parse_fail_validation(self): + self.system_dict['name'] = None + self.assertRaises(MarshmallowError, self.parser.parse_system, self.system_dict) + self.assertRaises(MarshmallowError, self.parser.parse_system, 'bad bad bad', from_string=False) + + def test_parse_non_strict_failure(self): + self.system_dict['name'] = None + self.parser.parse_system(self.system_dict, from_string=False, strict=False) + + def test_no_modify_arguments(self): + system_copy = copy.deepcopy(self.system_dict) + self.parser.parse_system(self.system_dict) + self.assertEqual(system_copy, self.system_dict) + + def test_parse_system(self): + assert_system_equal(self.system, self.parser.parse_system(self.system_dict), True) + + def test_parse_instance(self): + assert_instance_equal(self.instance, self.parser.parse_instance(self.instance_dict), True) + + def test_parse_command(self): + assert_command_equal(self.command, self.parser.parse_command(self.command_dict), True) + + def test_parse_parameter(self): + assert_parameter_equal(self.parameter, self.parser.parse_parameter(self.parameter_dict), True) + + def test_parse_request(self): + assert_request_equal(self.request, self.parser.parse_request(self.request_dict), True) + + def test_parse_patch(self): + assert_patch_equal(self.patch1, self.parser.parse_patch(self.patch_dict)[0], True) + + def test_parse_patch_ignore_many(self): + assert_patch_equal(self.patch1, self.parser.parse_patch(self.patch_dict, many=False)[0], True) + + def test_parse_patch_no_envelope(self): + assert_patch_equal(self.parser.parse_patch(self.patch_no_envelope_dict)[0], self.patch1) + + def test_parse_many_patch_no_envelope(self): + assert_patch_equal(self.parser.parse_patch([self.patch_no_envelope_dict])[0], self.patch1) + + def test_parse_patch_many(self): + patches = sorted(self.parser.parse_patch(self.patch_many_dict, many=True), key=lambda x: x.operation) + for index, patch in enumerate([self.patch1, self.patch2]): + assert_patch_equal(patch, patches[index]) + + def test_parse_logging_config(self): + assert_logging_config_equal(self.logging_config, self.parser.parse_logging_config(self.logging_config_dict)) + + def test_parse_logging_config_ignore_many(self): + assert_logging_config_equal(self.logging_config, self.parser.parse_logging_config(self.logging_config_dict, + many=True)) + + def test_parse_event(self): + assert_event_equal(self.event, self.parser.parse_event(self.event_dict)) + + def test_parse_queue(self): + assert_queue_equal(self.queue, self.parser.parse_queue(self.queue_dict)) + + def test_serialize_system(self): + self.assertEqual(self.system_dict, self.parser.serialize_system(self.system, to_string=False)) + + def test_serialize_system_no_commands(self): + self.system_dict.pop('commands') + self.assertEqual(self.system_dict, self.parser.serialize_system(self.system, to_string=False, + include_commands=False)) + + def test_serialize_system_no_commands_other_excludes(self): + self.system_dict.pop('commands') + self.system_dict.pop('icon_name') + self.assertEqual(self.system_dict, self.parser.serialize_system(self.system, to_string=False, + include_commands=False, exclude=('icon_name',))) + + def test_serialize_instance(self): + self.assertEqual(self.instance_dict, self.parser.serialize_instance(self.instance, to_string=False)) + + def test_serialize_command(self): + self.assertEqual(self.command_dict, self.parser.serialize_command(self.command, to_string=False)) + + def test_serialize_parameter(self): + self.assertEqual(self.parameter_dict, self.parser.serialize_parameter(self.parameter, to_string=False)) + + def test_serialize_request(self): + self.assertEqual(self.request_dict, self.parser.serialize_request(self.request, to_string=False)) + + def test_serialize_patch(self): + self.assertEqual(self.patch_dict, self.parser.serialize_patch(self.patch1, to_string=False, many=False)) + + def test_serialize_patch_many(self): + self.assertEqual(self.patch_many_dict, + self.parser.serialize_patch([self.patch1, self.patch2], to_string=False, many=True)) + + def test_serialize_logging_config(self): + self.assertEqual(self.logging_config_dict, + self.parser.serialize_logging_config(self.logging_config, to_string=False)) + + def test_serialize_event(self): + self.assertEqual(self.event_dict, + self.parser.serialize_event(self.event, to_string=False)) + + def test_serialize_queue(self): + self.assertEqual(self.queue_dict, + self.parser.serialize_queue(self.queue, to_string=False)) + + +class BrewmasterSchemaParserTest(unittest.TestCase): + + def test_deprecation(self): + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter('always') + + BrewmasterSchemaParser() + self.assertEqual(1, len(w)) + + warning = w[0] + self.assertEqual(warning.category, DeprecationWarning) + self.assertIn("'BrewmasterSchemaParser'", str(warning)) + self.assertIn("'SchemaParser'", str(warning)) + self.assertIn('3.0', str(warning)) diff --git a/test/schema_test.py b/test/schema_test.py new file mode 100644 index 00000000..fa52acbf --- /dev/null +++ b/test/schema_test.py @@ -0,0 +1,48 @@ +import unittest +from datetime import datetime + +import pytz + +from brewtils.models import System +from brewtils.schemas import DateTime, BaseSchema, SystemSchema + + +class SchemaTest(unittest.TestCase): + + def setUp(self): + self.test_epoch = 1451651214123 + self.test_dt = datetime(2016, 1, 1, hour=12, minute=26, second=54, microsecond=123000) + + self.test_epoch_tz = 1451668974123 + self.test_dt_tz = datetime(2016, 1, 1, hour=12, minute=26, second=54, microsecond=123000, + tzinfo=pytz.timezone('US/Eastern')) + + def test_make_object_no_model(self): + base_schema = BaseSchema() + self.assertEqual('input', base_schema.make_object('input')) + + def test_make_object_with_model(self): + system_schema = SystemSchema(context={'models': {'SystemSchema': System}}) + self.assertIsInstance(system_schema.make_object({'name': 'name'}), System) + + def test_get_attribute_name(self): + attributes = SystemSchema.get_attribute_names() + self.assertIn('id', attributes) + self.assertIn('name', attributes) + self.assertNotIn('__model__', attributes) + + def test_to_epoch_no_tz(self): + self.assertEqual(self.test_epoch, DateTime.to_epoch(self.test_dt)) + + def test_to_epoch_local_time(self): + self.assertEqual(self.test_epoch, DateTime.to_epoch(self.test_dt, localtime=True)) + + def test_to_epoch_tz(self): + self.assertNotEqual(self.test_epoch, DateTime.to_epoch(self.test_dt_tz)) + self.assertEqual(self.test_epoch_tz, DateTime.to_epoch(self.test_dt_tz)) + + def test_to_epoch_tz_local(self): + self.assertEqual(self.test_epoch, DateTime.to_epoch(self.test_dt_tz, localtime=True)) + + def test_from_epoch(self): + self.assertEqual(self.test_dt, DateTime.from_epoch(self.test_epoch)) diff --git a/test/stoppable_thread_test.py b/test/stoppable_thread_test.py new file mode 100644 index 00000000..1499df7d --- /dev/null +++ b/test/stoppable_thread_test.py @@ -0,0 +1,36 @@ +import unittest + +from mock import Mock + +from brewtils.stoppable_thread import StoppableThread + + +class StoppableThreadTest(unittest.TestCase): + + def setUp(self): + self.thread = StoppableThread() + + def test_init_stop_not_set(self): + self.assertFalse(self.thread._stop_event.isSet()) + + def test_init_logger_passed_in(self): + fake_logger = Mock() + t = StoppableThread(logger=fake_logger) + self.assertEqual(t.logger, fake_logger) + + def test_stop(self): + self.thread.stop() + self.assertTrue(self.thread._stop_event.isSet()) + + def test_stopped_true(self): + self.thread._stop_event.set() + self.assertTrue(self.thread.stopped()) + + def test_stopped_false(self): + self.assertFalse(self.thread.stopped()) + + def test_wait(self): + event_mock = Mock() + self.thread._stop_event = event_mock + self.thread.wait(1) + event_mock.wait.assert_called_once_with(1) diff --git a/test/utils/__init__.py b/test/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/utils/comparable.py b/test/utils/comparable.py new file mode 100644 index 00000000..ac6c967e --- /dev/null +++ b/test/utils/comparable.py @@ -0,0 +1,189 @@ +from brewtils.models import System, Command, Instance, Parameter, Request, PatchOperation, LoggingConfig, Event, Queue + + +def assert_system_equal(system1, system2, deep=False): + assert isinstance(system1, System), "system1 was not a System" + assert isinstance(system2, System), "system2 was not a System" + assert type(system1) is type(system2), "system1 and system2 are not the same type." + + deep_fields = ["instances", "commands"] + for key in system1.__dict__.keys(): + if key in deep_fields: + continue + + assert hasattr(system1, key), "System1 does not have an attribute '%s'" % key + assert hasattr(system2, key), "System2 does not have an attribute '%s'" % key + assert getattr(system1, key) == getattr(system2, key), \ + "%s was not the same (%s, %s)" % (key, getattr(system1, key), getattr(system2, key)) + + if deep: + assert hasattr(system1, "instances"), "System1 does not have attribute 'instances'" + assert hasattr(system1, "commands"), "System1 does not have attribute 'commands'" + assert hasattr(system2, "instances"), "System1 does not have attribute 'instances'" + assert hasattr(system2, "commands"), "System1 does not have attribute 'commands'" + + assert len(system1.commands) == len(system2.commands), "system1 has a different number " \ + "of commands than system2" + for command1, command2 in zip(sorted(system1.commands, key=lambda x: x.name), + sorted(system2.commands, key=lambda x: x.name)): + assert_command_equal(command1, command2, deep) + + for instance1, instance2 in zip(sorted(system1.instances, key=lambda x: x.name), + sorted(system2.instances, key=lambda x: x.name)): + assert_instance_equal(instance1, instance2, deep) + + +def assert_command_equal(command1, command2, deep=False): + assert isinstance(command1, Command), "command1 was not a Command" + assert isinstance(command2, Command), "command2 was not a Command" + assert type(command1) is type(command2), "command1 and command2 are not the same type." + + deep_fields = ["parameters", "system"] + for key in command1.__dict__.keys(): + if key in deep_fields: + continue + + assert hasattr(command1, key), "Command1 does not have an attribute '%s'" % key + assert hasattr(command2, key), "Command2 does not have an attribute '%s'" % key + assert getattr(command1, key) == getattr(command2, key), \ + "%s was not the same (%s, %s)" % (key, getattr(command1, key), getattr(command2, key)) + + if deep: + assert hasattr(command1, "parameters"), "command1 does not have attribute 'parameters'" + assert hasattr(command2, "parameters"), "command1 does not have attribute 'parameters'" + assert len(command1.parameters) == len(command2.parameters), "command1 has a different number " \ + "of parameters than command2" + for parameter1, parameter2 in zip(sorted(command1.parameters, key=lambda p: p.key), + sorted(command2.parameters, key=lambda p: p.key)): + assert_parameter_equal(parameter1, parameter2, deep) + + +def assert_instance_equal(instance1, instance2, deep=False): + assert isinstance(instance1, Instance), "instance1 was not an Instance" + assert isinstance(instance2, Instance), "instance2 was not an Instance" + assert type(instance1) is type(instance2), "instance1 and instance2 are not the same type." + + for key in instance1.__dict__.keys(): + assert hasattr(instance1, key), "instance1 does not have an attribute '%s'" % key + assert hasattr(instance2, key), "instance2 does not have an attribute '%s'" % key + assert getattr(instance1, key) == getattr(instance2, key), \ + "%s was not the same (%s, %s)" % (key, getattr(instance1, key), getattr(instance2, key)) + + +def assert_request_equal(request1, request2, deep=False): + assert isinstance(request1, Request), "request1 was not a Request" + assert isinstance(request2, Request), "request2 was not a Request" + assert type(request1) is type(request2), "request1 and request2 are not the same type." + + deep_fields = ["children", "parent"] + for key in request1.__dict__.keys(): + if key in deep_fields: + continue + + assert hasattr(request1, key), "request1 does not have an attribute '%s'" % key + assert hasattr(request2, key), "request2 does not have an attribute '%s'" % key + assert getattr(request1, key) == getattr(request2, key), \ + "%s was not the same (%s, %s)" % (key, getattr(request1, key), getattr(request2, key)) + + if deep: + assert hasattr(request1, "children"), "request1 does not have attribute 'children'" + assert hasattr(request2, "children"), "request2 does not have attribute 'children'" + if request1.children is not None: + assert len(request1.children) == len(request2.children), \ + "request1 has a different number of children than request 2" + for request1, request2 in zip(sorted(request1.children, key=lambda p: p.id), + sorted(request2.children, key=lambda p: p.id)): + assert_request_equal(request1, request2, deep) + + +def assert_parameter_equal(parameter1, parameter2, deep=False): + assert isinstance(parameter1, Parameter), "parameter1 was not a Parameter" + assert isinstance(parameter2, Parameter), "parameter2 was not a Parameter" + assert type(parameter1) is type(parameter2), "parameter1 and parameter2 are not the same type." + + deep_fields = ["parameters", "choices"] + for key in parameter1.__dict__.keys(): + if key in deep_fields: + continue + + assert hasattr(parameter1, key), "parameter1 does not have an attribute '%s'" % key + assert hasattr(parameter2, key), "parameter2 does not have an attribute '%s'" % key + assert getattr(parameter1, key) == getattr(parameter2, key), \ + "%s was not the same (%s, %s)" % (key, getattr(parameter1, key), getattr(parameter2, key)) + + if deep: + assert hasattr(parameter1, "parameters"), "parameter1 does not have attribute 'parameters'" + assert hasattr(parameter2, "parameters"), "parameter1 does not have attribute 'parameters'" + assert len(parameter1.parameters) == len(parameter2.parameters), "command1 has a different number " \ + "of parameters than command2" + for parameter1, parameter2 in zip(sorted(parameter1.parameters, key=lambda p: p.key), + sorted(parameter2.parameters, key=lambda p: p.key)): + assert_parameter_equal(parameter1, parameter2, deep) + + assert hasattr(parameter1, "choices"), "parameter1 does not have attribute 'choices'" + assert hasattr(parameter2, "choices"), "parameter2 does not have attribute 'choices'" + + assert_choices_equal(parameter1.choices, parameter2.choices) + + +def assert_choices_equal(choices1, choices2): + if choices1 is None and choices2 is None: + return + elif choices1 is None: + assert False, "choices1 is None and choices2 is not" + elif choices2 is None: + assert False, "choices2 is None and choices1 is not" + else: + for key in choices1.__dict__.keys(): + assert hasattr(choices1, key), "choices1 does not have an attribute '%s'" % key + assert hasattr(choices2, key), "choices2 does not have an attribute '%s'" % key + assert getattr(choices1, key) == getattr(choices2, key), \ + "%s was not the same (%s, %s)" % (key, getattr(choices1, key), getattr(choices2, key)) + + +def assert_patch_equal(patch1, patch2, deep=False): + assert isinstance(patch1, PatchOperation), "patch1 was not an PatchOperation" + assert isinstance(patch2, PatchOperation), "patch2 was not an PatchOperation" + assert type(patch1) is type(patch2), "patch1 and instance2 are not the same type." + + for key in patch1.__dict__.keys(): + assert hasattr(patch1, key), "patch1 does not have an attribute '%s'" % key + assert hasattr(patch2, key), "patch2 does not have an attribute '%s'" % key + assert getattr(patch1, key) == getattr(patch2, key), \ + "%s was not the same (%s, %s)" % (key, getattr(patch1, key), getattr(patch2, key)) + + +def assert_logging_config_equal(logging_config1, logging_config2, deep=False): + assert isinstance(logging_config1, LoggingConfig), "logging_config1 was not a LoggingConfig" + assert isinstance(logging_config2, LoggingConfig), "logging_config2 was not a LoggingConfig" + assert type(logging_config1) is type(logging_config2), "logging_config1 and logging_config2 are not the same type" + + for key in logging_config1.__dict__.keys(): + assert hasattr(logging_config1, key), "logging_config1 does not have attribute '%s'" % key + assert hasattr(logging_config2, key), "logging_config2 does not have attribute '%s'" % key + assert getattr(logging_config1, key) == getattr(logging_config2, key), \ + "%s was not the same (%s, %s)" % (key, getattr(logging_config1, key), getattr(logging_config2, key)) + + +def assert_event_equal(event1, event2, deep=False): + assert isinstance(event1, Event), "event1 was not an Event" + assert isinstance(event2, Event), "event2 was not an Event" + assert type(event1) is type(event2), "event1 and event2 are not the same type" + + for key in event1.__dict__.keys(): + assert hasattr(event1, key), "event1 does not have an attribute '%s'" % key + assert hasattr(event2, key), "event2 does not have an attribute '%s'" % key + assert getattr(event1, key) == getattr(event2, key), \ + "%s was not the same (%s, %s)" % (key, getattr(event1, key), getattr(event2, key)) + + +def assert_queue_equal(queue1, queue2, deep=False): + assert isinstance(queue1, Queue), "queue1 was not an Queue" + assert isinstance(queue2, Queue), "queue2 was not an Queue" + assert type(queue1) is type(queue2), "event1 and event2 are not the same type" + + for key in queue1.__dict__.keys(): + assert hasattr(queue1, key), "queue1 does not have an attribute '%s'" % key + assert hasattr(queue2, key), "queue2 does not have an attribute '%s'" % key + assert getattr(queue1, key) == getattr(queue2, key), \ + "%s was not the same (%s, %s)" % (key, getattr(queue1, key), getattr(queue2, key)) diff --git a/tox.ini b/tox.ini new file mode 100644 index 00000000..bba26019 --- /dev/null +++ b/tox.ini @@ -0,0 +1,39 @@ +# Tox (http://tox.testrun.org/) is a tool for running tests +# in multiple virtualenvs. This configuration file will run the +# test suite on all supported python versions. To use it, "pip install tox" +# and then run "tox" from this directory. + +[tox] +envlist = py27,py35,py36,coverage,docs +skipsdist = {env:TOXBUILD:false} + +[testenv] +passenv = LANG +whitelist_externals = + /bin/true +deps = + {py27,py35,py36,coverage,docs}: -rrequirements.txt +commands = + {env:TOXBUILD:nosetests} + +[testenv:coverage] +whitelist_externals = + make + /bin/true + /bin/mv +commands = + {env:TOXBUILD:{toxinidir}/bin/generate_coverage.sh} + {env:TOXBUILD:mv /app/output/python /src/output} + +# This environmnet will assume you are running from the Dockerfile.test +# image. It would be nice if we could specify what commands to run based +# on whether we are in Jenkins or not. +[testenv:docs] +whitelist_externals = + make + /bin/true + /bin/mv +commands = + {env:TOXBUILD:make -C docs/} + {env:TOXBUILD:mv /app/docs/_build/html /src/docs/_build/.} +