diff --git a/.gitignore b/.gitignore index 605b524..c65daad 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ tests/haystackClient/HaystackClientTest.py /docs/build /.cache /pyhaystack/client/ops/vendor/niagara4 +.idea diff --git a/.idea/misc.xml b/.idea/misc.xml deleted file mode 100644 index df245c4..0000000 --- a/.idea/misc.xml +++ /dev/null @@ -1,14 +0,0 @@ - - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml deleted file mode 100644 index 1542d7b..0000000 --- a/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/.idea/needle.iml b/.idea/needle.iml deleted file mode 100644 index d0876a7..0000000 --- a/.idea/needle.iml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 6564d52..0000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml deleted file mode 100644 index 5f60140..0000000 --- a/.idea/workspace.xml +++ /dev/null @@ -1,43 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - 1451219164897 - - - - - - - - - - \ No newline at end of file diff --git a/README.rst b/README.rst index a2b0070..852dac4 100644 --- a/README.rst +++ b/README.rst @@ -15,7 +15,7 @@ Actually, connection can be established with : * Niagara4_ by Tridium * NiagaraAX_ by Tridium -* Widesky_ by VRT_ +* Widesky_ by Widesky.cloud_ * Skyspark_ by SkyFoundry (version 2 and 3+) Connection to Niagara AX or Niagara 4 requires the nHaystack_ module by J2 Innovations to be installed @@ -88,7 +88,7 @@ Pyhaystack is robust and will be ready for asynchronous development. We have chosen a state machine approach with observer pattern. See the docs for more informations. -This implementation has been mostly supported by VRT_ and Servisys_. We are hoping +This implementation has been mostly supported by Widesky.cloud_ and Servisys_. We are hoping that more people will join us in our effort to build a well working open-source software that will open the door of building data analysis to Python users. @@ -123,7 +123,7 @@ to pyhaystack (ex. unit conversion) .. _Niagara4 : https://www.tridium.com/en/products-services/niagara4 -.. _VRT : http://www.vrt.com.au/ +.. _Widesky.cloud : http://widesky.cloud/ .. _Servisys : http://www.servisys.com diff --git a/debian/changelog b/debian/changelog deleted file mode 100644 index b01f247..0000000 --- a/debian/changelog +++ /dev/null @@ -1,55 +0,0 @@ -pyhaystack (0.9.0.5) unstable; urgency=low - * Pull in bugfixes for HTTP headers - * Pull in version detection updates for WideSky - -- Stuart Longland Mon, 19 Sep 2016 14:17:33 +1000 - -pyhaystack (0.9.0.4) unstable; urgency=low - * Fix dependencies on pandas. - -- Stuart Longland Tue, 06 Sep 2016 14:56:03 +1000 - -pyhaystack (0.9.0.3) unstable; urgency=low - * Fix handling of multiple entity read requests where IDs for non-existent - entities are given. (VRT JIRA issue 366; upstream Github PR #26) - -- Stuart Longland Tue, 30 Aug 2016 08:12:52 +1000 - -pyhaystack (0.9.0.2) unstable; urgency=low - * Implement auto-detection of multi-point hisRead/hisWrite support - -- Stuart Longland Fri, 29 Jul 2016 14:37:23 +1000 - -pyhaystack (0.9.0.1) unstable; urgency=low - * Bugfixes for multi-point hisRead/hisWrite - -- Stuart Longland Wed, 27 Jul 2016 10:57:52 +1000 - -pyhaystack (0.9) unstable; urgency=low - * Pull upstream release 0.9 - * Add bugfix for POST handling - -- Stuart Longland Sat, 16 Jul 2016 08:47:52 +1000 - -pyhaystack (0.71.1.8.2-6) unstable; urgency=low - * Fix to handle empty series when writing frames. - -- Stuart Longland Thu, 02 Jun 2016 16:19:00 +1000 - -pyhaystack (0.71.1.8.2-5) unstable; urgency=low - * Re-build using latest sources - -- Stuart Longland Thu, 12 May 2016 12:34:02 +1000 - -pyhaystack (0.71.1.8.2-4) unstable; urgency=low - * Throw out unused mixins that drag in pandas library unnecessarily. - -- Stuart Longland Fri, 29 Apr 2016 06:02:27 +1000 - -pyhaystack (0.71.1.8.2-3) unstable; urgency=low - * Further fixes. - * Clean up now dead and non-functional legacy client. - -- Stuart Longland Thu, 28 Apr 2016 14:25:00 +1000 - -pyhaystack (0.71.1.8.2-2) unstable; urgency=low - - * Additional fixes and improvements. - - -- Stuart Longland Thu, 21 Apr 2016 14:14:04 +1000 - -pyhaystack (0.71.1.8.2-1) unstable; urgency=low - - * Initial package built on 'widesky' branch. - - -- Stuart Longland Wed, 20 Apr 2016 02:19:27 +0000 diff --git a/debian/compat b/debian/compat deleted file mode 100644 index 7f8f011..0000000 --- a/debian/compat +++ /dev/null @@ -1 +0,0 @@ -7 diff --git a/debian/control b/debian/control deleted file mode 100644 index 2c3f2fe..0000000 --- a/debian/control +++ /dev/null @@ -1,35 +0,0 @@ -Source: pyhaystack -Maintainer: VRT Systems -Section: python -Priority: optional -Build-Depends: python-all (>= 2.6.6-3), debhelper (>= 7) -Standards-Version: 3.9.1 - - - -Package: python-pyhaystack -Architecture: all -Depends: python-hszinc, python-tz, python-requests, - python-six, python-fysom, python-signalslot, python-semver -Suggests: ${misc:Depends}, ${python:Depends} -Description: Python Haystack Utility - pyhaystack |build-status| |coverage| |docs| - =========================================== - . - Pyhaystack is a module that allow python programs to connect to a haystack server [project-haystack.org](http://www.project-haystack.org). - . - Actually, connection can be established with Niagara Platform running the nhaystack module. - . - It's a work in progress and actually, main goal is to connect to server and retrive histories to make numeric analysis. Eventually, other options will be implemented through the REST API. - . - For this to work with [Anaconda](http://continuum.io/downloads) IPython Notebook in Windows, be sure to use "python setup.py install" using the Anaconda Command Prompt in Windows. - If not, module will be installed for System path python but won't work in the environment of Anaconda IPython Notebook. - . - Chat with us on |Gitter| - . - .. |build-status| image:: https://travis-ci.org/ChristianTremblay/pyhaystack.svg?branch=master - :target: https://travis-ci.org/ChristianTremblay/pyhaystack - :alt: Build status - . - .. |docs| image:: https://readthedocs.org/projects/pyhaystack/badge/?version=latest - :target: http://pyhaystack.readthedocs.org/ diff --git a/debian/rules b/debian/rules deleted file mode 100755 index 68479c9..0000000 --- a/debian/rules +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/make -f - -# This file was automatically generated by stdeb 0.8.5 at -# Wed, 20 Apr 2016 02:19:27 +0000 - -%: - dh $@ --with python2 --buildsystem=python_distutils - - -override_dh_auto_clean: - python setup.py clean -a - find . -name \*.pyc -exec rm {} \; - - - -override_dh_auto_build: - python setup.py build --force - - - -override_dh_auto_install: - python setup.py install --force --root=debian/python-pyhaystack --no-compile -O0 --install-layout=deb - - - -override_dh_python2: - dh_python2 --no-guessing-versions - - - - diff --git a/debian/source/format b/debian/source/format deleted file mode 100644 index 89ae9db..0000000 --- a/debian/source/format +++ /dev/null @@ -1 +0,0 @@ -3.0 (native) diff --git a/debian/source/options b/debian/source/options deleted file mode 100644 index bcc4bbb..0000000 --- a/debian/source/options +++ /dev/null @@ -1 +0,0 @@ -extend-diff-ignore="\.egg-info$" \ No newline at end of file diff --git a/docs/source/conf.py b/docs/source/conf.py index 857747c..cce2ebb 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -21,43 +21,43 @@ # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -#sys.path.insert(0, os.path.abspath('.')) +# sys.path.insert(0, os.path.abspath('.')) # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' +# needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.doctest', - 'sphinx.ext.intersphinx', - 'sphinx.ext.todo', - 'sphinx.ext.coverage', - 'sphinx.ext.mathjax', - 'sphinx.ext.ifconfig', - 'sphinx.ext.viewcode', + "sphinx.ext.autodoc", + "sphinx.ext.doctest", + "sphinx.ext.intersphinx", + "sphinx.ext.todo", + "sphinx.ext.coverage", + "sphinx.ext.mathjax", + "sphinx.ext.ifconfig", + "sphinx.ext.viewcode", ] -autoclass_content = 'both' +autoclass_content = "both" # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # source_suffix = ['.rst', '.md'] -source_suffix = '.rst' +source_suffix = ".rst" # The encoding of source files. -#source_encoding = 'utf-8-sig' +# source_encoding = 'utf-8-sig' # The master toctree document. -master_doc = 'index' +master_doc = "index" # General information about the project. -project = 'pyhaystack' +project = "pyhaystack" copyright = info.__copyright__ author = info.__author__ @@ -79,9 +79,9 @@ # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: -#today = '' +# today = '' # Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' +# today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -89,27 +89,27 @@ # The reST default role (used for this markup: `text`) to use for all # documents. -#default_role = None +# default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True +# add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). -#add_module_names = True +# add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. -#show_authors = False +# show_authors = False # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = "sphinx" # A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] +# modindex_common_prefix = [] # If true, keep warnings as "system message" paragraphs in the built documents. -#keep_warnings = False +# keep_warnings = False # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = True @@ -119,156 +119,155 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'sphinx_rtd_theme' +html_theme = "sphinx_rtd_theme" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. -#html_theme_options = {} +# html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] +# html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". -#html_title = None +# html_title = None # A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None +# html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. -#html_logo = None +# html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. -#html_favicon = None +# html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied # directly to the root of the documentation. -#html_extra_path = [] +# html_extra_path = [] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' +# html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. -#html_use_smartypants = True +# html_use_smartypants = True # Custom sidebar templates, maps document names to template names. -#html_sidebars = {} +# html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. -#html_additional_pages = {} +# html_additional_pages = {} # If false, no module index is generated. -#html_domain_indices = True +# html_domain_indices = True # If false, no index is generated. -#html_use_index = True +# html_use_index = True # If true, the index is split into individual pages for each letter. -#html_split_index = False +# html_split_index = False # If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True +# html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True +# html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True +# html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. -#html_use_opensearch = '' +# html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None +# html_file_suffix = None # Language to be used for generating the HTML full-text search index. # Sphinx supports the following languages: # 'da', 'de', 'en', 'es', 'fi', 'fr', 'h', 'it', 'ja' # 'nl', 'no', 'pt', 'ro', 'r', 'sv', 'tr' -#html_search_language = 'en' +# html_search_language = 'en' # A dictionary with options for the search language support, empty by default. # Now only 'ja' uses this config value -#html_search_options = {'type': 'default'} +# html_search_options = {'type': 'default'} # The name of a javascript file (relative to the configuration directory) that # implements a search results scorer. If empty, the default will be used. -#html_search_scorer = 'scorer.js' +# html_search_scorer = 'scorer.js' # Output file base name for HTML help builder. -htmlhelp_basename = 'pyhaystackdoc' +htmlhelp_basename = "pyhaystackdoc" # -- Options for LaTeX output --------------------------------------------- latex_elements = { -# The paper size ('letterpaper' or 'a4paper'). -#'papersize': 'letterpaper', - -# The font size ('10pt', '11pt' or '12pt'). -#'pointsize': '10pt', - -# Additional stuff for the LaTeX preamble. -#'preamble': '', - -# Latex figure (float) alignment -#'figure_align': 'htbp', + # The paper size ('letterpaper' or 'a4paper'). + #'papersize': 'letterpaper', + # The font size ('10pt', '11pt' or '12pt'). + #'pointsize': '10pt', + # Additional stuff for the LaTeX preamble. + #'preamble': '', + # Latex figure (float) alignment + #'figure_align': 'htbp', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'pyhaystack.tex', 'pyhaystack Documentation', - 'Christian Tremblay, P.Eng.', 'manual'), + ( + master_doc, + "pyhaystack.tex", + "pyhaystack Documentation", + "Christian Tremblay, P.Eng.", + "manual", + ) ] # The name of an image file (relative to this directory) to place at the top of # the title page. -#latex_logo = None +# latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. -#latex_use_parts = False +# latex_use_parts = False # If true, show page references after internal links. -#latex_show_pagerefs = False +# latex_show_pagerefs = False # If true, show URL addresses after external links. -#latex_show_urls = False +# latex_show_urls = False # Documents to append as an appendix to all manuals. -#latex_appendices = [] +# latex_appendices = [] # If false, no module index is generated. -#latex_domain_indices = True +# latex_domain_indices = True # -- Options for manual page output --------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, 'pyhaystack', 'pyhaystack Documentation', - [author], 1) -] +man_pages = [(master_doc, "pyhaystack", "pyhaystack Documentation", [author], 1)] # If true, show URL addresses after external links. -#man_show_urls = False +# man_show_urls = False # -- Options for Texinfo output ------------------------------------------- @@ -277,22 +276,28 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'pyhaystack', 'pyhaystack Documentation', - author, 'pyhaystack', 'One line description of project.', - 'Miscellaneous'), + ( + master_doc, + "pyhaystack", + "pyhaystack Documentation", + author, + "pyhaystack", + "One line description of project.", + "Miscellaneous", + ) ] # Documents to append as an appendix to all manuals. -#texinfo_appendices = [] +# texinfo_appendices = [] # If false, no module index is generated. -#texinfo_domain_indices = True +# texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. -#texinfo_show_urls = 'footnote' +# texinfo_show_urls = 'footnote' # If true, do not generate a @detailmenu in the "Top" node's menu. -#texinfo_no_detailmenu = False +# texinfo_no_detailmenu = False # -- Options for Epub output ---------------------------------------------- @@ -304,66 +309,66 @@ epub_copyright = copyright # The basename for the epub file. It defaults to the project name. -#epub_basename = project +# epub_basename = project # The HTML theme for the epub output. Since the default themes are not optimized # for small screen space, using the same theme for HTML and epub output is # usually not wise. This defaults to 'epub', a theme designed to save visual # space. -#epub_theme = 'epub' +# epub_theme = 'epub' # The language of the text. It defaults to the language option # or 'en' if the language is not set. -#epub_language = '' +# epub_language = '' # The scheme of the identifier. Typical schemes are ISBN or URL. -#epub_scheme = '' +# epub_scheme = '' # The unique identifier of the text. This can be a ISBN number # or the project homepage. -#epub_identifier = '' +# epub_identifier = '' # A unique identification for the text. -#epub_uid = '' +# epub_uid = '' # A tuple containing the cover image and cover page html template filenames. -#epub_cover = () +# epub_cover = () # A sequence of (type, uri, title) tuples for the guide element of content.opf. -#epub_guide = () +# epub_guide = () # HTML files that should be inserted before the pages created by sphinx. # The format is a list of tuples containing the path and title. -#epub_pre_files = [] +# epub_pre_files = [] # HTML files shat should be inserted after the pages created by sphinx. # The format is a list of tuples containing the path and title. -#epub_post_files = [] +# epub_post_files = [] # A list of files that should not be packed into the epub file. -epub_exclude_files = ['search.html'] +epub_exclude_files = ["search.html"] # The depth of the table of contents in toc.ncx. -#epub_tocdepth = 3 +# epub_tocdepth = 3 # Allow duplicate toc entries. -#epub_tocdup = True +# epub_tocdup = True # Choose between 'default' and 'includehidden'. -#epub_tocscope = 'default' +# epub_tocscope = 'default' # Fix unsupported image types using the Pillow. -#epub_fix_images = False +# epub_fix_images = False # Scale large images. -#epub_max_image_width = 0 +# epub_max_image_width = 0 # How to display URL addresses: 'footnote', 'no', or 'inline'. -#epub_show_urls = 'inline' +# epub_show_urls = 'inline' # If false, no index is generated. -#epub_use_index = True +# epub_use_index = True # Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {'https://docs.python.org/': None} +intersphinx_mapping = {"https://docs.python.org/": None} diff --git a/mkdeb.sh b/mkdeb.sh new file mode 100644 index 0000000..ceda0a3 --- /dev/null +++ b/mkdeb.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# Build a Debian package of pyhaystack. +set -e + +: ${MY_DIR:=$( dirname "$0" )} +: ${PYTHON:=$( which python2 )} + +: ${BUILD_PY2:=True} +: ${BUILD_PY3:=True} + +# Set the output directory if not already given +: ${OUT_DIR:=${MY_DIR}/out} + +cd "${MY_DIR}" + +# Clean up +[ ! -d deb_dist ] || rm -fr deb_dist +[ ! -d dist ] || rm -fr dist + +# Build +"${PYTHON}" setup.py \ + --command-package stdeb.command sdist_dsc \ + --with-python2=${BUILD_PY2} --with-python3=${BUILD_PY3} \ + ${DEBIAN_VERSION:+--debian-version=}${DEBIAN_VERSION} \ + bdist_deb + +# Clean up source tree +find deb_dist -mindepth 1 -maxdepth 1 -type d | xargs rm -fr + +# Move out the resultant files +[ -d ${OUT_DIR} ] || mkdir ${OUT_DIR} +mv deb_dist/* ${OUT_DIR} diff --git a/pyhaystack/__init__.py b/pyhaystack/__init__.py index 47e3655..7e1ac55 100644 --- a/pyhaystack/__init__.py +++ b/pyhaystack/__init__.py @@ -13,10 +13,12 @@ try: from hszinc import Quantity, use_pint + Q_ = Quantity from .client.loader import get_instance as connect import requests.packages.urllib3 from requests.packages.urllib3.exceptions import SubjectAltNameWarning + requests.packages.urllib3.disable_warnings(SubjectAltNameWarning) diff --git a/pyhaystack/client/__init__.py b/pyhaystack/client/__init__.py index 521ce1e..cd018a1 100644 --- a/pyhaystack/client/__init__.py +++ b/pyhaystack/client/__init__.py @@ -6,4 +6,4 @@ from .loader import get_implementation, get_instance -__all__ = ['get_implementation', 'get_instance'] +__all__ = ["get_implementation", "get_instance"] diff --git a/pyhaystack/client/entity/entity.py b/pyhaystack/client/entity/entity.py index a849139..1d06f13 100644 --- a/pyhaystack/client/entity/entity.py +++ b/pyhaystack/client/entity/entity.py @@ -8,6 +8,7 @@ from hszinc import Ref from .tags import ReadOnlyEntityTags, MutableEntityTags + class Entity(object): """ A base class for Project Haystack entities. This is a base class that is @@ -30,7 +31,7 @@ def __init__(self, session, entity_id): self._session = session self._entity_id = entity_id - if hasattr(session, 'update'): + if hasattr(session, "update"): tags = MutableEntityTags(self) else: tags = ReadOnlyEntityTags(self) @@ -50,7 +51,7 @@ def dis(self): """ Return the description field of the entity. """ - return self._tags['dis'] + return self._tags["dis"] @property def tags(self): @@ -63,15 +64,16 @@ def __repr__(self): """ Return a string representation of the entity. """ - return '<%s: %s>' % (self.id, self.tags) + return "<%s: %s>" % (self.id, self.tags) def _update_tags(self, tags): """ Update the value of given tags. """ self._tags._update_tags(tags) - if hasattr(self._session, '_check_entity_type') \ - and (not self._session._check_entity_type(self)): + if hasattr(self._session, "_check_entity_type") and ( + not self._session._check_entity_type(self) + ): self._invalidate() def _invalidate(self): @@ -96,7 +98,7 @@ def delete(self, callback=None): """ if not self._valid: raise StaleEntityInstanceError() - raise NotImplementedError('TODO: implement CRUD ops') + raise NotImplementedError("TODO: implement CRUD ops") class StaleEntityInstanceError(Exception): @@ -104,4 +106,5 @@ class StaleEntityInstanceError(Exception): Exception thrown when an entity instance is "stale", that is, the entity class type no longer matches the tag set present in the entity. """ + pass diff --git a/pyhaystack/client/entity/mixins/equip.py b/pyhaystack/client/entity/mixins/equip.py index a1dee4f..bddbdc9 100644 --- a/pyhaystack/client/entity/mixins/equip.py +++ b/pyhaystack/client/entity/mixins/equip.py @@ -8,25 +8,25 @@ import hszinc from ....exception import HaystackError + class EquipMixin(object): """ A mix-in used for entities that carry the 'equip' marker tag. """ - def find_entity(self, filter_expr=None, limit=None, - single=False, callback=None): + def find_entity(self, filter_expr=None, limit=None, single=False, callback=None): """ Retrieve the entities that are linked to this equipment. This is a convenience around the session find_entity method. """ equip_ref = hszinc.dump_scalar(self.id) if filter_expr is None: - filter_expr = 'equipRef==%s' % equip_ref + filter_expr = "equipRef==%s" % equip_ref else: - filter_expr = '(equipRef==%s) and (%s)' % (equip_ref, filter_expr) + filter_expr = "(equipRef==%s) and (%s)" % (equip_ref, filter_expr) return self._session.find_entity(filter_expr, limit, single, callback) - - def __getitem__(self,key): + + def __getitem__(self, key): """ In a navigation context, component of an equipment is a point (tag/entity) """ @@ -38,35 +38,35 @@ def __getitem__(self,key): # if key not found in tags... we probably are searching a point # self will call __iter__ which will look for points in equipment for point in self: - #partial_results = [] + # partial_results = [] # Given an ID.... should return the point with this ID - if key.replace('@','') == str(point.id).replace('@',''): + if key.replace("@", "") == str(point.id).replace("@", ""): return point # Given a dis or navName... should return equip - if 'dis' in each.tags: - if key == each.tags['dis']: + if "dis" in each.tags: + if key == each.tags["dis"]: return each - if 'navName' in each.tags: - if key == each.tags['navName']: + if "navName" in each.tags: + if key == each.tags["navName"]: return each - if 'navNameFormat' in each.tags: - if key == each.tags['navNameFormat']: + if "navNameFormat" in each.tags: + if key == each.tags["navNameFormat"]: return each - else: + else: try: # Maybe key is a filter_expr request = self.find_entity(key) return request.result except HaystackError as e: - self._session._log.warning('{} not found'.format(key)) - + self._session._log.warning("{} not found".format(key)) + def __iter__(self): """ When iterating over an equipment, we iterate points. """ for point in self.points: yield point - + @property def points(self): """ @@ -75,7 +75,7 @@ def points(self): try: return self._list_of_points except AttributeError: - print('Reading points for this equipment...') + print("Reading points for this equipment...") self._add_points() return self._list_of_points @@ -85,15 +85,15 @@ def refresh(self): """ self._list_of_points = [] self._add_points() - + def _add_points(self): """ Store a local copy of equip for this site To accelerate browser """ - if not '_list_of_points' in self.__dict__.keys(): - self._list_of_points = [] - for point in self['point'].items(): + if not "_list_of_points" in self.__dict__.keys(): + self._list_of_points = [] + for point in self["point"].items(): self._list_of_points.append(point[1]) @@ -106,5 +106,6 @@ def get_equip(self, callback=None): """ Retrieve an instance of the equip this entity is linked to. """ - return self._session.get_entity(self.tags['equipRef'], - callback=callback, single=True) + return self._session.get_entity( + self.tags["equipRef"], callback=callback, single=True + ) diff --git a/pyhaystack/client/entity/mixins/point.py b/pyhaystack/client/entity/mixins/point.py index eca4333..5c39aca 100644 --- a/pyhaystack/client/entity/mixins/point.py +++ b/pyhaystack/client/entity/mixins/point.py @@ -5,15 +5,19 @@ 'point' related mix-ins for high-level interface. """ + class HisMixin(object): """ A mix-in used for 'point' entities that carry the 'his' marker tag. """ - def his(self, rng='today', tz=None, series_format=None, callback=None): + + def his(self, rng="today", tz=None, series_format=None, callback=None): """ Shortcut to read_series """ - return self.his_read_series(rng=rng, tz=tz, series_format=series_format, callback=callback) + return self.his_read_series( + rng=rng, tz=tz, series_format=series_format, callback=callback + ) def his_read_series(self, rng, tz=None, series_format=None, callback=None): """ @@ -23,8 +27,9 @@ def his_read_series(self, rng, tz=None, series_format=None, callback=None): :param tz: Optional timezone to translate timestamps to :param series_format: Optional desired format for the series """ - return self._session.his_read_series(point=self, rng=rng, - tz=tz, series_format=series_format, callback=callback) + return self._session.his_read_series( + point=self, rng=rng, tz=tz, series_format=series_format, callback=callback + ) def his_write_series(self, series, tz=None, callback=None): """ @@ -33,10 +38,12 @@ def his_write_series(self, series, tz=None, callback=None): :param series: Historical series to write :param tz: Optional timezone to translate timestamps to """ - return self._session.his_write_series(point=self, series=series, - tz=tz, callback=callback) + return self._session.his_write_series( + point=self, series=series, tz=tz, callback=callback + ) + -class PointMixin(object): - @property +class PointMixin(object): + @property def value(self): - return (self._session.read(ids=self.id).result)[0]['curVal'] + return (self._session.read(ids=self.id).result)[0]["curVal"] diff --git a/pyhaystack/client/entity/mixins/site.py b/pyhaystack/client/entity/mixins/site.py index c733508..16d6571 100644 --- a/pyhaystack/client/entity/mixins/site.py +++ b/pyhaystack/client/entity/mixins/site.py @@ -8,25 +8,24 @@ import hszinc from ....exception import HaystackError + class SiteMixin(object): """ A mix-in used for entities that carry the 'site' marker tag. """ - def find_entity(self, filter_expr=None, single=False, - limit=None, callback=None): + def find_entity(self, filter_expr=None, single=False, limit=None, callback=None): """ Retrieve the entities that are linked to this site. This is a convenience around the session find_entity method. """ site_ref = hszinc.dump_scalar(self.id) if filter_expr is None: - filter_expr = 'siteRef==%s' % site_ref + filter_expr = "siteRef==%s" % site_ref else: - filter_expr = '(siteRef==%s) and (%s)' % (site_ref, filter_expr) + filter_expr = "(siteRef==%s) and (%s)" % (site_ref, filter_expr) return self._session.find_entity(filter_expr, limit, single, callback) - def __getitem__(self, key): """ A site is typically the first level object... under it there are @@ -48,26 +47,26 @@ def __getitem__(self, key): # self will call __iter__ which will look for equipments for each in self: # Given an ID.... should return the equip with this ID - - if key.replace('@','') == str(each.id).replace('@',''): + + if key.replace("@", "") == str(each.id).replace("@", ""): return each # Given a dis or navName... should return equip - if 'dis' in each.tags: - if key == each.tags['dis']: + if "dis" in each.tags: + if key == each.tags["dis"]: return each - if 'navName' in each.tags: - if key == each.tags['navName']: + if "navName" in each.tags: + if key == each.tags["navName"]: return each - if 'navNameFormat' in each.tags: - if key == each.tags['navNameFormat']: + if "navNameFormat" in each.tags: + if key == each.tags["navNameFormat"]: return each - else: + else: try: # Maybe key is a filter_expr request = self.find_entity(key) return request.result except HaystackError as e: - self._session._log.warning('{} not found'.format(key)) + self._session._log.warning("{} not found".format(key)) def __iter__(self): """ @@ -85,7 +84,7 @@ def __iter__(self): """ for equip in self.equipments: yield equip - + @property def equipments(self): """ @@ -97,27 +96,27 @@ def equipments(self): # At first, this variable will not exist... will be created return self._list_of_equip except AttributeError: - print('Reading equipments for this site...') + print("Reading equipments for this site...") self._add_equip() return self._list_of_equip - + def refresh(self): """ Re-create local list of equipments """ self._list_of_equip = [] self._add_equip() - + def _add_equip(self): """ Store a local copy of equip names for this site To accelerate browser """ - if not '_list_of_equip' in self.__dict__.keys(): - self._list_of_equip = [] - for equip in self['equip'].items(): + if not "_list_of_equip" in self.__dict__.keys(): + self._list_of_equip = [] + for equip in self["equip"].items(): self._list_of_equip.append(equip[1]) - + class SiteRefMixin(object): """ @@ -128,5 +127,6 @@ def get_site(self, callback=None): """ Retrieve an instance of the site this entity is linked to. """ - return self._session.get_entity(self.tags['siteRef'], - callback=callback, single=True) + return self._session.get_entity( + self.tags["siteRef"], callback=callback, single=True + ) diff --git a/pyhaystack/client/entity/mixins/tz.py b/pyhaystack/client/entity/mixins/tz.py index e6de278..bcd1eae 100644 --- a/pyhaystack/client/entity/mixins/tz.py +++ b/pyhaystack/client/entity/mixins/tz.py @@ -8,6 +8,7 @@ import hszinc import pytz + class TzMixin(object): """ A mix-in used for entities that carry the 'tz' tag. @@ -18,7 +19,7 @@ def hs_tz(self): """ Return the Project Haystack timezone type. """ - return self.tags['tz'] + return self.tags["tz"] @property def iana_tz(self): @@ -26,8 +27,8 @@ def iana_tz(self): Return the IANA (Olson) database timezone name. """ hs_tz = self.hs_tz - if '/' in hs_tz: - return hs_tz # This is the IANA zone name. + if "/" in hs_tz: + return hs_tz # This is the IANA zone name. tz_map = hszinc.zoneinfo.get_tz_map() return tz_map[hs_tz] diff --git a/pyhaystack/client/entity/model.py b/pyhaystack/client/entity/model.py index 3a0320b..2cdaf00 100644 --- a/pyhaystack/client/entity/model.py +++ b/pyhaystack/client/entity/model.py @@ -8,6 +8,7 @@ import weakref from .entity import Entity, DeletableEntity + class TaggingModel(object): """ A base class for representing tagging models. The tagging model @@ -34,7 +35,7 @@ def create_entity(self, entity_id, tags): # Does the session instance support CRUD? Add the appropriate base # class to the end of the types list. - if hasattr(session, 'delete'): + if hasattr(session, "delete"): types += [DeletableEntity] else: types += [Entity] @@ -57,5 +58,4 @@ def _identify_types(self, tags): - a list of class instances that represent the add-on types for that object. """ - raise NotImplementedError('To be implemented in %s' \ - % self.__class__.__name__) + raise NotImplementedError("To be implemented in %s" % self.__class__.__name__) diff --git a/pyhaystack/client/entity/models/haystack.py b/pyhaystack/client/entity/models/haystack.py index 02f2117..69522a2 100644 --- a/pyhaystack/client/entity/models/haystack.py +++ b/pyhaystack/client/entity/models/haystack.py @@ -9,6 +9,7 @@ from ..model import TaggingModel from ..mixins import tz, site, equip, point + class HaystackTaggingModel(TaggingModel): """ An implementation of the Project Haystack tagging model. @@ -24,33 +25,33 @@ def _identify_types(self, tags): """ types = [] names = [] - if 'tz' in tags: + if "tz" in tags: # We have a timezone types.append(tz.TzMixin) - names.append('Tz') + names.append("Tz") - if 'site' in tags: + if "site" in tags: # This is a site types.append(site.SiteMixin) - names.append('Site') - elif 'siteRef' in tags: + names.append("Site") + elif "siteRef" in tags: # This links to a site types.append(site.SiteRefMixin) - names.append('SiteRef') + names.append("SiteRef") - if 'equip' in tags: + if "equip" in tags: # This is a site types.append(equip.EquipMixin) - names.append('Equip') - elif 'equipRef' in tags: + names.append("Equip") + elif "equipRef" in tags: # This links to an equip types.append(equip.EquipRefMixin) - names.append('EquipRef') + names.append("EquipRef") - if 'point' in tags: + if "point" in tags: types.append(point.PointMixin) - if 'his' in tags: + if "his" in tags: types.append(point.HisMixin) - names.append('His') + names.append("His") - return ('%sEntity' % ''.join(sorted(names)), types) + return ("%sEntity" % "".join(sorted(names)), types) diff --git a/pyhaystack/client/entity/ops/crud.py b/pyhaystack/client/entity/ops/crud.py index 5e64930..4adcccd 100644 --- a/pyhaystack/client/entity/ops/crud.py +++ b/pyhaystack/client/entity/ops/crud.py @@ -27,20 +27,21 @@ def __init__(self, entity, updates): :param session: Haystack HTTP session object. """ super(EntityTagUpdateOperation, self).__init__(result_copy=False) - self._log = entity._session._log.getChild('update_tags') + self._log = entity._session._log.getChild("update_tags") self._entity = entity self._updates = updates self._state_machine = fysom.Fysom( - initial='init', final='done', - events=[ - # Event Current State New State - ('do_update', 'init', 'update'), - ('update_done', 'update', 'done'), - ('exception', '*', 'done'), - ], callbacks={ - 'onenterdone': self._do_done, - }) + initial="init", + final="done", + events=[ + # Event Current State New State + ("do_update", "init", "update"), + ("update_done", "update", "done"), + ("exception", "*", "done"), + ], + callbacks={"onenterdone": self._do_done}, + ) def go(self): """ @@ -60,19 +61,22 @@ def _on_update(self, operation, **kwargs): # Iterate over each row: for row in grid: row = row.copy() - entity_id = row.pop('id') - if (entity_id is None) or (entity_id.name != \ - self._entity.id.name): + entity_id = row.pop("id") + if (entity_id is None) or (entity_id.name != self._entity.id.name): # Not for us! - self._log.debug('Ignoring row (%s does not match %s) %r', - entity_id, self._entity.id, row) + self._log.debug( + "Ignoring row (%s does not match %s) %r", + entity_id, + self._entity.id, + row, + ) continue self._entity.tags._update_tags(row) self._entity.tags.revert() - self._log.debug('Processed row %r', row) + self._log.debug("Processed row %r", row) self._state_machine.update_done(result=self._entity) - except: # Catch all exceptions to pass to caller. + except: # Catch all exceptions to pass to caller. self._state_machine.exception(result=AsynchronousException()) def _do_done(self, event): diff --git a/pyhaystack/client/entity/tags.py b/pyhaystack/client/entity/tags.py index 8c98bcc..999b092 100644 --- a/pyhaystack/client/entity/tags.py +++ b/pyhaystack/client/entity/tags.py @@ -7,11 +7,16 @@ """ import hszinc -import collections + +try: + import collections.abc as col +except ImportError: + import collections as col import weakref from ...util.asyncexc import AsynchronousException from .ops.crud import EntityTagUpdateOperation + class BaseEntityTags(object): """ A base class for storing entity tags. @@ -32,17 +37,19 @@ def __repr__(self): """ Dump a string representation of the tags. """ + def _dump_tag(ti): (t, v) = ti if v is hszinc.MARKER: return t elif v is hszinc.REMOVE: - return 'R(%s)' % t + return "R(%s)" % t else: - return '%s=%r' % (t, v) + return "%s=%r" % (t, v) + tags = list(map(_dump_tag, self.items())) tags.sort() - return '{%s}' % ', '.join(tags) + return "{%s}" % ", ".join(tags) def __iter__(self): """ @@ -104,7 +111,7 @@ def commit(self, callback=None): """ entity = self._entity() updates = self._tag_updates.copy() - updates['id'] = entity.id + updates["id"] = entity.id for tag in self._tag_deletions: updates[tag] = hszinc.REMOVE @@ -173,13 +180,14 @@ def _tag_names(self): """ Return a set of tag names present. """ - return (set(self._tags.keys()) | set(self._tag_updates.keys())) \ - - self._tag_deletions + return ( + set(self._tags.keys()) | set(self._tag_updates.keys()) + ) - self._tag_deletions -class ReadOnlyEntityTags(BaseEntityTags, collections.Mapping): +class ReadOnlyEntityTags(BaseEntityTags, col.Mapping): pass -class MutableEntityTags(BaseMutableEntityTags, collections.MutableMapping): +class MutableEntityTags(BaseMutableEntityTags, col.MutableMapping): pass diff --git a/pyhaystack/client/http/auth.py b/pyhaystack/client/http/auth.py index 36a036a..c685590 100644 --- a/pyhaystack/client/http/auth.py +++ b/pyhaystack/client/http/auth.py @@ -4,10 +4,12 @@ containers for authentication methods defined in the HTTP spec. """ + class AuthenticationCredentials(object): """ A base class to represent authentication credentials. """ + pass @@ -15,6 +17,7 @@ class UserPasswordAuthenticationCredentials(AuthenticationCredentials): """ A base class that represents username/password type authentication. """ + def __init__(self, username, password): self.username = username self.password = password @@ -24,6 +27,7 @@ class BasicAuthenticationCredentials(UserPasswordAuthenticationCredentials): """ A class that represents Basic authentication. """ + pass @@ -31,4 +35,5 @@ class DigestAuthenticationCredentials(UserPasswordAuthenticationCredentials): """ A class that represents Digest authentication. """ + pass diff --git a/pyhaystack/client/http/base.py b/pyhaystack/client/http/base.py index e47bfbf..c6d5bbb 100644 --- a/pyhaystack/client/http/base.py +++ b/pyhaystack/client/http/base.py @@ -10,6 +10,7 @@ import shlex import re + try: from urllib.parse import quote_plus except ImportError: @@ -22,6 +23,7 @@ from .auth import AuthenticationCredentials + class HTTPClient(object): """ The base HTTP client interface. This class defines methods for making @@ -29,14 +31,25 @@ class HTTPClient(object): asynchronous one, even for synchronous implementations. """ - PROTO_RE = re.compile(r'^[a-z]+://') - CONTENT_TYPE_HDR = b'Content-Type' - CONTENT_LENGTH_HDR = b'Content-Length' - - def __init__(self, uri=None, params=None, headers=None, cookies=None, - auth=None, timeout=None, proxies=None, tls_verify=None, - tls_cert=None, accept_status=None, log=None, - insecure_requests_warning=True): + PROTO_RE = re.compile(r"^[a-z]+://") + CONTENT_TYPE_HDR = b"Content-Type" + CONTENT_LENGTH_HDR = b"Content-Length" + + def __init__( + self, + uri=None, + params=None, + headers=None, + cookies=None, + auth=None, + timeout=None, + proxies=None, + tls_verify=None, + tls_cert=None, + accept_status=None, + log=None, + insecure_requests_warning=True, + ): """ Instantiate a HTTP client instance with some default parameters. These parameters are made accessible as properties to be modified at @@ -80,15 +93,30 @@ def __init__(self, uri=None, params=None, headers=None, cookies=None, self.tls_verify = tls_verify self.tls_cert = tls_cert self.log = log - + if not insecure_requests_warning: self.silence_insecured_warnings() - def request(self, method, uri, callback, body=None, params=None, - headers=None, cookies=None, auth=None, timeout=None, proxies=None, - tls_verify=None, tls_cert=None, exclude_params=None, - exclude_headers=None, exclude_cookies=None, exclude_proxies=None, - accept_status=None): + def request( + self, + method, + uri, + callback, + body=None, + params=None, + headers=None, + cookies=None, + auth=None, + timeout=None, + proxies=None, + tls_verify=None, + tls_cert=None, + exclude_params=None, + exclude_headers=None, + exclude_cookies=None, + exclude_proxies=None, + accept_status=None, + ): """ Perform a request with this client. Most parameters here exist to either add to or override the defaults given by the client attributes. The @@ -146,8 +174,7 @@ def request(self, method, uri, callback, body=None, params=None, if not self.PROTO_RE.match(uri): # Do we have a base URL? if self.uri is None: - raise ValueError('uri must be absolute or base '\ - 'set in uri attribute') + raise ValueError("uri must be absolute or base " "set in uri attribute") # Prepend our base URL uri = urljoin(self.uri, uri) @@ -164,8 +191,13 @@ def _merge(given, defaults, exclude): result.update(given) if self.log is not None: - self.log.debug('Merging %r with %r, exclude %s -> %r', - given, defaults, exclude, result) + self.log.debug( + "Merging %r with %r, exclude %s -> %r", + given, + defaults, + exclude, + result, + ) return result # Merge our parameters, headers and cookies together @@ -177,48 +209,71 @@ def _merge(given, defaults, exclude): timeout = timeout or self.timeout or None if not ((auth is None) or isinstance(auth, AuthenticationCredentials)): - raise TypeError('%s is not a subclass of the '\ - 'AuthenticationCredentials class.' \ - % auth.__class__.__name__) + raise TypeError( + "%s is not a subclass of the " + "AuthenticationCredentials class." % auth.__class__.__name__ + ) if tls_verify is None: tls_verify = self.tls_verify - if (tls_verify is None) and uri.startswith('https://'): + if (tls_verify is None) and uri.startswith("https://"): # If we're dealing with a https:// URL, turn on verification # by default for user safety. tls_verify = True tls_cert = tls_cert or self.tls_cert or None # Convert parameters to a query string - query_str = u'&'.join([ - '%s=%s' % (key, quote_plus(value)) - for key, value in params.items() - ]) + query_str = "&".join( + ["%s=%s" % (key, quote_plus(value)) for key, value in params.items()] + ) # Tack query string onto URL if query_str: - uri += u'?' + query_str + uri += "?" + query_str # Perform the actual request. if self.log is not None: - self.log.debug( 'Performing operation %s of %s, headers: %r, '\ - 'cookies: %r, body: %r', method, uri, headers, - cookies, body) - self._request(method=method, uri=uri, callback=callback, body=body, - headers=headers, cookies=cookies, auth=auth, - timeout=timeout, proxies=proxies, tls_verify=tls_verify, - tls_cert=tls_cert, accept_status=accept_status) + self.log.debug( + "Performing operation %s of %s, headers: %r, " "cookies: %r, body: %r", + method, + uri, + headers, + cookies, + body, + ) + self._request( + method=method, + uri=uri, + callback=callback, + body=body, + headers=headers, + cookies=cookies, + auth=auth, + timeout=timeout, + proxies=proxies, + tls_verify=tls_verify, + tls_cert=tls_cert, + accept_status=accept_status, + ) def get(self, uri, callback, **kwargs): """ Convenience function: perform a HTTP GET operation. Arguments are the same as for request. """ - kwargs.pop('body',None) - self.request('GET', uri, callback, **kwargs) - - def post(self, uri, callback, body=None, body_type=None, body_size=None, - headers=None, **kwargs): + kwargs.pop("body", None) + self.request("GET", uri, callback, **kwargs) + + def post( + self, + uri, + callback, + body=None, + body_type=None, + body_size=None, + headers=None, + **kwargs + ): """ Convenience function: perform a HTTP POST operation. Arguments are the same as for request. @@ -242,20 +297,37 @@ def post(self, uri, callback, body=None, body_type=None, body_size=None, if body_type is not None: headers[self.CONTENT_TYPE_HDR] = body_type - self.request(method='POST', uri=uri, callback=callback, - body=body, headers=headers, **kwargs) - - def _request(self, method, uri, callback, body, - headers, cookies, auth, timeout, proxies, - tls_verify, tls_cert, accept_status): + self.request( + method="POST", + uri=uri, + callback=callback, + body=body, + headers=headers, + **kwargs + ) + + def _request( + self, + method, + uri, + callback, + body, + headers, + cookies, + auth, + timeout, + proxies, + tls_verify, + tls_cert, + accept_status, + ): """ Perform a HTTP request using the underlying implementation. This is expected to take the arguments given, perform a query, then return the result via a callback. """ - raise NotImplementedError('TODO: implement in %s' \ - % self.__class__.__name__) - + raise NotImplementedError("TODO: implement in %s" % self.__class__.__name__) + def silence_insecured_warnings(self): """ Can be used to disable Insecure Requests Warnings @@ -263,19 +335,24 @@ def silence_insecured_warnings(self): Use with care. """ if self.log is not None: - self.log.warning('Disabling insecure requests warnings. Please use with care. Unverified HTTPS requests will be made. Adding Certificate verification is strongly advised.') + self.log.warning( + "Disabling insecure requests warnings. Please use with care. Unverified HTTPS requests will be made. Adding Certificate verification is strongly advised." + ) try: import requests from requests.packages.urllib3.exceptions import InsecureRequestWarning + requests.packages.urllib3.disable_warnings(InsecureRequestWarning) except ImportError: raise + class HTTPResponse(object): """ A class that represents the raw response from a HTTP request. """ + def __init__(self, status_code, headers, body, cookies=None): self.status_code = status_code self.headers = CaseInsensitiveDict(headers or {}) @@ -310,11 +387,11 @@ def text(self): """ if self._text is None: body = self.body - if not hasattr(body, 'decode'): + if not hasattr(body, "decode"): # It probably is a str/unicode return body - content_encoding = self.content_type_args.get('charset') + content_encoding = self.content_type_args.get("charset") if content_encoding is None: self._text = self.body.decode() else: @@ -322,14 +399,15 @@ def text(self): return self._text def _parse_content_type(self): - content_type = self.headers['content-type'] + content_type = self.headers["content-type"] # Is content encoding shoehorned in there? - if ';' in content_type: - (content_type, content_type_args) = content_type.split(';',1) + if ";" in content_type: + (content_type, content_type_args) = content_type.split(";", 1) content_type = content_type.strip() - content_type_args = dict([tuple(kv.split('=',1)) for kv in - shlex.split(content_type_args)]) + content_type_args = dict( + [tuple(kv.split("=", 1)) for kv in shlex.split(content_type_args)] + ) else: content_type_args = {} self._content_type = content_type @@ -340,11 +418,12 @@ class CaseInsensitiveDict(dict): """ A dict object that maps keys in a case-insensitive manner. """ + @classmethod def _key_to_str(cls, key): # Handle bytes if isinstance(key, bytes): - key = key.decode('utf-8') + key = key.decode("utf-8") return str(key).lower() def __init__(self, *args, **kwargs): @@ -356,15 +435,12 @@ def __getitem__(self, key, *args, **kwargs): key = self._key_map[self._key_to_str(key)] except KeyError: pass - return super(CaseInsensitiveDict, self).__getitem__( - key, *args, **kwargs) + return super(CaseInsensitiveDict, self).__getitem__(key, *args, **kwargs) def __setitem__(self, key, *args, **kwargs): self._key_map[self._key_to_str(key)] = key - return super(CaseInsensitiveDict, self).__setitem__( - key, *args, **kwargs) + return super(CaseInsensitiveDict, self).__setitem__(key, *args, **kwargs) def __delitem__(self, key, *args, **kwargs): self._key_map.pop(str(key).lower(), None) - return super(CaseInsensitiveDict, self).__delitem__( - key, *args, **kwargs) + return super(CaseInsensitiveDict, self).__delitem__(key, *args, **kwargs) diff --git a/pyhaystack/client/http/dummy.py b/pyhaystack/client/http/dummy.py index 691c727..efbc3f9 100644 --- a/pyhaystack/client/http/dummy.py +++ b/pyhaystack/client/http/dummy.py @@ -4,10 +4,14 @@ """ from .base import HTTPClient, HTTPResponse -from .auth import BasicAuthenticationCredentials, \ - DigestAuthenticationCredentials -from .exceptions import HTTPConnectionError, HTTPTimeoutError, \ - HTTPRedirectError, HTTPStatusError, HTTPBaseError +from .auth import BasicAuthenticationCredentials, DigestAuthenticationCredentials +from .exceptions import ( + HTTPConnectionError, + HTTPTimeoutError, + HTTPRedirectError, + HTTPStatusError, + HTTPBaseError, +) from ...util.asyncexc import AsynchronousException @@ -29,18 +33,42 @@ def __init__(self): self._rq_order = [] self._next_id = 0 - def submit_request(self, method, uri, callback, - body, headers, cookies, auth, timeout, proxies, - tls_verify, tls_cert, accept_status): + def submit_request( + self, + method, + uri, + callback, + body, + headers, + cookies, + auth, + timeout, + proxies, + tls_verify, + tls_cert, + accept_status, + ): """ Submit a request. """ rq_id = self._next_id self._next_id += 1 - rq = DummyHttpClientRequest(rq_id, method, uri, callback, - body, headers, cookies, auth, timeout, proxies, - tls_verify, tls_cert, accept_status) + rq = DummyHttpClientRequest( + rq_id, + method, + uri, + callback, + body, + headers, + cookies, + auth, + timeout, + proxies, + tls_verify, + tls_cert, + accept_status, + ) self._requests[rq_id] = rq self._rq_order.append(rq_id) @@ -78,12 +106,35 @@ def __init__(self, server, **kwargs): super(DummyHttpClient, self).__init__(**kwargs) self._server = server - def _request(self, method, uri, callback, body, - headers, cookies, auth, timeout, proxies, - tls_verify, tls_cert, accept_status): - self._server.submit_request(method, - uri, callback, body, headers, cookies, auth, - timeout, proxies, tls_verify, tls_cert, accept_status) + def _request( + self, + method, + uri, + callback, + body, + headers, + cookies, + auth, + timeout, + proxies, + tls_verify, + tls_cert, + accept_status, + ): + self._server.submit_request( + method, + uri, + callback, + body, + headers, + cookies, + auth, + timeout, + proxies, + tls_verify, + tls_cert, + accept_status, + ) class DummyHttpClientRequest(object): @@ -94,9 +145,22 @@ class DummyHttpClientRequest(object): response to the waiting client. """ - def __init__(self, rq_id, method, uri, callback, body, - headers, cookies, auth, timeout, proxies, - tls_verify, tls_cert, accept_status): + def __init__( + self, + rq_id, + method, + uri, + callback, + body, + headers, + cookies, + auth, + timeout, + proxies, + tls_verify, + tls_cert, + accept_status, + ): """ Collect all the parameters supplied in the request. """ @@ -166,10 +230,11 @@ def __str__(self): """ Return a string representation for debugging purposes. """ - return 'Request %d: %s of %s\n'\ - '\tHeaders: %s\n'\ - '\tBody:\n%s' % (self.rq_id, self.method, self.uri, - self.headers, self.body) + return ( + "Request %d: %s of %s\n" + "\tHeaders: %s\n" + "\tBody:\n%s" % (self.rq_id, self.method, self.uri, self.headers, self.body) + ) def __hash__(self): """ @@ -186,14 +251,15 @@ def respond(self, status, headers, content, cookies=None): if cookies is None: cookies = {} - if ((self._accept_status is None) and (status < 400)) \ - or (status in self._accept_status): - result = HTTPResponse(status, headers.copy(), content, - cookies.copy()) + if ((self._accept_status is None) and (status < 400)) or ( + status in self._accept_status + ): + result = HTTPResponse(status, headers.copy(), content, cookies.copy()) else: try: - raise HTTPStatusError('HTTP Status %d' % status, - status, headers.copy(), content) + raise HTTPStatusError( + "HTTP Status %d" % status, status, headers.copy(), content + ) except: # Catch it for the callback result = AsynchronousException() diff --git a/pyhaystack/client/http/exceptions.py b/pyhaystack/client/http/exceptions.py index e3620ba..7d6feb2 100644 --- a/pyhaystack/client/http/exceptions.py +++ b/pyhaystack/client/http/exceptions.py @@ -3,10 +3,12 @@ HTTP client exception classes. """ + class HTTPBaseError(IOError): """ Error class to represent a HTTP errors. """ + pass @@ -14,6 +16,7 @@ class HTTPConnectionError(HTTPBaseError): """ Error class to represent a failed attempt to connect to a host. """ + pass @@ -21,6 +24,7 @@ class HTTPTimeoutError(HTTPConnectionError): """ Error class to represent that a request timed out. """ + pass @@ -28,6 +32,7 @@ class HTTPRedirectError(HTTPBaseError): """ Error class to represent that the server's redirections are looping. """ + pass @@ -35,10 +40,9 @@ class HTTPStatusError(HTTPBaseError): """ Error class to represent a returned failed status from the host. """ + def __init__(self, message, status, headers=None, body=None): self.headers = headers self.body = body self.status = status super(HTTPStatusError, self).__init__(message, status) - - diff --git a/pyhaystack/client/http/sync.py b/pyhaystack/client/http/sync.py index 76ef8e3..c135db1 100644 --- a/pyhaystack/client/http/sync.py +++ b/pyhaystack/client/http/sync.py @@ -4,71 +4,106 @@ """ from .base import HTTPClient, HTTPResponse -from .auth import BasicAuthenticationCredentials, \ - DigestAuthenticationCredentials -from .exceptions import HTTPConnectionError, HTTPTimeoutError, \ - HTTPRedirectError, HTTPStatusError, HTTPBaseError +from .auth import BasicAuthenticationCredentials, DigestAuthenticationCredentials +from .exceptions import ( + HTTPConnectionError, + HTTPTimeoutError, + HTTPRedirectError, + HTTPStatusError, + HTTPBaseError, +) from ...util.asyncexc import AsynchronousException import requests # Handle different versions of requests -try : +try: from requests.exceptions import SSLError except ImportError: from requests.packages.urllib3.exceptions import SSLError + class SyncHttpClient(HTTPClient): def __init__(self, **kwargs): self._session = requests.Session() super(SyncHttpClient, self).__init__(**kwargs) - def _request(self, method, uri, callback, body, - headers, cookies, auth, timeout, proxies, - tls_verify, tls_cert, accept_status): + def _request( + self, + method, + uri, + callback, + body, + headers, + cookies, + auth, + timeout, + proxies, + tls_verify, + tls_cert, + accept_status, + ): if auth is not None: if isinstance(auth, BasicAuthenticationCredentials): - auth = requests.auth.HTTPBasicAuth( - auth.username, auth.password) + auth = requests.auth.HTTPBasicAuth(auth.username, auth.password) elif isinstance(auth, DigestAuthenticationCredentials): - auth = requests.auth.HTTPDigestAuth( - auth.username, auth.password) + auth = requests.auth.HTTPDigestAuth(auth.username, auth.password) else: raise NotImplementedError( - '%s does not implement support for %s' % ( - self.__class__.__name__, - auth.__class__.__name__)) + "%s does not implement support for %s" + % (self.__class__.__name__, auth.__class__.__name__) + ) try: try: try: response = self._session.request( - method=method, url=uri, data=body, - headers=headers, cookies=cookies, - auth=auth, timeout=timeout, - proxies=proxies, verify=tls_verify, - cert=tls_cert) - if (accept_status is None) or \ - (response.status_code not in accept_status): + method=method, + url=uri, + data=body, + headers=headers, + cookies=cookies, + auth=auth, + timeout=timeout, + proxies=proxies, + verify=tls_verify, + cert=tls_cert, + ) + if (accept_status is None) or ( + response.status_code not in accept_status + ): response.raise_for_status() except SSLError as e: if self.log is not None: - self.log.warning('Problem with the certificate : %s', e) - self.log.warning('You can use http_args={"tls_verify":False} to validate issue.') + self.log.warning("Problem with the certificate : %s", e) + self.log.warning( + 'You can use http_args={"tls_verify":False} to validate issue.' + ) raise except Exception as e: if self.log is not None: - self.log.debug('Exception in request %s of %s with '\ - 'body %r, headers %r, cookies %r, auth %r', - method, uri, body, headers, cookies, auth, - exc_info=1) + self.log.debug( + "Exception in request %s of %s with " + "body %r, headers %r, cookies %r, auth %r", + method, + uri, + body, + headers, + cookies, + auth, + exc_info=1, + ) raise except requests.exceptions.HTTPError as e: - raise HTTPStatusError(e.args[0], e.response.status_code, \ - dict(e.response.headers), e.response.content) + raise HTTPStatusError( + e.args[0], + e.response.status_code, + dict(e.response.headers), + e.response.content, + ) except requests.exceptions.Timeout as e: raise HTTPTimeoutError(e.strerror) except requests.exceptions.TooManyRedirects as e: @@ -79,17 +114,19 @@ def _request(self, method, uri, callback, body, # TODO: handle this with a more specific exception raise HTTPBaseError(e.message) - result = HTTPResponse(response.status_code, - dict(response.headers), response.content, - dict(response.cookies)) + result = HTTPResponse( + response.status_code, + dict(response.headers), + response.content, + dict(response.cookies), + ) except Exception as e: # Catch all exceptions and forward those to the callback function result = AsynchronousException() try: callback(result) - except: # pragma: no cover + except: # pragma: no cover # This should not happen! if self.log: - self.log.exception('Failure in callback with result: %r', - result) + self.log.exception("Failure in callback with result: %r", result) diff --git a/pyhaystack/client/loader.py b/pyhaystack/client/loader.py index 86e60f8..d72e35a 100644 --- a/pyhaystack/client/loader.py +++ b/pyhaystack/client/loader.py @@ -12,19 +12,20 @@ # IMPLEMENTATION ALIASES: These help map short-hand aliases to full session # instances. Further aliases can be added here. IMPLEMENTATION_ALIAS = { - 'niagara-ax': 'niagara.NiagaraHaystackSession', - 'ax': 'niagara.NiagaraHaystackSession', - 'niagara4': 'niagara.Niagara4HaystackSession', - 'n4': 'niagara.Niagara4HaystackSession', - 'skyspark2': 'skyspark.SkysparkHaystackSession', - 'skyspark': 'skyspark.SkysparkScramHaystackSession', - 'widesky': 'widesky.WideskyHaystackSession', + "niagara-ax": "niagara.NiagaraHaystackSession", + "ax": "niagara.NiagaraHaystackSession", + "niagara4": "niagara.Niagara4HaystackSession", + "n4": "niagara.Niagara4HaystackSession", + "skyspark2": "skyspark.SkysparkHaystackSession", + "skyspark": "skyspark.SkysparkScramHaystackSession", + "widesky": "widesky.WideskyHaystackSession", } # KNOWN IMPLEMENTATIONS: This is populated at run time with instances of session # classes as they are loaded. It should be empty at first. _known_implementations = {} + def get_implementation(implementation): """ Get an implementation of Project Haystack session manager based on @@ -41,14 +42,13 @@ def get_implementation(implementation): pass # Extract class name from implementation - implementation_parts = implementation.split('.') + implementation_parts = implementation.split(".") implementation_class = implementation_parts.pop() - implementation_mod = '.'.join(implementation_parts) + implementation_mod = ".".join(implementation_parts) # Try short name, e.g. widesky.WideskySession try: - mod = import_module('.%s' % implementation_mod, - package='pyhaystack.client') + mod = import_module(".%s" % implementation_mod, package="pyhaystack.client") except ImportError: # Nope, not a short name, try the absolute full name. mod = import_module(implementation_mod) @@ -58,14 +58,14 @@ def get_implementation(implementation): impl = getattr(mod, implementation_class) except AttributeError: # Is it aliased? - if hasattr(mod, 'IMPLEMENTATIONS'): - impl_alias = getattr(mod, 'IMPLEMENTATIONS') + if hasattr(mod, "IMPLEMENTATIONS"): + impl_alias = getattr(mod, "IMPLEMENTATIONS") try: impl = impl_alias[implementation_class] except KeyError: - raise ImportError('No implementation named %s' % implementation) + raise ImportError("No implementation named %s" % implementation) else: - raise ImportError('No implementation named %s' % implementation) + raise ImportError("No implementation named %s" % implementation) _known_implementations[implementation] = impl return impl diff --git a/pyhaystack/client/mixins/vendor/niagara/bql.py b/pyhaystack/client/mixins/vendor/niagara/bql.py index 5024bad..2795c61 100644 --- a/pyhaystack/client/mixins/vendor/niagara/bql.py +++ b/pyhaystack/client/mixins/vendor/niagara/bql.py @@ -18,8 +18,8 @@ # Python 2.7 dinosaur from urllib import quote as quote_uri + class BQLOperation(BaseAuthOperation): - def __init__(self, session, bql, args=None, **kwargs): """ Initialise a GET request for the BQL with the given request and arguments. @@ -28,12 +28,11 @@ def __init__(self, session, bql, args=None, **kwargs): :param bql: BQL Request :param args: Dictionary of key-value pairs to be given as arguments. """ - self._log = session._log.getChild('bql.%s' % bql) - bql_request = 'ord?' + quote_uri(bql) + '%7Cview:file:ITableToCsv' + self._log = session._log.getChild("bql.%s" % bql) + bql_request = "ord?" + quote_uri(bql) + "%7Cview:file:ITableToCsv" self.uri = urljoin(session._uri, bql_request) self._file_like_object = None - super(BQLOperation, self).__init__( - session=session, uri=self.uri,**kwargs) + super(BQLOperation, self).__init__(session=session, uri=self.uri, **kwargs) def _do_submit(self, event): """ @@ -41,12 +40,13 @@ def _do_submit(self, event): """ try: - self._session._get(self._uri, api=False, - headers=self._headers, callback=self._on_response) - except: # Catch all exceptions to pass to caller. - self._log.debug('Get fails', exc_info=1) + self._session._get( + self._uri, api=False, headers=self._headers, callback=self._on_response + ) + except: # Catch all exceptions to pass to caller. + self._log.debug("Get fails", exc_info=1) self._state_machine.exception(result=AsynchronousException()) - + def _on_response(self, response): """ Process the response given back by the HTTP server. @@ -55,7 +55,7 @@ def _on_response(self, response): # Does the session want to invoke any relevant hooks? # This allows a session to detect problems in the session and # abort the operation. - if hasattr(self._session, '_on_http_grid_response'): + if hasattr(self._session, "_on_http_grid_response"): self._session._on_http_grid_response(response) # Process the HTTP error, if any. @@ -64,35 +64,35 @@ def _on_response(self, response): # If we're expecting a raw response back, then just hand the # request object back and finish here. - self._file_like_object = io.StringIO(response.body.decode('UTF-8')) + self._file_like_object = io.StringIO(response.body.decode("UTF-8")) df = pd.read_csv(self._file_like_object) self._state_machine.response_ok(result=df) return - - except: # Catch all exceptions for the caller. - self._log.debug('Parse fails', exc_info=1) + except: # Catch all exceptions for the caller. + self._log.debug("Parse fails", exc_info=1) self._state_machine.exception(result=AsynchronousException()) + class BQLMixin(object): """ This will add function needed to implement the BQL ops for Niagara clients """ + def _get_bql(self, bql, callback, cache=False, **kwargs): """ Perform a HTTP GET of a BQL Request. """ - op = self._BQL_OPERATION(self, bql, - cache=cache, **kwargs) + op = self._BQL_OPERATION(self, bql, cache=cache, **kwargs) if callback is not None: op.done_sig.connect(callback) op.go() return op - - def get_bql(self, bql): + + def get_bql(self, bql): """ Helper to get a BQL sent to the Niagara device - """ + """ return self._get_bql(bql, callback=lambda *a, **k: None) diff --git a/pyhaystack/client/mixins/vendor/niagara/encoding.py b/pyhaystack/client/mixins/vendor/niagara/encoding.py new file mode 100644 index 0000000..aa929d6 --- /dev/null +++ b/pyhaystack/client/mixins/vendor/niagara/encoding.py @@ -0,0 +1,33 @@ +#!python +# -*- coding: utf-8 -*- +""" +Niagara makes some weird thing with encoding. + +""" + +import re + + +class EncodingMixin(object): + """ + This will add functions needed to decode characters coming + from Niagara in its weird format ~2d like + + """ + + @classmethod + def unescape(self, s): + """ + Niagara and nhaystack will spit out ~xy characters + Those are in fact unicode and can be escaped to be + easier to read + + "H.Client.Labo~2f222~2dBA~2fPC_D~e9bit_Alim" + becomes + "H.Client.Labo/222-BA/PC_Débit_Alim" + """ + _s = s + escape = re.compile(r"~(\w\w)") + for each in escape.finditer(_s): + _s = re.sub(each.group(0), chr(int(each.group(1), base=16)), _s) + return _s diff --git a/pyhaystack/client/mixins/vendor/skyspark/evalexpr.py b/pyhaystack/client/mixins/vendor/skyspark/evalexpr.py index adcaed1..427ed76 100644 --- a/pyhaystack/client/mixins/vendor/skyspark/evalexpr.py +++ b/pyhaystack/client/mixins/vendor/skyspark/evalexpr.py @@ -7,6 +7,7 @@ """ + class EvalOpsMixin(object): """ This will add function needed to implement the eval ops @@ -14,8 +15,8 @@ class EvalOpsMixin(object): [ref : https://www.beyon-d.net/doc/docSkySpark/Ops.html] """ - - def get_eval(self, arg_expr): + + def get_eval(self, arg_expr): """ Eval ==== @@ -33,10 +34,11 @@ def get_eval(self, arg_expr): expr "readAll(site)" """ - - url = 'eval?expr=%s' % arg_expr + + url = "eval?expr=%s" % arg_expr return self._get_grid(url, callback=lambda *a, **k: None) + # =========================== # This function is commented and not working. I don't have anything to test # This should return multiple grids and I don't know how pyhaystack will react @@ -46,25 +48,25 @@ def get_eval(self, arg_expr): # """ # Eval All # ======== -# If you have multiple expressions to evaluate, you can POST a grid to the evalAll URI. -# The posted grid specifies a row for each expression to evaluate with the expr column. -# The results are returned as a encoded a list of grids based on content negioation in the same +# If you have multiple expressions to evaluate, you can POST a grid to the evalAll URI. +# The posted grid specifies a row for each expression to evaluate with the expr column. +# The results are returned as a encoded a list of grids based on content negioation in the same # order as the request expressions. -# -# If an error occurs for any one expression, then an error grid is returned as the result -# of that expression. All expressions are evaluated regardless of any partial failure. -# If you wish to perform an atomic series of expressions, then you can evaluate one expression +# +# If an error occurs for any one expression, then an error grid is returned as the result +# of that expression. All expressions are evaluated regardless of any partial failure. +# If you wish to perform an atomic series of expressions, then you can evaluate one expression # inside a do block. # -# The evalAll operation is not a Haystack compliant operation. Its request is compliant, +# The evalAll operation is not a Haystack compliant operation. Its request is compliant, # but the response returns a list of multiple grids. -# +# # Reusing Intermediate Results # ---------------------------- # -# The evalAll API supports the ability to reuse intermediate expressions to feed -# additional expressions. For example lets say we want to return a history query -# and a daily rollup of both min and max. The expensive way would be to re-evaluate +# The evalAll API supports the ability to reuse intermediate expressions to feed +# additional expressions. For example lets say we want to return a history query +# and a daily rollup of both min and max. The expensive way would be to re-evaluate # the history query three times: # # ver:"2.0" @@ -72,24 +74,22 @@ def get_eval(self, arg_expr): # "readAll(kw).hisRead(thisWeek)" # "readAll(kw).hisRead(thisWeek).hisRollup(max,1day)" # "readAll(kw).hisRead(thisWeek).hisRollup(min,1day)" -# -# Instead we can add an args column to reuse previous expressions as arguments. +# +# Instead we can add an args column to reuse previous expressions as arguments. # In this case we can evaluate the history query once, then reuse it for the two rollups: -# +# # ver:"2.0" # expr,args # "readAll(kw).hisRead(thisWeek)", # "hisRollup(_,max,1day)","0" # "hisRollup(_,min,1day)","0" -# -# The args column is formatted as a list of strings separated by comma. The arguments -# must be an integer index into the row to use as argument. Or the value "prev" may be -# used to indicate the previous row. In order to use the args feature, the expression must -# evaluate to a function with the required number of parameters. This is typically +# +# The args column is formatted as a list of strings separated by comma. The arguments +# must be an integer index into the row to use as argument. Or the value "prev" may be +# used to indicate the previous row. In order to use the args feature, the expression must +# evaluate to a function with the required number of parameters. This is typically # accomplished via partial application. # """ -# +# # url = '/evalAll?expr=%s' % arg_expr # result = self.session._post_grid(url, callback=lambda *a, **k: None) - - diff --git a/pyhaystack/client/mixins/vendor/widesky/crud.py b/pyhaystack/client/mixins/vendor/widesky/crud.py index c146d1c..054acfa 100644 --- a/pyhaystack/client/mixins/vendor/widesky/crud.py +++ b/pyhaystack/client/mixins/vendor/widesky/crud.py @@ -11,6 +11,7 @@ import hszinc from six import string_types + class CRUDOpsMixin(object): """ The CRUD operations mix-in implements low-level support for entity @@ -31,7 +32,9 @@ def create(self, entities, callback=None): :param entities: The entities to be inserted. """ - return self._crud_op('createRec', entities, callback) + return self._crud_op( + "createRec", entities, callback, accept_status=(200, 400, 404) + ) def create_entity(self, entities, single=None, callback=None): """ @@ -65,7 +68,9 @@ def update(self, entities, callback=None): :param entities: The entities to be updated. """ - return self._crud_op('updateRec', entities, callback) + return self._crud_op( + "updateRec", entities, callback, accept_status=(200, 400, 404) + ) def delete(self, ids=None, filter_expr=None, callback=None): """ @@ -86,27 +91,28 @@ def delete(self, ids=None, filter_expr=None, callback=None): if bool(ids): if filter_expr is not None: - raise ValueError('Either specify ids or filter_expr, not both') + raise ValueError("Either specify ids or filter_expr, not both") ids = [self._obj_to_ref(r) for r in ids] if len(ids) == 1: # Reading a single entity - return self._get_grid('deleteRec', callback, - args={'id': ids[0]}) + return self._get_grid("deleteRec", callback, args={"id": ids[0]}) else: # Reading several entities grid = hszinc.Grid() - grid.column['id'] = {} - grid.extend([{'id': r} for r in ids]) - return self._post_grid('deleteRec', grid, callback) + grid.column["id"] = {} + grid.extend([{"id": r} for r in ids]) + return self._post_grid("deleteRec", grid, callback) else: - args = {'filter': filter_expr} - return self._get_grid('deleteRec', callback, args=args) + args = {"filter": filter_expr} + return self._get_grid( + "deleteRec", callback, args=args, accept_status=(200, 400, 404) + ) # Private methods - def _crud_op(self, op, entities, callback): + def _crud_op(self, op, entities, callback, **kwargs): """ Perform a repeated operation on the given entities with the given values for each entity. `entities` should be a list of dicts, each @@ -123,8 +129,11 @@ def _crud_op(self, op, entities, callback): all_columns = set() list(map(all_columns.update, [e.keys() for e in entities])) # We'll put 'id' first sort the others. - all_columns.discard('id') - all_columns = ['id'] + sorted(all_columns) + if "id" in all_columns: + all_columns.discard("id") + all_columns = ["id"] + sorted(all_columns) + else: + all_columns = sorted(all_columns) # Construct the grid grid = hszinc.Grid() @@ -136,7 +145,8 @@ def _crud_op(self, op, entities, callback): entity = entity.copy() # Ensure 'id' is a ref - entity['id'] = self._obj_to_ref(entity['id']) + if "id" in entity: + entity["id"] = self._obj_to_ref(entity["id"]) # Ensure all other columns are present for column in all_columns: @@ -147,4 +157,4 @@ def _crud_op(self, op, entities, callback): grid.append(entity) # Post the grid - return self._post_grid(op, grid, callback) + return self._post_grid(op, grid, callback, **kwargs) diff --git a/pyhaystack/client/mixins/vendor/widesky/multihis.py b/pyhaystack/client/mixins/vendor/widesky/multihis.py index 20f9e2b..75c0a64 100644 --- a/pyhaystack/client/mixins/vendor/widesky/multihis.py +++ b/pyhaystack/client/mixins/vendor/widesky/multihis.py @@ -11,6 +11,7 @@ import hszinc from six import string_types + class MultiHisOpsMixin(object): """ The Multi-His operations mix-in implements low-level support for @@ -24,19 +25,18 @@ def multi_his_read(self, points, rng, callback=None): a numbered column named idN where N starts counting from zero. """ if isinstance(rng, slice): - str_rng = ','.join([hszinc.dump_scalar(p) for p in - (rng.start, rng.stop)]) + str_rng = ",".join([hszinc.dump_scalar(p) for p in (rng.start, rng.stop)]) elif not isinstance(rng, string_types): str_rng = hszinc.dump_scalar(rng) else: # Better be valid! str_rng = rng - args = {'range': str_rng} + args = {"range": str_rng} for (col, point) in enumerate(points): - args['id%d' % col] = self._obj_to_ref(point) + args["id%d" % col] = self._obj_to_ref(point) - return self._get_grid('hisRead', callback, args=args) + return self._get_grid("hisRead", callback, args=args) def multi_his_write(self, timestamp_records, callback=None): """ @@ -51,35 +51,37 @@ def multi_his_write(self, timestamp_records, callback=None): """ # Grid grid = hszinc.Grid() - grid.column['ts'] = {} + grid.column["ts"] = {} # A mapping of IDs to column indexes point_idx = {} + def _get_idx(point_id): try: return point_idx[point_id] except KeyError: col = len(point_idx) point_idx[point_id] = col - grid.column['v%d' % col] = {'id': self._obj_to_ref(point_id)} + grid.column["v%d" % col] = {"id": self._obj_to_ref(point_id)} return col # Collate the grid data by timestamp. grid_data_by_ts = {} + def _get_ts(ts): try: ts_rec = grid_data_by_ts[ts] except KeyError: - ts_rec = {'ts': ts} + ts_rec = {"ts": ts} grid_data_by_ts[ts] = ts_rec return ts_rec - if hasattr(timestamp_records, 'to_dict'): + if hasattr(timestamp_records, "to_dict"): # Probably a Pandas DataFrame. Assume it returns # {column: {ts: value}} for point_id, col_data in timestamp_records.to_dict().items(): col_idx = _get_idx(point_id) - col = 'v%d' % col_idx + col = "v%d" % col_idx for ts, value in col_data.items(): ts_rec = _get_ts(ts) ts_rec[col] = value @@ -89,18 +91,18 @@ def _get_ts(ts): ts_rec = _get_ts(ts) for point_id, value in values.items(): col_idx = _get_idx(point_id) - ts_rec['v%d' % col_idx] = value + ts_rec["v%d" % col_idx] = value else: # A list of dicts, I hope! for rec in timestamp_records: - ts = rec.pop('ts') + ts = rec.pop("ts") ts_rec = _get_ts(ts) for point_id, value in rec.items(): col_idx = _get_idx(point_id) - ts_rec['v%d' % col_idx] = value + ts_rec["v%d" % col_idx] = value # Fill up the grid - grid.extend(sorted(grid_data_by_ts.values(), key=lambda r : r['ts'])) + grid.extend(sorted(grid_data_by_ts.values(), key=lambda r: r["ts"])) # Submit the data - return self._post_grid('hisWrite', grid, callback) + return self._post_grid("hisWrite", grid, callback) diff --git a/pyhaystack/client/niagara.py b/pyhaystack/client/niagara.py index 944690f..3dc1a7d 100644 --- a/pyhaystack/client/niagara.py +++ b/pyhaystack/client/niagara.py @@ -8,8 +8,12 @@ from .ops.vendor.niagara import NiagaraAXAuthenticateOperation from .ops.vendor.niagara_scram import Niagara4ScramAuthenticateOperation from .mixins.vendor.niagara.bql import BQLOperation, BQLMixin +from .mixins.vendor.niagara.encoding import EncodingMixin -class NiagaraHaystackSession(HaystackSession, BQLMixin): +import hszinc + + +class NiagaraHaystackSession(HaystackSession, BQLMixin, EncodingMixin): """ The NiagaraHaystackSession class implements some base support for NiagaraAX. This is mainly a convenience for @@ -18,7 +22,7 @@ class NiagaraHaystackSession(HaystackSession, BQLMixin): _AUTH_OPERATION = NiagaraAXAuthenticateOperation _BQL_OPERATION = BQLOperation - + def __init__(self, uri, username, password, **kwargs): """ Initialise a Nagara Project Haystack session handler. @@ -27,7 +31,7 @@ def __init__(self, uri, username, password, **kwargs): :param username: Authentication user name. :param password: Authentication password. """ - super(NiagaraHaystackSession, self).__init__(uri, 'haystack', **kwargs) + super(NiagaraHaystackSession, self).__init__(uri, "haystack", **kwargs) self._username = username self._password = password self._authenticated = False @@ -61,7 +65,8 @@ def _on_authenticate_done(self, operation, **kwargs): finally: self._auth_op = None -class Niagara4HaystackSession(HaystackSession, BQLMixin): + +class Niagara4HaystackSession(HaystackSession, BQLMixin, EncodingMixin): """ The Niagara4HaystackSession class implements some base support for Niagara4. This is mainly a convenience for @@ -79,7 +84,10 @@ def __init__(self, uri, username, password, **kwargs): :param username: Authentication user name. :param password: Authentication password. """ - super(Niagara4HaystackSession, self).__init__(uri, 'haystack', **kwargs) + + super(Niagara4HaystackSession, self).__init__( + uri, "haystack", grid_format=hszinc.MODE_JSON, **kwargs + ) self._username = username self._password = password self._authenticated = False @@ -100,7 +108,7 @@ def _on_authenticate_done(self, operation, **kwargs): """ try: op_result = operation.result - self._authenticated = op_result['authenticated'] + self._authenticated = op_result["authenticated"] except: self._authenticated = False @@ -108,5 +116,3 @@ def _on_authenticate_done(self, operation, **kwargs): self._client.cookies = None finally: self._auth_op = None - - diff --git a/pyhaystack/client/ops/entity.py b/pyhaystack/client/ops/entity.py index 092cbc6..3cc8f28 100644 --- a/pyhaystack/client/ops/entity.py +++ b/pyhaystack/client/ops/entity.py @@ -11,6 +11,7 @@ from ...util import state from ...util.asyncexc import AsynchronousException +from ...exception import HaystackError class EntityRetrieveOperation(state.HaystackOperation): @@ -27,7 +28,8 @@ def __init__(self, session, single): single = bool(single) super(EntityRetrieveOperation, self).__init__( - result_deepcopy=False, result_copy=not single) + result_deepcopy=False, result_copy=not single + ) self._session = session self._entities = {} self._single = single @@ -38,17 +40,24 @@ def _on_read(self, operation, **kwargs): """ try: # See if the read succeeded. - grid = operation.result - self._log.debug('Received grid: %s', grid) + try: + grid = operation.result + except HaystackError as e: + # Is this a "not found" error? + if str(e).startswith("HNotFoundError"): + raise NameError("No matching entity found") + raise + + self._log.debug("Received grid: %s", grid) # Iterate over each row: for row in grid: # Ignore rows that don't specify an ID. - if 'id' not in row: + if "id" not in row: continue row = row.copy() - entity_ref = row.pop('id') + entity_ref = row.pop("id") # This entity does not exist if entity_ref is None: @@ -65,7 +74,8 @@ def _on_read(self, operation, **kwargs): entity._update_tags(row) except KeyError: entity = self._session._tagging_model.create_entity( - entity_id, row) + entity_id, row + ) # Stash/update entity references. self._session._entities[entity_id] = entity @@ -75,11 +85,11 @@ def _on_read(self, operation, **kwargs): try: result = list(self._entities.values())[0] except IndexError: - raise NameError('No matching entity found') + raise NameError("No matching entity found") else: result = self._entities self._state_machine.read_done(result=result) - except: # Catch all exceptions to pass to caller. + except: # Catch all exceptions to pass to caller. self._state_machine.exception(result=AsynchronousException()) def _do_done(self, event): @@ -116,7 +126,6 @@ class GetEntityOperation(EntityRetrieveOperation): """ - def __init__(self, session, entity_ids, refresh_all, single): """ Initialise a request for the named IDs. @@ -126,24 +135,25 @@ def __init__(self, session, entity_ids, refresh_all, single): :param refresh_all: Refresh all entities, ignore existing content. """ - self._log = session._log.getChild('get_entity') + self._log = session._log.getChild("get_entity") super(GetEntityOperation, self).__init__(session, single) - self._entity_ids = set(map(lambda r : r.name \ - if isinstance(r, hszinc.Ref) else r, entity_ids)) + self._entity_ids = set( + map(lambda r: r.name if isinstance(r, hszinc.Ref) else r, entity_ids) + ) self._todo = self._entity_ids.copy() self._refresh_all = refresh_all self._state_machine = fysom.Fysom( - initial='init', final='done', - events=[ - # Event Current State New State - ('cache_checked', 'init', 'read'), - ('read_done', 'read', 'done'), - ('exception', '*', 'done'), - ], callbacks={ - 'onenterread': self._do_read, - 'onenterdone': self._do_done, - }) + initial="init", + final="done", + events=[ + # Event Current State New State + ("cache_checked", "init", "read"), + ("read_done", "read", "done"), + ("exception", "*", "done"), + ], + callbacks={"onenterread": self._do_read, "onenterdone": self._do_done}, + ) def go(self): """ @@ -155,8 +165,7 @@ def go(self): entity_id = entity_id.name try: - self._entities[entity_id] = \ - self._session._entities[entity_id] + self._entities[entity_id] = self._session._entities[entity_id] except KeyError: pass @@ -171,19 +180,18 @@ def _do_read(self, event): """ try: if bool(self._todo): - self._session.read(ids=list(self._todo), - callback=self._on_read) + self._session.read(ids=list(self._todo), callback=self._on_read) else: # Nothing needed to read. if self._single: try: result = list(self._entities.values())[0] except IndexError: - raise NameError('No matching entity found') + raise NameError("No matching entity found") else: result = self._entities self._state_machine.read_done(result=result) - except: # Catch all exceptions to pass to caller. + except: # Catch all exceptions to pass to caller. self._state_machine.exception(result=AsynchronousException()) @@ -213,26 +221,28 @@ def __init__(self, session, filter_expr, limit, single): :param limit: Maximum number of entities to fetch. """ - self._log = session._log.getChild('find_entity') + self._log = session._log.getChild("find_entity") super(FindEntityOperation, self).__init__(session, single) self._filter_expr = filter_expr self._limit = limit self._state_machine = fysom.Fysom( - initial='init', final='done', - events=[ - # Event Current State New State - ('go', 'init', 'read'), - ('read_done', 'read', 'done'), - ('exception', '*', 'done'), - ], callbacks={ - 'onenterdone': self._do_done, - }) + initial="init", + final="done", + events=[ + # Event Current State New State + ("go", "init", "read"), + ("read_done", "read", "done"), + ("exception", "*", "done"), + ], + callbacks={"onenterdone": self._do_done}, + ) def go(self): """ Start the request, check cache for existing entities. """ self._state_machine.go() - self._session.read(filter_expr=self._filter_expr, limit=self._limit, - callback=self._on_read) + self._session.read( + filter_expr=self._filter_expr, limit=self._limit, callback=self._on_read + ) diff --git a/pyhaystack/client/ops/feature.py b/pyhaystack/client/ops/feature.py index bd892db..ec45d2d 100644 --- a/pyhaystack/client/ops/feature.py +++ b/pyhaystack/client/ops/feature.py @@ -10,6 +10,7 @@ from ...util import state import fysom + class HasFeaturesOperation(state.HaystackOperation): """ A base class to detect if a given set of features is present. @@ -24,7 +25,7 @@ def __init__(self, session, features, cache=True): :param cache: Whether or not to use cache for this check. """ super(HasFeaturesOperation, self).__init__() - self._log = session._log.getChild('has_features') + self._log = session._log.getChild("has_features") self._session = session self._features = set(features) self._cache = cache @@ -33,8 +34,7 @@ def __init__(self, session, features, cache=True): # compare the features to the op names to see if they're present. self._need_about = False self._need_formats = False - self._need_ops = any([('/' not in feature) \ - for feature in self._features]) + self._need_ops = any([("/" not in feature) for feature in self._features]) # Retrieved feature data self._about = None @@ -45,29 +45,36 @@ def __init__(self, session, features, cache=True): self._ops_data = {} self._state_machine = fysom.Fysom( - initial='init', final='done', - events=[ - # Event Current State New State - ('go', 'init', 'get_about'), - ('about_done', 'get_about', 'get_formats'), - ('formats_done', 'get_formats', 'get_ops'), - ('ops_done', 'get_ops', 'check_features'), - ('checked', 'check_features', 'done'), - ('exception', '*', 'done'), - ], callbacks={ - 'onenterget_about': self._do_get_about, - 'onenterget_formats': self._do_get_formats, - 'onenterget_ops': self._do_get_ops, - 'onentercheck_features':self._do_check_features, - 'onenterdone': self._do_done, - }) + initial="init", + final="done", + events=[ + # Event Current State New State + ("go", "init", "get_about"), + ("about_done", "get_about", "get_formats"), + ("formats_done", "get_formats", "get_ops"), + ("ops_done", "get_ops", "check_features"), + ("checked", "check_features", "done"), + ("exception", "*", "done"), + ], + callbacks={ + "onenterget_about": self._do_get_about, + "onenterget_formats": self._do_get_formats, + "onenterget_ops": self._do_get_ops, + "onentercheck_features": self._do_check_features, + "onenterdone": self._do_done, + }, + ) def go(self): """ Start the request. """ - self._log.debug('Needed: about=%s, formats=%s, ops=%s', - self._need_about, self._need_formats, self._need_ops) + self._log.debug( + "Needed: about=%s, formats=%s, ops=%s", + self._need_about, + self._need_formats, + self._need_ops, + ) self._state_machine.go() def _do_get_about(self, event): @@ -76,13 +83,12 @@ def _do_get_about(self, event): """ try: if self._need_about: - self._log.debug('Retrieving about data') - self._session.about(callback=self._on_got_about, - cache=self._cache) + self._log.debug("Retrieving about data") + self._session.about(callback=self._on_got_about, cache=self._cache) else: - self._log.debug('Skipping about data') + self._log.debug("Skipping about data") self._state_machine.about_done() - except: # Catch all exceptions to pass to caller. + except: # Catch all exceptions to pass to caller. self._state_machine.exception(result=AsynchronousException()) def _on_got_about(self, operation, **kwargs): @@ -92,9 +98,9 @@ def _on_got_about(self, operation, **kwargs): try: self._about = operation.result self._about_data = self._about[0] - self._log.debug('Got about data: %s', self._about_data) + self._log.debug("Got about data: %s", self._about_data) self._state_machine.about_done() - except: # Catch all exceptions to pass to caller. + except: # Catch all exceptions to pass to caller. self._state_machine.exception(result=AsynchronousException()) def _do_get_formats(self, event): @@ -103,13 +109,12 @@ def _do_get_formats(self, event): """ try: if self._need_formats: - self._log.debug('Retrieving formats data') - self._session.formats(callback=self._on_got_formats, - cache=self._cache) + self._log.debug("Retrieving formats data") + self._session.formats(callback=self._on_got_formats, cache=self._cache) else: - self._log.debug('Skipping formats data') + self._log.debug("Skipping formats data") self._state_machine.formats_done() - except: # Catch all exceptions to pass to caller. + except: # Catch all exceptions to pass to caller. self._state_machine.exception(result=AsynchronousException()) def _on_got_formats(self, operation, **kwargs): @@ -119,10 +124,10 @@ def _on_got_formats(self, operation, **kwargs): try: self._formats = operation.result for row in self._formats: - self._formats_data[row['mime']] = row - self._log.debug('Got formats data: ', self._formats_data) + self._formats_data[row["mime"]] = row + self._log.debug("Got formats data: ", self._formats_data) self._state_machine.formats_done() - except: # Catch all exceptions to pass to caller. + except: # Catch all exceptions to pass to caller. self._state_machine.exception(result=AsynchronousException()) def _do_get_ops(self, event): @@ -131,13 +136,12 @@ def _do_get_ops(self, event): """ try: if self._need_ops: - self._log.debug('Retrieving ops data') - self._session.ops(callback=self._on_got_ops, - cache=self._cache) + self._log.debug("Retrieving ops data") + self._session.ops(callback=self._on_got_ops, cache=self._cache) else: - self._log.debug('Skipping ops data') + self._log.debug("Skipping ops data") self._state_machine.ops_done() - except: # Catch all exceptions to pass to caller. + except: # Catch all exceptions to pass to caller. self._state_machine.exception(result=AsynchronousException()) def _on_got_ops(self, operation, **kwargs): @@ -147,10 +151,10 @@ def _on_got_ops(self, operation, **kwargs): try: self._ops = operation.result for row in self._ops: - self._ops_data[row['name']] = row - self._log.debug('Got ops data: %s', self._ops_data) + self._ops_data[row["name"]] = row + self._log.debug("Got ops data: %s", self._ops_data) self._state_machine.ops_done() - except: # Catch all exceptions to pass to caller. + except: # Catch all exceptions to pass to caller. self._state_machine.exception(result=AsynchronousException()) def _do_check_features(self, event): @@ -159,7 +163,7 @@ def _do_check_features(self, event): """ try: result = self._check_features() - except: # Catch all exceptions to pass to caller. + except: # Catch all exceptions to pass to caller. self._state_machine.exception(result=AsynchronousException()) return self._state_machine.checked(result=result) @@ -171,7 +175,7 @@ def _check_features(self): """ res = {} for feature in self._features: - if '/' in feature: + if "/" in feature: # This is an extension res[feature] = False elif feature in self._ops_data: diff --git a/pyhaystack/client/ops/grid.py b/pyhaystack/client/ops/grid.py index 111b890..15a9178 100644 --- a/pyhaystack/client/ops/grid.py +++ b/pyhaystack/client/ops/grid.py @@ -15,12 +15,13 @@ from six import string_types from time import time + class BaseAuthOperation(state.HaystackOperation): """ A base class authentication operations. """ - def __init__(self, session, uri, retries=2, cache=False,): + def __init__(self, session, uri, retries=2, cache=False): """ Initialise a request for the authenticating with the given URI and arguments. @@ -36,7 +37,6 @@ def __init__(self, session, uri, retries=2, cache=False,): super(BaseAuthOperation, self).__init__() - self._retries = retries self._session = session self._uri = uri @@ -45,29 +45,32 @@ def __init__(self, session, uri, retries=2, cache=False,): self._cache = cache self._state_machine = fysom.Fysom( - initial='init', final='done', - events=[ - # Event Current State New State - ('auth_ok', 'init', 'check_cache'), - ('auth_not_ok', 'init', 'auth_attempt'), - ('auth_ok', 'auth_attempt', 'check_cache'), - ('auth_not_ok', 'auth_attempt', 'auth_failed'), - ('auth_failed', 'auth_attempt', 'done'), - ('cache_hit', 'check_cache', 'done'), - ('cache_miss', 'check_cache', 'submit'), - ('response_ok', 'submit', 'done'), - ('exception', '*', 'failed'), - ('retry', 'failed', 'init'), - ('abort', 'failed', 'done'), - ], callbacks={ - 'onretry': self._check_auth, - 'onenterauth_attempt': self._do_auth_attempt, - 'onenterauth_failed': self._do_auth_failed, - 'onentercheck_cache': self._do_check_cache, - 'onentersubmit': self._do_submit, - 'onenterfailed': self._do_fail_retry, - 'onenterdone': self._do_done, - }) + initial="init", + final="done", + events=[ + # Event Current State New State + ("auth_ok", "init", "check_cache"), + ("auth_not_ok", "init", "auth_attempt"), + ("auth_ok", "auth_attempt", "check_cache"), + ("auth_not_ok", "auth_attempt", "auth_failed"), + ("auth_failed", "auth_attempt", "done"), + ("cache_hit", "check_cache", "done"), + ("cache_miss", "check_cache", "submit"), + ("response_ok", "submit", "done"), + ("exception", "*", "failed"), + ("retry", "failed", "init"), + ("abort", "failed", "done"), + ], + callbacks={ + "onretry": self._check_auth, + "onenterauth_attempt": self._do_auth_attempt, + "onenterauth_failed": self._do_auth_failed, + "onentercheck_cache": self._do_check_cache, + "onentersubmit": self._do_submit, + "onenterfailed": self._do_fail_retry, + "onenterdone": self._do_done, + }, + ) def go(self): """ @@ -85,8 +88,8 @@ def _check_auth(self, *args): self._state_machine.auth_ok() else: self._state_machine.auth_not_ok() - except: # Catch all exceptions to pass to caller. - self._log.debug('Authentication check fails', exc_info=1) + except: # Catch all exceptions to pass to caller. + self._log.debug("Authentication check fails", exc_info=1) self._state_machine.exception(result=AsynchronousException()) def _do_auth_attempt(self, event): @@ -95,26 +98,26 @@ def _do_auth_attempt(self, event): """ try: self._session.authenticate(callback=self._on_authenticate) - except: # Catch all exceptions to pass to caller. + except: # Catch all exceptions to pass to caller. self._state_machine.exception(result=AsynchronousException()) def _on_authenticate(self, *args, **kwargs): """ Retry the authentication check. """ - self._log.debug('Authenticated, trying again') + self._log.debug("Authenticated, trying again") self.go() def _do_check_cache(self, event): """ Implement if needed """ - self._state_machine.cache_miss() # Nope + self._state_machine.cache_miss() # Nope return def _on_response(self, response): raise NotImplementedError() - + def _do_fail_retry(self, event): """ Determine whether we retry or fail outright. @@ -140,15 +143,25 @@ def _do_done(self, event): """ self._done(event.result) + class BaseGridOperation(BaseAuthOperation): """ A base class for GET and POST operations involving grids. """ - def __init__(self, session, uri, args=None, - expect_format=hszinc.MODE_ZINC, multi_grid=False, - raw_response=False, retries=2, cache=False, cache_key=None, - accept_status=None): + def __init__( + self, + session, + uri, + args=None, + expect_format=hszinc.MODE_ZINC, + multi_grid=False, + raw_response=False, + retries=2, + cache=False, + cache_key=None, + accept_status=None, + ): """ Initialise a request for the grid with the given URI and arguments. @@ -175,10 +188,17 @@ def __init__(self, session, uri, args=None, super(BaseGridOperation, self).__init__(session, uri) if args is not None: # Convert scalars to strings - args = dict([(param, hszinc.dump_scalar(value) \ - if not isinstance(value, string_types) \ - else value) - for param, value in args.items()]) + args = dict( + [ + ( + param, + hszinc.dump_scalar(value) + if not isinstance(value, string_types) + else value, + ) + for param, value in args.items() + ] + ) self._retries = retries self._session = session @@ -197,21 +217,21 @@ def __init__(self, session, uri, args=None, if not raw_response: if expect_format == hszinc.MODE_ZINC: - self._headers[b'Accept'] = 'text/zinc' + self._headers[b"Accept"] = "text/zinc" elif expect_format == hszinc.MODE_JSON: - self._headers[b'Accept'] = 'application/json' + self._headers[b"Accept"] = "application/json" elif expect_format is not None: raise ValueError( - 'expect_format must be one onf hszinc.MODE_ZINC '\ - 'or hszinc.MODE_JSON') - + "expect_format must be one onf hszinc.MODE_ZINC " + "or hszinc.MODE_JSON" + ) def _do_check_cache(self, event): """ See if there's cache for this grid. """ if not self._cache: - self._state_machine.cache_miss() # Nope + self._state_machine.cache_miss() # Nope return # Initialise data @@ -249,6 +269,7 @@ def _proxy(op): return self._state_machine.cache_hit(res) + op.done_sig.connect(_proxy) def _on_response(self, response): @@ -259,7 +280,7 @@ def _on_response(self, response): # Does the session want to invoke any relevant hooks? # This allows a session to detect problems in the session and # abort the operation. - if hasattr(self._session, '_on_http_grid_response'): + if hasattr(self._session, "_on_http_grid_response"): self._session._on_http_grid_response(response) # Process the HTTP error, if any. @@ -276,29 +297,31 @@ def _on_response(self, response): content_type = response.content_type body = response.text - if content_type in ('text/zinc', 'text/plain'): + if content_type in ("text/zinc", "text/plain"): # We have been given a grid in ZINC format. decoded = hszinc.parse(body, mode=hszinc.MODE_ZINC) - elif content_type == 'application/json': + elif content_type == "application/json": # We have been given a grid in JSON format. decoded = [hszinc.parse(body, mode=hszinc.MODE_JSON)] - elif content_type in ('text/html'): + elif content_type in ("text/html"): # We probably fell back to a login screen after auto logoff. self._state_machine.exception(AsynchronousException()) else: # We don't recognise this! - raise ValueError('Unrecognised content type %s' % content_type) + raise ValueError("Unrecognised content type %s" % content_type) # Check for exceptions def _check_err(grid): try: - if 'err' in grid.metadata: + if "err" in grid.metadata: raise HaystackError( - grid.metadata.get('dis', 'Unknown Error'), - grid.metadata.get('traceback', None)) + grid.metadata.get("dis", "Unknown Error"), + grid.metadata.get("traceback", None), + ) return grid except: return AsynchronousException() + decoded = [_check_err(g) for g in decoded] if not self._multi_grid: decoded = decoded[0] @@ -306,15 +329,17 @@ def _check_err(grid): # If we get here, then the request itself succeeded. if self._cache: with self._session._grid_lk: - self._session._grid_cache[self._cache_key] = \ - (None, time() + self._session._grid_expiry, decoded) + self._session._grid_cache[self._cache_key] = ( + None, + time() + self._session._grid_expiry, + decoded, + ) self._state_machine.response_ok(result=decoded) - except: # Catch all exceptions for the caller. - self._log.debug('Parse fails', exc_info=1) + except: # Catch all exceptions for the caller. + self._log.debug("Parse fails", exc_info=1) self._state_machine.exception(result=AsynchronousException()) - class GetGridOperation(BaseGridOperation): """ A state machine that performs a GET operation then reads back a ZINC grid. @@ -333,10 +358,10 @@ def __init__(self, session, uri, args=None, multi_grid=False, **kwargs): _always_ return a list, otherwise, it will _always_ return a single grid. """ - self._log = session._log.getChild('get_grid.%s' % uri) + self._log = session._log.getChild("get_grid.%s" % uri) super(GetGridOperation, self).__init__( - session=session, uri=uri, args=args, - multi_grid=multi_grid, **kwargs) + session=session, uri=uri, args=args, multi_grid=multi_grid, **kwargs + ) def _do_submit(self, event): """ @@ -344,11 +369,15 @@ def _do_submit(self, event): """ try: - self._session._get(self._uri, params=self._args, - headers=self._headers, callback=self._on_response, - accept_status=self._accept_status) - except: # Catch all exceptions to pass to caller. - self._log.debug('Get fails', exc_info=1) + self._session._get( + self._uri, + params=self._args, + headers=self._headers, + callback=self._on_response, + accept_status=self._accept_status, + ) + except: # Catch all exceptions to pass to caller. + self._log.debug("Get fails", exc_info=1) self._state_machine.exception(result=AsynchronousException()) @@ -358,8 +387,9 @@ class PostGridOperation(BaseGridOperation): read back a ZINC grid. """ - def __init__(self, session, uri, grid, args=None, - post_format=hszinc.MODE_ZINC, **kwargs): + def __init__( + self, session, uri, grid, args=None, post_format=hszinc.MODE_ZINC, **kwargs + ): """ Initialise a POST request for the grid with the given grid, URI and arguments. @@ -371,26 +401,32 @@ def __init__(self, session, uri, grid, args=None, :param post_format: What format to post grids in? :param args: Dictionary of key-value pairs to be given as arguments. """ - self._log = session._log.getChild('post_grid.%s' % uri) + self._log = session._log.getChild("post_grid.%s" % uri) super(PostGridOperation, self).__init__( - session=session, uri=uri, args=args, **kwargs) + session=session, uri=uri, args=args, **kwargs + ) # Convert the grids to their native format - self._body = hszinc.dump(grid, mode=post_format).encode('utf-8') + self._body = hszinc.dump(grid, mode=post_format).encode("utf-8") if post_format == hszinc.MODE_ZINC: - self._content_type = 'text/zinc' + self._content_type = "text/zinc" else: - self._content_type = 'application/json' + self._content_type = "application/json" def _do_submit(self, event): """ Submit the GET request to the haystack server. """ try: - self._session._post(self._uri, body=self._body, - body_type=self._content_type, params=self._args, - headers=self._headers, callback=self._on_response, - accept_status=self._accept_status) - except: # Catch all exceptions to pass to caller. - self._log.debug('Post fails', exc_info=1) + self._session._post( + self._uri, + body=self._body, + body_type=self._content_type, + params=self._args, + headers=self._headers, + callback=self._on_response, + accept_status=self._accept_status, + ) + except: # Catch all exceptions to pass to caller. + self._log.debug("Post fails", exc_info=1) self._state_machine.exception(result=AsynchronousException()) diff --git a/pyhaystack/client/ops/his.py b/pyhaystack/client/ops/his.py index fd68846..c81e393 100644 --- a/pyhaystack/client/ops/his.py +++ b/pyhaystack/client/ops/his.py @@ -18,8 +18,9 @@ try: from pandas import Series, DataFrame + HAVE_PANDAS = True -except ImportError: # pragma: no cover +except ImportError: # pragma: no cover # Not covered, since we'll always have 'pandas' available during tests. HAVE_PANDAS = False @@ -31,7 +32,7 @@ def _resolve_tz(tz): if (tz is None) or isinstance(tz, tzinfo): return tz if isinstance(tz, string_types): - if '/' in tz: + if "/" in tz: # Olson database name return pytz.timezone(tz) else: @@ -44,9 +45,9 @@ class HisReadSeriesOperation(state.HaystackOperation): format. """ - FORMAT_LIST = 'list' # [(ts1, value1), (ts2, value2), ...] - FORMAT_DICT = 'dict' # {ts1: value1, ts2: value2, ...} - FORMAT_SERIES = 'series' # pandas.Series + FORMAT_LIST = "list" # [(ts1, value1), (ts2, value2), ...] + FORMAT_DICT = "dict" # {ts1: value1, ts2: value2, ...} + FORMAT_SERIES = "series" # pandas.Series def __init__(self, session, point, rng, tz, series_format): """ @@ -60,12 +61,23 @@ def __init__(self, session, point, rng, tz, series_format): """ super(HisReadSeriesOperation, self).__init__() - if series_format not in (self.FORMAT_LIST, self.FORMAT_DICT, - self.FORMAT_SERIES): - raise ValueError('Unrecognised series_format %s' % series_format) + if series_format not in ( + self.FORMAT_LIST, + self.FORMAT_DICT, + self.FORMAT_SERIES, + ): + raise ValueError("Unrecognised series_format %s" % series_format) if (series_format == self.FORMAT_SERIES) and (not HAVE_PANDAS): - raise NotImplementedError('pandas not available.') + raise NotImplementedError("pandas not available.") + + if isinstance(rng, slice): + rng = ",".join( + [ + hszinc.dump_scalar(p, mode=hszinc.MODE_ZINC) + for p in (rng.start, rng.stop) + ] + ) self._session = session self._point = point @@ -74,16 +86,16 @@ def __init__(self, session, point, rng, tz, series_format): self._series_format = series_format self._state_machine = fysom.Fysom( - initial='init', final='done', - events=[ - # Event Current State New State - ('go', 'init', 'read'), - ('read_done', 'read', 'done'), - ('exception', '*', 'done'), - ], callbacks={ - 'onenterread': self._do_read, - 'onenterdone': self._do_done, - }) + initial="init", + final="done", + events=[ + # Event Current State New State + ("go", "init", "read"), + ("read_done", "read", "done"), + ("exception", "*", "done"), + ], + callbacks={"onenterread": self._do_read, "onenterdone": self._do_done}, + ) def go(self): self._state_machine.go() @@ -92,8 +104,9 @@ def _do_read(self, event): """ Request the data from the server. """ - self._session.his_read(point=self._point, rng=self._range, - callback=self._on_read) + self._session.his_read( + point=self._point, rng=self._range, callback=self._on_read + ) def _on_read(self, operation, **kwargs): """ @@ -104,12 +117,12 @@ def _on_read(self, operation, **kwargs): grid = operation.result if self._tz is None: - conv_ts = lambda ts : ts + conv_ts = lambda ts: ts else: - conv_ts = lambda ts : ts.astimezone(self._tz) + conv_ts = lambda ts: ts.astimezone(self._tz) # Convert grid to list of tuples - data = [(conv_ts(row['ts']), row['val']) for row in grid] + data = [(conv_ts(row["ts"]), row["val"]) for row in grid] if self._series_format == self.FORMAT_DICT: data = dict(data) @@ -122,19 +135,19 @@ def _on_read(self, operation, **kwargs): units = data[0].unit else: values = data - units = '' + units = "" except ValueError: values = [] index = [] - units = '' + units = "" - #ser = Series(data=data[0].value, index=index) + # ser = Series(data=data[0].value, index=index) meta_serie = MetaSeries(data=values, index=index) - meta_serie.add_meta('units', units) - meta_serie.add_meta('point', self._point) + meta_serie.add_meta("units", units) + meta_serie.add_meta("point", self._point) self._state_machine.read_done(result=meta_serie) - except: # Catch all exceptions to pass to caller. + except: # Catch all exceptions to pass to caller. self._state_machine.exception(result=AsynchronousException()) def _do_done(self, event): @@ -150,9 +163,9 @@ class HisReadFrameOperation(state.HaystackOperation): concise format. """ - FORMAT_LIST = 'list' # [{'ts': ts1, 'col1': val1, ...}, {...}, ...] - FORMAT_DICT = 'dict' # {ts1: {'col1': val1, ...}, ts2: ...} - FORMAT_FRAME = 'frame' # pandas.DataFrame + FORMAT_LIST = "list" # [{'ts': ts1, 'col1': val1, ...}, {...}, ...] + FORMAT_DICT = "dict" # {ts1: {'col1': val1, ...}, ts2: ...} + FORMAT_FRAME = "frame" # pandas.DataFrame def __init__(self, session, columns, rng, tz, frame_format): """ @@ -165,20 +178,27 @@ def __init__(self, session, columns, rng, tz, frame_format): :param frame_format: What format to present the frame in. """ super(HisReadFrameOperation, self).__init__() - self._log = session._log.getChild('his_read_frame') + self._log = session._log.getChild("his_read_frame") - if frame_format not in (self.FORMAT_LIST, self.FORMAT_DICT, - self.FORMAT_FRAME): - raise ValueError('Unrecognised frame_format %s' % frame_format) + if frame_format not in (self.FORMAT_LIST, self.FORMAT_DICT, self.FORMAT_FRAME): + raise ValueError("Unrecognised frame_format %s" % frame_format) if (frame_format == self.FORMAT_FRAME) and (not HAVE_PANDAS): - raise NotImplementedError('pandas not available.') + raise NotImplementedError("pandas not available.") + + if isinstance(rng, slice): + rng = ",".join( + [ + hszinc.dump_scalar(p, mode=hszinc.MODE_ZINC) + for p in (rng.start, rng.stop) + ] + ) # Convert the columns to a list of tuples. - strip_ref = lambda r : r.name if isinstance(r, hszinc.Ref) else r + strip_ref = lambda r: r.name if isinstance(r, hszinc.Ref) else r if isinstance(columns, dict): # Ensure all are strings to references - columns = [(str(c),strip_ref(r)) for c, r in columns.items()] + columns = [(str(c), strip_ref(r)) for c, r in columns.items()] else: # Translate to a dict: columns = [(strip_ref(c), c) for c in columns] @@ -192,46 +212,50 @@ def __init__(self, session, columns, rng, tz, frame_format): self._todo = set([c[0] for c in columns]) self._state_machine = fysom.Fysom( - initial='init', final='done', - events=[ - # Event Current State New State - ('probe_multi', 'init', 'probing'), - ('do_multi_read', 'probing', 'multi_read'), - ('all_read_done', 'multi_read', 'postprocess'), - ('do_single_read', 'probing', 'single_read'), - ('all_read_done', 'single_read', 'postprocess'), - ('process_done', 'postprocess', 'done'), - ('exception', '*', 'done'), - ], callbacks={ - 'onenterprobing': self._do_probe_multi, - 'onentermulti_read': self._do_multi_read, - 'onentersingle_read': self._do_single_read, - 'onenterpostprocess': self._do_postprocess, - 'onenterdone': self._do_done, - }) + initial="init", + final="done", + events=[ + # Event Current State New State + ("probe_multi", "init", "probing"), + ("do_multi_read", "probing", "multi_read"), + ("all_read_done", "multi_read", "postprocess"), + ("do_single_read", "probing", "single_read"), + ("all_read_done", "single_read", "postprocess"), + ("process_done", "postprocess", "done"), + ("exception", "*", "done"), + ], + callbacks={ + "onenterprobing": self._do_probe_multi, + "onentermulti_read": self._do_multi_read, + "onentersingle_read": self._do_single_read, + "onenterpostprocess": self._do_postprocess, + "onenterdone": self._do_done, + }, + ) def go(self): self._state_machine.probe_multi() def _do_probe_multi(self, event): - self._log.debug('Probing for multi-his-read support') - self._session.has_features([self._session.FEATURE_HISREAD_MULTI], - callback=self._on_probe_multi) + self._log.debug("Probing for multi-his-read support") + self._session.has_features( + [self._session.FEATURE_HISREAD_MULTI], callback=self._on_probe_multi + ) def _on_probe_multi(self, operation, **kwargs): try: result = operation.result - except: # Catch all exceptions to pass to caller. + except: # Catch all exceptions to pass to caller. self._state_machine.exception(result=AsynchronousException()) return if result.get(self._session.FEATURE_HISREAD_MULTI): # Session object supports multi-his-read - self._log.debug('Using multi-his-read support') + self._log.debug("Using multi-his-read support") self._state_machine.do_multi_read() else: # Emulate multi-his-read with separate - self._log.debug('No multi-his-read support, emulating') + self._log.debug("No multi-his-read support, emulating") self._state_machine.do_single_read() def _get_ts_rec(self, ts): @@ -246,8 +270,11 @@ def _do_multi_read(self, event): """ Request the data from the server as a single multi-read request. """ - self._session.multi_his_read(points=[c[1] for c in self._columns], - rng=self._range, callback=self._on_multi_read) + self._session.multi_his_read( + points=[c[1] for c in self._columns], + rng=self._range, + callback=self._on_multi_read, + ) def _on_multi_read(self, operation, **kwargs): """ @@ -257,21 +284,20 @@ def _on_multi_read(self, operation, **kwargs): grid = operation.result if self._tz is None: - conv_ts = lambda ts : ts + conv_ts = lambda ts: ts else: - conv_ts = lambda ts : ts.astimezone(self._tz) + conv_ts = lambda ts: ts.astimezone(self._tz) for row in grid: - ts = conv_ts(row['ts']) + ts = conv_ts(row["ts"]) rec = self._get_ts_rec(ts) for (col_idx, (col, _)) in enumerate(self._columns): - val = row.get('v%d' % col_idx) - if (val is not None) or \ - (self._frame_format != self.FORMAT_FRAME): + val = row.get("v%d" % col_idx) + if (val is not None) or (self._frame_format != self.FORMAT_FRAME): rec[col] = val self._state_machine.all_read_done() - except: # Catch all exceptions to pass to caller. - self._log.debug('Hit exception', exc_info=1) + except: # Catch all exceptions to pass to caller. + self._log.debug("Hit exception", exc_info=1) self._state_machine.exception(result=AsynchronousException()) def _do_single_read(self, event): @@ -279,92 +305,98 @@ def _do_single_read(self, event): Request the data from the server as multiple single-read requests. """ for col, point in self._columns: - self._log.debug('Column %s point %s', col, point) - self._session.his_read(point, self._range, - lambda operation, **kw : self._on_single_read(operation, - col=col)) + self._log.debug("Column %s point %s", col, point) + self._session.his_read( + point, + self._range, + lambda operation, **kw: self._on_single_read(operation, col=col), + ) def _on_single_read(self, operation, col, **kwargs): """ Handle the multi-valued grid. """ - self._log.debug('Response back for column %s', col) + self._log.debug("Response back for column %s", col) try: grid = operation.result - #print(grid) - #print('===========') + # print(grid) + # print('===========') if self._tz is None: - conv_ts = lambda ts : ts + conv_ts = lambda ts: ts else: - conv_ts = lambda ts : ts.astimezone(self._tz) + conv_ts = lambda ts: ts.astimezone(self._tz) - self._log.debug('%d records for %s: %s', len(grid), col, grid) + self._log.debug("%d records for %s: %s", len(grid), col, grid) for row in grid: - ts = conv_ts(row['ts']) + ts = conv_ts(row["ts"]) if self._tz is None: self._tz = ts.tzinfo rec = self._get_ts_rec(ts) - val = row.get('val') - if (val is not None) or \ - (self._frame_format != self.FORMAT_FRAME): + val = row.get("val") + if (val is not None) or (self._frame_format != self.FORMAT_FRAME): rec[col] = val self._todo.discard(col) - self._log.debug('Still waiting for: %s', self._todo) + self._log.debug("Still waiting for: %s", self._todo) if not self._todo: # No more to read self._state_machine.all_read_done() - except: # Catch all exceptions to pass to caller. - self._log.debug('Hit exception', exc_info=1) + except: # Catch all exceptions to pass to caller. + self._log.debug("Hit exception", exc_info=1) self._state_machine.exception(result=AsynchronousException()) def _do_postprocess(self, event): """ Convert the dict-of-dicts to the desired frame format. """ - self._log.debug('Post-processing') + self._log.debug("Post-processing") try: if self._frame_format == self.FORMAT_LIST: + def _merge_ts(item): rec = item[1].copy() - rec['ts'] = item[0] + rec["ts"] = item[0] return rec + data = list(map(_merge_ts, list(self._data_by_ts.items()))) - #print(data) + # print(data) elif self._frame_format == self.FORMAT_FRAME: # Build from dict - data = MetaDataFrame.from_dict(self._data_by_ts, orient='index') + data = MetaDataFrame.from_dict(self._data_by_ts, orient="index") + def convert_quantity(val): """ If value is Quantity, convert to value """ - if isinstance(val,hszinc.Quantity): + if isinstance(val, hszinc.Quantity): return val.value else: return val + def get_units(serie): try: first_element = serie.dropna()[0] - except IndexError: # needed for empty results - return '' + except IndexError: # needed for empty results + return "" if isinstance(first_element, hszinc.Quantity): return first_element.unit else: - return '' + return "" + for name, serie in data.iteritems(): """ Convert Quantity and put unit in metadata """ - data.add_meta(name,get_units(serie)) + data.add_meta(name, get_units(serie)) data[name] = data[name].apply(convert_quantity) else: data = self._data_by_ts self._state_machine.process_done(result=data) - except: # Catch all exceptions to pass to caller. - self._log.debug('Hit exception', exc_info=1) + except: # Catch all exceptions to pass to caller. + self._log.debug("Hit exception", exc_info=1) self._state_machine.exception(result=AsynchronousException()) def _do_done(self, event): @@ -406,38 +438,41 @@ def __init__(self, session, point, series, tz): self._tz = _resolve_tz(tz) self._state_machine = fysom.Fysom( - initial='init', final='done', - events=[ - # Event Current State New State - ('have_tz', 'init', 'write'), - ('have_point', 'init', 'get_point_tz'), - ('need_point', 'init', 'get_point'), - ('have_point', 'get_point', 'get_point_tz'), - ('have_tz', 'get_point_tz', 'write'), - ('need_equip', 'get_point_tz', 'get_equip'), - ('have_equip', 'get_equip', 'get_equip_tz'), - ('have_tz', 'get_equip_tz', 'write'), - ('need_site', 'get_equip_tz', 'get_site'), - ('have_site', 'get_site', 'get_site_tz'), - ('have_tz', 'get_site_tz', 'write'), - ('write_done', 'write', 'done'), - ('exception', '*', 'done'), - ], callbacks={ - 'onenterget_point': self._do_get_point, - 'onenterget_point_tz': self._do_get_point_tz, - 'onenterget_equip': self._do_get_equip, - 'onenterget_equip_tz': self._do_get_equip_tz, - 'onenterget_site': self._do_get_site, - 'onenterget_site_tz': self._do_get_site_tz, - 'onenterwrite': self._do_write, - 'onenterdone': self._do_done, - }) + initial="init", + final="done", + events=[ + # Event Current State New State + ("have_tz", "init", "write"), + ("have_point", "init", "get_point_tz"), + ("need_point", "init", "get_point"), + ("have_point", "get_point", "get_point_tz"), + ("have_tz", "get_point_tz", "write"), + ("need_equip", "get_point_tz", "get_equip"), + ("have_equip", "get_equip", "get_equip_tz"), + ("have_tz", "get_equip_tz", "write"), + ("need_site", "get_equip_tz", "get_site"), + ("have_site", "get_site", "get_site_tz"), + ("have_tz", "get_site_tz", "write"), + ("write_done", "write", "done"), + ("exception", "*", "done"), + ], + callbacks={ + "onenterget_point": self._do_get_point, + "onenterget_point_tz": self._do_get_point_tz, + "onenterget_equip": self._do_get_equip, + "onenterget_equip_tz": self._do_get_equip_tz, + "onenterget_site": self._do_get_site, + "onenterget_site_tz": self._do_get_site_tz, + "onenterwrite": self._do_write, + "onenterdone": self._do_done, + }, + ) def go(self): - if self._tz is not None: # Do we have a timezone? + if self._tz is not None: # Do we have a timezone? # We do! self._state_machine.have_tz() - elif self._point is not None: # Nope, do we have the point? + elif self._point is not None: # Nope, do we have the point? # We do! self._state_machine.have_point() else: @@ -448,8 +483,7 @@ def _do_get_point(self, event): """ Retrieve the point entity. """ - self._session.get_entity(self._entity_id, single=True, - callback=self._got_point) + self._session.get_entity(self._entity_id, single=True, callback=self._got_point) def _got_point(self, operation, **kwargs): """ @@ -465,7 +499,7 @@ def _do_get_point_tz(self, event): """ See if the point has a timezone? """ - if hasattr(self._point, 'tz') and isinstance(self._point.tz, tzinfo): + if hasattr(self._point, "tz") and isinstance(self._point.tz, tzinfo): # We have our timezone. self._tz = self._point.tz self._state_machine.have_tz() @@ -494,7 +528,7 @@ def _do_get_equip_tz(self, event): See if the equip has a timezone? """ equip = event.equip - if hasattr(equip, 'tz') and isinstance(equip.tz, tzinfo): + if hasattr(equip, "tz") and isinstance(equip.tz, tzinfo): # We have our timezone. self._tz = equip.tz self._state_machine.have_tz() @@ -523,15 +557,16 @@ def _do_get_site_tz(self, event): See if the site has a timezone? """ site = event.site - if hasattr(site, 'tz') and isinstance(site.tz, tzinfo): + if hasattr(site, "tz") and isinstance(site.tz, tzinfo): # We have our timezone. self._tz = site.tz self._state_machine.have_tz() else: try: # Nope, no idea then. - raise ValueError('No timezone specified for operation, '\ - 'point, equip or site.') + raise ValueError( + "No timezone specified for operation, " "point, equip or site." + ) except: self._state_machine.exception(result=AsynchronousException()) @@ -541,7 +576,7 @@ def _do_write(self, event): """ try: # Process the timestamp records into an appropriate format. - if hasattr(self._series, 'to_dict'): + if hasattr(self._series, "to_dict"): records = self._series.to_dict() elif not isinstance(self._series, dict): records = dict(self._series) @@ -554,18 +589,26 @@ def _do_write(self, event): return # Time-shift the records. - if hasattr(self._tz, 'localize'): - localise = lambda ts : self._tz.localize(ts) \ - if ts.tzinfo is None else ts.astimezone(self._tz) + if hasattr(self._tz, "localize"): + localise = ( + lambda ts: self._tz.localize(ts) + if ts.tzinfo is None + else ts.astimezone(self._tz) + ) else: - localise = lambda ts : ts.replace(tzinfo=self._tz) \ - if ts.tzinfo is None else ts.astimezone(self._tz) - records = dict([(localise(ts), val) \ - for ts, val in records.items()]) + localise = ( + lambda ts: ts.replace(tzinfo=self._tz) + if ts.tzinfo is None + else ts.astimezone(self._tz) + ) + records = dict([(localise(ts), val) for ts, val in records.items()]) # Write the data - self._session.his_write(point=self._entity_id, - timestamp_records=records, callback=self._on_write) + self._session.his_write( + point=self._entity_id, + timestamp_records=records, + callback=self._on_write, + ) except: self._state_machine.exception(result=AsynchronousException()) @@ -577,10 +620,10 @@ def _on_write(self, operation, **kwargs): # See if the write succeeded. grid = operation.result if not isinstance(grid, hszinc.Grid): - raise TypeError('Unexpected result: %r' % grid) + raise TypeError("Unexpected result: %r" % grid) # Move to the done state. self._state_machine.write_done(result=None) - except: # Catch all exceptions to pass to caller. + except: # Catch all exceptions to pass to caller. self._state_machine.exception(result=AsynchronousException()) def _do_done(self, event): @@ -605,25 +648,29 @@ def __init__(self, session, columns, frame, tz): :param tz: Timezone to translate timezones to. """ super(HisWriteFrameOperation, self).__init__() - self._log = session._log.getChild('his_write_frame') + self._log = session._log.getChild("his_write_frame") tz = _resolve_tz(tz) if tz is None: tz = pytz.utc - if hasattr(tz, 'localize'): - localise = lambda ts : tz.localize(ts) \ - if ts.tzinfo is None else ts.astimezone(tz) + if hasattr(tz, "localize"): + localise = ( + lambda ts: tz.localize(ts) if ts.tzinfo is None else ts.astimezone(tz) + ) else: - localise = lambda ts : ts.replace(tzinfo=tz) \ - if ts.tzinfo is None else ts.astimezone(tz) + localise = ( + lambda ts: ts.replace(tzinfo=tz) + if ts.tzinfo is None + else ts.astimezone(tz) + ) # Convert frame to list of records. if HAVE_PANDAS: # Convert Pandas frame to dict of dicts form. if isinstance(frame, DataFrame): - self._log.debug('Convert from Pandas DataFrame') - raw_frame = frame.to_dict(orient='dict') + self._log.debug("Convert from Pandas DataFrame") + raw_frame = frame.to_dict(orient="dict") frame = {} for col, col_data in raw_frame.items(): for ts, val in col_data.items(): @@ -637,49 +684,60 @@ def __init__(self, session, columns, frame, tz): # Convert dict of dicts to records, de-referencing column names. if isinstance(frame, dict): if columns is None: + def _to_rec(item): (ts, raw_record) = item record = raw_record.copy() - record['ts'] = ts + record["ts"] = ts return record + else: + def _to_rec(item): (ts, raw_record) = item record = {} for col, val in raw_record.items(): entity = columns[col] - if hasattr(entity, 'id'): + if hasattr(entity, "id"): entity = entity.id if isinstance(entity, hszinc.Ref): entity = entity.name record[entity] = val - record['ts'] = ts + record["ts"] = ts return record + frame = list(map(_to_rec, list(frame.items()))) elif columns is not None: # Columns are aliased. De-alias the column names. frame = deepcopy(frame) for row in frame: - ts = row.pop('ts') + ts = row.pop("ts") raw = row.copy() row.clear() - row['ts'] = ts + row["ts"] = ts for column, point in columns.items(): try: value = raw.pop(column) except KeyError: - self._log.debug('At %s missing column %s (for %s): %s', - ts, column, point, raw) + self._log.debug( + "At %s missing column %s (for %s): %s", + ts, + column, + point, + raw, + ) continue row[session._obj_to_ref(point).name] = value # Localise all timestamps, extract columns: columns = set() + def _localise_rec(r): - r['ts'] = localise(r['ts']) - columns.update(set(r.keys()) - set(['ts'])) + r["ts"] = localise(r["ts"]) + columns.update(set(r.keys()) - set(["ts"])) return r + frame = list(map(_localise_rec, frame)) self._session = session @@ -689,61 +747,63 @@ def _localise_rec(r): self._tz = _resolve_tz(tz) self._state_machine = fysom.Fysom( - initial='init', final='done', - events=[ - # Event Current State New State - ('probe_multi', 'init', 'probing'), - ('no_data', 'init', 'done'), - ('do_multi_write', 'probing', 'multi_write'), - ('all_write_done', 'multi_write', 'done'), - ('do_single_write', 'probing', 'single_write'), - ('all_write_done', 'single_write', 'done'), - ('exception', '*', 'done'), - ], callbacks={ - 'onenterprobing': self._do_probe_multi, - 'onentermulti_write': self._do_multi_write, - 'onentersingle_write': self._do_single_write, - 'onenterdone': self._do_done, - }) + initial="init", + final="done", + events=[ + # Event Current State New State + ("probe_multi", "init", "probing"), + ("no_data", "init", "done"), + ("do_multi_write", "probing", "multi_write"), + ("all_write_done", "multi_write", "done"), + ("do_single_write", "probing", "single_write"), + ("all_write_done", "single_write", "done"), + ("exception", "*", "done"), + ], + callbacks={ + "onenterprobing": self._do_probe_multi, + "onentermulti_write": self._do_multi_write, + "onentersingle_write": self._do_single_write, + "onenterdone": self._do_done, + }, + ) def go(self): if not bool(self._columns): - self._log.debug('No data to write') + self._log.debug("No data to write") self._state_machine.no_data(result=None) else: self._state_machine.probe_multi() def _do_probe_multi(self, event): - self._log.debug('Probing for multi-his-write support') - self._session.has_features([self._session.FEATURE_HISWRITE_MULTI], - callback=self._on_probe_multi) + self._log.debug("Probing for multi-his-write support") + self._session.has_features( + [self._session.FEATURE_HISWRITE_MULTI], callback=self._on_probe_multi + ) def _on_probe_multi(self, operation, **kwargs): try: result = operation.result - except: # Catch all exceptions to pass to caller. - self._log.warning('Unable to probe multi-his-write support', - exc_info=1) + except: # Catch all exceptions to pass to caller. + self._log.warning("Unable to probe multi-his-write support", exc_info=1) self._state_machine.exception(result=AsynchronousException()) result = {} return - self._log.debug('Got result: %s', result) + self._log.debug("Got result: %s", result) if result.get(self._session.FEATURE_HISWRITE_MULTI): # Session object supports multi-his-write - self._log.debug('Using multi-his-write support') + self._log.debug("Using multi-his-write support") self._state_machine.do_multi_write() else: # Emulate multi-his-write with separate - self._log.debug('No multi-his-write support, emulating') + self._log.debug("No multi-his-write support, emulating") self._state_machine.do_single_write() def _do_multi_write(self, event): """ Request the data from the server as a single multi-read request. """ - self._session.multi_his_write(self._frame, - callback=self._on_multi_write) + self._session.multi_his_write(self._frame, callback=self._on_multi_write) def _on_multi_write(self, operation, **kwargs): """ @@ -752,10 +812,10 @@ def _on_multi_write(self, operation, **kwargs): try: grid = operation.result if not isinstance(grid, hszinc.Grid): - raise ValueError('Unexpected result %r' % grid) + raise ValueError("Unexpected result %r" % grid) self._state_machine.all_write_done(result=None) - except: # Catch all exceptions to pass to caller. - self._log.debug('Hit exception', exc_info=1) + except: # Catch all exceptions to pass to caller. + self._log.debug("Hit exception", exc_info=1) self._state_machine.exception(result=AsynchronousException()) def _do_single_write(self, event): @@ -763,33 +823,41 @@ def _do_single_write(self, event): Submit the data in single write requests. """ for point in self._columns: - self._log.debug('Point %s', point) + self._log.debug("Point %s", point) # Extract a series for this column - series = dict([(r['ts'], r[point]) for r in \ - filter(lambda r : r.get(point) is not None, self._frame)]) - - self._session.his_write_series(point, series, - callback=lambda operation, **kw : \ - self._on_single_write(operation, point=point)) + series = dict( + [ + (r["ts"], r[point]) + for r in filter(lambda r: r.get(point) is not None, self._frame) + ] + ) + + self._session.his_write_series( + point, + series, + callback=lambda operation, **kw: self._on_single_write( + operation, point=point + ), + ) def _on_single_write(self, operation, point, **kwargs): """ Handle the single write. """ - self._log.debug('Response back for point %s', point) + self._log.debug("Response back for point %s", point) try: res = operation.result if res is not None: - raise ValueError('Unexpected result %r' % res) + raise ValueError("Unexpected result %r" % res) self._todo.discard(point) - self._log.debug('Still waiting for: %s', self._todo) + self._log.debug("Still waiting for: %s", self._todo) if not self._todo: # No more to read self._state_machine.all_write_done(result=None) - except: # Catch all exceptions to pass to caller. - self._log.debug('Hit exception', exc_info=1) + except: # Catch all exceptions to pass to caller. + self._log.debug("Hit exception", exc_info=1) self._state_machine.exception(result=AsynchronousException()) def _do_done(self, event): @@ -800,11 +868,14 @@ def _do_done(self, event): if HAVE_PANDAS: + class MetaSeries(Series): """ Custom Pandas Serie with meta data """ + meta = {} + @property def _constructor(self): return MetaSeries @@ -812,13 +883,14 @@ def _constructor(self): def add_meta(self, key, value): self.meta[key] = value - class MetaDataFrame(DataFrame): """ Custom Pandas Dataframe with meta data Made from MetaSeries """ + meta = {} + def __init__(self, *args, **kw): super(MetaDataFrame, self).__init__(*args, **kw) diff --git a/pyhaystack/client/ops/vendor/niagara.py b/pyhaystack/client/ops/vendor/niagara.py index 1d985e4..e751205 100644 --- a/pyhaystack/client/ops/vendor/niagara.py +++ b/pyhaystack/client/ops/vendor/niagara.py @@ -13,6 +13,7 @@ from ...http.auth import BasicAuthenticationCredentials from ...http.exceptions import HTTPStatusError + class NiagaraAXAuthenticateOperation(state.HaystackOperation): """ An implementation of the log-in procedure for Niagara AX. The procedure @@ -27,7 +28,7 @@ class NiagaraAXAuthenticateOperation(state.HaystackOperation): Future requests should include the basic authentication credentials. """ - _LOGIN_RE = re.compile('login', re.IGNORECASE) + _LOGIN_RE = re.compile("login", re.IGNORECASE) def __init__(self, session, retries=0): """ @@ -52,25 +53,29 @@ def __init__(self, session, retries=0): self._retries = retries self._session = session self._cookies = {} - self._auth = BasicAuthenticationCredentials(session._username, - session._password) + self._auth = BasicAuthenticationCredentials( + session._username, session._password + ) self._state_machine = fysom.Fysom( - initial='init', final='done', - events=[ - # Event Current State New State - ('get_new_session', 'init', 'newsession'), - ('do_login', 'newsession', 'login'), - ('login_done', 'login', 'done'), - ('exception', '*', 'failed'), - ('retry', 'failed', 'newsession'), - ('abort', 'failed', 'done'), - ], callbacks={ - 'onenternewsession': self._do_new_session, - 'onenterlogin': self._do_login, - 'onenterfailed': self._do_fail_retry, - 'onenterdone': self._do_done, - }) + initial="init", + final="done", + events=[ + # Event Current State New State + ("get_new_session", "init", "newsession"), + ("do_login", "newsession", "login"), + ("login_done", "login", "done"), + ("exception", "*", "failed"), + ("retry", "failed", "newsession"), + ("abort", "failed", "done"), + ], + callbacks={ + "onenternewsession": self._do_new_session, + "onenterlogin": self._do_login, + "onenterfailed": self._do_fail_retry, + "onenterdone": self._do_done, + }, + ) def go(self): """ @@ -79,7 +84,7 @@ def go(self): # Are we logged in? try: self._state_machine.get_new_session() - except: # Catch all exceptions to pass to caller. + except: # Catch all exceptions to pass to caller. self._state_machine.exception(result=AsynchronousException()) def _do_new_session(self, event): @@ -87,10 +92,16 @@ def _do_new_session(self, event): Request the log-in cookie. """ try: - self._session._get('login', self._on_new_session, - cookies={}, headers={}, exclude_cookies=True, - exclude_headers=True, api=False) - except: # Catch all exceptions to pass to caller. + self._session._get( + "login", + self._on_new_session, + cookies={}, + headers={}, + exclude_cookies=True, + exclude_headers=True, + api=False, + ) + except: # Catch all exceptions to pass to caller. self._state_machine.exception(result=AsynchronousException()) def _on_new_session(self, response): @@ -109,30 +120,36 @@ def _on_new_session(self, response): self._cookies = response.cookies.copy() self._state_machine.do_login() - except: # Catch all exceptions to pass to caller. + except: # Catch all exceptions to pass to caller. self._state_machine.exception(result=AsynchronousException()) def _do_login(self, event): try: # Cover Niagara AX 3.7 where cookies are handled differently... try: - niagara_session = self._cookies['niagara_session'] + niagara_session = self._cookies["niagara_session"] except KeyError: niagara_session = "" - self._session._post('login', self._on_login, - params={ - 'token':'', - 'scheme':'cookieDigest', - 'absPathBase':'/', - 'content-type':'application/x-niagara-login-support', - 'Referer':self._session._client.uri+'login/', - 'accept':'text/zinc; charset=utf-8', - 'cookiePostfix' : niagara_session, - }, - headers={}, cookies=self._cookies, - exclude_cookies=True, exclude_proxies=True, - api=False, auth=self._auth) - except: # Catch all exceptions to pass to caller. + self._session._post( + "login", + self._on_login, + params={ + "token": "", + "scheme": "cookieDigest", + "absPathBase": "/", + "content-type": "application/x-niagara-login-support", + "Referer": self._session._client.uri + "login/", + "accept": "text/zinc; charset=utf-8", + "cookiePostfix": niagara_session, + }, + headers={}, + cookies=self._cookies, + exclude_cookies=True, + exclude_proxies=True, + api=False, + auth=self._auth, + ) + except: # Catch all exceptions to pass to caller. self._state_machine.exception(result=AsynchronousException()) def _on_login(self, response): @@ -152,10 +169,10 @@ def _on_login(self, response): else: if self._LOGIN_RE.match(response.text): # No good. - raise IOError('Login failed') + raise IOError("Login failed") self._state_machine.login_done(result=(self._auth, self._cookies)) - except: # Catch all exceptions to pass to caller. + except: # Catch all exceptions to pass to caller. self._state_machine.exception(result=AsynchronousException()) def _do_fail_retry(self, event): diff --git a/pyhaystack/client/ops/vendor/niagara_scram.py b/pyhaystack/client/ops/vendor/niagara_scram.py index c64daee..ef8c151 100644 --- a/pyhaystack/client/ops/vendor/niagara_scram.py +++ b/pyhaystack/client/ops/vendor/niagara_scram.py @@ -16,6 +16,7 @@ from ....util.asyncexc import AsynchronousException from ...http.exceptions import HTTPStatusError + class Niagara4ScramAuthenticateOperation(state.HaystackOperation): """ An implementation of the log-in procedure for Niagara4. The procedure @@ -32,7 +33,7 @@ class Niagara4ScramAuthenticateOperation(state.HaystackOperation): Future requests should use the JSESSIONID cookies returned. """ - _COOKIE_RE = re.compile(r'^cookie[ \t]*:[ \t]*([^=]+)=(.*)$') + _COOKIE_RE = re.compile(r"^cookie[ \t]*:[ \t]*([^=]+)=(.*)$") def __init__(self, session, retries=0): """ @@ -53,37 +54,39 @@ def __init__(self, session, retries=0): self._algorithm = None self._handshake_token = None - self._server_first_msg = None + self._server_first_msg = None self._server_nonce = None self._server_salt = None self._server_iterations = None self._auth_token = None self._auth = None - self._login_uri = '%s' % \ - (session._client.uri) + self._login_uri = "%s" % (session._client.uri) self._state_machine = fysom.Fysom( - initial='init', final='done', - events=[ - # Event Current State New State - ('get_new_session', 'init', 'newsession'), - ('do_prelogin', 'newsession', 'prelogin'), - ('do_first_msg', 'prelogin', 'first_msg'), - ('do_second_msg', 'first_msg', 'second_msg'), - ('do_validate_login', 'second_msg', 'validate_login'), - ('login_done', 'validate_login', 'done'), - ('exception', '*', 'failed'), - ('retry', 'failed', 'newsession'), - ('abort', 'failed', 'done'), - ], callbacks={ - 'onenternewsession': self._do_new_session, - 'onenterprelogin': self._do_prelogin, - 'onenterfirst_msg': self._do_first_msg, - 'onentersecond_msg': self._do_second_msg, - 'onentervalidate_login': self._do_validate_login, - 'onenterfailed': self._do_fail_retry, - 'onenterdone': self._do_done, - }) + initial="init", + final="done", + events=[ + # Event Current State New State + ("get_new_session", "init", "newsession"), + ("do_prelogin", "newsession", "prelogin"), + ("do_first_msg", "prelogin", "first_msg"), + ("do_second_msg", "first_msg", "second_msg"), + ("do_validate_login", "second_msg", "validate_login"), + ("login_done", "validate_login", "done"), + ("exception", "*", "failed"), + ("retry", "failed", "newsession"), + ("abort", "failed", "done"), + ], + callbacks={ + "onenternewsession": self._do_new_session, + "onenterprelogin": self._do_prelogin, + "onenterfirst_msg": self._do_first_msg, + "onentersecond_msg": self._do_second_msg, + "onentervalidate_login": self._do_validate_login, + "onenterfailed": self._do_fail_retry, + "onenterdone": self._do_done, + }, + ) def go(self): """ @@ -91,7 +94,7 @@ def go(self): """ try: self._state_machine.get_new_session() - except: # Catch all exceptions to pass to caller. + except: # Catch all exceptions to pass to caller. self._state_machine.exception(result=AsynchronousException()) def _do_new_session(self, event): @@ -99,11 +102,16 @@ def _do_new_session(self, event): Reach the prelogin page and clear everything """ try: - self._session._get('%s/prelogin?clear=true' % self._login_uri, - callback=self._on_new_session, - cookies={}, headers={}, exclude_cookies=True, - exclude_headers=True, api=False) - except: # Catch all exceptions to pass to caller. + self._session._get( + "%s/prelogin?clear=true" % self._login_uri, + callback=self._on_new_session, + cookies={}, + headers={}, + exclude_cookies=True, + exclude_headers=True, + api=False, + ) + except: # Catch all exceptions to pass to caller. pass def _on_new_session(self, response): @@ -115,8 +123,8 @@ def _on_new_session(self, response): self._state_machine.do_prelogin() else: raise HTTPStatusError("Unable to connect to server") - - except Exception as e: # Catch all exceptions to pass to caller. + + except Exception as e: # Catch all exceptions to pass to caller. self._state_machine.exception(result=AsynchronousException()) def _do_prelogin(self, event): @@ -124,15 +132,18 @@ def _do_prelogin(self, event): Send the username to the prelogin page """ try: - self._session._post('%s/prelogin' % self._login_uri, - params={'j_username': self._session._username}, - callback=self._on_prelogin, - cookies={}, - headers={}, - exclude_cookies=True, - exclude_headers=True, api=False) - - except: # Catch all exceptions to pass to caller. + self._session._post( + "%s/prelogin" % self._login_uri, + params={"j_username": self._session._username}, + callback=self._on_prelogin, + cookies={}, + headers={}, + exclude_cookies=True, + exclude_headers=True, + api=False, + ) + + except: # Catch all exceptions to pass to caller. pass def _on_prelogin(self, response): @@ -145,23 +156,28 @@ def _on_prelogin(self, response): self.client_first_msg = "n=%s,r=%s" % (self._session._username, self._nonce) self._state_machine.do_first_msg() - except Exception as e: # Catch all exceptions to pass to caller. + except Exception as e: # Catch all exceptions to pass to caller. self._state_machine.exception(result=AsynchronousException()) def _do_first_msg(self, event): """ Send the client first message to server """ - msg = 'action=sendClientFirstMessage&clientFirstMessage=n,,%s' % self.client_first_msg - cookies = dict(niagara_userid = self._session._username) + msg = ( + "action=sendClientFirstMessage&clientFirstMessage=n,,%s" + % self.client_first_msg + ) + cookies = dict(niagara_userid=self._session._username) try: - self._session._post('%s/j_security_check' % (self._login_uri), - body=msg.encode('utf-8'), - callback=self._on_first_msg, - headers={"Content-Type": "application/x-niagara-login-support"}, - cookies=cookies, - api=False) + self._session._post( + "%s/j_security_check" % (self._login_uri), + body=msg.encode("utf-8"), + callback=self._on_first_msg, + headers={"Content-Type": "application/x-niagara-login-support"}, + cookies=cookies, + api=False, + ) except Exception as e: self._state_machine.exception(result=AsynchronousException()) @@ -172,12 +188,14 @@ def _on_first_msg(self, response): This includes the JSESSIONID """ try: - self.jsession = get_jession(response.headers['set-cookie']) - self.server_first_msg = response.body.decode('utf-8') + self.jsession = get_jession(response.headers["set-cookie"]) + self.server_first_msg = response.body.decode("utf-8") tab_response = self.server_first_msg.split(",") - self.server_nonce = scram.regex_after_equal( tab_response[0] ) - self.server_salt = hexlify( scram.b64decode( scram.regex_after_equal( tab_response[1] ) ) ) - self.server_iterations = scram.regex_after_equal( tab_response[2] ) + self.server_nonce = scram.regex_after_equal(tab_response[0]) + self.server_salt = hexlify( + scram.b64decode(scram.regex_after_equal(tab_response[1])) + ) + self.server_iterations = scram.regex_after_equal(tab_response[2]) self._algorithm_name = "sha256" self._algorithm = sha256 @@ -186,30 +204,44 @@ def _on_first_msg(self, response): except Exception as e: self._state_machine.exception(result=AsynchronousException()) - def _do_second_msg(self, event): """ Send the client second (final) message to server """ - self.salted_password = scram.salted_password_2( self.server_salt, self.server_iterations, self._algorithm_name, self._session._password ) - client_final_without_proof = "c=%s,r=%s" % ( scram.standard_b64encode(b'n,,').decode(), - self.server_nonce ) - self.auth_msg = "%s,%s,%s" % ( self.client_first_msg, self.server_first_msg, - client_final_without_proof ) - - client_proof = _createClientProof(self.salted_password, self.auth_msg, self._algorithm) + self.salted_password = scram.salted_password_2( + self.server_salt, + self.server_iterations, + self._algorithm_name, + self._session._password, + ) + client_final_without_proof = "c=%s,r=%s" % ( + scram.standard_b64encode(b"n,,").decode(), + self.server_nonce, + ) + self.auth_msg = "%s,%s,%s" % ( + self.client_first_msg, + self.server_first_msg, + client_final_without_proof, + ) + + client_proof = _createClientProof( + self.salted_password, self.auth_msg, self._algorithm + ) client_final_message = client_final_without_proof + ",p=" + client_proof - final_msg = 'action=sendClientFinalMessage&clientFinalMessage=%s' % (client_final_message) + final_msg = "action=sendClientFinalMessage&clientFinalMessage=%s" % ( + client_final_message + ) - cookies = dict(niagara_userid = self._session._username, - JSESSIONID = self.jsession) + cookies = dict(niagara_userid=self._session._username, JSESSIONID=self.jsession) try: - self._session._post('%s/j_security_check' % self._login_uri, - body=final_msg.strip().encode("utf-8"), - callback=self._on_second_msg, - headers={"Content-Type": "application/x-niagara-login-support"}, - cookies=cookies, - api=False) + self._session._post( + "%s/j_security_check" % self._login_uri, + body=final_msg.strip().encode("utf-8"), + callback=self._on_second_msg, + headers={"Content-Type": "application/x-niagara-login-support"}, + cookies=cookies, + api=False, + ) except: self._state_machine.exception(result=AsynchronousException()) @@ -219,32 +251,46 @@ def _on_second_msg(self, response): We will compare signatures to validate the authentication """ try: - server_final_message = response.body.decode('utf-8') - server_key = hmac.new( unhexlify( self.salted_password ), "Server Key".encode('UTF-8'), self._algorithm).hexdigest() - server_signature = hmac.new( unhexlify( server_key ) , self.auth_msg.encode() , self._algorithm ).hexdigest() - remote_server_signature = hexlify( scram.b64decode( scram.regex_after_equal( server_final_message ) ) ) + server_final_message = response.body.decode("utf-8") + server_key = hmac.new( + unhexlify(self.salted_password), + "Server Key".encode("UTF-8"), + self._algorithm, + ).hexdigest() + server_signature = hmac.new( + unhexlify(server_key), self.auth_msg.encode(), self._algorithm + ).hexdigest() + remote_server_signature = hexlify( + scram.b64decode(scram.regex_after_equal(server_final_message)) + ) if server_signature == remote_server_signature.decode(): - cookies = dict(JSESSIONID=self.jsession, niagara_userid=self._session._username) + cookies = dict( + JSESSIONID=self.jsession, niagara_userid=self._session._username + ) self._session._client.cookies = cookies self._state_machine.do_validate_login() else: - raise Exception('Login Failed, local and remote signature are different') + raise Exception( + "Login Failed, local and remote signature are different" + ) except Exception as e: - self._state_machine.exception(result=AsynchronousException()) + self._state_machine.exception(result=AsynchronousException()) def _do_validate_login(self, event): """ We need to send another request to the server to validate the login """ try: - self._session._post('%s/j_security_check' % self._login_uri, - body=None, - callback=self._on_validate_login, - headers={"Content-Type": "application/x-niagara-login-support"}, - api=False) + self._session._post( + "%s/j_security_check" % self._login_uri, + body=None, + callback=self._on_validate_login, + headers={"Content-Type": "application/x-niagara-login-support"}, + api=False, + ) except: self._state_machine.exception(result=AsynchronousException()) @@ -253,14 +299,15 @@ def _on_validate_login(self, response): Retrieve the response and set authenticated status """ try: + if isinstance(response, AsynchronousException): + response.reraise() if response.status_code == 200: - self._state_machine.login_done(result={'authenticated': True}) + self._state_machine.login_done(result={"authenticated": True}) else: - raise HTTPStatusError('Server refused the last message') + raise HTTPStatusError("Server refused the last message") except Exception as e: - self._state_machine.exception(result=AsynchronousException()) - + self._state_machine.exception(result=AsynchronousException()) def _do_fail_retry(self, event): """ @@ -278,26 +325,33 @@ def _do_done(self, event): """ self._done(event.result) -def binary_encoding(string, encoding = 'utf-8'): + +def binary_encoding(string, encoding="utf-8"): """ This helper function will allow compatibility with Python 2 and 3 """ try: return bytes(string, encoding) - except TypeError: # We are in Python 2 + except TypeError: # We are in Python 2 return str(string) + def get_jession(arg_header): - set_cookie = arg_header.split(',') + set_cookie = arg_header.split(",") for key in set_cookie: if "JSESSIONID=" in key: jsession = scram.regex_after_equal(key) jsession = jsession.split(";")[0] return jsession + def _createClientProof(salted_password, auth_msg, algorithm): - client_key = hmac.new( unhexlify( salted_password ), "Client Key".encode('UTF-8'), algorithm).hexdigest() - stored_key = scram._hash_sha256( unhexlify(client_key), algorithm ) - client_signature = hmac.new( unhexlify( stored_key ) , auth_msg.encode() , algorithm ).hexdigest() - client_proof = scram._xor (client_key, client_signature) - return b2a_base64(unhexlify(client_proof)).decode('utf-8') + client_key = hmac.new( + unhexlify(salted_password), "Client Key".encode("UTF-8"), algorithm + ).hexdigest() + stored_key = scram._hash_sha256(unhexlify(client_key), algorithm) + client_signature = hmac.new( + unhexlify(stored_key), auth_msg.encode(), algorithm + ).hexdigest() + client_proof = scram._xor(client_key, client_signature) + return b2a_base64(unhexlify(client_proof)).decode("utf-8") diff --git a/pyhaystack/client/ops/vendor/skyspark.py b/pyhaystack/client/ops/vendor/skyspark.py index 13c79fc..4875d17 100644 --- a/pyhaystack/client/ops/vendor/skyspark.py +++ b/pyhaystack/client/ops/vendor/skyspark.py @@ -14,6 +14,7 @@ from ....util import state from ....util.asyncexc import AsynchronousException + class SkysparkAuthenticateOperation(state.HaystackOperation): """ An implementation of the log-in procedure for Skyspark. The procedure @@ -34,7 +35,7 @@ class SkysparkAuthenticateOperation(state.HaystackOperation): Future requests should the cookies returned. """ - _COOKIE_RE = re.compile(r'^cookie[ \t]*:[ \t]*([^=]+)=(.*)$') + _COOKIE_RE = re.compile(r"^cookie[ \t]*:[ \t]*([^=]+)=(.*)$") def __init__(self, session, retries=2): """ @@ -52,24 +53,30 @@ def __init__(self, session, retries=2): self._username = None self._user_salt = None self._digest = None - self._login_uri = '%s/auth/%s/api?%s' % \ - (session._client.uri, session._project, session._username) + self._login_uri = "%s/auth/%s/api?%s" % ( + session._client.uri, + session._project, + session._username, + ) self._state_machine = fysom.Fysom( - initial='init', final='done', - events=[ - # Event Current State New State - ('get_new_session', 'init', 'newsession'), - ('do_login', 'newsession', 'login'), - ('login_done', 'login', 'done'), - ('exception', '*', 'failed'), - ('retry', 'failed', 'newsession'), - ('abort', 'failed', 'done'), - ], callbacks={ - 'onenternewsession': self._do_new_session, - 'onenterlogin': self._do_login, - 'onenterfailed': self._do_fail_retry, - 'onenterdone': self._do_done, - }) + initial="init", + final="done", + events=[ + # Event Current State New State + ("get_new_session", "init", "newsession"), + ("do_login", "newsession", "login"), + ("login_done", "login", "done"), + ("exception", "*", "failed"), + ("retry", "failed", "newsession"), + ("abort", "failed", "done"), + ], + callbacks={ + "onenternewsession": self._do_new_session, + "onenterlogin": self._do_login, + "onenterfailed": self._do_fail_retry, + "onenterdone": self._do_done, + }, + ) def go(self): """ @@ -78,7 +85,7 @@ def go(self): # Are we logged in? try: self._state_machine.get_new_session() - except: # Catch all exceptions to pass to caller. + except: # Catch all exceptions to pass to caller. self._state_machine.exception(result=AsynchronousException()) def _do_new_session(self, event): @@ -86,13 +93,17 @@ def _do_new_session(self, event): Request the log-in parameters. """ try: - self._session._get(self._login_uri, - callback=self._on_new_session, - - cookies={}, headers={}, exclude_cookies=True, - exclude_headers=True, api=False) - #args={'username': self._session._username}, - except: # Catch all exceptions to pass to caller. + self._session._get( + self._login_uri, + callback=self._on_new_session, + cookies={}, + headers={}, + exclude_cookies=True, + exclude_headers=True, + api=False, + ) + # args={'username': self._session._username}, + except: # Catch all exceptions to pass to caller. self._state_machine.exception(result=AsynchronousException()) def _on_new_session(self, response): @@ -104,36 +115,39 @@ def _on_new_session(self, response): response.reraise() login_params = {} - for line in response.text.split('\n'): - key, value = line.split(':') + for line in response.text.split("\n"): + key, value = line.split(":") login_params[key] = value - self._username = login_params['username'] - self._user_salt = login_params['userSalt'] - self._nonce = login_params['nonce'] + self._username = login_params["username"] + self._user_salt = login_params["userSalt"] + self._nonce = login_params["nonce"] self._state_machine.do_login() - except: # Catch all exceptions to pass to caller. + except: # Catch all exceptions to pass to caller. self._state_machine.exception(result=AsynchronousException()) def _do_login(self, event): try: login_params = { - 'username' : self._session._username, - 'password' : self._session._password, - 'userSalt' : self._user_salt, - 'nonce' : self._nonce + "username": self._session._username, + "password": self._session._password, + "userSalt": self._user_salt, + "nonce": self._nonce, } - self._digest = get_digest_info(login_params)['digest'] + self._digest = get_digest_info(login_params)["digest"] # Post - self._session._post(self._login_uri, - callback=self._on_login, - body='nonce:%s\ndigest:%s' % \ - (self._nonce, self._digest), - body_type='text/plain; charset=utf-8', - headers={}, exclude_cookies=True, - exclude_headers=True, api=False) - except: # Catch all exceptions to pass to caller. + self._session._post( + self._login_uri, + callback=self._on_login, + body="nonce:%s\ndigest:%s" % (self._nonce, self._digest), + body_type="text/plain; charset=utf-8", + headers={}, + exclude_cookies=True, + exclude_headers=True, + api=False, + ) + except: # Catch all exceptions to pass to caller. self._state_machine.exception(result=AsynchronousException()) def _on_login(self, response): @@ -147,12 +161,12 @@ def _on_login(self, response): # Locate the cookie in the response. cookie_match = self._COOKIE_RE.match(response.text) if not cookie_match: - raise IOError('No cookie in response, log-in failed.') + raise IOError("No cookie in response, log-in failed.") (cookie_name, cookie_value) = cookie_match.groups() self._state_machine.login_done(result={cookie_name: cookie_value}) - except: # Catch all exceptions to pass to caller. + except: # Catch all exceptions to pass to caller. self._state_machine.exception(result=AsynchronousException()) def _do_fail_retry(self, event): @@ -171,26 +185,32 @@ def _do_done(self, event): """ self._done(event.result) + def get_digest_info(param): - message = binary_encoding("%s:%s" % (param['username'], param['userSalt'])) - password_buf = binary_encoding(param['password']) - hmac_final = base64.b64encode(hmac.new(key=password_buf, msg=message, digestmod=hashlib.sha1).digest()) + message = binary_encoding("%s:%s" % (param["username"], param["userSalt"])) + password_buf = binary_encoding(param["password"]) + hmac_final = base64.b64encode( + hmac.new(key=password_buf, msg=message, digestmod=hashlib.sha1).digest() + ) - digest_msg = binary_encoding('%s:%s' % (hmac_final.decode('utf-8'), param['nonce'])) + digest_msg = binary_encoding("%s:%s" % (hmac_final.decode("utf-8"), param["nonce"])) digest = hashlib.sha1() digest.update(digest_msg) digest_final = base64.b64encode((digest.digest())) - res ={'hmac' : hmac_final.decode('utf-8'), - 'digest' : digest_final.decode('utf-8'), - 'nonce' : param['nonce']} + res = { + "hmac": hmac_final.decode("utf-8"), + "digest": digest_final.decode("utf-8"), + "nonce": param["nonce"], + } return res -def binary_encoding(string, encoding = 'utf-8'): + +def binary_encoding(string, encoding="utf-8"): """ This helper function will allow compatibility with Python 2 and 3 """ try: return bytes(string, encoding) - except TypeError: # We are in Python 2 + except TypeError: # We are in Python 2 return str(string) diff --git a/pyhaystack/client/ops/vendor/skyspark_scram.py b/pyhaystack/client/ops/vendor/skyspark_scram.py index 2993ede..6cff71a 100644 --- a/pyhaystack/client/ops/vendor/skyspark_scram.py +++ b/pyhaystack/client/ops/vendor/skyspark_scram.py @@ -18,6 +18,7 @@ from ....util.asyncexc import AsynchronousException from ...http.exceptions import HTTPStatusError + class SkysparkScramAuthenticateOperation(state.HaystackOperation): """ An implementation of the log-in procedure for Skyspark. The procedure @@ -33,7 +34,7 @@ class SkysparkScramAuthenticateOperation(state.HaystackOperation): Future requests should the cookies returned. """ - _COOKIE_RE = re.compile(r'^cookie[ \t]*:[ \t]*([^=]+)=(.*)$') + _COOKIE_RE = re.compile(r"^cookie[ \t]*:[ \t]*([^=]+)=(.*)$") def __init__(self, session, retries=2): """ @@ -54,35 +55,37 @@ def __init__(self, session, retries=2): self._algorithm = None self._handshake_token = None - self._server_first_msg = None + self._server_first_msg = None self._server_nonce = None self._server_salt = None self._server_iterations = None self._auth_token = None self._auth = None - self._login_uri = '%s' % \ - (session._client.uri) + self._login_uri = "%s" % (session._client.uri) self._state_machine = fysom.Fysom( - initial='init', final='done', - events=[ - # Event Current State New State - ('get_new_session', 'init', 'newsession'), - ('do_hs_token', 'newsession', 'handshake_token'), - ('do_second_msg', 'handshake_token', 'second_msg'), - ('do_server_token', 'second_msg', 'server_token'), - ('login_done', 'server_token', 'done'), - ('exception', '*', 'failed'), - ('retry', 'failed', 'newsession'), - ('abort', 'failed', 'done'), - ], callbacks={ - 'onenternewsession': self._do_new_session, - 'onenterhandshake_token': self._do_hs_token, - 'onentersecond_msg': self._do_second_msg, - 'onenterserver_token': self._do_server_token, - 'onenterfailed': self._do_fail_retry, - 'onenterdone': self._do_done, - }) + initial="init", + final="done", + events=[ + # Event Current State New State + ("get_new_session", "init", "newsession"), + ("do_hs_token", "newsession", "handshake_token"), + ("do_second_msg", "handshake_token", "second_msg"), + ("do_server_token", "second_msg", "server_token"), + ("login_done", "server_token", "done"), + ("exception", "*", "failed"), + ("retry", "failed", "newsession"), + ("abort", "failed", "done"), + ], + callbacks={ + "onenternewsession": self._do_new_session, + "onenterhandshake_token": self._do_hs_token, + "onentersecond_msg": self._do_second_msg, + "onenterserver_token": self._do_server_token, + "onenterfailed": self._do_fail_retry, + "onenterdone": self._do_done, + }, + ) def go(self): """ @@ -91,7 +94,7 @@ def go(self): # Are we logged in? try: self._state_machine.get_new_session() - except: # Catch all exceptions to pass to caller. + except: # Catch all exceptions to pass to caller. self._state_machine.exception(result=AsynchronousException()) def _do_new_session(self, event): @@ -99,49 +102,56 @@ def _do_new_session(self, event): Test if server respond... """ try: - self._session._get('%s/user/login' % self._login_uri, - callback=self._on_new_session, - cookies={}, headers={}, exclude_cookies=True, - exclude_headers=True, api=False) - #args={'username': self._session._username}, - except: # Catch all exceptions to pass to caller. + self._session._get( + "%s/user/login" % self._login_uri, + callback=self._on_new_session, + cookies={}, + headers={}, + exclude_cookies=True, + exclude_headers=True, + api=False, + ) + # args={'username': self._session._username}, + except: # Catch all exceptions to pass to caller. pass - def _on_new_session(self, response): """ Retrieve the log-in parameters. """ try: - #if isinstance(response, AsynchronousException): + # if isinstance(response, AsynchronousException): # response.reraise() self._nonce = scram.get_nonce() self._salt_username = scram.base64_no_padding(self._session._username) self.client_first_message = "HELLO username=%s" % (self._salt_username) self._state_machine.do_hs_token() - except Exception as e: # Catch all exceptions to pass to caller. + except Exception as e: # Catch all exceptions to pass to caller. self._state_machine.exception(result=AsynchronousException()) def _do_hs_token(self, event): try: - self._session._get('%s/ui' % self._login_uri, - callback=self._validate_hs_token, - headers={"Authorization": self.client_first_message}, - exclude_cookies=True, api=False) + self._session._get( + "%s/ui" % self._login_uri, + callback=self._validate_hs_token, + headers={"Authorization": self.client_first_message}, + exclude_cookies=True, + api=False, + ) except Exception as e: self._state_machine.exception(result=AsynchronousException()) def _validate_hs_token(self, response): try: - response.reraise() # ← AsynchronousException class + response.reraise() # ← AsynchronousException class except HTTPStatusError as e: if e.status != 401 and e.status != 303: raise try: - server_response = e.headers['WWW-Authenticate'] - header_response = server_response.split(',') - algorithm = scram.regex_after_equal( header_response[1] ) + server_response = e.headers["WWW-Authenticate"] + header_response = server_response.split(",") + algorithm = scram.regex_after_equal(header_response[1]) algorithm_name = algorithm.replace("-", "").lower() if algorithm_name == "sha256": self._algorithm = sha256 @@ -150,42 +160,47 @@ def _validate_hs_token(self, response): self._algorithm = sha1 self._algorithm_name = "sha1" else: - raise Exception('SHA not implemented') + raise Exception("SHA not implemented") self._handshake_token = scram.regex_after_equal(header_response[0]) self._state_machine.do_second_msg() except Exception as e: self._state_machine.exception(result=AsynchronousException()) - def _do_second_msg(self, event): self._client_second_msg = "n=%s,r=%s" % (self._session._username, self._nonce) client_second_msg_encoded = scram.base64_no_padding(self._client_second_msg) - authMsg = "SCRAM handshakeToken=%s, data=%s" % (self._handshake_token , client_second_msg_encoded ) + authMsg = "SCRAM handshakeToken=%s, data=%s" % ( + self._handshake_token, + client_second_msg_encoded, + ) try: # Post - self._session._get('%s/ui' % self._login_uri, - callback=self._validate_sec_msg, - headers={"Authorization": authMsg}, - exclude_cookies=True, - exclude_headers=True, api=False) + self._session._get( + "%s/ui" % self._login_uri, + callback=self._validate_sec_msg, + headers={"Authorization": authMsg}, + exclude_cookies=True, + exclude_headers=True, + api=False, + ) except: self._state_machine.exception(result=AsynchronousException()) def _validate_sec_msg(self, response): try: - response.reraise() # ← AsynchronousException class + response.reraise() # ← AsynchronousException class except HTTPStatusError as e: if e.status != 401 and e.status != 303: raise try: - header_response = e.headers['WWW-Authenticate'] - tab_header = header_response.split(',') + header_response = e.headers["WWW-Authenticate"] + tab_header = header_response.split(",") server_data = scram.regex_after_equal(tab_header[0]) missing_padding = len(server_data) % 4 if missing_padding != 0: - server_data += '='* (4 - missing_padding) + server_data += "=" * (4 - missing_padding) server_data = scram.b64decode(server_data).decode() - tab_response = server_data.split(',') + tab_response = server_data.split(",") self._server_first_msg = server_data self._server_nonce = scram.regex_after_equal(tab_response[0]) self._server_salt = scram.regex_after_equal(tab_response[1]) @@ -198,34 +213,62 @@ def _validate_sec_msg(self, response): self._state_machine.exception(result=AsynchronousException()) def _do_server_token(self, event): - client_final_no_proof = "c=%s,r=%s" % ( scram.standard_b64encode(b'n,,').decode() , self._server_nonce ) - auth_msg = "%s,%s,%s" % ( self._client_second_msg, self._server_first_msg, client_final_no_proof ) - client_key = hmac.new(unhexlify(scram.salted_password(self._server_salt, self._server_iterations, self._algorithm_name, self._session._password)), "Client Key".encode('UTF-8'), self._algorithm).hexdigest() - stored_key = scram._hash_sha256(unhexlify(client_key), self._algorithm) - client_signature = hmac.new( unhexlify(stored_key), auth_msg.encode('utf-8'), self._algorithm).hexdigest() - client_proof = scram._xor(client_key, client_signature) - client_proof_encode = b2a_base64(unhexlify(client_proof)).decode() - client_final = client_final_no_proof + ",p=" + client_proof_encode - client_final_base64 = scram.base64_no_padding(client_final) - final_msg = "scram handshaketoken=%s,data=%s" % (self._handshake_token , client_final_base64) + client_final_no_proof = "c=%s,r=%s" % ( + scram.standard_b64encode(b"n,,").decode(), + self._server_nonce, + ) + auth_msg = "%s,%s,%s" % ( + self._client_second_msg, + self._server_first_msg, + client_final_no_proof, + ) + client_key = hmac.new( + unhexlify( + scram.salted_password( + self._server_salt, + self._server_iterations, + self._algorithm_name, + self._session._password, + ) + ), + "Client Key".encode("UTF-8"), + self._algorithm, + ).hexdigest() + stored_key = scram._hash_sha256(unhexlify(client_key), self._algorithm) + client_signature = hmac.new( + unhexlify(stored_key), auth_msg.encode("utf-8"), self._algorithm + ).hexdigest() + client_proof = scram._xor(client_key, client_signature) + client_proof_encode = b2a_base64(unhexlify(client_proof)).decode() + client_final = client_final_no_proof + ",p=" + client_proof_encode + client_final_base64 = scram.base64_no_padding(client_final) + final_msg = "scram handshaketoken=%s,data=%s" % ( + self._handshake_token, + client_final_base64, + ) try: - self._session._get('%s/ui' % self._login_uri, - callback=self._validate_server_token, - headers={"Authorization": final_msg}, - exclude_cookies=True, - exclude_headers=True, api=False) + self._session._get( + "%s/ui" % self._login_uri, + callback=self._validate_server_token, + headers={"Authorization": final_msg}, + exclude_cookies=True, + exclude_headers=True, + api=False, + ) except Exception as e: self._state_machine.exception(result=AsynchronousException()) def _validate_server_token(self, response): try: - server_response = response.headers['Authentication-Info'] - tab_response = server_response.split(',') + server_response = response.headers["Authentication-Info"] + tab_response = server_response.split(",") self._auth_token = scram.regex_after_equal(tab_response[0]) self._auth = "BEARER authToken=%s" % self._auth_token - self._state_machine.login_done(result={'header': {'Authorization': self._auth} }) + self._state_machine.login_done( + result={"header": {"Authorization": self._auth}} + ) except Exception as e: self._state_machine.exception(result=AsynchronousException()) @@ -246,26 +289,32 @@ def _do_done(self, event): """ self._done(event.result) + def get_digest_info(param): - message = binary_encoding("%s:%s" % (param['username'], param['userSalt'])) - password_buf = binary_encoding(param['password']) - hmac_final = base64.b64encode(hmac.new(key=password_buf, msg=message, digestmod=hashlib.sha1).digest()) + message = binary_encoding("%s:%s" % (param["username"], param["userSalt"])) + password_buf = binary_encoding(param["password"]) + hmac_final = base64.b64encode( + hmac.new(key=password_buf, msg=message, digestmod=hashlib.sha1).digest() + ) - digest_msg = binary_encoding('%s:%s' % (hmac_final.decode('utf-8'), param['nonce'])) + digest_msg = binary_encoding("%s:%s" % (hmac_final.decode("utf-8"), param["nonce"])) digest = hashlib.sha1() digest.update(digest_msg) digest_final = base64.b64encode((digest.digest())) - res ={'hmac' : hmac_final.decode('utf-8'), - 'digest' : digest_final.decode('utf-8'), - 'nonce' : param['nonce']} + res = { + "hmac": hmac_final.decode("utf-8"), + "digest": digest_final.decode("utf-8"), + "nonce": param["nonce"], + } return res -def binary_encoding(string, encoding = 'utf-8'): + +def binary_encoding(string, encoding="utf-8"): """ This helper function will allow compatibility with Python 2 and 3 """ try: return bytes(string, encoding) - except TypeError: # We are in Python 2 + except TypeError: # We are in Python 2 return str(string) diff --git a/pyhaystack/client/ops/vendor/widesky.py b/pyhaystack/client/ops/vendor/widesky.py index 0cbd1a7..16cb13a 100644 --- a/pyhaystack/client/ops/vendor/widesky.py +++ b/pyhaystack/client/ops/vendor/widesky.py @@ -18,6 +18,7 @@ from ..feature import HasFeaturesOperation from ...session import HaystackSession + class WideskyAuthenticateOperation(state.HaystackOperation): """ An implementation of the log-in procedure for WideSky. WideSky uses @@ -53,37 +54,45 @@ def __init__(self, session, retries=0): super(WideskyAuthenticateOperation, self).__init__() self._auth_headers = { - 'Authorization': (u'Basic %s' % base64.b64encode( - ':'.join([session._client_id, - session._client_secret]).encode('utf-8') - ).decode('us-ascii') - ).encode('us-ascii'), - 'Accept': 'application/json', - 'Content-Type': 'application/json', + "Authorization": ( + u"Basic %s" + % base64.b64encode( + ":".join([session._client_id, session._client_secret]).encode( + "utf-8" + ) + ).decode("us-ascii") + ).encode("us-ascii"), + "Accept": "application/json", + "Content-Type": "application/json", } - self._auth_body = json.dumps({ - 'username': session._username, - 'password': session._password, - 'grant_type': 'password', - }).encode('utf-8') + self._auth_body = json.dumps( + { + "username": session._username, + "password": session._password, + "grant_type": "password", + } + ).encode("utf-8") self._session = session self._retries = retries self._auth_result = None self._state_machine = fysom.Fysom( - initial='init', final='done', - events=[ - # Event Current State New State - ('do_login', ['init', 'failed'], 'login'), - ('login_done', 'login', 'done'), - ('exception', '*', 'failed'), - ('retry', 'failed', 'login'), - ('abort', 'failed', 'done'), - ], callbacks={ - 'onenterlogin': self._do_login, - 'onenterfailed': self._do_fail_retry, - 'onenterdone': self._do_done, - }) + initial="init", + final="done", + events=[ + # Event Current State New State + ("do_login", ["init", "failed"], "login"), + ("login_done", "login", "done"), + ("exception", "*", "failed"), + ("retry", "failed", "login"), + ("abort", "failed", "done"), + ], + callbacks={ + "onenterlogin": self._do_login, + "onenterfailed": self._do_fail_retry, + "onenterdone": self._do_done, + }, + ) def go(self): """ @@ -92,16 +101,20 @@ def go(self): # Are we logged in? try: self._state_machine.do_login() - except: # Catch all exceptions to pass to caller. + except: # Catch all exceptions to pass to caller. self._state_machine.exception(result=AsynchronousException()) def _do_login(self, event): try: - self._session._post(self._session._auth_dir, - self._on_login, body=self._auth_body, - headers=self._auth_headers, exclude_headers=True, - api=False) - except: # Catch all exceptions to pass to caller. + self._session._post( + self._session._auth_dir, + self._on_login, + body=self._auth_body, + headers=self._auth_headers, + exclude_headers=True, + api=False, + ) + except: # Catch all exceptions to pass to caller. self._state_machine.exception(result=AsynchronousException()) def _on_login(self, response): @@ -114,19 +127,18 @@ def _on_login(self, response): content_type = response.content_type if content_type is None: - raise ValueError('No content-type given in reply') - if content_type != 'application/json': - raise ValueError('Invalid content type received: %s' % \ - content_type) + raise ValueError("No content-type given in reply") + if content_type != "application/json": + raise ValueError("Invalid content type received: %s" % content_type) # Decode JSON reply reply = json.loads(response.text) - for key in ('token_type', 'access_token', 'expires_in'): + for key in ("token_type", "access_token", "expires_in"): if key not in reply: - raise ValueError('Missing %s in reply :%s' % (key, reply)) + raise ValueError("Missing %s in reply :%s" % (key, reply)) self._state_machine.login_done(result=reply) - except: # Catch all exceptions to pass to caller. + except: # Catch all exceptions to pass to caller. self._state_machine.exception(result=AsynchronousException()) def _do_fail_retry(self, event): @@ -156,19 +168,20 @@ def __init__(self, session, entities, single): :param session: Haystack HTTP session object. :param entities: A list of entities to create. """ - self._log = session._log.getChild('create_entity') + self._log = session._log.getChild("create_entity") super(CreateEntityOperation, self).__init__(session, single) self._new_entities = entities self._state_machine = fysom.Fysom( - initial='init', final='done', - events=[ - # Event Current State New State - ('send_create', 'init', 'create'), - ('read_done', 'create', 'done'), - ('exception', '*', 'done'), - ], callbacks={ - 'onenterdone': self._do_done, - }) + initial="init", + final="done", + events=[ + # Event Current State New State + ("send_create", "init", "create"), + ("read_done", "create", "done"), + ("exception", "*", "done"), + ], + callbacks={"onenterdone": self._do_done}, + ) def go(self): """ @@ -178,23 +191,24 @@ def go(self): # Ensure IDs are basenames. def _preprocess_entity(e): if not isinstance(e, dict): - raise TypeError('%r is not a dict' % e) + raise TypeError("%r is not a dict" % e) e = e.copy() - e_id = e.pop('id') - if isinstance(e_id, hszinc.Ref): - e_id = e_id.name - if '.' in e_id: - e_id = e_id.split('.')[-1] - e['id'] = hszinc.Ref(e_id) + if "id" in e: + e_id = e.pop("id") + if isinstance(e_id, hszinc.Ref): + e_id = e_id.name + if "." in e_id: + e_id = e_id.split(".")[-1] + e["id"] = hszinc.Ref(e_id) return e + entities = list(map(_preprocess_entity, self._new_entities)) self._session.create(entities, callback=self._on_read) class WideSkyHasFeaturesOperation(HasFeaturesOperation): def __init__(self, session, features, **kwargs): - super(WideSkyHasFeaturesOperation, self).__init__( - session, features, **kwargs) + super(WideSkyHasFeaturesOperation, self).__init__(session, features, **kwargs) # Turn on retrieval of 'about' version data. self._need_about = True @@ -203,21 +217,28 @@ def _check_features(self): res = super(WideSkyHasFeaturesOperation, self)._check_features() # Ensure this is WideSky - if self._about_data['productName'] not in ( - 'Widesky Semantic Database Toolkit', 'WideSky'): + if self._about_data["productName"] not in ( + "Widesky Semantic Database Toolkit", + "WideSky", + ): # Not recognised, stop here. return res - # Get the WideSky version, preferring moduleVersion over productVersion - ver = self._about_data.get('moduleVersion', - self._about_data['productVersion']) + ver = self._about_data.get("moduleVersion", self._about_data["productVersion"]) for feature in self._features: - if feature in (HaystackSession.FEATURE_HISREAD_MULTI, - HaystackSession.FEATURE_HISWRITE_MULTI): + if feature in ( + HaystackSession.FEATURE_HISREAD_MULTI, + HaystackSession.FEATURE_HISWRITE_MULTI, + ): try: - res[feature] = semver.match(ver, '>=0.5.0') + res[feature] = semver.match(ver, ">=0.5.0") except ValueError: # Unrecognised version string return res + elif feature == HaystackSession.FEATURE_ID_UUID: + try: + res[feature] = semver.match(ver, ">=0.8.0") + except ValueError: + return res return res diff --git a/pyhaystack/client/session.py b/pyhaystack/client/session.py index a69a05c..1954e05 100644 --- a/pyhaystack/client/session.py +++ b/pyhaystack/client/session.py @@ -19,6 +19,7 @@ from .ops import feature as feature_ops from .entity.models.haystack import HaystackTaggingModel + class HaystackSession(object): """ The Haystack Session handler is responsible for presenting an API for @@ -56,10 +57,18 @@ class HaystackSession(object): _HAS_FEATURES_OPERATION = feature_ops.HasFeaturesOperation - def __init__(self, uri, api_dir, grid_format=hszinc.MODE_ZINC, - http_client=sync.SyncHttpClient, http_args=None, - tagging_model=HaystackTaggingModel, log=None, - pint=False, cache_expiry=3600.0): + def __init__( + self, + uri, + api_dir, + grid_format=hszinc.MODE_ZINC, + http_client=sync.SyncHttpClient, + http_args=None, + tagging_model=HaystackTaggingModel, + log=None, + pint=False, + cache_expiry=3600.0, + ): """ Initialise a base Project Haystack session handler. @@ -76,23 +85,22 @@ def __init__(self, uri, api_dir, grid_format=hszinc.MODE_ZINC, See : https://pint.readthedocs.io/ for details about pint """ if log is None: - log = logging.getLogger('pyhaystack.client.%s' \ - % self.__class__.__name__) + log = logging.getLogger("pyhaystack.client.%s" % self.__class__.__name__) self._log = log if http_args is None: http_args = {} - #Configure hszinc to use pint or not for Quantity definition + # Configure hszinc to use pint or not for Quantity definition self.config_pint(pint) if grid_format not in (hszinc.MODE_ZINC, hszinc.MODE_JSON): - raise ValueError('Unrecognised grid format %s' % grid_format) + raise ValueError("Unrecognised grid format %s" % grid_format) self._grid_format = grid_format # Create the HTTP client object - if bool(http_args.pop('debug',None)) and ('log' not in http_args): - http_args['log'] = log.getChild('http_client') + if bool(http_args.pop("debug", None)) and ("log" not in http_args): + http_args["log"] = log.getChild("http_client") self._client = http_client(uri=uri, **http_args) self._api_dir = api_dir @@ -108,7 +116,7 @@ def __init__(self, uri, api_dir, grid_format=hszinc.MODE_ZINC, # Grid cache self._grid_lk = Lock() self._grid_expiry = cache_expiry - self._grid_cache = {} # 'op' -> (op, expiry, grid) + self._grid_cache = {} # 'op' -> (op, expiry, grid) # Public methods/properties @@ -145,19 +153,19 @@ def about(self, cache=True, callback=None): """ Retrieve the version information of this Project Haystack server. """ - return self._get_grid('about', callback, cache=cache) + return self._on_about(cache=cache, callback=callback) def ops(self, cache=True, callback=None): """ Retrieve the operations supported by this Project Haystack server. """ - return self._get_grid('ops', callback, cache=cache) + return self._on_ops(cache=cache, callback=callback) def formats(self, cache=True, callback=None): """ Retrieve the grid formats supported by this Project Haystack server. """ - return self._get_grid('formats', callback, cache=cache) + return self._on_formats(cache=cache, callback=callback) def read(self, ids=None, filter_expr=None, limit=None, callback=None): """ @@ -174,31 +182,9 @@ def read(self, ids=None, filter_expr=None, limit=None, callback=None): of interest. :param limit: A limit on the number of entities to return. """ - if isinstance(ids, string_types) or isinstance(ids, hszinc.Ref): - # Make sure we always pass a list. - ids = [ids] - - if bool(ids): - if filter_expr is not None: - raise ValueError('Either specify ids or filter_expr, not both') - - ids = [self._obj_to_ref(r) for r in ids] - - if len(ids) == 1: - # Reading a single entity - return self._get_grid('read', callback, args={'id': ids[0]}) - else: - # Reading several entities - grid = hszinc.Grid() - grid.column['id'] = {} - grid.extend([{'id': r} for r in ids]) - return self._post_grid('read', grid, callback) - else: - args = {'filter': filter_expr} - if limit is not None: - args['limit'] = int(limit) - - return self._get_grid('read', callback, args=args) + return self._on_read( + ids=ids, filter_expr=filter_expr, limit=limit, callback=callback + ) def nav(self, nav_id=None, callback=None): """ @@ -206,25 +192,23 @@ def nav(self, nav_id=None, callback=None): operation allows servers to expose the database in a human-friendly tree (or graph) that can be explored. """ - return self._get_grid('nav', callback, args={'navId': nav_id}) + return self._on_nav(nav_id=nav_id, callback=callback) - def watch_sub(self, points, watch_id=None, watch_dis=None, - lease=None, callback=None): + def watch_sub( + self, points, watch_id=None, watch_dis=None, lease=None, callback=None + ): """ This creates a new watch with debug string watch_dis, identifier watch_id (string) and a lease time of lease (integer) seconds. points is a list of strings, Entity objects or hszinc.Ref objects. """ - grid = hszinc.Grid() - grid.column['id'] = {} - grid.extend([{'id': self._obj_to_ref(p)} for p in points]) - if watch_id is not None: - grid.metadata['watchId'] = watch_id - if watch_dis is not None: - grid.metadata['watchDis'] = watch_dis - if lease is not None: - grid.metadata['lease'] = lease - return self._post_grid('watchSub', grid, callback) + return self._on_watch_sub( + points=points, + watch_id=watch_id, + watch_dis=watch_dis, + lease=lease, + callback=callback, + ) def watch_unsub(self, watch, points=None, callback=None): """ @@ -235,18 +219,7 @@ def watch_unsub(self, watch, points=None, callback=None): hszinc.Ref objects which will be removed from the Watch object. Otherwise, it closes the Watch object. """ - grid = hszinc.Grid() - grid.column['id'] = {} - - if not isinstance(watch, string_types): - watch = watch.id - grid.metadata['watchId'] = watch - - if points is not None: - grid.extend([{'id': self._obj_to_ref(p)} for p in points]) - else: - grid.metadata['close'] = hszinc.MARKER - return self._post_grid('watchSub', grid, callback) + return self._on_watch_unsub(watch=watch, points=points, callback=callback) def watch_poll(self, watch, refresh=False, callback=None): """ @@ -256,20 +229,11 @@ def watch_poll(self, watch, refresh=False, callback=None): If refresh is True, then all points on the watch will be updated, not just those that have changed since the last poll. """ - grid = hszinc.Grid() - grid.column['empty'] = {} - - if not isinstance(watch, string_types): - watch = watch.id - grid.metadata['watchId'] = watch - - if refresh: - grid.metadata['refresh'] = hszinc.MARKER + return self._on_watch_poll(watch=watch, refresh=refresh, callback=callback) - return self._post_grid('watchPoll', grid, callback) - - def point_write(self, point, level=None, val=None, who=None, - duration=None, callback=None): + def point_write( + self, point, level=None, val=None, who=None, duration=None, callback=None + ): """ point is either the ID of the writeable point entity, or an instance of the writeable point entity to retrieve the write status of or write a @@ -279,24 +243,14 @@ def point_write(self, point, level=None, val=None, who=None, write status of the point is retrieved. Otherwise, a write is performed to the nominated point. """ - args = { - 'id': self._obj_to_ref(point), - } - if level is None: - if (val is not None) or (who is not None) or (duration is not None): - raise ValueError( - 'If level is None, val, who and duration must '\ - 'be None too.') - else: - args.update({ - 'level': level, - 'val': val, - }) - if who is not None: - args['who'] = who - if duration is not None: - args['duration'] = duration - return self._get_grid('pointWrite', callback, args=args) + return self._on_point_write( + point=point, + level=level, + val=val, + who=who, + duration=duration, + callback=callback, + ) def his_read(self, point, rng, callback=None): """ @@ -307,19 +261,7 @@ def his_read(self, point, rng, callback=None): datetime.datetime (providing all samples since the nominated time) or a slice of datetime.dates or datetime.datetimes. """ - if isinstance(rng, slice): - str_rng = ','.join([hszinc.dump_scalar(p) for p in - (rng.start, rng.stop)]) - elif not isinstance(rng, string_types): - str_rng = hszinc.dump_scalar(rng) - else: - # Better be valid! - str_rng = rng - - return self._get_grid('hisRead', callback, args={ - 'id': self._obj_to_ref(point), - 'range': str_rng, - }) + return self._on_his_read(point=point, rng=rng, callback=callback) def his_write(self, point, timestamp_records, callback=None): """ @@ -329,20 +271,9 @@ def his_write(self, point, timestamp_records, callback=None): (datetime.datetime) to the values to be written at those times, or a Pandas Series object. """ - grid = hszinc.Grid() - grid.metadata['id'] = self._obj_to_ref(point) - grid.column['ts'] = {} - grid.column['val'] = {} - - if hasattr(timestamp_records, 'to_dict'): - timestamp_records = timestamp_records.to_dict() - - timestamp_records = list(timestamp_records.items()) - timestamp_records.sort(key=lambda rec : rec[0]) - for (ts, val) in timestamp_records: - grid.append({'ts': ts, 'val': val}) - - return self._post_grid('hisWrite', grid, callback) + return self._on_his_write( + point=point, timestamp_records=timestamp_records, callback=callback + ) def invoke_action(self, entity, action, callback=None, **kwargs): """ @@ -350,14 +281,9 @@ def invoke_action(self, entity, action, callback=None, **kwargs): invoke the named action on. Keyword arguments give any additional parameters required for the user action. """ - grid = hszinc.Grid() - grid.metadata['id'] = self._obj_to_ref(entity) - grid.metadata['action'] = action - for arg in kwargs.keys(): - grid.column[arg] = {} - grid.append(kwargs) - - return self._post_grid('invokeAction', grid, callback) + return self._on_invoke_action( + entity=entity, action=action, callback=callback, action_args=kwargs + ) def get_entity(self, ids, refresh=False, single=None, callback=None): """ @@ -399,8 +325,7 @@ def find_entity(self, filter_expr, limit=None, single=False, callback=None): op.go() return op - def his_read_series(self, point, rng, tz=None, - series_format=None, callback=None): + def his_read_series(self, point, rng, tz=None, series_format=None, callback=None): """ Read the historical data of the given point and return it as a series. @@ -415,8 +340,7 @@ def his_read_series(self, point, rng, tz=None, else: series_format = self._HIS_READ_SERIES_OPERATION.FORMAT_LIST - op = self._HIS_READ_SERIES_OPERATION(self, point, - rng, tz, series_format) + op = self._HIS_READ_SERIES_OPERATION(self, point, rng, tz, series_format) if callback is not None: op.done_sig.connect(callback) op.go() @@ -436,8 +360,7 @@ def his_write_series(self, point, series, tz=None, callback=None): op.go() return op - def his_read_frame(self, columns, rng, tz=None, - frame_format=None, callback=None): + def his_read_frame(self, columns, rng, tz=None, frame_format=None, callback=None): """ Read the historical data of multiple given points and return them as a data frame. @@ -454,8 +377,7 @@ def his_read_frame(self, columns, rng, tz=None, else: frame_format = self._HIS_READ_FRAME_OPERATION.FORMAT_LIST - op = self._HIS_READ_FRAME_OPERATION(self, columns, - rng, tz, frame_format) + op = self._HIS_READ_FRAME_OPERATION(self, columns, rng, tz, frame_format) if callback is not None: op.done_sig.connect(callback) op.go() @@ -478,14 +400,14 @@ def his_write_frame(self, frame, columns=None, tz=None, callback=None): op.done_sig.connect(callback) op.go() return op - + @property def site(self): """ This helper will return the first site found on the server. This case is typical : having one site per server. """ - sites = self.find_entity('site').result + sites = self.find_entity("site").result return sites[list(sites.keys())[0]] @property @@ -493,12 +415,14 @@ def sites(self): """ This helper will return all sites found on the server. """ - sites = self.find_entity('site').result + sites = self.find_entity("site").result return sites # Extension feature support. - FEATURE_HISREAD_MULTI = 'hisRead/multi' # Multi-point hisRead - FEATURE_HISWRITE_MULTI = 'hisWrite/multi' # Multi-point hisWrite + FEATURE_HISREAD_MULTI = "hisRead/multi" # Multi-point hisRead + FEATURE_HISWRITE_MULTI = "hisWrite/multi" # Multi-point hisWrite + FEATURE_ID_UUID = "id_uuid" + def has_features(self, features, cache=True, callback=None): """ Determine if a given feature is supported. This is a helper function @@ -517,6 +441,137 @@ def has_features(self, features, cache=True, callback=None): # Protected methods/properties + def _on_about(self, cache, callback, **kwargs): + return self._get_grid("about", callback, cache=cache, **kwargs) + + def _on_ops(self, cache, callback, **kwargs): + return self._get_grid("ops", callback, cache=cache, **kwargs) + + def _on_formats(self, cache, callback, **kwargs): + return self._get_grid("formats", callback, cache=cache, **kwargs) + + def _on_read(self, ids, filter_expr, limit, callback, **kwargs): + if isinstance(ids, string_types) or isinstance(ids, hszinc.Ref): + # Make sure we always pass a list. + ids = [ids] + + if bool(ids): + if filter_expr is not None: + raise ValueError("Either specify ids or filter_expr, not both") + + ids = [self._obj_to_ref(r) for r in ids] + + if len(ids) == 1: + # Reading a single entity + return self._get_grid("read", callback, args={"id": ids[0]}, **kwargs) + else: + # Reading several entities + grid = hszinc.Grid() + grid.column["id"] = {} + grid.extend([{"id": r} for r in ids]) + return self._post_grid("read", grid, callback, **kwargs) + else: + args = {"filter": filter_expr} + if limit is not None: + args["limit"] = int(limit) + + return self._get_grid("read", callback, args=args, **kwargs) + + def _on_nav(self, nav_id, callback, **kwargs): + return self._get_grid("nav", callback, args={"nav_id": nav_id}, **kwargs) + + def _on_watch_sub(self, points, watch_id, watch_dis, lease, callback, **kwargs): + grid = hszinc.Grid() + grid.column["id"] = {} + grid.extend([{"id": self._obj_to_ref(p)} for p in points]) + if watch_id is not None: + grid.metadata["watchId"] = watch_id + if watch_dis is not None: + grid.metadata["watchDis"] = watch_dis + if lease is not None: + grid.metadata["lease"] = lease + return self._post_grid("watchSub", grid, callback, **kwargs) + + def _on_watch_unsub(self, watch, points, callback, **kwargs): + grid = hszinc.Grid() + grid.column["id"] = {} + + if not isinstance(watch, string_types): + watch = watch.id + grid.metadata["watchId"] = watch + + if points is not None: + grid.extend([{"id": self._obj_to_ref(p)} for p in points]) + else: + grid.metadata["close"] = hszinc.MARKER + return self._post_grid("watchSub", grid, callback, **kwargs) + + def _on_watch_poll(self, watch, refresh, callback, **kwargs): + grid = hszinc.Grid() + grid.column["empty"] = {} + + if not isinstance(watch, string_types): + watch = watch.id + grid.metadata["watchId"] = watch + return self._post_grid("watchPoll", grid, callback, **kwargs) + + def _on_point_write(self, point, level, val, who, duration, callback, **kwargs): + args = {"id": self._obj_to_ref(point)} + if level is None: + if (val is not None) or (who is not None) or (duration is not None): + raise ValueError( + "If level is None, val, who and duration must " "be None too." + ) + else: + args.update({"level": level, "val": val}) + if who is not None: + args["who"] = who + if duration is not None: + args["duration"] = duration + return self._get_grid("pointWrite", callback, args=args, **kwargs) + + def _on_his_read(self, point, rng, callback, **kwargs): + if isinstance(rng, slice): + str_rng = ",".join([hszinc.dump_scalar(p) for p in (rng.start, rng.stop)]) + elif not isinstance(rng, string_types): + str_rng = hszinc.dump_scalar(rng) + else: + # Better be valid! + str_rng = rng + + return self._get_grid( + "hisRead", + callback, + args={"id": self._obj_to_ref(point), "range": str_rng}, + **kwargs + ) + + def _on_his_write(self, point, timestamp_records, callback, **kwargs): + grid = hszinc.Grid() + grid.metadata["id"] = self._obj_to_ref(point) + grid.column["ts"] = {} + grid.column["val"] = {} + + if hasattr(timestamp_records, "to_dict"): + timestamp_records = timestamp_records.to_dict() + + timestamp_records = list(timestamp_records.items()) + timestamp_records.sort(key=lambda rec: rec[0]) + for (ts, val) in timestamp_records: + grid.append({"ts": ts, "val": val}) + + return self._post_grid("hisWrite", grid, callback, **kwargs) + + def _on_invoke_action(self, entity, action, callback, action_args, **kwargs): + grid = hszinc.Grid() + grid.metadata["id"] = self._obj_to_ref(entity) + grid.metadata["action"] = action + for arg in action_args.keys(): + grid.column[arg] = {} + grid.append(action_args) + + return self._post_grid("invokeAction", grid, callback, **kwargs) + def _get(self, uri, callback, api=True, **kwargs): """ Perform a raw HTTP GET operation. This is a convenience wrapper around @@ -524,47 +579,69 @@ def _get(self, uri, callback, api=True, **kwargs): the session instance. """ if api: - uri = '%s/%s' % (self._api_dir, uri) + uri = "%s/%s" % (self._api_dir, uri) return self._client.get(uri, callback, **kwargs) - def _get_grid(self, uri, callback, expect_format=None, - cache=False, **kwargs): + def _get_grid(self, uri, callback, expect_format=None, cache=False, **kwargs): """ Perform a HTTP GET of a grid. """ if expect_format is None: - expect_format=self._grid_format - op = self._GET_GRID_OPERATION(self, uri, - expect_format=expect_format, cache=cache, **kwargs) + expect_format = self._grid_format + op = self._GET_GRID_OPERATION( + self, uri, expect_format=expect_format, cache=cache, **kwargs + ) if callback is not None: op.done_sig.connect(callback) op.go() return op - def _post(self, uri, callback, body=None, body_type=None, body_size=None, - headers=None, api=True, **kwargs): + def _post( + self, + uri, + callback, + body=None, + body_type=None, + body_size=None, + headers=None, + api=True, + **kwargs + ): """ Perform a raw HTTP POST operation. This is a convenience wrapper around the HTTP client class that allows pre/post processing of the request by the session instance. """ if api: - uri = '%s/%s' % (self._api_dir, uri) - return self._client.post(uri=uri, callback=callback, - body=body, body_type=body_type, body_size=body_size, - headers=headers, **kwargs) - - def _post_grid(self, uri, grid, callback, post_format=None, - expect_format=None, **kwargs): + uri = "%s/%s" % (self._api_dir, uri) + return self._client.post( + uri=uri, + callback=callback, + body=body, + body_type=body_type, + body_size=body_size, + headers=headers, + **kwargs + ) + + def _post_grid( + self, uri, grid, callback, post_format=None, expect_format=None, **kwargs + ): """ Perform a HTTP POST of a grid. """ if expect_format is None: - expect_format=self._grid_format + expect_format = self._grid_format if post_format is None: - post_format=self._grid_format - op = self._POST_GRID_OPERATION(self, uri, grid, - expect_format=expect_format, post_format=post_format, **kwargs) + post_format = self._grid_format + op = self._POST_GRID_OPERATION( + self, + uri, + grid, + expect_format=expect_format, + post_format=post_format, + **kwargs + ) if callback is not None: op.done_sig.connect(callback) op.go() @@ -579,10 +656,11 @@ def _obj_to_ref(self, obj): return obj if isinstance(obj, string_types): return hszinc.Ref(obj) - if hasattr(obj, 'id'): + if hasattr(obj, "id"): return obj.id - raise NotImplementedError('Don\'t know how to get the ID from a %s' \ - % obj.__class__.__name__) + raise NotImplementedError( + "Don't know how to get the ID from a %s" % obj.__class__.__name__ + ) # Private methods/properties @@ -593,8 +671,7 @@ def _on_authenticate_done(self, operation, **kwargs): subclass to indicate the authentication state and clear the _auth_op attribute on the base class. """ - raise NotImplementedError('To be implemented in %s' % \ - self.__class__.__name__) + raise NotImplementedError("To be implemented in %s" % self.__class__.__name__) def config_pint(self, value=False): if value: diff --git a/pyhaystack/client/skyspark.py b/pyhaystack/client/skyspark.py index 7f3696b..323cbb9 100644 --- a/pyhaystack/client/skyspark.py +++ b/pyhaystack/client/skyspark.py @@ -9,8 +9,8 @@ from .ops.vendor.skyspark_scram import SkysparkScramAuthenticateOperation from .mixins.vendor.skyspark import evalexpr -class SkysparkHaystackSession(HaystackSession, - evalexpr.EvalOpsMixin): + +class SkysparkHaystackSession(HaystackSession, evalexpr.EvalOpsMixin): """ The SkysparkHaystackSession class implements some base support for Skyspark servers. @@ -18,7 +18,7 @@ class SkysparkHaystackSession(HaystackSession, _AUTH_OPERATION = SkysparkAuthenticateOperation - def __init__(self, uri, username, password, project = '', **kwargs): + def __init__(self, uri, username, password, project="", **kwargs): """ Initialise a Skyspark Project Haystack session handler. @@ -27,8 +27,7 @@ def __init__(self, uri, username, password, project = '', **kwargs): :param password: Authentication password. :param project: Skyspark project name """ - super(SkysparkHaystackSession, self).__init__(uri, - 'api/%s' % project, **kwargs) + super(SkysparkHaystackSession, self).__init__(uri, "api/%s" % project, **kwargs) self._project = project self._username = username self._password = password @@ -60,8 +59,8 @@ def _on_authenticate_done(self, operation, **kwargs): finally: self._auth_op = None -class SkysparkScramHaystackSession(HaystackSession, - evalexpr.EvalOpsMixin): + +class SkysparkScramHaystackSession(HaystackSession, evalexpr.EvalOpsMixin): """ The SkysparkHaystackSession class implements some base support for Skyspark servers. @@ -78,8 +77,9 @@ def __init__(self, uri, username, password, project, **kwargs): :param password: Authentication password. :param project: Skyspark project name """ - super(SkysparkScramHaystackSession, self).__init__(uri, - 'api/%s' % project,**kwargs) + super(SkysparkScramHaystackSession, self).__init__( + uri, "api/%s" % project, **kwargs + ) self._username = username self._password = password @@ -104,7 +104,7 @@ def _on_authenticate_done(self, operation, **kwargs): """ try: op_result = operation.result - header = op_result['header'] + header = op_result["header"] self._authenticated = True self._client.cookies = None self._client.headers = header diff --git a/pyhaystack/client/widesky.py b/pyhaystack/client/widesky.py index 4e55021..a33e5ce 100644 --- a/pyhaystack/client/widesky.py +++ b/pyhaystack/client/widesky.py @@ -6,24 +6,17 @@ from time import time from .session import HaystackSession -from .ops.vendor.widesky import WideskyAuthenticateOperation, \ - CreateEntityOperation, WideSkyHasFeaturesOperation +from .ops.vendor.widesky import ( + WideskyAuthenticateOperation, + CreateEntityOperation, + WideSkyHasFeaturesOperation, +) from .mixins.vendor.widesky import crud, multihis from ..util.asyncexc import AsynchronousException from .http.exceptions import HTTPStatusError -def _decode_str(s, enc='utf-8'): - """ - Try to decode a 'str' object to a Unicode string. - """ - try: - return s.decode(enc) - except AttributeError: - # This is probably already a Unicode string - return s - -def _decode_str(s, enc='utf-8'): +def _decode_str(s, enc="utf-8"): """ Try to decode a 'str' object to a Unicode string. """ @@ -34,9 +27,9 @@ def _decode_str(s, enc='utf-8'): return s -class WideskyHaystackSession(crud.CRUDOpsMixin, - multihis.MultiHisOpsMixin, - HaystackSession): +class WideskyHaystackSession( + crud.CRUDOpsMixin, multihis.MultiHisOpsMixin, HaystackSession +): """ The WideskyHaystackSession class implements some base support for Widesky servers. This is mainly a convenience for @@ -47,9 +40,17 @@ class WideskyHaystackSession(crud.CRUDOpsMixin, _CREATE_ENTITY_OPERATION = CreateEntityOperation _HAS_FEATURES_OPERATION = WideSkyHasFeaturesOperation - def __init__(self, uri, username, password, - client_id, client_secret, - api_dir='api', auth_dir='oauth2/token', **kwargs): + def __init__( + self, + uri, + username, + password, + client_id, + client_secret, + api_dir="api", + auth_dir="oauth2/token", + **kwargs + ): """ Initialise a VRT Widesky Project Haystack session handler. @@ -59,8 +60,7 @@ def __init__(self, uri, username, password, :param client_id: Authentication client ID. :param client_secret: Authentication client secret. """ - super(WideskyHaystackSession, self).__init__( - uri, api_dir, **kwargs) + super(WideskyHaystackSession, self).__init__(uri, api_dir, **kwargs) self._auth_dir = auth_dir self._username = username self._password = password @@ -76,10 +76,15 @@ def is_logged_in(self): if self._auth_result is None: return False # Return true if our token expires in the future. - return (self._auth_result.get('expires_in') or 0.0) > (1000.0 * time()) + return (self._auth_result.get("expires_in") or 0.0) > (1000.0 * time()) # Private methods/properties + def _on_read(self, ids, filter_expr, limit, callback, **kwargs): + return super(WideskyHaystackSession, self)._on_read( + ids, filter_expr, limit, callback, accept_status=(200, 404) + ) + def _on_http_grid_response(self, response): # If there's a '401' error, then we've lost the token. if isinstance(response, AsynchronousException): @@ -94,7 +99,7 @@ def _on_http_grid_response(self, response): status_code = response.status_code if (status_code == 401) and (self._auth_result is not None): - self._log.warning('Authentication lost due to HTTP error 401.') + self._log.warning("Authentication lost due to HTTP error 401.") self._auth_result = None self._client.headers = {} @@ -108,16 +113,17 @@ def _on_authenticate_done(self, operation, **kwargs): try: self._auth_result = operation.result self._client.headers = { - 'Authorization': (u'%s %s' % ( - _decode_str(self._auth_result['token_type'], - 'us-ascii'), - _decode_str(self._auth_result['access_token'], - 'us-ascii'), - )).encode('us-ascii') + "Authorization": ( + u"%s %s" + % ( + _decode_str(self._auth_result["token_type"], "us-ascii"), + _decode_str(self._auth_result["access_token"], "us-ascii"), + ) + ).encode("us-ascii") } except: self._auth_result = None self._client.headers = {} - self._log.warning('Log-in fails', exc_info=1) + self._log.warning("Log-in fails", exc_info=1) finally: self._auth_op = None diff --git a/pyhaystack/exception.py b/pyhaystack/exception.py index 57a0006..0f18fe3 100644 --- a/pyhaystack/exception.py +++ b/pyhaystack/exception.py @@ -4,30 +4,38 @@ """ + class HaystackError(Exception): - ''' + """ Exception thrown when an error grid is returned by the Haystack server. See http://project-haystack.org/doc/Rest#errorGrid - ''' + """ + def __init__(self, message, traceback=None, *args, **kwargs): super(HaystackError, self).__init__(message, *args, **kwargs) self.traceback = traceback + # Those exceptions have been made when working with Niagara AX class NoResponseFromServer(Exception): pass + class ProblemSendingRequestToServer(Exception): pass + class NoCookieReceived(Exception): pass + class ProblemReadingCookie(Exception): pass + class AuthenticationProblem(Exception): pass + class UnknownHistoryType(Exception): - pass \ No newline at end of file + pass diff --git a/pyhaystack/info.py b/pyhaystack/info.py index 10ba8b5..4f6cbc2 100644 --- a/pyhaystack/info.py +++ b/pyhaystack/info.py @@ -10,8 +10,8 @@ """ -__author__ = 'Christian Tremblay, Stuart Longland, @sudo-Whateverman, Igor' -__author_email__ = 'christian.tremblay@servisys.com' -__version__ = '0.92.7' -__license__ = 'Apache 2.0' +__author__ = "Christian Tremblay, Stuart Longland, @sudo-Whateverman, Igor" +__author_email__ = "christian.tremblay@servisys.com" +__version__ = "0.92.10" +__license__ = "Apache 2.0" __copyright__ = "Christian Tremblay / SERVISYS inc. | Stuart Longland / VRT | 2016" diff --git a/pyhaystack/util/asyncexc.py b/pyhaystack/util/asyncexc.py index c56fbce..ea4c504 100644 --- a/pyhaystack/util/asyncexc.py +++ b/pyhaystack/util/asyncexc.py @@ -20,6 +20,7 @@ def _some_async_function(…, callback_fn): from sys import exc_info from six import reraise + class AsynchronousException(object): def __init__(self): self._exc_info = exc_info() diff --git a/pyhaystack/util/filterbuilder.py b/pyhaystack/util/filterbuilder.py index 51f56e7..5b70307 100644 --- a/pyhaystack/util/filterbuilder.py +++ b/pyhaystack/util/filterbuilder.py @@ -20,6 +20,7 @@ import hszinc + class Base(object): def __and__(self, other): return And(self, other) @@ -37,32 +38,32 @@ def __init__(self, value): def __eq__(self, other): if not isinstance(other, Scalar): - raise TypeError('not a scalar: %r' % other) + raise TypeError("not a scalar: %r" % other) return Equal(self, other) def __ne__(self, other): if not isinstance(other, Scalar): - raise TypeError('not a scalar: %r' % other) + raise TypeError("not a scalar: %r" % other) return NotEqual(self, other) def __lt__(self, other): if not isinstance(other, Scalar): - raise TypeError('not a scalar: %r' % other) + raise TypeError("not a scalar: %r" % other) return LessThan(self, other) def __le__(self, other): if not isinstance(other, Scalar): - raise TypeError('not a scalar: %r' % other) + raise TypeError("not a scalar: %r" % other) return LessThanOrEqual(self, other) def __gt__(self, other): if not isinstance(other, Scalar): - raise TypeError('not a scalar: %r' % other) + raise TypeError("not a scalar: %r" % other) return GreaterThan(self, other) def __ge__(self, other): if not isinstance(other, Scalar): - raise TypeError('not a scalar: %r' % other) + raise TypeError("not a scalar: %r" % other) return GreaterThanOrEqual(self, other) def __str__(self): @@ -84,16 +85,16 @@ def __init__(self, x, y): def __str__(self): if isinstance(self.x, Binary): - x = '( %s )' % self.x + x = "( %s )" % self.x else: x = str(self.x) if isinstance(self.y, Binary): - y = '( %s )' % self.y + y = "( %s )" % self.y else: y = str(self.y) - return '%s %s %s' % (x, self.OP, y) + return "%s %s %s" % (x, self.OP, y) class Unary(Base): @@ -102,42 +103,42 @@ def __init__(self, value): def __str__(self): if isinstance(self.value, Binary): - return '%s ( %s )' % (self.OP, self.value) + return "%s ( %s )" % (self.OP, self.value) else: - return '%s %s' % (self.OP, self.value) + return "%s %s" % (self.OP, self.value) class Equal(Binary): - OP = '==' + OP = "==" class NotEqual(Binary): - OP = '!=' + OP = "!=" class LessThan(Binary): - OP = '<' + OP = "<" class LessThanOrEqual(Binary): - OP = '<=' + OP = "<=" class GreaterThan(Binary): - OP = '>' + OP = ">" class GreaterThanOrEqual(Binary): - OP = '>=' + OP = ">=" class And(Binary): - OP = 'and' + OP = "and" class Or(Binary): - OP = 'or' + OP = "or" class Not(Unary): - OP = 'not' + OP = "not" diff --git a/pyhaystack/util/scram.py b/pyhaystack/util/scram.py index 8992ef0..61da436 100644 --- a/pyhaystack/util/scram.py +++ b/pyhaystack/util/scram.py @@ -3,8 +3,7 @@ from binascii import b2a_hex, unhexlify, b2a_base64, hexlify from requests.auth import HTTPBasicAuth -from base64 import standard_b64encode, b64decode, urlsafe_b64encode, \ - urlsafe_b64decode +from base64 import standard_b64encode, b64decode, urlsafe_b64encode, urlsafe_b64decode from hashlib import sha1, sha256 try: @@ -18,37 +17,47 @@ import re import os + def get_nonce(): return b2a_hex(os.urandom(32)).decode() + def get_nonce_16(): - return urlsafe_b64encode( os.urandom(16) ).decode() + return urlsafe_b64encode(os.urandom(16)).decode() + def _hash_sha256(client_key, algorithm): hashFunc = algorithm() hashFunc.update(client_key) return hashFunc.hexdigest() + def salted_password(salt, iterations, algorithm_name, password): - dk = pbkdf2_hmac(algorithm_name, password.encode(), - urlsafe_b64decode(salt), int(iterations)) + dk = pbkdf2_hmac( + algorithm_name, password.encode(), urlsafe_b64decode(salt), int(iterations) + ) encrypt_password = hexlify(dk) return encrypt_password + def salted_password_2(salt, iterations, algorithm_name, password): - dk = pbkdf2_hmac(algorithm_name, password.encode(), - unhexlify(salt), int(iterations)) + dk = pbkdf2_hmac( + algorithm_name, password.encode(), unhexlify(salt), int(iterations) + ) encrypt_password = hexlify(dk) return encrypt_password + def base64_no_padding(s): encoded_str = urlsafe_b64encode(s.encode()) encoded_str = encoded_str.decode().replace("=", "") return encoded_str + def regex_after_equal(s): - tmp_str = re.search( "\=(.*)$" ,s, flags=0) + tmp_str = re.search("\=(.*)$", s, flags=0) return tmp_str.group(1) + def _xor(s1, s2): return hex(int(s1, 16) ^ int(s2, 16))[2:] diff --git a/pyhaystack/util/state.py b/pyhaystack/util/state.py index 5e6750e..c659e9f 100644 --- a/pyhaystack/util/state.py +++ b/pyhaystack/util/state.py @@ -15,6 +15,7 @@ class NotReadyError(Exception): Exception raised when an attempt is made to retrieve the result of an operation before it is ready. """ + pass @@ -23,6 +24,7 @@ class HaystackOperation(object): A core state machine object. This implements the basic interface presented for all operations in pyhaystack. """ + def __init__(self, result_copy=True, result_deepcopy=True): """ Initialisation. This should be overridden by subclasses to accept and @@ -38,7 +40,7 @@ def __init__(self, result_copy=True, result_deepcopy=True): self._done_evt = Event() # Signal emitted when the operation is "done" - self.done_sig = Signal(name='done', threadsafe=True) + self.done_sig = Signal(name="done", threadsafe=True) # Result returned by operation self._result = None @@ -52,8 +54,9 @@ def go(self): operation. """ # This needs to be implemented in the subclass. - raise NotImplementedError("To be implemented in subclass %s" \ - % self.__class__.__name__) + raise NotImplementedError( + "To be implemented in subclass %s" % self.__class__.__name__ + ) def wait(self, timeout=None): """ @@ -111,11 +114,11 @@ def __repr__(self): Return a representation of this object's state. """ if self.is_failed: - return '<%s failed>' % self.__class__.__name__ + return "<%s failed>" % self.__class__.__name__ elif self.is_done: - return '<%s done: %s>' % (self.__class__.__name__, self._result) + return "<%s done: %s>" % (self.__class__.__name__, self._result) else: - return '<%s %s>' % (self.__class__.__name__, self.state) + return "<%s %s>" % (self.__class__.__name__, self.state) def _done(self, result): """ diff --git a/pyhaystack/util/tools.py b/pyhaystack/util/tools.py index dfc0557..e51a407 100644 --- a/pyhaystack/util/tools.py +++ b/pyhaystack/util/tools.py @@ -1,11 +1,12 @@ # -*- coding: utf-8 -*- import json + def isfloat(value): """ Helper function to detect if a value is a float """ - if value != '': + if value != "": try: float(value) return True @@ -14,11 +15,12 @@ def isfloat(value): else: return False + def isBool(value): """ Helper function to detect if a value is boolean """ - if value != '': + if value != "": if isinstance(value, bool): return True else: @@ -26,8 +28,9 @@ def isBool(value): else: return False + def prettyprint(jsonData): """ Pretty print json object """ - print('%s' % json.dumps(jsonData, sort_keys=True, indent=4)) \ No newline at end of file + print("%s" % json.dumps(jsonData, sort_keys=True, indent=4)) diff --git a/setup.py b/setup.py index f0dbe85..5fd85e9 100644 --- a/setup.py +++ b/setup.py @@ -8,55 +8,59 @@ import os -os.environ['COPY_EXTENDED_ATTRIBUTES_DISABLE'] = 'true' -os.environ['COPYFILE_DISABLE'] = 'true' +os.environ["COPY_EXTENDED_ATTRIBUTES_DISABLE"] = "true" +os.environ["COPYFILE_DISABLE"] = "true" -setup(name='pyhaystack', - version=info.__version__, - description='Python Haystack Utility', - author=info.__author__, - author_email=info.__author_email__, - url='http://www.project-haystack.com/', - keywords = ['tags', 'hvac', 'project-haystack', 'building', 'automation', 'analytic'], - install_requires = [ - 'requests', - 'setuptools', - 'pandas', - 'parsimonious', - 'iso8601', - 'hszinc', - 'six', - 'fysom', - 'signalslot', - 'semver', - 'certifi'], - packages=[ - 'pyhaystack', - 'pyhaystack.client', - 'pyhaystack.client.mixins', - 'pyhaystack.client.mixins.vendor', - 'pyhaystack.client.mixins.vendor.widesky', - 'pyhaystack.client.mixins.vendor.skyspark', - 'pyhaystack.client.mixins.vendor.niagara', - 'pyhaystack.client.http', - 'pyhaystack.client.ops', - 'pyhaystack.client.ops.vendor', - 'pyhaystack.client.entity', - 'pyhaystack.client.entity.mixins', - 'pyhaystack.client.entity.models', - 'pyhaystack.client.entity.ops', - 'pyhaystack.util', - 'pyhaystack.server', - 'pyhaystack.util'], - long_description=open('README.rst').read(), - classifiers=[ - "Development Status :: 4 - Beta", - "Intended Audience :: Developers", - "Operating System :: OS Independent", - "Programming Language :: Python", - "Topic :: Software Development :: Libraries :: Python Modules", - "Topic :: System :: Networking", - "Topic :: Utilities", - "License :: OSI Approved :: Apache Software License", - ],) +setup( + name="pyhaystack", + version=info.__version__, + description="Python Haystack Utility", + author=info.__author__, + author_email=info.__author_email__, + url="http://www.project-haystack.com/", + keywords=["tags", "hvac", "project-haystack", "building", "automation", "analytic"], + install_requires=[ + "requests", + "setuptools", + "pandas", + "parsimonious", + "iso8601", + "hszinc", + "six", + "fysom", + "signalslot", + "semver", + "certifi", + ], + packages=[ + "pyhaystack", + "pyhaystack.client", + "pyhaystack.client.mixins", + "pyhaystack.client.mixins.vendor", + "pyhaystack.client.mixins.vendor.widesky", + "pyhaystack.client.mixins.vendor.skyspark", + "pyhaystack.client.mixins.vendor.niagara", + "pyhaystack.client.http", + "pyhaystack.client.ops", + "pyhaystack.client.ops.vendor", + "pyhaystack.client.entity", + "pyhaystack.client.entity.mixins", + "pyhaystack.client.entity.models", + "pyhaystack.client.entity.ops", + "pyhaystack.util", + "pyhaystack.server", + "pyhaystack.util", + ], + long_description=open("README.rst").read(), + classifiers=[ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Topic :: Software Development :: Libraries :: Python Modules", + "Topic :: System :: Networking", + "Topic :: Utilities", + "License :: OSI Approved :: Apache Software License", + ], +) diff --git a/tests/client/test_base.py b/tests/client/test_base.py index ce2b490..a4b4ab5 100644 --- a/tests/client/test_base.py +++ b/tests/client/test_base.py @@ -28,9 +28,10 @@ # Logging setup so we can see what's going on import logging + logging.basicConfig(level=logging.DEBUG) -BASE_URI = 'https://myserver/api/' +BASE_URI = "https://myserver/api/" # For testing _on_http_grid_response: class DummySession(widesky.WideskyHaystackSession): @@ -41,8 +42,11 @@ def __init__(self, **kwargs): def _on_http_grid_response(self, response, *args, **kwargs): self._log.debug( - 'Received grid response: response=%r args=%r, kwargs=%r', - response, args, kwargs) + "Received grid response: response=%r args=%r, kwargs=%r", + response, + args, + kwargs, + ) self._on_http_grid_response_called += 1 self._on_http_grid_response_last_args = (response, args, kwargs) @@ -54,31 +58,33 @@ def server_session(): """ server = dummy_http.DummyHttpServer() session = DummySession( - uri=BASE_URI, - username='testuser', - password='testpassword', - client_id='testclient', - client_secret='testclientsecret', - http_client=dummy_http.DummyHttpClient, - http_args={'server': server, 'debug': True}, - grid_format=hszinc.MODE_ZINC) + uri=BASE_URI, + username="testuser", + password="testpassword", + client_id="testclient", + client_secret="testclientsecret", + http_client=dummy_http.DummyHttpClient, + http_args={"server": server, "debug": True}, + grid_format=hszinc.MODE_ZINC, + ) # Force an authentication. op = session.authenticate() # Pop the request off the stack. We'll assume it's fine for now. rq = server.next_request() - assert server.requests() == 0, 'More requests waiting' - rq.respond(status=200, - headers={ - b'Content-Type': 'application/json' - }, - content='''{ + assert server.requests() == 0, "More requests waiting" + rq.respond( + status=200, + headers={b"Content-Type": "application/json"}, + content="""{ "token_type": "Bearer", "access_token": "DummyAccessToken", "refresh_token": "DummyRefreshToken", "expires_in": %f - }''' % ((time.time() + 86400) * 1000.0)) - assert op.state == 'done' - logging.debug('Result = %s', op.result) + }""" + % ((time.time() + 86400) * 1000.0), + ) + assert op.state == "done" + logging.debug("Result = %s", op.result) assert server.requests() == 0 assert session.is_logged_in return (server, session) @@ -94,7 +100,7 @@ def test_on_http_grid_response(self, server_session): session._on_http_grid_response_last_args = None # Fetch a grid - op = session._get_grid('dummy', callback=lambda *a, **kwa : None) + op = session._get_grid("dummy", callback=lambda *a, **kwa: None) # The operation should still be in progress assert not op.is_done @@ -105,10 +111,12 @@ def test_on_http_grid_response(self, server_session): # Make a grid to respond with expected = hszinc.Grid() - expected.column['empty'] = {} - rq.respond(status=200, headers={ - b'Content-Type': 'text/zinc', - }, content=hszinc.dump(expected, mode=hszinc.MODE_ZINC)) + expected.column["empty"] = {} + rq.respond( + status=200, + headers={b"Content-Type": "text/zinc"}, + content=hszinc.dump(expected, mode=hszinc.MODE_ZINC), + ) # Our dummy function should have been called assert session._on_http_grid_response_called == 1 @@ -132,43 +140,47 @@ def test_about(self, server_session): rq = server.next_request() # Request shall be a GET - assert rq.method == 'GET', 'Expecting GET, got %s' % rq + assert rq.method == "GET", "Expecting GET, got %s" % rq # Request shall be for base + 'api/about' - assert rq.uri == BASE_URI + 'api/about' + assert rq.uri == BASE_URI + "api/about" # Accept header shall be given - assert rq.headers[b'Accept'] == 'text/zinc' + assert rq.headers[b"Accept"] == "text/zinc" # Make a grid to respond with expected = hszinc.Grid() - expected.column['haystackVersion'] = {} - expected.column['tz'] = {} - expected.column['serverName'] = {} - expected.column['serverTime'] = {} - expected.column['serverBootTime'] = {} - expected.column['productName'] = {} - expected.column['productUri'] = {} - expected.column['productVersion'] = {} - expected.column['moduleName'] = {} - expected.column['moduleVersion'] = {} - expected.append({ - 'haystackVersion': '2.0', - 'tz': 'UTC', - 'serverName': 'pyhaystack dummy server', - 'serverTime': datetime.datetime.now(tz=pytz.UTC), - 'serverBootTime': datetime.datetime.now(tz=pytz.UTC), - 'productName': 'pyhaystack dummy server', - 'productVersion': '0.0.1', - 'productUri': hszinc.Uri('http://pyhaystack.readthedocs.io'), - 'moduleName': 'tests.client.base', - 'moduleVersion': '0.0.1', - }) - - rq.respond(status=200, headers={ - b'Content-Type': 'text/zinc', - }, content=hszinc.dump(expected, mode=hszinc.MODE_ZINC)) + expected.column["haystackVersion"] = {} + expected.column["tz"] = {} + expected.column["serverName"] = {} + expected.column["serverTime"] = {} + expected.column["serverBootTime"] = {} + expected.column["productName"] = {} + expected.column["productUri"] = {} + expected.column["productVersion"] = {} + expected.column["moduleName"] = {} + expected.column["moduleVersion"] = {} + expected.append( + { + "haystackVersion": "2.0", + "tz": "UTC", + "serverName": "pyhaystack dummy server", + "serverTime": datetime.datetime.now(tz=pytz.UTC), + "serverBootTime": datetime.datetime.now(tz=pytz.UTC), + "productName": "pyhaystack dummy server", + "productVersion": "0.0.1", + "productUri": hszinc.Uri("http://pyhaystack.readthedocs.io"), + "moduleName": "tests.client.base", + "moduleVersion": "0.0.1", + } + ) + + rq.respond( + status=200, + headers={b"Content-Type": "text/zinc"}, + content=hszinc.dump(expected, mode=hszinc.MODE_ZINC), + ) # State machine should now be done assert op.is_done @@ -187,60 +199,47 @@ def test_ops(self, server_session): rq = server.next_request() # Request shall be a GET - assert rq.method == 'GET', 'Expecting GET, got %s' % rq + assert rq.method == "GET", "Expecting GET, got %s" % rq # Request shall be for base + 'api/ops' - assert rq.uri == BASE_URI + 'api/ops' + assert rq.uri == BASE_URI + "api/ops" # Accept header shall be given - assert rq.headers[b'Accept'] == 'text/zinc' + assert rq.headers[b"Accept"] == "text/zinc" # Make a grid to respond with expected = hszinc.Grid() - expected.column['name'] = {} - expected.column['summary'] = {} - expected.extend([{ - "name": "about", - "summary": "Summary information for server" - }, { - "name": "ops", - "summary": "Operations supported by this server" - }, { - "name": "formats", - "summary": "Grid data formats supported by this server" - }, { - "name": "read", - "summary": "Read records by id or filter" - }, { - "name": "hisRead", - "summary": "Read historical records" - }, { - "name": "hisWrite", - "summary": "Write historical records" - }, { - "name": "nav", - "summary": "Navigate a project" - }, { - "name": "watchSub", - "summary": "Subscribe to change notifications" - }, { - "name": "watchUnsub", - "summary": "Unsubscribe from change notifications" - }, { - "name": "watchPoll", - "summary": "Poll for changes in watched points" - }, { - "name": "pointWrite", - "summary": "Write a real-time value to a point" - }, { - "name": "invokeAction", - "summary": "Invoke an action on an entity" - }]) - - rq.respond(status=200, headers={ - b'Content-Type': 'text/zinc', - }, content=hszinc.dump(expected, mode=hszinc.MODE_ZINC)) + expected.column["name"] = {} + expected.column["summary"] = {} + expected.extend( + [ + {"name": "about", "summary": "Summary information for server"}, + {"name": "ops", "summary": "Operations supported by this server"}, + { + "name": "formats", + "summary": "Grid data formats supported by this server", + }, + {"name": "read", "summary": "Read records by id or filter"}, + {"name": "hisRead", "summary": "Read historical records"}, + {"name": "hisWrite", "summary": "Write historical records"}, + {"name": "nav", "summary": "Navigate a project"}, + {"name": "watchSub", "summary": "Subscribe to change notifications"}, + { + "name": "watchUnsub", + "summary": "Unsubscribe from change notifications", + }, + {"name": "watchPoll", "summary": "Poll for changes in watched points"}, + {"name": "pointWrite", "summary": "Write a real-time value to a point"}, + {"name": "invokeAction", "summary": "Invoke an action on an entity"}, + ] + ) + + rq.respond( + status=200, + headers={b"Content-Type": "text/zinc"}, + content=hszinc.dump(expected, mode=hszinc.MODE_ZINC), + ) # State machine should now be done assert op.is_done @@ -259,37 +258,37 @@ def test_formats(self, server_session): rq = server.next_request() # Request shall be a GET - assert rq.method == 'GET', 'Expecting GET, got %s' % rq + assert rq.method == "GET", "Expecting GET, got %s" % rq # Request shall be for base + 'api/formats' - assert rq.uri == BASE_URI + 'api/formats' + assert rq.uri == BASE_URI + "api/formats" # Accept header shall be given - assert rq.headers[b'Accept'] == 'text/zinc' + assert rq.headers[b"Accept"] == "text/zinc" # Make a grid to respond with expected = hszinc.Grid() - expected.column['mime'] = {} - expected.column['receive'] = {} - expected.column['send'] = {} - expected.extend([{ - "mime": "text/csv", - "receive": hszinc.MARKER, - "send": hszinc.MARKER, - }, { - "mime": "text/zinc", - "receive": hszinc.MARKER, - "send": hszinc.MARKER, - }, { - "mime": "application/json", - "receive": hszinc.MARKER, - "send": hszinc.MARKER, - }]) - - rq.respond(status=200, headers={ - b'Content-Type': 'text/zinc', - }, content=hszinc.dump(expected, mode=hszinc.MODE_ZINC)) + expected.column["mime"] = {} + expected.column["receive"] = {} + expected.column["send"] = {} + expected.extend( + [ + {"mime": "text/csv", "receive": hszinc.MARKER, "send": hszinc.MARKER}, + {"mime": "text/zinc", "receive": hszinc.MARKER, "send": hszinc.MARKER}, + { + "mime": "application/json", + "receive": hszinc.MARKER, + "send": hszinc.MARKER, + }, + ] + ) + + rq.respond( + status=200, + headers={b"Content-Type": "text/zinc"}, + content=hszinc.dump(expected, mode=hszinc.MODE_ZINC), + ) # State machine should now be done assert op.is_done @@ -298,7 +297,7 @@ def test_formats(self, server_session): def test_read_one_id(self, server_session): (server, session) = server_session - op = session.read(hszinc.Ref('my.entity.id')) + op = session.read(hszinc.Ref("my.entity.id")) # The operation should still be in progress assert not op.is_done @@ -308,27 +307,26 @@ def test_read_one_id(self, server_session): rq = server.next_request() # Request shall be a GET - assert rq.method == 'GET', 'Expecting GET, got %s' % rq + assert rq.method == "GET", "Expecting GET, got %s" % rq # Request shall be for a specific URI - assert rq.uri == BASE_URI + 'api/read?id=%40my.entity.id' + assert rq.uri == BASE_URI + "api/read?id=%40my.entity.id" # Accept header shall be given - assert rq.headers[b'Accept'] == 'text/zinc' + assert rq.headers[b"Accept"] == "text/zinc" # Make a grid to respond with expected = hszinc.Grid() - expected.column['id'] = {} - expected.column['dis'] = {} - expected.extend([{ - "id": hszinc.Ref('my.entity.id'), - "dis": 'my entity' - }]) + expected.column["id"] = {} + expected.column["dis"] = {} + expected.extend([{"id": hszinc.Ref("my.entity.id"), "dis": "my entity"}]) - rq.respond(status=200, headers={ - b'Content-Type': 'text/zinc', - }, content=hszinc.dump(expected, mode=hszinc.MODE_ZINC)) + rq.respond( + status=200, + headers={b"Content-Type": "text/zinc"}, + content=hszinc.dump(expected, mode=hszinc.MODE_ZINC), + ) # State machine should now be done assert op.is_done @@ -337,11 +335,13 @@ def test_read_one_id(self, server_session): def test_read_many_id(self, server_session): (server, session) = server_session - op = session.read([ - hszinc.Ref('my.entity.id1'), - hszinc.Ref('my.entity.id2'), - hszinc.Ref('my.entity.id3'), - ]) + op = session.read( + [ + hszinc.Ref("my.entity.id1"), + hszinc.Ref("my.entity.id2"), + hszinc.Ref("my.entity.id3"), + ] + ) # The operation should still be in progress assert not op.is_done @@ -351,51 +351,49 @@ def test_read_many_id(self, server_session): rq = server.next_request() # Request shall be a GET - assert rq.method == 'POST', 'Expecting POST, got %s' % rq + assert rq.method == "POST", "Expecting POST, got %s" % rq # Request shall be for a specific URI - assert rq.uri == BASE_URI + 'api/read' + assert rq.uri == BASE_URI + "api/read" # Body shall be in ZINC - assert rq.headers[b'Content-Type'] == 'text/zinc' + assert rq.headers[b"Content-Type"] == "text/zinc" # Body shall be a single valid grid of this form: expected = hszinc.Grid() - expected.column['id'] = {} - expected.extend([{ - "id": hszinc.Ref('my.entity.id1'), - }, { - "id": hszinc.Ref('my.entity.id2'), - }, { - "id": hszinc.Ref('my.entity.id3'), - }]) - actual = hszinc.parse(rq.body.decode('utf-8'), - mode=hszinc.MODE_ZINC) + expected.column["id"] = {} + expected.extend( + [ + {"id": hszinc.Ref("my.entity.id1")}, + {"id": hszinc.Ref("my.entity.id2")}, + {"id": hszinc.Ref("my.entity.id3")}, + ] + ) + actual = hszinc.parse(rq.body.decode("utf-8"), mode=hszinc.MODE_ZINC) assert len(actual) == 1 grid_cmp(expected, actual[0]) # Accept header shall be given - assert rq.headers[b'Accept'] == 'text/zinc' + assert rq.headers[b"Accept"] == "text/zinc" # Make a grid to respond with expected = hszinc.Grid() - expected.column['id'] = {} - expected.column['dis'] = {} - expected.extend([{ - "id": hszinc.Ref('my.entity.id1'), - "dis": 'my entity 1' - }, { - "id": hszinc.Ref('my.entity.id2'), - "dis": 'my entity 2' - }, { - "id": hszinc.Ref('my.entity.id3'), - "dis": 'my entity 3' - }]) - - rq.respond(status=200, headers={ - b'Content-Type': 'text/zinc', - }, content=hszinc.dump(expected, mode=hszinc.MODE_ZINC)) + expected.column["id"] = {} + expected.column["dis"] = {} + expected.extend( + [ + {"id": hszinc.Ref("my.entity.id1"), "dis": "my entity 1"}, + {"id": hszinc.Ref("my.entity.id2"), "dis": "my entity 2"}, + {"id": hszinc.Ref("my.entity.id3"), "dis": "my entity 3"}, + ] + ) + + rq.respond( + status=200, + headers={b"Content-Type": "text/zinc"}, + content=hszinc.dump(expected, mode=hszinc.MODE_ZINC), + ) # State machine should now be done assert op.is_done diff --git a/tests/client/test_entity.py b/tests/client/test_entity.py index 7a12440..1cbe914 100644 --- a/tests/client/test_entity.py +++ b/tests/client/test_entity.py @@ -28,9 +28,11 @@ # Logging setup so we can see what's going on import logging + logging.basicConfig(level=logging.DEBUG) -BASE_URI = 'https://myserver/api/' +BASE_URI = "https://myserver/api/" + @pytest.fixture def server_session(): @@ -39,31 +41,33 @@ def server_session(): """ server = dummy_http.DummyHttpServer() session = widesky.WideskyHaystackSession( - uri=BASE_URI, - username='testuser', - password='testpassword', - client_id='testclient', - client_secret='testclientsecret', - http_client=dummy_http.DummyHttpClient, - http_args={'server': server, 'debug': True}, - grid_format=hszinc.MODE_ZINC) + uri=BASE_URI, + username="testuser", + password="testpassword", + client_id="testclient", + client_secret="testclientsecret", + http_client=dummy_http.DummyHttpClient, + http_args={"server": server, "debug": True}, + grid_format=hszinc.MODE_ZINC, + ) # Force an authentication. op = session.authenticate() # Pop the request off the stack. We'll assume it's fine for now. rq = server.next_request() - assert server.requests() == 0, 'More requests waiting' - rq.respond(status=200, - headers={ - b'Content-Type': 'application/json' - }, - content='''{ + assert server.requests() == 0, "More requests waiting" + rq.respond( + status=200, + headers={b"Content-Type": "application/json"}, + content="""{ "token_type": "Bearer", "access_token": "DummyAccessToken", "refresh_token": "DummyRefreshToken", "expires_in": %f - }''' % ((time.time() + 86400) * 1000.0)) - assert op.state == 'done' - logging.debug('Result = %s', op.result) + }""" + % ((time.time() + 86400) * 1000.0), + ) + assert op.state == "done" + logging.debug("Result = %s", op.result) assert server.requests() == 0 assert session.is_logged_in return (server, session) @@ -71,11 +75,10 @@ def server_session(): @pytest.mark.usefixtures("server_session") class TestSession(object): - def test_get_single_entity(self, server_session): (server, session) = server_session # Try retrieving an existing single entity - op = session.get_entity('my.entity.id', single=True) + op = session.get_entity("my.entity.id", single=True) # The operation should still be in progress assert not op.is_done @@ -85,44 +88,48 @@ def test_get_single_entity(self, server_session): rq = server.next_request() # Request shall be a GET - assert rq.method == 'GET', 'Expecting GET, got %s' % rq + assert rq.method == "GET", "Expecting GET, got %s" % rq # Request shall be for base + 'api/read?id=@my.entity.id' - assert rq.uri == BASE_URI + 'api/read?id=%40my.entity.id' + assert rq.uri == BASE_URI + "api/read?id=%40my.entity.id" # Accept header shall be given - assert rq.headers[b'Accept'] == 'text/zinc' + assert rq.headers[b"Accept"] == "text/zinc" # Make a grid to respond with response = hszinc.Grid() - response.column['id'] = {} - response.column['dis'] = {} - response.column['randomTag'] = {} - response.append({ - 'id': hszinc.Ref('my.entity.id', value='id'), - 'dis': 'A test entity', - 'randomTag': hszinc.MARKER - }) - - rq.respond(status=200, headers={ - b'Content-Type': 'text/zinc', - }, content=hszinc.dump(response, mode=hszinc.MODE_ZINC)) + response.column["id"] = {} + response.column["dis"] = {} + response.column["randomTag"] = {} + response.append( + { + "id": hszinc.Ref("my.entity.id", value="id"), + "dis": "A test entity", + "randomTag": hszinc.MARKER, + } + ) + + rq.respond( + status=200, + headers={b"Content-Type": "text/zinc"}, + content=hszinc.dump(response, mode=hszinc.MODE_ZINC), + ) # State machine should now be done assert op.is_done entity = op.result # Response should be an entity - assert isinstance(entity, Entity), '%r not an entity' % entity + assert isinstance(entity, Entity), "%r not an entity" % entity # The tags should be passed through from the response - assert entity.id.name == 'my.entity.id' - assert entity.tags['dis'] == response[0]['dis'] - assert entity.tags['randomTag'] == response[0]['randomTag'] + assert entity.id.name == "my.entity.id" + assert entity.tags["dis"] == response[0]["dis"] + assert entity.tags["randomTag"] == response[0]["randomTag"] def test_get_single_entity_missing(self, server_session): (server, session) = server_session # Try retrieving an existing single entity that does not exist - op = session.get_entity('my.nonexistent.id', single=True) + op = session.get_entity("my.nonexistent.id", single=True) # The operation should still be in progress assert not op.is_done @@ -132,24 +139,26 @@ def test_get_single_entity_missing(self, server_session): rq = server.next_request() # Request shall be a GET - assert rq.method == 'GET', 'Expecting GET, got %s' % rq + assert rq.method == "GET", "Expecting GET, got %s" % rq # Request shall be for base + 'api/read?id=@my.nonexistent.id' - assert rq.uri == BASE_URI + 'api/read?id=%40my.nonexistent.id' + assert rq.uri == BASE_URI + "api/read?id=%40my.nonexistent.id" # Accept header shall be given - assert rq.headers[b'Accept'] == 'text/zinc' + assert rq.headers[b"Accept"] == "text/zinc" # Make a grid to respond with. Note, server might also choose to # throw an error, but we'll pretend it doesn't. response = hszinc.Grid() - response.column['id'] = {} - response.column['dis'] = {} + response.column["id"] = {} + response.column["dis"] = {} - rq.respond(status=200, headers={ - b'Content-Type': 'text/zinc', - }, content=hszinc.dump(response, mode=hszinc.MODE_ZINC)) + rq.respond( + status=200, + headers={b"Content-Type": "text/zinc"}, + content=hszinc.dump(response, mode=hszinc.MODE_ZINC), + ) # State machine should now be done assert op.is_done @@ -158,13 +167,14 @@ def test_get_single_entity_missing(self, server_session): entity = op.result assert entity is None except NameError as e: - assert str(e) == 'No matching entity found' + assert str(e) == "No matching entity found" def test_get_multi_entity_missng(self, server_session): (server, session) = server_session # Try retrieving existing multiple entities, with one that doesn't exist - op = session.get_entity(['my.entity.id1', 'my.entity.id2', - 'my.nonexistent.id'], single=False) + op = session.get_entity( + ["my.entity.id1", "my.entity.id2", "my.nonexistent.id"], single=False + ) # The operation should still be in progress assert not op.is_done @@ -174,83 +184,87 @@ def test_get_multi_entity_missng(self, server_session): rq = server.next_request() # Request shall be a POST - assert rq.method == 'POST', 'Expecting POST, got %s' % rq + assert rq.method == "POST", "Expecting POST, got %s" % rq # Request shall be for base + 'api/read?id=@my.entity.id' - assert rq.uri == BASE_URI + 'api/read' + assert rq.uri == BASE_URI + "api/read" # Accept header shall be given - assert rq.headers[b'Accept'] == 'text/zinc' - assert rq.headers[b'Content-Type'] == 'text/zinc' + assert rq.headers[b"Accept"] == "text/zinc" + assert rq.headers[b"Content-Type"] == "text/zinc" # Body shall be a single grid: - rq_grid = hszinc.parse(rq.body.decode('utf-8'), mode=hszinc.MODE_ZINC) + rq_grid = hszinc.parse(rq.body.decode("utf-8"), mode=hszinc.MODE_ZINC) assert len(rq_grid) == 1 rq_grid = rq_grid[0] # It shall have one column; id - assert set(rq_grid.column.keys()) == set(['id']) + assert set(rq_grid.column.keys()) == set(["id"]) # It shall have 3 rows assert len(rq_grid) == 3 # Each row should only have 'id' values - assert all([(set(r.keys()) == set(['id'])) for r in rq_grid]) + assert all([(set(r.keys()) == set(["id"])) for r in rq_grid]) # The rows' 'id' column should *only* contain Refs. - assert all([isinstance(r['id'], hszinc.Ref) for r in rq_grid]) + assert all([isinstance(r["id"], hszinc.Ref) for r in rq_grid]) # Both IDs shall be listed, we don't about order - assert set([r['id'].name for r in rq_grid]) \ - == set(['my.entity.id1', 'my.entity.id2', 'my.nonexistent.id']) + assert set([r["id"].name for r in rq_grid]) == set( + ["my.entity.id1", "my.entity.id2", "my.nonexistent.id"] + ) # Make a grid to respond with response = hszinc.Grid() - response.column['id'] = {} - response.column['dis'] = {} - response.column['randomTag'] = {} - response.extend([{ - 'id': hszinc.Ref('my.entity.id2', value='id2'), - 'dis': 'A test entity #2', - 'randomTag': hszinc.MARKER - },{ - 'id': None, - 'dis': None, - 'randomTag': None - },{ - 'id': hszinc.Ref('my.entity.id1', value='id1'), - 'dis': 'A test entity #1', - 'randomTag': hszinc.MARKER - }]) - - rq.respond(status=200, headers={ - b'Content-Type': 'text/zinc', - }, content=hszinc.dump(response, mode=hszinc.MODE_ZINC)) + response.column["id"] = {} + response.column["dis"] = {} + response.column["randomTag"] = {} + response.extend( + [ + { + "id": hszinc.Ref("my.entity.id2", value="id2"), + "dis": "A test entity #2", + "randomTag": hszinc.MARKER, + }, + {"id": None, "dis": None, "randomTag": None}, + { + "id": hszinc.Ref("my.entity.id1", value="id1"), + "dis": "A test entity #1", + "randomTag": hszinc.MARKER, + }, + ] + ) + + rq.respond( + status=200, + headers={b"Content-Type": "text/zinc"}, + content=hszinc.dump(response, mode=hszinc.MODE_ZINC), + ) # State machine should now be done assert op.is_done entities = op.result # Response should be a dict - assert isinstance(entities, dict), '%r not a dict' % entity + assert isinstance(entities, dict), "%r not a dict" % entity # Response should have these keys - assert set(entities.keys()) == set(['my.entity.id1', 'my.entity.id2']) + assert set(entities.keys()) == set(["my.entity.id1", "my.entity.id2"]) - entity = entities.pop('my.entity.id1') - assert isinstance(entity, Entity), '%r not an entity' % entity + entity = entities.pop("my.entity.id1") + assert isinstance(entity, Entity), "%r not an entity" % entity # The tags should be passed through from the response - assert entity.id.name == 'my.entity.id1' - assert entity.tags['dis'] == response[2]['dis'] - assert entity.tags['randomTag'] == response[2]['randomTag'] + assert entity.id.name == "my.entity.id1" + assert entity.tags["dis"] == response[2]["dis"] + assert entity.tags["randomTag"] == response[2]["randomTag"] - entity = entities.pop('my.entity.id2') - assert isinstance(entity, Entity), '%r not an entity' % entity + entity = entities.pop("my.entity.id2") + assert isinstance(entity, Entity), "%r not an entity" % entity # The tags should be passed through from the response - assert entity.id.name == 'my.entity.id2' - assert entity.tags['dis'] == response[0]['dis'] - assert entity.tags['randomTag'] == response[0]['randomTag'] + assert entity.id.name == "my.entity.id2" + assert entity.tags["dis"] == response[0]["dis"] + assert entity.tags["randomTag"] == response[0]["randomTag"] def test_get_multi_entity(self, server_session): (server, session) = server_session # Try retrieving existing multiple entities - op = session.get_entity(['my.entity.id1', 'my.entity.id2'], - single=False) + op = session.get_entity(["my.entity.id1", "my.entity.id2"], single=False) # The operation should still be in progress assert not op.is_done @@ -260,70 +274,78 @@ def test_get_multi_entity(self, server_session): rq = server.next_request() # Request shall be a POST - assert rq.method == 'POST', 'Expecting POST, got %s' % rq + assert rq.method == "POST", "Expecting POST, got %s" % rq # Request shall be for base + 'api/read?id=@my.entity.id' - assert rq.uri == BASE_URI + 'api/read' + assert rq.uri == BASE_URI + "api/read" # Accept header shall be given - assert rq.headers[b'Accept'] == 'text/zinc' - assert rq.headers[b'Content-Type'] == 'text/zinc' + assert rq.headers[b"Accept"] == "text/zinc" + assert rq.headers[b"Content-Type"] == "text/zinc" # Body shall be a single grid: - rq_grid = hszinc.parse(rq.body.decode('utf-8'), mode=hszinc.MODE_ZINC) + rq_grid = hszinc.parse(rq.body.decode("utf-8"), mode=hszinc.MODE_ZINC) assert len(rq_grid) == 1 rq_grid = rq_grid[0] # It shall have one column; id - assert set(rq_grid.column.keys()) == set(['id']) + assert set(rq_grid.column.keys()) == set(["id"]) # It shall have 2 rows assert len(rq_grid) == 2 # Each row should only have 'id' values - assert all([(set(r.keys()) == set(['id'])) for r in rq_grid]) + assert all([(set(r.keys()) == set(["id"])) for r in rq_grid]) # The rows' 'id' column should *only* contain Refs. - assert all([isinstance(r['id'], hszinc.Ref) for r in rq_grid]) + assert all([isinstance(r["id"], hszinc.Ref) for r in rq_grid]) # Both IDs shall be listed, we don't about order - assert set([r['id'].name for r in rq_grid]) \ - == set(['my.entity.id1', 'my.entity.id2']) + assert set([r["id"].name for r in rq_grid]) == set( + ["my.entity.id1", "my.entity.id2"] + ) # Make a grid to respond with response = hszinc.Grid() - response.column['id'] = {} - response.column['dis'] = {} - response.column['randomTag'] = {} - response.extend([{ - 'id': hszinc.Ref('my.entity.id1', value='id1'), - 'dis': 'A test entity #1', - 'randomTag': hszinc.MARKER - },{ - 'id': hszinc.Ref('my.entity.id2', value='id2'), - 'dis': 'A test entity #2', - 'randomTag': hszinc.MARKER - }]) - - rq.respond(status=200, headers={ - b'Content-Type': 'text/zinc', - }, content=hszinc.dump(response, mode=hszinc.MODE_ZINC)) + response.column["id"] = {} + response.column["dis"] = {} + response.column["randomTag"] = {} + response.extend( + [ + { + "id": hszinc.Ref("my.entity.id1", value="id1"), + "dis": "A test entity #1", + "randomTag": hszinc.MARKER, + }, + { + "id": hszinc.Ref("my.entity.id2", value="id2"), + "dis": "A test entity #2", + "randomTag": hszinc.MARKER, + }, + ] + ) + + rq.respond( + status=200, + headers={b"Content-Type": "text/zinc"}, + content=hszinc.dump(response, mode=hszinc.MODE_ZINC), + ) # State machine should now be done assert op.is_done entities = op.result # Response should be a dict - assert isinstance(entities, dict), '%r not a dict' % entity + assert isinstance(entities, dict), "%r not a dict" % entity # Response should have these keys - assert set(entities.keys()) == set(['my.entity.id1','my.entity.id2']) + assert set(entities.keys()) == set(["my.entity.id1", "my.entity.id2"]) - entity = entities.pop('my.entity.id1') - assert isinstance(entity, Entity), '%r not an entity' % entity + entity = entities.pop("my.entity.id1") + assert isinstance(entity, Entity), "%r not an entity" % entity # The tags should be passed through from the response - assert entity.id.name == 'my.entity.id1' - assert entity.tags['dis'] == response[0]['dis'] - assert entity.tags['randomTag'] == response[0]['randomTag'] + assert entity.id.name == "my.entity.id1" + assert entity.tags["dis"] == response[0]["dis"] + assert entity.tags["randomTag"] == response[0]["randomTag"] - entity = entities.pop('my.entity.id2') - assert isinstance(entity, Entity), '%r not an entity' % entity + entity = entities.pop("my.entity.id2") + assert isinstance(entity, Entity), "%r not an entity" % entity # The tags should be passed through from the response - assert entity.id.name == 'my.entity.id2' - assert entity.tags['dis'] == response[1]['dis'] - assert entity.tags['randomTag'] == response[1]['randomTag'] + assert entity.id.name == "my.entity.id2" + assert entity.tags["dis"] == response[1]["dis"] + assert entity.tags["randomTag"] == response[1]["randomTag"] diff --git a/tests/manual/multiRWtest.py b/tests/manual/multiRWtest.py index 062bfd9..03a1661 100644 --- a/tests/manual/multiRWtest.py +++ b/tests/manual/multiRWtest.py @@ -1,10 +1,10 @@ # Simple application that uses the pyhaystack library # For this program to work you will need to have; # 1) the API server up and running at port 3000 -# 2) Insert the point AUTH_Site.1.Equip.multiPt04 and AUTH_Site.1.Equip.multiPt05 +# 2) Insert the point AUTH_Site.1.Equip.multiPt04 and AUTH_Site.1.Equip.multiPt05 # - You can the API system test to insert these points -# Expected output: +# Expected output: # Setup Widesky session. Object= # Written some data using multi hisWrite. response should be blank = [] # Read the written data using multi hisRead. response is = @@ -30,37 +30,45 @@ import pytz # Edit the following parameters to suit your system -WS_URI='http://localhost:3000' -WS_USERNAME='youruser@example.com' -WS_PASSWORD='yourpassword' -WS_CLIENT_ID='xxxx' -WS_CLIENT_SECRET='yyyy' +WS_URI = "http://localhost:3000" +WS_USERNAME = "youruser@example.com" +WS_PASSWORD = "yourpassword" +WS_CLIENT_ID = "xxxx" +WS_CLIENT_SECRET = "yyyy" + def doStuff(): session = WideskyHaystackSession( - uri=WS_URI, - username=WS_USERNAME, - password=WS_PASSWORD, - client_id=WS_CLIENT_ID, - client_secret=WS_CLIENT_SECRET) + uri=WS_URI, + username=WS_USERNAME, + password=WS_PASSWORD, + client_id=WS_CLIENT_ID, + client_secret=WS_CLIENT_SECRET, + ) - print "Setup Widesky session. Object=", session + print("Setup Widesky session. Object=", session) result = session.multi_his_write( - timestamp_records= { - datetime.datetime.now(tz=pytz.timezone('Australia/Brisbane')).replace(microsecond=0): - { - 'AUTH_Site.1.Equip.1.multiPt04': 800.8, - 'AUTH_Site.1.Equip.1.multiPt05': 123 - }}).result[:] + timestamp_records={ + datetime.datetime.now(tz=pytz.timezone("Australia/Brisbane")).replace( + microsecond=0 + ): { + "AUTH_Site.1.Equip.1.multiPt04": 800.8, + "AUTH_Site.1.Equip.1.multiPt05": 123, + } + } + ).result[:] + + print("Written some data using multi hisWrite. response should be blank = ", result) - print "Written some data using multi hisWrite. response should be blank = ", result + result = session.multi_his_read( + points=["AUTH_Site.1.Equip.1.multiPt04", "AUTH_Site.1.Equip.1.multiPt05"], + rng="today", + ) + print("Read the written data using multi hisRead. response is = ", result) - result = session.multi_his_read(points=['AUTH_Site.1.Equip.1.multiPt04', 'AUTH_Site.1.Equip.1.multiPt05'], rng='today') - print "Read the written data using multi hisRead. response is = ", result if __name__ == "__main__": try: doStuff() except KeyboardInterrupt: sys.exit(0) - diff --git a/tests/test_niagara_escape.py b/tests/test_niagara_escape.py new file mode 100644 index 0000000..ccc0dbb --- /dev/null +++ b/tests/test_niagara_escape.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +""" +First test... just import something... +""" +import pytest +import sys + +from pyhaystack.client.niagara import Niagara4HaystackSession + + +@pytest.mark.skipif(sys.version_info < (3, 4), reason="requires python3 or higher") +def test_conversion_of_str(): + unescape = Niagara4HaystackSession.unescape + dct = { + "H.Client.Labo~2f227~2d2~2fBA~2fPC_D~e9bit_Alim": "H.Client.Labo/227-2/BA/PC_Débit_Alim" + } + for k, v in dct.items(): + assert unescape(k) == v diff --git a/tests/test_niagaraax.py b/tests/test_niagaraax.py index 60ff84c..1c806a0 100644 --- a/tests/test_niagaraax.py +++ b/tests/test_niagaraax.py @@ -6,20 +6,24 @@ from pyhaystack.client.niagara import NiagaraHaystackSession -@pytest.fixture(scope='module') + +@pytest.fixture(scope="module") def session(request): - session = NiagaraHaystackSession(uri='http://www.myserver.com', - username='user_name', - password='M87h$&') - + session = NiagaraHaystackSession( + uri="http://www.myserver.com", username="user_name", password="M87h$&" + ) + def terminate(): session = None print("It's over") + request.addfinalizer(terminate) return session - + + def test_session_username(session): - assert session._username == 'user_name' - + assert session._username == "user_name" + + def test_session_password(session): - assert session._password == 'M87h$&' \ No newline at end of file + assert session._password == "M87h$&" diff --git a/tests/test_skyspark.py b/tests/test_skyspark.py index c206eba..4582f9e 100644 --- a/tests/test_skyspark.py +++ b/tests/test_skyspark.py @@ -7,14 +7,15 @@ from pyhaystack.client.ops.vendor.skyspark import get_digest_info + def test_digest_creation(): test_param = { - 'username' : "alice", - 'password' : "secret", - 'userSalt' : "6s6Q5Rn0xZP0LPf89bNdv+65EmMUrTsey2fIhim/wKU=", - 'nonce' : "3da210bdb1163d0d41d3c516314cbd6e" - } + "username": "alice", + "password": "secret", + "userSalt": "6s6Q5Rn0xZP0LPf89bNdv+65EmMUrTsey2fIhim/wKU=", + "nonce": "3da210bdb1163d0d41d3c516314cbd6e", + } test_result = get_digest_info(test_param) - assert test_result['digest'] == "B2B3mIzE/+dqcqOJJ/ejSGXRKvE=" - assert test_result['hmac'] == "z9NILqJ3QHSG5+GlDnXsV9txjgo=" + assert test_result["digest"] == "B2B3mIzE/+dqcqOJJ/ejSGXRKvE=" + assert test_result["hmac"] == "z9NILqJ3QHSG5+GlDnXsV9txjgo=" diff --git a/tests/test_widesky.py b/tests/test_widesky.py index 749c332..c72f2ce 100644 --- a/tests/test_widesky.py +++ b/tests/test_widesky.py @@ -26,11 +26,13 @@ # Logging setup so we can see what's going on import logging + logging.basicConfig(level=logging.DEBUG) from pyhaystack.client import widesky -BASE_URI = 'https://myserver/api/' +BASE_URI = "https://myserver/api/" + class TestIsLoggedIn(object): """ @@ -44,12 +46,13 @@ def test_returns_false_if_no_auth_result(self): server = dummy_http.DummyHttpServer() session = widesky.WideskyHaystackSession( uri=BASE_URI, - username='testuser', - password='testpassword', - client_id='testclient', - client_secret='testclientsecret', + username="testuser", + password="testpassword", + client_id="testclient", + client_secret="testclientsecret", http_client=dummy_http.DummyHttpClient, - http_args={'server': server, 'debug': True}) + http_args={"server": server, "debug": True}, + ) # Straight off the bat, this should be None assert session._auth_result is None @@ -61,12 +64,13 @@ def test_returns_false_if_no_expires_in(self): server = dummy_http.DummyHttpServer() session = widesky.WideskyHaystackSession( uri=BASE_URI, - username='testuser', - password='testpassword', - client_id='testclient', - client_secret='testclientsecret', + username="testuser", + password="testpassword", + client_id="testclient", + client_secret="testclientsecret", http_client=dummy_http.DummyHttpClient, - http_args={'server': server, 'debug': True}) + http_args={"server": server, "debug": True}, + ) # Inject our own auth result, empty dict. session._auth_result = {} @@ -78,17 +82,19 @@ def test_returns_false_if_expires_in_past(self): server = dummy_http.DummyHttpServer() session = widesky.WideskyHaystackSession( uri=BASE_URI, - username='testuser', - password='testpassword', - client_id='testclient', - client_secret='testclientsecret', + username="testuser", + password="testpassword", + client_id="testclient", + client_secret="testclientsecret", http_client=dummy_http.DummyHttpClient, - http_args={'server': server, 'debug': True}) + http_args={"server": server, "debug": True}, + ) # Inject our own auth result, expiry in the past. session._auth_result = { - # Milliseconds here! - 'expires_in': (time.time() - 1.0) * 1000.0 + # Milliseconds here! + "expires_in": (time.time() - 1.0) + * 1000.0 } # We should see a False result here @@ -98,17 +104,19 @@ def test_returns_true_if_expires_in_future(self): server = dummy_http.DummyHttpServer() session = widesky.WideskyHaystackSession( uri=BASE_URI, - username='testuser', - password='testpassword', - client_id='testclient', - client_secret='testclientsecret', + username="testuser", + password="testpassword", + client_id="testclient", + client_secret="testclientsecret", http_client=dummy_http.DummyHttpClient, - http_args={'server': server, 'debug': True}) + http_args={"server": server, "debug": True}, + ) # Inject our own auth result, expiry in the future. session._auth_result = { - # Milliseconds here! - 'expires_in': (time.time() + 1.0) * 1000.0 + # Milliseconds here! + "expires_in": (time.time() + 1.0) + * 1000.0 } # We should see a True result here @@ -127,33 +135,36 @@ def test_no_op_if_response_not_401(self): server = dummy_http.DummyHttpServer() session = widesky.WideskyHaystackSession( uri=BASE_URI, - username='testuser', - password='testpassword', - client_id='testclient', - client_secret='testclientsecret', + username="testuser", + password="testpassword", + client_id="testclient", + client_secret="testclientsecret", http_client=dummy_http.DummyHttpClient, - http_args={'server': server, 'debug': True}) + http_args={"server": server, "debug": True}, + ) # Seed this with parameters auth_result = { - 'expires_in': (time.time() + 3600.0) * 1000.0, - 'access_token': 'abcdefgh', - 'refresh_token': '12345678', + "expires_in": (time.time() + 3600.0) * 1000.0, + "access_token": "abcdefgh", + "refresh_token": "12345678", } session._auth_result = auth_result # A dummy response, we don't care much about the content. - res = HTTPResponse(status_code=200, headers={}, body='') + res = HTTPResponse(status_code=200, headers={}, body="") # This should do nothing session._on_http_grid_response(res) # Same keys - assert set(auth_result.keys()) == set(session._auth_result.keys()), \ - 'Keys mismatch' + assert set(auth_result.keys()) == set( + session._auth_result.keys() + ), "Keys mismatch" for key in auth_result.keys(): - assert auth_result[key] == session._auth_result[key], \ - 'Mismatching key %s' % key + assert auth_result[key] == session._auth_result[key], ( + "Mismatching key %s" % key + ) def test_logout_if_response_401(self): """ @@ -162,23 +173,24 @@ def test_logout_if_response_401(self): server = dummy_http.DummyHttpServer() session = widesky.WideskyHaystackSession( uri=BASE_URI, - username='testuser', - password='testpassword', - client_id='testclient', - client_secret='testclientsecret', + username="testuser", + password="testpassword", + client_id="testclient", + client_secret="testclientsecret", http_client=dummy_http.DummyHttpClient, - http_args={'server': server, 'debug': True}) + http_args={"server": server, "debug": True}, + ) # Seed this with parameters auth_result = { - 'expires_in': (time.time() + 3600.0) * 1000.0, - 'access_token': 'abcdefgh', - 'refresh_token': '12345678', + "expires_in": (time.time() + 3600.0) * 1000.0, + "access_token": "abcdefgh", + "refresh_token": "12345678", } session._auth_result = auth_result # A dummy response, we don't care much about the content. - res = HTTPResponse(status_code=401, headers={}, body='') + res = HTTPResponse(status_code=401, headers={}, body="") # This should drop our session session._on_http_grid_response(res) @@ -191,24 +203,25 @@ def test_logout_if_response_401_via_exception(self): server = dummy_http.DummyHttpServer() session = widesky.WideskyHaystackSession( uri=BASE_URI, - username='testuser', - password='testpassword', - client_id='testclient', - client_secret='testclientsecret', + username="testuser", + password="testpassword", + client_id="testclient", + client_secret="testclientsecret", http_client=dummy_http.DummyHttpClient, - http_args={'server': server, 'debug': True}) + http_args={"server": server, "debug": True}, + ) # Seed this with parameters auth_result = { - 'expires_in': (time.time() + 3600.0) * 1000.0, - 'access_token': 'abcdefgh', - 'refresh_token': '12345678', + "expires_in": (time.time() + 3600.0) * 1000.0, + "access_token": "abcdefgh", + "refresh_token": "12345678", } session._auth_result = auth_result # Generate a HTTPStatusError, wrap it up in an AsynchronousException try: - raise HTTPStatusError('Unauthorized', 401) + raise HTTPStatusError("Unauthorized", 401) except HTTPStatusError: res = AsynchronousException() @@ -223,18 +236,19 @@ def test_no_op_if_exception_not_http_status_error(self): server = dummy_http.DummyHttpServer() session = widesky.WideskyHaystackSession( uri=BASE_URI, - username='testuser', - password='testpassword', - client_id='testclient', - client_secret='testclientsecret', + username="testuser", + password="testpassword", + client_id="testclient", + client_secret="testclientsecret", http_client=dummy_http.DummyHttpClient, - http_args={'server': server, 'debug': True}) + http_args={"server": server, "debug": True}, + ) # Seed this with parameters auth_result = { - 'expires_in': (time.time() + 3600.0) * 1000.0, - 'access_token': 'abcdefgh', - 'refresh_token': '12345678', + "expires_in": (time.time() + 3600.0) * 1000.0, + "access_token": "abcdefgh", + "refresh_token": "12345678", } session._auth_result = auth_result @@ -243,7 +257,7 @@ class DummyError(Exception): pass try: - raise DummyError('Testing') + raise DummyError("Testing") except DummyError: res = AsynchronousException() @@ -251,8 +265,10 @@ class DummyError(Exception): session._on_http_grid_response(res) # Same keys - assert set(auth_result.keys()) == set(session._auth_result.keys()), \ - 'Keys mismatch' + assert set(auth_result.keys()) == set( + session._auth_result.keys() + ), "Keys mismatch" for key in auth_result.keys(): - assert auth_result[key] == session._auth_result[key], \ - 'Mismatching key %s' % key + assert auth_result[key] == session._auth_result[key], ( + "Mismatching key %s" % key + ) diff --git a/tests/util.py b/tests/util.py index 51c27b1..7b7995f 100644 --- a/tests/util.py +++ b/tests/util.py @@ -9,39 +9,39 @@ import hszinc + def grid_meta_cmp(msg, expected, actual): errors = [] for key in set(expected.keys()) | set(expected.keys()): if key not in expected: - errors.append('%s key %s unexpected' % (msg, key)) + errors.append("%s key %s unexpected" % (msg, key)) elif key not in actual: - errors.append('%s key %s missing' % (msg, key)) + errors.append("%s key %s missing" % (msg, key)) else: ev = expected[key] av = actual[key] if ev != av: - errors.append('%s key %s was %r not %r' \ - % (msg, key, av, ev)) + errors.append("%s key %s was %r not %r" % (msg, key, av, ev)) return errors + def grid_col_cmp(expected, actual): errors = [] for col in set(expected.keys()) | set(actual.keys()): if col not in expected: - errors.append('unexpected column %s' % col) + errors.append("unexpected column %s" % col) elif col not in actual: - errors.append('missing column %s' % col) + errors.append("missing column %s" % col) else: - errors.extend(grid_meta_cmp('column %s' % col, - expected[col], actual[col])) + errors.extend(grid_meta_cmp("column %s" % col, expected[col], actual[col])) return errors + def grid_cmp(expected, actual): - assert isinstance(expected, hszinc.Grid), 'expected is not a grid' - assert isinstance(actual, hszinc.Grid), 'actual is not a grid' + assert isinstance(expected, hszinc.Grid), "expected is not a grid" + assert isinstance(actual, hszinc.Grid), "actual is not a grid" - errors = grid_meta_cmp('grid metadata', - expected.metadata, actual.metadata) + errors = grid_meta_cmp("grid metadata", expected.metadata, actual.metadata) errors.extend(grid_col_cmp(expected.column, actual.column)) for idx in range(0, max(len(expected), len(actual))): @@ -55,11 +55,11 @@ def grid_cmp(expected, actual): arow = None if erow is None: - errors.append('Unexpected row %d: %r' % (idx, arow)) + errors.append("Unexpected row %d: %r" % (idx, arow)) elif arow is None: - errors.append('Missing row %d: %r' % (idx, erow)) + errors.append("Missing row %d: %r" % (idx, erow)) else: - errors.extend(grid_meta_cmp('Row %d' % idx, erow, arow)) + errors.extend(grid_meta_cmp("Row %d" % idx, erow, arow)) if errors: - assert False, 'Grids do not match:\n- %s' % ('\n- '.join(errors)) + assert False, "Grids do not match:\n- %s" % ("\n- ".join(errors))