Develop (#17)

- Support namespaces in hello messages -- primarily to support "rfc-compliant" mode in JunOS -- thank you [Gary Napier](https://github.com/napierg) for finding this and coming up with the fix!
- Another fixup to chunk checker -- think that the itty bitty chunk issues have now been solved :)
- Minor house keeping bits... nothing exciting!

CHANGELOG.md

@@ -1,6 +1,12 @@
 CHANGELOG
 =========
 
+# 2020.11.15
+- Support namespaces in hello messages -- primarily to support "rfc-compliant" mode in JunOS -- thank you 
+ [Gary Napier](https://github.com/napierg) for finding this and coming up with the fix!
+- Another fixup to chunk checker -- think that the itty bitty chunk issues have now been solved :)
+
+
 # 2020.10.24
 - Improve the "echo" checker -- and add this for sync as well, because...
 - SSH2 and Paramiko are now supported transports!

README.md

@@ -1,10 +1,8 @@
-![](https://github.com/scrapli/scrapli_netconf/workflows/Weekly%20Build/badge.svg)
+[![Supported Versions](https://img.shields.io/pypi/pyversions/scrapli.svg)](https://pypi.org/project/scrapli)
 [![PyPI version](https://badge.fury.io/py/scrapli_netconf.svg)](https://badge.fury.io/py/scrapli_netconf)
-[![Python 3.6](https://img.shields.io/badge/python-3.6-blue.svg)](https://www.python.org/downloads/release/python-360/)
-[![Python 3.7](https://img.shields.io/badge/python-3.7-blue.svg)](https://www.python.org/downloads/release/python-370/)
-[![Python 3.8](https://img.shields.io/badge/python-3.8-blue.svg)](https://www.python.org/downloads/release/python-380/)
+[![Weekly Build](https://github.com/scrapli/scrapli_netconf/workflows/Weekly%20Build/badge.svg)](https://github.com/scrapli/scrapli_netconf/actions?query=workflow%3A%22Weekly+Build%22)
 [![Code Style](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/ambv/black)
-
+[![License: MIT](https://img.shields.io/badge/License-MIT-blueviolet.svg)](https://opensource.org/licenses/MIT)
 
 scrapli_netconf
 ===============

docs/scrapli_netconf/driver/base_driver.html

@@ -121,8 +121,11 @@ <h1 class="title">Module <code>scrapli_netconf.driver.base_driver</code></h1>
             CouldNotExchangeCapabilities: if server capabilities cannot be parsed
 
         &#34;&#34;&#34;
+        # matches hello with or without namespace
         filtered_raw_server_capabilities = re.search(
-            pattern=rb&#34;(&lt;hello.*&lt;\/hello&gt;)&#34;, string=raw_server_capabilities, flags=re.I | re.S
+            pattern=rb&#34;(&lt;(\w+\:){0,1}hello.*&lt;\/(\w+\:){0,1}hello&gt;)&#34;,
+            string=raw_server_capabilities,
+            flags=re.I | re.S,
         )
         if filtered_raw_server_capabilities is None:
             msg = f&#34;Failed to parse server capabilities from host {self._host}&#34;
@@ -726,7 +729,7 @@ <h1 class="title">Module <code>scrapli_netconf.driver.base_driver</code></h1>
                 channel inputs (string and xml)
 
         Raises:
-            CapabilityNotSupported: if `validate` ca
+            CapabilityNotSupported: if `validate` capability does not exist
 
         &#34;&#34;&#34;
         self.logger.debug(&#34;Building payload for `validate` operation.&#34;)
@@ -1054,8 +1057,11 @@ <h2 id="raises">Raises</h2>
             CouldNotExchangeCapabilities: if server capabilities cannot be parsed
 
         &#34;&#34;&#34;
+        # matches hello with or without namespace
         filtered_raw_server_capabilities = re.search(
-            pattern=rb&#34;(&lt;hello.*&lt;\/hello&gt;)&#34;, string=raw_server_capabilities, flags=re.I | re.S
+            pattern=rb&#34;(&lt;(\w+\:){0,1}hello.*&lt;\/(\w+\:){0,1}hello&gt;)&#34;,
+            string=raw_server_capabilities,
+            flags=re.I | re.S,
         )
         if filtered_raw_server_capabilities is None:
             msg = f&#34;Failed to parse server capabilities from host {self._host}&#34;
@@ -1659,7 +1665,7 @@ <h2 id="raises">Raises</h2>
                 channel inputs (string and xml)
 
         Raises:
-            CapabilityNotSupported: if `validate` ca
+            CapabilityNotSupported: if `validate` capability does not exist
 
         &#34;&#34;&#34;
         self.logger.debug(&#34;Building payload for `validate` operation.&#34;)

docs/scrapli_netconf/helper.html

@@ -123,7 +123,7 @@ <h1 class="title">Module <code>scrapli_netconf.helper</code></h1>
 <h2 class="section-title" id="header-functions">Functions</h2>
 <dl>
 <dt id="scrapli_netconf.helper.remove_namespaces"><code class="name flex">
-<span>def <span class="ident">remove_namespaces</span></span>(<span>tree: <cyfunction Element at 0x7f852836a2b0>) ‑> <cyfunction Element at 0x7f852836a2b0></span>
+<span>def <span class="ident">remove_namespaces</span></span>(<span>tree: <cyfunction Element at 0x7fd2882252b0>) ‑> <cyfunction Element at 0x7fd2882252b0></span>
 </code></dt>
 <dd>
 <div class="desc"><p>Remove all namespace tags from Element object</p>

docs/scrapli_netconf/index.html

@@ -30,8 +30,8 @@ <h1 class="title">Package <code>scrapli_netconf</code></h1>
 <pre><code class="python">&#34;&#34;&#34;scrapli_netconf scrapli netconf plugin&#34;&#34;&#34;
 from scrapli_netconf.driver import AsyncNetconfScrape, NetconfScrape
 
-__version__ = &#34;2020.10.24&#34;
-__all__ = (&#34;NetconfScrape&#34;, &#34;AsyncNetconfScrape&#34;)</code></pre>
+__version__ = &#34;2020.11.15&#34;
+__all__ = (&#34;NetconfScrape&#34;, &#34;AsyncNetconfScrape&#34;, &#34;__version__&#34;)</code></pre>
 </details>
 </section>
 <section>

docs/scrapli_netconf/response.html

@@ -231,9 +231,11 @@ <h1 class="title">Module <code>scrapli_netconf.response</code></h1>
             extraneous_trailing_newline_count = 1
         trimmed_newline_len = actual_len - extraneous_trailing_newline_count
 
-        if expected_len == 1:
-            # at least nokia tends to have itty bitty chunks of one element, deal w/ that
-            actual_len = 1
+        if rstripped_len == 0:
+            # at least nokia tends to have itty bitty chunks of one element, and/or chunks that have
+            # *only* whitespace and our regex ignores this, so if there was/is nothing in the result
+            # section we can assume it was just whitespace and move on w/our lives
+            actual_len = expected_len
 
         if expected_len == actual_len:
             return
@@ -267,7 +269,7 @@ <h1 class="title">Module <code>scrapli_netconf.response</code></h1>
         # validate all received data
         for result in result_sections:
             self._validate_chunk_size_netconf_1_1(result=result)
-        # assert 0
+
         self.xml_result = etree.fromstring(
             b&#34;\n&#34;.join(
                 [
@@ -376,7 +378,7 @@ <h2 class="section-title" id="header-classes">Classes</h2>
 <dl>
 <dt id="scrapli_netconf.response.NetconfResponse"><code class="flex name class">
 <span>class <span class="ident">NetconfResponse</span></span>
-<span>(</span><span>netconf_version: <a title="scrapli_netconf.constants.NetconfVersion" href="constants.html#scrapli_netconf.constants.NetconfVersion">NetconfVersion</a>, xml_input: <cyfunction Element at 0x7f852836a2b0>, strip_namespaces: bool = True, failed_when_contains: Union[bytes, List[bytes], NoneType] = None, **kwargs: Any)</span>
+<span>(</span><span>netconf_version: <a title="scrapli_netconf.constants.NetconfVersion" href="constants.html#scrapli_netconf.constants.NetconfVersion">NetconfVersion</a>, xml_input: <cyfunction Element at 0x7fd2882252b0>, strip_namespaces: bool = True, failed_when_contains: Union[bytes, List[bytes], NoneType] = None, **kwargs: Any)</span>
 </code></dt>
 <dd>
 <div class="desc"><p>Scrapli Netconf NetconfResponse</p>
@@ -587,9 +589,11 @@ <h2 id="raises">Raises</h2>
             extraneous_trailing_newline_count = 1
         trimmed_newline_len = actual_len - extraneous_trailing_newline_count
 
-        if expected_len == 1:
-            # at least nokia tends to have itty bitty chunks of one element, deal w/ that
-            actual_len = 1
+        if rstripped_len == 0:
+            # at least nokia tends to have itty bitty chunks of one element, and/or chunks that have
+            # *only* whitespace and our regex ignores this, so if there was/is nothing in the result
+            # section we can assume it was just whitespace and move on w/our lives
+            actual_len = expected_len
 
         if expected_len == actual_len:
             return
@@ -623,7 +627,7 @@ <h2 id="raises">Raises</h2>
         # validate all received data
         for result in result_sections:
             self._validate_chunk_size_netconf_1_1(result=result)
-        # assert 0
+
         self.xml_result = etree.fromstring(
             b&#34;\n&#34;.join(
                 [
@@ -763,7 +767,7 @@ <h2 id="raises">Raises</h2>
 </details>
 </dd>
 <dt id="scrapli_netconf.response.NetconfResponse.get_xml_elements"><code class="name flex">
-<span>def <span class="ident">get_xml_elements</span></span>(<span>self) ‑> Dict[str, <cyfunction Element at 0x7f852836a2b0>]</span>
+<span>def <span class="ident">get_xml_elements</span></span>(<span>self) ‑> Dict[str, <cyfunction Element at 0x7fd2882252b0>]</span>
 </code></dt>
 <dd>
 <div class="desc"><p>Parse each section under "data" into a dict of {tag: Element} for easy viewing/parsing</p>

docs/scrapli_netconf/transport/asyncssh_.html

@@ -51,7 +51,7 @@ <h1 class="title">Module <code>scrapli_netconf.transport.asyncssh_</code></h1>
             bytes: bytes output from server captured while opening the connection
 
         Raises:
-            N/A
+            ConnectionNotOpened: if unable to open connection (`ChannelOpenError` from asyncssh)
 
         &#34;&#34;&#34;
         from asyncssh.misc import ChannelOpenError  # pylint: disable=C0415
@@ -188,7 +188,7 @@ <h2 id="raises">Raises</h2>
             bytes: bytes output from server captured while opening the connection
 
         Raises:
-            N/A
+            ConnectionNotOpened: if unable to open connection (`ChannelOpenError` from asyncssh)
 
         &#34;&#34;&#34;
         from asyncssh.misc import ChannelOpenError  # pylint: disable=C0415
@@ -279,7 +279,10 @@ <h2 id="returns">Returns</h2>
 <dd>bytes output from server captured while opening the connection</dd>
 </dl>
 <h2 id="raises">Raises</h2>
-<p>N/A</p></div>
+<dl>
+<dt><code>ConnectionNotOpened</code></dt>
+<dd>if unable to open connection (<code>ChannelOpenError</code> from asyncssh)</dd>
+</dl></div>
 <details class="source">
 <summary>
 <span>Expand source code</span>
@@ -295,7 +298,7 @@ <h2 id="raises">Raises</h2>
         bytes: bytes output from server captured while opening the connection
 
     Raises:
-        N/A
+        ConnectionNotOpened: if unable to open connection (`ChannelOpenError` from asyncssh)
 
     &#34;&#34;&#34;
     from asyncssh.misc import ChannelOpenError  # pylint: disable=C0415

docs/scrapli_netconf/transport/index.html

@@ -213,6 +213,26 @@ <h2 id="raises">Raises</h2>
         self.logger.debug(f&#34;Authenticated to host {self.host} successfully&#34;)
         return login_bytes
 
+    @staticmethod
+    def _authenticate_check_hello(output: bytes) -&gt; bool:
+        &#34;&#34;&#34;
+        Check if &#34;hello&#34; message is in output
+
+        Args:
+            output: bytes output from the channel
+
+        Returns:
+            bool: true if hello message is seen, otherwise false
+
+        Raises:
+            N/A
+
+        &#34;&#34;&#34;
+        hello_match = re.search(pattern=HELLO_MATCH, string=output)
+        if hello_match:
+            return True
+        return False
+
     @OperationTimeout(&#34;_timeout_ops&#34;, &#34;Timed out looking for SSH login password prompt&#34;)
     def _authenticate(self) -&gt; bytes:
         &#34;&#34;&#34;
@@ -261,7 +281,7 @@ <h2 id="raises">Raises</h2>
                     &#34;likely failed authentication&#34;
                 )
                 raise ScrapliAuthenticationFailed(msg)
-            if b&#34;&lt;hello&#34; in output.lower():
+            if self._authenticate_check_hello(output=output):
                 self.logger.info(&#34;Found start of server capabilities, authentication successful&#34;)
                 self._isauthenticated = True
                 return output</code></pre>

docs/scrapli_netconf/transport/systemssh.html

@@ -28,13 +28,16 @@ <h1 class="title">Module <code>scrapli_netconf.transport.systemssh</code></h1>
 <span>Expand source code</span>
 </summary>
 <pre><code class="python">&#34;&#34;&#34;scrapli_netconf.transport.systemssh&#34;&#34;&#34;
+import re
 from typing import Any
 
 from scrapli.decorators import OperationTimeout
 from scrapli.exceptions import ScrapliAuthenticationFailed
 from scrapli.transport import SystemSSHTransport
 from scrapli.transport.ptyprocess import PtyProcess
 
+HELLO_MATCH = re.compile(pattern=rb&#34;&lt;(\w+\:){0,1}hello&#34;, flags=re.I)
+
 
 class NetconfSystemSSHTransport(SystemSSHTransport):
     def __init__(self, **kwargs: Any):
@@ -85,6 +88,26 @@ <h1 class="title">Module <code>scrapli_netconf.transport.systemssh</code></h1>
         self.logger.debug(f&#34;Authenticated to host {self.host} successfully&#34;)
         return login_bytes
 
+    @staticmethod
+    def _authenticate_check_hello(output: bytes) -&gt; bool:
+        &#34;&#34;&#34;
+        Check if &#34;hello&#34; message is in output
+
+        Args:
+            output: bytes output from the channel
+
+        Returns:
+            bool: true if hello message is seen, otherwise false
+
+        Raises:
+            N/A
+
+        &#34;&#34;&#34;
+        hello_match = re.search(pattern=HELLO_MATCH, string=output)
+        if hello_match:
+            return True
+        return False
+
     @OperationTimeout(&#34;_timeout_ops&#34;, &#34;Timed out looking for SSH login password prompt&#34;)
     def _authenticate(self) -&gt; bytes:
         &#34;&#34;&#34;
@@ -133,7 +156,7 @@ <h1 class="title">Module <code>scrapli_netconf.transport.systemssh</code></h1>
                     &#34;likely failed authentication&#34;
                 )
                 raise ScrapliAuthenticationFailed(msg)
-            if b&#34;&lt;hello&#34; in output.lower():
+            if self._authenticate_check_hello(output=output):
                 self.logger.info(&#34;Found start of server capabilities, authentication successful&#34;)
                 self._isauthenticated = True
                 return output</code></pre>
@@ -300,6 +323,26 @@ <h2 id="raises">Raises</h2>
         self.logger.debug(f&#34;Authenticated to host {self.host} successfully&#34;)
         return login_bytes
 
+    @staticmethod
+    def _authenticate_check_hello(output: bytes) -&gt; bool:
+        &#34;&#34;&#34;
+        Check if &#34;hello&#34; message is in output
+
+        Args:
+            output: bytes output from the channel
+
+        Returns:
+            bool: true if hello message is seen, otherwise false
+
+        Raises:
+            N/A
+
+        &#34;&#34;&#34;
+        hello_match = re.search(pattern=HELLO_MATCH, string=output)
+        if hello_match:
+            return True
+        return False
+
     @OperationTimeout(&#34;_timeout_ops&#34;, &#34;Timed out looking for SSH login password prompt&#34;)
     def _authenticate(self) -&gt; bytes:
         &#34;&#34;&#34;
@@ -348,7 +391,7 @@ <h2 id="raises">Raises</h2>
                     &#34;likely failed authentication&#34;
                 )
                 raise ScrapliAuthenticationFailed(msg)
-            if b&#34;&lt;hello&#34; in output.lower():
+            if self._authenticate_check_hello(output=output):
                 self.logger.info(&#34;Found start of server capabilities, authentication successful&#34;)
                 self._isauthenticated = True
                 return output</code></pre>

noxfile.py

@@ -117,8 +117,8 @@ def pylama(session):
         N/A
 
     """
+    session.install("-e", ".")
     session.install("-r", "requirements-dev.txt")
-    session.install(".")
     session.run("pylama", ".")
 
 

requirements-dev.txt

@@ -2,9 +2,9 @@ nox==2020.8.22
 black==20.8b1
 isort==5.6.4
 mypy==0.790
-pytest==6.1.1
+pytest==6.1.2
 pytest-cov==2.10.1
-pyfakefs==4.1.0
+pyfakefs==4.2.1
 pylama==7.7.1
 pycodestyle>=2.6.0
 pydocstyle==5.1.1

requirements.txt

@@ -1,3 +1,3 @@
-scrapli>=2020.10.10
+scrapli>=2020.11.15
 scrapli-asyncssh>=2020.10.10
 lxml>=4.5.1,<5.0.0

scrapli_netconf/__init__.py

@@ -1,5 +1,5 @@
 """scrapli_netconf scrapli netconf plugin"""
 from scrapli_netconf.driver import AsyncNetconfScrape, NetconfScrape
 
-__version__ = "2020.10.24"
-__all__ = ("NetconfScrape", "AsyncNetconfScrape")
+__version__ = "2020.11.15"
+__all__ = ("NetconfScrape", "AsyncNetconfScrape", "__version__")

scrapli_netconf/driver/base_driver.py

@@ -92,8 +92,11 @@ def _parse_server_capabilities(self, raw_server_capabilities: bytes) -> None:
             CouldNotExchangeCapabilities: if server capabilities cannot be parsed
 
         """
+        # matches hello with or without namespace
         filtered_raw_server_capabilities = re.search(
-            pattern=rb"(<hello.*<\/hello>)", string=raw_server_capabilities, flags=re.I | re.S
+            pattern=rb"(<(\w+\:){0,1}hello.*<\/(\w+\:){0,1}hello>)",
+            string=raw_server_capabilities,
+            flags=re.I | re.S,
         )
         if filtered_raw_server_capabilities is None:
             msg = f"Failed to parse server capabilities from host {self._host}"
@@ -697,7 +700,7 @@ def _pre_validate(self, source: str) -> NetconfResponse:
                 channel inputs (string and xml)
 
         Raises:
-            CapabilityNotSupported: if `validate` ca
+            CapabilityNotSupported: if `validate` capability does not exist
 
         """
         self.logger.debug("Building payload for `validate` operation.")