diff --git a/.appveyor.yml b/.appveyor.yml index ffc40f7..1ee5d9d 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -88,7 +88,8 @@ for: # https://github.com/appveyor/ci/issues/401#issuecomment-301649481 # https://www.appveyor.com/docs/packaging-artifacts/#pushing-artifacts-from-scripts on_finish: +- ps: $ErrorActionPreference = 'SilentlyContinue' - ps: > - $root = Resolve-Path .nox; + $root = Resolve-Path .; [IO.Directory]::GetFiles($root.Path, '*.log', 'AllDirectories') | % { Push-AppveyorArtifact $_ -FileName $_.Substring($root.Path.Length + 1) -DeploymentName to-publish } diff --git a/.idea/dictionaries/pavel.xml b/.idea/dictionaries/pavel.xml index 29464ee..de347ad 100644 --- a/.idea/dictionaries/pavel.xml +++ b/.idea/dictionaries/pavel.xml @@ -61,6 +61,7 @@ deallocated debian decaxta + decel deduplicator deduplicators demultiplexer @@ -81,11 +82,13 @@ dsdlgp dsonar dtype + eangvel edgeitems elif emptor endfor endmacro + envvar ethertype facto ffee @@ -127,6 +130,7 @@ isfinite jmsc kibibyte + kibibytes kirienko koopman levelname @@ -187,6 +191,7 @@ pkgutil popen powershell + pppwm prio prog protip @@ -212,7 +217,9 @@ qwertyuiop raii rankdir + ratiometric rawsource + rbat readlines readthedocs reasm @@ -256,6 +263,8 @@ sitecustomize sitl slcan + smca + smcf sockaddr socketcan socketcanfd @@ -263,6 +272,7 @@ sssss ssssssss stdint + stid strictification subcommand subcommands @@ -288,6 +298,7 @@ tocfile toctree todos + torq tradeoff tsvfc tsvh @@ -296,6 +307,7 @@ uavcan uber udpros + udral ulong uname undisable @@ -323,6 +335,7 @@ worl wpcap xbee + xcoupling xddddddd xfer xxxx diff --git a/README.md b/README.md index 467ea22..0a1f7c0 100644 --- a/README.md +++ b/README.md @@ -3,17 +3,9 @@ OpenCyphal logo -[![Build status](https://ci.appveyor.com/api/projects/status/knl63ojynybi3co6/branch/main?svg=true)](https://ci.appveyor.com/project/Zubax/yakut/branch/main) -[![PyPI - Version](https://img.shields.io/pypi/v/yakut.svg)](https://pypi.org/project/yakut/) -[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) -[![Forum](https://img.shields.io/discourse/users.svg?server=https%3A%2F%2Fforum.opencyphal.org&color=1700b3)](https://forum.opencyphal.org) - -Yakút is a simple cross-platform command-line interface (CLI) tool for diagnostics and debugging of -[Cyphal](https://opencyphal.org) networks. -By virtue of being based on [PyCyphal](https://github.com/OpenCyphal/pycyphal), -Yakut supports all Cyphal transports (UDP, serial, CAN, ...) -and is compatible with all major features of the protocol. -It is designed to be usable with GNU/Linux, Windows, and macOS. +[![PyPI - Version](https://img.shields.io/pypi/v/yakut.svg)](https://pypi.org/project/yakut/) [![Forum](https://img.shields.io/discourse/users.svg?server=https%3A%2F%2Fforum.opencyphal.org&color=1700b3)](https://forum.opencyphal.org) + +Yakút is a simple cross-platform command-line interface (CLI) tool for diagnostics and debugging of [Cyphal](https://opencyphal.org) networks. By virtue of being based on [PyCyphal](https://github.com/OpenCyphal/pycyphal), Yakut supports all Cyphal transports (UDP, serial, CAN, ...) and is compatible with all major features of the protocol. It is designed to be usable with GNU/Linux, Windows, and macOS. yakut monitor @@ -21,30 +13,22 @@ Ask questions and get assistance at [forum.opencyphal.org](https://forum.opencyp ## Installing -First, make sure to [have Python installed](https://docs.python.org/3/using/index.html). -Windows users are recommended to grab the official distribution from Windows Store. +First, make sure to [have Python installed](https://docs.python.org/3/using/index.html). Windows users are recommended to grab the official distribution from Windows Store. Install Yakut: **`pip install yakut`** -By default, Yakut does not support joysticks or MIDI controllers -(this feature is described below in section [Publishing messages](#publishing-messages)). -To enable the support for input devices, install the optional dependency: **`pip install yakut[joystick]`**. -GNU/Linux users will need to also install: [SDL2](https://libsdl.org), -possibly libjack (with headers), possibly libasound2 (with headers) -(if you are using a Debian-based distro, the required packages are: `libsdl2-dev libasound2-dev libjack-dev`). +The default installation comes without support for input devices like joysticks or MIDI controllers. If you need this, enable this option explicitly: **`pip install yakut[joystick]`**. GNU/Linux users will need to also install: [SDL2](https://libsdl.org), possibly libjack (with headers), possibly libasound2 (with headers) (if you are using a Debian-based distro, the required packages are: `libsdl2-dev libasound2-dev libjack-dev`). Afterward do endeavor to read the docs: **`yakut --help`** Check for new versions every now and then: **`pip install --upgrade yakut`** -Installation & configuration screencasts are available for -[Windows](https://forum.opencyphal.org/t/screencast-of-installing-configuring-yakut/1197/2?u=pavel.kirienko), -[GNU/Linux](https://forum.opencyphal.org/t/screencast-of-installing-configuring-yakut/1197/1?u=pavel.kirienko), -and -[macOS](https://www.youtube.com/watch?v=dQw4w9WgXcQ). +Installation & configuration screencasts are available for [Windows](https://forum.opencyphal.org/t/screencast-of-installing-configuring-yakut/1197/2?u=pavel.kirienko), [GNU/Linux](https://forum.opencyphal.org/t/screencast-of-installing-configuring-yakut/1197/1?u=pavel.kirienko), and [macOS](https://www.youtube.com/watch?v=dQw4w9WgXcQ). ### Additional third-party tools +Since Yakut heavily relies on YAML/JSON documents exchanged via stdin/stdout, [**`jq`**](https://stedolan.github.io/jq/) is often needed for any non-trivial usage of the tool, so consider installing it as well. Users of GNU/Linux will likely find it in the default software repositories (`pacman -S jq`, `apt install jq`, etc.). + - Cyphal/CAN on GNU/Linux: [`can-utils`](https://github.com/linux-can/can-utils) - Cyphal/UDP or Cyphal/CAN: [Wireshark](https://www.wireshark.org/) (n.b.: Wireshark might label Cyphal captures as UAVCAN due to rebranding) @@ -52,10 +36,7 @@ and ## Invoking commands Any option can be supplied either as a command-line argument or as an environment variable named like -`YAKUT_[subcommand_]option`. -If both are provided, command-line options take precedence over environment variables. -You can use this feature to configure desired defaults by exporting environment variables from the -rc-file of your shell (for bash/zsh this is `~/.bashrc`/`~/.zshrc`, for PowerShell see `$profile`). +`YAKUT_[subcommand_]option`. If both are provided, command-line options take precedence over environment variables. You can use this feature to configure desired defaults by exporting environment variables from the rc-file of your shell (for bash/zsh this is `~/.bashrc`/`~/.zshrc`, for PowerShell see `$profile`). Options for the main command shall be specified before the subcommand when invoking Yakut: @@ -71,15 +52,13 @@ Yakut may also be invoked via its alias **`y`** as long as this name does not co ## Compiling DSDL -Suppose we have our custom DSDL namespace that we want to use. -First, it needs to be *compiled*: +Suppose we have our custom DSDL namespace that we want to use. First, it needs to be *compiled*: ```bash yakut compile ~/custom_data_types/sirius_cyber_corp ``` -Most of the commands require the standard namespace to be available, -so let's compile it too, along with the regulated namespace: +Most of the commands require the standard namespace to be available, so let's compile it too, along with the regulated namespace: ```bash yakut compile ~/public_regulated_data_types/uavcan ~/public_regulated_data_types/reg @@ -94,26 +73,14 @@ yakut compile \ https://github.com/Zubax/zubax_dsdl/archive/refs/heads/master.zip ``` -Compilation outputs will be stored in the current working directory, but it can be overridden if needed -via `--output` or `YAKUT_COMPILE_OUTPUT`. -Naturally, Yakut needs to know where the outputs are located to use them; -by default it looks in the current directory. -You can specify additional search locations using `--path` or `YAKUT_PATH`. +Compilation outputs will be stored in the current working directory, but it can be overridden if needed via `--output` or `YAKUT_COMPILE_OUTPUT`. Naturally, Yakut needs to know where the outputs are located to use them; by default it looks in the current directory. You can specify additional search locations using `--path` or `YAKUT_PATH`. A question one is likely to ask here is: *Why don't you ship precompiled regulated DSDL together with the tool?* -Indeed, that would be trivial to do, but we avoid that on purpose to emphasize our commitment to -supporting vendor-specific and regulated DSDL at the same level. -In the past we used to give regulated namespaces special treatment, -which caused our users to acquire misconceptions about the purpose of DSDL. -Specifically, there have been forks of the standard namespace extended with vendor-specific types, -which is harmful to the ecosystem. - -Having to manually compile the regulated namespaces is not an issue because it is just a single command to run. -You may opt to keeping compiled namespaces that you use often somewhere in a dedicated directory and put -`YAKUT_PATH=/your/directory` into your shell's rc-file so that you don't have to manually specify -the path when invoking Yakut. -Similarly, you can configure it to use that directory as the default destination for compiled DSDL: +Indeed, that would be trivial to do, but we avoid that on purpose to emphasize our commitment to supporting vendor-specific and regulated DSDL at the same level. In the past we used to give regulated namespaces special treatment, which caused our users to acquire misconceptions about the purpose of DSDL. Specifically, there have been forks of the standard namespace extended with vendor-specific types, which is harmful to the ecosystem. + +Having to manually compile the regulated namespaces is not an issue because it is just a single command to run. You may opt to keeping compiled namespaces that you use often somewhere in a dedicated directory and put +`YAKUT_PATH=/your/directory` into your shell's rc-file so that you don't have to manually specify the path when invoking Yakut. Similarly, you can configure it to use that directory as the default destination for compiled DSDL: ```bash # bash/zsh on GNU/Linux or macOS @@ -132,21 +99,14 @@ knowing that the outputs will be always stored to and read from a fixed place un ## Communicating -Commands that access the network need to know how to do so. -This is typically configured via standard Cyphal registers assigned from environment variables. +Commands that access the network need to know how to do so. This is typically configured via standard Cyphal registers assigned from environment variables. -Cyphal registers are named values that contain various configuration parameters of a Cyphal application/node. -They are extensively described in the [Cyphal Specification](https://opencyphal.org/specification). -When starting a new process, it is possible to pass arbitrary registers via environment variables. +Cyphal registers are named values that contain various configuration parameters of a Cyphal application/node. They are extensively described in the [Cyphal Specification](https://opencyphal.org/specification). When starting a new process, it is possible to pass arbitrary registers via environment variables. -There are certain registers that are looked at by Cyphal nodes to determine how to connect to the network. -Some of them are given below, but the list is not exhaustive. -The full description of supported registers is available in the API documentation for -[`pycyphal.application.make_transport()`](https://pycyphal.readthedocs.io/en/stable/api/pycyphal.application.html#pycyphal.application.make_transport). +There are certain registers that are looked at by Cyphal nodes to determine how to connect to the network. Some of them are given below, but the list is not exhaustive. The full description of supported registers is available in the API documentation for [`pycyphal.application.make_transport()`](https://pycyphal.readthedocs.io/en/stable/api/pycyphal.application.html#pycyphal.application.make_transport) +. -If the available registers define more than one transport configuration, a redundant transport will be initialized. -It is not necessary to assign all of these registers to use a particular transport -because all of them except `uavcan.*.iface` come with defaults. +If the available registers define more than one transport configuration, a redundant transport will be initialized. It is not necessary to assign all of these registers to use a particular transport because all of them except `uavcan.*.iface` come with defaults. | Transport | Register name | Register type | Environment variable name | Semantics | Example environment variable value | |-----------|-----------------------|----------------|---------------------------|------------------------------------------------------------|-------------------------------------| @@ -160,11 +120,8 @@ because all of them except `uavcan.*.iface` come with defaults. ### Protip on environment variables -Defining the required environment variables manually is unergonomic and time-consuming. -A better option is to have relevant configuration that you use often defined in a dedicated file (or several) -that is sourced into the current shell session as necessary -(conceptually this is similar to virtual environments used in Python, etc). -Here is an example for a doubly-redundant CAN bus (assuming sh/bash/zsh here): +Defining the required environment variables manually is unergonomic and time-consuming. A better option is to have relevant configuration that you use often defined in a dedicated file (or several) +that is sourced into the current shell session as necessary (conceptually this is similar to virtual environments used in Python, etc). Here is an example for a doubly-redundant CAN bus (assuming sh/bash/zsh here): ```bash # Common Cyphal register configuration for testing & debugging. @@ -185,46 +142,33 @@ $ yakut monitor # Whatever. ### Subscribing to subjects -Subscribe to subject 33 of type `uavcan.si.unit.angle.Scalar` as shown below; -notice how we specify the subject-ID before the data type name. -If the data type version number(s) are not specified (minor or both), the latest available is chosen automatically. -You will see output if/when there is a publisher on this subject (more on this in the next section). +Subscribe to subject 33 of type `uavcan.si.unit.angle.Scalar` as shown below; notice how we specify the subject-ID before the data type name, and that the short data type name can be given in lowercase for convenience. If the data type version number(s) are not specified (minor or both), the latest available is chosen automatically. You will see output if/when there is a publisher on this subject (more on this in the next section). ```bash $ export UAVCAN__UDP__IFACE=127.63.0.0 -$ yakut sub 33:uavcan.si.unit.angle.Scalar --with-metadata +$ yakut sub 33:uavcan.si.unit.angle.scalar --with-metadata --- 33: - _metadata_: - timestamp: {system: 1608987583.298886, monotonic: 788272.540747} - priority: nominal - transfer_id: 0 - source_node_id: 42 + _meta_: {ts_system: 1651525532.267223, ts_monotonic: 1827406.498846, source_node_id: 112, transfer_id: 2, priority: nominal, dtype: uavcan.si.unit.angle.Scalar.1.0} radian: 2.309999942779541 - --- 33: - _metadata_: - timestamp: {system: 1608987583.298886, monotonic: 788272.540747} - priority: nominal - transfer_id: 1 - source_node_id: 42 + _meta_: {ts_system: 1651525533.274571, ts_monotonic: 1827407.506242, source_node_id: 112, transfer_id: 3, priority: nominal, dtype: uavcan.si.unit.angle.Scalar.1.0} radian: 2.309999942779541 ``` -If more than one subject is specified, -a synchronizer will be used to group messages from multiple subjects into synchronized groups, -which are then printed all at once. -If subjects are not updated in lockstep some or all messages may be dropped. +#### Synchronization + +If more than one subject is specified, the default behavior is to output each received message separately. Often there are synchronous subjects that are updated in lockstep, where it is desirable to group received messages pertaining to the same time point into *synchronized groups*. This can be achieved with options like `--sync...`, which select different [synchronization policies](https://pycyphal.readthedocs.io/en/stable/api/pycyphal.presentation.subscription_synchronizer.html) (see `--help` for technical details). If the synchronized subjects are not updated in lockstep some or all messages may be dropped. subject synchronization -Yakut can determine the data type names automatically if the publisher node(s) support the required -network introspection services. -In the following example only the subject-IDs are provided and the type information is discovered automatically: +#### Data type discovery + +Yakut can determine the data type names automatically if the other node(s) utilizing this subject support the required network introspection services (most nodes do). In the following example only the subject-IDs are provided and the type information is discovered automatically: ```bash -$ y sub 100 110 120 140 150 +$ y sub 100 110 120 140 150 --sync --- 100: heartbeat: @@ -246,78 +190,56 @@ $ y sub 100 110 120 140 150 voltage: {volt: 24.92441749572754} 140: {dc_voltage: 125, dc_current: 0, phase_current_amplitude: 4, velocity: 111, ratiometric_setpoint: 9} 150: - current: [0.0020294189453125, 0.70458984375] + current: [0.0020294189453, 0.7045898] voltage: [0.1893310546875, 1.3359375] ``` -Use `--help` to see additional options (`--redraw` is often useful). - -#### Exporting data to computer algebra systems or spreadsheet processors +#### Exporting data for offline analysis -Here the `reg.udral.physics.dynamics.rotation.PlanarTs` message is formatted using the TSV formatter -with headers prepended. -The resulting data can be imported as-is into Excel, Wolfram Mathematica, etc. +The TSV format option can be used to export data for offline analysis using Pandas, Excel, etc. To see all available options use `yakut --help`. ```bash -yakut --format=TSVH subscribe 142:reg.udral.physics.dynamics.rotation.PlanarTs > rotation_data.tsv +y --tsvh sub 1252 1255 --sync > ~/out.tsv ``` +Pandas import + ### Publishing messages -Publishing two messages synchronously twice (four messages total): +Publishing a message twice (you can use a subscriber as explained earlier to see it on the bus): ```bash export UAVCAN__UDP__IFACE=127.63.0.0 export UAVCAN__NODE__ID=42 -yakut pub -N2 33:uavcan.si.unit.angle.Scalar 2.31 uavcan.diagnostic.Record '{text: "2.31 radian"}' +yakut pub -N2 33:uavcan.si.unit.angle.scalar 2.31 ``` -We did not specify the subject-ID for the second subject, so Yakut defaulted to the fixed subject-ID. Like in the case of subscriber, automatic subject type discovery is also available here. -The above example will publish constant values which is rarely useful. -You can define arbitrary Python expressions that are evaluated by Yakut before every publication. -Such expressions are entered as strings marked with a [YAML tag](https://yaml.org/spec/1.2/spec.html#id2761292) `!$`. -There may be an arbitrary number of such expressions in a YAML document, -and their results may be arbitrary as long as the final structure can initialize the specified message. -The following example will publish a sinewave with frequency 1 Hz, amplitude 10 meters: +The above example will publish constant values which is rarely useful. You can define arbitrary Python expressions that are evaluated by Yakut before every publication. Such expressions are entered as strings marked with a [YAML tag](https://yaml.org/spec/1.2/spec.html#id2761292) `!$`. There may be an arbitrary number of such expressions in a YAML document, and their results may be arbitrary as long as the final structure can initialize the specified message. The following example will publish a sinewave with frequency 1 Hz, amplitude 10 meters: ```bash yakut pub -T 0.01 1234:uavcan.si.unit.length.Scalar '{meter: !$ "sin(t * pi * 2) * 10"}' ``` -Notice that we make use of entities like the variable `t` or the standard function `sin` in the expression. -You will see the full list of available entities if you run `y pub --help`. +Notice that we make use of entities like the variable `t` or the standard function `sin` in the expression. You will see the full list of available entities if you run `y pub --help`. -One particularly important capability of this command is the ability to read data from connected -joysticks or MIDI controllers. -It allows the user to control distributed processes or equipment in real time, simulate sensor feeds, etc. -Function `A(x,y)` returns the normalized value of axis `y` from connected controller `x` -(for full details see `yakut pub --help`); -likewise, there is `B(x,y)` for push buttons and `T(x,y)` for toggle switches. -The next example will publish 3D angular velocity setpoint, thrust setpoint, and the arming switch state, -allowing the user to control these parameters interactively: +One particularly important capability of this command is the ability to read data from connected joysticks or MIDI controllers. It allows the user to control distributed processes or equipment in real time, simulate sensor feeds, etc. Function `A(x,y)` returns the normalized value of axis `y` from connected controller `x` (for full details see `yakut pub --help`); likewise, there is `B(x,y)` for push buttons and `T(x,y)` for toggle switches. The next example will simultaneously publish 3D angular velocity setpoint, thrust setpoint, and the arming switch state, allowing the user to control these parameters interactively: ```bash yakut pub -T 0.1 \ - 5:uavcan.si.unit.angular_velocity.Vector3 '!$ "[A(1,0)*10, A(1,1)*10, (A(1,2)-A(1,5))*5]"' \ - 6:uavcan.si.unit.power.Scalar '!$ A(2,10)*1e3' \ - 7:uavcan.primitive.scalar.Bit '!$ T(1,5)' + 5:uavcan.si.unit.angular_velocity.vector3 '!$ "[A(1,0)*10, A(1,1)*10, (A(1,2)-A(1,5))*5]"' \ + 6:uavcan.si.unit.power.scalar '!$ A(2,10)*1e3' \ + 7:uavcan.primitive.scalar.bit '!$ T(1,5)' ``` To see the published values, either launch a subscriber in a new terminal as `y sub 5 6 7`, or add `--verbose`. Observe that we didn't spell out the field names here (`radian_per_second`, `watt`, `value`) -because it is actually not required; -information on the accepted formats can be found in the documentation in PyCyphal API for -[`pycyphal.dsdl.update_from_builtin()`](https://pycyphal.readthedocs.io/en/stable/api/pycyphal.dsdl.html#pycyphal.dsdl.update_from_builtin). - -The list of connected controllers and how their axes are mapped can be seen using `yakut joystick`, -as shown in the video: - -[![yakut joystick](https://img.youtube.com/vi/YPr98KM1RFM/maxresdefault.jpg)](https://www.youtube.com/watch?v=YPr98KM1RFM) +because positional initialization is also supported; information on the accepted formats can be found in the documentation in PyCyphal API for [`pycyphal.dsdl.update_from_builtin()`](https://pycyphal.readthedocs.io/en/stable/api/pycyphal.dsdl.html#pycyphal.dsdl.update_from_builtin) +. -Here is an example where a MIDI controller is used to interactively change the frequency and amplitude of a sinewave: +The list of connected controllers and how their axes are mapped can be seen using `yakut joystick` (video: ). Here is an example where a MIDI controller is used to interactively change the frequency and amplitude of a sinewave: [![yakut publish](https://img.youtube.com/vi/DSsI882ZYh0/maxresdefault.jpg)](https://www.youtube.com/watch?v=DSsI882ZYh0) @@ -356,17 +278,15 @@ $ yakut call 42 123:sirius_cyber_corp.PerformLinearLeastSquaresFit 'points: [{x: ``` You might notice that the verbose initialization form used in this example is hard to type: -`points: [{x: 10, y: 1}, {x: 20, y: 2}]`. -Instead, you can use positional initialization for convenience: `[[10, 1], [20, 2]]`. +`points: [{x: 10, y: 1}, {x: 20, y: 2}]`. Instead, you can use positional initialization for convenience: `[[10, 1], [20, 2]]`. The data type name can also be given in all-lowercase for ease of typing. Automatic data type discovery is also available here but the service has to be referred by name, not port-ID: ```bash -y q 42 least_squares '[[10, 1], [20, 2]]' # "y q" is shorthand for "yakut call" (see --help for more) +y q 42 least_squares '[[10, 1], [20, 2]]' # "y q" is shorthand for "yakut call" ``` -You can still override the type if you want to use a different one -(e.g., if the remote node names the type of this service differently): +You can still override the type if you want to use a different one (e.g., if the remote node names the type of this service differently): ```bash y q 42 least_squares:my_namespace.MySpecialType '[[10, 1], [20, 2]]' @@ -374,11 +294,7 @@ y q 42 least_squares:my_namespace.MySpecialType '[[10, 1], [20, 2]]' ## Monitoring the network -The command `yakut monitor` can be used to display *all* activity on the network in a compact representation. -It tracks online nodes and maintains real-time statistics on all transfers exchanged between each node -on the network. -It may also be able to detect some common network configuration issues like zombie nodes -(nodes that do not publish `uavcan.node.Heartbeat`). +The command `yakut monitor` can be used to display *all* activity on the network in a compact representation. It tracks online nodes and maintains real-time statistics on all transfers exchanged between each node on the network. It may also be able to detect some common network configuration issues like zombie nodes (nodes that do not publish `uavcan.node.Heartbeat`). Read `yakut monitor --help` for details. @@ -391,17 +307,310 @@ $ y mon # "y mon" is shorthand for "yaku yakut monitor -The monitor can be an anonymous node or it can be given a node-ID of its own. -In the latter case it will actively query other nodes using the standard introspection services. +The monitor can be an anonymous node or it can be given a node-ID of its own. In the latter case it will actively query other nodes using the standard introspection services. + +Some transports, Cyphal/UDP in particular, require elevated privileges to run this tool due to the security implications of low-level packet capture. + +## Working with registers + +Cyphal registers are typed named values stored locally per-node. They are used to establish node configuration, sample diagnostics, read calibration parameters, and more. The network service is defined in the standard namespace `uavcan.register`. + +Read the list of register names available on a node (we are using short names here for brevity; long names are also available but are inconvenient for interactive use): + +```shell +$ y rl 125 # rl is short for register-list +[drv.acm.std.pppwm_threshold, drv.acm.std.xcoupling_inductance_compensation, drv.obs.ekf.q_eangvel, ...] +``` + +You can specify a set of node-IDs like `x-y` which denotes interval `[x, y)`, or a simple list `x,y,z`, a list with exclusion `x-y!w,z` which means `[x, y) except w and z`, and so on. This is convenient when you are working with a large network interactively. One important distinction is that a single node-ID like `125` and a set of node-IDs of one element like `125,` are treated differently (kind of like tuples of one element in some programming languages): + +```shell +$ y rl 125, # Mind the comma! It changes the output to be a mapping of one element. +125: [drv.acm.std.pppwm_threshold, drv.acm.std.xcoupling_inductance_compensation, drv.obs.ekf.q_eangvel, ...] + +$ y rl 122-126 # Produce list of register names per node +122: [drv.acm.std.pppwm_threshold, drv.acm.std.xcoupling_inductance_compensation, drv.obs.ekf.q_eangvel, ...] +123: [drv.acm.std.pppwm_threshold, drv.acm.std.xcoupling_inductance_compensation, drv.obs.ekf.q_eangvel, ...] +124: [vsi.pins, vsi.pwm_dead_time, vsi.pwm_freq, vsi.shortest_time_in_disabled_state, ...] +125: [vsi.pins, vsi.pwm_dead_time, vsi.pwm_freq, vsi.shortest_time_in_disabled_state, ...] +``` + +The distinction between grouped and flat output becomes relevant when this command is combined with others, like `register-batch` (short `rb`): + +```shell +$ y rl 122-126 | y rb # Read all registers from nodes 122,123,124,125 +122: + drv.acm.std.pppwm_threshold: 0.8999999761581421 + drv.acm.std.xcoupling_inductance_compensation: false + drv.obs.ekf.q_eangvel: 3000000.0 + # output truncated for clarity... +123: + drv.acm.std.pppwm_threshold: 0.8999999761581421 + drv.acm.std.xcoupling_inductance_compensation: false + drv.obs.ekf.q_eangvel: 3000000.0 + # ... +124: + vsi.pins: [0, 0, 3, 1, 1] + vsi.pwm_dead_time: 0.0 + vsi.pwm_freq: 47009.6640625 + vsi.shortest_time_in_disabled_state: 1.9999999494757503e-05 + # ... +125: + vsi.pins: [0, 0, 3, 1, 1] + vsi.pwm_dead_time: 0.0 + vsi.pwm_freq: 47009.6640625 + vsi.shortest_time_in_disabled_state: 1.9999999494757503e-05 + # ... +``` + +In the above invocation the register-batch (`rb`) command will obtain the list of register names per node from stdin. It is also possible to provide a flat list of names to sample the same registers from multiple nodes, but in this case the register-batch command needs to be given a list of nodes explicitly, since it is no longer contained in the directive read from stdin: + +```shell +$ y rl 125 | jq 'map(select(test("sys.+")))' > sys_register_names.json # using jq to filter the list + +$ cat sys_register_names.json +[ + "sys.debug", + "sys.info.mem", + "sys.info.time" +] + +$ cat sys_register_names.json | y rb 122-126 +122: + sys.debug: true + sys.info.mem: [4096, 160, 16384, 13536, 100704, 40064, 44160, 888, 0] + sys.info.time: [6.333333431030042e-07, 5.0099999498343095e-05] +123: + sys.debug: true + sys.info.mem: [4096, 160, 16384, 13536, 100704, 40064, 44160, 888, 0] + sys.info.time: [6.333333431030042e-07, 4.9966667575063184e-05] +124: + sys.debug: true + sys.info.mem: [4096, 160, 16384, 13536, 100704, 40064, 44160, 888, 0] + sys.info.time: [6.333333431030042e-07, 5.008888911106624e-05] +125: + sys.debug: true + sys.info.mem: [4096, 160, 16384, 13536, 100704, 40832, 44928, 888, 0] + sys.info.time: [6.333333431030042e-07, 5.008888911106624e-05] +``` + +Notice that unless the output format is given explicitly (as `--json`, `--yaml`, etc.), Yakut defaults to YAML if stdout is connected to the terminal for the benefit of the human, otherwise (in case of piping/redirection) the default is JSON which enables compatibility with `jq` and similar tools. + +Option `--only=mp` can be given to `register-batch` (`rb`) to show only mutable-persistent registers; see `--help` for more info. This is useful when you want to store the configuration parameters of a given node (or several). + +To interactively read/write a single register on one or several nodes use `yakut register`, or simply `y r`: + +```shell +# Read register: +$ y r 125 sys.debug +true + +# Write register: +$ y r 125 sys.debug 0 +false # New value returned by the node after assignment + +# Read from several nodes; output grouped by node-ID: +$ y r 122-126 m.inductance_dq +122: [1.2549953680718318e-05, 1.2549953680718318e-05] +123: [1.2549953680718318e-05, 1.2549953680718318e-05] +124: [1.2549953680718318e-05, 1.2549953680718318e-05] +125: [1.2549953680718318e-05, 1.2549953680718318e-05] + +# Change register on several nodes: +$ y r 122-126 m.inductance_dq 13e-6 12e-6 +122: [1.3e-05, 1.2e-05] +123: [1.3e-05, 1.2e-05] +124: [1.3e-05, 1.2e-05] +125: [1.3e-05, 1.2e-05] + +# Show type and force node-ID grouping with comma: +$ y r 125, m.inductance_dq --detailed +125: + real32: + value: [1.2999999853491317e-05, 1.2000000424450263e-05] + +# Show even more information: +$ y r 125, m.inductance_dq --detailed --detailed +125: + real32: + value: [1.2999999853491317e-05, 1.2000000424450263e-05] + _meta_: {mutable: true, persistent: true} + +# If there is no such register, we get a null (empty): +$ y r 125 no.such.register +null +``` + +The accepted value representations are specified in the standard register service documentation `uavcan.register.Access` in the section on environment variables. + +Perhaps one of the most important use cases for the register tools is to save the node configuration into a YAML file, then edit that file manually to keep only relevant parameters, and use that later to configure the entire network in one command. + +## Execute standard and custom commands + +The standard RPC-service `uavcan.node.ExecuteCommand` allows one to perform certain standard and vendor-specific activities on the node. Yakut provides `execute-command`, or simply `cmd`, as a more convenient alternative to calling `yakut call uavcan.node.ExecuteCommand` manually: + +```shell +# "emergency" is an abbreviation of "COMMAND_EMERGENCY_STOP", the code is 65531. +$ y cmd 125 emergency +{status: 0} + +# Restart nodes 122,123,124,125; instead of "restart" one could say 65535. +$ y cmd 122-126 restart +122: {status: 0} +123: {status: 0} +124: {status: 0} +125: {status: 0} + +# Reset 128 nodes to factory defaults concurrently, do not wait/check responses. +$ y cmd -e 0-128 factory_reset +# ... + +# Install the same software image on multiple nodes +# (a file server would be required though; there is a separate command for that). +$ y cmd 122-126 begin_software_update "/path/to/firmware.app.bin" +# ... + +# Execute a vendor-specific command 42 with some argument. +$ y cmd 122-126 42 'some command argument' +# ... +``` + +## Node configuration example + +Suppose we have a bunch of similar nodes that we want to configure. First we need to dump the mutable-persistent registers into a YAML file: + +```shell +# This option ↓↓↓↓ is needed to force YAML output. It would default to JSON because the output is redirected. +y rl 125, | y --yaml rb --only=mp > cyphal_config.yaml +# The filter option here ↑↑↑↑↑↑↑ keeps only mutable persistent registers in the output. +``` + +Then edit that file manually to remove irrelevant parameters and copy those that should be different per node. Suppose that when we are done with editing we end up with something like this (notice how we use the YAML dict merge syntax to avoid repetition): + +```yaml +122: &prototype_esc + m.pole_count: 24 + m.current_max: 50 + m.resistance: 0.03427 + m.inductance_dq: [ 12.55e-06, 12.55e-06 ] + m.flux_linkage: 0.001725 + m.current_ramp: 1000.0 + m.voltage_ramp: 20.0 + m.velocity_accel_decel: [ 7000.0, 5000.0 ] + m.fw_voltage_boost: 1.0 + + mns.pub_interval_min: 0.005 + mns.ratiometric_setpoint_min: 0.03 + mns.ratiometric_to_absolute_mul: 0.0 + mns.setpoint_index: 0 + + uavcan.can.bitrate: [ 1000000, 0 ] + uavcan.can.count: 1 + + uavcan.pub.dynamics.id: 1220 + uavcan.pub.feedback.id: 1221 + uavcan.pub.power.id: 1222 + uavcan.pub.compact.id: 0xFFFF # disabled + uavcan.pub.dq.id: 0xFFFF # disabled + uavcan.pub.status.id: 0xFFFF # disabled + + uavcan.sub.readiness.id: 10 + uavcan.sub.setpoint_dyn.id: 0xFFFF # disabled + uavcan.sub.setpoint_r_torq.id: 0xFFFF # disabled + uavcan.sub.setpoint_r_torq_u9.id: 0xFFFF # disabled + uavcan.sub.setpoint_r_volt.id: 14 + uavcan.sub.setpoint_r_volt_u9.id: 0xFFFF # disabled + uavcan.sub.setpoint_vel.id: 0xFFFF # disabled + + uavcan.srv.low_level_io.id: 0xFFFF # disabled + +123: # This item is for node-ID 123, and so on. + # The construct below is the YAML dict merge statement. + # It makes this entry inherit all parameters from the above + # but the inherited keys can be overridden. + <<: *prototype_esc + uavcan.pub.dynamics.id: 1230 # Override this subject. + uavcan.pub.feedback.id: 1231 # and so on... + uavcan.pub.power.id: 1232 + mns.setpoint_index: 1 + +124: + <<: *prototype_esc + uavcan.pub.dynamics.id: 1240 + uavcan.pub.feedback.id: 1241 + uavcan.pub.power.id: 1242 + mns.setpoint_index: 2 + +125: + <<: *prototype_esc + uavcan.pub.dynamics.id: 1250 + uavcan.pub.feedback.id: 1251 + uavcan.pub.power.id: 1252 + mns.setpoint_index: 3 +``` + +The above is a valid register file that is both human-friendly and can be understood by `yakut register-batch`. Then to deploy the configuration to the network we need to do simply: + +```shell +cat cyphal_config.yaml | y rb +``` + +Alternatively (same result, different syntax; this option may be more convenient for Windows users): + +```shell +y rb --file=cyphal_config.yaml +``` + +When the configuration is deployed, we will probably need to restart the nodes for the changes to take effect: + +```shell +$ y cmd 122-126 restart -e +Responses not checked as requested +122: {status: 0} +123: {status: 0} +124: {status: 0} +125: {status: 0} +``` + +Now in this example we are dealing with motor controllers, so let's spin one motor (controlled by joystick) for testing purposes (the subject mapping is defined in the YAML file we just wrote): + +```shell +# Publish on two subjects: 10 with auto-discovered type; 14 with explicit type. +y pub -T 0.01 \ + 10 '!$ 3*T(1,23)' \ + 14:reg.udral.service.actuator.common.sp.Vector6.0.1 '!$ "[0, 0, 0, A(1,3)-A(1,4), 0, 0]" +``` + +Subscribe to telemetry from one of the nodes: + +y sub 1250 1251 1252 --sync --redraw + +Fetch the port-ID and type information directly from the running nodes: + +```shell +# You can pipe the last output through "jq" to get a nicely formatted and colored JSON. +$ y rl 122-126 | jq 'map_values([.[] | select(test("uavcan.+(id|type)"))])' | y rb +122: {uavcan.node.id: 122, uavcan.pub.compact.id: 65535, uavcan.pub.compact.type: zubax.telega.CompactFeedback.0.1, uavcan.pub.dq.id: 65535, uavcan.pub.dq.type: zubax.telega.DQ.0.1, uavcan.pub.dynamics.id: 1220, uavcan.pub.dynamics.type: reg.udral.physics.dynamics.rotation.PlanarTs.0.1, uavcan.pub.feedback.id: 1221, uavcan.pub.feedback.type: reg.udral.service.actuator.common.Feedback.0.1, uavcan.pub.power.id: 1222, uavcan.pub.power.type: reg.udral.physics.electricity.PowerTs.0.1, uavcan.pub.status.id: 65535, uavcan.pub.status.type: reg.udral.service.actuator.common.Status.0.1, uavcan.srv.low_level_io.id: 65535, uavcan.srv.low_level_io.type: zubax.low_level_io.Access.0.1, uavcan.sub.readiness.id: 10, uavcan.sub.readiness.type: reg.udral.service.common.Readiness.0.1, uavcan.sub.setpoint_dyn.id: 65535, uavcan.sub.setpoint_dyn.type: reg.udral.physics.dynamics.rotation.Planar.0.1, uavcan.sub.setpoint_r_torq.id: 65535, uavcan.sub.setpoint_r_torq.type: reg.udral.service.actuator.common.sp.Vector31.0.1, uavcan.sub.setpoint_r_torq_u9.id: 65535, uavcan.sub.setpoint_r_torq_u9.type: zubax.telega.setpoint.Raw9x56.0.1, uavcan.sub.setpoint_r_volt.id: 14, uavcan.sub.setpoint_r_volt.type: reg.udral.service.actuator.common.sp.Vector31.0.1, uavcan.sub.setpoint_r_volt_u9.id: 65535, uavcan.sub.setpoint_r_volt_u9.type: zubax.telega.setpoint.Raw9x56.0.1, uavcan.sub.setpoint_vel.id: 65535, uavcan.sub.setpoint_vel.type: reg.udral.service.actuator.common.sp.Vector31.0.1} +123: {uavcan.node.id: 123, uavcan.pub.compact.id: 65535, uavcan.pub.compact.type: zubax.telega.CompactFeedback.0.1, uavcan.pub.dq.id: 65535, uavcan.pub.dq.type: zubax.telega.DQ.0.1, uavcan.pub.dynamics.id: 1230, uavcan.pub.dynamics.type: reg.udral.physics.dynamics.rotation.PlanarTs.0.1, uavcan.pub.feedback.id: 1231, uavcan.pub.feedback.type: reg.udral.service.actuator.common.Feedback.0.1, uavcan.pub.power.id: 1232, uavcan.pub.power.type: reg.udral.physics.electricity.PowerTs.0.1, uavcan.pub.status.id: 65535, uavcan.pub.status.type: reg.udral.service.actuator.common.Status.0.1, uavcan.srv.low_level_io.id: 65535, uavcan.srv.low_level_io.type: zubax.low_level_io.Access.0.1, uavcan.sub.readiness.id: 10, uavcan.sub.readiness.type: reg.udral.service.common.Readiness.0.1, uavcan.sub.setpoint_dyn.id: 65535, uavcan.sub.setpoint_dyn.type: reg.udral.physics.dynamics.rotation.Planar.0.1, uavcan.sub.setpoint_r_torq.id: 65535, uavcan.sub.setpoint_r_torq.type: reg.udral.service.actuator.common.sp.Vector31.0.1, uavcan.sub.setpoint_r_torq_u9.id: 65535, uavcan.sub.setpoint_r_torq_u9.type: zubax.telega.setpoint.Raw9x56.0.1, uavcan.sub.setpoint_r_volt.id: 14, uavcan.sub.setpoint_r_volt.type: reg.udral.service.actuator.common.sp.Vector31.0.1, uavcan.sub.setpoint_r_volt_u9.id: 65535, uavcan.sub.setpoint_r_volt_u9.type: zubax.telega.setpoint.Raw9x56.0.1, uavcan.sub.setpoint_vel.id: 65535, uavcan.sub.setpoint_vel.type: reg.udral.service.actuator.common.sp.Vector31.0.1} +124: {uavcan.node.id: 124, uavcan.pub.compact.id: 65535, uavcan.pub.compact.type: zubax.telega.CompactFeedback.0.1, uavcan.pub.dq.id: 65535, uavcan.pub.dq.type: zubax.telega.DQ.0.1, uavcan.pub.dynamics.id: 1240, uavcan.pub.dynamics.type: reg.udral.physics.dynamics.rotation.PlanarTs.0.1, uavcan.pub.feedback.id: 1241, uavcan.pub.feedback.type: reg.udral.service.actuator.common.Feedback.0.1, uavcan.pub.power.id: 1242, uavcan.pub.power.type: reg.udral.physics.electricity.PowerTs.0.1, uavcan.pub.status.id: 65535, uavcan.pub.status.type: reg.udral.service.actuator.common.Status.0.1, uavcan.srv.low_level_io.id: 65535, uavcan.srv.low_level_io.type: zubax.low_level_io.Access.0.1, uavcan.sub.readiness.id: 10, uavcan.sub.readiness.type: reg.udral.service.common.Readiness.0.1, uavcan.sub.setpoint_dyn.id: 65535, uavcan.sub.setpoint_dyn.type: reg.udral.physics.dynamics.rotation.Planar.0.1, uavcan.sub.setpoint_r_torq.id: 65535, uavcan.sub.setpoint_r_torq.type: reg.udral.service.actuator.common.sp.Vector31.0.1, uavcan.sub.setpoint_r_torq_u9.id: 65535, uavcan.sub.setpoint_r_torq_u9.type: zubax.telega.setpoint.Raw9x56.0.1, uavcan.sub.setpoint_r_volt.id: 14, uavcan.sub.setpoint_r_volt.type: reg.udral.service.actuator.common.sp.Vector31.0.1, uavcan.sub.setpoint_r_volt_u9.id: 65535, uavcan.sub.setpoint_r_volt_u9.type: zubax.telega.setpoint.Raw9x56.0.1, uavcan.sub.setpoint_vel.id: 65535, uavcan.sub.setpoint_vel.type: reg.udral.service.actuator.common.sp.Vector31.0.1} +125: {uavcan.node.id: 125, uavcan.pub.compact.id: 65535, uavcan.pub.compact.type: zubax.telega.CompactFeedback.0.1, uavcan.pub.dq.id: 65535, uavcan.pub.dq.type: zubax.telega.DQ.0.1, uavcan.pub.dynamics.id: 1250, uavcan.pub.dynamics.type: reg.udral.physics.dynamics.rotation.PlanarTs.0.1, uavcan.pub.feedback.id: 1251, uavcan.pub.feedback.type: reg.udral.service.actuator.common.Feedback.0.1, uavcan.pub.power.id: 1252, uavcan.pub.power.type: reg.udral.physics.electricity.PowerTs.0.1, uavcan.pub.status.id: 65535, uavcan.pub.status.type: reg.udral.service.actuator.common.Status.0.1, uavcan.srv.low_level_io.id: 65535, uavcan.srv.low_level_io.type: zubax.low_level_io.Access.0.1, uavcan.sub.readiness.id: 10, uavcan.sub.readiness.type: reg.udral.service.common.Readiness.0.1, uavcan.sub.setpoint_dyn.id: 65535, uavcan.sub.setpoint_dyn.type: reg.udral.physics.dynamics.rotation.Planar.0.1, uavcan.sub.setpoint_r_torq.id: 65535, uavcan.sub.setpoint_r_torq.type: reg.udral.service.actuator.common.sp.Vector31.0.1, uavcan.sub.setpoint_r_torq_u9.id: 65535, uavcan.sub.setpoint_r_torq_u9.type: zubax.telega.setpoint.Raw9x56.0.1, uavcan.sub.setpoint_r_volt.id: 14, uavcan.sub.setpoint_r_volt.type: reg.udral.service.actuator.common.sp.Vector31.0.1, uavcan.sub.setpoint_r_volt_u9.id: 65535, uavcan.sub.setpoint_r_volt_u9.type: zubax.telega.setpoint.Raw9x56.0.1, uavcan.sub.setpoint_vel.id: 65535, uavcan.sub.setpoint_vel.type: reg.udral.service.actuator.common.sp.Vector31.0.1} +``` + +The most important diagnostic tool is `yakut monitor`. If you run it you should see the state of the entire network at a glance: + +Yakut monitor on a live network with one motor spinning + +You can see the data published by our publisher command to subjects 10 and 14 in the upper-left corner of the matrix, and further to the right you can see that several other nodes consume data from these subjects (specifically they are our motor controllers and the monitor itself). Then there is the staggered diagonal structure showing each of the motor controllers publishing telemetry on its own set of subjects. The one that is currently running the motor publishes at 100 Hz, others (that are idle) are limited to 1 Hz. + +The service traffic is shown to be zero which is usually the normal state for any operational network. If you run a command that needs network discovery you will see some brief activity there. + +The summary rows/columns show the total traffic in transfers per second and bytes per second per subject and per node. The totals shown at the very bottom-right corner are the total network utilization estimated at the application layer (in this specific example the network carries 516 transfers per second, 6 kibibytes of application-layer payload per second). -Some transports, Cyphal/UDP in particular, require elevated privileges to run this tool due to the security -implications of low-level packet capture. +There are some transport-layer errors reported by the tool that are due to the suboptimal wiring configuration. ## Updating node software -The file server command can be used to serve files, -run a plug-and-play node-ID allocator (some embedded bootloader implementations require that), -and automatically send software update requests `uavcan.node.ExecuteCommand` to nodes whose software is old. +The file server command can be used to serve files, run a plug-and-play node-ID allocator (some embedded bootloader implementations require that), and automatically send software update requests `uavcan.node.ExecuteCommand` to nodes whose software is old. To demonstrate this capability, suppose that the network contains the following nodes: @@ -409,13 +618,7 @@ To demonstrate this capability, suppose that the network contains the following - nodes 3, 4 named `com.example.bar`, hardware v4.2, software v3.4 - node 5 named `com.example.baz` -Software updates are distributed as atomic package files. -In case of embedded systems, the package is usually just the firmware image, -possibly compressed or amended with some metadata. -For the file server this is irrelevant since it never looks inside the files it serves. -However, the name is relevant as it shall follow a particular pattern to make the server recognize -the file as a software package. -The full specification is given in the command help: `yakut file-server --help`. +Software updates are distributed as atomic package files. In case of embedded systems, the package is usually just the firmware image, possibly compressed or amended with some metadata. For the file server this is irrelevant since it never looks inside the files it serves. However, the name is relevant as it shall follow a particular pattern to make the server recognize the file as a software package. The full specification is given in the command help: `yakut file-server --help`. Suppose that we have the following packages that we need to deploy: @@ -431,9 +634,7 @@ com.example.bar-4-3.3.app.pkg # Hardware v4.x com.example.bar-5.6-3.5.app.bin # Hardware v5.6 only ``` -The server rescans its root directory whenever a new node is found online, -meaning that packages can be added/removed at runtime and the server will pick up the changes on the fly. -Launch the server: +The server rescans its root directory whenever a new node is found online, meaning that packages can be added/removed at runtime and the server will pick up the changes on the fly. Launch the server: ```shell $ export UAVCAN__UDP__IFACE=127.63.0.0 @@ -441,23 +642,14 @@ $ export UAVCAN__NODE__ID=42 $ yakut file-server --plug-and-play=allocation_table.db --update-software ``` -If there are any nodes online (or if they join the network later), -the server will check the version of each by sending `uavcan.node.GetInfo`, -and if a newer package is available locally, it will request the node to install it -by sending `uavcan.node.ExecuteCommand`. +If there are any nodes online (or if they join the network later), the server will check the version of each by sending `uavcan.node.GetInfo`, and if a newer package is available locally, it will request the node to install it by sending `uavcan.node.ExecuteCommand`. In this specific case, the following will happen: - Nodes 1 and 2 will be updated to v1.1. -- Nodes 3 and 4 will not be updated because the newer package v3.5 is incompatible with hardware v4.2, - and the compatible version v3.3 is too old. +- Nodes 3 and 4 will not be updated because the newer package v3.5 is incompatible with hardware v4.2, and the compatible version v3.3 is too old. - Node 5 will not be updated because there are no suitable packages. Add `--verbose` to see how exactly the decisions are made. -This command can be used to implement **automatic network-wide configuration management**. -Start the server and leave it running. -Store all relevant packages into its root directory. -When a node is connected or restarted, the server will automatically compare the version of its software -against the local files and perform an update if necessary. -Therefore, the entire network will be kept up-to-date without manual intervention. +This command can be used to implement **automatic network-wide configuration management**. Start the server and leave it running. Store all relevant packages into its root directory. When a node is connected or restarted, the server will automatically compare the version of its software against the local files and perform an update if necessary. Therefore, the entire network will be kept up-to-date without manual intervention. diff --git a/docs/jupyter.png b/docs/jupyter.png new file mode 100644 index 0000000..a2b8b0c Binary files /dev/null and b/docs/jupyter.png differ diff --git a/docs/monitor_esc_spinning.png b/docs/monitor_esc_spinning.png new file mode 100644 index 0000000..5bedfb8 Binary files /dev/null and b/docs/monitor_esc_spinning.png differ diff --git a/docs/subscribe.gif b/docs/subscribe.gif new file mode 100644 index 0000000..6445640 Binary files /dev/null and b/docs/subscribe.gif differ diff --git a/setup.cfg b/setup.cfg index 6560b82..4344434 100644 --- a/setup.cfg +++ b/setup.cfg @@ -185,7 +185,9 @@ disable= too-many-statements, too-many-instance-attributes, eval-used, - unspecified-encoding + unspecified-encoding, + not-callable, + unbalanced-tuple-unpacking [pylint.REPORTS] output-format=colorized diff --git a/tests/cmd/call.py b/tests/cmd/call.py index 26f10cf..68728c0 100644 --- a/tests/cmd/call.py +++ b/tests/cmd/call.py @@ -64,10 +64,10 @@ async def handle_request( # Invoke the service without discovery and then run the server for a few seconds to let it process the request. proc = Subprocess.cli( # Windows compat: -v blocks stderr pipe on Windows. - "--format=json", + "-j", "call", "22", - "222:sirius_cyber_corp.PerformLinearLeastSquaresFit", + "222:sirius_cyber_corp.performlinearleastsquaresfit", "points: [{x: 10, y: 1}, {x: 20, y: 2}]", "--priority=SLOW", "--with-metadata", @@ -85,18 +85,18 @@ async def handle_request( # Parse the output and validate it. parsed = json.loads(stdout) print("PARSED RESPONSE:", parsed) - assert parsed["222"]["_metadata_"]["priority"] == "slow" - assert parsed["222"]["_metadata_"]["source_node_id"] == 22 + assert parsed["222"]["_meta_"]["priority"] == "slow" + assert parsed["222"]["_meta_"]["source_node_id"] == 22 assert parsed["222"]["slope"] == pytest.approx(0.1) assert parsed["222"]["y_intercept"] == pytest.approx(0.0) # Invoke the service with ID discovery and static type. last_metadata = None proc = Subprocess.cli( # Windows compat: -v blocks stderr pipe on Windows. - "--format=json", + "-j", "call", "22", - "least_squares:sirius_cyber_corp.PerformLinearLeastSquaresFit", + "least_squares:sirius_cyber_corp.PERFORMLINEARLEASTSQUARESFIT", "points: [{x: 0, y: 0}, {x: 10, y: 3}]", "--priority=FAST", "--with-metadata", @@ -115,15 +115,15 @@ async def handle_request( # Parse the output and validate it. parsed = json.loads(stdout) print("PARSED RESPONSE:", parsed) - assert parsed["222"]["_metadata_"]["priority"] == "fast" - assert parsed["222"]["_metadata_"]["source_node_id"] == 22 + assert parsed["222"]["_meta_"]["priority"] == "fast" + assert parsed["222"]["_meta_"]["source_node_id"] == 22 assert parsed["222"]["slope"] == pytest.approx(0.3) assert parsed["222"]["y_intercept"] == pytest.approx(0.0) # Invoke the service with full discovery. last_metadata = None proc = Subprocess.cli( # Windows compat: -v blocks stderr pipe on Windows. - "--format=json", + "-j", "call", "22", "least_squares", # Type not specified -- discovered. @@ -144,8 +144,8 @@ async def handle_request( # Parse the output and validate it. parsed = json.loads(stdout) print("PARSED RESPONSE:", parsed) - assert parsed["222"]["_metadata_"]["priority"] == "nominal" - assert parsed["222"]["_metadata_"]["source_node_id"] == 22 + assert parsed["222"]["_meta_"]["priority"] == "nominal" + assert parsed["222"]["_meta_"]["source_node_id"] == 22 assert parsed["222"]["slope"] == pytest.approx(0.4) assert parsed["222"]["y_intercept"] == pytest.approx(0.0) @@ -205,7 +205,7 @@ async def _unittest_call_fixed(transport_factory: TransportFactory, compiled_dsd # Invoke a fixed port-ID service. proc = Subprocess.cli( # Windows compat: -v blocks stderr pipe on Windows. - "--format=json", + "-j", "call", "22", "uavcan.node.GetInfo", diff --git a/tests/cmd/execute_command.py b/tests/cmd/execute_command.py new file mode 100644 index 0000000..df0d7d2 --- /dev/null +++ b/tests/cmd/execute_command.py @@ -0,0 +1,182 @@ +# Copyright (c) 2022 OpenCyphal +# This software is distributed under the terms of the MIT License. +# Author: Pavel Kirienko + +from __future__ import annotations +import asyncio +from typing import Any, AsyncIterable, Callable, Awaitable +import json +import concurrent.futures +import pytest +import pycyphal +from tests.dsdl import OUTPUT_DIR +from tests.transport import TransportFactory +from tests.subprocess import execute_cli +from yakut.util import EXIT_CODE_UNSUCCESSFUL + + +class Remote: + def __init__(self, name: str, env: dict[str, str]) -> None: + from pycyphal.application import make_registry, make_node, NodeInfo + from uavcan.node import ExecuteCommand_1 + + self._node = make_node( + NodeInfo(name=name), + make_registry(environment_variables=env), + ) + self.last_request: ExecuteCommand_1.Request | None = None + self.next_response: ExecuteCommand_1.Response | None = None + + async def serve_execute_command( + req: ExecuteCommand_1.Request, + _meta: pycyphal.presentation.ServiceRequestMetadata, + ) -> ExecuteCommand_1.Response | None: + # print(self._node, req, _meta, self.next_response, sep="\n\t") + self.last_request = req + return self.next_response + + self._srv = self._node.get_server(ExecuteCommand_1) + self._srv.serve_in_background(serve_execute_command) + self._node.start() + + def close(self) -> None: + self._srv.close() + self._node.close() + + +Runner = Callable[..., Awaitable[Any]] + + +@pytest.fixture +async def _context( + compiled_dsdl: Any, + transport_factory: TransportFactory, +) -> AsyncIterable[tuple[Runner, tuple[Remote, Remote]]]: + asyncio.get_running_loop().slow_callback_duration = 10.0 + _ = compiled_dsdl + remote_nodes = ( + Remote(f"remote_10", env=transport_factory(10).environment), + Remote(f"remote_11", env=transport_factory(11).environment), + ) + background_executor = concurrent.futures.ThreadPoolExecutor() + + async def run(*args: str) -> tuple[int, Any]: + def call() -> tuple[int, Any]: + status, stdout, _stderr = execute_cli( + "cmd", + *args, + environment_variables={ + **transport_factory(100).environment, + "YAKUT_PATH": str(OUTPUT_DIR), + }, + timeout=10, + ensure_success=False, + ) + return status, json.loads(stdout) if stdout else None + + return await asyncio.get_running_loop().run_in_executor(background_executor, call) + + yield run, remote_nodes + for rn in remote_nodes: + rn.close() + await asyncio.sleep(1.0) + + +@pytest.mark.asyncio +async def _unittest_basic(_context: tuple[Runner, tuple[Remote, Remote]]) -> None: + from uavcan.node import ExecuteCommand_1 + + run, (remote_10, remote_11) = _context + + # SUCCESS + remote_10.next_response = ExecuteCommand_1.Response(status=0) + remote_11.next_response = ExecuteCommand_1.Response(status=0) + assert await run("10-12", "restart", "--timeout=3") == ( + 0, + { + "10": {"status": 0}, + "11": {"status": 0}, + }, + ) + assert await run("10-12", "111", "COMMAND ARGUMENT", "--timeout=3") == ( + 0, + { + "10": {"status": 0}, + "11": {"status": 0}, + }, + ) + assert ( + remote_10.last_request + and remote_10.last_request.command == 111 + and remote_10.last_request.parameter.tobytes().decode() == "COMMAND ARGUMENT" + ) + assert ( + remote_11.last_request + and remote_11.last_request.command == 111 + and remote_11.last_request.parameter.tobytes().decode() == "COMMAND ARGUMENT" + ) + + # REMOTE ERROR; PROPAGATED AND IGNORED + remote_10.next_response = ExecuteCommand_1.Response(status=100) + remote_11.next_response = ExecuteCommand_1.Response(status=200) + assert await run("10-12", "restart", "--timeout=3") == ( + EXIT_CODE_UNSUCCESSFUL, + { + "10": {"status": 100}, + "11": {"status": 200}, + }, + ) + assert await run("10-12", "123", "--expect=100,200", "--timeout=3") == ( + 0, + { + "10": {"status": 100}, + "11": {"status": 200}, + }, + ) + assert remote_10.last_request and remote_10.last_request.command == 123 + assert remote_11.last_request and remote_11.last_request.command == 123 + + # ONE TIMED OUT; ERROR PROPAGATED AND IGNORED + remote_10.next_response = None + remote_11.next_response = ExecuteCommand_1.Response(status=0) + assert await run("10-12", "123", "--timeout=3") == ( + EXIT_CODE_UNSUCCESSFUL, + { + "10": None, + "11": {"status": 0}, + }, + ) + assert await run("10-12", "123", "--expect") == ( + 0, + { + "10": None, + "11": {"status": 0}, + }, + ) + + # FLAT OUTPUT (NOT GROUPED BY NODE-ID) + remote_11.next_response = ExecuteCommand_1.Response(status=210) + assert await run("11", "123", "FOO BAR", "--timeout=3") == ( + EXIT_CODE_UNSUCCESSFUL, + {"status": 210}, + ) + assert ( + remote_11.last_request + and remote_11.last_request.command == 123 + and remote_11.last_request.parameter.tobytes().decode() == "FOO BAR" + ) + assert await run("11", "222", "--timeout=3", "--expect=0..256") == ( + 0, + {"status": 210}, + ) + assert ( + remote_11.last_request + and remote_11.last_request.command == 222 + and remote_11.last_request.parameter.tobytes().decode() == "" + ) + + # ERRORS + assert (await run("bad"))[0] != 0 + assert (await run("10", "invalid_command"))[0] != 0 + assert (await run("10", "99999999999"))[0] != 0 # Bad command code, serialization will fail + assert (await run("10", "0", "z" * 1024))[0] != 0 # Bad parameter, serialization will fail diff --git a/tests/cmd/file_server.py b/tests/cmd/file_server.py index 683abe9..e202bbd 100644 --- a/tests/cmd/file_server.py +++ b/tests/cmd/file_server.py @@ -170,7 +170,7 @@ def __del__(self) -> None: "file-server", root, f"--plug-and-play={root}/allocation_table.db", - f"+U", + f"-u", environment_variables={ "UAVCAN__SERIAL__IFACE": serial_broker, "UAVCAN__NODE__ID": "42", diff --git a/tests/cmd/publish/expression.py b/tests/cmd/publish/expression.py index db5fcc8..0b72097 100644 --- a/tests/cmd/publish/expression.py +++ b/tests/cmd/publish/expression.py @@ -20,7 +20,7 @@ def _unittest_publish_expression_a(compiled_dsdl: typing.Any, serial_broker: str } proc_sub = Subprocess.cli( - "--format=json", + "-j", "sub", "7654:uavcan.primitive.array.Real64.1.0", environment_variables=env, @@ -71,7 +71,7 @@ def _unittest_publish_expression_b(compiled_dsdl: typing.Any, serial_broker: str } proc_sub = Subprocess.cli( - "--format=json", + "-j", "sub", "7654:uavcan.primitive.String.1.0", environment_variables=env, diff --git a/tests/cmd/pubsub.py b/tests/cmd/pubsub.py index 5215926..e2f2bce 100644 --- a/tests/cmd/pubsub.py +++ b/tests/cmd/pubsub.py @@ -22,31 +22,31 @@ def _unittest_pub_sub_regular(transport_factory: TransportFactory, compiled_dsdl "YAKUT_PATH": str(OUTPUT_DIR), } proc_sub_heartbeat = Subprocess.cli( - "--format=json", + "-j", "sub", - "uavcan.node.Heartbeat", + "uavcan.node.heartbeat", "--with-metadata", environment_variables=env, ) proc_sub_diagnostic = Subprocess.cli( - "--format=json", + "-j", "sub", - "4321:uavcan.diagnostic.Record", + "4321:uavcan.diagnostic.record", "--count=3", "--with-metadata", environment_variables=env, ) proc_sub_diagnostic_wrong_pid = Subprocess.cli( - "--format=yaml", + "-y", "sub", - "uavcan.diagnostic.Record", + "uavcan.diagnostic.record", "--count=3", environment_variables=env, ) proc_sub_temperature = Subprocess.cli( - "--format=json", + "-j", "sub", - "555:uavcan.si.sample.temperature.Scalar", + "555:uavcan.si.sample.temperature.scalar", "--count=3", "--no-metadata", environment_variables=env, @@ -64,7 +64,7 @@ def _unittest_pub_sub_regular(transport_factory: TransportFactory, compiled_dsdl '{severity: 6, timestamp: 123456, text: "Hello world!"}', # Use shorthand init for severity, timestamp "1234:uavcan.diagnostic.Record", '{text: "Goodbye world."}', - "555:uavcan.si.sample.temperature.Scalar", + "555:uavcan.si.sample.temperature.scalar", "{kelvin: 123.456}", "--count=3", "--period=3", @@ -77,9 +77,10 @@ def _unittest_pub_sub_regular(transport_factory: TransportFactory, compiled_dsdl _, stdout, _ = execute_cli( f"--transport={transport_factory(52).expression}", f"--path={OUTPUT_DIR}", + "-y", "call", "51", - "uavcan.node.GetInfo.1.0", + "uavcan.node.getinfo.1.0", "--no-metadata", "--timeout=5", timeout=10.0, @@ -114,24 +115,24 @@ def _unittest_pub_sub_regular(transport_factory: TransportFactory, compiled_dsdl assert 1 <= len(heartbeats) <= 20 for m in heartbeats: - src_nid = m["7509"]["_metadata_"]["source_node_id"] + src_nid = m["7509"]["_meta_"]["source_node_id"] if src_nid == 51: # The publisher - assert "high" in m["7509"]["_metadata_"]["priority"].lower() - assert m["7509"]["_metadata_"]["transfer_id"] >= 0 + assert "high" in m["7509"]["_meta_"]["priority"].lower() + assert m["7509"]["_meta_"]["transfer_id"] >= 0 assert m["7509"]["uptime"] in range(10) assert m["7509"]["vendor_specific_status_code"] == 54 elif src_nid == 52: # The caller (GetInfo) - assert "nominal" in m["7509"]["_metadata_"]["priority"].lower() - assert m["7509"]["_metadata_"]["transfer_id"] >= 0 + assert "nominal" in m["7509"]["_meta_"]["priority"].lower() + assert m["7509"]["_meta_"]["transfer_id"] >= 0 assert m["7509"]["uptime"] in range(4) else: assert False assert len(diagnostics) == 3 for m in diagnostics: - assert "slow" in m["4321"]["_metadata_"]["priority"].lower() - assert m["4321"]["_metadata_"]["transfer_id"] >= 0 - assert m["4321"]["_metadata_"]["source_node_id"] == 51 + assert "slow" in m["4321"]["_meta_"]["priority"].lower() + assert m["4321"]["_meta_"]["transfer_id"] >= 0 + assert m["4321"]["_meta_"]["source_node_id"] == 51 assert m["4321"]["timestamp"]["microsecond"] == 123456 assert m["4321"]["text"] == "Hello world!" @@ -149,23 +150,23 @@ def _unittest_slow_cli_pub_sub_anon(transport_factory: TransportFactory, compile "YAKUT_PATH": str(OUTPUT_DIR), } proc_sub_heartbeat = Subprocess.cli( # Windows compat: -v blocks stderr pipe on Windows. - "--format=json", + "-j", "sub", - "uavcan.node.Heartbeat", + "uavcan.node.heartbeat", "--with-metadata", environment_variables=env, ) proc_sub_diagnostic_with_meta = Subprocess.cli( # Windows compat: -v blocks stderr pipe on Windows. - "--format=json", + "-j", "sub", - "uavcan.diagnostic.Record", + "uavcan.diagnostic.record", "--with-metadata", environment_variables=env, ) proc_sub_diagnostic_no_meta = Subprocess.cli( # Windows compat: -v blocks stderr pipe on Windows. - "--format=json", + "-j", "sub", - "uavcan.diagnostic.Record", + "uavcan.diagnostic.record", "--no-metadata", environment_variables=env, ) @@ -175,7 +176,7 @@ def _unittest_slow_cli_pub_sub_anon(transport_factory: TransportFactory, compile if transport_factory(None).can_transmit: proc = Subprocess.cli( "pub", - "uavcan.diagnostic.Record", + "uavcan.diagnostic.record", "{}", "--count=2", "--period=2", @@ -197,9 +198,9 @@ def _unittest_slow_cli_pub_sub_anon(transport_factory: TransportFactory, compile # Hence, to support the case of redundant transports, we use 'greater or equal' here. assert len(diagnostics) >= 2 for m in diagnostics: - assert "nominal" in m["8184"]["_metadata_"]["priority"].lower() - assert m["8184"]["_metadata_"]["transfer_id"] >= 0 - assert m["8184"]["_metadata_"]["source_node_id"] is None + assert "nominal" in m["8184"]["_meta_"]["priority"].lower() + assert m["8184"]["_meta_"]["transfer_id"] >= 0 + assert m["8184"]["_meta_"]["source_node_id"] is None assert m["8184"]["timestamp"]["microsecond"] == 0 assert m["8184"]["text"] == "" @@ -224,10 +225,11 @@ def _unittest_slow_cli_pub_sub_anon(transport_factory: TransportFactory, compile def _unittest_e2e_discovery_pub(transport_factory: TransportFactory, compiled_dsdl: typing.Any) -> None: _ = compiled_dsdl proc_sub = Subprocess.cli( - "--format=json", + "-j", "sub", - "1000:uavcan.primitive.String", - "2000:uavcan.primitive.String", + "1000:uavcan.primitive.string", + "2000:uavcan.primitive.string", + "--smca", "--no-metadata", "--count=3", environment_variables={ @@ -235,7 +237,7 @@ def _unittest_e2e_discovery_pub(transport_factory: TransportFactory, compiled_ds "YAKUT_PATH": str(OUTPUT_DIR), }, ) - time.sleep(10.0) # Let the subscriber boot up. + time.sleep(3.0) # Let the subscriber boot up. proc_pub = Subprocess.cli( # Windows compat: -v blocks stderr pipe on Windows. "pub", "1000", # Use discovery. @@ -260,9 +262,9 @@ def _unittest_e2e_discovery_sub(transport_factory: TransportFactory, compiled_ds _ = compiled_dsdl proc_pub = Subprocess.cli( # Windows compat: -v blocks stderr pipe on Windows. "pub", - "1000:uavcan.primitive.String", + "1000:uavcan.primitive.string", "hello", - "2000:uavcan.primitive.String", + "2000:uavcan.primitive.string", "world", "--period=3", environment_variables={ @@ -270,13 +272,14 @@ def _unittest_e2e_discovery_sub(transport_factory: TransportFactory, compiled_ds "YAKUT_PATH": str(OUTPUT_DIR), }, ) - time.sleep(10.0) # Let the publisher boot up. + time.sleep(3.0) # Let the publisher boot up. proc_sub = Subprocess.cli( - "--format=json", + "-j", "sub", "1000", # Use discovery. "2000", # Use discovery. "--no-metadata", + "--smca", "--count=3", environment_variables={ "YAKUT_TRANSPORT": transport_factory(10).expression, diff --git a/tests/cmd/pubsub_sync.py b/tests/cmd/pubsub_sync.py index 6180ea0..9af019c 100644 --- a/tests/cmd/pubsub_sync.py +++ b/tests/cmd/pubsub_sync.py @@ -14,19 +14,19 @@ def _unittest_monoclust_ts_field_auto(transport_factory: TransportFactory, compiled_dsdl: typing.Any) -> None: _ = compiled_dsdl proc_sub = Subprocess.cli( - "--format=json", + "-j", "sub", "1000:uavcan.si.sample.mass.Scalar", "2000:uavcan.si.sample.mass.Scalar", "--no-metadata", "--count=3", - "--sync-monoclust", # Automatic tolerance setting. + "--smcf", # Automatic tolerance setting. environment_variables={ "YAKUT_TRANSPORT": transport_factory(10).expression, "YAKUT_PATH": str(OUTPUT_DIR), }, ) - time.sleep(10.0) + time.sleep(3.0) proc_pub = Subprocess.cli( # Windows compat: -v blocks stderr pipe on Windows. "pub", "1000:uavcan.si.sample.mass.Scalar", @@ -60,18 +60,18 @@ def _unittest_monoclust_ts_field_auto(transport_factory: TransportFactory, compi def _unittest_monoclust_ts_field_manual(transport_factory: TransportFactory, compiled_dsdl: typing.Any) -> None: _ = compiled_dsdl proc_sub = Subprocess.cli( - "--format=json", + "-j", "sub", "1000:uavcan.si.sample.mass.Scalar", "2000:uavcan.si.sample.mass.Scalar", "--no-metadata", - "--sync-monoclust=0.25", # Fixed tolerance setting; count not limited + "--smcf=0.25", # Fixed tolerance setting; count not limited environment_variables={ "YAKUT_TRANSPORT": transport_factory(10).expression, "YAKUT_PATH": str(OUTPUT_DIR), }, ) - time.sleep(10.0) + time.sleep(3.0) proc_pub = Subprocess.cli( # Windows compat: -v blocks stderr pipe on Windows. "pub", "1000:uavcan.si.sample.mass.Scalar", @@ -112,7 +112,7 @@ def _unittest_monoclust_ts_field_type_not_timestamped( "sub", "1000:uavcan.si.unit.mass.Scalar", "2000:uavcan.si.unit.mass.Scalar", - "--sync-monoclust", # Require timestamp field matching but the data types have no such field + "--smcf", # Require timestamp field matching but the data types have no such field environment_variables={ "YAKUT_TRANSPORT": transport_factory(10).expression, "YAKUT_PATH": str(OUTPUT_DIR), @@ -129,13 +129,13 @@ def _unittest_monoclust_ts_field_type_not_timestamped( def _unittest_monoclust_ts_arrival_auto(transport_factory: TransportFactory, compiled_dsdl: typing.Any) -> None: _ = compiled_dsdl proc_sub = Subprocess.cli( - "--format=json", + "-j", "sub", "1000:uavcan.primitive.String", "2000:uavcan.primitive.String", "--no-metadata", "--count=3", - "--sync-monoclust-arrival", # Automatic tolerance setting. + "--smca", # Automatic tolerance setting. environment_variables={ "YAKUT_TRANSPORT": transport_factory(10).expression, "YAKUT_PATH": str(OUTPUT_DIR), @@ -166,13 +166,13 @@ def _unittest_monoclust_ts_arrival_auto(transport_factory: TransportFactory, com def _unittest_transfer_id(transport_factory: TransportFactory, compiled_dsdl: typing.Any) -> None: _ = compiled_dsdl proc_sub = Subprocess.cli( - "--format=json", + "-j", "sub", "1000:uavcan.primitive.String", "2000:uavcan.primitive.String", "--no-metadata", "--count=3", - "--sync-transfer-id", + "--stid", environment_variables={ "YAKUT_TRANSPORT": transport_factory(10).expression, "YAKUT_PATH": str(OUTPUT_DIR), @@ -198,3 +198,35 @@ def _unittest_transfer_id(transport_factory: TransportFactory, compiled_dsdl: ty {"1000": {"value": "1"}, "2000": {"value": "1"}}, {"1000": {"value": "2"}, "2000": {"value": "2"}}, ] + + +def _unittest_async(transport_factory: TransportFactory, compiled_dsdl: typing.Any) -> None: + _ = compiled_dsdl + proc_sub = Subprocess.cli( + "-j", + "sub", + "1000:uavcan.primitive.String", + "2000:uavcan.primitive.String", + "--no-metadata", + "--count=4", + environment_variables={ + "YAKUT_TRANSPORT": transport_factory(10).expression, + "YAKUT_PATH": str(OUTPUT_DIR), + }, + ) + time.sleep(3.0) + env = { + **transport_factory(11).environment, + "YAKUT_PATH": str(OUTPUT_DIR), + } + execute_cli("pub", "--count=1", "1000:uavcan.primitive.String", "abc", environment_variables=env) + execute_cli("pub", "--count=1", "2000:uavcan.primitive.String", "def", environment_variables=env) + execute_cli("pub", "--count=2", "1000:uavcan.primitive.String", "ghi", environment_variables=env) + out_sub = proc_sub.wait(30.0)[1].splitlines() + msgs = list(map(json.loads, out_sub)) + assert msgs == [ + {"1000": {"value": "abc"}}, + {"2000": {"value": "def"}}, + {"1000": {"value": "ghi"}}, + {"1000": {"value": "ghi"}}, + ] diff --git a/tests/cmd/register_access.py b/tests/cmd/register_access.py new file mode 100644 index 0000000..7efbd9c --- /dev/null +++ b/tests/cmd/register_access.py @@ -0,0 +1,205 @@ +# Copyright (c) 2022 OpenCyphal +# This software is distributed under the terms of the MIT License. +# Author: Pavel Kirienko + +from __future__ import annotations +import asyncio +import time +import json +from typing import Any +import pytest +from tests.dsdl import OUTPUT_DIR +from tests.transport import TransportFactory +from tests.subprocess import execute_cli, Subprocess + + +@pytest.mark.asyncio +async def _unittest_logic(compiled_dsdl: Any) -> None: + from pycyphal.transport.loopback import LoopbackTransport + import pycyphal.application + from pycyphal.application.register import ValueProxy, Natural64 + from yakut.cmd.register_access._logic import access, Result + from yakut.cmd.register_access._cmd import _make_representer + + _ = compiled_dsdl + repr_simple = _make_representer(simplify=True, metadata=False) + repr_full = _make_representer(simplify=False, metadata=True) + + node = pycyphal.application.make_node(pycyphal.application.NodeInfo(), transport=LoopbackTransport(10)) + try: + node.registry.clear() + node.registry["a"] = ValueProxy("a") + node.registry["b"] = ValueProxy(Natural64([1, 2, 3])) + node.start() + + async def once(**kwargs: Any) -> Result: + print() # Separate from prior output + out = await access( # pylint: disable=missing-kwoa + node, + lambda text: print(f"Progress: {text!r}"), + timeout=1.0, + **kwargs, + ) + print("Result:", out.value_per_node) + print("Errors:", out.errors) + print("Warns :", out.warnings) + return out + + # READ EXISTING REGISTER FROM EXISTING NODE + res = await once( + node_ids=[10], + reg_name="a", + reg_val_str=None, + optional_service=False, + optional_register=False, + ) + assert not res.errors + assert not res.warnings + assert repr_simple(res.value_per_node[10]) == "a" + assert repr_full(res.value_per_node[10])["string"]["value"] == "a" + + # READ EXISTING REGISTER BUT ONE NODE IS MISSING + res = await once( + node_ids=[10, 11], + reg_name="b", + reg_val_str=None, + optional_service=False, + optional_register=False, + ) + assert len(res.errors) == 1 + assert not res.warnings + assert repr_simple(res.value_per_node[10]) == [1, 2, 3] + assert repr_simple(res.value_per_node[11]) is None + + # READ EXISTING REGISTER, ONE NODE IS MISSING, BUT OPTIONAL SERVICE IS ENABLED + res = await once( + node_ids=[10, 11], + reg_name="b", + reg_val_str=None, + optional_service=True, + optional_register=False, + ) + assert not res.errors + assert len(res.warnings) == 1 + assert repr_simple(res.value_per_node[10]) == [1, 2, 3] + assert repr_simple(res.value_per_node[11]) is None + + # READ NONEXISTENT REGISTER, NO ERROR BECAUSE NOT ASKED TO MODIFY IT + res = await once( + node_ids=[10], + reg_name="c", + reg_val_str=None, + optional_service=False, + optional_register=False, + ) + assert not res.errors + assert not res.warnings + assert repr_simple(res.value_per_node[10]) is None + + # WRITE NONEXISTENT REGISTER, ERROR + res = await once( + node_ids=[10], + reg_name="c", + reg_val_str="VALUE", + optional_service=False, + optional_register=False, + ) + assert len(res.errors) == 1 + assert not res.warnings + assert repr_simple(res.value_per_node[10]) is None + + # WRITE NONEXISTENT REGISTER, NO ERROR BECAUSE OPTIONAL REGISTER MODE + res = await once( + node_ids=[10], + reg_name="c", + reg_val_str="VALUE", + optional_service=False, + optional_register=True, + ) + assert not res.errors + assert len(res.warnings) == 1 + assert repr_simple(res.value_per_node[10]) is None + + # WRITE REGISTER, DATA NOT COERCIBLE + res = await once( + node_ids=[10], + reg_name="b", + reg_val_str="NOT VALID", + optional_service=False, + optional_register=False, + ) + assert len(res.errors) == 1 + assert len(res.warnings) == 0 + assert repr_simple(res.value_per_node[10]) == [1, 2, 3] + + finally: + node.close() + await asyncio.sleep(1) + + +def _unittest_cmd(compiled_dsdl: Any, transport_factory: TransportFactory) -> None: + _ = compiled_dsdl + # Run a dummy node which we can query. + bg_node = Subprocess.cli( + "sub", + "1000:uavcan.primitive.empty", + environment_variables={ + **transport_factory(10).environment, + "YAKUT_PATH": str(OUTPUT_DIR), + }, + ) + time.sleep(2) + expect_register = "uavcan.node.description" + try: + # READ EXISTING REGISTER, MAP OUTPUT + status, stdout, _ = execute_cli( + "-j", + "register-access", + "10,", + expect_register, + environment_variables={ + "YAKUT_TRANSPORT": transport_factory(100).expression, + "YAKUT_PATH": str(OUTPUT_DIR), + }, + ) + assert status == 0 + data = json.loads(stdout.strip()) + print(json.dumps(data, indent=4)) + assert len(data) == 1 + assert data["10"] == "" + + # READ EXISTING REGISTER, FLAT OUTPUT + status, stdout, _ = execute_cli( + "-j", + "register-access", + "10", + expect_register, + environment_variables={ + "YAKUT_TRANSPORT": transport_factory(100).expression, + "YAKUT_PATH": str(OUTPUT_DIR), + }, + ) + assert status == 0 + data = json.loads(stdout.strip()) + print(json.dumps(data, indent=4)) + assert data == "" + + # MODIFY REGISTER, MAP OUTPUT + status, stdout, _ = execute_cli( + "-j", + "register-access", + "10,", + expect_register, + "Reference value", + environment_variables={ + "YAKUT_TRANSPORT": transport_factory(100).expression, + "YAKUT_PATH": str(OUTPUT_DIR), + }, + ) + assert status == 0 + data = json.loads(stdout.strip()) + print(json.dumps(data, indent=4)) + assert len(data) == 1 + assert data["10"] == "Reference value" + finally: + bg_node.wait(10, interrupt=True) diff --git a/tests/cmd/register_batch.py b/tests/cmd/register_batch.py new file mode 100644 index 0000000..a265cb6 --- /dev/null +++ b/tests/cmd/register_batch.py @@ -0,0 +1,255 @@ +# Copyright (c) 2022 OpenCyphal +# This software is distributed under the terms of the MIT License. +# Author: Pavel Kirienko + +from __future__ import annotations +import asyncio +import time +from typing import Any +import json +import tempfile +from pathlib import Path +from pprint import pprint +import pytest +from tests.dsdl import OUTPUT_DIR +from tests.transport import TransportFactory +from tests.subprocess import execute_cli, Subprocess + + +@pytest.mark.asyncio +async def _unittest_caller(compiled_dsdl: Any) -> None: + from pycyphal.transport.loopback import LoopbackTransport + import pycyphal.application + from pycyphal.application.register import ValueProxy, Natural64, Value, String + from yakut.cmd.register_batch._directive import Directive + from yakut.cmd.register_batch._caller import Skipped, Timeout, TypeCoercionFailure, do_calls + + _ = compiled_dsdl + + node = pycyphal.application.make_node(pycyphal.application.NodeInfo(), transport=LoopbackTransport(10)) + try: + node.registry.clear() + node.registry["a"] = ValueProxy("a") + node.registry["b"] = ValueProxy(Natural64([1, 2, 3])) + node.registry["c"] = ValueProxy(Natural64([3, 2, 1])) + node.start() + + res = await do_calls( + node, + lambda x: print("Progress:", x), + timeout=1.0, + directive=Directive( + registers_per_node={ + 10: { + "c": lambda _: None, # Type coercion failure does not interrupt further processing. + "a": Value(string=String("z")), + "d": Value(string=String("n")), # No such register. + "b": lambda v: v, + }, + 11: { + "y": lambda _: None, + "z": lambda _: None, + }, + } + ), + ) + pprint(res.responses_per_node) + assert res.responses_per_node.keys() == {10, 11} + + assert res.responses_per_node[10]["a"].value.string.value.tobytes().decode() == "z" # type: ignore + assert list(res.responses_per_node[10]["b"].value.natural64.value) == [1, 2, 3] # type: ignore + assert isinstance(res.responses_per_node[10]["c"], TypeCoercionFailure) + assert res.responses_per_node[10]["d"].value.empty # type: ignore + + assert res.responses_per_node[11]["y"] == Timeout() + assert res.responses_per_node[11]["z"] == Skipped() + + finally: + node.close() + await asyncio.sleep(1) + + +def _unittest_cmd(compiled_dsdl: Any, transport_factory: TransportFactory) -> None: + _ = compiled_dsdl + file = Path(tempfile.mktemp("yakut_register_batch_test.yaml")) + # Run dummy nodes which we can query. + bg_nodes = [ + Subprocess.cli( + "sub", + "1000:uavcan.primitive.empty", + environment_variables={ + **transport_factory(10 + idx).environment, + "YAKUT_PATH": str(OUTPUT_DIR), + }, + ) + for idx in range(2) + ] + time.sleep(1) + try: + # READ INPUT KEYED + file.write_text("{10: [uavcan.node.id, uavcan.node.description], 11: [uavcan.node.id]}") + status, stdout, _ = execute_cli( + "register-batch", + f"--file={file}", + environment_variables={ + **transport_factory(100).environment, + "YAKUT_PATH": str(OUTPUT_DIR), + }, + ) + assert status == 0 + data = json.loads(stdout.strip()) + print(json.dumps(data, indent=4)) + assert len(data) == 2 + assert data["10"]["uavcan.node.id"] == 10 + assert data["10"]["uavcan.node.description"] == "" + assert data["11"]["uavcan.node.id"] == 11 + + # MODIFY INPUT KEYED + file.write_text("{10: {uavcan.node.description: TEN}, 11: {uavcan.node.description: ELEVEN}}") + status, stdout, _ = execute_cli( + "register-batch", + f"--file={file}", + environment_variables={ + **transport_factory(100).environment, + "YAKUT_PATH": str(OUTPUT_DIR), + }, + ) + assert status == 0 + data = json.loads(stdout.strip()) + print(json.dumps(data, indent=4)) + assert len(data) == 2 + assert data["10"]["uavcan.node.description"] == "TEN" + assert data["11"]["uavcan.node.description"] == "ELEVEN" + + # READ INPUT FLAT, OUTPUT FLAT + file.write_text("[uavcan.node.id, uavcan.node.description]") + status, stdout, _ = execute_cli( + "register-batch", + f"--file={file}", + "10", + environment_variables={ + **transport_factory(100).environment, + "YAKUT_PATH": str(OUTPUT_DIR), + }, + ) + assert status == 0 + data = json.loads(stdout.strip()) + print(json.dumps(data, indent=4)) + assert len(data) == 2 + assert data["uavcan.node.id"] == 10 + assert data["uavcan.node.description"] == "TEN" + + # MODIFY INPUT FLAT, OUTPUT KEYED + file.write_text("{uavcan.node.description: 'TEN OR ELEVEN'}") + status, stdout, _ = execute_cli( + "register-batch", + f"--file={file}", + "10,11", + environment_variables={ + **transport_factory(100).environment, + "YAKUT_PATH": str(OUTPUT_DIR), + }, + ) + assert status == 0 + data = json.loads(stdout.strip()) + print(json.dumps(data, indent=4)) + assert len(data) == 2 + assert data["10"]["uavcan.node.description"] == "TEN OR ELEVEN" + assert data["11"]["uavcan.node.description"] == "TEN OR ELEVEN" + + # MODIFY INPUT FLAT, OUTPUT KEYED, ONE TIMED OUT WITH ERROR + file.write_text("{uavcan.node.description: XXX}") + status, stdout, _ = execute_cli( + "register-batch", + f"--file={file}", + "10-13", + environment_variables={ + **transport_factory(100).environment, + "YAKUT_PATH": str(OUTPUT_DIR), + }, + ensure_success=False, + ) + assert status != 0 + data = json.loads(stdout.strip()) + print(json.dumps(data, indent=4)) + assert len(data) == 3 + assert data["10"]["uavcan.node.description"] == "XXX" + assert data["11"]["uavcan.node.description"] == "XXX" + assert not data["12"] + + # MODIFY INPUT FLAT, OUTPUT KEYED, NO SUCH REGISTER ERROR + file.write_text("{nonexistent.register: 123}") + status, stdout, _ = execute_cli( + "register-batch", + f"--file={file}", + "10,11", + environment_variables={ + **transport_factory(100).environment, + "YAKUT_PATH": str(OUTPUT_DIR), + }, + ensure_success=False, + ) + assert status != 0 + data = json.loads(stdout.strip()) + print(json.dumps(data, indent=4)) + assert len(data) == 2 + assert data["10"]["nonexistent.register"] is None + assert data["11"]["nonexistent.register"] is None + + # MODIFY INPUT FLAT, OUTPUT KEYED, NO SUCH REGISTER, ERROR IGNORED + file.write_text("{nonexistent.register: 123}") + status, stdout, _ = execute_cli( + "register-batch", + f"--file={file}", + "10,11", + "--optional-register", + environment_variables={ + **transport_factory(100).environment, + "YAKUT_PATH": str(OUTPUT_DIR), + }, + ) + assert status == 0 + data = json.loads(stdout.strip()) + print(json.dumps(data, indent=4)) + assert len(data) == 2 + assert data["10"]["nonexistent.register"] is None + assert data["11"]["nonexistent.register"] is None + + # MODIFY INPUT FLAT, OUTPUT FLAT, DETAILED + file.write_text("[uavcan.node.id]") + status, stdout, _ = execute_cli( + "register-batch", + f"--file={file}", + "10", + "--detailed", + environment_variables={ + **transport_factory(100).environment, + "YAKUT_PATH": str(OUTPUT_DIR), + }, + ) + assert status == 0 + data = json.loads(stdout.strip()) + print(json.dumps(data, indent=4)) + assert len(data) == 1 + assert data["uavcan.node.id"]["natural16"]["value"] == [10] + + # MODIFY INPUT FLAT, OUTPUT FLAT, DETAILED, FILTERED EMPTY + file.write_text("[uavcan.node.id]") + status, stdout, _ = execute_cli( + "register-batch", + f"--file={file}", + "10", + "--only=iv", # The requested register is not immutable-volatile so it will be skipped. + environment_variables={ + **transport_factory(100).environment, + "YAKUT_PATH": str(OUTPUT_DIR), + }, + ) + assert status == 0 + data = json.loads(stdout.strip()) + print(json.dumps(data, indent=4)) + assert data == {} + finally: + for bg in bg_nodes: + bg.wait(10, interrupt=True) + file.unlink() diff --git a/tests/cmd/register_list.py b/tests/cmd/register_list.py new file mode 100644 index 0000000..7e00ca5 --- /dev/null +++ b/tests/cmd/register_list.py @@ -0,0 +1,162 @@ +# Copyright (c) 2022 OpenCyphal +# This software is distributed under the terms of the MIT License. +# Author: Pavel Kirienko + +from __future__ import annotations +import asyncio +import time +import json +from typing import Any +import pytest +from tests.dsdl import OUTPUT_DIR +from tests.transport import TransportFactory +from tests.subprocess import execute_cli, Subprocess + + +@pytest.mark.asyncio +async def _unittest_logic(compiled_dsdl: Any) -> None: + from pycyphal.transport.loopback import LoopbackTransport + import pycyphal.application + from pycyphal.application.register import ValueProxy + from yakut.cmd.register_list._logic import list_names + + _ = compiled_dsdl + + node = pycyphal.application.make_node(pycyphal.application.NodeInfo(), transport=LoopbackTransport(10)) + try: + node.registry.clear() + node.registry["a"] = ValueProxy("a") + node.registry["b"] = ValueProxy("b") + node.registry["c"] = ValueProxy("c") + node.start() + + res = await list_names( + node, + lambda text: print(f"Progress: {text!r}"), + node_ids=[10], + optional_service=False, + timeout=1.0, + ) + assert not res.errors + assert not res.warnings + assert res.names_per_node == { + 10: ["a", "b", "c"], + } + + res = await list_names( + node, + lambda text: print(f"Progress: {text!r}"), + node_ids=[10, 3], + optional_service=False, + timeout=1.0, + ) + assert len(res.errors) == 1 + assert not res.warnings + assert res.names_per_node == { + 10: ["a", "b", "c"], + 3: None, + } + + res = await list_names( + node, + lambda text: print(f"Progress: {text!r}"), + node_ids=[10, 3], + optional_service=True, + timeout=1.0, + ) + assert not res.errors + assert len(res.warnings) == 1 + assert res.names_per_node == { + 10: ["a", "b", "c"], + 3: None, + } + finally: + node.close() + await asyncio.sleep(1) + + +def _unittest_cmd(compiled_dsdl: Any, transport_factory: TransportFactory) -> None: + _ = compiled_dsdl + # Run a dummy node which we can query. + bg_node = Subprocess.cli( + "sub", + "1000:uavcan.primitive.empty", + environment_variables={ + **transport_factory(10).environment, + "YAKUT_PATH": str(OUTPUT_DIR), + }, + ) + time.sleep(2) + expect_register = "uavcan.node.description" + try: + # Not keyed by node-ID. + status, stdout, _ = execute_cli( + "-j", + "register-list", + "10", + environment_variables={ + "YAKUT_TRANSPORT": transport_factory(100).expression, + "YAKUT_PATH": str(OUTPUT_DIR), + }, + ) + assert status == 0 + data = json.loads(stdout.strip()) + print(json.dumps(data, indent=4)) + assert len(data) > 1 + assert expect_register in data + + # Keyed by node-ID. + status, stdout, _ = execute_cli( + "-j", + "register-list", + "10,", # Mind the comma! + environment_variables={ + "YAKUT_TRANSPORT": transport_factory(100).expression, + "YAKUT_PATH": str(OUTPUT_DIR), + }, + ) + assert status == 0 + data = json.loads(stdout.strip()) + print(json.dumps(data, indent=4)) + assert len(data) == 1 + assert expect_register in data["10"] + + # Poll non-existent nodes. + status, stdout, _ = execute_cli( + "-j", + "register-list", + "10..13", + environment_variables={ + "YAKUT_TRANSPORT": transport_factory(100).expression, + "YAKUT_PATH": str(OUTPUT_DIR), + }, + ensure_success=False, + ) + assert status != 0 # Because timed out + data = json.loads(stdout.strip()) + print(json.dumps(data, indent=4)) + assert len(data) == 3 + assert expect_register in data["10"] + assert data["11"] is None + assert data["12"] is None + + # Same but no error. + status, stdout, _ = execute_cli( + "-j", + "register-list", + "10..13", + "--optional-service", + environment_variables={ + "YAKUT_TRANSPORT": transport_factory(100).expression, + "YAKUT_PATH": str(OUTPUT_DIR), + }, + ) + assert status == 0 + data = json.loads(stdout.strip()) + print(json.dumps(data, indent=4)) + assert len(data) == 3 + assert expect_register in data["10"] + assert data["11"] is None + assert data["12"] is None + finally: + bg_node.wait(10, interrupt=True) diff --git a/tests/dtype_loader.py b/tests/dtype_loader.py index 9871468..8e22eb0 100644 --- a/tests/dtype_loader.py +++ b/tests/dtype_loader.py @@ -26,7 +26,7 @@ def _unittest_dtype_loader(compiled_dsdl: Any) -> None: assert model.full_name == "uavcan.node.Heartbeat" assert model.version == (1, 0) - ty = load_dtype("sirius_cyber_corp.Foo.1.0") + ty = load_dtype("sirius_cyber_corp.foo.1.0") assert isinstance(ty, type) model = pycyphal.dsdl.get_model(ty) assert model.full_name == "sirius_cyber_corp.Foo" @@ -35,7 +35,7 @@ def _unittest_dtype_loader(compiled_dsdl: Any) -> None: with pytest.raises(NotFoundError): _ = load_dtype("sirius_cyber_corp.Foo.1.1") - ty = load_dtype("sirius_cyber_corp.Foo.1.1", allow_minor_version_mismatch=True) # Same but relaxed. + ty = load_dtype("sirius_cyber_corp.foo.1.1", allow_minor_version_mismatch=True) # Same but relaxed. assert isinstance(ty, type) model = pycyphal.dsdl.get_model(ty) assert model.full_name == "sirius_cyber_corp.Foo" @@ -47,7 +47,7 @@ def _unittest_dtype_loader(compiled_dsdl: Any) -> None: assert model.full_name == "sirius_cyber_corp.Foo" assert model.version == (1, 9) - ty = load_dtype("sirius_cyber_corp.Foo.2") + ty = load_dtype("sirius_cyber_corp.foo.2") assert isinstance(ty, type) model = pycyphal.dsdl.get_model(ty) assert model.full_name == "sirius_cyber_corp.Foo" diff --git a/tests/transport.py b/tests/transport.py index edd7dfc..e722b47 100644 --- a/tests/transport.py +++ b/tests/transport.py @@ -2,6 +2,7 @@ # This software is distributed under the terms of the MIT License. # Author: Pavel Kirienko +from __future__ import annotations import os import sys import time @@ -13,8 +14,18 @@ @dataclasses.dataclass(frozen=True) class TransportConfig: + """ + Either "expression" or "environment" can be used to initialize the node. + The configurations should be nearly equivalent. + """ + expression: str + """ + Please do not use this in new tests, + consider using the environment variables instead as they are the recommended form now. + """ can_transmit: bool + environment: dict[str, str] TransportFactory = typing.Callable[[typing.Optional[int]], TransportConfig] @@ -31,6 +42,13 @@ def _generate() -> typing.Iterator[typing.Callable[[], typing.Iterator[Transport Sensible transport configurations supported by the CLI to test against. Don't forget to extend when adding support for new transports. """ + + def mk_env(node_id: int | None, **items: typing.Any) -> dict[str, str]: + return { + **{k: str(v) for k, v in items.items()}, + **({"UAVCAN__NODE__ID": str(node_id)} if node_id is not None else {}), + } + if sys.platform == "linux": # pragma: no branch def sudo(cmd: str, ensure_success: bool = True) -> None: @@ -52,6 +70,11 @@ def vcan() -> typing.Iterator[TransportFactory]: yield lambda nid: TransportConfig( expression=f"CAN(can.media.socketcan.SocketCANMedia('vcan0',64),local_node_id={nid})", can_transmit=True, + environment=mk_env( + nid, + UAVCAN__CAN__IFACE="socketcan:vcan0", + UAVCAN__CAN__MTU=64, + ), ) def vcan_tmr() -> typing.Iterator[TransportFactory]: @@ -64,6 +87,11 @@ def vcan_tmr() -> typing.Iterator[TransportFactory]: ) ), can_transmit=True, + environment=mk_env( + nid, + UAVCAN__CAN__IFACE="socketcan:vcan0 socketcan:vcan1 socketcan:vcan2", + UAVCAN__CAN__MTU="64", + ), ) yield vcan @@ -87,6 +115,10 @@ def serial_tunneled_via_tcp() -> typing.Iterator[TransportFactory]: yield lambda nid: TransportConfig( expression=f"Serial('{serial_endpoint}',local_node_id={nid})", can_transmit=True, + environment=mk_env( + nid, + UAVCAN__SERIAL__IFACE=serial_endpoint, + ), ) assert broker.alive time.sleep(1.0) # Ensure all clients have disconnected to avoid warnings in the test logs. @@ -95,9 +127,23 @@ def serial_tunneled_via_tcp() -> typing.Iterator[TransportFactory]: def udp_loopback() -> typing.Iterator[TransportFactory]: yield lambda nid: ( - TransportConfig(expression=f"UDP('127.0.0.0',{nid})", can_transmit=True) + TransportConfig( + expression=f"UDP('127.0.0.0',{nid})", + can_transmit=True, + environment=mk_env( + nid, + UAVCAN__UDP__IFACE="127.0.0.0", + ), + ) if nid is not None - else TransportConfig(expression="UDP('127.0.0.1',None)", can_transmit=False) + else TransportConfig( + expression="UDP('127.0.0.1',None)", + can_transmit=False, + environment=mk_env( + nid, + UAVCAN__UDP__IFACE="127.0.0.1", + ), + ) ) def heterogeneous_udp_serial() -> typing.Iterator[TransportFactory]: @@ -113,6 +159,11 @@ def heterogeneous_udp_serial() -> typing.Iterator[TransportFactory]: ) ), can_transmit=nid is not None, + environment=mk_env( + nid, + UAVCAN__SERIAL__IFACE=serial_endpoint, + UAVCAN__UDP__IFACE="127.0.0.1", + ), ) assert broker.alive time.sleep(1.0) # Ensure all clients have disconnected to avoid warnings in the test logs. diff --git a/yakut/VERSION b/yakut/VERSION index 78bc1ab..d9df1bb 100644 --- a/yakut/VERSION +++ b/yakut/VERSION @@ -1 +1 @@ -0.10.0 +0.11.0 diff --git a/yakut/cmd/call.py b/yakut/cmd/call.py index fb2d2c0..e251b2b 100644 --- a/yakut/cmd/call.py +++ b/yakut/cmd/call.py @@ -3,15 +3,15 @@ # Author: Pavel Kirienko from __future__ import annotations +import sys from typing import Any, Callable, TYPE_CHECKING import decimal from functools import lru_cache -import asyncio import click import pycyphal import yakut from yakut.enum_param import EnumParam -from yakut.param.formatter import Formatter +from yakut.param.formatter import Formatter, FormatterHints from yakut.util import convert_transfer_metadata_to_builtin from yakut import dtype_loader @@ -60,7 +60,7 @@ def _validate_request_fields(ctx: click.Context, param: click.Parameter, value: "+M/-M", default=False, show_default=True, - help="When enabled, the response object is prepended with an extra field named `_metadata_`.", + help="When enabled, the response object is prepended with an extra field named `_meta_`.", ) @yakut.pass_purser @yakut.asynchronous() @@ -88,6 +88,7 @@ async def call( [SERVICE_ID:]TYPE_NAME[.MAJOR[.MINOR]] SERVICE_NAME[:TYPE_NAME[.MAJOR[.MINOR]]] + The short data type name is case-insensitive for convenience. In the data type name, "/" or "\\" can be used instead of "." for convenience and filesystem autocompletion. If the data type is specified without the service-ID, a fixed service-ID shall be defined for this data type. Missing version numbers default to the latest available. @@ -105,7 +106,7 @@ async def call( Examples: \b - yakut call 42 uavcan.node.GetInfo +M -T3 -Pe + yakut call 42 uavcan.node.getinfo +M -T3 -Pe yakut call 42 least_squares 'points: [{x: 10, y: 1}, {x: 20, y: 2}]' yakut call 42 least_squares:sirius_cyber_corp.PerformLinearLeastSquaresFit '[[10, 1], [20, 2]]' """ @@ -127,7 +128,7 @@ async def call( ) finalizers: list[Callable[[], None]] = [] try: - formatter = purser.make_formatter() + formatter = purser.make_formatter(FormatterHints(single_document=True)) # The cached factory is needed to postpone node initialization as much as possible because it disturbs # the network and the networking hardware (if any) and is usually costly. @@ -157,7 +158,6 @@ def get_node() -> Node: await _run(client, request, formatter, with_metadata=with_metadata) finally: pycyphal.util.broadcast(finalizers[::-1])() - await asyncio.sleep(1e-3) # Let the background tasks finalize before leaving the loop. async def _run( @@ -177,7 +177,7 @@ def on_transfer_feedback(fb: pycyphal.transport.Feedback) -> None: request_ts_application = pycyphal.transport.Timestamp.now() result = await client.call(request) response_ts_application = pycyphal.transport.Timestamp.now() - if result is None: + if result is None: # TODO this should exit with yakut.util.EXIT_CODE_UNSUCCESSFUL raise click.ClickException(f"The request has timed out after {client.response_timeout:0.1f} seconds") if not request_ts_transport: # pragma: no cover request_ts_transport = request_ts_application @@ -205,15 +205,14 @@ def on_transfer_feedback(fb: pycyphal.transport.Feedback) -> None: bi.update( convert_transfer_metadata_to_builtin( transfer, - roundtrip_time={ - "transport_layer": (transfer.timestamp.monotonic - request_ts_transport.monotonic).quantize(qnt), - "application_layer": application_duration.quantize(qnt), - }, + dtype=client.dtype, + rtt=(transfer.timestamp.monotonic - request_ts_transport.monotonic).quantize(qnt), ) ) bi.update(pycyphal.dsdl.to_builtin(response)) - print(formatter({client.port_id: bi})) + sys.stdout.write(formatter({client.port_id: bi})) + sys.stdout.flush() async def _resolve( @@ -258,7 +257,7 @@ async def _resolve( (ty_or_srv,) = specs del specs - possibly_dtype_name = ty_or_srv.count(".") >= 1 and any(x.isupper() for x in ty_or_srv) + possibly_dtype_name = ty_or_srv.count(".") >= 1 if possibly_dtype_name: try: dtype = dtype_loader.load_dtype(ty_or_srv) diff --git a/yakut/cmd/execute_command/__init__.py b/yakut/cmd/execute_command/__init__.py new file mode 100644 index 0000000..a269f8b --- /dev/null +++ b/yakut/cmd/execute_command/__init__.py @@ -0,0 +1,5 @@ +# Copyright (c) 2022 OpenCyphal +# This software is distributed under the terms of the MIT License. +# Author: Pavel Kirienko + +from ._cmd import execute_command diff --git a/yakut/cmd/execute_command/_cmd.py b/yakut/cmd/execute_command/_cmd.py new file mode 100644 index 0000000..c2222e3 --- /dev/null +++ b/yakut/cmd/execute_command/_cmd.py @@ -0,0 +1,267 @@ +# Copyright (c) 2022 OpenCyphal +# This software is distributed under the terms of the MIT License. +# Author: Pavel Kirienko + +from __future__ import annotations +import sys +from typing import TYPE_CHECKING, Callable, Sequence, Optional, Any +import asyncio +import click +import pycyphal +import yakut +from yakut.int_set_parser import parse_int_set, INT_SET_USER_DOC +from yakut.ui import ProgressReporter, show_error, show_warning +from yakut.param.formatter import FormatterHints +from yakut.util import EXIT_CODE_UNSUCCESSFUL + +if TYPE_CHECKING: + import pycyphal.application + from uavcan.node import ExecuteCommand_1 + +_logger = yakut.get_logger(__name__) + + +_HELP = f""" +Invoke uavcan.node.ExecuteCommand on a group of nodes. + +The response objects are returned as a per-node mapping unless the set of node-IDs is one integer, +in which case only that object is returned without the node-ID. + +The command may be specified either as an explicit integer or as a mnemonic name defined in the +uavcan.node.ExecuteCommand service specification; e.g., restart=65535, begin_software_update=65533; +abbreviations are also accepted. + +Restart many nodes at once, some of which may not be present (for Cyphal/CAN this would be the entire network): + +\b + yakut execute-command 0-128 restart -e + yakut execute-command 0-128 65535 -e # Same, with the command code given explicitly. + +Request multiple nodes to install the same software image +(requires a file server node) (also see the file server command): + +\b + y cmd 122-126 begin_software_update /path/to/software/image + +{INT_SET_USER_DOC} +""" + + +def _parse_status_set(inp: str) -> set[int] | None: + """ + >>> _parse_status_set("") is None + True + >>> _parse_status_set("1") == {1} + True + >>> _parse_status_set("1-5") == {1, 2, 3, 4} + True + """ + if not inp: + return None + ins = parse_int_set(inp) + return set(ins) if not isinstance(ins, (int, float)) else {ins} + + +@yakut.subcommand(aliases="cmd", help=_HELP) +@click.argument("node_ids", type=parse_int_set) +@click.argument("command") +@click.argument("parameter", default="") +@click.option( + "--expect", + "-e", + type=_parse_status_set, + default="0", + metavar="[STATUS_CODES]", + is_flag=False, + flag_value="", + help=f""" +Specify the set of status codes that shall be accepted as success (the default is only zero). +The set notation is supported; e.g., use 0-256 to accept any status code as success. + +Using this option without any value (or an empty string) +will result in sending the requests simultaneously without requiring the nodes to respond and not validating the status +(fire and forget). +This is occasionally useful because some nodes may be unable to respond to certain requests such as COMMAND_RESTART; +some nodes may not support this service at all. + +{INT_SET_USER_DOC} +""", +) +@click.option( + "--timeout", + "-T", + type=float, + default=pycyphal.presentation.DEFAULT_SERVICE_REQUEST_TIMEOUT, + show_default=True, + metavar="SECONDS", + help="Service response timeout.", +) +@yakut.pass_purser +@yakut.asynchronous() +async def execute_command( + purser: yakut.Purser, + node_ids: set[int] | int, + command: str, + parameter: str, + expect: set[int] | None, + timeout: float, +) -> int: + _logger.debug( + "node_ids=%r command=%r parameter=%r expect=%r timeout=%r", node_ids, command, parameter, expect, timeout + ) + command_parsed = _parse_command(command) + del command + formatter = purser.make_formatter(FormatterHints(single_document=True)) + try: + from uavcan.node import ExecuteCommand_1 + except ImportError as ex: + from yakut.cmd.compile import make_usage_suggestion + + raise click.ClickException(make_usage_suggestion(ex.name)) from None + + request = ExecuteCommand_1.Request(command=command_parsed, parameter=parameter) + # Ensure the parameters are valid before constructing the node. + # Unfortunately, PyCyphal does not allow us to pass pre-serialized objects to a presentation-layer port instance. + # Perhaps this capability should be added one day. + _ = pycyphal.dsdl.serialize(request) + + with purser.get_node("execute_command", allow_anonymous=False) as node: + with ProgressReporter() as prog: + result = await _run( + node, + prog, + list(sorted(node_ids)) if isinstance(node_ids, set) else [node_ids], + request, + timeout=timeout, + fire_and_forget=expect is None, + ) + + success = True + + def error(text: str) -> None: + nonlocal success + success = False + show_error(text) + + if expect is not None: + for nid, resp in result.items(): + if resp is None: + error(f"Timed out while waiting for response from node {nid}") + elif resp.status not in expect: + desc = _status_code_to_name(resp.status) or "(unknown status code)" + error(f"Node {nid} returned unexpected status code: {resp.status} {desc}") + else: + _logger.debug("Success @%r: %r", nid, resp) + else: + show_warning("Responses not checked as requested") + + final: dict[int, Any] = { + nid: pycyphal.dsdl.to_builtin(resp) if resp is not None else None for nid, resp in result.items() + } + if isinstance(node_ids, int): + final = final[node_ids] + sys.stdout.write(formatter(final)) + sys.stdout.flush() + return 0 if success else EXIT_CODE_UNSUCCESSFUL + + +async def _run( + local_node: "pycyphal.application.Node", + progress: Callable[[str], None], + node_ids: Sequence[int], + request: "ExecuteCommand_1.Request", + *, + timeout: float, + fire_and_forget: bool, +) -> dict[int, Optional["ExecuteCommand_1.Response"]]: + from uavcan.node import ExecuteCommand_1 + + async def once(nid: int) -> None: + cln = local_node.make_client(ExecuteCommand_1, nid) + try: + cln.response_timeout = timeout + result[nid] = await cln(request) + finally: + cln.close() + + result: dict[int, ExecuteCommand_1.Response | None] = {} + if not fire_and_forget: + for nid in node_ids: + progress(f"{nid: 5}") + await once(nid) + else: + for r in await asyncio.gather( + *(asyncio.ensure_future(once(nid)) for nid in node_ids), + return_exceptions=True, + ): + if isinstance(r, BaseException): + raise r + # noinspection PyTypeChecker + result = dict(sorted(result.items())) + + return result + + +_DSDL_REQUEST_COMMAND_CONSTANT_PREFIX = "COMMAND_" +_DSDL_RESPONSE_STATUS_CONSTANT_PREFIX = "STATUS_" + + +def _parse_command(inp: str) -> int: + """ + This function shall not be invoked before the command handler is entered because it requires the standard DSDL + namespace to be available (or you would need to handle the DSDL not found error here). + + >>> _parse_command("0") + 0 + >>> _parse_command("0x100") + 256 + >>> _parse_command("restart") == _parse_command("ReStArT") == _parse_command("res") == 0xFFFF + True + >>> _parse_command("store_persistent_states") == 65530 + True + >>> _parse_command("no matching option") # doctest:+IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + ClickException: ... + >>> _parse_command("") # doctest:+IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + ClickException: ... + """ + for fun in (int, lambda x: int(x, 0)): + try: + return fun(inp) # type: ignore + except ValueError: + pass + + from uavcan.node import ExecuteCommand_1 + + ty = ExecuteCommand_1.Request + std = { + x[len(_DSDL_REQUEST_COMMAND_CONSTANT_PREFIX) :]: getattr(ty, x) + for x in dir(ty) + if x.startswith(_DSDL_REQUEST_COMMAND_CONSTANT_PREFIX) + } + matches = {k: v for k, v in std.items() if k.lower().startswith(inp.lower())} + if len(matches) == 1: + return int(next(iter(matches.values()))) + raise click.ClickException(f"Command not understood: {inp!r}") + + +def _status_code_to_name(code: int) -> str | None: + """ + >>> _status_code_to_name(0) + 'SUCCESS' + >>> _status_code_to_name(1) + 'FAILURE' + >>> _status_code_to_name(254) is None + True + """ + from uavcan.node import ExecuteCommand_1 + + ty = ExecuteCommand_1.Response + return { + getattr(ty, x): x[len(_DSDL_RESPONSE_STATUS_CONSTANT_PREFIX) :] + for x in dir(ty) + if x.startswith(_DSDL_RESPONSE_STATUS_CONSTANT_PREFIX) + }.get(code) diff --git a/yakut/cmd/file_server/_cmd.py b/yakut/cmd/file_server/_cmd.py index d58a4f1..855dcfa 100644 --- a/yakut/cmd/file_server/_cmd.py +++ b/yakut/cmd/file_server/_cmd.py @@ -56,15 +56,26 @@ def _validate_root_directory(ctx: click.Context, param: click.Parameter, value: """, ) @click.option( - "--update-software/--no-update-software", - "+U/-U", - default=False, - show_default=True, + "--update-software", + "-u", + type=int, + is_flag=False, + flag_value=-1, + multiple=True, + metavar="[NODE_ID]", help=f""" Check if all online nodes are running up-to-date software; request update if not. The software version is determined by invoking uavcan.node.GetInfo for every node that is online or became online (or restarted). +This option may take an optional value which shall be a node-ID, +in which case the specified node will be explicitly commanded to begin an update once regardless of its state +(even if an update is already in progress, but this is done only once to avoid infinite loop and only +if a newer software image is available), +while no other nodes will be updated. +This option can be given multiple times to select multiple nodes to update; +if it is given at least once without a value, all nodes will be considered candidates for an update. + When a node responds to uavcan.node.GetInfo, the root directory of the file server is scanned for software packages that are suitable for the node. If the node is already executing one of the available software packages or no suitable packages are found, @@ -122,7 +133,10 @@ def _validate_root_directory(ctx: click.Context, param: click.Parameter, value: @yakut.pass_purser @yakut.asynchronous(interrupted_ok=True) async def file_server( - purser: yakut.Purser, roots: list[Path], plug_and_play: Optional[str], update_software: bool + purser: yakut.Purser, + roots: list[Path], + plug_and_play: Optional[str], + update_software: tuple[int, ...], ) -> None: """ Run a standard Cyphal file server; optionally run a plug-and-play node-ID allocator and software updater. @@ -148,6 +162,9 @@ async def file_server( raise click.ClickException(make_usage_suggestion(ex.name)) + update_all_nodes = any(x < 0 for x in update_software) + explicit_nodes = set(filter(lambda x: x >= 0, update_software)) + _logger.info("Total update: %r; trigger explicitly: %r", update_all_nodes, sorted(explicit_nodes)) with purser.get_node("file_server", allow_anonymous=False) as node: node_tracker: Optional[NodeTracker] = None # Initialized lazily only if needed. @@ -184,14 +201,23 @@ def check_software_update(node_id: int, _old_entry: Optional[Entry], entry: Opti return heartbeat = entry.heartbeat assert isinstance(heartbeat, Heartbeat) + if node_id not in explicit_nodes and not update_all_nodes: + _logger.warning( + "Node %r ignored because total update mode not selected and its ID is not given explicitly " + "(explicit node-IDs are: %s)", + node_id, + ",".join(map(str, sorted(explicit_nodes))) or "(none given)", + ) + return if ( heartbeat.mode.value == heartbeat.mode.SOFTWARE_UPDATE and heartbeat.health.value < heartbeat.health.WARNING - ): + ) and node_id not in explicit_nodes: # Do not skip an update request if the health is WARNING because it indicates a problem, possibly # caused by a missing application. Details: https://github.com/OpenCyphal/yakut/issues/27 - _logger.info( - "Node %r does not require an update because it is in the software update mode already " + _logger.warning( + "Node %r does not require an update because it is in the software update mode already, " + "we are not instructed to trigger it explicitly, " "and its health is acceptable: %r", node_id, heartbeat, @@ -233,16 +259,22 @@ async def do_call() -> None: ) return _logger.info("Node %r confirmed software update command %r", node_id, cmd_request) + try: + explicit_nodes.remove(node_id) + except LookupError: + pass asyncio.create_task(do_call()) else: - _logger.info("Node %r does not require a software update.", node_id) + _logger.warning("Node %r does not require a software update.", node_id) if update_software: _logger.info("Initializing the software update checker") # The check should be run in a separate thread because on a system with slow/busy disk IO this may cause # the file server to slow down significantly because the event loop would be blocked here on disk reads. get_node_tracker().add_update_handler(check_software_update) + else: + _logger.info("Software update checker is not required") await asyncio.sleep(1e100) diff --git a/yakut/cmd/monitor/_ui.py b/yakut/cmd/monitor/_ui.py index 1cd8fcb..63dff2d 100644 --- a/yakut/cmd/monitor/_ui.py +++ b/yakut/cmd/monitor/_ui.py @@ -21,7 +21,9 @@ def refresh_screen(contents: str) -> None: click.clear() else: _TEXT_STREAM.write("\n" * 3) - _TEXT_STREAM.flush() # Synchronize clear with the following output since it is buffered separately. + # Synchronize clear with the following output since it is buffered separately. + # Note that we MUST flush stdout, not _TEXT_STREAM, because it is buffered separately. + sys.stdout.flush() click.echo(contents, file=_TEXT_STREAM, nl=False) diff --git a/yakut/cmd/publish/_cmd.py b/yakut/cmd/publish/_cmd.py index 2398c75..24f723c 100644 --- a/yakut/cmd/publish/_cmd.py +++ b/yakut/cmd/publish/_cmd.py @@ -3,7 +3,6 @@ # Author: Pavel Kirienko from __future__ import annotations -import asyncio import math from typing import Tuple, List, Sequence, Callable, Any, Dict import dataclasses @@ -80,13 +79,13 @@ def load(what: Sequence[ExpressionContextModule]) -> Dict[str, Any]: Example: publish constant messages (no embedded expressions, just regular YAML): \b - yakut pub uavcan.diagnostic.Record '{text: "Hello world!", severity: {value: 4}}' -N3 -T0.1 -P hi - yakut pub 33:uavcan.si.unit.angle.Scalar 2.31 uavcan.diagnostic.Record 'text: "2.31 radian"' + yakut pub uavcan.diagnostic.record '{text: "Hello world!", severity: {value: 4}}' -N3 -T0.1 -P hi + yakut pub 33:uavcan.si.unit.angle.scalar 2.31 uavcan.diagnostic.Record 'text: "2.31 radian"' Example: publish sinewave with frequency 1 Hz, amplitude 10 meters: \b - yakut pub -T 0.01 1234:uavcan.si.unit.length.Scalar '!$ "sin(t * pi * 2) * 10"' + yakut pub -T 0.01 1234:uavcan.si.unit.length.scalar '!$ "sin(t * pi * 2) * 10"' Example: as above, but control the frequency of the sinewave and its amplitude using sliders 10 and 11 of the first connected controller (use `yakut joystick` to find connected controllers and their axis mappings): @@ -105,7 +104,7 @@ def load(what: Sequence[ExpressionContextModule]) -> Dict[str, Any]: Example: simulate timestamped measurement of voltage affected by white noise with standard deviation 0.25 V: \b - yakut pub -T 0.1 6:uavcan.si.sample.voltage.Scalar \\ + yakut pub -T 0.1 6:uavcan.si.sample.voltage.scalar \\ '{timestamp: !$ time()*1e6, volt: !$ "A(2,10)*100+normalvariate(0,0.25)"}' """.strip() @@ -179,7 +178,7 @@ def _validate_message_spec( return [(s, f) for s, f in (value[i : i + 2] for i in range(0, len(value), 2))] # pylint: disable=R1721 -@yakut.subcommand(help=_HELP, aliases="pub") +@yakut.subcommand(help=_HELP, aliases=["pub", "p"]) @click.argument( "message", type=str, @@ -310,7 +309,6 @@ def get_subject_resolver() -> SubjectResolver: _log_final_report(node.presentation) finally: pycyphal.util.broadcast(finalizers[::-1])() - await asyncio.sleep(0.1) # let background tasks finalize before leaving the loop def _log_final_report(presentation: pycyphal.presentation.Presentation) -> None: diff --git a/yakut/cmd/register_access/__init__.py b/yakut/cmd/register_access/__init__.py new file mode 100644 index 0000000..7bf6cea --- /dev/null +++ b/yakut/cmd/register_access/__init__.py @@ -0,0 +1,5 @@ +# Copyright (c) 2022 OpenCyphal +# This software is distributed under the terms of the MIT License. +# Author: Pavel Kirienko + +from ._cmd import register_access diff --git a/yakut/cmd/register_access/_cmd.py b/yakut/cmd/register_access/_cmd.py new file mode 100644 index 0000000..47788e4 --- /dev/null +++ b/yakut/cmd/register_access/_cmd.py @@ -0,0 +1,174 @@ +# Copyright (c) 2022 OpenCyphal +# This software is distributed under the terms of the MIT License. +# Author: Pavel Kirienko + +from __future__ import annotations +import sys +from typing import Any, Sequence, TYPE_CHECKING, Optional, Callable +import click +import pycyphal +import yakut +from yakut.int_set_parser import parse_int_set, INT_SET_USER_DOC +from yakut.ui import ProgressReporter, show_error, show_warning +from yakut.param.formatter import FormatterHints +from yakut.register import explode_value, get_access_response_metadata +from yakut.util import EXIT_CODE_UNSUCCESSFUL +from ._logic import access + +if TYPE_CHECKING: + import pycyphal.application + from uavcan.register import Access_1 + +_logger = yakut.get_logger(__name__) + + +_HELP = f""" +Read or modify a register on one or multiple remote nodes. + +If no value is given, the register will be only read from the specified nodes, +and only one request will be sent per node. + +If a value is given, it shall follow the standard environment variable notation described in the +uavcan.register.Access service specification. +The assignment value will be parsed depending on the type of the register reported by the remote node +(which may be different per node). + +If a value is given, two requests will be sent: the first one to discover the type, +the second to actually write the value +(unless the provided value could not be applied to the register type). +The final value returned by the node is always printed at the end, +which may be different from the assigned value depending on the internal logic of the node +(e.g., the value could be adjusted to satisfy constraints, the register could be read-only, etc.). + +The output (but not the behavior) depends on how the node-ID set is specified: +if it's a single number, the result will be only the value of the specified register; +if multiple node-IDs are given or the sole node-ID is given in the set notation, +the output will be a map of (node_id->register_value). + +Examples: + +\b + yakut reg 125 m.inductance_dq # Outputs only the value + yakut reg 125, m.inductance_dq # Outputs {{125: value}} + yakut reg 122-126 m.inductance_dq '12.0e-6 14.7e-6' + yakut reg 122-126 m.inductance_dq 12.0e-6 14.7e-6 # Quotes are optional + y r 122-126 m.inductance_dq | jq '[.[]]|unique' # Remove duplicate values from output + y r 125 uavcan.node.description "Motor rear-left #3" + +{INT_SET_USER_DOC} +""" + + +@yakut.subcommand(aliases=["r", "reg"], help=_HELP) +@click.argument("node_ids", type=parse_int_set) +@click.argument("register_name") +@click.argument("register_value_element", nargs=-1) +@click.option( + "--timeout", + "-T", + type=float, + default=pycyphal.presentation.DEFAULT_SERVICE_REQUEST_TIMEOUT, + show_default=True, + metavar="SECONDS", + help="Service response timeout.", +) +@click.option( + "--optional-service", + "-s", + is_flag=True, + help=""" +Ignore nodes that fail to respond to the first RPC-service request instead of reporting an error +assuming that the register service is not supported. +If a node responded at least once it is assumed to support the service and any future timeout +will be always treated as error. +Best-effort output will always be produced regardless of this option; that is, it only affects the exit code. +""", +) +@click.option( + "--optional-register", + "-r", + is_flag=True, + help=""" +If modification is requested (i.e., if a value is given), +nodes that report that they don't have such register are silently ignored instead of reporting an error. +Best-effort output will always be produced regardless of this option; that is, it only affects the exit code. +""", +) +@click.option( + "--detailed", + "-d", + count=True, + help="Display the value as-is, with DSDL type information, do not simplify. Specify twice to also show metadata.", +) +@yakut.pass_purser +@yakut.asynchronous() +async def register_access( + purser: yakut.Purser, + node_ids: Sequence[int] | int, + register_name: str, + register_value_element: Sequence[str], + timeout: float, + optional_service: bool, + optional_register: bool, + detailed: int, +) -> int: + _logger.debug( + "node_ids=%r, register_name=%r, register_value_element=%r timeout=%r", + node_ids, + register_name, + register_value_element, + timeout, + ) + _logger.debug( + "optional_service=%r optional_register=%r detailed=%r", + optional_service, + optional_register, + detailed, + ) + reg_val_str = " ".join(register_value_element) if len(register_value_element) > 0 else None + del register_value_element + formatter = purser.make_formatter(FormatterHints(single_document=True)) + representer = _make_representer(simplify=detailed < 1, metadata=detailed > 1) + + with purser.get_node("register_access", allow_anonymous=False) as node: + with ProgressReporter() as prog: + result = await access( + node, + prog, + list(sorted(node_ids)) if not isinstance(node_ids, int) else [node_ids], + reg_name=register_name, + reg_val_str=reg_val_str, + optional_service=optional_service, + optional_register=optional_register, + timeout=timeout, + ) + # The node is no longer needed. + for msg in result.errors: + show_error(msg) + for msg in result.warnings: + show_warning(msg) + + final = ( + {k: representer(v) for k, v in result.value_per_node.items()} + if not isinstance(node_ids, int) + else representer(result.value_per_node[node_ids]) + ) + sys.stdout.write(formatter(final)) + sys.stdout.flush() + + return EXIT_CODE_UNSUCCESSFUL if result.errors else 0 + + +def _make_representer(simplify: bool, metadata: bool) -> Callable[[Optional["Access_1.Response"]], Any]: + def represent(response: Optional["Access_1.Response"]) -> Any: + return ( + explode_value( + response.value, + simplify=simplify, + metadata=get_access_response_metadata(response) if metadata else None, + ) + if response is not None + else None + ) + + return represent diff --git a/yakut/cmd/register_access/_logic.py b/yakut/cmd/register_access/_logic.py new file mode 100644 index 0000000..e18d307 --- /dev/null +++ b/yakut/cmd/register_access/_logic.py @@ -0,0 +1,141 @@ +# Copyright (c) 2022 OpenCyphal +# This software is distributed under the terms of the MIT License. +# Author: Pavel Kirienko + +from __future__ import annotations +import dataclasses +from typing import Sequence, TYPE_CHECKING, Union, Optional, Callable +import pycyphal +import yakut + + +if TYPE_CHECKING: + import pycyphal.application + from uavcan.register import Access_1 + + +@dataclasses.dataclass +class Result: + value_per_node: dict[int, Optional["Access_1.Response"]] = dataclasses.field(default_factory=dict) + errors: list[str] = dataclasses.field(default_factory=list) + warnings: list[str] = dataclasses.field(default_factory=list) + + +async def access( + local_node: "pycyphal.application.Node", + progress: Callable[[str], None], + node_ids: Sequence[int], + *, + reg_name: str, + reg_val_str: str | None, + optional_service: bool, + optional_register: bool, + timeout: float, +) -> Result: + res = Result() + for nid, item in (await _access(local_node, progress, node_ids, reg_name, reg_val_str, timeout=timeout)).items(): + _logger.debug("Register @%r: %r", nid, item) + res.value_per_node[nid] = None # Error state is default state + if isinstance(item, _NoService): + if optional_service: + res.warnings.append(f"Service not accessible at node {nid}, ignoring as requested") + else: + res.errors.append(f"Service not accessible at node {nid}") + + elif isinstance(item, _Timeout): + res.errors.append(f"Request to node {nid} has timed out") + + elif isinstance(item, tuple): + resp, exc = item + res.value_per_node[nid] = resp + res.errors.append(f"Assignment failed at node {nid}: {type(exc).__name__}: {exc}") + + else: + res.value_per_node[nid] = item + if item.value.empty and reg_val_str is not None: + if optional_register: + res.warnings.append(f"Nonexistent register {reg_name!r} at node {nid} ignored as requested") + else: + res.errors.append(f"Cannot assign nonexistent register {reg_name!r} at node {nid}") + return res + + +class _NoService: + pass + + +class _Timeout: + pass + + +async def _access( + local_node: pycyphal.application.Node, + progress: Callable[[str], None], + node_ids: Sequence[int], + reg_name: str, + reg_val_str: str | None, + *, + timeout: float, +) -> dict[ + int, + Union[ + _NoService, + _Timeout, + "Access_1.Response", + tuple["Access_1.Response", "pycyphal.application.register.ValueConversionError"], + ], +]: + from uavcan.register import Access_1 + + out: dict[ + int, + Access_1.Response | _NoService | _Timeout | pycyphal.application.register.ValueConversionError, + ] = {} + for nid in node_ids: + progress(f"{nid: 5}: {reg_name!r}") + cln = local_node.make_client(Access_1, nid) + try: + cln.response_timeout = timeout + out[nid] = await _access_one(cln, reg_name, reg_val_str) + finally: + cln.close() + return out + + +async def _access_one( + client: pycyphal.presentation.Client["Access_1"], + reg_name: str, + reg_val_str: str | None, +) -> Union[ + _NoService, + _Timeout, + "Access_1.Response", + tuple["Access_1.Response", "pycyphal.application.register.ValueConversionError"], +]: + from uavcan.register import Access_1, Name_1 + from pycyphal.application.register import ValueProxy, ValueConversionError + + resp = await client(Access_1.Request(name=Name_1(reg_name))) + if resp is None: + return _NoService() + assert isinstance(resp, Access_1.Response) + if reg_val_str is None or resp.value.empty: # Modification is not required or there is no such register. + return resp + + # Coerce the supplied value to the type of the remote register. + assert not resp.value.empty + val = ValueProxy(resp.value) + try: + val.assign_environment_variable(reg_val_str) + except ValueConversionError as ex: # Oops, not coercible (e.g., register is float[], value is string) + return resp, ex + + # Write the coerced value to the node; it may also modify it so return the response, not the coercion result. + resp = await client(Access_1.Request(name=Name_1(reg_name), value=val.value)) + if resp is None: # We got a response before but now we didn't, something is messed up so the result is different. + return _Timeout() + assert isinstance(resp, Access_1.Response) + return resp + + +_logger = yakut.get_logger(__name__) diff --git a/yakut/cmd/register_batch/__init__.py b/yakut/cmd/register_batch/__init__.py new file mode 100644 index 0000000..bbed969 --- /dev/null +++ b/yakut/cmd/register_batch/__init__.py @@ -0,0 +1,5 @@ +# Copyright (c) 2022 OpenCyphal +# This software is distributed under the terms of the MIT License. +# Author: Pavel Kirienko + +from ._cmd import register_batch diff --git a/yakut/cmd/register_batch/_caller.py b/yakut/cmd/register_batch/_caller.py new file mode 100644 index 0000000..30c8d75 --- /dev/null +++ b/yakut/cmd/register_batch/_caller.py @@ -0,0 +1,121 @@ +# Copyright (c) 2022 OpenCyphal +# This software is distributed under the terms of the MIT License. +# Author: Pavel Kirienko + +from __future__ import annotations +import dataclasses +from typing import TYPE_CHECKING, Union, Callable +import pycyphal +import yakut +from ._directive import Directive, RegisterDirective + +if TYPE_CHECKING: + import pycyphal.application + from uavcan.register import Access_1 + + +@dataclasses.dataclass(frozen=True) +class Tag: + def __eq__(self, other: object) -> bool: + """ + >>> TypeCoercionFailure("foo") == TypeCoercionFailure("bar") + True + >>> TypeCoercionFailure("") == Timeout() + False + """ + return issubclass(type(self), type(other)) and issubclass(type(other), type(self)) + + +@dataclasses.dataclass(frozen=True) +class TypeCoercionFailure(Tag): + msg: str + + __eq__ = Tag.__eq__ + + +@dataclasses.dataclass(frozen=True) +class Timeout(Tag): + pass + + +@dataclasses.dataclass(frozen=True) +class Skipped(Tag): + pass + + +@dataclasses.dataclass +class Result: + responses_per_node: dict[int, dict[str, Union[Tag, "Access_1.Response"]]] = dataclasses.field(default_factory=dict) + """ + Keys of the innermost dict are always the same as those in the directive regardless of success. + Processing of a node stops at first timeout, further items set to Skipped. + Empty value means that there is no such register. + """ + + +async def do_calls( + local_node: "pycyphal.application.Node", + progress: Callable[[str], None], + *, + timeout: float, + directive: Directive, +) -> Result: + from uavcan.register import Access_1 + + out = Result() + for node_id, node_dir in directive.registers_per_node.items(): + cln = local_node.make_client(Access_1, node_id) + try: + cln.response_timeout = timeout + responses: dict[str, Access_1.Response | Tag] = {k: Skipped() for k in node_dir} + for idx, (reg_name, reg_dir) in enumerate(node_dir.items()): + progress(f"{node_id: 5}: {reg_name!r}") + resp = await _process_one(cln, reg_name, reg_dir) + responses[reg_name] = resp + _logger.info("Result for %r@%r (%r/%r): %r", reg_name, node_id, idx + 1, len(node_dir), resp) + if isinstance(resp, Timeout): + break + finally: + cln.close() + out.responses_per_node[node_id] = responses + + assert out.responses_per_node.keys() == directive.registers_per_node.keys() + assert all( + out.responses_per_node[node_id].keys() == directive.registers_per_node[node_id].keys() + for node_id in directive.registers_per_node + ) + return out + + +async def _process_one( + client: pycyphal.presentation.Client["Access_1"], + register_name: str, + directive: RegisterDirective, +) -> Union[TypeCoercionFailure, Timeout, "Access_1.Response"]: + from pycyphal.application.register import Value + from uavcan.register import Access_1, Name_1 + + # Construct the request object. If we are given the full type information then we can do the job in one query. + req = Access_1.Request(name=Name_1(register_name), value=directive if isinstance(directive, Value) else None) + + # Go through the network. Empty response means there is no such register, no point proceeding. + resp = await client(req) + if resp is None: + return Timeout() + assert isinstance(resp, Access_1.Response) + if resp.value.empty or isinstance(directive, Value): + return resp + + # Perform type coercion to the discovered type. + assert callable(directive) + coerced = directive(resp.value) + if coerced is None: + return TypeCoercionFailure(f"Value not coercible to {resp.value}") + + # Send the write request with the updated coerced value. + req = Access_1.Request(name=Name_1(register_name), value=coerced) + assert isinstance(req.value, Value) + return await client(req) or Timeout() + + +_logger = yakut.get_logger(__name__) diff --git a/yakut/cmd/register_batch/_cmd.py b/yakut/cmd/register_batch/_cmd.py new file mode 100644 index 0000000..483dd7c --- /dev/null +++ b/yakut/cmd/register_batch/_cmd.py @@ -0,0 +1,239 @@ +# Copyright (c) 2022 OpenCyphal +# This software is distributed under the terms of the MIT License. +# Author: Pavel Kirienko + +from __future__ import annotations +import logging +import sys +from typing import TYPE_CHECKING, TextIO, Union, Callable, Any +import click +import pycyphal +import yakut +from yakut.ui import ProgressReporter, show_error, show_warning +from yakut.param.formatter import FormatterHints +from yakut.yaml import Loader +from yakut.int_set_parser import parse_int_set, INT_SET_USER_DOC +from yakut.register import explode_value, get_access_response_metadata +from yakut.util import EXIT_CODE_UNSUCCESSFUL +from ._directive import Directive, SCHEMA_USER_DOC +from ._caller import do_calls, TypeCoercionFailure, Timeout, Skipped, Tag + +if TYPE_CHECKING: + import pycyphal.application + from uavcan.register import Access_1 + +_logger = yakut.get_logger(__name__) + + +Predicate = Callable[["Access_1.Response"], bool] + + +_PREDICATES = { + "m": lambda r: r if r.mutable else None, + "i": lambda r: r if not r.mutable else None, + "p": lambda r: r if r.persistent else None, + "v": lambda r: r if not r.persistent else None, + "mp": lambda r: r if r.mutable and r.persistent else None, + "mv": lambda r: r if r.mutable and not r.persistent else None, + "ip": lambda r: r if not r.mutable and r.persistent else None, + "iv": lambda r: r if not r.mutable and not r.persistent else None, +} + + +_HELP = f""" +Read/write multiple registers at multiple nodes (useful for network configuration management). + +Accepts a YAML/JSON file containing register names and/or values per node; the default is to read from stdin. +Acceptable formats are generated either by register-list (in which case registers will be only read) +or by this command (in which case the specified values will be written). + +The registers will be processed strictly sequentially in the order they are specified +(this matters if register access has side effects). + +Save configuration parameters into a file (using verbose form for clarity here): + +\b + yakut register-list 125 | yakut register-batch 125 --only=mp > pure_config.json + yakut register-list 125, | yakut register-batch --only=mp > node_125_config.json + yakut register-list 120-128 | yakut register-batch --only=mp > network_config.json + +To get human-friendly output either add --format=yaml to the last command, or pipe the output through "jq". + +You can also remove the node-ID keys using jq '.[]' (this may be useful if you already have an existing file): + +\b + cat node_125_config.json | jq '.[]' + +Apply the same parameters to nodes 10,11,12,13,14: + +\b + cat pure_config.json | y rb 10-15 + +You can also convert a pure config file that is not keyed by node-ID by adding the IDs using jq +(this is obviously not intended for interactive use): + +\b + cat pure_config.json | \\ + jq '. as $in | [range(10;15) | {{key: .|tostring, value: $in}}] | from_entries' | \\ + y rb + +...same but applied to one node 125: + +\b + cat pure_config.json | jq '{{"125": .}}' | y rb + +Filter output registers by name preserving the node-ID keys; in this case those matching "uavcan*id": + +\b + y rl 125, | y rb | jq 'map_values(with_entries(select(.key | test("uavcan.+id"))))' + +Read diagnostic registers from two similar nodes +(protip: save the register names into a file instead of calling register-list each time): + +\b + y rl 124 | y rb 124,125 -oiv + +{INT_SET_USER_DOC} +""" + + +@yakut.subcommand(aliases=["rbat", "rb"], help=_HELP) +@click.argument( + "node_ids", + required=False, + type=lambda x: parse_int_set(x) if x is not None else None, +) +@click.option( + "--file", + "-f", + type=click.File("r"), + default=sys.stdin, + help=f""" +Defaults to stdin. Supports YAML/JSON. + +{SCHEMA_USER_DOC} +""", +) +@click.option( + "--timeout", + "-T", + type=float, + default=pycyphal.presentation.DEFAULT_SERVICE_REQUEST_TIMEOUT, + show_default=True, + metavar="SECONDS", + help="Service response timeout.", +) +@click.option( + "--optional-register", + "-r", + is_flag=True, + help=""" +If modification is requested (i.e., if a value is given), +nodes that report that they don't have such register are silently ignored instead of reporting an error. +Best-effort output will always be produced regardless of this option; that is, it only affects the exit code. +""", +) +@click.option( + "--detailed", + "-d", + count=True, + help="Display the values as-is, with DSDL type information, do not simplify. Specify twice to include metadata.", +) +@click.option( + "--only", + "-o", + type=click.Choice(list(_PREDICATES.keys()), case_sensitive=False), + help=""" +Filter the output to include only mutable/immutable, persistent/volatile registers. +All registers are always written regardless, this option only affects the final output. +""", +) +@yakut.pass_purser +@yakut.asynchronous() +async def register_batch( + purser: yakut.Purser, + node_ids: set[int] | int | None, + file: TextIO, + timeout: float, + optional_register: bool, + detailed: int, + only: str | None, +) -> int: + predicate: Predicate = _PREDICATES[only] if only else lambda _: True + formatter = purser.make_formatter(FormatterHints(single_document=True)) + representer = _make_representer(detail=detailed) + with file: + directive = Directive.load( + Loader().load(file), + node_ids=node_ids if not isinstance(node_ids, int) else {node_ids}, + ) + _logger.debug("Loaded directive: %r", directive) + with purser.get_node("register_batch", allow_anonymous=False) as node: + with ProgressReporter() as prog: + result = await do_calls(node, prog, directive=directive, timeout=timeout) + if _logger.isEnabledFor(logging.INFO): + _logger.info("%s", node.presentation.transport.sample_statistics()) + + from uavcan.register import Access_1 + + success = True + + def error(msg: str) -> None: + nonlocal success + success = False + show_error(msg) + + for node_id, per_node in result.responses_per_node.items(): + failed_count = 0 + for reg_name, response in per_node.items(): + if isinstance(response, Access_1.Response) and not response.value.empty: + continue + failed_count += 1 + prefix = f"{node_id}:{reg_name!r}: " + if isinstance(response, Access_1.Response) and response.value.empty: + if optional_register: + show_warning(prefix + "No such register, ignored as requested") + else: + error(prefix + "No such register") + elif isinstance(response, TypeCoercionFailure): + error(prefix + f"Original value unchanged because coercion failed: {response.msg}") + elif isinstance(response, Timeout): + error(prefix + "Timed out and gave up on this node") + elif isinstance(response, Skipped): + assert not success + else: + assert False, response + _logger.info("Node %r: total %r, failed %r", node_id, len(per_node), failed_count) + if failed_count > 0: + show_warning(f"{node_id}: {failed_count} failed of {len(per_node)} total. Output incomplete.") + + final: Any = { + node_id: { + reg_name: representer(response) + for reg_name, response in per_node.items() + if isinstance(response, Access_1.Response) and predicate(response) + } + for node_id, per_node in result.responses_per_node.items() + } + if isinstance(node_ids, int): + final = final[node_ids] + sys.stdout.write(formatter(final)) + sys.stdout.flush() + return 0 if success else EXIT_CODE_UNSUCCESSFUL + + +def _make_representer(detail: int) -> Callable[[Union[Tag, "Access_1.Response"]], Any]: + from uavcan.register import Access_1 + + def represent(response: Union[Tag, Access_1.Response]) -> Any: + return ( + explode_value( + response.value, + simplify=detail < 1, + metadata=get_access_response_metadata(response) if detail > 1 else None, + ) + if isinstance(response, Access_1.Response) + else None + ) + + return represent diff --git a/yakut/cmd/register_batch/_directive.py b/yakut/cmd/register_batch/_directive.py new file mode 100644 index 0000000..1245e6d --- /dev/null +++ b/yakut/cmd/register_batch/_directive.py @@ -0,0 +1,187 @@ +# Copyright (c) 2022 OpenCyphal +# This software is distributed under the terms of the MIT License. +# Author: Pavel Kirienko + +from __future__ import annotations +import dataclasses +from typing import TYPE_CHECKING, Any, Union, Callable, Optional, Iterable, Mapping, Sequence +import yakut +from yakut.register import unexplode_value + +if TYPE_CHECKING: + from pycyphal.application.register import Value + + +Unexploder = Callable[["Value"], Optional["Value"]] +""" +Returns None if the value is not coercible to the prototype. +""" + +RegisterDirective = Union["Value", Unexploder] + + +class InvalidDirectiveError(ValueError): + pass + + +SCHEMA_USER_DOC = """ +If node-IDs are given explicitly, the following schemas are accepted: + +\b + [register_name] + {register_name->register_value} + +If node-IDs are not given, they shall be contained in the input: + +\b + {node_id->[register_name]} + {node_id->{register_name->register_value}} +""" + + +@dataclasses.dataclass(frozen=True) +class Directive: + registers_per_node: dict[int, dict[str, RegisterDirective]] + """ + The directive contains either values or factories that take a prototype and return the value (or None on error). + When a value is available, it is to be written immediately in one request. + Otherwise, a read request will need to be executed first to discover the type of the register; + the factory will then use that information to perform coercion (which may fail). + """ + + @staticmethod + def load(ast: Any, node_ids: Iterable[int] | None) -> Directive: + node_ids = list(sorted(node_ids)) if node_ids is not None else None + if node_ids is not None: + ast = {n: ast for n in node_ids} + _logger.debug("Decorated: %r", ast) + + if isinstance(ast, Mapping): + registers_per_node: dict[int, dict[str, RegisterDirective]] = {} + for node_id_orig, node_spec in ast.items(): + try: + nid = int(node_id_orig) + except ValueError: + raise InvalidDirectiveError(f"Not a valid node-ID: {node_id_orig}") from None + nd = _load_node(node_spec) + registers_per_node[nid] = nd + _logger.debug("Loaded node directive for %d: %r", nid, nd) + return Directive(registers_per_node=registers_per_node) + + if ast is None: + _logger.warning("Empty directive, nothing to do") + return Directive({}) + + raise InvalidDirectiveError(f"Invalid directive: expected mapping (node_id->...), found {type(ast).__name__}") + + +def _load_node(ast: Any) -> dict[str, RegisterDirective]: + from pycyphal.application.register import Value + + if isinstance(ast, Sequence) and all(isinstance(x, str) for x in ast): + return {str(x): Value() for x in ast} + + if isinstance(ast, Mapping) and all(isinstance(x, str) for x in ast.keys()): + return {reg_name: _load_leaf(reg_spec) for reg_name, reg_spec in ast.items()} + + if ast is None: + return {} + + raise InvalidDirectiveError( + f"Invalid node specifier: expected [register_name] or (register_name->register_value) or null; " + f"found {type(ast).__name__}" + ) + + +def _load_leaf(exploded: Any) -> RegisterDirective: + return unexplode_value(exploded) or (lambda proto: unexplode_value(exploded, proto)) + + +_logger = yakut.get_logger(__name__) + + +def _unittest_directive() -> None: + from pytest import raises + from pycyphal.application.register import Value, String, Integer32, Bit + + class CV(Value): # type: ignore + def __eq__(self, other: object) -> bool: + if isinstance(other, Value): + return repr(self) == repr(other) + return NotImplemented + + # Load full form + dr = Directive.load( + { + "0": { + "a": {"string": {"value": "z"}, "_meta_": {"ignored": "very ignored"}}, + }, + 1: { + "b": {"string": {"value": "y"}}, + "c": None, + "d": {"empty": {}}, + }, + " 2 ": ["e", "f"], + 3: None, + }, + node_ids=None, + ) + assert dr.registers_per_node == { + 0: {"a": CV(string=String("z"))}, + 1: {"b": CV(string=String("y")), "c": CV(), "d": CV()}, + 2: {"e": CV(), "f": CV()}, + 3: {}, + } + + # Load full form with explicit node-IDs + dr = Directive.load( + { + "a": {"string": {"value": "z"}}, + "b": None, + }, + node_ids=[0, 1], + ) + assert dr.registers_per_node == { + 0: {"a": CV(string=String("z")), "b": CV()}, + 1: {"a": CV(string=String("z")), "b": CV()}, + } + + # Load names only with explicit node-IDs + dr = Directive.load( + ["a", "b"], + node_ids=[0, 1], + ) + assert dr.registers_per_node == { + 0: {"a": CV(), "b": CV()}, + 1: {"a": CV(), "b": CV()}, + } + + # Load simplified form, deferred callables returned. + dr = Directive.load( + { + "0": { + "a": [0, 1, 2], + "b": 456, + }, + }, + node_ids=None, + ) + assert dr and len(dr.registers_per_node) == 1 + assert {"a", "b"} == dr.registers_per_node[0].keys() + uxp = dr.registers_per_node[0]["a"] + assert callable(uxp) + assert CV(integer32=Integer32([0, 1, 2])) == uxp(Value(integer32=Integer32([0] * 3))) + assert CV(bit=Bit([False, True, True])) == uxp(Value(bit=Bit([False] * 3))) + uxp = dr.registers_per_node[0]["b"] + assert callable(uxp) + assert CV(integer32=Integer32([456])) == uxp(Value(integer32=Integer32([0]))) + + # Errors. + with raises(InvalidDirectiveError): + Directive.load([], node_ids=None) + with raises(InvalidDirectiveError): + Directive.load("", node_ids=None) + with raises(InvalidDirectiveError): + Directive.load({"z": []}, node_ids=None) + with raises(InvalidDirectiveError): + Directive.load({"z": "q"}, node_ids=None) diff --git a/yakut/cmd/register_list/__init__.py b/yakut/cmd/register_list/__init__.py new file mode 100644 index 0000000..d6e1457 --- /dev/null +++ b/yakut/cmd/register_list/__init__.py @@ -0,0 +1,5 @@ +# Copyright (c) 2022 OpenCyphal +# This software is distributed under the terms of the MIT License. +# Author: Pavel Kirienko + +from ._cmd import register_list diff --git a/yakut/cmd/register_list/_cmd.py b/yakut/cmd/register_list/_cmd.py new file mode 100644 index 0000000..7cf83fd --- /dev/null +++ b/yakut/cmd/register_list/_cmd.py @@ -0,0 +1,107 @@ +# Copyright (c) 2022 OpenCyphal +# This software is distributed under the terms of the MIT License. +# Author: Pavel Kirienko + +from __future__ import annotations +import sys +from typing import TYPE_CHECKING +import click +import pycyphal +import yakut +from yakut.int_set_parser import parse_int_set, INT_SET_USER_DOC +from yakut.ui import ProgressReporter, show_error, show_warning +from yakut.param.formatter import FormatterHints +from yakut.util import EXIT_CODE_UNSUCCESSFUL +from ._logic import list_names + +if TYPE_CHECKING: + import pycyphal.application + +_logger = yakut.get_logger(__name__) + + +_HELP = f""" +List registers available on the specified remote node(s). + +If the specified node-ID is a single integer, then the output contains simply the list of register names on that node. + +If the specified node-ID is a set of integers (or a single integer followed by element separator, like `123,`), +then the output is a mapping (node_id->[register_name]). + +The output can be piped to another command like yakut register-batch. + +Examples: + +\b + y rl 42 > registers.json + y rl 90,100..125,!110-115 | y rb > registers_all.json + y rl '[1,2,42,105]' | jq > register_names_all.json + +Filter by name, in this case those matching "uavcan*id" (mind the comma!): + +\b + y rl 125 | jq 'map(select(test("uavcan.+id")))' + y rl 125, | jq 'map_values([.[] | select(test("uavcan.+id"))])' + +Compute intersection -- registers that are available in all of the queried nodes: + +\b + y rl 120-128 | jq '. as $in|reduce .[] as $item ($in|flatten|flatten;.-(.-$item))|unique' + +{INT_SET_USER_DOC} +""" + + +@yakut.subcommand(aliases="rl", help=_HELP) +@click.argument("node_ids", type=parse_int_set) +@click.option( + "--timeout", + "-T", + type=float, + default=pycyphal.presentation.DEFAULT_SERVICE_REQUEST_TIMEOUT, + show_default=True, + metavar="SECONDS", + help="Service response timeout.", +) +@click.option( + "--optional-service", + "-s", + is_flag=True, + help=""" +Ignore nodes that fail to respond to the first RPC-service request instead of reporting an error +assuming that the register service is not supported. +If a node responded at least once it is assumed to support the service and any future timeout +will be always treated as an error. +""", +) +@yakut.pass_purser +@yakut.asynchronous() +async def register_list( + purser: yakut.Purser, + node_ids: set[int] | int, + timeout: float, + optional_service: bool, +) -> int: + _logger.debug("node_ids=%r, timeout=%r optional_service=%r", node_ids, timeout, optional_service) + node_ids_list = list(sorted(node_ids)) if isinstance(node_ids, set) else [node_ids] + assert isinstance(node_ids_list, list) and all(isinstance(x, int) for x in node_ids_list) + formatter = purser.make_formatter(FormatterHints(single_document=True)) + with purser.get_node("register_list", allow_anonymous=False) as node: + with ProgressReporter() as prog: + result = await list_names( + node, + prog, + node_ids_list, + optional_service=optional_service, + timeout=timeout, + ) + # The node is no longer needed. + for msg in result.errors: + show_error(msg) + for msg in result.warnings: + show_warning(msg) + final = result.names_per_node if not isinstance(node_ids, int) else result.names_per_node[node_ids] + sys.stdout.write(formatter(final)) + sys.stdout.flush() + + return EXIT_CODE_UNSUCCESSFUL if result.errors else 0 diff --git a/yakut/cmd/register_list/_logic.py b/yakut/cmd/register_list/_logic.py new file mode 100644 index 0000000..1ae66d1 --- /dev/null +++ b/yakut/cmd/register_list/_logic.py @@ -0,0 +1,95 @@ +# Copyright (c) 2022 OpenCyphal +# This software is distributed under the terms of the MIT License. +# Author: Pavel Kirienko + +from __future__ import annotations +import dataclasses +from typing import Sequence, TYPE_CHECKING, Callable +import bisect +import yakut + +if TYPE_CHECKING: + import pycyphal.application + + +@dataclasses.dataclass +class Result: + names_per_node: dict[int, list[str] | None] = dataclasses.field(default_factory=dict) + errors: list[str] = dataclasses.field(default_factory=list) + warnings: list[str] = dataclasses.field(default_factory=list) + + +async def list_names( + local_node: "pycyphal.application.Node", + progress: Callable[[str], None], + node_ids: Sequence[int], + *, + optional_service: bool, + timeout: float, +) -> Result: + res = Result() + for nid, names in (await _impl_list_names(local_node, progress, node_ids, timeout=timeout)).items(): + _logger.debug("Names @%r: %r", nid, names) + if isinstance(names, _NoService): + res.names_per_node[nid] = None + if optional_service: + res.warnings.append(f"Register list service is not accessible at node {nid}, ignoring as requested") + else: + res.errors.append(f"Register list service is not accessible at node {nid}") + else: + lst = res.names_per_node.setdefault(nid, []) + assert isinstance(lst, list) + for idx, n in enumerate(names): + if isinstance(n, _Timeout): + res.errors.append(f"Request #{idx} to node {nid} has timed out, data incomplete") + else: + bisect.insort(lst, n) + return res + + +class _NoService: + pass + + +class _Timeout: + pass + + +async def _impl_list_names( + local_node: "pycyphal.application.Node", + progress: Callable[[str], None], + node_ids: Sequence[int], + *, + timeout: float, +) -> dict[int, list[str | _Timeout] | _NoService]: + from uavcan.register import List_1 + + out: dict[int, list[str | _Timeout] | _NoService] = {} + for nid in node_ids: + cln = local_node.make_client(List_1, nid) + try: + cln.response_timeout = timeout + name_list: list[str | _Timeout] | _NoService = [] + for idx in range(2**16): + progress(f"{nid: 5}: {idx: 5}") + resp = await cln(List_1.Request(index=idx)) + assert isinstance(name_list, list) + if resp is None: + if 0 == idx: # First request timed out, assume service not supported or node is offline + name_list = _NoService() + else: # Non-first request has timed out, assume network error + name_list.append(_Timeout()) + break + assert isinstance(resp, List_1.Response) + name = resp.name.name.tobytes().decode(errors="replace") + if not name: + break + name_list.append(name) + finally: + cln.close() + _logger.debug("Register names fetched from node %r: %r", nid, name_list) + out[nid] = name_list + return out + + +_logger = yakut.get_logger(__name__) diff --git a/yakut/cmd/subscribe/_cmd.py b/yakut/cmd/subscribe/_cmd.py index 3cf8e6f..eb30190 100644 --- a/yakut/cmd/subscribe/_cmd.py +++ b/yakut/cmd/subscribe/_cmd.py @@ -5,7 +5,6 @@ from __future__ import annotations import sys import math -import asyncio from typing import Any, Sequence, TYPE_CHECKING, Callable, Iterable import logging from functools import lru_cache @@ -19,7 +18,6 @@ from yakut.util import convert_transfer_metadata_to_builtin from yakut.subject_specifier_processor import process_subject_specifier, SubjectResolver from ._sync import Synchronizer, SynchronizerFactory -from ._sync_unary import make_sync_unary if TYPE_CHECKING: import pycyphal.application @@ -51,16 +49,9 @@ def set_synchronizer_factory(self, val: SynchronizerFactory) -> None: def get_synchronizer_factory(self) -> SynchronizerFactory: if self._synchronizer_factory is None: - from ._sync_monoclust import make_sync_monoclust - from pycyphal.presentation.subscription_synchronizer import get_local_reception_timestamp - - self.set_synchronizer_factory( - lambda subs: make_sync_monoclust( - subs, - f_key=get_local_reception_timestamp, - tolerance_minmax=SYNC_MONOCLUST_TOLERANCE_MINMAX_TS_ARRIVAL, - ) - ) + from ._sync_async import make_sync_async + + self.set_synchronizer_factory(make_sync_async) assert self._synchronizer_factory is not None return self._synchronizer_factory @@ -128,14 +119,14 @@ def _handle_option_synchronizer_transfer_id(ctx: click.Context, _param: click.Pa ctx.ensure_object(Config).set_synchronizer_factory(make_sync_transfer_id) -@yakut.subcommand(aliases="sub") +@yakut.subcommand(aliases=["sub", "s"]) @click.argument("subject", type=str, nargs=-1) @click.option( "--with-metadata/--no-metadata", "+M/-M", default=False, show_default=True, - help="When enabled, each message object is prepended with an extra field named `_metadata_`.", + help="When enabled, each message object is prepended with an extra field named `_meta_`.", ) @click.option( "--count", @@ -151,11 +142,11 @@ def _handle_option_synchronizer_transfer_id(ctx: click.Context, _param: click.Pa "--no-scroll", "-R", is_flag=True, - help="Clear terminal output before printing output. This option only has effect if stdout is a tty.", + help="Clear terminal before printing output. This option only has effect if stdout is a tty.", ) @click.option( - "--sync-monoclust", - "--sync-mc", + "--sync-monoclust-field", + "--smcf", callback=_handle_option_synchronizer_monoclust_timestamp_field, expose_value=False, type=float, @@ -169,7 +160,8 @@ def _handle_option_synchronizer_transfer_id(ctx: click.Context, _param: click.Pa ) @click.option( "--sync-monoclust-arrival", - "--sync-mca", + "--smca", + "--sync", # This is currently the default because it works with any data type. callback=_handle_option_synchronizer_monoclust_timestamp_arrival, expose_value=False, type=float, @@ -177,13 +169,13 @@ def _handle_option_synchronizer_transfer_id(ctx: click.Context, _param: click.Pa flag_value=float("nan"), help=f""" Use the monotonic clustering synchronizer with the local arrival timestamp as the clustering key. -Works with all data types but may perform poorly depending on the timing and system latency. +Works with all data types but may perform poorly depending on the local timestamping accuracy and system latency. The optional value is the synchronization tolerance in seconds; autodetect if not specified. """, ) @click.option( "--sync-transfer-id", - "--sync-tid", + "--stid", callback=_handle_option_synchronizer_transfer_id, expose_value=False, is_flag=True, @@ -210,6 +202,7 @@ async def subscribe( the subject-ID may be omitted if the data type defines a fixed one; or the type can be omitted to engage automatic type discovery (discovery may fail if the local node is anonymous as it will be unable to issue RPC-service requests). + The short data type name is case-insensitive for convenience. The accepted forms are: \b @@ -217,21 +210,28 @@ async def subscribe( TYPE_NAME[.MAJOR[.MINOR]] SUBJECT_ID - If multiple subjects are specified, a synchronous subscription will be used. - It is intended for subscribing to a group of coupled subjects like lockstep sensor feeds or other coupled objects. + The output documents (YAML/JSON/etc depending on the chosen format) will contain one message object each, + unless synchronization is enabled via --sync-*. + In that case, each output document will contain a complete synchronized group of messages (ordering retained). + Synchronous subscription is intended for subscribing to a group of coupled subjects like coupled sensor feeds. More on subscription synchronization is available in the PyCyphal docs. - If no synchronizer is specified, Yakut will choose one automatically. - Each received object or synchronized group is emitted to stdout as a key-value mapping, - where the number of elements equals the number of subjects the command is asked to subscribe to; - the keys are subject-IDs and values are the received message objects. + Each received message or synchronized group is emitted to stdout as a key-value mapping, + where the keys are subject-IDs and values are the received message objects. Examples: \b - yakut sub 33:uavcan.si.unit.angle.Scalar --with-metadata --count=1 + yakut sub 33:uavcan.si.unit.angle.scalar --with-metadata --count=1 yakut sub 33 42 5789 --sync-monoclust-arrival=0.1 - yakut sub uavcan.node.Heartbeat + yakut sub uavcan.node.heartbeat + y sub 1220 1230 1240 1250 --sync-monoclust + + Extracting a specific field, sub-object, data manipulation: + + \b + y sub uavcan.node.heartbeat | jq '.[].health.value' + y sub uavcan.node.heartbeat +M | jq '.[]|[._meta_.transfer_id, ._meta_.timestamp.system]' """ config = click.get_current_context().ensure_object(Config) try: @@ -261,13 +261,10 @@ def get_node() -> Node: subscribers: list[Subscriber[Any]] = await _make_subscribers(subject, get_node) finalizers += [s.close for s in subscribers] - if len(subscribers) > 1: - synchronizer = config.get_synchronizer_factory()(subscribers) - elif len(subscribers) == 1: - synchronizer = make_sync_unary([subscribers[0]]) - else: + if len(subscribers) == 0: _logger.warning("Nothing to do because no subjects are specified") return + synchronizer = config.get_synchronizer_factory()(subscribers) # The node is closed through the finalizers at exit. # Note that we can't close the node before closing the subscribers to avoid resource errors inside PyCyphal. @@ -284,7 +281,6 @@ def get_node() -> Node: _logger.info("% 4s: %s", sub.port_id, sub.sample_statistics()) finally: pycyphal.util.broadcast(finalizers[::-1])() - await asyncio.sleep(0.1) # let background tasks finalize before leaving the loop async def _make_subscribers( @@ -316,33 +312,24 @@ def get_resolver() -> SubjectResolver: async def _run(synchronizer: Synchronizer, formatter: Formatter, with_metadata: bool, count: int, redraw: bool) -> None: - metadata_cache: dict[object, dict[str, Any]] = {} - - def get_extra_metadata(sub: Subscriber[Any]) -> dict[str, Any]: - try: - return metadata_cache[sub] - except LookupError: # This may be expensive so we only do it once. - model = pycyphal.dsdl.get_model(sub.dtype) - metadata_cache[sub] = { - "dtype": str(model), - } - return metadata_cache[sub] - - def process_group(group: tuple[tuple[tuple[Any, TransferFrom], Subscriber[Any]], ...]) -> None: + def process_group(group: tuple[tuple[tuple[Any, TransferFrom] | None, Subscriber[Any]], ...]) -> None: nonlocal count outer: dict[int, dict[str, Any]] = {} - # noinspection PyTypeChecker - for (msg, meta), subscriber in group: + for maybe_msg_meta, subscriber in group: + if maybe_msg_meta is None: + continue # Asynchronous mode. + msg, meta = maybe_msg_meta assert isinstance(meta, TransferFrom) and isinstance(subscriber, Subscriber) bi: dict[str, Any] = {} # We use updates to ensure proper dict ordering: metadata before data if with_metadata: - bi.update(convert_transfer_metadata_to_builtin(meta, **get_extra_metadata(subscriber))) + bi.update(convert_transfer_metadata_to_builtin(meta, dtype=subscriber.dtype)) bi.update(pycyphal.dsdl.to_builtin(msg)) outer[subscriber.port_id] = bi if redraw: click.clear() - print(formatter(outer)) # Use print to properly handle end-of-line for both TTY and files on all platforms + sys.stdout.write(formatter(outer)) + sys.stdout.flush() count -= 1 if count <= 0: _logger.debug("Reached the specified synchronized group count, stopping") diff --git a/yakut/cmd/subscribe/_sync.py b/yakut/cmd/subscribe/_sync.py index 2a82943..aab02f2 100644 --- a/yakut/cmd/subscribe/_sync.py +++ b/yakut/cmd/subscribe/_sync.py @@ -4,12 +4,20 @@ from __future__ import annotations -from typing import Any, Callable, Awaitable, Iterable, Tuple +from typing import Any, Callable, Awaitable, Iterable, Tuple, Optional import pycyphal SynchronizerOutput = Callable[ - [Tuple[Tuple[Tuple[Any, pycyphal.transport.TransferFrom], pycyphal.presentation.Subscriber[Any]], ...]], + [ + Tuple[ + Tuple[ + Optional[Tuple[Any, pycyphal.transport.TransferFrom]], + pycyphal.presentation.Subscriber[Any], + ], + ..., + ] + ], None, ] diff --git a/yakut/cmd/subscribe/_sync_async.py b/yakut/cmd/subscribe/_sync_async.py new file mode 100644 index 0000000..c3a59d0 --- /dev/null +++ b/yakut/cmd/subscribe/_sync_async.py @@ -0,0 +1,110 @@ +# Copyright (c) 2022 OpenCyphal +# This software is distributed under the terms of the MIT License. +# Author: Pavel Kirienko + +from __future__ import annotations +import asyncio +from typing import Any, Iterable, Callable +import pycyphal +from pycyphal.transport import TransferFrom +from pycyphal.presentation import Subscriber +from ._sync import SynchronizerOutput, Synchronizer + + +def make_sync_async(subscribers: Iterable[Subscriber[Any]]) -> Synchronizer: + """ + This synchronizer delivers one message at a time immediately without any synchronization, + where all other messages and their metadata are set to None. + """ + subscribers = list(subscribers) + queue: asyncio.Queue[tuple[tuple[tuple[Any, TransferFrom] | None, Subscriber[Any]], ...]] = asyncio.Queue() + + def mk_handler(index: int) -> Callable[[Any, TransferFrom], None]: + def hdl(msg: Any, meta: TransferFrom) -> None: + assert isinstance(meta, TransferFrom) + queue.put_nowait( + tuple( + ( + ((msg, meta) if idx == index else None), + sub, + ) + for idx, sub in enumerate(subscribers) + ) + ) + + return hdl + + async def fun(output: SynchronizerOutput) -> None: + try: + for idx, sub in enumerate(subscribers): + sub.receive_in_background(mk_handler(idx)) + while True: + item = await queue.get() + output(item) + finally: + pycyphal.util.broadcast((x.close for x in subscribers))() + + return fun + + +def _unittest_sync_async() -> None: + from tests.dsdl import ensure_compiled_dsdl + + ensure_compiled_dsdl() + + from pycyphal.transport.loopback import LoopbackTransport + from pycyphal.presentation import Presentation + from uavcan.primitive.scalar import Integer8_1 + + async def run() -> None: + pre = Presentation(LoopbackTransport(10)) + sub_a = pre.make_subscriber(Integer8_1, 1000) + sub_b = pre.make_subscriber(Integer8_1, 1001) + pub_a = pre.make_publisher(sub_a.dtype, sub_a.port_id) + pub_b = pre.make_publisher(sub_b.dtype, sub_b.port_id) + try: + syn = make_sync_async([sub_a, sub_b]) + results: list[tuple[tuple[tuple[Any, TransferFrom] | None, Subscriber[Any]], ...]] = [] + # noinspection PyTypeChecker + tsk = asyncio.create_task(syn(results.append)) + try: + await asyncio.sleep(0.1) + assert not results + + await pub_a.publish(Integer8_1(50)) + await asyncio.sleep(0.1) + ((((msg, meta), rx_sub_a), (none, rx_sub_b)),) = results # type: ignore + results.clear() + assert rx_sub_a is sub_a and rx_sub_b is sub_b and none is None + assert isinstance(msg, Integer8_1) + assert msg.value == 50 + assert meta.source_node_id == 10 + + await pub_a.publish(Integer8_1(51)) + await asyncio.sleep(0.1) + ((((msg, meta), rx_sub_a), (none, rx_sub_b)),) = results # type: ignore + results.clear() + assert rx_sub_a is sub_a and rx_sub_b is sub_b and none is None + assert isinstance(msg, Integer8_1) + assert msg.value == 51 + assert meta.source_node_id == 10 + + await pub_b.publish(Integer8_1(52)) + await asyncio.sleep(0.1) + (((none, rx_sub_a), ((msg, meta), rx_sub_b)),) = results # type: ignore + results.clear() + assert rx_sub_a is sub_a and rx_sub_b is sub_b and none is None + assert isinstance(msg, Integer8_1) + assert msg.value == 52 + assert meta.source_node_id == 10 + + finally: + tsk.cancel() + finally: + sub_a.close() + sub_b.close() + pub_a.close() + pub_b.close() + pre.close() + + asyncio.run(run()) diff --git a/yakut/cmd/subscribe/_sync_unary.py b/yakut/cmd/subscribe/_sync_unary.py deleted file mode 100644 index 456ba99..0000000 --- a/yakut/cmd/subscribe/_sync_unary.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright (c) 2022 OpenCyphal -# This software is distributed under the terms of the MIT License. -# Author: Pavel Kirienko - - -from __future__ import annotations -from typing import Any, Iterable -import pycyphal -from ._sync import SynchronizerOutput, Synchronizer - - -def make_sync_unary(subscribers: Iterable[pycyphal.presentation.Subscriber[Any]]) -> Synchronizer: - subscribers = list(subscribers) - if len(subscribers) != 1: - raise ValueError(f"Unary synchronizer requires exactly one subscriber; got {subscribers}") - (subscriber,) = subscribers - - async def fun(output: SynchronizerOutput) -> None: - async for msg, meta in subscriber: - assert isinstance(meta, pycyphal.transport.TransferFrom) - output((((msg, meta), subscriber),)) - - return fun diff --git a/yakut/dtype_loader.py b/yakut/dtype_loader.py index c290609..2164dc0 100644 --- a/yakut/dtype_loader.py +++ b/yakut/dtype_loader.py @@ -28,8 +28,9 @@ def load_dtype(name: str, allow_minor_version_mismatch: bool = False) -> Type[An Parses a data specifier string of the form ``full_data_type_name[.major_version[.minor_version]]``. Name separators may be replaced with ``/`` or ``\`` for compatibility with file system paths. Missing version numbers substituted with the latest one available. + The short data type name is case-insensitive. - :param name: Examples: ``uavcan.Heartbeat``, ``uavcan.Heartbeat.1``, ``uavcan.Heartbeat.1.0``. + :param name: Examples: ``uavcan.heartbeat``, ``uavcan.Heartbeat.1``, ``uavcan.HEARTBEAT.1.0``. :param allow_minor_version_mismatch: If the minor version is specified and there is no matching data type, @@ -71,7 +72,7 @@ def _load(name_components: list[str], major: int | None, minor: int | None) -> T (x.groups(), getattr(mod, x.string)) for x in filter(None, map(_RE_SHORT_TYPE_NAME_IDENTIFIER.match, dir(mod))) if ( - short_name == x.group(1) + short_name.lower() == x.group(1).lower() and (major is None or int(x.group(2)) == major) and (minor is None or int(x.group(3)) == minor) ) diff --git a/yakut/int_set_parser.py b/yakut/int_set_parser.py new file mode 100644 index 0000000..904d124 --- /dev/null +++ b/yakut/int_set_parser.py @@ -0,0 +1,101 @@ +# Copyright (c) 2019 OpenCyphal +# This software is distributed under the terms of the MIT License. +# Author: Pavel Kirienko + +from __future__ import annotations +import re +import logging + +_logger = logging.getLogger(__name__) + + +class IntSetError(ValueError): + pass + + +INT_SET_USER_DOC = """ +Integer set notation examples: + +\b + Discrete elements (, or ;): 1,56;-3 + Intervals [lo,hi) (- or ...): 10-23,-5--7,-10..-2 + Exclusion with ! prefix: 5-9,!6,!5...7 + Arbitrary combination: -9--5;+4,!-8..-5 + JSON/YAML compatibility: [1,53,78] +""".strip() + + +def parse_int_set(text: str) -> set[int] | int: + """ + Unpacks the integer set notation. + Accepts JSON-list (subset of YAML) of integers at input, too. + A single scalar is returned as-is unless there is a separator at the end ("125,") or JSON list is used. + Raises :class:`IntSetError` on syntax error. + Usage: + + >>> parse_int_set("") + set() + >>> parse_int_set("123"), parse_int_set("[123]"), parse_int_set("123,") + (123, {123}, {123}) + >>> parse_int_set("-0"), parse_int_set("[-0]"), parse_int_set("-0,") + (0, {0}, {0}) + >>> sorted(parse_int_set("0..0x0A")) # Half-open interval with .. or ... or - + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + >>> sorted(parse_int_set("-9...-5,")) + [-9, -8, -7, -6] + >>> sorted(parse_int_set("-9--5; +4, !-8..-5")) # Exclusion with ! prefix + [-9, 4] + >>> sorted(parse_int_set("-10..+10,!-9-+9")) # Valid separators are , and ; + [-10, 9] + >>> sorted(parse_int_set("6-6")) + [] + >>> sorted(parse_int_set("[1,53,78]")) + [1, 53, 78] + >>> parse_int_set("123,456,9-") # doctest:+IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + IntSetError: ... + """ + + def try_parse(val: str) -> int | None: + try: + return int(val, 0) + except ValueError: + return None + + collapse = not _RE_JSON_LIST.match(text) + incl: set[int] = set() + excl: set[int] = set() + for item in _RE_SPLIT.split(_RE_JSON_LIST.sub(r"\1", text)): + item = item.strip() + if not item: + collapse = False + continue + if item.startswith("!"): + target_set = excl + item = item[1:] + else: + target_set = incl + x = try_parse(item) + if x is not None: + target_set.add(x) + continue + match = _RE_RANGE.match(item) + if match: + lo, hi = map(try_parse, match.groups()) + if lo is not None and hi is not None: + target_set |= set(range(lo, hi)) + continue + raise IntSetError(f"Item {item!r} of the integer set {text!r} could not be parsed") + + result: set[int] | int = incl - excl + assert isinstance(result, set) + if collapse and len(result) == 1: + (result,) = result + _logger.debug("Int set %r parsed as %r", text, result) + return result + + +_RE_JSON_LIST = re.compile(r"^\s*\[([^]]*)]\s*$") +_RE_SPLIT = re.compile(r"[,;]") +_RE_RANGE = re.compile(r"([+-]?\w+)(?:-|\.\.\.?)([+-]?\w+)") diff --git a/yakut/main.py b/yakut/main.py index a2d0593..2360a6e 100644 --- a/yakut/main.py +++ b/yakut/main.py @@ -14,7 +14,7 @@ import click import yakut from yakut.param.transport import transport_factory_option, TransportFactory, Transport -from yakut.param.formatter import formatter_factory_option, FormatterFactory, Formatter +from yakut.param.formatter import formatter_factory_option, FormatterFactory, Formatter, FormatterHints from yakut.param.node import node_factory_option, NodeFactory if TYPE_CHECKING: @@ -59,8 +59,8 @@ def __init__( def paths(self) -> list[Path]: return list(self._paths) - def make_formatter(self) -> Formatter: - return self._f_formatter() + def make_formatter(self, hints: FormatterHints = FormatterHints()) -> Formatter: + return self._f_formatter(hints) def get_registry(self) -> pycyphal.application.register.Registry: """ @@ -167,7 +167,7 @@ def format_commands(self, ctx: click.Context, formatter: click.HelpFormatter) -> subcmd = ",".join([subcmd] + list(sorted(self._commands[subcmd]))) rows.append((subcmd, cmd.get_short_help_str(limit))) if rows: - with formatter.section("Commands"): + with formatter.section("Commands (with aliases)"): formatter.write_dl(rows) @staticmethod @@ -256,8 +256,7 @@ def _click_main( def main() -> None: # https://click.palletsprojects.com/en/8.1.x/exceptions/ - def err(text: str) -> None: - click.secho(text, err=True, fg="red", bold=True) + from yakut.ui import show_error status: Any = 1 # noinspection PyBroadException @@ -281,11 +280,11 @@ def err(text: str) -> None: click.secho("", err=True, nl=False) except Exception as ex: # pylint: disable=broad-except - err(f"{type(ex).__name__}: {ex}") + show_error(f"{type(ex).__name__}: {ex}") _logger.debug("EXCEPTION %s: %s", type(ex).__name__, ex, exc_info=True) except BaseException as ex: # pylint: disable=broad-except - err(f"Internal error, please report: {ex}") + show_error(f"Internal error, please report: {ex}") _logger.error("%s: %s", type(ex).__name__, ex, exc_info=True) _logger.debug("EXIT %r", status) @@ -319,7 +318,13 @@ def proxy(*args: Any, **kwargs: Any) -> Any: finally: _logger.debug("Event loop finalization with exc=%r", sys.exc_info()) try: - loop.set_exception_handler(handle_task_exception) + loop.set_exception_handler(handle_task_exception) # Reduce severity of exception reports + + # Suppress finalization errors from PyCyphal https://github.com/OpenCyphal/pycyphal/issues/227 + pycyphal_logger = logging.getLogger("pycyphal") + if not pycyphal_logger.isEnabledFor(logging.INFO): # Do not suppress if verbose + pycyphal_logger.setLevel(logging.CRITICAL) + orphans = asyncio.all_tasks(loop) if orphans: for ts in orphans: diff --git a/yakut/param/formatter.py b/yakut/param/formatter.py index cda7943..d569d83 100644 --- a/yakut/param/formatter.py +++ b/yakut/param/formatter.py @@ -3,34 +3,59 @@ # Author: Pavel Kirienko from __future__ import annotations +import sys +import dataclasses +import logging from typing import Callable, Any, cast from collections.abc import Mapping, Collection import click + +@dataclasses.dataclass(frozen=True) +class FormatterHints: + single_document: bool = False + """ + If true, document separators will not be emitted (if applicable). + """ + + Formatter = Callable[[Any], str] -FormatterFactory = Callable[[], Formatter] +""" +The output represents a complete document with the terminating newline if such is necessary. +The caller need not add anything. +If sending this to stdout, do not use print() as-is because it will add an extra newline at the end. +""" + +FormatterFactory = Callable[[FormatterHints], Formatter] def formatter_factory_option(f: Callable[..., Any]) -> Callable[..., Any]: - def validate(ctx: click.Context, param: object, value: str) -> FormatterFactory: - _ = ctx, param + override: str | None = None + + def install_override(_ctx: click.Context, _param: click.Parameter, value: str) -> None: + nonlocal override + override = value or override + + def validate(ctx: click.Context, param: click.Parameter, value: str) -> FormatterFactory: try: - return _FORMATTERS[value.upper()] + return _FORMATTERS[(override or value).upper()] except LookupError: - raise click.BadParameter(f"Invalid format name: {value!r}") from None + raise click.BadParameter(f"Invalid format name: {value!r}", ctx=ctx, param=param) from None - choices = list(_FORMATTERS.keys()) - default = choices[0] doc = f""" -The format of data printed into stdout. -This option is only relevant for commands that generate structured outputs, like pub or call; other commands ignore it. +The format of data printed to stdout. +This option is only relevant for commands that produce data (sub, call, etc.). The final representation of the output data is constructed from an intermediate "builtin-based" representation, which is a simplified form that is stripped of the detailed DSDL type information, like JSON. For more info please read the PyCyphal documentation on builtin-based representations. -YAML separates objects with `---`. +Option "auto" (default) selects YAML if the output is shown on the terminal for the benefit of the user; +if the output is redirected (e.g., piped to another command or to a file), +JSON is selected to enable compatibility with jq and other 3rd-party stream processing tools. +It helps to remember that JSON is a subset of YAML. +YAML separates objects with `---`. JSON and TSV (tab separated values) keep exactly one object per line. TSV is intended for use with third-party software @@ -46,29 +71,45 @@ def validate(ctx: click.Context, param: object, value: str) -> FormatterFactory: "-F", "formatter_factory", envvar="YAKUT_FORMAT", - type=click.Choice(choices, case_sensitive=False), + type=click.Choice(list(_FORMATTERS.keys()), case_sensitive=False), callback=validate, - default=default, + default=list(_FORMATTERS.keys())[0], show_default=True, help=doc, )(f) + + def shortcut(opt: str) -> None: + nonlocal f + f = click.option( + f"--{opt}", + "-" + opt[0], + flag_value=opt, + callback=install_override, + help=f"Same as --format={opt}", + is_eager=True, + expose_value=False, + )(f) + + shortcut("yaml") + shortcut("json") + shortcut("tsvh") return f -def _make_yaml_formatter() -> Formatter: +def _make_yaml_formatter(hints: FormatterHints) -> Formatter: from yakut.yaml import Dumper - dumper = Dumper(explicit_start=True) + dumper = Dumper(explicit_start=not hints.single_document) return dumper.dumps -def _make_json_formatter() -> Formatter: +def _make_json_formatter(_hints: FormatterHints) -> Formatter: # We prefer simplejson over the standard json because the native json lacks important capabilities: # - simplejson preserves dict ordering, which is very important for UX. # - simplejson supports Decimal. import simplejson as json # type: ignore - return lambda data: cast(str, json.dumps(data, ensure_ascii=False, separators=(",", ":"))) + return lambda data: cast(str, json.dumps(data, ensure_ascii=False, separators=(",", ":"))) + _NEWLINE def _insert_format_specifier( @@ -92,14 +133,14 @@ def _insert_format_specifier( def _flatten_start( - d: dict[Any, Any] | Collection[Any], + outer: Any, parent_key: str = "", sep: str = ".", - do_put_format_specifiers: bool = False, + with_format_specifiers: bool = False, ) -> dict[str, Any]: - def flatten(d: dict[Any, Any] | Collection[Any], parent_key: str = "") -> dict[str, Any]: + def flatten(data: Any, parent_key: str = "") -> dict[str, Any]: def add_item(items: list[tuple[str, Any]], new_key: str, v: Mapping[Any, Any] | Collection[Any]) -> None: - if do_put_format_specifiers: + if with_format_specifiers: _insert_format_specifier(items, new_key, v) if isinstance(v, Mapping) or (isinstance(v, Collection) and not isinstance(v, str)): for_extension = flatten(v, new_key) @@ -108,75 +149,94 @@ def add_item(items: list[tuple[str, Any]], new_key: str, v: Mapping[Any, Any] | items.extend(for_extension.items()) else: items.append((new_key, v)) - if do_put_format_specifiers: + if with_format_specifiers: _insert_format_specifier(items, new_key, v, is_start=False) - if isinstance(d, Mapping): + if isinstance(data, Mapping): items: list[tuple[str, Any]] = [] - for k, v in d.items(): + for k, v in data.items(): new_key = parent_key + sep + str(k) if parent_key else str(k) add_item(items, new_key, v) return dict(items) - if isinstance(d, Collection) and not isinstance(d, str): + if isinstance(data, Collection) and not isinstance(data, str): items = [] - for i, v in enumerate(d): + for i, v in enumerate(data): new_key = parent_key + sep + f"[{i}]" if parent_key else str(f"[{i}]") add_item(items, new_key, v) return dict(items) return {} - return flatten(d, parent_key) + return flatten(outer, parent_key) -def _make_tsv_formatter() -> Formatter: - def tsv_format_function(data: dict[Any, Any]) -> str: - return "\t".join([str(v) for k, v in _flatten_start(data).items()]) +def _make_tsv_formatter(hints: FormatterHints) -> Formatter: + # TODO: if single_document, transpose the top-level dict to have the keys on the leftmost row. + # Transpose lists in a similar manner. + _ = hints # TODO not used yet + + def tsv_format_function(data: Any) -> str: + return "\t".join(str(v) for k, v in _flatten_start(data).items()) + _NEWLINE return tsv_format_function -def _make_tsvh_formatter_factory(do_put_format_specifiers: bool = False) -> Callable[[], Formatter]: - def _make_tsvh_formatter() -> Formatter: +def _make_tsvh_formatter_factory(with_format_specifiers: bool) -> FormatterFactory: + def make_tsvh_formatter(hints: FormatterHints) -> Formatter: + _ = hints # TODO see above is_first_time = True - def tsv_format_function_with_header(data: dict[Any, Any]) -> str: + def tsv_format_function_with_header(data: Any) -> str: nonlocal is_first_time if is_first_time: is_first_time = False - return ( - "\t".join( - [ + return _NEWLINE.join( + ( + "\t".join( str(k) - for k, v in _flatten_start(data, do_put_format_specifiers=do_put_format_specifiers).items() - ] - ) - + "\n" - + "\t".join( - [ + for k, v in _flatten_start(data, with_format_specifiers=with_format_specifiers).items() + ), + "\t".join( str(v) - for k, v in _flatten_start(data, do_put_format_specifiers=do_put_format_specifiers).items() - ] + for k, v in _flatten_start(data, with_format_specifiers=with_format_specifiers).items() + ), + "", ) ) - return "\t".join( - [str(v) for k, v in _flatten_start(data, do_put_format_specifiers=do_put_format_specifiers).items()] + return ( + "\t".join( + str(v) for k, v in _flatten_start(data, with_format_specifiers=with_format_specifiers).items() + ) + + _NEWLINE ) return tsv_format_function_with_header - return _make_tsvh_formatter + return make_tsvh_formatter + + +def _make_auto(hints: FormatterHints) -> Formatter: + fac = _make_yaml_formatter if sys.stdout.isatty() else _make_json_formatter + _logger.debug("Automatically selected formatter: %r", fac) + return fac(hints) _FORMATTERS = { + "AUTO": _make_auto, "YAML": _make_yaml_formatter, "JSON": _make_json_formatter, "TSV": _make_tsv_formatter, - "TSVH": _make_tsvh_formatter_factory(do_put_format_specifiers=False), - "TSVFC": _make_tsvh_formatter_factory(do_put_format_specifiers=True), + "TSVH": _make_tsvh_formatter_factory(with_format_specifiers=False), + "TSVFC": _make_tsvh_formatter_factory(with_format_specifiers=True), } +_NEWLINE = "\n" + +_logger = logging.getLogger(__name__) + def _unittest_formatter() -> None: + default_hints = FormatterHints() + obj = { 2345: { "abc": { @@ -186,7 +246,7 @@ def _unittest_formatter() -> None: } } assert ( - _FORMATTERS["YAML"]()(obj) + _FORMATTERS["YAML"](default_hints)(obj) == """--- 2345: abc: @@ -194,19 +254,19 @@ def _unittest_formatter() -> None: ghi: 789 """ ) - assert _FORMATTERS["JSON"]()(obj) == '{"2345":{"abc":{"def":[123,456]},"ghi":789}}' - assert _FORMATTERS["TSV"]()(obj) == "123\t456\t789" - tsvh_formatter = _FORMATTERS["TSVH"]() + assert _FORMATTERS["JSON"](default_hints)(obj) == '{"2345":{"abc":{"def":[123,456]},"ghi":789}}\n' + assert _FORMATTERS["TSV"](default_hints)(obj) == "123\t456\t789\n" + tsvh_formatter = _FORMATTERS["TSVH"](default_hints) # first time should include a header - assert tsvh_formatter(obj) == "2345.abc.def.[0]\t2345.abc.def.[1]\t2345.ghi\n123\t456\t789" + assert tsvh_formatter(obj) == "2345.abc.def.[0]\t2345.abc.def.[1]\t2345.ghi\n123\t456\t789\n" # subsequent calls shouldn't include a header - assert tsvh_formatter(obj) == "123\t456\t789" + assert tsvh_formatter(obj) == "123\t456\t789\n" from decimal import Decimal from math import nan obj = { 142: { - "_metadata_": { + "_meta_": { "timestamp": {"system": Decimal("1640611164.396007"), "monotonic": Decimal("4765.594161")}, "priority": "nominal", "transfer_id": 28, @@ -223,12 +283,12 @@ def _unittest_formatter() -> None: }, } } - tsvfc_formatter = _FORMATTERS["TSVFC"]() + tsvfc_formatter = _FORMATTERS["TSVFC"](default_hints) assert ( - tsvfc_formatter(obj) == "142{ 142._metadata_{ 142._metadata_.timestamp{" - " 142._metadata_.timestamp.system 142._metadata_.timestamp.monotonic" - " 142._metadata_.timestamp} 142._metadata_.priority 142._metadata_.transfer_id" - " 142._metadata_.source_node_id 142._metadata_} 142.timestamp{ 142.timestamp.microsecond" + tsvfc_formatter(obj) == "142{ 142._meta_{ 142._meta_.timestamp{" + " 142._meta_.timestamp.system 142._meta_.timestamp.monotonic" + " 142._meta_.timestamp} 142._meta_.priority 142._meta_.transfer_id" + " 142._meta_.source_node_id 142._meta_} 142.timestamp{ 142.timestamp.microsecond" " 142.timestamp} 142.value{ 142.value.kinematics{ 142.value.kinematics.angular_position{" " 142.value.kinematics.angular_position.radian 142.value.kinematics.angular_position}" " 142.value.kinematics.angular_velocity{ 142.value.kinematics.angular_velocity.radian_per_second" @@ -238,17 +298,20 @@ def _unittest_formatter() -> None: " 142.value.torque.newton_meter 142.value.torque} 142.value} 142}\n{ { {" " 1640611164.396007 4765.594161 } nominal 28 21 }" " { 309697890 } { { { nan } { 0.0 }" - " { 0.0 } } { nan } } }" + " { 0.0 } } { nan } } }\n" + ) + assert ( + _FORMATTERS["TSV"](default_hints)(obj) + == "1640611164.396007\t4765.594161\tnominal\t28\t21\t309697890\tnan\t0.0\t0.0\tnan\n" ) - assert _FORMATTERS["TSV"]()(obj) == "1640611164.396007\t4765.594161\tnominal\t28\t21\t309697890\tnan\t0.0\t0.0\tnan" assert ( - _FORMATTERS["TSVH"]()(obj) - == "142._metadata_.timestamp.system\t142._metadata_.timestamp.monotonic\t142._metadata_.priority\t" - "142._metadata_.transfer_id\t142._metadata_.source_node_id\t142.timestamp.microsecond\t" + _FORMATTERS["TSVH"](default_hints)(obj) + == "142._meta_.timestamp.system\t142._meta_.timestamp.monotonic\t142._meta_.priority\t" + "142._meta_.transfer_id\t142._meta_.source_node_id\t142.timestamp.microsecond\t" "142.value.kinematics.angular_position.radian" "\t142.value.kinematics.angular_velocity.radian_per_second\t142.value.kinematics.angular_acceleration." "radian_per_second_per_second\t142.value.torque.newton_meter" "\n1640611164.396007\t4765.594161\tnominal\t2" - "8\t21\t309697890\tnan\t0.0\t0.0\tnan" + "8\t21\t309697890\tnan\t0.0\t0.0\tnan\n" ) diff --git a/yakut/register.py b/yakut/register.py new file mode 100644 index 0000000..6f600ca --- /dev/null +++ b/yakut/register.py @@ -0,0 +1,198 @@ +# Copyright (c) 2019 OpenCyphal +# This software is distributed under the terms of the MIT License. +# Author: Pavel Kirienko + +from __future__ import annotations +from typing import Any, TYPE_CHECKING, Callable, Optional +import logging +import pycyphal +from yakut.util import METADATA_KEY + +if TYPE_CHECKING: + import pycyphal.application + from pycyphal.application.register import Value + from uavcan.register import Access_1 + + +async def fetch_registers( + presentation: pycyphal.presentation.Presentation, + node_id: int, + *, + predicate: Callable[[str], bool] = lambda *_: True, + timeout: float = pycyphal.presentation.DEFAULT_SERVICE_REQUEST_TIMEOUT, + priority: pycyphal.transport.Priority = pycyphal.transport.Priority.LOW, +) -> dict[str, "pycyphal.application.register.ValueProxy"] | None: + """ + Obtain registers from the specified remote node for whose names the predicate is true. + Returns None on network timeout. + """ + from pycyphal.application.register import ValueProxy as RegisterValue + from uavcan.register import Access_1, List_1, Name_1 + + # Fetch register names. + c_list = presentation.make_client_with_fixed_service_id(List_1, node_id) + c_list.response_timeout = timeout + c_list.priority = priority + names: list[str] = [] + while True: + req: Any = List_1.Request(len(names)) + resp = await c_list(req) + if resp is None: + _logger.warning("Request to %r has timed out: %s", node_id, req) + return None + assert isinstance(resp, List_1.Response) + if not resp.name.name.tobytes(): + break + names.append(resp.name.name.tobytes().decode()) + _logger.debug("Register names fetched from node %r: %s", node_id, names) + c_list.close() + del c_list + + names = list(filter(predicate, names)) + + # Then fetch the registers themselves. + c_access = presentation.make_client_with_fixed_service_id(Access_1, node_id) + c_access.response_timeout = timeout + c_access.priority = priority + regs: dict[str, RegisterValue] = {} + for nm in names: + req = Access_1.Request(name=Name_1(nm)) + resp = await c_access(req) + if resp is None: + _logger.warning("Request to %r has timed out: %s", node_id, req) + return None + assert isinstance(resp, Access_1.Response) + regs[nm] = RegisterValue(resp.value) + c_access.close() + + return regs + + +def unexplode_value(xpl: Any, prototype: Optional["Value"] = None) -> Optional["Value"]: + """ + Reverse the effect of :func:`explode`. + Returns None if the exploded form is invalid or not applicable to the prototype. + Some simplified exploded forms can be unexploded only if the prototype + is given because simplification erases type information. + Some unambiguous simplified forms may be unexploded autonomously. + + >>> from tests.dsdl import ensure_compiled_dsdl + >>> ensure_compiled_dsdl() + >>> from pycyphal.application.register import Value, Natural16 + >>> ux = unexplode_value + + >>> ux(None) # None is a simplified form of Empty. + uavcan.register.Value...(empty=...) + >>> ux({"integer8": {"value": [1,2,3]}, "_meta_": {"whatever": 0}}) # Metadata ignored. + uavcan.register.Value...(integer8=...[1,2,3])) + >>> ux({"integer8": {"value": [1,2,3]}}) # Pure Value (same as above) + uavcan.register.Value...(integer8=...[1,2,3])) + >>> ux([1,2,3]) is None # Prototype required. + True + >>> ux([1,2,3], Value(natural16=Natural16([0,0,0]))) + uavcan.register.Value...(natural16=...[1,2,3])) + >>> ux(123, Value(natural16=Natural16([0]))) + uavcan.register.Value...(natural16=...[123])) + >>> ux("abc", Value(natural16=Natural16([0]))) is None # Not applicable + True + + Roundtrip: + + >>> unexplode_value(explode_value(Value(natural16=Natural16([0,1,2])), metadata={"a": 654})) + uavcan.register.Value...(natural16=...[0,1,2])) + """ + from pycyphal.dsdl import update_from_builtin + from pycyphal.application.register import ValueProxy, Value, ValueConversionError + + if xpl is None: + return Value() + if isinstance(xpl, dict) and xpl: # Empty dict is not a valid representation. + try: + res = update_from_builtin( + Value(), + {k: v for k, v in xpl.items() if k.strip("_") == k}, # Strip metadata fields. + ) + assert isinstance(res, Value) + return res + except (ValueError, TypeError): + pass + if prototype is not None: + ret = ValueProxy(prototype) + try: + ret.assign(xpl) + assert isinstance(ret.value, Value) + return ret.value + except ValueConversionError: + pass + return None + + +def explode_value(val: "Value", *, simplify: bool = False, metadata: dict[str, Any] | None = None) -> Any: + """ + Represent the register value using primitives (list, dict, string, etc.). + If simplified mode is selected, + the metadata and type information will be discarded and only a human-friendly representation of the + value will be constructed. + The reconstruction back to the original form is a bit involved but we provide :func:`unexplode` for that. + The metadata is added under a key ``_meta_``, if there is any, but it is ignored in simplified mode. + """ + if not simplify: + out = pycyphal.dsdl.to_builtin(val) + if metadata is not None: + out[METADATA_KEY] = dict(metadata) + return out + return _simplify_value(val) + + +def _simplify_value(msg: "Value") -> Any: + """ + Construct simplified human-friendly representation of the register value using primitives (list, string, etc.). + Designed for use with commands that output compact register values in YAML/JSON/TSV/whatever, + discarding the detailed type information. + + >>> from tests.dsdl import ensure_compiled_dsdl + >>> ensure_compiled_dsdl() + >>> from pycyphal.application.register import Value, Empty + >>> from pycyphal.application.register import Integer8, Natural8, Integer32, String, Unstructured + + >>> None is _simplify_value(Value()) # empty is none + True + >>> _simplify_value(Value(integer8=Integer8([123]))) + 123 + >>> _simplify_value(Value(natural8=Natural8([123, 23]))) + [123, 23] + >>> _simplify_value(Value(integer32=Integer32([123, -23, 105]))) + [123, -23, 105] + >>> _simplify_value(Value(integer32=Integer32([99999]))) + 99999 + >>> _simplify_value(Value(string=String("Hello world"))) + 'Hello world' + >>> _simplify_value(Value(unstructured=Unstructured(b"Hello world"))) + b'Hello world' + """ + # This is kinda crude, perhaps needs improvement. + if msg.empty: + return None + if msg.unstructured: + return msg.unstructured.value.tobytes() + if msg.string: + return msg.string.value.tobytes().decode(errors="replace") + ((_ty, val),) = pycyphal.dsdl.to_builtin(msg).items() + val = val["value"] + val = list(val.encode() if isinstance(val, str) else val) + if len(val) == 1: # One-element arrays shown as scalars. + (val,) = val + return val + + +def get_access_response_metadata(val: "Access_1.Response") -> dict[str, Any]: + """ + This is for use with :func:`explode_value`. + """ + return { + "mutable": val.mutable, + "persistent": val.persistent, + } + + +_logger = logging.getLogger(__name__) diff --git a/yakut/subject_resolver.py b/yakut/subject_resolver.py index b40c155..c131378 100644 --- a/yakut/subject_resolver.py +++ b/yakut/subject_resolver.py @@ -8,7 +8,7 @@ import logging from typing import TYPE_CHECKING import pycyphal -from yakut.util import fetch_registers +from yakut.register import fetch_registers if TYPE_CHECKING: import pycyphal.application diff --git a/yakut/ui.py b/yakut/ui.py new file mode 100644 index 0000000..609b901 --- /dev/null +++ b/yakut/ui.py @@ -0,0 +1,53 @@ +# Copyright (c) 2022 OpenCyphal +# This software is distributed under the terms of the MIT License. +# Author: Pavel Kirienko + +from __future__ import annotations +import sys +from typing import Callable, Any +import click + + +ProgressCallback = Callable[[str], None] + + +class ProgressReporter: + def __init__(self) -> None: + self._widest = 0 + self._impl = _mk_impl() + + def __call__(self, text: str) -> None: + self._widest = max(self._widest, len(text)) + # Add extra space after the text is to improve appearance when the text is shortened. + self._impl(text.ljust(self._widest)) + + def clear(self) -> None: + """ + Call this once at the end to erase the progress line from the screen. + Does nothing if no output was generated. + """ + if self._widest > 0: + self._impl(" " * self._widest) + + def __enter__(self) -> ProgressReporter: + return self + + def __exit__(self, *_: Any) -> None: + """ + Invokes :meth:`clear` upon leaving the context. + """ + self.clear() + + +def _mk_impl() -> ProgressCallback: + if sys.stderr.isatty(): + return lambda text: click.secho(f"\r{text}\r", nl=False, file=sys.stderr, fg="green") + return lambda _: None + + +def show_error(msg: str) -> None: + click.secho(msg, err=True, fg="red", bold=True) + + +def show_warning(msg: str) -> None: + click.secho(msg, err=True, fg="yellow") diff --git a/yakut/util.py b/yakut/util.py index 1cc1b08..54fd0fa 100644 --- a/yakut/util.py +++ b/yakut/util.py @@ -3,85 +3,56 @@ # Author: Pavel Kirienko from __future__ import annotations -from typing import Any, TYPE_CHECKING, Callable -import logging +from typing import Any, Callable, TypeVar import decimal +import functools import pycyphal -if TYPE_CHECKING: - import pycyphal.application -_logger = logging.getLogger(__name__) +T = TypeVar("T") +METADATA_KEY = "_meta_" -async def fetch_registers( - presentation: pycyphal.presentation.Presentation, - node_id: int, - *, - predicate: Callable[[str], bool] = lambda *_: True, - timeout: float = pycyphal.presentation.DEFAULT_SERVICE_REQUEST_TIMEOUT, - priority: pycyphal.transport.Priority = pycyphal.transport.Priority.LOW, -) -> dict[str, "pycyphal.application.register.ValueProxy"] | None: - """ - Obtain registers from the specified remote node for whose names the predicate is true. - Returns None on network timeout. - """ - from pycyphal.application.register import ValueProxy as RegisterValue - from uavcan.register import Access_1, List_1, Name_1 +EXIT_CODE_UNSUCCESSFUL = 100 +""" +The command was invoked and executed correctly but the desired goal could not be attained for external reasons. +""" - # Fetch register names. - c_list = presentation.make_client_with_fixed_service_id(List_1, node_id) - c_list.response_timeout = timeout - c_list.priority = priority - names: list[str] = [] - while True: - req: Any = List_1.Request(len(names)) - resp = await c_list(req) - if resp is None: - _logger.warning("Request to %r has timed out: %s", node_id, req) - return None - assert isinstance(resp, List_1.Response) - if not resp.name.name.tobytes(): - break - names.append(resp.name.name.tobytes().decode()) - _logger.debug("Register names fetched from node %r: %s", node_id, names) - c_list.close() - del c_list - names = list(filter(predicate, names)) +def compose(*fs: Callable[..., T]) -> Callable[..., T]: + """ + >>> compose(lambda x: x+2, lambda x: x*2)(3) + 10 + """ + return functools.reduce(_compose_unit, fs[::-1]) - # Then fetch the registers themselves. - c_access = presentation.make_client_with_fixed_service_id(Access_1, node_id) - c_access.response_timeout = timeout - c_access.priority = priority - regs: dict[str, RegisterValue] = {} - for nm in names: - req = Access_1.Request(name=Name_1(nm)) - resp = await c_access(req) - if resp is None: - _logger.warning("Request to %r has timed out: %s", node_id, req) - return None - assert isinstance(resp, Access_1.Response) - regs[nm] = RegisterValue(resp.value) - c_access.close() - return regs +def _compose_unit(f: Callable[..., T], g: Callable[..., T]) -> Callable[..., T]: + return lambda *a, **kw: f(g(*a, **kw)) def convert_transfer_metadata_to_builtin( - transfer: pycyphal.transport.TransferFrom, **extra_fields: dict[str, Any] -) -> dict[str, Any]: - out = { - "timestamp": { - "system": transfer.timestamp.system.quantize(_MICRO), - "monotonic": transfer.timestamp.monotonic.quantize(_MICRO), - }, - "priority": transfer.priority.name.lower(), - "transfer_id": transfer.transfer_id, - "source_node_id": transfer.source_node_id, + transfer: pycyphal.transport.TransferFrom, + *, + dtype: Any, + **extra_fields: dict[str, Any], +) -> dict[str, dict[str, Any]]: + return { + METADATA_KEY: { + "ts_system": transfer.timestamp.system.quantize(_MICRO), + "ts_monotonic": transfer.timestamp.monotonic.quantize(_MICRO), + "source_node_id": transfer.source_node_id, + "transfer_id": transfer.transfer_id, + "priority": transfer.priority.name.lower(), + "dtype": get_dtype_full_name_with_version(dtype), + **extra_fields, + } } - out.update(extra_fields) - return {"_metadata_": out} + + +@functools.lru_cache(None) +def get_dtype_full_name_with_version(dtype: Any) -> str: + return str(pycyphal.dsdl.get_model(dtype)) _MICRO = decimal.Decimal("0.000001") diff --git a/yakut/yaml/_dumper.py b/yakut/yaml/_dumper.py index 3123809..25b7360 100644 --- a/yakut/yaml/_dumper.py +++ b/yakut/yaml/_dumper.py @@ -14,12 +14,12 @@ class Dumper: Natively represents decimal.Decimal as floats in the output. """ - def __init__(self, explicit_start: bool = False): + def __init__(self, explicit_start: bool = False, prefer_block_style: bool = False): # We need to use the roundtrip representer to retain ordering of mappings, which is important for usability. self._impl = ruamel.yaml.YAML(typ="rt") # noinspection PyTypeHints self._impl.explicit_start = explicit_start - self._impl.default_flow_style = None # Choose between block/inline automatically + self._impl.default_flow_style = False if prefer_block_style else None self._impl.width = 2**31 # Unlimited width def dump(self, data: Any, stream: TextIO) -> None: @@ -28,7 +28,13 @@ def dump(self, data: Any, stream: TextIO) -> None: def dumps(self, data: Any) -> str: s = io.StringIO() self.dump(data, s) - return s.getvalue() + out = s.getvalue() + + # FIXME HACK configure ruamel.yaml to not emit the ellipsis (how?) + suf = "\n...\n" + out = out[:-4] if out.endswith(suf) else out # The trailing end-of-line shall be kept. + + return out def _represent_decimal(self: ruamel.yaml.BaseRepresenter, data: decimal.Decimal) -> ruamel.yaml.ScalarNode: @@ -62,5 +68,20 @@ def _unittest_yaml() -> None: def: - .nan - {qaz: 789.0} +""" + ) + ref = Dumper(explicit_start=False, prefer_block_style=True).dumps( + { + "abc": decimal.Decimal("-inf"), + "def": [decimal.Decimal("nan"), {"qaz": decimal.Decimal("789"), "qqq": 123}], + } + ) + assert ( + ref + == """abc: -.inf +def: +- .nan +- qaz: 789.0 + qqq: 123 """ ) diff --git a/yakut/yaml/_eval_loader.py b/yakut/yaml/_eval_loader.py index dea7e72..be15f47 100644 --- a/yakut/yaml/_eval_loader.py +++ b/yakut/yaml/_eval_loader.py @@ -2,7 +2,8 @@ # This software is distributed under the terms of the MIT License. # Author: Pavel Kirienko -from typing import Any, Dict, Callable +from __future__ import annotations +from typing import Any, Dict, Callable, TextIO import time import ruamel.yaml import ruamel.yaml.constructor @@ -50,7 +51,7 @@ def evaluation_context(self) -> Dict[str, Any]: """ return self._evaluation_context - def load(self, text: str, **evaluation_context: Any) -> Any: + def load(self, text: str | TextIO, **evaluation_context: Any) -> Any: """ Loads and evaluates the evaluable YAML in one operation. It is not recommended to use this method if the same YAML document needs to be evaluated multiple times @@ -59,7 +60,7 @@ def load(self, text: str, **evaluation_context: Any) -> Any: """ return self.load_unevaluated(text)(**evaluation_context) - def load_unevaluated(self, text: str) -> Callable[..., Any]: + def load_unevaluated(self, text: str | TextIO) -> Callable[..., Any]: """ Loads the document without evaluation. The result is a closure that accepts keyword arguments that extend/override the evaluation context diff --git a/yakut/yaml/_loader.py b/yakut/yaml/_loader.py index fdfa096..68834cf 100644 --- a/yakut/yaml/_loader.py +++ b/yakut/yaml/_loader.py @@ -2,7 +2,8 @@ # This software is distributed under the terms of the MIT License. # Author: Pavel Kirienko -from typing import Any +from __future__ import annotations +from typing import Any, TextIO import decimal import ruamel.yaml import ruamel.yaml.constructor @@ -16,7 +17,7 @@ class Loader: def __init__(self) -> None: self._impl = ruamel.yaml.YAML() - def load(self, text: str) -> Any: + def load(self, text: str | TextIO) -> Any: return self._impl.load(text)