diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9dac0765..501702c4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,10 +4,9 @@ You can fork the repo and submit a pull request in Github. All developers must have signed the [P4.org](http://p4.org) CLA. -### Madoko style checker +### AsciiDoc style checker The P4Runtime specification is written using -[Madoko](http://madoko.org/reference.html). We provide a lint tool to catch +[AsciiDoc](https://docs.asciidoctor.org/). We provide a lint tool to catch basic formatting issues and try to keep the spec uniform. The lint tool will be -run as part of CI and patches cannot be merged until it returns success. You can -run the lint tool locally with `./tools/madokolint.py`. +run as part of CI and patches cannot be merged until it returns success. diff --git a/docs/v1/Makefile b/docs/v1/Makefile index 05b7f7ee..c2e4bbe5 100644 --- a/docs/v1/Makefile +++ b/docs/v1/Makefile @@ -1,4 +1,3 @@ - SPEC=P4Runtime-Spec ROUGE_STYLE=github @@ -26,8 +25,8 @@ ${SPEC}.html: ${SPEC}.adoc figs time asciidoctor -v \ -r asciidoctor-mathematical \ -r asciidoctor-bibtex \ - -r asciidoctor-lists \ + -r asciidoctor-lists \ -a rouge-css=$(ROUGE_CSS) $< clean: - /bin/rm -f ${SPEC}.pdf ${SPEC}.html resources/figs/stem-*.png + /bin/rm -f ${SPEC}.pdf ${SPEC}.html resources/figs/stem-*.png \ No newline at end of file diff --git a/docs/v1/P4Runtime-Spec.adoc b/docs/v1/P4Runtime-Spec.adoc index 498210d7..0c97d016 100755 --- a/docs/v1/P4Runtime-Spec.adoc +++ b/docs/v1/P4Runtime-Spec.adoc @@ -1577,8 +1577,7 @@ Both `Meter` and `DirectMeter` messages share the following fields: ** `BYTES`, which signifies that this meter can be configured with rates expressed in bytes/second. ** `PACKETS`, for rates expressed in packets/second. -+ --- + The meter type can be any of the `MeterSpec.Type` enum values: ** `TWO_RATE_THREE_COLOR`: This is the *Two Rate Three Color Marker* (trTCM) @@ -1607,7 +1606,7 @@ direct_meter(MeterType.bytes) my_meter; specified by the Committed Information Rate (CIR) and Committed Burst Size (CBS) to mark packets GREEN or RED. In a P4 program, this mode can be set by adding the `@single_rate_two_color` annotation to the meter definition. --- + For indexed meters, the `Meter` message contains also a `size` field, an `int64` representing the maximum number of independent cells that can be held by this meter. Conversely, the `DirectMeter` message contains a `direct_table_id` field @@ -2109,7 +2108,7 @@ read should match if the RPCs finished successfully (with the exception of parts of the response known to be data plane volatile, as explained in <<#sec-data-plane-volatile-objects>>). Consider the following pseudocode as an example: -[source,pseudo] +[source,python] ---- intended_value = value @@ -2435,8 +2434,8 @@ specification. [.center,%autowidth] [#tab-p4-type-usage] |=== -| Element type 3+^| Container type -| | header | header_union | struct or tuple +| *Element type* 3+^| *Container type* +| | *header* | *header_union* | *struct or tuple* | `bit` | allowed | error | allowed | `int` | allowed | error | allowed | `varbit` | allowed | error | allowed @@ -2638,7 +2637,7 @@ type_info { ---- Here's a `p4.WriteRequest` to set the value of `register_ip[12]`: -[source,proto] +[source,protobuf] ---- update { type: INSERT @@ -2760,7 +2759,7 @@ type bit<9> PortId_32_t; In this case, the P4Info message would include the following `P4TypeInfo` messages: -[source,proto] +[source,protobuf] ---- type_info { new_types { @@ -2857,7 +2856,7 @@ table t { This table would have the following representation in the generated P4Info message: -[source,proto] +[source,protobuf] ---- tables { preamble { @@ -3036,7 +3035,7 @@ In this P4Runtime request, the client omits the table's `TERNARY` field from the repeated `match` field to indicate a "don't care" match. As shown below, the `match` specifies only the `EXACT` field given by `field_id: 2`. -[source,proto] +[source,protobuf] ---- device_id: 3 entities { @@ -3066,7 +3065,7 @@ P4Runtime server must return an `INVALID_ARGUMENT` error code. * The binary string encoding of the value must conform to the xref:sec-bytestrings[Bytestrings] requirements. -[source,pseudo] +[source,python] ---- assert(BytestringValid(match.exact().value())) ---- @@ -3077,7 +3076,7 @@ assert(BytestringValid(match.exact().value())) * "Don't care" match must be omitted. * "Don't care" bits must be 0 in value. -[source,pseudo] +[source,python] ---- assert(BytestringValid(match.lpm().value())) @@ -3099,7 +3098,7 @@ assert(trailing_zeros >= field_bits - pLen) most-significant value bits need zero-padding before any logical operations with the mask. -[source,pseudo] +[source,python] ---- assert(BytestringValid(match.ternary().value())) assert(BytestringValid(match.ternary().mask())) @@ -3120,7 +3119,7 @@ assert(value & mask == value) * Low bound must be less than or equal to the high bound. * "Don't care" match must be omitted. -[source,pseudo] +[source,python] ---- assert(BytestringValid(match.range().low())) assert(BytestringValid(match.range().high())) @@ -3137,7 +3136,7 @@ assert(low != min_field_value || high != max_field_value) * The binary string encoding of the value must conform to the xref:sec-bytestrings[Bytestrings] requirements. -[source,pseudo] +[source,python] ---- assert(BytestringValid(match.optional().value())) ---- @@ -3364,7 +3363,7 @@ fields may be used to select and filter results: For example, in order to read all entries from all tables from device 3, the client can use the following `ReadRequest` message. -[source,proto] +[source,protobuf] ---- device_id: 3 entities { @@ -3381,7 +3380,7 @@ In order to read all entries with priority 11 from a specific table (with id 0x0212ab34) from device 3, the client can use the following `ReadRequest` message: -[source,proto] +[source,protobuf] ---- device_id: 3 entities { @@ -3421,7 +3420,7 @@ table t { The following `WriteRequest` message can be used to add 2 entries: -[source,proto] +[source,protobuf] ---- device_id: 3 entities { @@ -3447,7 +3446,7 @@ The first entry is a "don't care" entry, while the second one matches all The following `ReadRequest` message will return *all* entries in the table, not just the "don't care" entry. -[source,proto] +[source,protobuf] ---- device_id: 3 entities { @@ -3897,7 +3896,7 @@ the one shot message. For example, consider the action selector table defined xref:sec-action-profile-member-and-group[role]. This table could be programmed with the following one shot update: -[source,proto] +[source,protobuf] ---- table_entry { table_id: 0x0212ab34 @@ -4062,7 +4061,7 @@ P4Runtime message can be used for all three types of PSA counters --- `PACKETS`, * `byte_count` is an `int64`, corresponding to the number of octets. * `packet_count` is an `int64`, corresponding to the number of packets. -[source,proto] +[source,protobuf] ---- message CounterData { int64 byte_count = 1; @@ -4085,7 +4084,7 @@ P4Runtime client may modify the associated direct counter entry using the `DirectCounterEntry` message. Once the table entry is deleted the associated direct counter entry can no longer be accessed. -[source,proto] +[source,protobuf] ---- message DirectCounterEntry { TableEntry table_entry = 1; @@ -4208,7 +4207,7 @@ primary purpose of the color counters is for debugging purposes. A meter may be configured as a direct or indirect instance, similar to a counter. The `MeterConfig` P4Runtime message represents meter configuration. -[source,proto] +[source,protobuf] ---- message MeterConfig { int64 cir = 1; // Committed Information Rate @@ -4246,7 +4245,7 @@ P4Runtime client may modify the associated direct meter entry using the `DirectMeterEntry` message. Once the table entry is deleted the associated direct meter entry can no longer be accessed. -[source,proto] +[source,protobuf] ---- message DirectMeterEntry { TableEntry table_entry = 1; @@ -4310,7 +4309,7 @@ follows: represents the per color counter values associated with the corresponding meter. -[source,proto] +[source,protobuf] ---- message MeterEntry { uint32 meter_id = 1; @@ -4402,7 +4401,7 @@ control arp_multicast(inout H hdr, inout M smeta) { At runtime, the client writes the following update in the target (shown in Protobuf text format). -[source,proto] +[source,protobuf] ---- type: INSERT entity { @@ -4539,7 +4538,7 @@ control clone_low_ttl(inout H hdr, inout M smeta) { At runtime, the client writes the following update in the target (shown in Protobuf text format). -[source,proto] +[source,protobuf] ---- type: INSERT entity { @@ -4646,7 +4645,7 @@ state parse_l2 { The corresponding entry in the P4Info for this Value Set is: -[source,proto] +[source,protobuf] ---- value_sets { preamble { @@ -4665,7 +4664,7 @@ value_sets { At runtime, the client writes the following update in the target (shown in Protobuf text format). -[source,proto] +[source,protobuf] ---- type: MODIFY entity { @@ -4898,7 +4897,7 @@ while (true) { This is used to support a P4 extern entity that is not part of PSA. It is defined as: -[source,proto] +[source,protobuf] ---- message ExternEntry { uint32 extern_type_id = 1; @@ -4930,7 +4929,7 @@ explain how error reporting works in the failure case. gRPC uses `grpc::Status` cite:[gRPCStatus] to represent the status returned by an RPC. It has 3 attributes: -[source,cpp] +[source,c++] ---- StatusCode code_; grpc::string error_message_; @@ -4942,7 +4941,7 @@ overall RPC status. The `error_message_` is a developer-facing error message, which should be in English. The `binary_error_details_` carries a serialized `google.rpc.Status` message cite:[ProtoStatus] message, which has 3 fields: -[source,proto] +[source,protobuf] ---- int32 code = 1; // see code.proto string message = 2; @@ -5067,7 +5066,7 @@ to reject the program. The `Write` RPC updates one or more P4 entities on the target. The request is defined as follows: -[source,proto] +[source,protobuf] ---- message WriteRequest { uint64 device_id = 1; @@ -5102,7 +5101,7 @@ before processing the `updates` list: The updates field is a list of P4 entity updates to be applied. Each update is defined as: -[source,proto] +[source,protobuf] ---- message Update { enum Type { @@ -5307,7 +5306,7 @@ binary_error_details { The `Read` RPC retrieves one or more P4 entities from the P4Runtime server. The request is defined as: -[source,proto] +[source,protobuf] ---- message ReadRequest { uint64 device_id = 1; @@ -5333,7 +5332,7 @@ require an `election_id`, and they do not require the presence of an open The `Read `response consists of a sequence of messages (a gRPC `stream`) with each message defined as: -[source,proto] +[source,protobuf] ---- message ReadResponse { repeated Entity entities = 1; @@ -5378,7 +5377,7 @@ For example, in a *request* of type `CounterEntry`: To read the entire forwarding state for a given device, the P4Runtime client can generate the following `ReadRequest`: -[source,proto] +[source,protobuf] ---- device_id: entities { @@ -5509,7 +5508,7 @@ be difficult to implement correctly. A P4Runtime client may configure the P4Runtime target with a new P4 pipeline by invoking the `SetForwardingPipelineConfig RPC`. The request is defined as: -[source,proto] +[source,protobuf] ---- message SetForwardingPipelineConfigRequest { enum Action { @@ -5590,7 +5589,7 @@ different mechanism). In such cases, the RPC should return an The forwarding-pipeline configuration of the target can be retrieved by invoking the `GetForwardingPipelineConfig RPC`. The request is defined as: -[source,proto] +[source,protobuf] ---- message GetForwardingPipelineConfigRequest { enum ResponseType { @@ -5626,7 +5625,7 @@ its value can be one of: The response contains the `ForwardingPipelineConfig` for the specified device: -[source,proto] +[source,protobuf] ---- message GetForwardingPipelineConfigResponse { ForwardingPipelineConfig config = 1; @@ -5677,7 +5676,7 @@ in the P4Info using the `ControllerPacketMetadata` messages. Both `PacketIn` and `PacketOut` stream messages share the same fields and are defined as follows: -[source,proto] +[source,protobuf] ---- // Packet sent from the controller to the switch. message PacketOut { @@ -5759,7 +5758,7 @@ reset. In fact, a full restart is the only way to reset the `election_id`. The `MasterArbitrationUpdate` message is defined as follows: -[source,proto] +[source,protobuf] ---- message MasterArbitrationUpdate { uint64 device_id = 1; @@ -5862,7 +5861,7 @@ server software, the channel or the client can handle. Here is a reasonable pseudo-code implementation for idle timeout for table entries: -[source,cpp] +[source,c++] ---- IdleTimeoutStream stream; @@ -6089,7 +6088,7 @@ ports in the device-specific port number space. P4Runtime reserves device-independent and controller-specific 32-bit constants for the CPU port and the recirculation port as follows: -[source,proto] +[source,protobuf] ---- enum SdnPort { SDN_PORT_UNKNOWN = 0; @@ -6369,7 +6368,7 @@ extern MyNewPacketCounter { externs. It is recommended that *p4info-ext* include a `P4Ids` message based on the one in p4info.proto that the P4 compiler can refer to when xref:sec-id-allocation[assigning IDs] to each extern instance. -[source,proto] +[source,protobuf] ---- message P4Ids { enum Prefix { @@ -6386,7 +6385,7 @@ message P4Ids { xref:sec-p4info-extern[`p4.config.v1.ExternInstance`] message as the `info` field, which is of type `Any` cite:[ProtoAny]. -[source,proto] +[source,protobuf] ---- message MyNewPacketCounter { // corresponds to the T type parameter in the P4 extern definition @@ -6407,7 +6406,7 @@ embedded in an xref:sec-extern-entry[`ExternEntry`] message generated by the P4Runtime client. Here is a possible Protobuf message for our `MyNewPacketCounter` P4 extern: -[source,proto] +[source,protobuf] ---- // This message enables reading / writing data to the counter at the provided // index @@ -6754,7 +6753,7 @@ channel. As a rule of thumb, it might make sense to allow for at least `8192 + MAX_UPDATES_PER_WRITE * 100` bytes of metadata. For example, in C++, one can create a client channel as follows: -[source,cpp] +[source,c++] ---- const int MAX_UPDATES_PER_WRITE = 100; ::grpc::ChannelArguments arguments; @@ -6778,7 +6777,7 @@ setting the `grpc.max_receive_message_length` when building the gRPC server. For example, in C++, one can set the maximum receive message size as follows: -[source,cpp] +[source,c++] ---- const int MAX_RECEIVE_MESSAGE_SIZE = 128 * 1024 * 1024; // 128MB ::grpc::ServerBuilder server_builder; diff --git a/docs/v1/README.md b/docs/v1/README.md index e0d2141a..4d69cac9 100644 --- a/docs/v1/README.md +++ b/docs/v1/README.md @@ -5,62 +5,67 @@ specification document. # Markup version -The markup version uses Madoko (https://www.madoko.net) to produce +The markup version uses AsciiDoc (https://docs.asciidoctor.org/) to produce HTML and PDF versions of the documentation. Pre-built versions of the documentation are available on the [P4.org specifications page](https://p4.org/specs). Files: -- `P4Runtime-spec.mdk` is the main file. -- assets: Figures - - `*.odg` - OfficeLibre source drawing file used to export images. These are +- `P4Runtime-Spec.adoc` is the main file. +- resources: + - figs + - `*.odg` - OfficeLibre source drawing file used to export images. These are bulk-rendered at build time into .svg and .png images via `soffice` command-line (required in build environment) + - fonts + - `*.ttf` - Type font source file used to export fonts. + - theme: + - `*.yaml` - Describes how PDF P4Runtime specification will be displayed. + - `*.css` - Describes how HTML P4Runtime specification will displayed. + - `*.bib` - Bibliography file that contains a list of bibliographical item, such as articles, books, and theses. - `Makefile` builds documentation in the build subdirectory -- `p4.json` is providing custom syntax highlighting for P4. It is a rip off from - the cpp.json provided by Madoko (the "extend" clause does not work, my version - of Madoko asks for a "tokenizer" to be defined). Style customization for each - token can be done using CSS style attributes (see `token.keyword` in - `P4Runtime-spec.mdk`). + ## Building -The easiest way to render the Madoko specification documentation is to use the -`p4lang/p4rt-madoko:latest` Docker` image: +The easiest way to render the AsciiDoc specification documentation is to use the +`p4lang/p4rt-asciidoc:latest` Docker` image: - docker run -v `pwd`/docs/v1:/usr/src/p4-spec p4lang/p4rt-madoko:latest make + docker run -v `pwd`/docs/v1:/usr/src/p4-spec p4lang/p4rt-asciidoc:latest make ### Linux ``` -sudo apt-get install nodejs -sudo npm install madoko -g -sudo apt-get install libreoffice -sudo apt-get install texlive-science texlive-xetex -make [all | html | pdf ] +rvm install ruby-3.3.1 +rvm use 3.3.1 +gem install asciidoctor +gem install asciidoctor-pdf +gem install asciidoctor-bibtex +gem install asciidoctor-mathematical +gem install prawn-gmagick +gem install rouge +gem install asciidoctor-bibtex +gem install asciidoctor-lists +gem install prawn-gmagick +make ``` -In particular (on Ubuntu 16.04 at least), don't try `sudo apt-get install npm` -because `npm` is already included and this will yield a bunch of confusing error -messages from `apt-get`. - -`dvipng` is used to render "math" inside BibTex (used for bibliography) -titles, so you will need to install it as well. On Debian-based Linux, it can be -done with `sudo apt-get install dvipng`. +You can use the [local +installation](https://github.com/p4lang/p4-spec/blob/main/p4-16/spec/install-asciidoctor-linux.sh) +method. ### MacOS We use the [local -installation](http://research.microsoft.com/en-us/um/people/daan/madoko/doc/reference.html#sec-installation-and-usage) -method. For Mac OS, I installed node.js using Homebrew and then Madoko using -npm: +installation](https://github.com/p4lang/p4-spec/blob/main/p4-16/spec/install-asciidoctor-linux.sh) +method. For Mac OS you can install AsciiDoc using Homebrew via: ``` -brew install node.js -npm install madoko -g +brew install asciidoctor ``` -Note that to build the PDF you need a functional TeX version installed. ### Windows -You need to install miktex [http://miktex.org/], madoko -[https://www.madoko.net/] and node.js [https://nodejs.org/en/]. To -build you can invoke the make.bat script. +You need to install chocolatey [https://chocolatey.org/] or rubyInstaller [https://rubyinstaller.org/downloads/]. +Once you’ve installed Ruby , open a terminal and type: +``` + gem install asciidoctor +``` diff --git a/docs/v1/bibref_no_title.js b/docs/v1/bibref_no_title.js deleted file mode 100644 index a67417f0..00000000 --- a/docs/v1/bibref_no_title.js +++ /dev/null @@ -1,5 +0,0 @@ -$(document).ready(function(){ - $('a').filter(".bibref").hover(function(e){ - $(this).attr('title', ''); - }); -}); diff --git a/docs/v1/cpp.json b/docs/v1/cpp.json deleted file mode 100644 index 19c08a42..00000000 --- a/docs/v1/cpp.json +++ /dev/null @@ -1,105 +0,0 @@ -{ - "displayName": "C++", - "name": "cpp", - "mimeTypes": ["text/cpp","text/c"], - "fileExtensions": ["cpp","c++","h","c"], - - "lineComment": "//", - "blockCommentStart": "/*", - "blockCommentEnd": "*/", - - "keywords": [ - "alignas", "alignof", "and", "and_eq", "asm", "auto", "bitand", "bitor", "bool", "break", "case", - "catch", "char", "char16_t", "char32_t", "class", "compl", "const", "constexpr", "const_cast", - "continue", "decltype", "default", "delete", "do", "double", "dynamic_cast", "else", - "enum", "explicit", "export", "extern", "false", "float", "for", "friend", "goto", "if", "inline", - "int", "long", "mutable", "namespace", "new", "noexcept", "not", "not_eq", "nullptr", "operator", - "or", "or_eq", "private", "protected", "public", "register", "reinterpret_cast", - "return", "short", "signed", "sizeof", "static", "static_assert", "static_cast", "struct", - "switch", "template", "this", "thread_local", "throw", "true", "try", "typedef", "typeid", - "typename", "union", "unsigned", "using", "virtual", "void", "volatile", "wchar_t", "while", - "xor", "xor_eq" - ], - - "typeKeywords": [ - "bool", "double", "byte", "int", "short", "char", "void", "long", "float", - "char32_t", "unsigned", "wchar_t", "char16_t" - ], - - "directives": [ - "include","if","endif","ifdef","define","line","warning","error" - ], - - "operators": [ - "=", ">", "<", "!", "~", "?", ":", - "==", "<=", ">=", "!=", "&&", "||", "++", "--", - "+", "-", "*", "/", "&", "|", "^", "%", "<<", - ">>", ">>>", "+=", "-=", "*=", "/=", "&=", "|=", - "^=", "%=", "<<=", ">>=", ">>>=" - ], - - "symbols": "[=>)([A-Z][\\w]*)", ["keyword", "identifier"] ], - ["[A-Z][\\w]*(?!\\s*[\\w\\(])", "type.identifier" ], - ["[A-Z][A-Z0-9_]*(?![\\w\\(])", "type.identifier" ], - - ["^(\\s*#)(\\w+)(.*)", { "cases": { - "$2@directives": ["namespace","namespace","string"], - "@default": ["meta","meta","string"] - } } ], - - { "include": "@whitespace" }, - - ["[{}()\\[\\]]", "@brackets"], - ["[<>](?!@symbols)", "@brackets"], - ["@symbols", { "cases": { "@operators": "operator", - "@default" : "" } } ], - - ["\\d*\\.\\d+([eE][\\-+]?\\d+)?[fFdD]?", "number.float"], - ["0[xX][0-9a-fA-F_]*[0-9a-fA-F][Ll]?", "number.hex"], - ["0[0-7_]*[0-7][Ll]?", "number.octal"], - ["0[bB][0-1_]*[0-1][Ll]?", "number.binary"], - ["\\d+[lL]?", "number"], - - ["[;,.]", "delimiter"], - - ["[lL]\"([^\"\\\\]|\\\\.)*$", "string.invalid" ], - ["\"", "string", "@string" ], - - ["'[^\\\\']'", "string"], - ["(')(@escapes)(')", ["string","string.escape","string"]], - ["'", "string.invalid"] - ], - - "whitespace": [ - ["[ \\t\\r\\n]+", "white"], - ["\\/\\*", "comment", "@comment" ], - ["\\/\\/.*$", "comment"] - ], - - "comment": [ - ["[^\\/*]+", "comment" ], - ["\\/\\*", "comment.invalid" ], - ["\\*/", "comment", "@pop" ], - ["[\\/*]", "comment" ] - ], - - "string": [ - ["[^\\\\\"]+", "string"], - ["@escapes", "string.escape"], - ["\\\\.", "string.escape.invalid"], - ["\"", "string", "@pop" ] - ] - } -} diff --git a/docs/v1/install-asciidoctor-linux.sh b/docs/v1/install-asciidoctor-linux.sh deleted file mode 100755 index f80999c9..00000000 --- a/docs/v1/install-asciidoctor-linux.sh +++ /dev/null @@ -1,210 +0,0 @@ -#! /bin/bash - -# Copyright 2024 Andy Fingerhut - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -linux_version_warning() { - 1>&2 echo "Found ID ${ID} and VERSION_ID ${VERSION_ID} in /etc/os-release" - 1>&2 echo "This script only supports these:" - 1>&2 echo " ID ubuntu, VERSION_ID in 20.04 22.04 24.04" - #1>&2 echo " ID fedora, VERSION_ID in 36 37 38" - 1>&2 echo "" - 1>&2 echo "Proceed installing manually at your own risk of" - 1>&2 echo "significant time spent figuring out how to make it all" - 1>&2 echo "work, or consider getting VirtualBox and creating a" - 1>&2 echo "virtual machine with one of the tested versions." -} - -get_used_disk_space_in_mbytes() { - echo $(df --output=used --block-size=1M . | tail -n 1) -} - -abort_script=0 - -if [ ! -r /etc/os-release ] -then - 1>&2 echo "No file /etc/os-release. Cannot determine what OS this is." - linux_version_warning - exit 1 -fi -source /etc/os-release -PROCESSOR=`uname --machine` - -supported_distribution=0 -tried_but_got_build_errors=0 -if [ "${ID}" = "ubuntu" ] -then - case "${VERSION_ID}" in - 20.04) - supported_distribution=1 - OS_SPECIFIC_PACKAGES="libgdk-pixbuf2.0-dev" - ;; - 22.04) - supported_distribution=1 - OS_SPECIFIC_PACKAGES="libgdk-pixbuf-2.0-dev" - ;; - 24.04) - supported_distribution=1 - OS_SPECIFIC_PACKAGES="libgdk-pixbuf-2.0-dev" - ;; - esac -elif [ "${ID}" = "fedora" ] -then - case "${VERSION_ID}" in - 38) - supported_distribution=0 - ;; - 39) - supported_distribution=0 - ;; - 40) - supported_distribution=0 - ;; - esac -fi - -if [ ${supported_distribution} -eq 1 ] -then - echo "Found supported ID ${ID} and VERSION_ID ${VERSION_ID} in /etc/os-release" -else - linux_version_warning - if [ ${tried_but_got_build_errors} -eq 1 ] - then - 1>&2 echo "" - 1>&2 echo "This OS has been tried at least onc before, but" - 1>&2 echo "there were errors during a compilation or build" - 1>&2 echo "step that have not yet been fixed. If you have" - 1>&2 echo "experience in fixing such matters, your help is" - 1>&2 echo "appreciated." - fi - exit 1 -fi - -min_free_disk_MBytes=`expr 1 \* 1024` -free_disk_MBytes=`df --output=avail --block-size=1M . | tail -n 1` - -if [ "${free_disk_MBytes}" -lt "${min_free_disk_MBytes}" ] -then - free_disk_comment="too low" - abort_script=1 -else - free_disk_comment="enough" -fi - -echo "Minimum free disk space to run this script: ${min_free_disk_MBytes} MBytes" -echo "Free disk space on this system from df output: ${free_disk_MBytes} MBytes -> $free_disk_comment" - -if [ "${abort_script}" == 1 ] -then - echo "" - echo "Aborting script because system has too little free disk space" - exit 1 -fi - -echo "Passed all sanity checks" - -DISK_USED_START=`get_used_disk_space_in_mbytes` - -set -e -set -x - -echo "------------------------------------------------------------" -echo "Time and disk space used before installation begins:" -set -x -date -df -h . -df -BM . -TIME_START=$(date +%s) - -# On new systems if you have never checked repos you should do that first - -# Install a few packages (vim is not strictly necessary -- installed for -# my own convenience): -if [ "${ID}" = "ubuntu" ] -then - sudo apt-get --yes install gnupg2 curl -elif [ "${ID}" = "fedora" ] -then - sudo dnf -y update - sudo dnf -y install git vim -fi - -gpg2 --keyserver keyserver.ubuntu.com --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB -curl -sSL https://get.rvm.io | bash -if [[ $UID == 0 ]]; then - source /usr/local/rvm/scripts/rvm -else - source $HOME/.rvm/scripts/rvm -fi -rvm install ruby-3.3.1 -rvm use 3.3.1 -gem install asciidoctor -gem install asciidoctor-pdf -gem install asciidoctor-bibtex -# Additional installations to enable installing -# asciidoctor-mathematical and prawn-gmagick -sudo apt-get --yes install cmake flex libglib2.0-dev libcairo2-dev libpango1.0-dev libxml2-dev libwebp-dev libzstd-dev libgraphicsmagick1-dev libmagickwand-dev ${OS_SPECIFIC_PACKAGES} -gem install asciidoctor-mathematical -gem install prawn-gmagick -gem install rouge -gem install asciidoctor-bibtex -gem install asciidoctor-lists -gem install prawn-gmagick - -which ruby -ruby --version -which gem -gem --version -which asciidoctor -asciidoctor --version -which asciidoctor-pdf -asciidoctor-pdf --version - -set +e - -set +x -echo "------------------------------------------------------------" -echo "Time and disk space used when installation was complete:" -set -x -date -df -h . -df -BM . -TIME_END=$(date +%s) -set +x -echo "" -echo "Elapsed time for various install steps:" -echo "Total time : $(($TIME_END-$TIME_START)) sec" -set -x - -DISK_USED_END=`get_used_disk_space_in_mbytes` - -set +x -echo "All disk space utilizations below are in MBytes:" -echo "" -echo "DISK_USED_START ${DISK_USED_START}" -echo "DISK_USED_END ${DISK_USED_END}" -echo "DISK_USED_END - DISK_USED_START : $((${DISK_USED_END}-${DISK_USED_START})) MBytes" - -echo "----------------------------------------------------------------------" -echo "CONSIDER READING WHAT IS BELOW" -echo "----------------------------------------------------------------------" -echo "" -echo "You should add this command in a shell startup script, e.g." -echo "$HOME/.bashrc if you use the Bash shell:" -echo "" -echo " source \$HOME/.rvm/scripts/rvm" -echo "" -echo "----------------------------------------------------------------------" -echo "CONSIDER READING WHAT IS ABOVE" -echo "----------------------------------------------------------------------" diff --git a/docs/v1/make.bat b/docs/v1/make.bat deleted file mode 100644 index 2f789eaa..00000000 --- a/docs/v1/make.bat +++ /dev/null @@ -1 +0,0 @@ -madoko --pdf -vv --png --odir=build P4Runtime-spec.mdk \ No newline at end of file diff --git a/docs/v1/p4.json b/docs/v1/p4.json deleted file mode 100644 index f8148c37..00000000 --- a/docs/v1/p4.json +++ /dev/null @@ -1,113 +0,0 @@ -{ - "displayName": "P4", - "name": "p4", - - "lineComment": "//", - "blockCommentStart": "/*", - "blockCommentEnd": "*/", - - "keywords": [ "action", "apply", "control", "default", "else", - "extern", "exit", "false", "if", - "package", "parser", "return", "select", "state", "switch", - "table", "transition", "true", "typedef", "verify" - ], - - "extraKeywords": [], - - "typeKeywords": [ - "bool", "bit", "const", "enum", "entries", "error", "header", "header_union", "in", "inout", "int", "match_kind", "out", "tuple", "struct", "varbit", "void", "type" - ], - - "extraTypeKeywords": [], - - "directives": [ - "include","if","endif","ifdef","define","ifndef","undef","line" - ], - - "annotations": [ - "atomic", "defaultonly", "name", "priority", "tableonly", "hidden", "globalname" - ], - - "operators": [ - "=", ">", "<", "!", "~", "?", ":", - "==", "<=", ">=", "!=", "&&", "||", "++", - "+", "-", "*", "/", "&", "|", "^", "%", "<<", - ">>", "&&&", ".." - ], - - "extraOperators": [], - - "symbols": "[=>](?!@symbols)", "@brackets"], - ["@symbols", { "cases": { - "@operators": "operator", - "@extraOperators": "operator.extra", - "@default" : "" } } ], - - ["\\d*\\.\\d+([eE][\\-+]?\\d+)?[fFdD]?", "number.float"], - ["0[xX][0-9a-fA-F_]*[0-9a-fA-F][Ll]?", "number.hex"], - ["0[0-7_]*[0-7][Ll]?", "number.octal"], - ["0[bB][0-1_]*[0-1][Ll]?", "number.binary"], - ["\\d+[lL]?", "number"], - - ["[;,.]", "delimiter"], - - ["[lL]\"([^\"\\\\]|\\\\.)*$", "string.invalid"], - ["\"", "string", "@string" ], - - - ["'[^\\\\']'", "string"], - ["(')(@escapes)(')", ["string","string.escape","string"]], - ["'", "string.invalid"] - ], - - "whitespace": [ - ["[ \\t\\r\\n]+", "white"], - ["\\/\\*", "comment", "@comment" ], - ["\\/\\/.*$", "comment"] - ], - - "comment": [ - ["[^\\/*]+", "comment" ], - ["\\/\\*", "comment.invalid" ], - ["\\*/", "comment", "@pop" ], - ["[\\/*]", "comment" ] - ], - - "string": [ - ["[^\\\\\"]+", "string"], - ["@escapes", "string.escape"], - ["\\\\.", "string.escape.invalid"], - ["\"", "string", "@pop" ] - ] - } -} diff --git a/docs/v1/proto.json b/docs/v1/proto.json deleted file mode 100644 index 0a09fb9c..00000000 --- a/docs/v1/proto.json +++ /dev/null @@ -1,86 +0,0 @@ -{ - "displayName": "Protobuf", - "name": "proto", - - "lineComment": "//", - "blockCommentStart": "/*", - "blockCommentEnd": "*/", - - "keywords": [ - "syntax", "import", "package", "service", - "rpc", "returns", - "message", "oneof", "repeated", - "reserved", - "=" - ], - - "extraKeywords": ["optimize_for", "cc_enable_arenas", "objc_class_prefix", "deprecated"], - - "typeKeywords": [ - "enum", - "double", "float", - "int32", "int64", "uint32", "uint64", "sint32", "sint64", - "fixed32", "fixed64", "sfixed32", "sfixed64", - "bool", - "string", "bytes" - ], - - "directives": [], - - "symbols": "[=>&2 echo " Ubuntu 16.04" - 1>&2 echo " Ubuntu 18.04" - 1>&2 echo " Ubuntu 20.04" - 1>&2 echo " Ubuntu 22.04" -} - -echo "------------------------------------------------------------" -echo "Purpose of this script:" -echo "" -echo "On a supported operating system that has not had any additional" -echo "packages installed yet, install a set of packages that are" -echo "needed to successfully create the HTML and PDF versions of these" -echo "documents from their Madoko source files (files with names that" -echo "end with '.mdk'):" -echo "" -echo "+ The P4_16 language specification" -echo "+ The Portable Switch Architecture (PSA) specification" -echo "" -echo "While it would be nice if I could assure you that this script" -echo "will work on a system that already had many packages installed," -echo "I do not know which packages might have conflicts with each" -echo "other." -echo "------------------------------------------------------------" - -# This is where the application gnome-font-viewer copies font files -# when a user clicks the "Install" button. -FONT_INSTALL_DIR="${HOME}/.local/share/fonts" - -warning() { - 1>&2 echo "This script has only been tested on these OS versions:" - print_supported_os_versions -} - -lsb_release >& /dev/null -if [ $? != 0 ] -then - 1>&2 echo "No 'lsb_release' found in your command path." - warning - exit 1 -fi - -DISTRIBUTOR_ID=`lsb_release -si` -UBUNTU_RELEASE=`lsb_release -sr` - -if [ ${DISTRIBUTOR_ID} != "Ubuntu" -o \( ${UBUNTU_RELEASE} != "16.04" -a ${UBUNTU_RELEASE} != "18.04" -a ${UBUNTU_RELEASE} != "20.04" -a ${UBUNTU_RELEASE} != "22.04" \) ] -then - warning - 1>&2 echo "" - 1>&2 echo "Here is what command 'lsb_release -a' shows this OS to be:" - lsb_release -a - exit 1 -fi - -set -ex - -set +x -echo "------------------------------------------------------------" -echo "Time and disk space used before installation begins:" -set -x -date -df -BM . - -# Common packages to install on all tested Ubuntu versions -sudo apt-get --yes install git curl make nodejs npm texlive-xetex dvipng - -if [[ "${UBUNTU_RELEASE}" > "18" ]] -then - # Only needed for Ubuntu 18.04 and later - sudo apt-get --yes install texlive-science -else - # Only needed for Ubuntu 16.04 - sudo apt-get --yes install nodejs-legacy texlive-generic-extra texlive-math-extra -fi - -# Common packages to install on all tested Ubuntu versions -sudo npm install madoko -g - -set +x -echo "------------------------------------------------------------" -echo "Time and disk space used just before 'apt clean':" -set -x -date -df -BM . - -# After install of the packages above, this command often seems to -# help reduce the disk space used by a gigabyte or so. -sudo apt clean - -# On a freshly installed Ubuntu 16.04 system, added about 1.3G to the -# used disk space, although temporarily went about 1 GB more than that -# before 'sudo apt clean'. - -# On a freshly installed Ubuntu 18.04 system, added about 0.8G. - -# Retrieve and install fonts -mkdir -p "${FONT_INSTALL_DIR}" -curl -fsSL --output "${FONT_INSTALL_DIR}/UtopiaStd-Regular.otf" https://raw.github.com/p4lang/p4-spec/gh-pages/fonts/UtopiaStd-Regular.otf -curl -fsSL --output "${FONT_INSTALL_DIR}/luximr.ttf" https://raw.github.com/p4lang/p4-spec/gh-pages/fonts/luximr.ttf - -set +x -echo "------------------------------------------------------------" -echo "Time and disk space used when installation was complete:" -set -x -date -df -BM . - -if [ ${DISTRIBUTOR_ID} == "Ubuntu" -a ${UBUNTU_RELEASE} == "22.04" ] -then - set +x - 1>&2 echo "" - 1>&2 echo "------------------------------------------------------------" - 1>&2 echo "WARNING!" - 1>&2 echo "------------------------------------------------------------" - 1>&2 echo "" - 1>&2 echo "While this script can successfully install the required" - 1>&2 echo "packages on an Ubuntu 22.04 system like this one, these" - 1>&2 echo "installed packages are different versions than for" - 1>&2 echo "other supported OS's, and in our testing they GIVE" - 1>&2 echo "ERRORS and FAIL TO BUILD HTML and PDF files for the P4" - 1>&2 echo "specifications. See this issue:" - 1>&2 echo "" - 1>&2 echo " https://github.com/p4lang/p4-spec/issues/1115" - 1>&2 echo "" - 1>&2 echo "If you know how to fix this, your help is much appreciated." -fi diff --git a/tools/madokolint.conf.json b/tools/madokolint.conf.json deleted file mode 100644 index 3da443b6..00000000 --- a/tools/madokolint.conf.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "keywords": [ - { - "category": "P4Runtime message", - "keywords": [ - "WriteRequest", - "WriteResponse", - "ReadRequest", - "ReadResponse", - "SetForwardingPipelineConfigRequest", - "SetForwardingPipelineConfigResponse", - "GetForwardingPipelineConfigRequest", - "GetForwardingPipelineConfigResponse", - "StreamMessageRequest", - "StreamMessageResponse" - ] - }, - { - "category": "gRPC error code", - "keywords": [ - "CANCELLED", - "UNKNOWN", - "INVALID_ARGUMENT", - "DEADLINE_EXCEEDED", - "NOT_FOUND", - "ALREADY_EXISTS", - "PERMISSION_DENIED", - "UNAUTHENTICATED", - "RESOURCE_EXHAUSTED", - "FAILED_PRECONDITION", - "ABORTED", - "OUT_OF_RANGE", - "UNIMPLEMENTED", - "INTERNAL", - "UNAVAILABLE", - "DATA_LOSS" - ] - } - ] -} diff --git a/tools/madokolint.py b/tools/madokolint.py deleted file mode 100755 index 1438728f..00000000 --- a/tools/madokolint.py +++ /dev/null @@ -1,327 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2019 VMware, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -# DISCLAIMER: This is a work in progress. This linter was written specifically -# for the P4Runtime specification document and may not be useful for other -# Madoko documents, as it may be making some assumptions as to how the document -# was written. - -# TODO: handle Madoko includes (we do not use them for the P4Runtime spec)? - - -import argparse -from collections import namedtuple -import json -import os.path -import re -import sys -import traceback - - -DEFAULT_CONF = 'madokolint.conf.json' -LINE_WRAP_LENGTH = 80 - - -parser = argparse.ArgumentParser(description='Lint tool for Madoko code') -parser.add_argument('files', metavar='FILE', type=str, nargs='+', - help='Input files') -parser.add_argument('--conf', type=str, - help='Configuration file for lint tool') - - -class MadokoFmtError(Exception): - def __init__(self, filename, lineno, description): - self.filename = filename - self.lineno = lineno - self.description = description - - def __str__(self): - return "Unexpected Madoko code in file {} at line {}: {}".format( - self.filename, self.lineno, self.description) - - -class LintState: - def __init__(self): - self.errors_cnt = 0 - - def error(self, filename, lineno, line, description): - # TODO: print line later? - print("Error in file {} at line {}: {}.".format(filename, lineno, description)) - self.errors_cnt += 1 - - -lint_state = LintState() - - -class LintConf: - class BadConfException(Exception): - def __init__(self, what): - self.what = what - - def __str__(self): - return self.what - - - def __init__(self): - self.keywords = {} - - def build_from(self, conf_fp): - try: - conf_d = json.load(conf_fp) - for entry in conf_d['keywords']: - category = entry['category'] - for keyword in entry['keywords']: - if keyword in self.keywords: - raise LintConf.BadConfException( - "Keyword '{}' is present multiple times in configuration".format( - keyword)) - self.keywords[keyword] = category - except json.JSONDecodeError: - print("Provided configuration file is not a valid JSON file") - sys.exit(1) - except KeyError: - print("Provided JSON configuration file has missing attributes") - traceback.print_exc() - sys.exit(1) - except LintConf.BadConfException as e: - print(str(e)) - sys.exit(1) - - -lint_conf = LintConf() - - -class Context: - """A context is an object that is used to determine whether a specific "checker" (check_* - method) should visit a given line.""" - - def enter(self, line, filename, lineno): - """Called before visiting a line. - Returns True iff the checker should visit the given line. - """ - return True - - def exit(self, line, filename, lineno): - """Called after visiting a line.""" - pass - - -class ContextSkipBlocks(Context): - """A context used to only visit Madoko code outside of blocks.""" - - Block = namedtuple('Block', ['num_tildes', 'name']) - - def __init__(self): - self.p_block = re.compile('^ *(?P~+) *(?:(?PBegin|End)(?: +))?(?P\w+)?') - self.blocks_stack = [] - - def enter(self, line, filename, lineno): - m = self.p_block.match(line) - if m: - num_tildes = len(m.group("tildes")) - has_begin = m.group("cmd") == "Begin" - has_end = m.group("cmd") == "End" - blockname = m.group("name") - - if has_begin: - self.blocks_stack.append(self.Block(num_tildes, blockname)) - return False - if has_end: - if not self.blocks_stack: - raise MadokoFmtError(filename, lineno, "Block end line but no block was begun") - expected = self.blocks_stack.pop() - if num_tildes != expected.num_tildes or blockname != expected.name: - raise MadokoFmtError( - filename, lineno, - "Block end line does not match last visited block begin line") - return False - if blockname is None: - if not self.blocks_stack: - raise MadokoFmtError(filename, lineno, "Block end line but no block was begun") - expected = self.blocks_stack.pop() - if num_tildes != expected.num_tildes: - raise MadokoFmtError( - filename, lineno, - "Block end line does not match last visited block begin line") - return False - self.blocks_stack.append(self.Block(num_tildes, blockname)) - return False - if self.blocks_stack: - return False - return True - - -# TODO: would "skip metadata" be more generic? -class ContextAfterTitle(Context): - """A context used to visit only Madoko code after the [TITLE] block element. - """ - - def __init__(self, *args): - self.title_found = False - self.p_title = re.compile('^ *\[TITLE\] *$') - - def enter(self, line, filename, lineno): - if self.title_found: - return True - self.title_found = self.p_title.match(line) is not None - return False - - -class ContextSkipHeadings(Context): - """A context used to skip headings (lines starting with #).""" - - def __init__(self, *args): - self.p_headings = re.compile('^ *#') - - def enter(self, line, filename, lineno): - return self.p_headings.match(line) is None - - -class ContextCompose(Context): - """A special context used to combine an arbitrary number of contexts.""" - - def __init__(self, *args): - self.contexts = list(args) - - def enter(self, line, filename, lineno): - res = True - for c in self.contexts: - # we use a short-circuit on purpose, if a context returns False we do not even enter - # subsequent contexts. This has some implications on how contexts are used. - res = res and c.enter(line, filename, lineno) - return res - - def exit(self, line, filename, lineno): - for c in self.contexts: - c.exit(line, filename, lineno) - - -def foreach_line(path, context, fn): - """Iterate over every line in the file. For each line, call fn iff the enter method of the - provided context returns True.""" - lineno = 1 - with open(path, 'r') as f: - for line in f: - if context.enter(line, path, lineno): - fn(line, lineno) - lineno += 1 - context.exit(line, path, lineno) - - -def check_line_wraps(path): - def check(line, lineno): - if "http" in line: # TODO: we can probably do better than this - return - if len(line) > LINE_WRAP_LENGTH + 1: # +1 for the newline characted - lint_state.error(path, lineno, line, - "is more than {} characters long".format(LINE_WRAP_LENGTH)) - - foreach_line(path, - ContextCompose(ContextAfterTitle(), ContextSkipBlocks(), ContextSkipHeadings()), - check) - - -def check_trailing_whitespace(path): - def check(line, lineno): - if len(line) >= 2 and line[-2].isspace(): - lint_state.error(path, lineno, line, "trailing whitespace") - - foreach_line(path, Context(), check) - - -def check_predefined_abbreviations(path): - abbreviations = { - 'e.g.': '⪚', - 'i.e.': '&ie;', - 'et al.': '&etal;', - } - - def check(line, lineno): - for k, v in abbreviations.items(): - if k in line: - lint_state.error(path, lineno, line, - "contains '{}', use '{}' instead".format(k, v)) - - foreach_line(path, ContextCompose(ContextAfterTitle(), ContextSkipBlocks()), check) - - -def check_keywords(path): - def check(line, lineno): - for word in line.split(): - if word not in lint_conf.keywords: - continue - category = lint_conf.keywords[word] - lint_state.error( - path, lineno, line, - "'{}' is a known keyword ({}), highlight it with backticks".format(word, category)) - - foreach_line(path, ContextCompose(ContextAfterTitle(), ContextSkipBlocks()), check) - - -def process_one(path): - check_line_wraps(path) - check_predefined_abbreviations(path) - check_trailing_whitespace(path) - check_keywords(path) - - -def main(): - args = parser.parse_args() - - for f in args.files: - if not os.path.isfile(f): - print("'{}' is not a valid file path".format(f)) - sys.exit(1) - _, ext = os.path.splitext(f) - if ext != ".mdk": - print("'{}' does not have an .mdk extension") - sys.exit(1) - - conf_path = None - if args.conf is not None: - if not os.path.isfile(args.conf): - print("'{}' is not a valid file path".format(args.conf)) - sys.exit(1) - conf_path = args.conf - elif os.path.isfile(DEFAULT_CONF): # search working directory - conf_path = DEFAULT_CONF - else: # search directory of Python script - this_dir = os.path.dirname(os.path.abspath(__file__)) - path = os.path.join(this_dir, DEFAULT_CONF) - if os.path.isfile(path): - conf_path = path - - if conf_path is not None: - with open(conf_path, 'r') as conf_fp: - lint_conf.build_from(conf_fp) - - for f in args.files: - try: - process_one(f) - except MadokoFmtError as e: - print(e) - - errors_cnt = lint_state.errors_cnt - print("**********") - print("Errors found: {}".format(errors_cnt)) - rc = 0 if errors_cnt == 0 else 2 - sys.exit(rc) - - -if __name__ == '__main__': - main()