Merge branch 'wellenvogel:master' into oldenv

.github/workflows/ci.yml

@@ -10,6 +10,8 @@ jobs:
         os: [ubuntu-latest]
 
     runs-on: ${{ matrix.os }}
+    env:
+      PIP_BREAK_SYSTEM_PACKAGES: 1
     steps:
       - uses: actions/checkout@v2
         with: 

.github/workflows/release.yml

@@ -18,6 +18,9 @@ jobs:
     # The type of runner that the job will run on
     runs-on: ubuntu-latest
 
+    env:
+      PIP_BREAK_SYSTEM_PACKAGES: 1
+
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it

Readme.md

@@ -50,6 +50,10 @@ For the list of hardware set ups refer to [Hardware](doc/Hardware.md).
 
 There is a couple of prebuild binaries that you can directly flash to your device. For other combinations of hardware there is an [online build service](doc/BuildService.md) that will allow you to select your hardware and trigger a build.
 
+Connectivity
+------------
+For details of the usage of serial devices and the USB connection refer to [Serial and USB](doc/serial-usb.md).<br>
+For details on the networking capabilities refer to [Networking](doc/network.md).
 
 Installation
 ------------
@@ -166,6 +170,28 @@ For details refer to the [example description](lib/exampletask/Readme.md).
 
 Changelog
 ---------
+[20241114](../../releases/tag/20241114)
+**********
+* UDP writer and reader - [#79](../../issues/79)
+* extensions for [user tasks](lib/exampletask/Readme.md)
+  * extend the Web UI with js and css
+  * register handler for Web Requests
+* Naming of the config file [#87](../../issues/87)
+* MTW from PGN130311 [#83](../../issues/83)
+* USB connection on S3 stops [#81](../../issues/81)
+* remove invalid true wind calc, allow to configure some mapping - partly [#75](../../issues/75)
+* correctly parse GSV messages [#50](../../issues/50)
+* minor adaptations from new version [#66](../../issues/66) 
+* new platform version 6.8.1
+* internally restructure the channel handling
+* add docs for [networking](doc/network.md) and [serial/USB](doc/serial-usb.md)
+* allow to configure the timeout(s) for the data display
+* new library versions - nmea2000 4.22.0, nmea0183 1.10.1
+* allow for builds completely without FastLED
+* wipe the nvs partition on factory reset (to handle corrupted config)
+* do not store the wifi settings in nvs on the system level [#78](../../issues/78)
+* rename of data: HDG->HDT, MHDG->HDM
+* adapt crash decoder tool to s3
 [20240428](../../releases/tag/20240428)
 **********
 * fix build error with M5 gps kit

doc/network.md

@@ -0,0 +1,80 @@
+Networking
+==========
+The gateway includes a couple of network functions to send and receive NMEA data on network connections and to use a WebBrowser for configuration and status display.
+
+To understand the networking functions you always have to consider 2 parts:
+
+The "physical" connection
+-------------------------
+For the gateway this is always a Wifi connection.
+It provides the ability to use it as an access point and/or to connect to another wifi network (wifi client).
+
+Access Point
+************
+When starting up it will create an own Wifi network (access point or AP) with the name of the system. You connect to this network like to any Wifi Hotspot.
+The initial password is esp32nmea2k. You should change this password on the configuration page in the Web UI.
+When you connect the gateway will provide you with an IP address (DHCP) - and it will also have an own IP address in this range (default: 192.168.15.1). If this IP address (range) conflicts with other devices (especially if you run multiple gateways and connect them) you can change the range at the system tab on the configuration page.<br>
+If you do not need the access point during normal operation you can set "stopApTime" so that it will shut down after this time (in minutes) if no device is connected. This will also reduce the power consumption.
+
+Wifi Client
+***********
+Beside the own access point the gateway can also connect to another Wifi network. You need to configure the SSID and password at the "wifi client" tab.
+On the status page you can see if the gateway is connected and which address it got within the network.
+
+You can use both networks to connect to the gateway. It announces it's services via [bonjour](https://developer.apple.com/bonjour/). So you normally should be able to reach your system using Name.local (name being the system name, default ESP32NMEA2K). Or you can use an app that allows for bonjour browsing.
+
+The "logical" connection
+------------------------
+After you connected a device to the gateway on either the access point or by using the same Wifi network you can easily access the Web UI with a browser - e.g. using the Name.local url.
+
+To send or receive NMEA data you additionally need to configure some connection between the gateway and your device(s).
+The gateway provides the following options for NMEA connections:
+
+TCP Server
+**********
+When using the TCP server you must configure a connection to the gateway _on the device_ that you would like to connect. The gateway listens at port 10110 (can be changed at the TCP server tab). So on your device you need to configure the address of the gateway and this port. The address depends on the network that you use to connect your device - either the address from the gateway access point (default: 192.168.15.1) - or the address that the gateway was given in the Wifi client network (check this on the status page).<br>
+If your device supports this you can also use the Name.local for the address - or let the device browse for a bonjour service.<br>
+The TCP server has a limit for the number of connections (can be configured, default: 6). As for any channel you can define what it will write and read on it's connections and apply filters.
+If you want to send NMEA2000 data via TCP you can enable "Seasmart out".
+
+TCP Client
+**********
+With this settings you can configure the gateway to establish a connection to another device that provides data via TCP. You need to know the address and port for that device. If the other device also uses bonjour (like e.g. a second gateway device) you can also use this name instead of the address.
+Like for the TCP server you can define what should be send or received with filters.
+
+UDP Reader
+**********
+UDP is distinct from TCP in that it does not establish a connection directly. Instead in sends/receives messages - without checking if they have been received by someone at all. Therefore it is also not able to ensure that really all messages are reaching their destination. But as the used protocols (NMEA0183, NMEA2000) are prepared for unreliable connections any way (for serial connections you normally have no idea if the data arrives) UDP is still a valid way of connecting devices.<br>
+One advantage of UDP is that you can send out messages as broadcast or multicast - thus allowing for multiple receivers.
+
+Small hint for __multicast__:<br>
+Normally in the environment the gateway will work you will not use multicast. If you want to send to multiple devices you will use broadcast. The major difference between them are 2 points:<br>
+  1. broadcast messages will not pass a real router (but they will be available to all devices connected to one access point)
+  2. broadcast messages will always be send to all devices - regardless whether they would like to receive them or not. This can create a bit more network traffic.
+
+Multicast requires that receivers announce their interest in receiving messages (by "joining" a so called multicast group) and this way only interested devices will receive such messages - and it is possible to configure routers in a way that they route multicast messages.
+
+To use the gateway UDP reader you must select from where you would like to receive messages. In any case you need to set up a port (default: 10110). Afterwards you need to decide on the allowed sources:
+  * _all_ (default): accept messages from both the access point and the wifi client network - both broadcast messages and directly addressed ones
+  * _ap_: only accept messages from devices that are connected to the access point
+  * _cli_: only accept messages from devices on the Wifi client network
+  * _mp-all_: you need to configure the multicast address(group) you would like to join and will receive multicast from both the access point and the wifi client network
+  * _mp-ap_: multicast only from the access point network
+  * _mp-cli_: multicast only from the wifi client network
+
+UDP Writer
+**********
+The UDP writer basically is the counterpart for the UDP reader.
+You also have to select where do you want the UDP messages to be sent to.
+  * _bc-all_ (default): Send the messages as broadcast to devices connected to the own access point and to devices on the wifi client network
+  * _bc-ap_: send the messages as broadcast only to the access point network
+  * _bc-cli_: send the messages as broadcast to the wifi client network
+  * _normal_: you need to configure a destination address (one device) that you want the messages to be send to
+  * _mc-all_: send messages as multicast to both the access point network and the wifi client network. _Hint_: Only devices that configured the same multicast address will receive such messages.
+  * _mc-ap_: multicast only to the access point network
+  * _mc-cli_: multicast only to the wifi client network.
+
+With the combination of UDP writer and UDP reader you can easily connect multiple gateway devices without a lot of configuration. Just configure one device as UDP writer (with the default settings) and configure other devices as UDP reader (also with default settings) - this way it does not matter how you connect the devices - all devices will receive the data that is sent by the first one.<br>
+__Remark:__ be carefull not to create loops when you would like to send data in both directions between 2 devices using UDP. Either use filters - or use TCP connections as they are able to send data in both directions on one connection (without creating a loop).
+
+If you want to forward NMEA2000 data from one gateway device to another, just enable "Seasmart out" at the sender side. This will encapsulate the NMEA2000 data in a NMEA0183 compatible format. The receiver will always automatically detect this format and handle the data correctly.

doc/serial-usb.md

@@ -0,0 +1,51 @@
+Serial and USB
+==============
+The gateway software uses the [arduino layer](https://github.com/espressif/arduino-esp32) on top of the [espressif IDF framework](https://github.com/espressif/esp-idf).
+The handling of serial devices is mainly done by the implementations in the arduino-espressif layer.
+The gateway code adds some buffering on top of this implementation and ensures that normally only full nmea records are sent.
+If the external connection is to slow the gateway will drop complete records.
+All handling of the serial channels is done in the main task of the gateway. 
+
+Serial Devices
+--------------
+THe arduino espressif layer provides the serial devices as [Streams](https://github.com/espressif/arduino-esp32/blob/master/cores/esp32/Stream.h#L48).
+Main implementations are [HardwareSerial](https://github.com/espressif/arduino-esp32/blob/3670e2bf2aca822f2e1225fdb0e0796e490005a8/cores/esp32/HardwareSerial.h#L71) - for the UARTS and [HWCDC](https://github.com/espressif/arduino-esp32/blob/3670e2bf2aca822f2e1225fdb0e0796e490005a8/cores/esp32/HWCDC.h#L43)(C3/S3 only) - for the USB CDC hardware device.
+
+For the github versions: arduino-espressif 3.20009 maps to github tag 2.0.9.
+
+The arduino espressif layer creates a couple of global instances for the serial devices (potentially depending on some defines).
+The important defines for C3/S3 are:
+
+ * ARDUINO_USB_MODE - the gateway always expects this to be 1
+ * ARDUINO_USB_CDC_ON_BOOT - 0 or 1 (CB in the table below)
+
+The created devices for framework 3.20009:
+
+| Device(Variable) | Type(ESP32) | C3/S3 CB=0 | C3/S3 CB=1 |
+| ------------ | ------- | ------ | -----|
+| Serial0 | --- | --- | HardwareSerial(0) |
+| Serial1 | HardwareSerial(1) | HardwareSerial(1) | HardwareSerial(1) |
+| Serial2 | HardwareSerial(2) | HardwareSerial(2) | HardwareSerial(2) |
+| USBSerial | --- | HWCDC | ---- | 
+| Serial | HardwareSerial(0) | HardwareSerial(0) | HWCDC | 
+
+Unfortunately it seems that in newer versions of the framework the devices could change.
+
+The gateway will use the following serial devices:
+
+* USBserial:<br> 
+  For debug output and NMEA as USB connection. If you do not use an S3/C3 with ARDUINO_USB_CDC_ON_BOOT=0 you need to add a <br>_define USBSerial Serial_ somewhere in your build flags or in your task header.<br>
+  Currently the gateway does not support setting the pins for the USB channel (that would be possible in principle only if an external PHY device is used and the USB is connected to a normal UART).
+* Serial1:<br>
+  If you define GWSERIAL_TYPE (1,2,3,4) - see [defines](../lib/hardware/GwHardware.h#23) or GWSERIAL_MODE ("UNI","BI","TX","RX") it will be used for the first serial channel.
+* Serial2:<br>
+  If you define GWSERIAL2_TYPE (1,2,3,4) - see [defines](../lib/hardware/GwHardware.h#23) or GWSERIAL2_MODE ("UNI","BI","TX","RX") it will be used for the second serial channel.
+
+Hints
+-----
+For normal ESP32 chips you need to set <br>_define USBSerial Serial_ <br>and you can use up to 2 serial channels beside the USB channel.
+For C3/S3 chips you can go for 2 options:
+1. set ARDUINO_USB_CDC_ON_BOOT=1: in This case you still need to set<br> _define USBSerial Serial_<br> You can use up to 2 serial channels in the gateway core - but you still have Serial0 available for a third channel (not yet supported by the core - but can be used in your user code)
+2. set ARDUINO_USB_CDC_ON_BOOT=0: in this case USBSerial is already defined as the USB channel. You can use 2 channels in the gateway core and optional you can use Serial in your user code.
+
+If you do not set any of the GWSERIAL* defines (and they are not set by the core hardware definitions) you can freely use Serial1 and / or Serial2 in your user code.

extra_script.py

@@ -18,6 +18,8 @@
 GEN_DIR='lib/generated'
 CFG_FILE='web/config.json'
 XDR_FILE='web/xdrconfig.json'
+INDEXJS="index.js"
+INDEXCSS="index.css"
 CFG_INCLUDE='GwConfigDefinitions.h'
 CFG_INCLUDE_IMPL='GwConfigDefImpl.h'
 XDR_INCLUDE='GwXdrTypeMappings.h'
@@ -66,6 +68,7 @@ def isCurrent(infile,outfile):
 def compressFile(inFile,outfile):
     if isCurrent(inFile,outfile):
         return
+    print("compressing %s"%inFile)
     with open(inFile, 'rb') as f_in:
         with gzip.open(outfile, 'wb') as f_out:
             shutil.copyfileobj(f_in, f_out)
@@ -372,6 +375,32 @@ def getLibs():
             rt.append(e)
     return rt
 
+
+
+def joinFiles(target,pattern,dirlist):
+    flist=[]
+    for dir in dirlist:
+            fn=os.path.join(dir,pattern)
+            if os.path.exists(fn):
+                flist.append(fn)
+    current=False
+    if os.path.exists(target):
+        current=True
+        for f in flist:
+            if not isCurrent(f,target):
+                current=False
+                break
+    if current:
+        print("%s is up to date"%target)
+        return
+    print("creating %s"%target)
+    with gzip.open(target,"wb") as oh:
+        for fn in flist:
+            print("adding %s to %s"%(fn,target))
+            with open(fn,"rb") as rh:
+                shutil.copyfileobj(rh,oh)
+
+
 OWNLIBS=getLibs()+["FS","WiFi"]
 GLOBAL_INCLUDES=[]
 
@@ -440,6 +469,8 @@ def prebuild(env):
     compressFile(mergedConfig,mergedConfig+".gz")
     generateCfg(mergedConfig,os.path.join(outPath(),CFG_INCLUDE),False)
     generateCfg(mergedConfig,os.path.join(outPath(),CFG_INCLUDE_IMPL),True)
+    joinFiles(os.path.join(outPath(),INDEXJS+".gz"),INDEXJS,["web"]+userTaskDirs)
+    joinFiles(os.path.join(outPath(),INDEXCSS+".gz"),INDEXCSS,["web"]+userTaskDirs)
     embedded=getEmbeddedFiles(env)
     filedefs=[]
     for ef in embedded:
@@ -453,7 +484,6 @@ def prebuild(env):
         filedefs.append((pureName,usname,ct))
         inFile=os.path.join(basePath(),"web",pureName)
         if os.path.exists(inFile):
-            print("compressing %s"%inFile)
             compressFile(inFile,ef)
         else:
             print("#WARNING: infile %s for %s not found"%(inFile,ef))

lib/api/GwApi.h

@@ -6,7 +6,9 @@
 #include "GWConfig.h"
 #include "GwBoatData.h"
 #include "GwXDRMappings.h"
+#include "GwSynchronized.h"
 #include <map>
+#include <ESPAsyncWebServer.h>
 class GwApi;
 typedef void (*GwUserTaskFunction)(GwApi *);
 //API to be used for additional tasks
@@ -95,6 +97,8 @@ class GwApi{
                 unsigned long ser2Tx=0;
                 unsigned long tcpSerRx=0;
                 unsigned long tcpSerTx=0;
+                unsigned long udpwTx=0;
+                unsigned long udprRx=0;
                 int tcpClients=0;
                 unsigned long tcpClRx=0;
                 unsigned long tcpClTx=0;
@@ -169,6 +173,20 @@ class GwApi{
         virtual void remove(int idx){}
         virtual TaskInterfaces * taskInterfaces()=0;
 
+        /**
+         * register handler for web URLs
+         * Please be aware that this handler function will always be called from a separate
+         * task. So you must ensure proper synchronization!
+        */
+        using HandlerFunction=std::function<void(AsyncWebServerRequest *)>;
+        /**
+         * @param url: the url of that will trigger the handler.
+         *             it will be prefixed with /api/user/<taskname>
+         *             taskname is the name that you used in addUserTask
+         * @param handler: the handler function (see remark above about thread synchronization)
+        */
+        virtual void registerRequestHandler(const String &url,HandlerFunction handler)=0;
+
         /**
          * only allowed during init methods
         */