Add support for sqlite (#3575)

Co-authored-by: Romain Beauxis <[email protected]>

.github/scripts/build-details.sh

@@ -65,7 +65,7 @@ else
   IS_SNAPSHOT=
 fi
 
-MINIMAL_EXCLUDE_DEPS="alsa ao bjack camlimages dssi faad fdkaac flac frei0r gd graphics gstreamer imagelib irc-client-unix ladspa lame lastfm lilv lo mad magic ogg opus osc-unix portaudio pulseaudio samplerate shine soundtouch speex srt taglib tls theora tsdl vorbis"
+MINIMAL_EXCLUDE_DEPS="alsa ao bjack camlimages dssi faad fdkaac flac frei0r gd graphics gstreamer imagelib irc-client-unix ladspa lame lastfm lilv lo mad magic ogg opus osc-unix portaudio pulseaudio samplerate shine soundtouch speex srt taglib tls theora tsdl sqlite3 vorbis"
 
 {
   echo "branch=${BRANCH}"

.github/workflows/ci.yml

@@ -245,6 +245,10 @@ jobs:
           sudo -u opam -E git remote set-url origin https://github.com/savonet/liquidsoap.git
           sudo -u opam -E git fetch origin ${{ github.sha }}
           sudo -u opam -E git checkout ${{ github.sha }}
+      - name: Install sqlite
+        run: |
+          sudo apt-get -y install libsqlite3-dev
+          sudo -u opam -E opam install -y sqlite3
       - name: Install pandoc
         run: |
           cd /tmp
@@ -395,10 +399,15 @@ jobs:
           sudo apt-get update
           sudo apt-get -y dist-upgrade
           sudo apt-get -y autoremove
+          sudo apt-get -y install libsqlite3-dev
       - name: Update alpine packages
         if: matrix.os == 'alpine'
         run: |
           apk -U --force-overwrite upgrade
+          apk add -i sqlite-dev
+      - name: Install additional libraries
+        run: |
+          sudo -u opam -E opam install -y sqlite3
       - name: Install pandoc
         run: |
           cd /tmp

.pre-commit-config.yaml

@@ -28,7 +28,7 @@ repos:
       - id: ocamlformat
 
   - repo: https://github.com/savonet/pre-commit-liquidsoap
-    rev: e8b1a43
+    rev: 5fadc94
     hooks:
       - id: liquidsoap-prettier
 

CHANGES.md

@@ -13,8 +13,9 @@ New:
 - Add `metadata.replaygain` method to extract unified replay gain value from metadata (#3438).
 - Add `compute` parameter to `file.replaygain` to control gain calculation (#3438).
 - Add `compute` parameter to `enable_replaygain_metadata` to control replay gain calculation (#3438).
-- Add `copy:` protocol (##3506)
+- Add `copy:` protocol (#3506)
 - Add `file.touch`.
+- Add support for sqlite databases (#3575).
 
 Changed:
 

doc/content/cookbook.md

@@ -145,6 +145,15 @@ length 5:
 
 ```
 
+The default implementation of `medialib` uses standard Liquidsoap functions and
+can be pretty expensive in terms of memory. A more efficient implementation is
+available if you compiled with support for sqlite3 databases. In this case, you
+can use the `medialib.sqlite` operator as follows:
+
+```{.liquidsoap include="medialib.sqlite.liq"}
+
+```
+
 ## Force a file/playlist to be played at least every XX minutes
 
 It can be useful to have a special playlist that is played at least every 20 minutes for instance (3 times per hour).

doc/content/database.md

@@ -0,0 +1,126 @@
+# Database support
+
+Liquidsoap supports SQL databases through the sqlite library. If you build
+Liquidsoap by yourself, you should install the
+[SQLite3-OCaml](https://github.com/mmottl/sqlite3-ocaml) library, e.g. with
+`opam install sqlite3`{.bash}.
+
+In order to create or open a database, you should use the `sqlite` function,
+which takes as argument the file where the database is stored and returns an
+object whose methods can be used to modify or query the database:
+
+```{.liquidsoap include="sqlite.liq" from="open-begin" to="open-end"}
+
+```
+
+A table in the database can then be created by calling the `table.create` method
+on the object with as arguments the table name (labeled by `table`) and the list
+of columns specified by pairs consisting of the column name, and its
+type. Setting the `preserve` argument to `true`{.liquidsoap} allows not creating
+the table if one already exists under this name. In our example, we want to use
+our database to store metadata for files so that we create a table named
+`"metadata"`{.liquidsoap} with columns corresponding to the artist, title, etc.:
+
+```{.liquidsoap include="sqlite.liq" from="create-begin" to="create-end"}
+
+```
+
+Inserting a row is then performed using the `insert` method, which takes as
+argument the table and a record containing the data for the row:
+
+```{.liquidsoap include="sqlite.liq" from="insert-begin" to="insert-end"}
+
+```
+
+Since the field `filename` is a primary key, it has to be unique (two rows
+cannot have the same file name), so that inserting two files with the same
+filename in the database will result in an error. If we want that the second
+insertion replace the first one, we can pass the `replace=true`{.liquidsoap}
+argument to the `insert` function.
+
+We can query the database with the `select` method. For instance, to obtain all
+the files whose year is posterior to 2000, we can write
+
+```{.liquidsoap include="sqlite.liq" from="select-begin" to="select-end"}
+
+```
+
+In the case where you want to use strings in your queries, you should always use
+`sqlite.escape` to properly escape it and avoid injections:
+
+```{.liquidsoap include="sqlite.liq" from="select2-begin" to="select2-end"}
+
+```
+
+The `select` function, returns a list of rows. To each row will correspond a
+list of pairs strings consisting of
+
+- a string: the name of the column,
+- a nullable string: its value (this is nullable because the contents of a
+  column can be NULL in databases).
+
+We could thus extract the filenames from the above queries and use those in
+order to build a playlist as follows:
+
+```{.liquidsoap include="sqlite.liq" from="play-begin" to="play-end"}
+
+```
+
+This can be read as follows: for each row (by `list.map`{.liquidsoap}), we
+convert the row to a list of pairs of strings as described above (by calling the
+`to_list`{.liquidsoap} method), we replace take the field labeled
+`"filename"`{.liquidsoap} (by `list.assoc`{.liquidsoap}) and take its value,
+assuming that it is not null (by `null.get`{.liquidsoap}).
+
+Since manipulating rows as lists of pairs of strings is not convenient,
+Liquidsoap offers the possibility to represent them as records with
+constructions of the form
+
+```liquidsoap
+let sqlite.row (r : {a : string; b : int}) = row
+```
+
+which instructs to parse the row `row`{.liquidsoap} as a record `r` with fields
+`a` and `b` of respective types `string`{.liquidsoap} and
+`int`{.liquidsoap}. The above filename extraction is thus more conveniently
+written as
+
+```{.liquidsoap include="sqlite.liq" from="play2-begin" to="play2-end"}
+
+```
+
+Other useful methods include
+
+- `count` to count the number of rows satisfying a condition
+
+  ```{.liquidsoap include="sqlite.liq" from="count-begin" to="count-end"}
+
+  ```
+
+- `delete` to delete rows from a table
+
+  ```{.liquidsoap include="sqlite.liq" from="play-begin" to="play-end"}
+
+  ```
+
+- `table.drop` to delete tables from the database
+
+  ```{.liquidsoap include="sqlite.liq" from="drop-begin" to="drop-end"}
+
+  ```
+
+- `exec` to execute an arbitrary SQL query which does not return anything:
+
+  ```{.liquidsoap include="sqlite.liq" from="exec-begin" to="exec-end"}
+
+  ```
+
+- `query` to execute an arbitrary SQL query returning rows
+
+  ```{.liquidsoap include="sqlite.liq" from="query-begin" to="query-end"}
+
+  ```
+
+Finally, if your aim is to index file metadata, you might be interested in the
+`medialib.sqlite`{.liquidsoap} operator which is implemented in the standard
+library as described above (see the [quickstart](quick_start.html)).

doc/content/documentation.md

@@ -36,6 +36,7 @@ If you are migrating from a previous version, you might want to checkout
 - [JSON import/export](json.html): Importing and exporting language values in JSON.
 - [Playlist parsers](playlist_parsers.html): Supported playlist formats.
 - [LADSPA plugins](ladspa.html): Using LADSPA plugins.
+- [Database](database.html): Support for SQL databases.
 
 ## Core
 

doc/content/index.md

@@ -58,4 +58,4 @@ Liquidsoap is a flexible tool for processing audio and video streams, that's all
 
 Liquidsoap itself doesn't have a nice GUI or any graphical programming environment. You'll have to write the script by hand, and the only possible interaction with a running liquidsoap is the telnet server.
 
-Liquidsoap doesn't do any database or website stuff. It won't index your audio files, it won't allow your users to score songs on the web, etc. However, liquidsoap makes the interfacing with other tools easy, since it can call an external application (reading from the database) to get audio tracks, another one (updating last-played information) to notify that some file has been successfully played. An example of this is [Beets](beets.html), RadioPi also has a more complex system of its own along these lines.
+Liquidsoap makes the interfacing with other tools easy, since it can call an external application (reading from the database) to get audio tracks, another one (updating last-played information) to notify that some file has been successfully played. An example of this is [Beets](beets.html).

doc/content/liq/medialib.sqlite.liq

@@ -0,0 +1,4 @@
+m = medialib.sqlite(database="/tmp/medialib.sql", "~/music/")
+l = m.find(artist_contains="Brassens")
+l = list.shuffle(l)
+output(playlist.list(l))

doc/content/liq/playlists.liq

@@ -5,7 +5,8 @@ s1 = playlist("/my/playlist.txt")
 s2 = playlist(mode="normal", "/my/pl.m3u")
 
 # The playlist can come from any URI, can be reloaded every 10 minutes.
-s3 = playlist(reload=600,"http://my/playlist.txt")
+s3 = playlist(reload=600, "http://my/playlist.txt")
+
 # END
 output.dummy(fallible=true, s1)
 output.dummy(fallible=true, s2)

doc/content/liq/sqlite.liq

@@ -0,0 +1,120 @@
+#!../../../liquidsoap
+
+%ifdef sqlite
+# open-begin
+db = sqlite("/tmp/database.sql")
+
+# open-end
+# drop-begin
+db.table.drop("metadata")
+
+# drop-end
+# create-begin
+db.table.create(
+  "metadata",
+  preserve=true,
+  [
+    (
+      "filename",
+      "STRING PRIMARY KEY"
+    ),
+    ("artist", "STRING"),
+    ("title", "STRING"),
+    ("year", "INT")
+  ]
+)
+
+# create-end
+# insert-begin
+db.insert(
+  table="metadata",
+  {
+    artist="Naps",
+    title=
+      "Best life",
+    year=2021,
+    filename="naps.mp3"
+  }
+)
+db.insert(
+  table="metadata",
+  {
+    artist="Orelsan",
+    title=
+      "L'odeur de l'essence",
+    year=2021,
+    filename="orelsan.mp3"
+  }
+)
+
+# insert-end
+
+# count-begin
+n = db.count(table="metadata", where="year=2023")
+
+# count-end
+ignore(n)
+
+# select-begin
+l =
+  db.select(
+    table="metadata",
+    where=
+      "year >= 2000"
+  )
+
+# select-end
+# select2-begin
+find_artist = "Brassens"
+l' =
+  db.select(
+    table="metadata",
+    where=
+      "artist = #{sqlite.escape(find_artist)}"
+  )
+
+# select2-end
+ignore(l')
+
+# query-begin
+l'' =
+  db.query(
+    "SELECT * FROM metadata WHERE artist = 'bla'"
+  )
+
+# query-end
+ignore(l'')
+
+# play-begin
+files =
+  list.map(fun (row) -> null.get(list.assoc("filename", row.to_list())), l)
+s = playlist.list(files)
+output(s)
+
+# play-end
+# play2-begin
+def f(row) =
+  let sqlite.row (r :
+    {filename: string, artist: string, title: string, year: int}
+  ) = row
+  r.filename
+end
+files = list.map(f, l)
+s = playlist.list(files)
+output(s)
+
+# play2-end
+# delete-begin
+db.delete(
+  table="metadata",
+  where=
+    "year < 1900"
+)
+
+# delete-end
+# exec-begin
+db.exec(
+  "DROP TABLE IF EXISTS metadata"
+)
+# exec-end
+%endif