Merge pull request #14 from sine-fdn/postgres-integration

SQL Integration Example

.github/workflows/test.yml

@@ -20,3 +20,5 @@ jobs:
         working-directory: ./examples/http-single-server-channels
       - run: cargo test -- --nocapture
         working-directory: ./examples/iroh-p2p-channels
+      - run: docker compose -f docker-compose.yml up -d && cargo build --release && cargo test --release -- --nocapture
+        working-directory: ./examples/sql-integration

Cargo.toml

@@ -2,6 +2,7 @@ workspace = { members = [
     "examples/http-multi-server-channels",
     "examples/http-single-server-channels",
     "examples/iroh-p2p-channels",
+    "examples/sql-integration",
 ] }
 [package]
 name = "parlay"
@@ -12,7 +13,7 @@ edition = "2021"
 bincode = "1.3.3"
 blake3 = "1.5.0"
 chacha20poly1305 = "0.10.1"
-garble_lang = "0.2.0"
+garble_lang = { version = "0.3.0", features = ["serde"] }
 rand = "0.8.5"
 serde = { version = "1.0.195", features = ["derive"] }
 tokio = { version = "1.35.1", features = ["full"] }

examples/http-multi-server-channels/src/main.rs

@@ -66,7 +66,8 @@ async fn main() -> Result<(), Error> {
         Commands::Pre { urls } => {
             let parties = urls.len() - 1;
             let channel = HttpChannel::new(urls, parties).await?;
-            fpre(channel, parties).await.context("FPre")
+            fpre(channel, parties).await.context("FPre").unwrap();
+            Ok(())
         }
         Commands::Party {
             urls,

examples/http-single-server-channels/src/main.rs

@@ -82,7 +82,7 @@ async fn main() {
                     sleep(Duration::from_secs(1)).await;
                 }
             }
-            fpre(channel, parties).await.unwrap()
+            fpre(channel, parties).await.unwrap();
         }
         Commands::Party {
             url,

examples/sql-integration/.example.garble.rs

@@ -0,0 +1,38 @@
+const ROWS_0: usize = PARTY_0::ROWS;
+const ROWS_1: usize = PARTY_1::ROWS;
+const ROWS_RESULT: usize = max(PARTY_0::ROWS, PARTY_1::ROWS);
+const STR_LEN: usize = max(PARTY_0::STR_LEN, PARTY_1::STR_LEN);
+
+enum InsuranceStatus {
+    Foo,
+    Bar,
+    FooBar,
+}
+
+pub fn main(
+    residents: [(i32, [u8; STR_LEN], i32); ROWS_0],
+    insurance: [(i32, [u8; STR_LEN], InsuranceStatus); ROWS_1],
+) -> [[u8; STR_LEN]; ROWS_RESULT] {
+    let mut result = [[0u8; STR_LEN]; ROWS_RESULT];
+    let mut i = 0usize;
+    for resident in residents {
+        for insurance in insurance {
+            let (_, name0, age) = resident;
+            let (_, name1, insurance_status) = insurance;
+            if name0 == name1 && age > 65i32 {
+                match insurance_status {
+                    InsuranceStatus::Foo => {
+                        result[i] = name0;
+                        i = i + 1usize;
+                    }
+                    InsuranceStatus::Bar => {
+                        result[i] = name0;
+                        i = i + 1usize;
+                    }
+                    _ => {}
+                }
+            }
+        }
+    }
+    result
+}

examples/sql-integration/Cargo.toml

@@ -0,0 +1,27 @@
+[package]
+name = "parlay-sql-integration"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+anyhow = "1.0.79"
+axum = "0.7.4"
+blake3 = "1.5.1"
+clap = { version = "4.4.18", features = ["derive"] }
+handlebars = "5.1.2"
+parlay = { path = "../../", version = "0.1.0" }
+reqwest = { version = "0.11.23", features = ["json"] }
+serde = "1.0.197"
+serde_json = "1.0.115"
+sqlx = { version = "0.7.4", features = [
+    "runtime-tokio",
+    "any",
+    "postgres",
+    "mysql",
+    "sqlite",
+] }
+tokio = { version = "1.35.1", features = ["macros", "rt", "rt-multi-thread"] }
+tower-http = { version = "0.5.1", features = ["fs", "trace"] }
+tracing = "0.1.40"
+tracing-subscriber = "0.3.18"
+url = { version = "2.5.0", features = ["serde"] }

examples/sql-integration/docker-compose.yml

@@ -0,0 +1,41 @@
+version: "3.9"
+services:
+  db0:
+    image: postgres:16.2
+    restart: always
+    shm_size: 128mb
+    volumes:
+      - ./seeds/db0:/docker-entrypoint-initdb.d
+    ports:
+      - 5550:5432
+    environment:
+      POSTGRES_PASSWORD: test
+
+  db1:
+    image: mysql:8.4
+    restart: always
+    shm_size: 128mb
+    volumes:
+      - ./seeds/db1:/docker-entrypoint-initdb.d
+      - ./seeds/db1/insurance.csv:/var/lib/mysql-files/insurance.csv
+    ports:
+      - 5551:3306
+    environment:
+      MYSQL_ROOT_PASSWORD: test
+
+  db_out:
+    image: postgres:16.2
+    restart: always
+    shm_size: 128mb
+    volumes:
+      - ./seeds/db_out:/docker-entrypoint-initdb.d
+    ports:
+      - 5555:5432
+    environment:
+      POSTGRES_PASSWORD: test
+
+  adminer:
+    image: adminer
+    restart: always
+    ports:
+      - 8080:8080

examples/sql-integration/policies0.json

@@ -0,0 +1,46 @@
+{
+  "accepted": [
+    {
+      "participants": [
+        "http://localhost:8000",
+        "http://localhost:8001",
+        "http://localhost:8002"
+      ],
+      "program": ".example.garble.rs",
+      "leader": 0,
+      "party": 0,
+      "input": "SELECT id, name, age FROM residents",
+      "input_db": "postgres://postgres:test@localhost:5550/postgres",
+      "setup": "TRUNCATE results",
+      "output": "INSERT INTO results (name) VALUES ($1)",
+      "output_db": "postgres://postgres:test@localhost:5555/postgres",
+      "constants": {
+        "ROWS": {
+          "query": "SELECT COUNT(id) FROM residents",
+          "ty": "usize"
+        },
+        "STR_LEN": {
+          "query": "SELECT MAX(LENGTH(name)) FROM residents",
+          "ty": "usize"
+        }
+      }
+    },
+    {
+      "participants": [
+        "http://localhost:8000",
+        "http://localhost:8001",
+        "http://localhost:8002"
+      ],
+      "program": "does_not_exist.garble.rs",
+      "leader": 1,
+      "party": 0,
+      "input": "SELECT id, name, age FROM residents",
+      "input_db": "postgres://postgres:test@localhost:5550/postgres",
+      "max_rows": 4,
+      "setup": "TRUNCATE results",
+      "output": "INSERT INTO results (name) VALUES ($1)",
+      "output_db": "postgres://postgres:test@localhost:5555/postgres",
+      "constants": {}
+    }
+  ]
+}

examples/sql-integration/policies1.json

@@ -0,0 +1,26 @@
+{
+  "accepted": [
+    {
+      "participants": [
+        "http://localhost:8000",
+        "http://localhost:8001",
+        "http://localhost:8002"
+      ],
+      "program": ".example.garble.rs",
+      "leader": 0,
+      "party": 1,
+      "input": "SELECT id, name, status FROM insurance",
+      "input_db": "mysql://root:test@localhost:5551/test",
+      "constants": {
+        "ROWS": {
+          "query": "SELECT COUNT(id) FROM insurance",
+          "ty": "usize"
+        },
+        "STR_LEN": {
+          "query": "SELECT MAX(LENGTH(name)) FROM insurance",
+          "ty": "usize"
+        }
+      }
+    }
+  ]
+}

examples/sql-integration/policies2.json

@@ -0,0 +1,16 @@
+{
+  "accepted": [
+    {
+      "participants": [
+        "http://localhost:8000",
+        "http://localhost:8001",
+        "http://localhost:8002",
+        "http://localhost:8003"
+      ],
+      "program": ".example.garble.rs",
+      "leader": 0,
+      "party": 2,
+      "input": "65u32"
+    }
+  ]
+}

examples/sql-integration/preprocessor.json

@@ -0,0 +1,16 @@
+{
+  "accepted": [
+    {
+      "participants": [
+        "http://localhost:8000",
+        "http://localhost:8001",
+        "http://localhost:8002"
+      ],
+      "program": "",
+      "leader": 0,
+      "party": 0,
+      "input": "",
+      "constants": {}
+    }
+  ]
+}

examples/sql-integration/seeds/db0/init.sql

@@ -0,0 +1,11 @@
+CREATE TABLE residents (
+    id SERIAL PRIMARY KEY,
+    name TEXT,
+    age INT,
+    address TEXT
+);
+
+COPY residents
+FROM '/docker-entrypoint-initdb.d/residents.csv'
+DELIMITER ','
+CSV HEADER;

examples/sql-integration/seeds/db0/residents.csv

@@ -0,0 +1,5 @@
+Id,Name,Age,Address
+0,"John Doe",22,"Parkstraße 1, 12345 Berlin"
+1,"Jane Doe",33,"Schlossallee 2, 12345 Berlin"
+2,"Max Doe",66,"Parkstraße 1, 12345 Berlin"
+3,"Bob Doe",77,"Schlossallee 2, 12345 Berlin"

examples/sql-integration/seeds/db1/init.sql

@@ -0,0 +1,16 @@
+CREATE DATABASE test;
+USE test;
+
+CREATE TABLE insurance (
+    id SERIAL PRIMARY KEY,
+    name TEXT,
+    status TEXT,
+    address TEXT
+);
+
+LOAD DATA INFILE '/var/lib/mysql-files/insurance.csv'
+INTO TABLE insurance
+FIELDS TERMINATED BY ','
+OPTIONALLY ENCLOSED BY '"'
+LINES TERMINATED BY '\n'
+IGNORE 1 ROWS

examples/sql-integration/seeds/db1/insurance.csv

@@ -0,0 +1,4 @@
+Id,Name,Status,Address
+1,"Max Doe","foo","Parkstraße 1, 12345 Berlin"
+2,"Bob Doe","bar","Schlossallee 2, 12345 Berlin"
+3,"Jane Doe","foobar","Schlossallee 2, 12345 Berlin"

examples/sql-integration/seeds/db_out/init.sql

@@ -0,0 +1,4 @@
+CREATE TABLE results (
+    id SERIAL PRIMARY KEY,
+    name TEXT
+);