feat: tus with linked buckets

Dockerfile

@@ -14,7 +14,7 @@ RUN npm run build
 FROM node:18-alpine
 WORKDIR /app
 COPY migrations migrations
-COPY ecosystem.config.js package.json ./
+COPY package.json ./
 COPY --from=0 /app/node_modules node_modules
 COPY --from=1 /app/dist dist
 EXPOSE 5000

src/test/db/docker-compose.yml → docker-compose-infra.yml

@@ -3,17 +3,27 @@
 version: '3'
 services:
   db:
-    image: postgres:13
+    image: postgres:15
     ports:
       - '5432:5432'
     volumes:
-      - ./src/test/db:/docker-entrypoint-initdb.d/
+      - ./migrations/base:/docker-entrypoint-initdb.d/
+    shm_size: 1g
     environment:
       POSTGRES_DB: postgres
       POSTGRES_USER: postgres
       POSTGRES_PASSWORD: postgres
       POSTGRES_PORT: 5432
 
+  multitenant_db:
+    image: postgres:15
+    ports:
+      - '5433:5432'
+    environment:
+      POSTGRES_DB: postgres
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+
   pg_bouncer:
     image: bitnami/pgbouncer:latest
     ports:
@@ -28,15 +38,6 @@ services:
     depends_on:
       - db
 
-  multitenant_db:
-    image: postgres:13
-    ports:
-      - '5433:5432'
-    environment:
-      POSTGRES_DB: postgres
-      POSTGRES_USER: postgres
-      POSTGRES_PASSWORD: postgres
-
   redis:
     image: redis:6.2-alpine
     restart: always
@@ -53,14 +54,16 @@ services:
       MINIO_ROOT_PASSWORD: secret1234
     command: server --console-address ":9001" /data
 
-  createbuckets:
+  create_default_bucket:
     image: minio/mc
     depends_on:
       - minio
+    environment:
+      BUCKET_NAME: ${GLOBAL_S3_BUCKET:-local-bucket}
     entrypoint: >
       /bin/sh -c "
       /usr/bin/mc alias set supa-minio http://minio:9000 supa-storage secret1234;
-      /usr/bin/mc mb supa-minio/supa-storage-staging-ap-southeast-1;
+      /usr/bin/mc mb "supa-minio/$${BUCKET_NAME}";
       exit 0;
       "
 

docker-compose.yml

@@ -0,0 +1,103 @@
+# docker-compose.yml
+
+version: '3'
+services:
+  storage-api:
+    build: .
+    ports:
+      - '5003:5000' # service port
+      - '5004:5001' # admin port
+    env_file: .env
+    environment:
+      DATABASE_URL: postgres://postgres:postgres@db/postgres
+      DATABASE_POOL_URL: postgresql://postgres:postgres@pg_bouncer:6432/postgres
+      RATE_LIMITER_REDIS_URL: redis://redis:6379
+      GLOBAL_S3_ENDPOINT: http://minio:9000
+      IS_MULTITENANT: true
+      MULTITENANT_DATABASE_URL: postgresql://postgres:postgres@multitenant_db/postgres
+    depends_on:
+      - db
+      - multitenant_db
+      - minio
+      - create_default_bucket
+      - imgproxy
+      - redis
+
+  db:
+    image: postgres:15
+    ports:
+      - '5432:5432'
+    volumes:
+      - ./migrations/base:/docker-entrypoint-initdb.d/
+    shm_size: 1g
+    environment:
+      POSTGRES_DB: postgres
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+      POSTGRES_PORT: 5432
+
+  multitenant_db:
+    image: postgres:15
+    ports:
+      - '5433:5432'
+    environment:
+      POSTGRES_DB: postgres
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+
+  pg_bouncer:
+    image: bitnami/pgbouncer:latest
+    ports:
+      - '6453:6432'
+    environment:
+      POSTGRESQL_USERNAME: postgres
+      POSTGRESQL_HOST: db
+      POSTGRESQL_PASSWORD: postgres
+      PGBOUNCER_POOL_MODE: transaction
+      PGBOUNCER_IGNORE_STARTUP_PARAMETERS: "extra_float_digits, options"
+      PGBOUNCER_STATS_USERS: postgres
+    depends_on:
+      - db
+
+  redis:
+    image: redis:6.2-alpine
+    restart: always
+    ports:
+      - '6379:6379'
+
+  minio:
+    image: minio/minio
+    ports:
+      - '9000:9000'
+      - '9001:9001'
+    environment:
+      MINIO_ROOT_USER: supa-storage
+      MINIO_ROOT_PASSWORD: secret1234
+    command: server --console-address ":9001" /data
+
+  create_default_bucket:
+    image: minio/mc
+    depends_on:
+      - minio
+    environment:
+      BUCKET_NAME: ${GLOBAL_S3_BUCKET:-local-bucket}
+    entrypoint: >
+      /bin/sh -c "
+      /usr/bin/mc alias set supa-minio http://minio:9000 supa-storage secret1234;
+      /usr/bin/mc mb "supa-minio/$${BUCKET_NAME}";
+      exit 0;
+      "
+
+  imgproxy:
+    image: darthsim/imgproxy
+    ports:
+      - '50020:8080'
+    volumes:
+      - ./data:/images/data
+    environment:
+      - IMGPROXY_WRITE_TIMEOUT=20
+      - IMGPROXY_READ_TIMEOUT=20
+      - IMGPROXY_REQUESTS_QUEUE_SIZE=24
+      - IMGPROXY_LOCAL_FILESYSTEM_ROOT=/images
+      - IMGPROXY_USE_ETAG=true
+      - IMGPROXY_ENABLE_WEBP_DETECTION=true

src/test/db/02-storage-schema.sql → migrations/base/01-storage-schema.sql

@@ -12,7 +12,6 @@ CREATE TABLE "storage"."buckets" (
     "owner" uuid,
     "created_at" timestamptz DEFAULT now(),
     "updated_at" timestamptz DEFAULT now(),
-    CONSTRAINT "buckets_owner_fkey" FOREIGN KEY ("owner") REFERENCES "auth"."users"("id"),
     PRIMARY KEY ("id")
 );
 CREATE UNIQUE INDEX "bname" ON "storage"."buckets" USING BTREE ("name");
@@ -28,7 +27,6 @@ CREATE TABLE "storage"."objects" (
     "last_accessed_at" timestamptz DEFAULT now(),
     "metadata" jsonb,
     CONSTRAINT "objects_bucketId_fkey" FOREIGN KEY ("bucket_id") REFERENCES "storage"."buckets"("id"),
-    CONSTRAINT "objects_owner_fkey" FOREIGN KEY ("owner") REFERENCES "auth"."users"("id"),
     PRIMARY KEY ("id")
 );
 CREATE UNIQUE INDEX "bucketid_objname" ON "storage"."objects" USING BTREE ("bucket_id","name");

migrations/tenant/0017-add_owner_id_column_deprecate_owner.sql

@@ -0,0 +1,8 @@
+alter table storage.objects add column if not exists owner_id text default null;
+alter table storage.buckets add column if not exists owner_id text default null;
+
+comment on column storage.objects.owner is 'Field is deprecated, use owner_id instead';
+comment on column storage.buckets.owner is 'Field is deprecated, use owner_id instead';
+
+ALTER TABLE storage.buckets
+    DROP CONSTRAINT IF EXISTS buckets_owner_fkey;

migrations/tenant/0018-external-buckets.sql

@@ -0,0 +1,14 @@
+CREATE TABLE bucket_credentials (
+      "id" uuid NOT NULL DEFAULT extensions.uuid_generate_v4(),
+      name text NOT NULL unique,
+      access_key text NULL,
+      secret_key text NULL,
+      role text null,
+      region text not null,
+      endpoint text NULL,
+      force_path_style boolean NOT NULL default false,
+      PRIMARY KEY (id)
+);
+
+ALTER TABLE storage.buckets ADD COLUMN credential_id uuid DEFAULT NULL;
+ALTER TABLE storage.buckets ADD CONSTRAINT fk_bucket_credential FOREIGN KEY (credential_id) REFERENCES bucket_credentials(id);