Initial Working POC

.gitignore

@@ -0,0 +1,5 @@
+# Ignore the entire vendor directory
+vendor/
+
+# Ignore all .idea files and directories
+.idea/

Makefile

@@ -0,0 +1,70 @@
+# Define Docker Compose command
+DOCKER_COMPOSE := docker-compose
+
+# Docker Compose file
+COMPOSE_FILE := ./deployment/docker-compose.yml
+
+# Variables
+BINARY_NAME=heartbeat_shard_app
+MAIN_GO=main.go
+PID_FILE=app.pid
+
+# Targets
+build:
+	@echo "Building the application..."
+	@go build -o $(BINARY_NAME) $(MAIN_GO)
+
+run: build
+	@echo "Running the application..."
+	@./$(BINARY_NAME) & echo $$! > $(PID_FILE)
+
+clean:
+	@echo "Cleaning up..."
+	@rm -f $(BINARY_NAME)
+
+#Generate SQL
+generate-sql:
+	sqlc generate
+
+# Build Docker containers
+db-build:
+	$(DOCKER_COMPOSE) -f $(COMPOSE_FILE) build
+
+# Run Docker containers
+db-up:
+	$(DOCKER_COMPOSE) -f $(COMPOSE_FILE) up -d
+
+# Stop Docker containers
+db-down:
+	$(DOCKER_COMPOSE) -f $(COMPOSE_FILE) down
+
+# Restart Docker containers
+db-restart: down up
+
+# View logs of Docker containers
+db-logs:
+	$(DOCKER_COMPOSE) -f $(COMPOSE_FILE) logs -f
+
+# Remove Docker containers and associated volumes
+db-clean:
+	$(DOCKER_COMPOSE) -f $(COMPOSE_FILE) down -v
+
+# Sleep for 1 second
+sleep:
+	@sleep 1
+
+# Create Setup
+setup: db-up sleep run
+
+# Bring down the setup
+teardown: db-clean
+	@echo "Stopping the application..."
+	@if [ -f $(PID_FILE) ]; then \
+		PID=$$(cat $(PID_FILE)); \
+		kill $$PID && rm -f $(PID_FILE); \
+		echo "Application stopped."; \
+	else \
+		echo "No PID file found. Is the application running?"; \
+	fi
+	@rm -f $(BINARY_NAME)
+

README.md

@@ -0,0 +1,92 @@
+# postgres-static shard
+* Implement simple static sharding using Postgres in the backend.
+
+## Prerequisites
+* Docker
+* Docker Compose
+* Golang 1.22 or later
+
+## Infrastructure
+* Machine: MacBook M2 Pro
+
+## Setup/TearDown
+* Clone the repository
+* The repository contains a `Makefile` with the following targets of interest:
+  * `make setup` - Setup the below environment:
+    * 2 independent Postgres docker instances with pre-populated heartbeat data for multiple machines.
+    * Builds and runs simple Web Application that runs on port **8080** and routes user-requested queries on one of the shards based on 
+    machine_id value.
+  * `make teardown` - Teardown the above environment
+
+## Shard(Postgres Instances) Coordinates
+* shard0:
+  * Host: localhost
+  * Port: 5432
+  * User: test_user
+  * Password: test_password
+  * Database: shard0
+
+* shard1:
+  * Host: localhost
+  * Port: 5433
+  * User: test_user
+  * Password: test_password
+  * Database: shard1
+
+## DB Schema
+* Schema:
+  * Table: `heartbeat`
+    * Columns:
+      * `machine_id` - TEXT
+      * `last_heartbeat` - BIGINT
+
+## DB-Topology and Sharding Strategy
+* The setup consists of 2 Postgres instances:
+  * shard0 running on port 5432.
+  * shard1 running on and 5433.
+
+* The sharding strategy is simple and static:
+  * The machines have the following naming convention: `machine_<id>`, for example machine_001.
+  * All the even numbered machines are stored in shard0.
+  * All the odd numbered machines are stored in the shard1.
+
+## Prepopulated Test Data
+* Shard 1:
+
+|  machine_id   |  last_heartbeat  |
+|:-------------:|:----------------:|
+|  machine_001  |    1622548800    |
+|  machine_003  |    1622721600    |
+|  machine_005  |    1622894400    |
+|  machine_007  |    1623067200    |
+|  machine_009  |    1623240000    |
+
+* Shard 2:
+
+|  machine_id  |  last_heartbeat  |
+|:------------:|:----------------:|
+| machine_002  |    1622635200    |
+| machine_004  |    1622808000    |
+| machine_006  |    1622980800    |
+| machine_008  |    1623153600    |
+| machine_0010 |    1623326400    |
+
+## Supported Operations:
+* The Web Application supports the following operations:
+  * `get` - Get the last heartbeat of a machine.
+    * Example Invocation: `curl -X GET http://localhost:8080/heartbeat/machine_001` 
+  * `set` - Set the last heartbeat of a machine.
+    * Example Invocation: `curl -X PUT http://localhost:8080/heartbeat/machine_001 -H "Content-Type: application/json" -d '{"last_heartbeat": 1625160900}'`
+
+## Example Test Cases
+* Requests for even machines, that should land on shard0:
+  * Get the last heartbeat of machine_002:
+    * `curl -X GET http://localhost:8080/heartbeat/machine_002`
+  * Set the last heartbeat of machine_002:
+    * `curl -X PUT http://localhost:8080/heartbeat/machine_002 -H "Content-Type: application/json" -d '{"last_heartbeat": 1625160900}'`
+* Requests for odd machines, that land on shard1:
+  * Get the last heartbeat of machine_001:
+    * `curl -X GET http://localhost:8080/heartbeat/machine_001`
+  * Set the last heartbeat of machine_001:
+    * `curl -X PUT http://localhost:8080/heartbeat/machine_001 -H "Content-Type: application/json" -d '{"last_heartbeat": 1625150700}'`
+* **Note:** Verify the requests were successful and landed on the correct shards by checking the DB entries/updates in the corresponding shards.

deployment/db/query/heartbeat.sql

@@ -0,0 +1,5 @@
+-- name: UpdateLastHeartbeat :one
+UPDATE heartbeat SET last_heartbeat = $2 WHERE machine_id = $1 RETURNING *;
+
+-- name: GetLastHeartbeat :one
+SELECT last_heartbeat FROM heartbeat WHERE machine_id = $1;

deployment/db/schema/0001-create_and_populate_tables.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+set -e
+
+echo "creating schema"
+
+# Connect to PostgreSQL and execute the commands
+psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
+
+CREATE TABLE IF NOT EXISTS heartbeat (
+    machine_id TEXT,
+    last_heartbeat BIGINT
+);
+
+ALTER TABLE heartbeat ADD CONSTRAINT unique_machine_id UNIQUE (machine_id);
+
+\copy heartbeat from '/var/lib/data/init/heartbeat.csv' with delimiter E'\t' null ''
+
+EOSQL
+echo "Data import complete"

deployment/db/schema/schema.sql

@@ -0,0 +1,7 @@
+CREATE TABLE IF NOT EXISTS heartbeat (
+machine_id TEXT,
+last_heartbeat BIGINT
+);
+
+-- Ensure the unique constraint on user_id
+ALTER TABLE heartbeat ADD CONSTRAINT unique_machine_id UNIQUE (machine_id);

deployment/docker-compose.yml

@@ -0,0 +1,41 @@
+version: '3.8'
+
+services:
+  postgres1:
+    image: postgres:latest
+    restart: always
+    environment:
+      POSTGRES_USER: test_user
+      POSTGRES_PASSWORD: test_password
+      POSTGRES_DB: shard0
+    volumes:
+      - postgres1_data:/var/lib/postgresql/data
+      - ../testdata/db/shard0/heartbeat.csv:/var/lib/data/init/heartbeat.csv
+      - ./db/schema/0001-create_and_populate_tables.sh:/docker-entrypoint-initdb.d/0001-create_and_populate_tables.sh
+    ports:
+      - "5432:5432"
+    networks:
+      - postgres_network
+
+  postgres2:
+    image: postgres:latest
+    restart: always
+    environment:
+      POSTGRES_USER: test_user
+      POSTGRES_PASSWORD: test_password
+      POSTGRES_DB: shard1
+    volumes:
+      - postgres2_data:/var/lib/postgresql/data
+      - ../testdata/db/shard1/heartbeat.csv:/var/lib/data/init/heartbeat.csv
+      - ./db/schema/0001-create_and_populate_tables.sh:/docker-entrypoint-initdb.d/0001-create_and_populate_tables.sh
+    ports:
+      - "5433:5432"
+    networks:
+      - postgres_network
+
+volumes:
+  postgres1_data:
+  postgres2_data:
+
+networks:
+  postgres_network:

go.mod

@@ -0,0 +1,41 @@
+module github.com/soumya-codes/postgres-static-shard
+
+go 1.22
+
+require (
+	github.com/gin-gonic/gin v1.10.0
+	github.com/jackc/pgx/v5 v5.6.0
+)
+
+require (
+	github.com/bytedance/sonic v1.11.6 // indirect
+	github.com/bytedance/sonic/loader v0.1.1 // indirect
+	github.com/cloudwego/base64x v0.1.4 // indirect
+	github.com/cloudwego/iasm v0.2.0 // indirect
+	github.com/gabriel-vasile/mimetype v1.4.3 // indirect
+	github.com/gin-contrib/sse v0.1.0 // indirect
+	github.com/go-playground/locales v0.14.1 // indirect
+	github.com/go-playground/universal-translator v0.18.1 // indirect
+	github.com/go-playground/validator/v10 v10.20.0 // indirect
+	github.com/goccy/go-json v0.10.2 // indirect
+	github.com/jackc/pgpassfile v1.0.0 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.7 // indirect
+	github.com/kr/text v0.2.0 // indirect
+	github.com/leodido/go-urn v1.4.0 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
+	github.com/rogpeppe/go-internal v1.12.0 // indirect
+	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
+	github.com/ugorji/go/codec v1.2.12 // indirect
+	golang.org/x/arch v0.8.0 // indirect
+	golang.org/x/crypto v0.23.0 // indirect
+	golang.org/x/net v0.25.0 // indirect
+	golang.org/x/sys v0.20.0 // indirect
+	golang.org/x/text v0.15.0 // indirect
+	google.golang.org/protobuf v1.34.1 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)