From 5ea7bb90efee01139b9bfe92c80b1f8d5e3ccbf4 Mon Sep 17 00:00:00 2001
From: Jun Lin Chen <contact@mc256.dev>
Date: Sat, 3 Feb 2024 22:55:52 -0500
Subject: [PATCH 1/4] add terraform deployment

---
 demo/terraform/.gitignore          |  17 +++++
 demo/terraform/.terraform.lock.hcl |  25 +++++++
 demo/terraform/locals.tf           |   3 +
 demo/terraform/main.tf             | 110 +++++++++++++++++++++++++++++
 demo/terraform/outputs.tf          |   5 ++
 demo/terraform/providers.tf        |  18 +++++
 demo/terraform/terraform.tfvars    |   2 +
 demo/terraform/variables.tf        |  30 ++++++++
 8 files changed, 210 insertions(+)
 create mode 100644 demo/terraform/.gitignore
 create mode 100644 demo/terraform/.terraform.lock.hcl
 create mode 100644 demo/terraform/locals.tf
 create mode 100644 demo/terraform/main.tf
 create mode 100644 demo/terraform/outputs.tf
 create mode 100644 demo/terraform/providers.tf
 create mode 100644 demo/terraform/terraform.tfvars
 create mode 100644 demo/terraform/variables.tf

diff --git a/demo/terraform/.gitignore b/demo/terraform/.gitignore
new file mode 100644
index 0000000..c69b464
--- /dev/null
+++ b/demo/terraform/.gitignore
@@ -0,0 +1,17 @@
+# Ignore Terraform state files
+*.tfstate
+*.tfstate.backup
+*.tfplan
+
+# Ignore .terraform directory
+.terraform/
+
+# Ignore override files
+override.tf
+override.tf.json
+*_override.tf
+*_override.tf.json
+
+# Ignore CLI configuration files
+.terraformrc
+terraform.rc
diff --git a/demo/terraform/.terraform.lock.hcl b/demo/terraform/.terraform.lock.hcl
new file mode 100644
index 0000000..87e36d6
--- /dev/null
+++ b/demo/terraform/.terraform.lock.hcl
@@ -0,0 +1,25 @@
+# This file is maintained automatically by "terraform init".
+# Manual edits may be lost in future updates.
+
+provider "registry.terraform.io/hashicorp/aws" {
+  version     = "5.35.0"
+  constraints = "~> 5.35.0"
+  hashes = [
+    "h1:KlFlsBQpmSzE+vrYnXQeYEwX/K2E/yUIf5bX4ilOS7Q=",
+    "zh:3a2a6f40db82d30ea8c5e3e251ca5e16b08e520570336e7e342be823df67e945",
+    "zh:420a23b69b412438a15b8b2e2c9aac2cf2e4976f990f117e4bf8f630692d3949",
+    "zh:4d8b887f6a71b38cff77ad14af9279528433e279eed702d96b81ea48e16e779c",
+    "zh:4edd41f8e1c7d29931608a7b01a7ae3d89d6f95ef5502cf8200f228a27917c40",
+    "zh:6337544e2ded5cf37b55a70aa6ce81c07fd444a2644ff3c5aad1d34680051bdc",
+    "zh:668faa3faaf2e0758bf319ea40d2304340f4a2dc2cd24460ddfa6ab66f71b802",
+    "zh:79ddc6d7c90e59fdf4a51e6ea822ba9495b1873d6a9d70daf2eeaf6fc4eb6ff3",
+    "zh:885822027faf1aa57787f980ead7c26e7d0e55b4040d926b65709b764f804513",
+    "zh:8c50a8f397b871388ff2e048f5eb280af107faa2e8926694f1ffd9f32a7a7cdf",
+    "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425",
+    "zh:a2f5d2553df5573a060641f18ee7585587047c25ba73fd80617f59b5893d22b4",
+    "zh:c43833ae2a152213ee92eb5be7653f9493779eddbe0ce403ea49b5f1d87fd766",
+    "zh:dab01527a3a55b4f0f958af6f46313d775e27f9ad9d10bedbbfea4a35a06dc5f",
+    "zh:ed49c65620ec42718d681a7fc00c166c295ff2795db6cede2c690b83f9fb3e65",
+    "zh:f0a358c0ae1087c466d0fbcc3b4da886f33f881a145c3836ec43149878b86a1a",
+  ]
+}
diff --git a/demo/terraform/locals.tf b/demo/terraform/locals.tf
new file mode 100644
index 0000000..bf278ec
--- /dev/null
+++ b/demo/terraform/locals.tf
@@ -0,0 +1,3 @@
+locals {
+  project_name = var.project_id
+}
diff --git a/demo/terraform/main.tf b/demo/terraform/main.tf
new file mode 100644
index 0000000..2cb8cf0
--- /dev/null
+++ b/demo/terraform/main.tf
@@ -0,0 +1,110 @@
+## EC2 Image
+data "aws_ami" "ubuntu" {
+  most_recent = true
+  owners      = ["099720109477"] # Canonical
+
+  filter {
+    name   = "name"
+    values = ["ubuntu/images/hvm-ssd/ubuntu-jammy-22.04-amd64-server-*"]
+  }
+
+  filter {
+    name   = "virtualization-type"
+    values = ["hvm"]
+  }
+
+  filter {
+    name   = "root-device-type"
+    values = ["ebs"]
+  }
+
+  filter {
+    name   = "architecture"
+    values = ["x86_64"]
+  }
+
+}
+
+## create vpc
+resource "aws_vpc" "ec2_vpc" {
+  cidr_block = "10.0.0.0/16"
+  tags = merge(
+    var.default_tags,
+    {
+      Name = "${local.project_name}-ec2-vpc"
+    },
+  )
+}
+
+resource "aws_subnet" "ec2_subnet" {
+  vpc_id     = aws_vpc.ec2_vpc.id
+  cidr_block = "10.0.1.0/24"
+  tags = merge(
+    var.default_tags,
+    {
+      Name = "${local.project_name}-ec2-subnet"
+    },
+  )
+}
+
+## Security Group
+resource "aws_security_group" "ec2_security_group" {
+  name        = "${local.project_name}-ec2-sg"
+  description = "Allow inbound traffic from port 8080 and all outbound traffic"
+  vpc_id      = aws_vpc.ec2_vpc.id
+
+  tags = merge(
+    var.default_tags,
+    {
+      Name = "${local.project_name}-ec2-sg"
+    },
+  )
+}
+
+resource "aws_vpc_security_group_egress_rule" "all_egress" {
+  security_group_id = aws_security_group.ec2_security_group.id
+
+  cidr_ipv4   = "0.0.0.0/0" # all
+  ip_protocol = "-1"        # all
+
+}
+
+resource "aws_vpc_security_group_ingress_rule" "http_ingress" {
+  security_group_id = aws_security_group.ec2_security_group.id
+
+  cidr_ipv4   = "0.0.0.0/0"
+  from_port   = 8080
+  to_port     = 8080
+  ip_protocol = "tcp"
+
+}
+
+
+## EC2 Host
+resource "aws_instance" "ec2" {
+  ami                    = data.aws_ami.ubuntu.id
+  instance_type          = var.instance_type
+  subnet_id              = aws_subnet.ec2_subnet.id
+  key_name               = var.ssh_keyname
+  vpc_security_group_ids = [aws_security_group.ec2_security_group.id]
+  monitoring             = true
+
+  lifecycle {
+    ignore_changes = [subnet_id, ami]
+  }
+
+  root_block_device {
+    volume_type           = "gp3"
+    volume_size           = var.ebs_size_in_gb
+    encrypted             = false
+    delete_on_termination = true
+  }
+
+
+  tags = merge(
+    var.default_tags,
+    {
+      Name = "${local.project_name}-ec2-cloud"
+    },
+  )
+}
diff --git a/demo/terraform/outputs.tf b/demo/terraform/outputs.tf
new file mode 100644
index 0000000..e3736fd
--- /dev/null
+++ b/demo/terraform/outputs.tf
@@ -0,0 +1,5 @@
+output "id" {
+  description = "The ec2 instance id"
+  value       = aws_instance.ec2.id
+  sensitive   = false
+}
diff --git a/demo/terraform/providers.tf b/demo/terraform/providers.tf
new file mode 100644
index 0000000..f683d71
--- /dev/null
+++ b/demo/terraform/providers.tf
@@ -0,0 +1,18 @@
+terraform {
+  required_providers {
+    aws = {
+      version = "~> 5.35.0"
+      source  = "hashicorp/aws"
+    }
+  }
+}
+
+provider "aws" {
+  profile = "default"
+  # Hard-coded credentials are not recommended in any Terraform configuration and 
+  # risks secret leakage should this file ever be committed to a public version control system.
+  # Instead, use environment variables or shared credentials file
+  # ref: https://registry.terraform.io/providers/-/aws/latest/docs#environment-variables
+
+  # Please set the credentials in the environment variables or in $HOME/.aws/credentials and $HOME/.aws/config files
+}
diff --git a/demo/terraform/terraform.tfvars b/demo/terraform/terraform.tfvars
new file mode 100644
index 0000000..091c44a
--- /dev/null
+++ b/demo/terraform/terraform.tfvars
@@ -0,0 +1,2 @@
+# required
+ssh_keyname = "starlight-key"
diff --git a/demo/terraform/variables.tf b/demo/terraform/variables.tf
new file mode 100644
index 0000000..feec96c
--- /dev/null
+++ b/demo/terraform/variables.tf
@@ -0,0 +1,30 @@
+variable "default_tags" {
+  default = {
+    project     = "starlight-experiment"
+    environment = "dev"
+  }
+}
+
+variable "ssh_keyname" {
+  type        = string
+  description = "the ssh key to access the ec2 instance"
+}
+
+variable "instance_type" {
+  type        = string
+  default     = "t3a.nano"
+  description = "the instance type to use"
+}
+
+variable "project_id" {
+  type        = string
+  default     = "starlight"
+  description = "the project name"
+}
+
+variable "ebs_size_in_gb" {
+  type        = number
+  default     = 10
+  description = "the ebs size in gb"
+}
+

From 0cf8e8de32e319b2683ddb02eaa36821153d6f70 Mon Sep 17 00:00:00 2001
From: Jun Lin Chen <contact@mc256.dev>
Date: Sun, 4 Feb 2024 15:42:11 -0500
Subject: [PATCH 2/4] add terraform deployment

---
 demo/terraform/.gitignore       |  2 +
 demo/terraform/main.tf          | 93 ++++++++++++++++++++++++++++++---
 demo/terraform/outputs.tf       | 22 +++++++-
 demo/terraform/terraform.tfvars |  6 ++-
 demo/terraform/variables.tf     | 14 ++++-
 docs/starlight-proxy.md         |  1 +
 docs/starlight-snapshotter.md   |  2 +
 7 files changed, 129 insertions(+), 11 deletions(-)

diff --git a/demo/terraform/.gitignore b/demo/terraform/.gitignore
index c69b464..0fc2ec7 100644
--- a/demo/terraform/.gitignore
+++ b/demo/terraform/.gitignore
@@ -15,3 +15,5 @@ override.tf.json
 # Ignore CLI configuration files
 .terraformrc
 terraform.rc
+
+.terraform.tfstate.lock.info
\ No newline at end of file
diff --git a/demo/terraform/main.tf b/demo/terraform/main.tf
index 2cb8cf0..7cb1384 100644
--- a/demo/terraform/main.tf
+++ b/demo/terraform/main.tf
@@ -28,6 +28,7 @@ data "aws_ami" "ubuntu" {
 ## create vpc
 resource "aws_vpc" "ec2_vpc" {
   cidr_block = "10.0.0.0/16"
+
   tags = merge(
     var.default_tags,
     {
@@ -36,9 +37,11 @@ resource "aws_vpc" "ec2_vpc" {
   )
 }
 
-resource "aws_subnet" "ec2_subnet" {
-  vpc_id     = aws_vpc.ec2_vpc.id
-  cidr_block = "10.0.1.0/24"
+resource "aws_subnet" "ec2_subnet_public" {
+  vpc_id                  = aws_vpc.ec2_vpc.id
+  cidr_block              = "10.0.1.0/24"
+  map_public_ip_on_launch = true
+
   tags = merge(
     var.default_tags,
     {
@@ -47,6 +50,7 @@ resource "aws_subnet" "ec2_subnet" {
   )
 }
 
+
 ## Security Group
 resource "aws_security_group" "ec2_security_group" {
   name        = "${local.project_name}-ec2-sg"
@@ -67,9 +71,21 @@ resource "aws_vpc_security_group_egress_rule" "all_egress" {
   cidr_ipv4   = "0.0.0.0/0" # all
   ip_protocol = "-1"        # all
 
+  description = "Allow all outbound traffic"
+}
+
+resource "aws_vpc_security_group_ingress_rule" "starlight_proxy_ingress" {
+  security_group_id = aws_security_group.ec2_security_group.id
+
+  cidr_ipv4   = "0.0.0.0/0"
+  from_port   = 8090
+  to_port     = 8090
+  ip_protocol = "tcp"
+
+  description = "Allow inbound traffic for Starlight Proxy"
 }
 
-resource "aws_vpc_security_group_ingress_rule" "http_ingress" {
+resource "aws_vpc_security_group_ingress_rule" "adminer_ingress" {
   security_group_id = aws_security_group.ec2_security_group.id
 
   cidr_ipv4   = "0.0.0.0/0"
@@ -77,17 +93,49 @@ resource "aws_vpc_security_group_ingress_rule" "http_ingress" {
   to_port     = 8080
   ip_protocol = "tcp"
 
+  description = "Allow inbound traffic for Adminer - Database management in a single PHP file"
+}
+
+resource "aws_vpc_security_group_ingress_rule" "registry_ingress" {
+  security_group_id = aws_security_group.ec2_security_group.id
+
+  cidr_ipv4   = "0.0.0.0/0"
+  from_port   = 5000
+  to_port     = 5000
+  ip_protocol = "tcp"
+
+  description = "Allow inbound traffic for Container Registry"
+}
+
+resource "aws_vpc_security_group_ingress_rule" "ssh_ingress" {
+  security_group_id = aws_security_group.ec2_security_group.id
+
+  cidr_ipv4   = "0.0.0.0/0"
+  from_port   = 22
+  to_port     = 22
+  ip_protocol = "tcp"
+
+  description = "Allow inbound traffic for Container Registry"
+}
+
+
+## Key Pair
+resource "aws_key_pair" "deployer" {
+  count      = var.ssh_public_key != "" ? 1 : 0
+  key_name   = var.ssh_key_name
+  public_key = var.ssh_public_key
 }
 
 
 ## EC2 Host
-resource "aws_instance" "ec2" {
+resource "aws_instance" "starlight_cloud" {
   ami                    = data.aws_ami.ubuntu.id
   instance_type          = var.instance_type
-  subnet_id              = aws_subnet.ec2_subnet.id
-  key_name               = var.ssh_keyname
+  subnet_id              = aws_subnet.ec2_subnet_public.id
+  key_name               = var.ssh_public_key == "" ? var.ssh_key_name : aws_key_pair.deployer[0].key_name
   vpc_security_group_ids = [aws_security_group.ec2_security_group.id]
   monitoring             = true
+  private_ip             = "10.0.1.21"
 
   lifecycle {
     ignore_changes = [subnet_id, ami]
@@ -108,3 +156,34 @@ resource "aws_instance" "ec2" {
     },
   )
 }
+
+
+
+resource "aws_instance" "starlight_edge" {
+  ami                    = data.aws_ami.ubuntu.id
+  instance_type          = var.instance_type
+  subnet_id              = aws_subnet.ec2_subnet_public.id
+  key_name               = var.ssh_public_key == "" ? var.ssh_key_name : aws_key_pair.deployer[0].key_name
+  vpc_security_group_ids = [aws_security_group.ec2_security_group.id]
+  monitoring             = true
+  private_ip             = "10.0.1.22"
+
+  lifecycle {
+    ignore_changes = [subnet_id, ami]
+  }
+
+  root_block_device {
+    volume_type           = "gp3"
+    volume_size           = var.ebs_size_in_gb
+    encrypted             = false
+    delete_on_termination = true
+  }
+
+
+  tags = merge(
+    var.default_tags,
+    {
+      Name = "${local.project_name}-ec2-edge"
+    },
+  )
+}
diff --git a/demo/terraform/outputs.tf b/demo/terraform/outputs.tf
index e3736fd..bca2051 100644
--- a/demo/terraform/outputs.tf
+++ b/demo/terraform/outputs.tf
@@ -1,5 +1,23 @@
-output "id" {
+output "cloud-instance-id" {
   description = "The ec2 instance id"
-  value       = aws_instance.ec2.id
+  value       = aws_instance.starlight_cloud.id
+  sensitive   = false
+}
+
+output "cloud-instance-public-ip" {
+  description = "The ec2 instance public ip"
+  value       = aws_instance.starlight_cloud.public_ip
+  sensitive   = false
+}
+
+output "edge-instance-id" {
+  description = "The ec2 instance id"
+  value       = aws_instance.starlight_edge.id
+  sensitive   = false
+}
+
+output "edge-instance-public-ip" {
+  description = "The ec2 instance public ip"
+  value       = aws_instance.starlight_edge.public_ip
   sensitive   = false
 }
diff --git a/demo/terraform/terraform.tfvars b/demo/terraform/terraform.tfvars
index 091c44a..5dbc703 100644
--- a/demo/terraform/terraform.tfvars
+++ b/demo/terraform/terraform.tfvars
@@ -1,2 +1,6 @@
 # required
-ssh_keyname = "starlight-key"
+ssh_key_name = "starlight-key"
+
+# please replace with your own public key
+# this is the key for accessing the EC2 instances, if empty, we assume the key above is already created
+ssh_public_key = "ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAgEA08XpxYFsakw1hVCcFYTVvXMF+sxyF9PQ16dF+D1xP3Vr5MGq275FIyKQs0oQxKCCBtUGMIgCVrQ+V3CVu9CsfBTHoJlk1nW0dfJcB3j9kG3+3NRIHZ+FTrIO5hvMWJ+SzXTIDYgdcKLJ/YIboqBI4Rm/rNx7fVIib0K+43RcD4YRbwcMXK1lWkZ2czj+ixKmX5P0EeTLNz2J6XEsLT3V39B4QoXLKJM3TTjlRx2w980UI2lUED+3aW8jeW6Y9v9TUW5ZiZHxFLULQGD4UoivxLHUZCVJbnSaBX5r1yiE2jOvlhCXgaEa/EyYt6gBacMMC/qdIIX6nsts5+K4i/+37Jixd1qyJ5tWcxY9tUQZeiD1kuL0qy/mKtR9ONZLOUFvmWXG7t5i9axVaIyjj+Yb/4PXvEVd3jZ0jC8gcPjkkjOV22CLCCLgwtUHcDTE/OM8/oUem4Tnd/9blBjd47RfuHTdyVsujLwue5hpUo64E3JDSdVee0s1Yso1Wx8ZEfhJgqfAc+E5gSprY2pdFZUwffN52I8+72OfNnsmw1h10LqvN5Xpu+12eARr/LQHY/0E45kJRAcFbAgjUwKmKtdf0UAepb/AwLF37I9UT4uLxs56dvw4z1rJQDGQJdm8GVv0CJ8pzjBsYFYzf43Mp4+lYJ+V7BBMMo5YLa6em/bqi5s= mc256"
diff --git a/demo/terraform/variables.tf b/demo/terraform/variables.tf
index feec96c..099cfdb 100644
--- a/demo/terraform/variables.tf
+++ b/demo/terraform/variables.tf
@@ -5,11 +5,17 @@ variable "default_tags" {
   }
 }
 
-variable "ssh_keyname" {
+variable "ssh_key_name" {
   type        = string
   description = "the ssh key to access the ec2 instance"
 }
 
+variable "ssh_public_key" {
+  type        = string
+  description = "the private key to access the ec2 instance. If not provided, we assume the public key is already in the AWS account"
+  default     = ""
+}
+
 variable "instance_type" {
   type        = string
   default     = "t3a.nano"
@@ -28,3 +34,9 @@ variable "ebs_size_in_gb" {
   description = "the ebs size in gb"
 }
 
+
+variable "starlight_version" {
+  type        = string
+  default     = "0.6.2"
+  description = "the version of the starlight software to deploy"
+}
diff --git a/docs/starlight-proxy.md b/docs/starlight-proxy.md
index 5ded918..456fb03 100644
--- a/docs/starlight-proxy.md
+++ b/docs/starlight-proxy.md
@@ -1,5 +1,6 @@
 # Starlight Proxy
 
+**⚠️ This document is outdated. Please use https://github.com/mc256/starlight/blob/master/docs/newbie.md instead**
 
 This is the **Step 1** to use Starlight:
 
diff --git a/docs/starlight-snapshotter.md b/docs/starlight-snapshotter.md
index 84eb68f..f84c15b 100644
--- a/docs/starlight-snapshotter.md
+++ b/docs/starlight-snapshotter.md
@@ -1,5 +1,7 @@
 # Starlight Snapshotter Plugin
 
+**⚠️ This document is outdated. Please use https://github.com/mc256/starlight/blob/master/docs/newbie.md instead**
+
 This is the **Step 2** to use Starlight:
 
 Set up the worker to be able to run Starlight. 

From 4d67c08571051864f9d3eaba95f509436a6a559c Mon Sep 17 00:00:00 2001
From: Jun Lin Chen <contact@mc256.dev>
Date: Sun, 4 Feb 2024 19:51:43 -0500
Subject: [PATCH 3/4] add terraform deployment

---
 README.md                       |  22 +-
 demo/terraform/main.tf          | 142 +++++++++++-
 demo/terraform/outputs.tf       |  13 ++
 demo/terraform/terraform.tfvars |  15 ++
 demo/terraform/variables.tf     |  18 +-
 docs/newbie.md                  | 387 --------------------------------
 docs/starlight-proxy.md         | 208 -----------------
 docs/starlight-snapshotter.md   | 207 -----------------
 docs/starlight-workflow.md      |  19 --
 9 files changed, 192 insertions(+), 839 deletions(-)
 delete mode 100644 docs/newbie.md
 delete mode 100644 docs/starlight-proxy.md
 delete mode 100644 docs/starlight-snapshotter.md
 delete mode 100644 docs/starlight-workflow.md

diff --git a/README.md b/README.md
index 22d8c0f..cc4ee1c 100644
--- a/README.md
+++ b/README.md
@@ -52,7 +52,8 @@ Starlight is compatible with Kubernetes and can replace the default `overlayfs`
 We could use helm to deploy Starlight on a Kubernetes cluster.
 
 - [I am familiar with **K8s** & Helm. **TL;DR**](https://github.com/mc256/starlight/blob/master/docs/helm.md)
-- [I have 2 **Virtual Machines**, but **TL;DR**](https://github.com/mc256/starlight/blob/master/docs/newbie.md)
+- [I have 2 **Virtual Machines**, but **TL;DR**](https://github.com/mc256/starlight/blob/master/docs/2vm.md)
+- [I know **Terraform** and have an **AWS** account](https://github.com/mc256/starlight/blob/master/docs/terraform.md)
 
 ---
 
@@ -62,7 +63,6 @@ You need to:
 1) Set up a **Starlight proxy**, 
 ideally close to the **registry** server you are using. Configure the proxy server to point to the registry and run it.
 Starlight supports any standard registry. (It can be deployed to k8s using ***Helm***)
-<br>[Find out how to install **Starlight proxy** ➡️](https://github.com/mc256/starlight/blob/master/docs/starlight-proxy.md) 
 
 
 2) Set up the worker to be able to run Starlight. 
@@ -71,7 +71,6 @@ installing **containerd** and the **Starlight snapshotter plugin**,
 configuring containerd to use the plugin, 
 and starting the Starlight snapshotter daemon
 (you also need to tell the snapshotter the address of the proxy server).
-<br>[Find out how to install **containerd** & **Starlight snapshotter plugin** ➡️](https://github.com/mc256/starlight/blob/master/docs/starlight-snapshotter.md)
 
 
 3) Convert the container image to the **Starlight format** container image.
@@ -179,8 +178,9 @@ Starlight is not complete. Our roadmap:
 | [v0.1.3](https://github.com/mc256/starlight/tree/v0.1.3) |              |2022-10-12|
 | [v0.2.7](https://github.com/mc256/starlight/tree/v0.2.7) |              |2022-11-27|
 | [v0.3.2](https://github.com/mc256/starlight/tree/v0.3.2) |              |2023-01-27|
-| [v0.4.7](https://github.com/mc256/starlight/tree/v0.4.7) | stable       |2023-06-05|
-| [v0.5.x](https://github.com/mc256/starlight)             | in progress  |  |
+| [v0.4.7](https://github.com/mc256/starlight/tree/v0.4.7) |              |2023-06-05|
+| [v0.5.x](https://github.com/mc256/starlight/tree/v0.5.8) | stable       |2023-11-26|
+| [v0.6.x](https://github.com/mc256/starlight/)            | in progress  |2024|
 
 Feature List:
 - [x] Scalable database backend (v0.2)
@@ -196,9 +196,9 @@ Feature List:
   - [x] Goharbor support (v0.2)
   - [x] Multiple platforms image support (v0.2) 
   - [x] Jointly optimizing multiple containers deployments (v0.4)
-- [ ] Argo CI/CD support (v0.6)
-  - [ ] Hook/ Scanner for automatic image conversion (v0.5)
-  - [ ] Converting containers that have already been fully retrieved using Starlight to use OverlayFS. (v0.5)
-- [ ] Starlight new features (v0.6)
-  - [ ] Resume interrupted pull connection (v0.5)
-  - [ ] Garbage Collection (v0.5)
\ No newline at end of file
+- [ ] Argo CI/CD support (v0.7)
+  - [ ] Hook/ Scanner for automatic image conversion (v0.7)
+  - [ ] Converting containers that have already been fully retrieved using Starlight to use OverlayFS. (v0.7)
+- [ ] Starlight new features (v0.7)
+  - [ ] Resume interrupted pull connection (v0.7)
+  - [ ] Garbage Collection (v0.7)
\ No newline at end of file
diff --git a/demo/terraform/main.tf b/demo/terraform/main.tf
index 7cb1384..b1c4c3b 100644
--- a/demo/terraform/main.tf
+++ b/demo/terraform/main.tf
@@ -118,6 +118,40 @@ resource "aws_vpc_security_group_ingress_rule" "ssh_ingress" {
   description = "Allow inbound traffic for Container Registry"
 }
 
+## Internet Gateway
+resource "aws_internet_gateway" "ec2_igw" {
+  vpc_id = aws_vpc.ec2_vpc.id
+
+  tags = merge(
+    var.default_tags,
+    {
+      Name = "${local.project_name}-ec2-igw"
+    },
+  )
+}
+
+## Route Table
+resource "aws_route_table" "ec2_route_table" {
+  vpc_id = aws_vpc.ec2_vpc.id
+
+  route {
+    cidr_block = "0.0.0.0/0"
+    gateway_id = aws_internet_gateway.ec2_igw.id
+  }
+
+  tags = merge(
+    var.default_tags,
+    {
+      Name = "${local.project_name}-ec2-route-table"
+    },
+  )
+}
+
+resource "aws_route_table_association" "ec2_route_table_association" {
+  subnet_id      = aws_subnet.ec2_subnet_public.id
+  route_table_id = aws_route_table.ec2_route_table.id
+}
+
 
 ## Key Pair
 resource "aws_key_pair" "deployer" {
@@ -130,7 +164,7 @@ resource "aws_key_pair" "deployer" {
 ## EC2 Host
 resource "aws_instance" "starlight_cloud" {
   ami                    = data.aws_ami.ubuntu.id
-  instance_type          = var.instance_type
+  instance_type          = var.cloud_instance_type
   subnet_id              = aws_subnet.ec2_subnet_public.id
   key_name               = var.ssh_public_key == "" ? var.ssh_key_name : aws_key_pair.deployer[0].key_name
   vpc_security_group_ids = [aws_security_group.ec2_security_group.id]
@@ -143,7 +177,7 @@ resource "aws_instance" "starlight_cloud" {
 
   root_block_device {
     volume_type           = "gp3"
-    volume_size           = var.ebs_size_in_gb
+    volume_size           = var.cloud_ebs_size_in_gb
     encrypted             = false
     delete_on_termination = true
   }
@@ -155,13 +189,51 @@ resource "aws_instance" "starlight_cloud" {
       Name = "${local.project_name}-ec2-cloud"
     },
   )
+
+  user_data = <<-EOF
+#!/bin/bash
+echo "cloud" | sudo tee /etc/hostname > /dev/null
+sudo hostname -F /etc/hostname
+echo "10.0.1.21 cloud.cluster.local" | sudo tee -a /etc/hosts > /dev/null
+
+sudo apt update && \
+sudo apt upgrade -y && \
+sudo apt install -y docker-compose git && \
+sudo usermod -aG docker ubuntu && \
+sudo systemctl enable docker && \
+sudo systemctl start docker
+
+cd /home/ubuntu && \
+git clone https://github.com/mc256/starlight.git && \
+cd /home/ubuntu/starlight && \
+git checkout v${var.starlight_version} && \
+cd /home/ubuntu/starlight/demo/compose/ && \
+cp docker-compose-example.yaml docker-compose.yaml && \
+docker-compose up -d
+
+cat <<EOT | sudo tee -a /etc/sysctl.conf > /dev/null
+net.core.wmem_max=125829120
+net.core.rmem_max=125829120
+net.ipv4.tcp_rmem= 10240 87380 125829120
+net.ipv4.tcp_wmem= 10240 87380 125829120
+net.ipv4.tcp_window_scaling = 1
+net.ipv4.tcp_timestamps = 1
+net.ipv4.tcp_sack = 1
+net.ipv4.tcp_no_metrics_save = 1
+net.core.netdev_max_backlog = 10000
+EOT
+sudo sysctl -p
+
+touch /home/ubuntu/.completed
+              EOF
+
 }
 
 
 
 resource "aws_instance" "starlight_edge" {
   ami                    = data.aws_ami.ubuntu.id
-  instance_type          = var.instance_type
+  instance_type          = var.edge_instance_type
   subnet_id              = aws_subnet.ec2_subnet_public.id
   key_name               = var.ssh_public_key == "" ? var.ssh_key_name : aws_key_pair.deployer[0].key_name
   vpc_security_group_ids = [aws_security_group.ec2_security_group.id]
@@ -174,7 +246,7 @@ resource "aws_instance" "starlight_edge" {
 
   root_block_device {
     volume_type           = "gp3"
-    volume_size           = var.ebs_size_in_gb
+    volume_size           = var.edge_ebs_size_in_gb
     encrypted             = false
     delete_on_termination = true
   }
@@ -186,4 +258,66 @@ resource "aws_instance" "starlight_edge" {
       Name = "${local.project_name}-ec2-edge"
     },
   )
+
+  user_data = <<-EOF
+#!/bin/bash
+echo "edge" | sudo tee /etc/hostname > /dev/null
+sudo hostname -F /etc/hostname
+echo "10.0.1.21 cloud.cluster.local cloud" | sudo tee -a /etc/hosts > /dev/null
+
+sudo apt update && sudo apt upgrade -y && \
+sudo apt install -y build-essential containerd
+
+sudo systemctl enable containerd  && \
+sudo systemctl start containerd
+
+wget https://go.dev/dl/go1.20.8.linux-amd64.tar.gz && \
+sudo rm -rf /usr/local/go && sudo tar -C /usr/local -xzf go1.20.8.linux-amd64.tar.gz
+
+echo "export PATH=$PATH:/usr/local/go/bin" | sudo tee -a /home/ubuntu/.bashrc > /dev/null
+
+export PATH=$PATH:/usr/local/go/bin
+export GOPATH=/home/ubuntu/go
+export HOME=/home/
+source /home/ubuntu/.bashrc
+
+
+cat <<EOT | sudo tee -a /etc/sysctl.conf > /dev/null
+net.core.wmem_max=125829120
+net.core.rmem_max=125829120
+net.ipv4.tcp_rmem= 10240 87380 125829120
+net.ipv4.tcp_wmem= 10240 87380 125829120
+net.ipv4.tcp_window_scaling = 1
+net.ipv4.tcp_timestamps = 1
+net.ipv4.tcp_sack = 1
+net.ipv4.tcp_no_metrics_save = 1
+net.core.netdev_max_backlog = 10000
+EOT
+sudo sysctl -p
+
+
+cd /home/ubuntu && \
+git clone https://github.com/mc256/starlight.git && \
+cd /home/ubuntu/starlight && \
+git checkout v${var.starlight_version} && \
+make starlight-daemon ctr-starlight && \
+sudo make install install-systemd-service 
+
+sudo systemctl enable starlight-daemon
+sudo systemctl start starlight-daemon
+
+sudo ctr-starlight add myproxy http cloud.cluster.local:8090
+
+sudo mkdir /etc/containerd/ && \
+cat <<EOT | sudo tee -a /etc/containerd/config.toml > /dev/null
+  [proxy_plugins]
+    [proxy_plugins.starlight]
+      type = "snapshot"
+      address = "/run/starlight/starlight-snapshotter.sock"
+EOT
+
+sudo systemctl restart containerd
+
+touch /home/ubuntu/.completed
+              EOF
 }
diff --git a/demo/terraform/outputs.tf b/demo/terraform/outputs.tf
index bca2051..e8d1f67 100644
--- a/demo/terraform/outputs.tf
+++ b/demo/terraform/outputs.tf
@@ -10,6 +10,12 @@ output "cloud-instance-public-ip" {
   sensitive   = false
 }
 
+output "cloud-instance-private-ip" {
+  description = "The ec2 instance private ip"
+  value       = aws_instance.starlight_cloud.private_ip
+  sensitive   = false
+}
+
 output "edge-instance-id" {
   description = "The ec2 instance id"
   value       = aws_instance.starlight_edge.id
@@ -21,3 +27,10 @@ output "edge-instance-public-ip" {
   value       = aws_instance.starlight_edge.public_ip
   sensitive   = false
 }
+
+output "edge-instance-private-ip" {
+  description = "The ec2 instance private ip"
+  value       = aws_instance.starlight_edge.private_ip
+  sensitive   = false
+}
+
diff --git a/demo/terraform/terraform.tfvars b/demo/terraform/terraform.tfvars
index 5dbc703..b40a803 100644
--- a/demo/terraform/terraform.tfvars
+++ b/demo/terraform/terraform.tfvars
@@ -4,3 +4,18 @@ ssh_key_name = "starlight-key"
 # please replace with your own public key
 # this is the key for accessing the EC2 instances, if empty, we assume the key above is already created
 ssh_public_key = "ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAgEA08XpxYFsakw1hVCcFYTVvXMF+sxyF9PQ16dF+D1xP3Vr5MGq275FIyKQs0oQxKCCBtUGMIgCVrQ+V3CVu9CsfBTHoJlk1nW0dfJcB3j9kG3+3NRIHZ+FTrIO5hvMWJ+SzXTIDYgdcKLJ/YIboqBI4Rm/rNx7fVIib0K+43RcD4YRbwcMXK1lWkZ2czj+ixKmX5P0EeTLNz2J6XEsLT3V39B4QoXLKJM3TTjlRx2w980UI2lUED+3aW8jeW6Y9v9TUW5ZiZHxFLULQGD4UoivxLHUZCVJbnSaBX5r1yiE2jOvlhCXgaEa/EyYt6gBacMMC/qdIIX6nsts5+K4i/+37Jixd1qyJ5tWcxY9tUQZeiD1kuL0qy/mKtR9ONZLOUFvmWXG7t5i9axVaIyjj+Yb/4PXvEVd3jZ0jC8gcPjkkjOV22CLCCLgwtUHcDTE/OM8/oUem4Tnd/9blBjd47RfuHTdyVsujLwue5hpUo64E3JDSdVee0s1Yso1Wx8ZEfhJgqfAc+E5gSprY2pdFZUwffN52I8+72OfNnsmw1h10LqvN5Xpu+12eARr/LQHY/0E45kJRAcFbAgjUwKmKtdf0UAepb/AwLF37I9UT4uLxs56dvw4z1rJQDGQJdm8GVv0CJ8pzjBsYFYzf43Mp4+lYJ+V7BBMMo5YLa6em/bqi5s= mc256"
+
+# recommended to change to machine with more memory
+#
+# Current setting is tide to AWS free tier limit 750hours of t3.micro (1GB memory).
+cloud_instance_type = "t3.micro"
+edge_instance_type  = "t3.micro"
+
+
+# EBS volume size in GB
+# Cloud will need more space for storing the container image and metadata than the edge.
+# Please adjust the size according to your needs.
+#
+# Current setting is tide to AWS EBS free tier limit 30GB
+cloud_ebs_size_in_gb = 20
+edge_ebs_size_in_gb  = 10
diff --git a/demo/terraform/variables.tf b/demo/terraform/variables.tf
index 099cfdb..dffcdc8 100644
--- a/demo/terraform/variables.tf
+++ b/demo/terraform/variables.tf
@@ -16,9 +16,15 @@ variable "ssh_public_key" {
   default     = ""
 }
 
-variable "instance_type" {
+variable "cloud_instance_type" {
   type        = string
-  default     = "t3a.nano"
+  default     = "m5a.large"
+  description = "the instance type to use"
+}
+
+variable "edge_instance_type" {
+  type        = string
+  default     = "t2.micro"
   description = "the instance type to use"
 }
 
@@ -28,7 +34,13 @@ variable "project_id" {
   description = "the project name"
 }
 
-variable "ebs_size_in_gb" {
+variable "cloud_ebs_size_in_gb" {
+  type        = number
+  default     = 20
+  description = "the ebs size in gb"
+}
+
+variable "edge_ebs_size_in_gb" {
   type        = number
   default     = 10
   description = "the ebs size in gb"
diff --git a/docs/newbie.md b/docs/newbie.md
deleted file mode 100644
index 6b32e3f..0000000
--- a/docs/newbie.md
+++ /dev/null
@@ -1,387 +0,0 @@
-# TL;DR All-in-one Quick Start Guide
-
-To finish this guide, you will need TWO machines (or VMs) far away from each other. 
-One acts as the Cloud, and the other acts as the Edge. You will need to identify the IP address of the Cloud server.
-
-The following instructions have been tested using AWS EC2 t2.micro with Ubuntu 22.04 LTS and `starlight v0.3.2`.
-
-`git checkout v0.3.2`
-
----
-
-## The "Cloud"
-
-In this machine you will need to set up the Starlight Proxy and a standard container registry. 
-If you are using AWS EC2, please add the following ports to the Security Group whitelist when you create the VM:
- - TCP 8090: Starlight Proxy (set the source to be your VPC)
- - TCP 8080: Adminer - for Metadata database  (set the source to be your own IP)
- - TCP 5000: Container Registry  (set the source to be your VPC)
- - TCP 80: Container Registry (set the source to be your VPC)
-
-
-1. Change the hostname of the server (Don't copy and paste, replace `<ip address>` with your server's IP address)
-
-    ```shell
-    echo "cloud" | sudo tee /etc/hostname > /dev/null
-    sudo hostname -F /etc/hostname
-    echo "<ip address> cloud.cluster.local" | sudo tee -a /etc/hosts > /dev/null
-    ```
-
-2. Install [Docker](https://docs.docker.com/engine/install/ubuntu/#install-using-the-repository) and [Docker Compose](https://docs.docker.com/compose/install/)  
-
-    If using Ubuntu 22.04 LTS, you could install Docker and Docker Compose using the following commands: 
-    ```shell
-    sudo apt update && \
-    sudo apt upgrade -y && \
-    sudo apt install -y docker-compose && \
-    sudo usermod -aG docker $USER
-    ```
-    After adding the current user to the `docker` group, you (may) **need to log out and log in** to take effect.
-    To confirm that Docker is working with correct permission, `docker ps` should not print any errors.
-    ```shell
-    docker ps
-    # CONTAINER ID   IMAGE     COMMAND   CREATED   STATUS    PORTS     NAMES
-    ```
-   
-3. Clone this project and launch the registry and proxy containers from `./demo/compose/registry+proxy`
-
-    ```shell
-    git clone https://github.com/mc256/starlight.git && \
-    cd starlight/demo/compose/registry+proxy && \
-    git checkout v0.3.1 && \
-    docker-compose up -d
-    # Creating network "registryproxy_default" with the default driver
-    # Creating registryproxy_db_1       ... done
-    # Creating registryproxy_registry_1 ... done
-    # Creating registryproxy_dbadmin_1  ... done
-    # Creating registryproxy_proxy_1    ... done
-    ```
-    The Starlight proxy writes image metadata to the Postgres database, and
-    the container registry saves container images to `./data_registry`.
-
-    
-3. Verify the registry and proxy are running.
-    ```shell
-    # This checks the Starlight Proxy
-    curl http://cloud.cluster.local:8090/
-    # {"status":"OK","code":200,"message":"Starlight Proxy"}
-    # This checks the container registry
-    curl http://cloud.cluster.local:5000/v2/
-    # {}
-    ```
-
-    If it does not work, please restart the containers after the database has been created (missing a db health check).
-    We could put a Nginx reverse proxy to handle SSL certificates or load balancing.
-    But for simplicity, this part is ignored in this example.
-
-
-3. Adjust the TCP window size (Optional). 
-   If the edge node is far away, we will need to adjust the TCP window size so that the connection can speed up to the speed limit faster. (You could calculate the best TCP window size using https://www.speedguide.net/bdp.php later)
-   If you skip this step, the connection will be much slower (the impact on speed is worse for other methods!).
-
-    ```shell
-    cat <<EOT | sudo tee -a /etc/sysctl.conf > /dev/null
-    net.core.wmem_max=125829120
-    net.core.rmem_max=125829120
-    net.ipv4.tcp_rmem= 10240 87380 125829120
-    net.ipv4.tcp_wmem= 10240 87380 125829120
-    net.ipv4.tcp_window_scaling = 1
-    net.ipv4.tcp_timestamps = 1
-    net.ipv4.tcp_sack = 1
-    net.ipv4.tcp_no_metrics_save = 1
-    net.core.netdev_max_backlog = 10000
-    EOT
-    sudo sysctl -p
-    ```
-
-🙌 That's it. Please obtain the IP address of this machine.
-
----
-
-## The "Edge"
-
-Please get another machine (or VM), you will need to set up a container worker with Starlight Snapshotter plugin.
-
-### 1. Install Dependencies
-
-The worker machine needs `build-essential` and `containerd`.
-```shell
-sudo apt update && sudo apt upgrade -y && \
-sudo apt install -y build-essential containerd
-```
-
-Enable `containerd`
-```shell
-sudo systemctl enable containerd  && \
-sudo systemctl start containerd
-```
-
-Verify `containerd` is running
-```shell
-sudo systemctl status containerd
-#      Active: active
-```
-
-Install Go https://go.dev/doc/install ➡️
-```shell
-wget https://go.dev/dl/go1.18.3.linux-amd64.tar.gz && \
-sudo rm -rf /usr/local/go && sudo tar -C /usr/local -xzf go1.18.3.linux-amd64.tar.gz
-```
-
-Add Go to the environment variable (You may want to change `.zshrc` or `.bashrc` file to permanently add this folder to the `PATH` environment variable)
-```shell
-export PATH=$PATH:/usr/local/go/bin
-```
-
-Verify Go is available
-```shell
-go version
-# go version go1.18.3 linux/amd64
-```
-
-### 2. (Optional) Tune the network
-
-Adjust the TCP window size. If the edge node is far away, we will need to adjust the TCP window size so that the connection can speed up to the speed limit faster. (You could calculate the best TCP window size using https://www.speedguide.net/bdp.php later)
-
-```shell
-cat <<EOT | sudo tee -a /etc/sysctl.conf > /dev/null
-net.core.wmem_max=125829120
-net.core.rmem_max=125829120
-net.ipv4.tcp_rmem= 10240 87380 125829120
-net.ipv4.tcp_wmem= 10240 87380 125829120
-net.ipv4.tcp_window_scaling = 1
-net.ipv4.tcp_timestamps = 1
-net.ipv4.tcp_sack = 1
-net.ipv4.tcp_no_metrics_save = 1
-net.core.netdev_max_backlog = 10000
-EOT
-sudo sysctl -p
-```
-
-### 3. Clone and Build
-Clone the Starlight repository
-```shell
-git clone https://github.com/mc256/starlight.git && \
-cd starlight && \ 
-git checkout v0.3.1
-```
-
-Build the snapshotter plugin and CLI tool
-```shell
-make starlight-daemon ctr-starlight
-```
-
-### 4. Configure Starlight Snapshotter
-
-Find out the IP address / DNS of the Starlight Proxy server and set these two environment variables (Don't Copy-Paste!)
-```shell
-# DO NOT COPY! 
-# This is just an example !!! Get the real address of your server !!!
-echo "<ip address> cloud.cluster.local" | sudo tee -a /etc/hosts > /dev/null
-````
-
-Verify that the Starlight proxy is accessible from the worker. 
-```shell
-curl http://cloud.cluster.local:8090/
-# {"status":"OK","code":200,"message":"Starlight Proxy"}
-curl http://cloud.cluster.local:5000/v2/
-# {}
-```
-If it does not work, please check the firewall configurations, 
-Please add port 8090 and 5000 to the firewall whitelist, the worker has to access these ports.
-
-Install Starlight Snapshotter `systemd` service and CLI tool.
-```shell
-sudo make install install-systemd-service
-```
-
-Enable Starlight snapshotter service
-```shell
-sudo systemctl enable starlight   && \
-sudo systemctl start starlight
-```
-
-Verify Starlight is running
-```shell
-sudo systemctl status starlight
-# it should be "active".
-```
-
-Add Starlight Proxy profile to the Snapshotter's configuration file
-```shell
-sudo ctr-starlight add myproxy http cloud.cluster.local:8090
-```
-
-Confirm that the proxy has been added
-```shell
-sudo ctr-starlight ls
-# [starlight-shared] https://starlight.yuri.moe
-#          [myproxy] http://cloud.cluster.local:8090
-```
-
-Test the proxy is working
-```shell
-sudo ctr-starlight test myproxy
-# ping test success: ok! - http://cloud.cluster.local:8090
-# latency: XX ms
-```
-
-
-### 5. Configure `contaienrd`
-
-Add configuration to `/etc/containerd/config.toml`. 
-(If you have set other `proxy_plugins`, please manually edit the file)
-```shell
-sudo mkdir /etc/containerd/ && \
-cat <<EOT | sudo tee -a /etc/containerd/config.toml > /dev/null
-  [proxy_plugins]
-    [proxy_plugins.starlight]
-      type = "snapshot"
-      address = "/run/starlight/starlight-snapshotter.sock"
-EOT
-```
-
-Restart `containerd` service
-```shell
-sudo systemctl restart containerd
-```
-
-Verify the Starlight snapshotter plugin is functioning
-```shell
-sudo ctr plugin ls | grep starlight 
-# io.containerd.snapshotter.v1    starlight                -              ok
-```
-
-### 6. Convert Container Image
-
-Convert the container image to the **Starlight format** container image and report to the Starlight proxy.
-
-
-```shell
-sudo ctr-starlight convert \
-    --insecure-destination \
-    --notify --profile myproxy \
-    --platform linux/amd64 \
-    docker.io/library/redis:6.2.1 cloud.cluster.local/redis:6.2.1-starlight && \
-sudo ctr-starlight convert \
-    --insecure-destination \
-    --notify --profile myproxy \
-    --platform linux/amd64 \
-    docker.io/library/redis:6.2.2 cloud.cluster.local/redis:6.2.2-starlight
-```
-
-In this example, we load two versions of the Redis container image from docker hub and convert them to the Starlight 
-format container image and notify the Starlight proxy (using `--notify` flag).
-
-### 7. Optimize Container Image
-
-Set `starlight` as the default containerd snapshotter in command line (optional).
-```shell
-export CONTAINERD_SNAPSHOTTER=starlight
-```
-
-Collect traces on the worker for container startup.
-```shell
-sudo ctr-starlight optimizer on && \
-sudo ctr-starlight pull --profile myproxy cloud.cluster.local/redis:6.2.1-starlight && \
-mkdir /tmp/test-redis-data && \
-sudo ctr c create \
-   --snapshotter=starlight \
-   --mount type=bind,src=/tmp/test-redis-data,dst=/data,options=rbind:rw \
-   --env-file ./demo/config/all.env \
-   --net-host \
-   cloud.cluster.local/redis:6.2.1-starlight \
-   instance1 /usr/local/bin/redis-server && \
-sudo ctr task start instance1
-```
-
-You may terminate the container using `Ctrl-C`, and remove the container:
-```shell
-sudo ctr container rm instance1
-```
-
-Repeat the same thing for `redis:6.2.2`
-```shell
-sudo ctr-starlight pull --profile myproxy cloud.cluster.local/redis:6.2.2-starlight && \
-sudo ctr c create \
-   --snapshotter=starlight \
-   --mount type=bind,src=/tmp/test-redis-data,dst=/data,options=rbind:rw \
-   --env-file ./demo/config/all.env \
-   --net-host \
-   cloud.cluster.local/redis:6.2.2-starlight \
-   instance2 /usr/local/bin/redis-server && \
-sudo ctr task start instance2
-```
-
-Terminate the container using `Ctrl-C`, and remove the container:
-```shell
-sudo ctr container rm instance2
-```
-
-Report traces to the Starlight Proxy.
-```shell
-sudo ctr-starlight optimizer off && \
-sudo ctr-starlight report --profile myproxy
-# set optimizer: completed request
-#	sha256:291220ae234f1aa9655359d7e553b05fa9e288fd811b7429f229e5aecd64a181: collected 40.966s file access traces - okay
-#	sha256:561c5b8bb95e26feb56b2bfda1d1fe2aee3229e5d9a2cc879e7524f05ad427a8: collected 13.459s file access traces - okay
-# reported traces: uploaded traces
-```
-
-
-### 8. Clear all the cache and reset the environment
-```shell
-# This script will kill containerd and starlight.
-# And then it removes ALL contents
-# in /var/lib/containerd, /var/lib/starlight, and /tmp/test-redis-data (the redis data directory)
-# and unmount all the mount points in /var/lib/starlight/mounts
-sudo ./demo/reset.sh 
-
-# Confirm that the cache is cleared
-sudo ls -al /var/lib/containerd
-# > ls: cannot access '/var/lib/containerd': No such file or directory
-sudo ls -al /var/lib/starlight
-# > ls: cannot access '/var/lib/starlight': No such file or directory
-
-
-# restart the processes
-sudo systemctl start starlight containerd
-```
-
-### 9. (Optional) Check the metadata database
-
-You could also inspect the metadata database using the Adminer Web UI at `http://<public ip address of the cloud>:8080/` (system: `PostgreQL`,server: `db`, username: `postgres`, password is the same as the username).
-
-In the `file` table, you could see it records the access order of the file.
-
-![adminer](./images/metadatadb-screenshot.png)
-
-
-### 10. Deploying and update container
-
-Start a container using Starlight (it should be much faster)
-```shell
-sudo ctr-starlight pull --profile myproxy cloud.cluster.local/redis:6.2.1-starlight && \
-mkdir /tmp/test-redis-data && \
-sudo ctr c create \
-   --snapshotter=starlight \
-   --mount type=bind,src=/tmp/test-redis-data,dst=/data,options=rbind:rw \
-   --env-file ./demo/config/all.env \
-   --net-host \
-   cloud.cluster.local/redis:6.2.1-starlight \
-   instance3 /usr/local/bin/redis-server && \
-sudo ctr task start instance3
-```
-
-Update a container using Starlight (also way faster)
-```shell
-sudo ctr-starlight pull --profile myproxy cloud.cluster.local/redis:6.2.2-starlight && \
-sudo ctr c create \
-   --snapshotter=starlight \
-   --mount type=bind,src=/tmp/test-redis-data,dst=/data,options=rbind:rw \
-   --env-file ./demo/config/all.env \
-   --net-host \
-   cloud.cluster.local/redis:6.2.2-starlight \
-   instance4 /usr/local/bin/redis-server && \
-sudo ctr task start instance4
-```
diff --git a/docs/starlight-proxy.md b/docs/starlight-proxy.md
deleted file mode 100644
index 456fb03..0000000
--- a/docs/starlight-proxy.md
+++ /dev/null
@@ -1,208 +0,0 @@
-# Starlight Proxy
-
-**⚠️ This document is outdated. Please use https://github.com/mc256/starlight/blob/master/docs/newbie.md instead**
-
-This is the **Step 1** to use Starlight:
-
-Set up a Starlight proxy, ideally close to the registry server you are using. 
-Configure the proxy server to point to the registry and run it. Starlight supports any standard registry.
-
-[⬅️ Back to README.md](https://github.com/mc256/starlight#getting-started)
-
----
-## Method 0. Use Helm (Recommended)
-
-```shell
-helm install my-starlight-proxy oci://ghcr.io/mc256/starlight/starlight-proxy-chart --version 0.1.2
-```
-
-You may need to customize the chart. Helm chart configuration are [available here](https://github.com/mc256/starlight/blob/master/docs/helm.md).
-
-
-## Method 1. Use Docker Compose to deploy Starlight Proxy + Container Registry (Recommended)
-
-This is an all-in-one example in case you don't have full access to a container registry.
-We could use Docker Compose to deploy both the proxy and the registry on the same machine.
-
-
-0. Install [Docker](https://docs.docker.com/engine/install/ubuntu/#install-using-the-repository) and [Docker Compose](https://docs.docker.com/compose/install/)  
-
-If using Ubuntu 20.04 LTS, you could install Docker and Docker Compose using the following commands: 
-```shell
-sudo apt update && \
-sudo apt upgrade -y && \
-sudo apt install -y docker-compose && \
-sudo usermod -aG docker $USER
-```
-After adding the current user to the `docker` group, you may _need to log out and log in_ to take effect.
-To confirm that Docker is working with correct permission, `docker ps` should not print any errors.
-```shell
-docker ps
-# CONTAINER ID   IMAGE     COMMAND   CREATED   STATUS    PORTS     NAMES
-```
-
-1. Clone this project and launch the registry and proxy containers from `./demo/compose/registry+proxy`
-
-```shell
-git clone https://github.com/mc256/starlight.git && \
-cd starlight/demo/compose/registry+proxy && \
-docker-compose up -d
-# Creating network "registryproxy_default" with the default driver
-# Creating starlightproxy    ... done
-# Creating starlightregistry ... done
-```
-The Starlight proxy writes image metadata to `./data_proxy` folder, and
-the container registry saves container images to `./data_registry`
-
-
-2. Verify the registry and proxy are running.
-```shell
-curl http://localhost:8090/
-# Starlight Proxy OK!
-curl http://localhost:5000/v2/
-# {}
-```
-
-The Starlight proxy listens on port 8090. 
-We could put a Nginx reverse proxy to handle SSL certificates or load balancing.
-But for simplicity, this part is ignored in this example.
-Please add port 8090 and 5000 to the firewall whitelist, the worker has to access these ports.
-
-🙌 That's it. Please obtain the IP address of the server and proceed to the **Step 2**.
-
-```shell
-# update the IP address keep this for future use. 
-export STARLIGHT_PROXY=<ip address of your server>:8090
-export REGISTRY=<ip address of your server>:5000
-```
-
-[⬅️ Back to README.md](https://github.com/mc256/starlight#getting-started) 
-
----
-## Method 2. Use Docker Compose (Starlight Only)
-
-The prebuilt Starlight proxy container image is available at  `ghcr.io/mc256/starlight/proxy:latest`.
-
-0. Install [Docker](https://docs.docker.com/engine/install/ubuntu/#install-using-the-repository) and [Docker Compose](https://docs.docker.com/compose/install/)  
-
-If using Ubuntu 20.04 LTS, you could install Docker and Docker Compose using the following commands: 
-```shell
-sudo apt update && \
-sudo apt upgrade -y && \
-sudo apt install -y docker-compose && \
-sudo usermod -aG docker $USER
-```
-After adding the current user to the `docker` group, you may need to log out and log in to take effect.
-To confirm that Docker is working with correct permission, `docker ps` should not print any errors.
-```shell
-docker ps
-# CONTAINER ID   IMAGE     COMMAND   CREATED   STATUS    PORTS     NAMES
-```
-
-1. Clone this project 
-
-```shell
-git clone https://github.com/mc256/starlight.git && \
-```
-
-2. Set `REGISTRY` environment variable to your own container registry. 
-
-```shell
-echo "REGISTRY=http://starlightregistry:5000" >> ./starlight/demo/compose/proxy/.env
-```
-
-3. Launch the proxy
-```shell
-cd ./starlight/demo/compose/proxy && \
-docker-compose up -d
-# Creating starlightproxy ... done
-```
-
-The Starlight proxy writes image metadata to `./data_proxy` folder.
-
-2. Verify the registry and proxy are running.
-```shell
-curl http://localhost:8090/
-# Starlight Proxy OK!
-```
-
-The Starlight proxy listens on port 8090. 
-We could put a Nginx reverse proxy to handle SSL certificates or load balancing.
-But for simplicity, this part is ignored in this example.
-
-🙌 That's it. Please obtain the IP address of the server and proceed to the **Step 2**.
-
-```shell
-# update the IP address keep this for future use. 
-export STARLIGHT_PROXY=<ip address of your server>:8090
-export REGISTRY=<ip address of your server>:5000
-```
-
-[⬅️ Back to README.md](https://github.com/mc256/starlight#getting-started)
-
----
-## Method 3. Build from source
-
-0. Install Go https://go.dev/doc/install ➡️
-```shell
-wget https://go.dev/dl/go1.17.8.linux-amd64.tar.gz &&
-sudo rm -rf /usr/local/go && sudo tar -C /usr/local -xzf go1.17.8.linux-amd64.tar.gz
-```
-
-1. Add Go to the environment variable (You may want to change `.zshrc` or `.bashrc` file to permanently add this folder to the `PATH` environment variable)
-```
-export PATH=$PATH:/usr/local/go/bin
-```
-
-2. Verify Go is available with `go version`
-```shell
-go version
-#go version go1.17.8 linux/amd64
-```
-
-4. Install necessary tools to build this project
-
-```shell
-sudo apt update && \
-sudo apt upgrade -y && \
-sudo apt install build-essential
-```
-
-4. Clone this project.
-
-```shell
-git clone https://github.com/mc256/starlight.git && \
-cd starlight
-```
-
-5. Build Starlight proxy
-```shell
-make starlight-proxy
-```
-
-6. Run Starlight
-```shell
-cd ./out && \
-mkdir ./data && \
-./starlight-proxy --registry=http://myregistry:5000 &
-```
-
-7. Verify the Starlight Proxy is working
-```shell
-curl http://localhost:8090/
-# Starlight Proxy OK!
-```
-
-The Starlight proxy listens on port 8090. 
-We could put a Nginx reverse proxy to handle SSL certificates or load balancing.
-But for simplicity, this part is ignored in this example.
-
-🙌 That's it. Please obtain the IP address of the server and proceed to the **Step 2**.
-
-```shell
-# update the IP address keep this for future use. 
-export STARLIGHT_PROXY=<ip address of your server>:8090
-export REGISTRY=<ip address of your server>:5000
-```
-
-[⬅️ Back to README.md](https://github.com/mc256/starlight#getting-started)
diff --git a/docs/starlight-snapshotter.md b/docs/starlight-snapshotter.md
deleted file mode 100644
index f84c15b..0000000
--- a/docs/starlight-snapshotter.md
+++ /dev/null
@@ -1,207 +0,0 @@
-# Starlight Snapshotter Plugin
-
-**⚠️ This document is outdated. Please use https://github.com/mc256/starlight/blob/master/docs/newbie.md instead**
-
-This is the **Step 2** to use Starlight:
-
-Set up the worker to be able to run Starlight. 
-This involves 
-installing **containerd** and the **Starlight snapshotter plugin**, 
-configuring containerd to use the plugin, 
-and starting the Starlight snapshotter daemon
-(you also need to tell the snapshotter the address of the proxy server).
-
-[⬅️ Back to README.md](https://github.com/mc256/starlight)
-
----
-
-## Method 1. Install Pre-built Package (Recommended)
-
-Pre-build deb package is available for `amd64`, `armhf`, and `arm64`.
-
-### 1. Install Starlight Snapshotter
-
-Download and install the `.deb` package from the [release page](https://github.com/mc256/starlight/releases).
-
-```shell
-export ARCH=amd64
-export SL_VERSION=0.1.2
-wget "https://github.com/mc256/starlight/releases/download/v${SL_VERSION}/starlight-snapshotter_${SL_VERSION}_$ARCH.deb"
-sudo apt install -f "./starlight-snapshotter_${SL_VERSION}_$ARCH.deb"
-```
-
-Update systemd service file `/lib/systemd/system/starlight.service`. 
-- Change `STARLIGHT_PROXY` to the address of the Starlight Proxy.
-- remove `--plain-http` if the Starlight Proxy is behind a HTTPS reverse proxy.
-```
-ExecStart=/usr/bin/starlight-grpc run --plain-http starlight.lan
-```
-
-Reload systemd service
-```shell
-sudo systemctl daemon-reload
-sudo systemctl restart starlight-snapshotter
-```
-
-### 2. Configure `contaienrd`
-
-Add configuration to `/etc/containerd/config.toml`. 
-(If you have set other `proxy_plugins`, please manually edit the file)
-```shell
-sudo mkdir /etc/containerd/ && \
-cat <<EOT | sudo tee -a /etc/containerd/config.toml > /dev/null
-[proxy_plugins]
-  [proxy_plugins.starlight]
-    type = "snapshot"
-    address = "/run/starlight-grpc/starlight-snapshotter.socket"
-EOT
-```
-
-Restart `containerd` service
-```shell
-sudo systemctl restart containerd
-```
-
-Verify the Starlight snapshotter plugin is functioning
-```shell
-sudo ctr plugin ls | grep starlight 
-# io.containerd.snapshotter.v1    starlight                -              ok
-```
-
-
-🙌 That's it. Please proceed to the **Step 3**.
-
-[⬅️ Back to README.md](https://github.com/mc256/starlight#getting-started) 
-
-
-
----
-
-
-
-## Method 2. Build from source
-
-### 1. Install Dependencies
- 
-The worker machine is supposed to be far away (in latency) to the registry and proxy.
-Please install **containerd** and **Starlight snapshotter** on a new machine (or VM), not the same machine that runs the proxy or the registry. 
-
-The worker machine needs `build-essential` and `containerd`.
-```shell
-sudo apt update && sudo apt upgrade -y && \
-sudo apt install -y build-essential containerd
-```
-
-Enable `containerd`
-```shell
-sudo systemctl enable containerd  && \
-sudo systemctl start containerd
-```
-
-Verify `containerd` is running
-```shell
-sudo systemctl status containerd
-#      Active: active
-```
-
-Install Go https://go.dev/doc/install ➡️
-```shell
-wget https://go.dev/dl/go1.17.8.linux-amd64.tar.gz && \
-sudo rm -rf /usr/local/go && sudo tar -C /usr/local -xzf go1.17.8.linux-amd64.tar.gz
-```
-
-Add Go to the environment variable (You may want to change `.zshrc` or `.bashrc` file to permanently add this folder to the `PATH` environment variable)
-```shell
-export PATH=$PATH:/usr/local/go/bin
-```
-
-Verify Go is available
-```shell
-go version
-# go version go1.17.8 linux/amd64
-```
-
-### 2. Clone and Build
-Clone the Starlight repository
-```shell
-git clone https://github.com/mc256/starlight.git && \
-cd starlight
-```
-
-Build the snapshotter plugin and CLI tool
-```shell
-make starlight-grpc ctr-starlight
-```
-
-### 3. Configure Starlight Snapshotter
-
-You need to find out the IP address / DNS of the Starlight Proxy server (in **Step 1**. [Find out how to install **Starlight proxy** ➡️](docs/starlight-proxy.md) )
-
-```shell
-# This is an example
-export STARLIGHT_PROXY=172.18.1.3:8090
-export REGISTRY=172.18.1.3:5000
-```
-
-Verify that the Starlight proxy is accessible from the worker. 
-```shell
-curl http://$STARLIGHT_PROXY
-# Starlight Proxy OK!
-```
-
-Install Starlight Snapshotter `systemd` service and CLI tool.
-Please follow the prompt, enter 
-```shell
-sudo make install install-systemd-service
-#Please enter Starlight Proxy address (example: proxy.mc256.dev:8090):172.18.1.3:8090
-#Enable HTTPS Certificate (requires load balancer like Nginx) (y/N):n
-#Created systemd service file (/lib/systemd/system/starlight.service)
-#Reloaded systemd daemon
-```
-
-Enable Starlight snapshotter service
-```shell
-sudo systemctl enable starlight   && \
-sudo systemctl start starlight
-```
-
-Verify Starlight is running
-```shell
-sudo systemctl status starlight
-# it should be "active".
-```
-
-### 4. Configure `contaienrd`
-
-Add configuration to `/etc/containerd/config.toml`. 
-(If you have set other `proxy_plugins`, please manually edit the file)
-```shell
-sudo mkdir /etc/containerd/ && \
-cat <<EOT | sudo tee -a /etc/containerd/config.toml > /dev/null
-[proxy_plugins]
-  [proxy_plugins.starlight]
-    type = "snapshot"
-    address = "/run/starlight-grpc/starlight-snapshotter.socket"
-EOT
-```
-
-Restart `containerd` service
-```shell
-sudo systemctl restart containerd
-```
-
-Verify the Starlight snapshotter plugin is functioning
-```shell
-sudo ctr plugin ls | grep starlight 
-# io.containerd.snapshotter.v1    starlight                -              ok
-```
-
-
-🙌 That's it. Please proceed to the **Step 3**.
-
-[⬅️ Back to README.md](https://github.com/mc256/starlight#getting-started) 
-
-
----
-
-For more information, please see `ctr-starlight --help` and `starlight-grpc --help`
diff --git a/docs/starlight-workflow.md b/docs/starlight-workflow.md
deleted file mode 100644
index e5eac96..0000000
--- a/docs/starlight-workflow.md
+++ /dev/null
@@ -1,19 +0,0 @@
-# How Starlight Works (Overview)
-![starlight-workflow](images/starlight-workflow.png)
-
-Once the user issues a worker `PULL` command to download a set of containers ①,
-the command is received by the standard **containerd** daemon.
-**containerd** then forwards the command to the **Starlight snapshotter** daemon ②, 
-and waits for confirmation that the requested images have been found.
-The Starlight snapshotter opens a connection to the **Starlight proxy** 
-and sends the list of requested containers as well as the list of relevant containers that already exist on the worker ③. 
-The proxy queries the directory database ④ for the list of files in the various layers of the 
-requested container image, as well in the image already available on the worker.
-
-The proxy will then begin computing the **delta bundle** that includes the set of distinct compressed file contents that the worker does not already have, specifically organized to speed up deployment;
-In the background, the proxy also responds with HTTP 200 OK header to the snapshotter, which notifies **containerd** that the `PULL` phase has finished successfully; the snapshotter however, remains active and keeps the connection open to receive the data from the proxy.
-In the background, the proxy issues a series of requests to the registry ⑦ to retrieve the compressed contents of files needed for delta bundle.
-Once the contents of the delta bundle has been computed, the proxy creates a **Starlight manifest** (SLM) -- the list of file metadata, container manifests, and other required metadata -- and sends it to the snapshotter ⑤,
-which notifies **containerd** that the `PULL` phase has finished successfully.
-
-Please read our [NSDI '22 paper](https://www.usenix.org/conference/nsdi22/presentation/chen-jun-lin) for more details.

From e267a6af8b8332a1eb15c182626a79771770b1c8 Mon Sep 17 00:00:00 2001
From: Jun Lin Chen <contact@mc256.dev>
Date: Sun, 4 Feb 2024 19:56:32 -0500
Subject: [PATCH 4/4] add terraform deployment

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index cc4ee1c..ea9d743 100644
--- a/README.md
+++ b/README.md
@@ -51,9 +51,9 @@ Starlight is implemented on top of **containerd**. It is comprised of cloud and
 Starlight is compatible with Kubernetes and can replace the default `overlayfs` snapshotter.
 We could use helm to deploy Starlight on a Kubernetes cluster.
 
-- [I am familiar with **K8s** & Helm. **TL;DR**](https://github.com/mc256/starlight/blob/master/docs/helm.md)
-- [I have 2 **Virtual Machines**, but **TL;DR**](https://github.com/mc256/starlight/blob/master/docs/2vm.md)
+- [I am familiar with **K8s** & Helm.](https://github.com/mc256/starlight/blob/master/docs/helm.md)
 - [I know **Terraform** and have an **AWS** account](https://github.com/mc256/starlight/blob/master/docs/terraform.md)
+- [I just have 2 **Virtual Machines**.](https://github.com/mc256/starlight/blob/master/docs/2vm.md)
 
 ---