Merge branch 'v2_master'

# Conflicts: # Dockerfile # README.md # cmd/ctr-starlight/report/report.go # demo/deb-package/debian/starlight-snapshotter.service # go.mod # grpc/api.go
mc256 · Nov 10, 2022 · 9124455 · 9124455
2 parents 66866b9 + 1779895
commit 9124455
Show file tree

Hide file tree

Showing 112 changed files with 9,806 additions and 2,800 deletions.
diff --git a/.github/workflows/debian-package.yml b/.github/workflows/debian-package.yml
@@ -7,7 +7,7 @@ on:
       - 'v*.*.*'
 jobs:
   build:
-    runs-on: ubuntu-22.04
+    runs-on: self-hosted
     steps:
       - name: Check out repository code
         uses: actions/checkout@v3
@@ -18,6 +18,8 @@ jobs:
           sudo apt update -y
           sudo apt upgrade -y
           sudo apt install net-tools
+          curl -fsSL https://get.docker.com -o /tmp/get-docker.sh
+          sh /tmp/get-docker.sh
           docker run --privileged --rm tonistiigi/binfmt --install all
           docker build -t helper:latest  -f ./.github/.helper/Dockerfile ./.github/.helper/
           docker run -d --hostname helper --expose 8080 --name helper -v "$(pwd)"/sandbox:/app/upload:rw helper:latest

diff --git a/Dockerfile b/Dockerfile
@@ -7,7 +7,7 @@ ENV GO111MODULE=on
 ENV REGISTRY=registry2
 ENV LOGLEVEL=info
 
-RUN make change-version-number build-starlight-proxy-for-alpine && mkdir ./out/data
+RUN make change-version-number set-production build-starlight-proxy-for-alpine && mkdir ./out/data
 
 #CMD ["/go/src/app/out/starlight-proxy"]
 FROM alpine:3.12

diff --git a/Makefile b/Makefile
@@ -1,41 +1,40 @@
 ######################################################################
 # Build
 ######################################################################
-TARGETS=starlight-proxy starlight-grpc ctr-starlight
+TARGETS=starlight-proxy starlight-daemon ctr-starlight
 COMMONENVVAR=GOOS=$(shell uname -s | tr A-Z a-z)
 BUILDENVVAR=CGO_ENABLED=0
 VERSION=$(shell git describe --tags --match "v*" || echo "v0.0.0")
 VERSIONNUMBER=$(shell echo $(VERSION) | sed 's/v//g')
 COMPILEDATE=$(shell date +%Y%m%d)
 
 
-.PHONY: build clean build-starlight-proxy build-starlight-grpc build-ctr-starlight
+.PHONY: build clean build-starlight-proxy build-starlight-daemon build-ctr-starlight
 
 .SILENT: install-systemd-service
 
 ######################################################################
 # Build
 ######################################################################
 .PHONY: build
-build: build-starlight-proxy build-starlight-grpc build-ctr-starlight
+build: build-starlight-proxy build-starlight-daemon build-ctr-starlight
 
 .PHONY: build-starlight-proxy
 build-starlight-proxy:
 	-mkdir ./out 2>/dev/null | true
 	go mod tidy
 	go build -o ./out/starlight-proxy ./cmd/starlight-proxy/main.go
 
-.PHONY: build-starlight-grpc
-build-starlight-grpc:
+.PHONY: build-starlight-daemon
+build-starlight-daemon:
 	-mkdir ./out 2>/dev/null | true
-	go build -o ./out/starlight-grpc ./cmd/starlight-grpc/main.go
+	go build -o ./out/starlight-daemon ./cmd/starlight-daemon/main.go
 
 .PHONY: build-ctr-starlight
 build-ctr-starlight:
 	-mkdir ./out 2>/dev/null | true
 	go build -o ./out/ctr-starlight ./cmd/ctr-starlight/main.go
 
-
 .PHONY: build-starlight-proxy-for-alpine
 build-starlight-proxy-for-alpine:
 	-mkdir ./out 2>/dev/null | true
@@ -55,13 +54,17 @@ push-helm-package:
 change-version-number:
 	sed -i 's/var Version = "0.0.0"/var Version = "$(VERSIONNUMBER)-$(COMPILEDATE)"/g' ./util/version.go
 
+.PHONY: set-production
+set-production:
+	sed -i 's/production = false/production = true/g' ./util/config.go
+
 .PHONY: generate-changelog
 generate-changelog: 
 	mkdir -p ./sandbox/starlight-snapshotter-$(VERSIONNUMBER)-$(COMPILEDATE)/debian/ 2>/dev/null | true
 	sh -c ./demo/deb-package/generate-changelog.sh > ./sandbox/starlight-snapshotter-$(VERSIONNUMBER)-$(COMPILEDATE)/debian/changelog
 
 .PHONY: create-deb-package
-create-deb-package: change-version-number build-starlight-grpc build-ctr-starlight generate-changelog
+create-deb-package: change-version-number set-production build-starlight-daemon build-ctr-starlight generate-changelog
 	mkdir -p ./sandbox/starlight-snapshotter-$(VERSIONNUMBER)-$(COMPILEDATE)/ 2>/dev/null | true
 	cp -r ./demo/deb-package/debian ./sandbox/starlight-snapshotter-$(VERSIONNUMBER)-$(COMPILEDATE)/
 	mkdir -p ./sandbox/starlight-snapshotter-$(VERSIONNUMBER)-$(COMPILEDATE)/debian/starlight-snapshotter/usr/bin/ 2>/dev/null | true
@@ -76,6 +79,13 @@ create-deb-package: change-version-number build-starlight-grpc build-ctr-starlig
 	dh_builddeb
 	dpkg-deb --info ./sandbox/starlight-snapshotter_$(VERSIONNUMBER)_amd64.deb
 
+.PHONY: update-protobuf
+update-protobuf:
+	protoc --go_out=. --go_opt=paths=source_relative \
+    --go-grpc_out=. --go-grpc_opt=paths=source_relative \
+    client/api/daemon.proto
+
+
 ######################################################################
 ###### Platform dependent build
 
@@ -84,7 +94,7 @@ create-deb-package.amd64: create-deb-package
 
 
 .PHONY: create-deb-package.armv6l
-create-deb-package.armv6l: change-version-number build-starlight-grpc build-ctr-starlight generate-changelog
+create-deb-package.armv6l: change-version-number set-production build-starlight-daemon build-ctr-starlight generate-changelog
 	mkdir -p ./sandbox/starlight-snapshotter-$(VERSIONNUMBER)-$(COMPILEDATE)/ 2>/dev/null | true
 	cp -r ./demo/deb-package/debian ./sandbox/starlight-snapshotter-$(VERSIONNUMBER)-$(COMPILEDATE)/
 	mkdir -p ./sandbox/starlight-snapshotter-$(VERSIONNUMBER)-$(COMPILEDATE)/debian/starlight-snapshotter/usr/bin/ 2>/dev/null | true
@@ -101,7 +111,7 @@ create-deb-package.armv6l: change-version-number build-starlight-grpc build-ctr-
 	dpkg-deb --info ./sandbox/starlight-snapshotter_$(VERSIONNUMBER)_armhf.deb
 
 .PHONY: create-deb-package.arm64
-create-deb-package.arm64: change-version-number build-starlight-grpc build-ctr-starlight generate-changelog
+create-deb-package.arm64: change-version-number set-production build-starlight-daemon build-ctr-starlight generate-changelog
 	mkdir -p ./sandbox/starlight-snapshotter-$(VERSIONNUMBER)-$(COMPILEDATE)/ 2>/dev/null | true
 	cp -r ./demo/deb-package/debian ./sandbox/starlight-snapshotter-$(VERSIONNUMBER)-$(COMPILEDATE)/
 	mkdir -p ./sandbox/starlight-snapshotter-$(VERSIONNUMBER)-$(COMPILEDATE)/debian/starlight-snapshotter/usr/bin/ 2>/dev/null | true
@@ -179,4 +189,4 @@ install-systemd-service:
 	#systemctl daemon-reload
 
 docker-image:
-	docker build
+	docker build -t harbor.yuri.moe/public/starlight-proxy:latest -f Dockerfile .
diff --git a/README.md b/README.md
@@ -1,3 +1,4 @@
+
 # Starlight: Fast Container Provisioning
 
 [![Docker Image](https://img.shields.io/github/workflow/status/mc256/starlight/Docker%20Image?label=Proxy%20Docker%20Image&logo=docker&logoColor=white&style=flat-square)](https://github.com/mc256/starlight/actions/workflows/docker-image.yml)
@@ -72,47 +73,37 @@ and starting the Starlight snapshotter daemon
    The Starlight format is **backwards compatible** and almost the same size, so there is no need to store compressed layers twice. In other words, non-Starlight workers will descrompress Starlight images with no chanages.
    The **Starlight CLI tool** features the image conversion, example:
    ```shell
-    export REGISTRY=172.18.2.3:5000 && \
-    ctr-starlight convert \
-        --insecure-source --insecure-destination \
-        $REGISTRY/redis:6.2.1 $REGISTRY/redis:6.2.1-starlight
-   ```
-   `$REGISTRY` is your container registry (e.g. `172.18.2.3:5000`).
-
-   In addition, the proxy needs some metadata about the list of files in the container to compute the data for deployment.
-   ```shell
-    export STARLIGHT_PROXY=172.18.2.3:8090 && \
-    curl http://$STARLIGHT_PROXY/prepare/redis:6.2.1-starlight
-    #Cached TOC: redis:6.2.1-starlight
+    ctr-starlight convert --notify harbor.yuri.moe/public/redis:6.2.7 harbor.yuri.moe/starlight/redis:6.2.7
    ```
-   `$STARLIGHT_PROXY` is the address of your Starlight Proxy (e.g. `172.18.2.3:8090`)
+    In addition, the proxy needs some metadata about the list of files in the container to compute the data for deployment. 
+    The `--nofity` flag tells the proxy to fetch the metadata from the registry and store it in the metadata database.
+
 
 4) Collect traces on the worker for container startup. 
    This entails starting the container on the worker while collecting file access traces that are sent to the proxy.
 
    The **Starlight CLI tool** features trace collection, example:
    ```shell
-   sudo ctr-starlight pull redis:6.2.1-starlight && \
+   sudo ctr-starlight optimizer on
+   sudo ctr-starlight pull harbor.yuri.moe/starlight/redis:6.2.7 && \
    mkdir /tmp/test-redis-data && \
-   sudo ctr-starlight create --optimize \
-          --mount type=bind,src=/tmp/test-redis-data,dst=/data,options=rbind:rw \
-       --env-file ./demo/config/all.env \
-       --net-host \
-       redis:6.2.1-starlight \
-       redis:6.2.1-starlight \
-       instance1 && \
+   sudo ctr create --snapshotter=starlight \
+        --mount type=bind,src=/tmp/test-redis-data,dst=/data,options=rbind:rw \
+        --env-file ./demo/config/all.env --net-host \
+        harbor.yuri.moe/starlight/redis:6.2.7
+        instance1 && \
    sudo ctr task start instance1
    ```
 
    You may terminate the container using `Ctrl-C`, and remove the container:
    ```shell
    sudo ctr container rm instance1
    ```
-   Traces will be saved to `/tmp/starlight-optimizer` folder.
 
    After finished running the container several times, then we can report all the traces to the proxy, using:
    ```shell
-   ctr-starlight report --plain-http
+   sudo ctr-starlight optimizer off
+   sudo ctr-starlight report
    ```
 
 5) Reset `containerd` and `starlight`. Clean up all the downloaded containers and cache.
@@ -129,27 +120,25 @@ The good news is that they should be quick, a few minutes for each container.
 
 Start a container using Starlight
 ```shell
-sudo ctr-starlight pull redis:6.2.1-starlight && \
+sudo ctr-starlight pull harbor.yuri.moe/starlight/redis:6.2.7 && \
 mkdir /tmp/test-redis-data && \
-sudo ctr-starlight create \
-	--mount type=bind,src=/tmp/test-redis-data,dst=/data,options=rbind:rw \
-	--env-file ./demo/config/all.env \
-	--net-host \
-	redis:6.2.1-starlight \
-	redis:6.2.1-starlight \
+sudo ctr container create --snapshotter=starlight \
+    --mount type=bind,src=/tmp/test-redis-data,dst=/data,options=rbind:rw \
+    --env-file ./demo/config/all.env \
+    --net-host \
+    harbor.yuri.moe/starlight/redis:6.2.7 \
     instance3 && \
 sudo ctr task start instance3
 ```
 
-Update a container using Starlight (Step 3 and Step 4 need to be done for `redis:6.2.2`)
+Update a container using Starlight (Step 3 and Step 4 need to be done for `redis:7.0.5`)
 ```shell
-sudo ctr-starlight pull redis:6.2.1-starlight redis:6.2.2-starlight && \
-sudo ctr-starlight create \
-	--mount type=bind,src=/tmp/test-redis-data,dst=/data,options=rbind:rw \
-	--env-file ./demo/config/all.env \
-	--net-host \
-	redis:6.2.2-starlight \
-	redis:6.2.2-starlight \
+sudo ctr-starlight pull harbor.yuri.moe/starlight/redis:7.0.5 && \
+sudo ctr container create --snapshotter=starlight  \
+    --mount type=bind,src=/tmp/test-redis-data,dst=/data,options=rbind:rw \
+    --env-file ./demo/config/all.env \
+    --net-host \
+    harbor.yuri.moe/starlight/redis:7.0.5 \
     instance4 && \
 sudo ctr task start instance4
 ```
@@ -171,32 +160,30 @@ If you find Starlight useful in your work, please cite our NSDI 2022 paper:
 }
 ```
 
+
 ## Roadmap
 Starlight is not complete. Our roadmap:
 
+| Version                                                  | Status      | Scheduled Release |
+|----------------------------------------------------------|-------------|-------------------|
+| [v0.1.3](https://github.com/mc256/starlight/tree/v0.1.3) | stable      |                   |
+| [v0.2.0](https://github.com/mc256/starlight)             | development |                   |
+| v0.3.0                                                   |             | 2022-12-01        |
+| v0.4.0                                                   |             | 2023-01-01        |
 
-| Version                                                     | Scheduled Release |
-|-------------------------------------------------------------|-------------------|
-| [v0.2.0](https://github.com/mc256/starlight/tree/v2_master) | 2022-11-01        |
-| v0.3.0                                                      | 2022-12-01        |
-| v0.4.0                                                      | 2023-01-01        |
-
-
+Feature List:
 - [x] Scalable database backend (v0.2)
   - [x] Postgres Database Schema (v0.2)
   - [x] Starlight Proxy Server (v0.2)
 - [ ] Kubernetes support (v0.3)
   - [x] Starlight Proxy (v0.2)
     - [x] Helm Chart (v0.2)
     - [x] Starlight Proxy authentication (v0.2)
-  - [ ] New Starlight Snapshotter / Content Plugin for containerd (v0.2)
-  - [ ] Kubernetes CRD for more flexible container image pulling strategies in challenging network condition (v0.3)
-    - [ ] Sidecar container/DaemonSet for container image pulling (v0.3)
-    - [ ] Speed up multi-container image pulling (v0.4)
+  - [x] New Starlight Snapshotter / Content Plugin for containerd (v0.2)
+  - [ ] initContainer tool (v0.3)
 - [ ] OCI container registry support (v0.3)
   - [x] Goharbor support (v0.2)
   - [x] Multiple platforms image support (v0.2) 
   - [ ] Goharbor Hook/ Scanner for automatic image conversion (v0.3)
   - [ ] Jointly optimizing multiple containers deployments (v0.4)
-  - [ ] Converting containers that have already been fully retrieved using Starlight to use OverlayFS. (v0.4)
-
+  - [ ] Converting containers that have already been fully retrieved using Starlight to use OverlayFS. (v0.4)