diff --git a/.changeset/config.json b/.changeset/config.json index 72b2f31bf..4ea34313c 100644 --- a/.changeset/config.json +++ b/.changeset/config.json @@ -10,5 +10,9 @@ "ignore": [], "snapshot": { "useCalculatedVersion": true + }, + "privatePackages": { + "version": true, + "tag": true } } diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1748db068..49bc03c32 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -18,6 +18,8 @@ jobs: release: name: Release runs-on: ubuntu-latest + outputs: + is-blob-local-release: ${{ steps.check-blob-local-release.outputs.result }} steps: - name: Checkout uses: actions/checkout@v4 @@ -43,3 +45,50 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} NPM_TOKEN: ${{ secrets.NPM_TOKEN_ELEVATED }} + + - name: Check for @vercel/blob-local + if: steps.changesets.outputs.published == 'true' + id: check-blob-local-release + uses: actions/github-script@v6 + with: + result-encoding: string + script: | + const publishedPackages = ${{ steps.changesets.outputs.publishedPackages }}; + const hasBlobLocal = publishedPackages.some(item => item.name === "@vercel/blob-local"); + return hasBlobLocal ? 'true' : 'false' + + release-blob-local: + name: Release @vercel/blob-local + needs: release + if: needs.release.outputs.is-blob-local-release == 'true' + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Configure Buildx + uses: docker/setup-buildx-action@v2 + with: + # This configures an alias for `docker build` to run `docker buildx` + # so we don't have to update our build script to be platform-aware. + install: true + platforms: linux/amd64,linux/arm64 + + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + password: ${{ secrets.DOCKER_PASSWORD }} + username: ${{ secrets.DOCKER_USERNAME }} + + - name: Read package.json + working-directory: ./services/blob-local + id: package-version + run: echo "version=$(jq -r .version package.json)" >> $GITHUB_OUTPUT + + - name: Docker build and push + uses: docker/build-push-action@v4 + with: + context: ./services/blob-local + platforms: linux/amd64,linux/arm64 + push: true + tags: zeit/vercel-blob-local:${{ steps.package-version.outputs.version }} diff --git a/.gitignore b/.gitignore index f8abbdade..aad4caa9d 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,5 @@ npm-debug.log .turbo .DS_Store .vscode + +blob_fs diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 9d457c715..5166b166c 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -2,3 +2,4 @@ packages: - 'packages/*' - 'tooling/*' - 'test/*' + - 'services/*' diff --git a/services/blob-local/.dockerignore b/services/blob-local/.dockerignore new file mode 100644 index 000000000..be61d7218 --- /dev/null +++ b/services/blob-local/.dockerignore @@ -0,0 +1,3 @@ +blob_fs +.DS_STORE +blob-local diff --git a/services/blob-local/Dockerfile b/services/blob-local/Dockerfile new file mode 100644 index 000000000..7a0cbf88e --- /dev/null +++ b/services/blob-local/Dockerfile @@ -0,0 +1,15 @@ +# syntax=docker/dockerfile:1 + +FROM golang:1.22 + +WORKDIR /app + +# pre-copy/cache go.mod for pre-downloading dependencies and only redownloading them in subsequent builds if they change +COPY go.mod go.sum ./ +RUN go mod download && go mod verify + +COPY ./ ./ +RUN CGO_ENABLED=0 GOOS=linux go build -o /vercel-blob-local + +EXPOSE 3001 +CMD ["/vercel-blob-local"] diff --git a/services/blob-local/go.mod b/services/blob-local/go.mod new file mode 100644 index 000000000..d7e62aa09 --- /dev/null +++ b/services/blob-local/go.mod @@ -0,0 +1,45 @@ +module vercel/blob-local + +go 1.22.0 + +require ( + github.com/dgraph-io/badger v1.6.2 + github.com/gin-gonic/gin v1.10.0 +) + +require ( + github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect + github.com/bytedance/sonic v1.11.6 // indirect + github.com/bytedance/sonic/loader v0.1.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cloudwego/base64x v0.1.4 // indirect + github.com/cloudwego/iasm v0.2.0 // indirect + github.com/dgraph-io/ristretto v0.1.1 // indirect + github.com/dustin/go-humanize v1.0.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.3 // indirect + github.com/gin-contrib/sse v0.1.0 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.20.0 // indirect + github.com/goccy/go-json v0.10.2 // indirect + github.com/golang/glog v1.0.0 // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/cpuid/v2 v2.2.7 // indirect + github.com/leodido/go-urn v1.4.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/pelletier/go-toml/v2 v2.2.2 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/ugorji/go/codec v1.2.12 // indirect + golang.org/x/arch v0.8.0 // indirect + golang.org/x/crypto v0.23.0 // indirect + golang.org/x/net v0.25.0 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect + golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect + google.golang.org/protobuf v1.34.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/services/blob-local/go.sum b/services/blob-local/go.sum new file mode 100644 index 000000000..e084f15c6 --- /dev/null +++ b/services/blob-local/go.sum @@ -0,0 +1,159 @@ +github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIoKjsnZuH8vjyaysT/ses3EvZeaV/1UkF2M= +github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0= +github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= +github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM= +github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= +github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= +github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= +github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/badger v1.6.2 h1:mNw0qs90GVgGGWylh0umH5iag1j6n/PeJtNvL6KY/x8= +github.com/dgraph-io/badger v1.6.2/go.mod h1:JW2yswe3V058sS0kZ2h/AXeDSqFjxnZcRrVH//y2UQE= +github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= +github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= +github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= +github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= +github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= +github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= +github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= +github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8= +github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= +github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= +github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= +github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= +github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= +github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= +github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= +golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc= +golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/services/blob-local/main.go b/services/blob-local/main.go new file mode 100644 index 000000000..5ac125857 --- /dev/null +++ b/services/blob-local/main.go @@ -0,0 +1,77 @@ +package main + +import ( + "fmt" + "os" + "path" + "vercel/blob-local/operation" + "vercel/blob-local/provider" + "vercel/blob-local/server" + + "github.com/gin-gonic/gin" +) + +func main() { + env := server.ENV() + if env != "dev" { + gin.SetMode(gin.ReleaseMode) + } + + r := gin.Default() + r.SetTrustedProxies(nil) + + dir := path.Join(os.TempDir(), "blob_fs") + + server.Debug("Using tmp dir: ", dir) + + fp := provider.NewFileProvider(dir) + mp := provider.NewMetadataProvider(dir, env) + defer mp.Close() + + op := provider.NewObjectProvider(fp, mp) + + put := operation.NewPut(op) + head := operation.NewHead(op) + del := operation.NewDel(op) + list := operation.NewList(op) + copy := operation.NewCopy(op) + + authorized := r.Group(operation.BasePath) + authorized.Use(server.AuthRequired()) + + authorized.PUT("/*path", func(c *gin.Context) { + copyRequest := c.Query("fromUrl") + + if copyRequest == "" { + put.Handle(c) + } else { + copy.Handle(c) + } + }) + + authorized.GET("/*path", func(c *gin.Context) { + + if c.Query("url") != "" { + head.Handle(c) + } else { + list.Handle(c) + } + }) + + authorized.POST("/delete", del.Handle) + + download := operation.NewDownload(op) + + r.GET("/public/*rest", download.Handle) + + r.GET("/", func(c *gin.Context) { + c.Redirect(302, "https://vercel.com/docs/storage/vercel-blob") + }) + + server.Log("starting server on ", server.BaseUrl) + + err := r.Run(":" + server.Port) + if err != nil { + panic(fmt.Errorf("Error running server: %w", err)) + } +} diff --git a/services/blob-local/operation/copy.go b/services/blob-local/operation/copy.go new file mode 100644 index 000000000..a95f7b153 --- /dev/null +++ b/services/blob-local/operation/copy.go @@ -0,0 +1,63 @@ +package operation + +import ( + "net/http" + "vercel/blob-local/provider" + "vercel/blob-local/server" + + "github.com/gin-gonic/gin" +) + +type Copy struct { + provider *provider.ObjectProvider +} + +type CopyOutput struct { + Url string `json:"url"` + DownloadUrl string `json:"downloadUrl"` + Pathname string `json:"pathname"` + ContentType string `json:"contentType"` + ContentDisposition string `json:"contentDisposition"` +} + +func NewCopy(p *provider.ObjectProvider) *Copy { + return &Copy{ + provider: p, + } +} + +func (p *Copy) Handle(c *gin.Context) { + + fromUrl := c.Query("fromUrl") + if fromUrl == "" { + c.String(http.StatusBadRequest, "Missing 'url' query parameter") + return + } + + pathname := FormatApiPath(c.Request.URL.Path) + + addRandomSuffix := c.Request.Header.Get("x-add-random-suffix") == "1" + contentType := server.CreateContentType(c.Request.Header.Get("x-content-type"), c, pathname) + + o, err := p.provider.Copy(StoreId(c), fromUrl, provider.CopyOptions{ + Pathname: pathname, + AddRandomSuffix: addRandomSuffix, + ContentType: contentType, + CacheControlMaxAge: c.Request.Header.Get("x-cache-control-max-age"), + }) + + if err != nil { + server.Debug("Copy error:", err) + + c.String(http.StatusNotFound, "From blob doesn't exist") + return + } + + c.JSON(200, CopyOutput{ + Url: o.Url, + DownloadUrl: o.DownloadUrl, + Pathname: o.Pathname, + ContentType: o.ContentType, + ContentDisposition: o.ContentDisposition, + }) +} diff --git a/services/blob-local/operation/del.go b/services/blob-local/operation/del.go new file mode 100644 index 000000000..e4926e7b4 --- /dev/null +++ b/services/blob-local/operation/del.go @@ -0,0 +1,39 @@ +package operation + +import ( + "net/http" + "vercel/blob-local/provider" + "vercel/blob-local/server" + + "github.com/gin-gonic/gin" +) + +type Del struct { + provider *provider.ObjectProvider +} + +type Body struct { + Urls []string `json:"urls"` +} + +func NewDel(p *provider.ObjectProvider) *Del { + return &Del{ + provider: p, + } +} + +func (p *Del) Handle(c *gin.Context) { + + var input Body + + if err := c.ShouldBindJSON(&input); err != nil { + server.Debug("Body error:", err) + + c.String(http.StatusBadRequest, "Wrong body") + return + } + + p.provider.Del(input.Urls...) + + c.JSON(200, gin.H{}) +} diff --git a/services/blob-local/operation/download.go b/services/blob-local/operation/download.go new file mode 100644 index 000000000..edf7f8efa --- /dev/null +++ b/services/blob-local/operation/download.go @@ -0,0 +1,68 @@ +package operation + +import ( + "net/http" + "strings" + "time" + "vercel/blob-local/provider" + "vercel/blob-local/server" + + "github.com/gin-gonic/gin" +) + +type Download struct { + provider *provider.ObjectProvider +} + +func NewDownload(p *provider.ObjectProvider) *Download { + return &Download{ + provider: p, + } +} + +func (d *Download) setContentDisposition(c *gin.Context, object *provider.Object) { + contentDisposition := "inline" + if object.ContentDisposition != "" { + contentDisposition = object.ContentDisposition + } + if c.Query("download") == "1" { + contentDisposition = strings.Replace(contentDisposition, "inline", "attachment", 1) + } + + c.Header("Content-Disposition", contentDisposition) +} + +func (d *Download) Handle(c *gin.Context) { + + publicUrl := server.BaseUrl + c.Request.URL.Path + + object, err := d.provider.Get(publicUrl) + if err != nil { + server.Debug("Download error:", err) + + c.String(http.StatusNotFound, "Blob not found") + return + } + + file, err := object.GetFile() + if err != nil { + server.Debug("Download error:", err) + + c.String(http.StatusNotFound, "Blob not found") + return + } + + defer file.Close() + + d.setContentDisposition(c, object) + + modtime, _ := time.Parse(time.RFC3339, object.UploadedAt) + + http.ServeContent( + c.Writer, + c.Request, + object.Pathname, + modtime, + file, + ) +} diff --git a/services/blob-local/operation/head.go b/services/blob-local/operation/head.go new file mode 100644 index 000000000..5508e6008 --- /dev/null +++ b/services/blob-local/operation/head.go @@ -0,0 +1,61 @@ +package operation + +import ( + "net/http" + "net/url" + "vercel/blob-local/provider" + "vercel/blob-local/server" + + "github.com/gin-gonic/gin" +) + +type Head struct { + provider *provider.ObjectProvider +} + +type HeadOutput struct { + Url string `json:"url"` + DownloadUrl string `json:"downloadUrl"` + Size int64 `json:"size"` + UploadedAt string `json:"uploadedAt"` + Pathname string `json:"pathname"` + ContentType string `json:"contentType"` + ContentDisposition string `json:"contentDisposition"` + CacheControl string `json:"cacheControl"` +} + +func NewHead(p *provider.ObjectProvider) *Head { + return &Head{ + provider: p, + } +} + +func (p *Head) Handle(c *gin.Context) { + + u := c.Query("url") + + _, err := url.Parse(u) + if err != nil { + c.AbortWithError(500, err) + } + + o, err := p.provider.Get(u) + + if err != nil { + server.Debug("GetObject error:", err) + + c.String(http.StatusNotFound, "Blob not found") + return + } + + c.JSON(200, HeadOutput{ + Url: o.Url, + DownloadUrl: o.DownloadUrl, + Size: o.Size, + UploadedAt: o.UploadedAt, + Pathname: o.Pathname, + ContentType: o.ContentType, + ContentDisposition: o.ContentDisposition, + CacheControl: o.CacheControl, + }) +} diff --git a/services/blob-local/operation/list.go b/services/blob-local/operation/list.go new file mode 100644 index 000000000..f20862cba --- /dev/null +++ b/services/blob-local/operation/list.go @@ -0,0 +1,75 @@ +package operation + +import ( + "net/http" + "vercel/blob-local/provider" + "vercel/blob-local/server" + + "github.com/gin-gonic/gin" +) + +type List struct { + provider *provider.ObjectProvider +} + +type ListBlobOutput struct { + Url string `json:"url"` + DownloadUrl string `json:"downloadUrl"` + Size int64 `json:"size"` + UploadedAt string `json:"uploadedAt"` + Pathname string `json:"pathname"` + ContentDisposition string `json:"contentDisposition"` + ContentType string `json:"contentType"` +} + +type ListOutput struct { + Blobs []ListBlobOutput `json:"blobs"` + HasMore bool `json:"hasMore"` + Cursor string `json:"cursor"` + Folders []string `json:"folders"` +} + +func NewList(p *provider.ObjectProvider) *List { + return &List{ + provider: p, + } +} + +func (p *List) Handle(c *gin.Context) { + + os, err := p.provider.List(StoreId(c)) + if err != nil { + server.Debug("List error:", err) + c.String(http.StatusInternalServerError, "Something went wrong") + return + } + + if os == nil { + c.JSON(200, ListOutput{ + Blobs: []ListBlobOutput{}, + HasMore: false, + }) + return + } + + blobs := make([]ListBlobOutput, len(os)) + for i := range os { + o := os[i] + + blobs[i] = ListBlobOutput{ + Url: o.Url, + DownloadUrl: o.DownloadUrl, + Pathname: o.Pathname, + ContentType: o.ContentType, + ContentDisposition: o.ContentDisposition, + Size: o.Size, + UploadedAt: o.UploadedAt, + } + } + + c.JSON(200, ListOutput{ + Blobs: blobs, + HasMore: false, + // TODO: pagination, folders + }) +} diff --git a/services/blob-local/operation/operation.go b/services/blob-local/operation/operation.go new file mode 100644 index 000000000..dcc0ebb80 --- /dev/null +++ b/services/blob-local/operation/operation.go @@ -0,0 +1,26 @@ +package operation + +import ( + "errors" + "strings" + "vercel/blob-local/server" + + "github.com/gin-gonic/gin" +) + +const ( + BasePath = "/api/" +) + +func FormatApiPath(pathname string) string { + return strings.Replace(pathname, BasePath, "", 1) +} + +func StoreId(c *gin.Context) string { + i := c.GetString(server.StoreId) + if i == "" { + panic(errors.New("missing storeId in context")) + } + + return i +} diff --git a/services/blob-local/operation/put.go b/services/blob-local/operation/put.go new file mode 100644 index 000000000..1a7032560 --- /dev/null +++ b/services/blob-local/operation/put.go @@ -0,0 +1,58 @@ +package operation + +import ( + "net/http" + "vercel/blob-local/provider" + "vercel/blob-local/server" + + "github.com/gin-gonic/gin" +) + +type Put struct { + provider *provider.ObjectProvider +} + +type PutOutput struct { + Url string `json:"url"` + DownloadUrl string `json:"downloadUrl"` + Pathname string `json:"pathname"` + ContentType string `json:"contentType"` + ContentDisposition string `json:"contentDisposition"` +} + +func NewPut(p *provider.ObjectProvider) *Put { + return &Put{ + provider: p, + } +} + +func (p *Put) Handle(c *gin.Context) { + + pathname := FormatApiPath(c.Request.URL.Path) + + addRandomSuffix := c.Request.Header.Get("x-add-random-suffix") == "1" + contentType := server.CreateContentType(c.Request.Header.Get("x-content-type"), c, pathname) + + o, err := p.provider.Put(StoreId(c), provider.PutOptions{ + Pathname: pathname, + Data: c.Request.Body, + AddRandomSuffix: addRandomSuffix, + ContentType: contentType, + CacheControlMaxAge: c.Request.Header.Get("x-cache-control-max-age"), + }) + + if err != nil { + server.Debug("Put error:", err) + + c.String(http.StatusInternalServerError, "Something went wrong") + return + } + + c.JSON(200, PutOutput{ + Url: o.Url, + DownloadUrl: o.DownloadUrl, + Pathname: o.Pathname, + ContentType: o.ContentType, + ContentDisposition: o.ContentDisposition, + }) +} diff --git a/services/blob-local/package.json b/services/blob-local/package.json new file mode 100644 index 000000000..a50569268 --- /dev/null +++ b/services/blob-local/package.json @@ -0,0 +1,21 @@ +{ + "name": "@vercel/blob-local", + "version": "0.0.1", + "private": true, + "description": "Server that emulates the Vercel Blob API for local development", + "homepage": "https://vercel.com/storage/blob", + "repository": { + "type": "git", + "url": "https://github.com/vercel/storage.git", + "directory": "services/blob-local" + }, + "license": "MIT", + "scripts": { + "dev": "ENV=dev wgo run main.go", + "docker:build": "docker build -t zeit/vercel-blob-local .", + "docker:run": "docker run -d -p 3001:3001 --name vercel-blob-local zeit/vercel-blob-local", + "docker:start": "docker start vercel-blob-local", + "docker:stop": "docker stop vercel-blob-local", + "start": "ENV=dev go run main.go" + } +} diff --git a/services/blob-local/provider/file.go b/services/blob-local/provider/file.go new file mode 100644 index 000000000..e131c0732 --- /dev/null +++ b/services/blob-local/provider/file.go @@ -0,0 +1,65 @@ +package provider + +import ( + "fmt" + "io" + "os" + "path" +) + +const ( + // read, write, and execute permissions + Permissions = 0755 +) + +type FileProvider struct { + dir string +} + +func NewFileProvider(dir string) *FileProvider { + + err := os.MkdirAll(dir, Permissions) + if err != nil { + panic(fmt.Errorf("Error creating temporary directory: %w", err)) + } + + return &FileProvider{dir} +} + +func (p *FileProvider) Put(store string, pathname string, data io.ReadCloser) (int64, string, error) { + relPath := path.Join(store, pathname) + pathname = path.Join(p.dir, relPath) + + err := os.MkdirAll(path.Dir(pathname), 0755) + if err != nil { + return 0, "", err + } + + outFile, err := os.Create(pathname) + if err != nil { + return 0, "", err + } + + defer outFile.Close() + size, err := io.Copy(outFile, data) + + return size, relPath, err +} + +func (p *FileProvider) Del(pathname string) error { + name := path.Join(p.dir, pathname) + + return os.Remove(name) +} + +func (p *FileProvider) Get(pathname string) (*os.File, error) { + name := path.Join(p.dir, pathname) + + file, err := os.Open(name) + + if err != nil { + return nil, err + } + + return file, nil +} diff --git a/services/blob-local/provider/metadata.go b/services/blob-local/provider/metadata.go new file mode 100644 index 000000000..c1ac13887 --- /dev/null +++ b/services/blob-local/provider/metadata.go @@ -0,0 +1,121 @@ +package provider + +import ( + "encoding/json" + "log" + + "github.com/dgraph-io/badger" +) + +type MetadataProvider struct { + db *badger.DB +} + +func NewMetadataProvider(dir string, mode string) *MetadataProvider { + os := badger.DefaultOptions(dir) + if mode != "dev" { + os = os.WithLogger(nil) + } + + db, err := badger.Open(os) + if err != nil { + log.Fatal(err) + } + + return &MetadataProvider{db} +} + +func (p *MetadataProvider) Put(key string, object Object) error { + err := p.db.Update(func(txn *badger.Txn) error { + + buf, err := json.Marshal(object) + if err != nil { + return err + } + + return txn.Set([]byte(key), buf) + }) + + return err +} + +func parseObject(i *badger.Item) (*Object, error) { + var object *Object + + buf, err := i.ValueCopy(nil) + if err != nil { + return nil, err + } + + err = json.Unmarshal(buf, &object) + + return object, err +} + +func (p *MetadataProvider) Get(key string) (*Object, error) { + var object *Object + err := p.db.View(func(txn *badger.Txn) error { + + val, err := txn.Get([]byte(key)) + if err != nil || val == nil { + return err + } + + o, err := parseObject(val) + if err != nil { + return err + } + + object = o + return nil + }) + + return object, err +} + +func (p *MetadataProvider) Del(key string) error { + err := p.db.Update(func(txn *badger.Txn) error { + + err := txn.Delete([]byte(key)) + if err != nil { + return err + } + + return nil + }) + + return err +} + +func (p *MetadataProvider) List(prefix string) ([]Object, error) { + + pre := []byte(prefix) + var out []Object + + err := p.db.View(func(txn *badger.Txn) error { + it := txn.NewIterator(badger.DefaultIteratorOptions) + defer it.Close() + + for it.Seek(pre); it.ValidForPrefix(pre); it.Next() { + item := it.Item() + if item == nil { + return nil + } + + o, err := parseObject(item) + if err != nil { + return err + } + + out = append(out, *o) + } + + return nil + }) + + return out, err +} + +func (p *MetadataProvider) Close() { + p.db.Close() +} diff --git a/services/blob-local/provider/object.go b/services/blob-local/provider/object.go new file mode 100644 index 000000000..a632e9a33 --- /dev/null +++ b/services/blob-local/provider/object.go @@ -0,0 +1,157 @@ +package provider + +import ( + "errors" + "io" + "os" + "time" + "vercel/blob-local/server" +) + +type Object struct { + Url string + DownloadUrl string + Size int64 + UploadedAt string + Pathname string + ContentType string + ContentDisposition string + CacheControl string + + FilePath string + fp *FileProvider +} + +func (o *Object) GetFile() (*os.File, error) { + file, err := o.fp.Get(o.FilePath) + if err != nil { + return nil, errors.New("GetData Error: " + err.Error()) + } + + return file, nil +} + +type ObjectProvider struct { + fp *FileProvider + mp *MetadataProvider +} + +func NewObjectProvider(fp *FileProvider, mp *MetadataProvider) *ObjectProvider { + return &ObjectProvider{fp, mp} +} + +type PutOptions struct { + Pathname string + Data io.ReadCloser + AddRandomSuffix bool + ContentType string + CacheControlMaxAge string +} + +func (p *ObjectProvider) Put(store string, options PutOptions) (*Object, error) { + pathname := options.Pathname + if options.AddRandomSuffix { + pathname = AddRandomSuffix(pathname) + } + + size, filepath, err := p.fp.Put(store, pathname, options.Data) + if err != nil { + return nil, err + } + + url := server.CreatePublicUrl(store, pathname) + + o := Object{ + // path without random suffix + Pathname: options.Pathname, + Size: size, + Url: url, + DownloadUrl: server.CreatePublicDownloadUrl(store, pathname), + // format as ISO 8601 + UploadedAt: time.Now().Format(time.RFC3339), + ContentType: options.ContentType, + ContentDisposition: server.CreateContentDisposition(options.Pathname), + CacheControl: server.CreateCacheControl(options.CacheControlMaxAge), + + FilePath: filepath, + fp: p.fp, + } + + err = p.mp.Put(url, o) + if err != nil { + return nil, err + } + + return &o, nil +} + +func (p *ObjectProvider) Get(url string) (*Object, error) { + + o, err := p.mp.Get(url) + if err != nil { + return nil, errors.New("GetObject Error: " + err.Error()) + } + + return o, nil +} + +func (p *ObjectProvider) Del(urls ...string) { + for _, url := range urls { + + o, err := p.mp.Get(url) + if err != nil { + continue + } + + err = p.mp.Del(url) + if err != nil { + continue + } + + p.fp.Del(o.FilePath) + } +} + +func (p ObjectProvider) List(store string) ([]Object, error) { + prefix := server.CreatePublicUrl(store, "") + + o, err := p.mp.List(prefix) + if err != nil { + return nil, err + } + + return o, nil +} + +type CopyOptions struct { + Pathname string + AddRandomSuffix bool + ContentType string + CacheControlMaxAge string +} + +func (p ObjectProvider) Copy(store string, from string, options CopyOptions) (*Object, error) { + fromO, err := p.mp.Get(from) + if err != nil || fromO == nil { + return nil, err + } + + file, err := p.fp.Get(fromO.FilePath) + if err != nil || file == nil { + return nil, err + } + + o, err := p.Put(store, PutOptions{ + Pathname: options.Pathname, + Data: file, + AddRandomSuffix: options.AddRandomSuffix, + ContentType: options.ContentType, + CacheControlMaxAge: options.CacheControlMaxAge, + }) + + if err != nil { + return nil, err + } + + return o, nil +} diff --git a/services/blob-local/provider/random.go b/services/blob-local/provider/random.go new file mode 100644 index 000000000..6fe795b66 --- /dev/null +++ b/services/blob-local/provider/random.go @@ -0,0 +1,28 @@ +package provider + +import ( + "math/rand" + "path" + "strings" + "time" +) + +const charset = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + +func generateRandomString(n int) string { + seed := rand.NewSource(time.Now().UnixNano()) + r := rand.New(seed) + b := make([]byte, n) + for i := range b { + b[i] = charset[r.Intn(len(charset))] + } + return string(b) +} + +func AddRandomSuffix(pathname string) string { + ext := path.Ext(pathname) + + suffix := generateRandomString(30) + + return strings.Replace(pathname, ext, "-"+suffix+ext, 1) +} diff --git a/services/blob-local/readme.md b/services/blob-local/readme.md new file mode 100644 index 000000000..31b58a441 --- /dev/null +++ b/services/blob-local/readme.md @@ -0,0 +1,70 @@ +# Blob-Local + +Blob-Local is a server for `@vercel/blob` which writes to the local filesystem instead of the cloud. It allows you to test and develop your applications locally without needing access to the hosted `@vercel/blob` store. + +## Installation + +First of all you should add these Environment Variables to you project so that the `@vercel/blob` SDK talks to your local server: + +```bash +NEXT_PUBLIC_VERCEL_BLOB_API_URL=http://localhost:3001/api +VERCEL_BLOB_API_URL=http://localhost:3001/api +``` + +### Using Docker + +You need to have [Docker](https://www.docker.com/) installed on your machine. + +```bash +docker run -d -p 3001:3001 --name vercel-blob-local zeit/vercel-blob-local +``` + +After this you can start and stop the `vercel-blob-local` container like this: + +```bash +docker start vercel-blob-local +docker stop vercel-blob-local +``` + +## Development + +### With Docker + +You need to have [Docker](https://www.docker.com/) installed on your machine. + +Build and run the container: + +```bash +pnpm --filter @vercel/blob-local docker:build +pnpm --filter @vercel/blob-local docker:run +``` + +After this you can start and stop the `vercel-blob-local` container like this: + +```bash +docker start vercel-blob-local +docker stop vercel-blob-local +``` + +or using the pnpm scripts: + +```bash +pnpm --filter @vercel/blob-local docker:start +pnpm --filter @vercel/blob-local docker:stop +``` + +### With Go + +You need to install [Go](https://go.dev/doc/install) and [WGO](https://github.com/bokwoon95/wgo). + +Start the server: + +```bash +pnpm --filter @vercel/blob-local dev +``` + +If you want a watcher that restarts the server on file changes you can use something like [wgo](https://github.com/bokwoon95/wgo) + +```bash +ENV=dev wgo run main.go +``` diff --git a/services/blob-local/server/auth.go b/services/blob-local/server/auth.go new file mode 100644 index 000000000..f99da8cf1 --- /dev/null +++ b/services/blob-local/server/auth.go @@ -0,0 +1,34 @@ +package server + +import ( + "net/http" + "strings" + + "github.com/gin-gonic/gin" +) + +const ( + StoreId = "storeId" +) + +func AuthRequired() gin.HandlerFunc { + return func(c *gin.Context) { + h := c.Request.Header["Authorization"] + if len(h) == 0 { + c.String(http.StatusUnauthorized, "Authorization header is required") + return + } + + t := strings.ReplaceAll(h[0], "Bearer ", "") + p := strings.Split(t, "_") + + if len(p) < 3 { + c.String(http.StatusUnauthorized, "Mailformed Authorization header") + return + } + + c.Set(StoreId, p[3]) + + c.Next() + } +} diff --git a/services/blob-local/server/env.go b/services/blob-local/server/env.go new file mode 100644 index 000000000..b203b5acb --- /dev/null +++ b/services/blob-local/server/env.go @@ -0,0 +1,7 @@ +package server + +import "os" + +func ENV() string { + return os.Getenv("ENV") +} diff --git a/services/blob-local/server/headers.go b/services/blob-local/server/headers.go new file mode 100644 index 000000000..1dda37873 --- /dev/null +++ b/services/blob-local/server/headers.go @@ -0,0 +1,46 @@ +package server + +import ( + "fmt" + "mime" + "path" + "strconv" + + "github.com/gin-gonic/gin" +) + +func CreateContentDisposition(pathname string) string { + filename := path.Base(pathname) + + return "inline; filename=\"" + filename + "\"" +} + +const oneYearInSeconds = 365 * 24 * 60 * 60 +const fiveMinutesInSeconds = 5 * 60 + +func CreateCacheControl(maxAge string) string { + num, err := strconv.Atoi(maxAge) + if err != nil { + num = oneYearInSeconds + } + + edge := min(num, fiveMinutesInSeconds) + + return fmt.Sprintf("public, max-age=%d,s-maxage=%d", num, edge) +} + +func CreateContentType(contentType string, c *gin.Context, pathname string) string { + if contentType == "" { + contentType, _, _ = mime.ParseMediaType(c.Request.Header.Get("content-type")) + } + + if contentType == "" { + contentType = mime.TypeByExtension(path.Ext(pathname)) + } + + if contentType == "" { + contentType = "application/octet-stream" + } + + return contentType +} diff --git a/services/blob-local/server/logger.go b/services/blob-local/server/logger.go new file mode 100644 index 000000000..00e42fece --- /dev/null +++ b/services/blob-local/server/logger.go @@ -0,0 +1,21 @@ +package server + +import ( + "fmt" +) + +func Log(a ...any) { + fmt.Print("@vercel/blob: ") + fmt.Print(a...) + fmt.Print("\n") +} + +func Debug(a ...any) { + if ENV() != "dev" { + return + } + + fmt.Print("@vercel/blob: ") + fmt.Print(a...) + fmt.Print("\n") +} diff --git a/services/blob-local/server/public.go b/services/blob-local/server/public.go new file mode 100644 index 000000000..9bf8f16b1 --- /dev/null +++ b/services/blob-local/server/public.go @@ -0,0 +1,21 @@ +package server + +import ( + "path" +) + +const ( + Port = "3001" + PublicBasePath = "/public" + Scheme = "http" + Host = "localhost" + BaseUrl = Scheme + "://" + Host + ":" + Port +) + +func CreatePublicUrl(store string, pathname string) string { + return BaseUrl + path.Join(PublicBasePath, store, pathname) +} + +func CreatePublicDownloadUrl(store string, pathname string) string { + return CreatePublicUrl(store, pathname) + "?download=1" +}