support s2 compression based snapshots

Signed-off-by: R.I.Pienaar <[email protected]>
nats-io · Jan 18, 2021 · 3742f7f · 3742f7f
1 parent f01998f
commit 3742f7f
Show file tree

Hide file tree

Showing 4 changed files with 22 additions and 18 deletions.
diff --git a/dependencies.md b/dependencies.md
@@ -8,3 +8,4 @@ This file lists the dependencies used in this repository.
 | github.com/dustin/go-humanize | MIT License |
 | github.com/nats-io/nats-server | Apache License 2.0 |
 | github.com/nats-io/nats.go | Apache License 2.0 |
+| github.com/klauspost/compress | BSD 3-Clause "New" or "Revised" License |
diff --git a/go.mod b/go.mod
@@ -4,6 +4,7 @@ go 1.14
 
 require (
 	github.com/dustin/go-humanize v1.0.0
-	github.com/nats-io/nats-server/v2 v2.1.8-0.20201204171240-e1b590db604e
-	github.com/nats-io/nats.go v1.10.1-0.20201111151633-9e1f4a0d80d8
+	github.com/klauspost/compress v1.11.7
+	github.com/nats-io/nats-server/v2 v2.1.8-0.20210117220321-51a321e73932
+	github.com/nats-io/nats.go v1.10.1-0.20210115180731-7fb8bacca613
 )
diff --git a/go.sum b/go.sum
@@ -11,22 +11,29 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/klauspost/compress v1.11.4 h1:kz40R/YWls3iqT9zX9AHN3WoVsrAWVyui5sxuLqiXqU=
+github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
+github.com/klauspost/compress v1.11.7 h1:0hzRabrMN4tSTvMfnL3SCv1ZGeAP23ynzodBgaHeMeg=
+github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/minio/highwayhash v1.0.0 h1:iMSDhgUILCr0TNm8LWlSjF8N0ZIj2qbO8WHp6Q/J2BA=
 github.com/minio/highwayhash v1.0.0/go.mod h1:xQboMTeM9nY9v/LlAOxFctujiv5+Aq2hR5dxBpaMbdc=
 github.com/nats-io/jwt v0.3.2 h1:+RB5hMpXUUA2dfxuhBTEkMOrYmM+gKIZYS1KjSostMI=
 github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU=
 github.com/nats-io/jwt v0.3.3-0.20200519195258-f2bf5ce574c7 h1:RnGotxlghqR5D2KDAu4TyuLqyjuylOsJiAFhXvMvQIc=
 github.com/nats-io/jwt v0.3.3-0.20200519195258-f2bf5ce574c7/go.mod h1:n3cvmLfBfnpV4JJRN7lRYCyZnw48ksGsbThGXEk4w9M=
+github.com/nats-io/jwt v1.1.0 h1:+vOlgtM0ZsF46GbmUoadq0/2rChNS45gtxHEa3H1gqM=
+github.com/nats-io/jwt v1.1.0/go.mod h1:n3cvmLfBfnpV4JJRN7lRYCyZnw48ksGsbThGXEk4w9M=
 github.com/nats-io/jwt/v2 v2.0.0-20200916203241-1f8ce17dff02/go.mod h1:vs+ZEjP+XKy8szkBmQwCB7RjYdIlMaPsFPs4VdS4bTQ=
 github.com/nats-io/jwt/v2 v2.0.0-20201015190852-e11ce317263c h1:Hc1D9ChlsCMVwCxJ6QT5xqfk2zJ4XNea+LtdfaYhd20=
 github.com/nats-io/jwt/v2 v2.0.0-20201015190852-e11ce317263c/go.mod h1:vs+ZEjP+XKy8szkBmQwCB7RjYdIlMaPsFPs4VdS4bTQ=
+github.com/nats-io/jwt/v2 v2.0.0-20210107222814-18c5cc45d263 h1:M2bkT/arzYFYPtRqQWN2m+LSLfcqFmPxlxvTxdB4aVE=
+github.com/nats-io/jwt/v2 v2.0.0-20210107222814-18c5cc45d263/go.mod h1:PuO5FToRL31ecdFqVjc794vK0Bj0CwzveQEDvkb7MoQ=
 github.com/nats-io/nats-server/v2 v2.1.8-0.20200524125952-51ebd92a9093/go.mod h1:rQnBf2Rv4P9adtAs/Ti6LfFmVtFG6HLhl/H7cVshcJU=
 github.com/nats-io/nats-server/v2 v2.1.8-0.20200601203034-f8d6dd992b71/go.mod h1:Nan/1L5Sa1JRW+Thm4HNYcIDcVRFc5zK9OpSZeI2kk4=
 github.com/nats-io/nats-server/v2 v2.1.8-0.20200929001935-7f44d075f7ad/go.mod h1:TkHpUIDETmTI7mrHN40D1pzxfzHZuGmtMbtb83TGVQw=
-github.com/nats-io/nats-server/v2 v2.1.8-0.20201103213111-0965a20b516d h1:zNXqQAfBphOwTdcjnd+6wM4dmMKpW5u3a9vyBhtcEIc=
-github.com/nats-io/nats-server/v2 v2.1.8-0.20201103213111-0965a20b516d/go.mod h1:XD0zHR/jTXdZvWaQfS5mQgsXj6x12kMjKLyAk/cOGgY=
-github.com/nats-io/nats-server/v2 v2.1.8-0.20201204171240-e1b590db604e h1:fnpbBmRJVwTKpHOK5mDVbEfTBaYTTZSbAx/HIfPhYo4=
-github.com/nats-io/nats-server/v2 v2.1.8-0.20201204171240-e1b590db604e/go.mod h1:XD0zHR/jTXdZvWaQfS5mQgsXj6x12kMjKLyAk/cOGgY=
+github.com/nats-io/nats-server/v2 v2.1.8-0.20201129161730-ebe63db3e3ed/go.mod h1:XD0zHR/jTXdZvWaQfS5mQgsXj6x12kMjKLyAk/cOGgY=
+github.com/nats-io/nats-server/v2 v2.1.8-0.20210117220321-51a321e73932 h1:HUxuK5DohVIxv2JvndCjMhOddDDB3Z9bz4jT3nwnEx0=
+github.com/nats-io/nats-server/v2 v2.1.8-0.20210117220321-51a321e73932/go.mod h1:FA26zYxU8tj8QcKb0Ize7HaYub6zgI18BPd/yX7zoUY=
 github.com/nats-io/nats.go v1.10.0 h1:L8qnKaofSfNFbXg0C5F71LdjPRnmQwSsA4ukmkt1TvY=
 github.com/nats-io/nats.go v1.10.0/go.mod h1:AjGArbfyR50+afOUotNX2Xs5SYHf+CoOa5HH1eEl2HE=
 github.com/nats-io/nats.go v1.10.1-0.20200531124210-96f2130e4d55 h1:AaYp5amXO8fkWw6ZUEjGFiWdj8FfjOGDquae7Ne5JOU=
@@ -35,8 +42,8 @@ github.com/nats-io/nats.go v1.10.1-0.20200606002146-fc6fed82929a h1:gzSKZOBlu/Dp
 github.com/nats-io/nats.go v1.10.1-0.20200606002146-fc6fed82929a/go.mod h1:8eAIv96Mo9QW6Or40jUHejS7e4VwZ3VRYD6Sf0BTDp4=
 github.com/nats-io/nats.go v1.10.1-0.20201021145452-94be476ad6e0 h1:s//xxvMgNRVJNTRXTRNvLlES0zvJoDLXfcmbK0YgN0E=
 github.com/nats-io/nats.go v1.10.1-0.20201021145452-94be476ad6e0/go.mod h1:VU2zERjp8xmF+Lw2NH4u2t5qWZxwc7jB3+7HVMWQXPI=
-github.com/nats-io/nats.go v1.10.1-0.20201111151633-9e1f4a0d80d8 h1:OCzS7FpkXX7pyWRzIkXZVWvzGleUKbG0E0YGs1IWXMQ=
-github.com/nats-io/nats.go v1.10.1-0.20201111151633-9e1f4a0d80d8/go.mod h1:hHejHU2mytFORoW+P6jfwfMHh8Y1HLEX3o2KLIpIMYk=
+github.com/nats-io/nats.go v1.10.1-0.20210115180731-7fb8bacca613 h1:Zio4IMHHsFjtTeksjF4PySxFNcKwSH9urNIiIW7A/FQ=
+github.com/nats-io/nats.go v1.10.1-0.20210115180731-7fb8bacca613/go.mod h1:Sa3kLIonafChP5IF0b55i9uvGR10I3hPETFbi4+9kOI=
 github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w=
 github.com/nats-io/nkeys v0.1.4 h1:aEsHIssIk6ETN5m2/MD8Y4B2X7FfXrBAUdkyRvbVYzA=
 github.com/nats-io/nkeys v0.1.4/go.mod h1:XdZpAbhgyyODYqjTawOnIOI7VlbKSarI9Gfy1tqEu/s=
@@ -56,6 +63,8 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7 h1:HmbHVPwrPEKPGLAcHSrMe6+hqSUlvZU0rab6x5EXfGU=
 golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1 h1:NusfzzA6yGQ+ua51ck7E3omNUX/JuqbFSaRGqU8CcLI=
 golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=

diff --git a/snapshots.go b/snapshots.go
@@ -15,7 +15,6 @@ package jsm
 
 import (
 	"archive/tar"
-	"compress/gzip"
 	"context"
 	"fmt"
 	"io"
@@ -27,6 +26,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/klauspost/compress/s2"
 	"github.com/nats-io/nats.go"
 
 	"github.com/nats-io/jsm.go/api"
@@ -237,14 +237,7 @@ func (sp *snapshotProgress) trackBlockProgress(r io.Reader, debug bool, errc cha
 	seenMetaSum := false
 	seenMetaInf := false
 
-	zr, err := gzip.NewReader(r)
-	if err != nil {
-		errc <- fmt.Errorf("progress tracker failed to start gzip: %s", err)
-		return
-	}
-	defer zr.Close()
-
-	tr := tar.NewReader(zr)
+	tr := tar.NewReader(s2.NewReader(r))
 
 	for {
 		hdr, err := tr.Next()
@@ -443,7 +436,7 @@ func (m *Manager) RestoreSnapshotFromFile(ctx context.Context, stream string, fi
 	return progress, &createResp.State, nil
 }
 
-// SnapshotToFile creates a backup into gzipped tar file
+// SnapshotToFile creates a backup into s2 compressed tar file
 func (s *Stream) SnapshotToFile(ctx context.Context, file string, opts ...SnapshotOption) (SnapshotProgress, error) {
 	sopts := &snapshotOptions{
 		file:      file,