support TLS configuration

* extend cluster config: add _tls knobs * bump config meta-version to 3 (was 2) * part three Signed-off-by: Alex Aizman <[email protected]>
NVIDIA · Oct 24, 2023 · 6a0fc5e · 6a0fc5e
1 parent b887530
commit 6a0fc5e
Show file tree

Hide file tree

Showing 17 changed files with 77 additions and 41 deletions.
diff --git a/ais/backend/http.go b/ais/backend/http.go
@@ -40,7 +40,7 @@ func NewHTTP(t cluster.TargetPut, config *cmn.Config) cluster.BackendProvider {
 			ReadBufferSize:  config.Net.HTTP.ReadBufferSize,
 		}
 		sargs = cmn.TLSArgs{
-			SkipVerify: config.Net.HTTP.SkipVerify,
+			SkipVerify: config.Net.HTTP.SkipVerifyTLS,
 		}
 	)
 	hp.httpClient = cmn.NewClient(cargs)

diff --git a/ais/htrun.go b/ais/htrun.go
@@ -306,7 +306,7 @@ func (h *htrun) initCtrlClient(config *cmn.Config) {
 			UseHTTPS:        config.Net.HTTP.UseHTTPS,
 		}
 		sargs = cmn.TLSArgs{
-			SkipVerify: config.Net.HTTP.SkipVerify,
+			SkipVerify: config.Net.HTTP.SkipVerifyTLS,
 		}
 	)
 	if config.Net.HTTP.UseHTTPS {
@@ -333,7 +333,7 @@ func (h *htrun) initDataClient(config *cmn.Config) {
 			UseHTTPS:        config.Net.HTTP.UseHTTPS,
 		}
 		sargs = cmn.TLSArgs{
-			SkipVerify: config.Net.HTTP.SkipVerify,
+			SkipVerify: config.Net.HTTP.SkipVerifyTLS,
 		}
 	)
 	if config.Net.HTTP.UseHTTPS {

diff --git a/ais/proxy.go b/ais/proxy.go
@@ -2054,7 +2054,7 @@ func newRevProxyTransport(config *cmn.Config) *http.Transport {
 		transport = cmn.NewTransport(cmn.TransportArgs{UseHTTPS: config.Net.HTTP.UseHTTPS})
 	)
 	if config.Net.HTTP.UseHTTPS {
-		transport.TLSClientConfig, err = cmn.NewTLS(cmn.TLSArgs{SkipVerify: config.Net.HTTP.SkipVerify})
+		transport.TLSClientConfig, err = cmn.NewTLS(cmn.TLSArgs{SkipVerify: config.Net.HTTP.SkipVerifyTLS})
 		cos.AssertNoErr(err)
 	}
 	return transport

diff --git a/api/env/ais.go b/api/env/ais.go
@@ -11,6 +11,9 @@ var (
 		PrimaryID     string
 		SkipVerifyCrt string
 		UseHTTPS      string
+		DomainTLS     string
+		ClientCA      string
+		ClientAuthTLS string
 		NumTarget     string
 		NumProxy      string
 		K8sPod        string
@@ -20,6 +23,9 @@ var (
 		PrimaryID:     "AIS_PRIMARY_ID",
 		SkipVerifyCrt: "AIS_SKIP_VERIFY_CRT",
 		UseHTTPS:      "AIS_USE_HTTPS",
+		DomainTLS:     "AIS_DOMAIN_TLS",
+		ClientCA:      "AIS_CLIENT_CA_TLS",
+		ClientAuthTLS: "AIS_CLIENT_AUTH_TLS",
 
 		// Env variables used for tests or CI
 		NumTarget: "NUM_TARGET",

diff --git a/cmd/cli/cli/init.go b/cmd/cli/cli/init.go
@@ -36,20 +36,30 @@ func Init() (err error) {
 	// http transport and clients: the main one and auth, if enabled
 	//
 	clusterURL = _clusterURL(cfg)
-	defaultHTTPClient = cmn.NewClient(cmn.TransportArgs{
+
+	useHTTPS := cos.IsHTTPS(clusterURL)
+	skipVerify := cfg.Cluster.SkipVerifyCrt
+	if useHTTPS {
+		if s := os.Getenv(env.AIS.SkipVerifyCrt); s != "" {
+			skipVerify = cos.IsParseBool(s)
+		}
+	}
+
+	cargs := cmn.TransportArgs{
 		DialTimeout: cfg.Timeout.TCPTimeout,
 		Timeout:     cfg.Timeout.HTTPTimeout,
-		UseHTTPS:    cos.IsHTTPS(clusterURL),
-		SkipVerify:  cfg.Cluster.SkipVerifyCrt,
-	})
+		UseHTTPS:    useHTTPS,
+	}
+	if useHTTPS {
+		sargs := cmn.TLSArgs{SkipVerify: skipVerify}
+		defaultHTTPClient = cmn.NewClientTLS(cargs, sargs)
+	} else {
+		defaultHTTPClient = cmn.NewClient(cargs)
+	}
 
 	if authnURL := cliAuthnURL(cfg); authnURL != "" {
-		authnHTTPClient = cmn.NewClient(cmn.TransportArgs{
-			DialTimeout: cfg.Timeout.TCPTimeout,
-			Timeout:     cfg.Timeout.HTTPTimeout,
-			UseHTTPS:    cos.IsHTTPS(authnURL),
-			SkipVerify:  cfg.Cluster.SkipVerifyCrt,
-		})
+		debug.Assert(useHTTPS == cos.IsHTTPS(authnURL))
+		authnHTTPClient = defaultHTTPClient
 		authParams = api.BaseParams{
 			Client: authnHTTPClient,
 			URL:    authnURL,

diff --git a/cmd/cli/go.mod b/cmd/cli/go.mod
@@ -4,7 +4,7 @@ go 1.21
 
 // direct
 require (
-	github.com/NVIDIA/aistore v1.3.21-0.20231020160354-468c3004def8
+	github.com/NVIDIA/aistore v1.3.21-0.20231024170343-fab6dc20181c
 	github.com/fatih/color v1.15.0
 	github.com/json-iterator/go v1.1.12
 	github.com/onsi/ginkgo v1.16.5

diff --git a/cmd/cli/go.sum b/cmd/cli/go.sum
@@ -1,7 +1,7 @@
 code.cloudfoundry.org/bytefmt v0.0.0-20190710193110-1eb035ffe2b6/go.mod h1:wN/zk7mhREp/oviagqUXY3EwuHhWyOvAdsn5Y4CzOrc=
 github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
-github.com/NVIDIA/aistore v1.3.21-0.20231020160354-468c3004def8 h1:PyRwhNw44rc4qTZoJWsr0/a3pm3skTD3ez2AEIvGCQ0=
-github.com/NVIDIA/aistore v1.3.21-0.20231020160354-468c3004def8/go.mod h1:+iSnZg0ovMaLgaT9fLAs2WmYBP7IfeTW1WYkbKrwP4g=
+github.com/NVIDIA/aistore v1.3.21-0.20231024170343-fab6dc20181c h1:w6p12hr68uTfkS2tzagStTmjiKLoLTIvPCQLj53EGyA=
+github.com/NVIDIA/aistore v1.3.21-0.20231024170343-fab6dc20181c/go.mod h1:+iSnZg0ovMaLgaT9fLAs2WmYBP7IfeTW1WYkbKrwP4g=
 github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8=
 github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q=
 github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=

diff --git a/cmn/config.go b/cmn/config.go
@@ -397,22 +397,28 @@ type (
 
 	HTTPConf struct {
 		Proto           string `json:"-"`                 // http or https (set depending on `UseHTTPS`)
-		Certificate     string `json:"server_crt"`        // HTTPS: openssl certificate
-		Key             string `json:"server_key"`        // HTTPS: openssl key
+		Certificate     string `json:"server_crt"`        // HTTPS: X509 certificate
+		Key             string `json:"server_key"`        // HTTPS: X509 key
+		ServerNameTLS   string `json:"domain_tls"`        // #6410
+		ClientCA        string `json:"client_ca_tls"`     // #6410
+		ClientAuthTLS   int    `json:"client_auth_tls"`   // #6410
 		WriteBufferSize int    `json:"write_buffer_size"` // http.Transport.WriteBufferSize; zero defaults to 4KB
 		ReadBufferSize  int    `json:"read_buffer_size"`  // http.Transport.ReadBufferSize; ditto
-		UseHTTPS        bool   `json:"use_https"`         // use HTTPS instead of HTTP
-		SkipVerify      bool   `json:"skip_verify"`       // skip HTTPS cert verification (used with self-signed certs)
-		Chunked         bool   `json:"chunked_transfer"`  // NOTE: not used Feb 2023
+		UseHTTPS        bool   `json:"use_https"`         // use HTTPS
+		SkipVerifyTLS   bool   `json:"skip_verify"`       // skip X509 cert verification (used with self-signed certs)
+		Chunked         bool   `json:"chunked_transfer"`  // (https://tools.ietf.org/html/rfc7230#page-36; not used since 02/23)
 	}
 	HTTPConfToSet struct {
 		Certificate     *string `json:"server_crt,omitempty"`
 		Key             *string `json:"server_key,omitempty"`
+		ServerNameTLS   *string `json:"domain_tls,omitempty"`
+		ClientCA        *string `json:"client_ca_tls,omitempty"`
 		WriteBufferSize *int    `json:"write_buffer_size,omitempty" list:"readonly"`
 		ReadBufferSize  *int    `json:"read_buffer_size,omitempty" list:"readonly"`
+		ClientAuthTLS   *int    `json:"client_auth_tls,omitempty"`
 		UseHTTPS        *bool   `json:"use_https,omitempty"`
-		SkipVerify      *bool   `json:"skip_verify,omitempty"`
-		Chunked         *bool   `json:"chunked_transfer,omitempty"` // https://tools.ietf.org/html/rfc7230#page-36
+		SkipVerifyTLS   *bool   `json:"skip_verify,omitempty"`
+		Chunked         *bool   `json:"chunked_transfer,omitempty"`
 	}
 
 	FSHCConf struct {
@@ -573,11 +579,15 @@ var (
 	_ jsp.Opts = (*ConfigToSet)(nil)
 )
 
-var configJspOpts = jsp.CCSign(MetaverConfig)
+func _jspOpts() jsp.Options {
+	opts := jsp.CCSign(MetaverConfig)
+	opts.OldMetaverOk = 2
+	return opts
+}
 
-func (*ClusterConfig) JspOpts() jsp.Options { return configJspOpts }
 func (*LocalConfig) JspOpts() jsp.Options   { return jsp.Plain() }
-func (*ConfigToSet) JspOpts() jsp.Options   { return configJspOpts }
+func (*ClusterConfig) JspOpts() jsp.Options { return _jspOpts() }
+func (*ConfigToSet) JspOpts() jsp.Options   { return _jspOpts() }
 
 // interface guard
 var (

diff --git a/cmn/jsp/io.go b/cmn/jsp/io.go
@@ -13,6 +13,7 @@ import (
 
 	"github.com/NVIDIA/aistore/cmn/cos"
 	"github.com/NVIDIA/aistore/cmn/debug"
+	"github.com/NVIDIA/aistore/cmn/nlog"
 	"github.com/OneOfOne/xxhash"
 	jsoniter "github.com/json-iterator/go"
 	"github.com/pierrec/lz4/v3"
@@ -116,9 +117,13 @@ func Decode(reader io.ReadCloser, v any, opts Options, tag string) (checksum *co
 		}
 		metaVer = binary.BigEndian.Uint32(prefix[cos.SizeofI64:])
 		if metaVer != opts.Metaver {
-			// NOTE: potential backward compatibility case for the caller
-			err = newErrVersion(tag, metaVer, opts.Metaver)
-			return
+			if opts.OldMetaverOk == 0 || metaVer > opts.Metaver || metaVer < opts.OldMetaverOk {
+				// not backward compatible
+				err = newErrVersion(tag, metaVer, opts.Metaver)
+				return
+			}
+			erw := newErrVersion(tag, metaVer, opts.Metaver, opts.OldMetaverOk)
+			nlog.Warningln(erw, "- proceeding anyway") // nlog depth 3
 		}
 		flags := binary.BigEndian.Uint32(prefix[cos.SizeofI64+cos.SizeofI32:])
 		opts.Compress = flags&(1<<0) != 0

diff --git a/cmn/jsp/opts.go b/cmn/jsp/opts.go
@@ -1,7 +1,7 @@
 // Package jsp (JSON persistence) provides utilities to store and load arbitrary
 // JSON-encoded structures with optional checksumming and compression.
 /*
- * Copyright (c) 2018-2021, NVIDIA CORPORATION. All rights reserved.
+ * Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.
  */
 package jsp
 
@@ -10,6 +10,8 @@ type (
 		// when non-zero, formatting version of the structure that's being (de)serialized
 		// (not to confuse with the jsp encoding version - see above)
 		Metaver uint32
+		// warn and keep loading
+		OldMetaverOk uint32
 
 		Compress  bool // lz4 when [version == 1 || version == 2]
 		Checksum  bool // xxhash when [version == 1 || version == 2]

diff --git a/cmn/ver_const.go b/cmn/ver_const.go
@@ -24,9 +24,9 @@ import "github.com/NVIDIA/aistore/cmn/jsp"
 //   `jsp` formats its *signature* and other implementation details.
 
 const (
-	VersionAIStore = "3.20"
-	VersionCLI     = "1.6.7"
-	VersionLoader  = "1.8"
+	VersionAIStore = "3.21.rc1"
+	VersionCLI     = "1.7"
+	VersionLoader  = "1.9"
 	VersionAuthN   = "1.0"
 )
 
@@ -39,7 +39,7 @@ const (
 
 	MetaverLOM = 1 // LOM
 
-	MetaverConfig      = 2 // Global Configuration (jsp)
+	MetaverConfig      = 3 // Global Configuration (jsp)
 	MetaverAuthNConfig = 1 // Authn config (jsp) // ditto
 	MetaverAuthTokens  = 1 // Authn tokens (jsp) // ditto
 

diff --git a/deploy/dev/local/aisnode_config.sh b/deploy/dev/local/aisnode_config.sh
@@ -122,6 +122,9 @@ cat > $AIS_CONF_FILE <<EOL
 			"use_https":         ${AIS_USE_HTTPS:-false},
 			"server_crt":        "${AIS_SERVER_CRT:-server.crt}",
 			"server_key":        "${AIS_SERVER_KEY:-server.key}",
+			"domain_tls":        "${AIS_DOMAIN_TLS:-localhost}",
+			"client_ca_tls":     "${AIS_CLIENT_CA_TLS}",
+			"client_auth_tls":   ${AIS_CLIENT_AUTH_TLS:-0},
 			"write_buffer_size": ${HTTP_WRITE_BUFFER_SIZE:-0},
 			"read_buffer_size":  ${HTTP_READ_BUFFER_SIZE:-0},
 			"chunked_transfer":  ${AIS_HTTP_CHUNKED_TRANSFER:-true},

diff --git a/ec/getxaction.go b/ec/getxaction.go
@@ -147,7 +147,7 @@ func (r *XactGet) newGetJogger(mpath string) *getJogger {
 		client *http.Client
 	)
 	if r.config.Net.HTTP.UseHTTPS {
-		client = cmn.NewClientTLS(cargs, cmn.TLSArgs{SkipVerify: r.config.Net.HTTP.SkipVerify})
+		client = cmn.NewClientTLS(cargs, cmn.TLSArgs{SkipVerify: r.config.Net.HTTP.SkipVerifyTLS})
 	} else {
 		client = cmn.NewClient(cargs)
 	}

diff --git a/ext/dsort/manager.go b/ext/dsort/manager.go
@@ -163,7 +163,7 @@ func newClient() {
 		}
 	)
 	if config.Net.HTTP.UseHTTPS {
-		bcastClient = cmn.NewClientTLS(cargs, cmn.TLSArgs{SkipVerify: config.Net.HTTP.SkipVerify})
+		bcastClient = cmn.NewClientTLS(cargs, cmn.TLSArgs{SkipVerify: config.Net.HTTP.SkipVerifyTLS})
 	} else {
 		bcastClient = cmn.NewClient(cargs)
 	}
@@ -212,7 +212,7 @@ func (m *Manager) init(pars *parsedReqSpec) error {
 		UseHTTPS:    m.config.Net.HTTP.UseHTTPS,
 	}
 	if m.config.Net.HTTP.UseHTTPS {
-		m.client = cmn.NewClientTLS(cargs, cmn.TLSArgs{SkipVerify: m.config.Net.HTTP.SkipVerify})
+		m.client = cmn.NewClientTLS(cargs, cmn.TLSArgs{SkipVerify: m.config.Net.HTTP.SkipVerifyTLS})
 	} else {
 		m.client = cmn.NewClient(cargs)
 	}

diff --git a/reb/globrun.go b/reb/globrun.go
@@ -121,7 +121,7 @@ func New(t cluster.Target, config *cmn.Config) *Reb {
 		client *http.Client
 	)
 	if config.Net.HTTP.UseHTTPS {
-		client = cmn.NewClientTLS(cargs, cmn.TLSArgs{SkipVerify: config.Net.HTTP.SkipVerify})
+		client = cmn.NewClientTLS(cargs, cmn.TLSArgs{SkipVerify: config.Net.HTTP.SkipVerifyTLS})
 	} else {
 		client = cmn.NewClient(cargs)
 	}

diff --git a/transport/client_fasthttp.go b/transport/client_fasthttp.go
@@ -52,7 +52,7 @@ func NewIntraDataClient() Client {
 		WriteBufferSize: wbuf,
 	}
 	if config.Net.HTTP.UseHTTPS {
-		tlsConfig, err := cmn.NewTLS(cmn.TLSArgs{SkipVerify: config.Net.HTTP.SkipVerify})
+		tlsConfig, err := cmn.NewTLS(cmn.TLSArgs{SkipVerify: config.Net.HTTP.SkipVerifyTLS})
 		cos.AssertNoErr(err)
 		cl.TLSConfig = tlsConfig
 	}

diff --git a/transport/client_nethttp.go b/transport/client_nethttp.go
@@ -49,7 +49,7 @@ func NewIntraDataClient() (client *http.Client) {
 		UseHTTPS:        config.Net.HTTP.UseHTTPS,
 	}
 	if config.Net.HTTP.UseHTTPS {
-		client = cmn.NewClientTLS(cargs, cmn.TLSArgs{SkipVerify: config.Net.HTTP.SkipVerify})
+		client = cmn.NewClientTLS(cargs, cmn.TLSArgs{SkipVerify: config.Net.HTTP.SkipVerifyTLS})
 	} else {
 		client = cmn.NewClient(cargs)
 	}