support TLS configuration

* pub, data, and control: use tls config (prev. commit)
* in-cluster clients: ditto
* tools: use tls env
* CLI: add _tls knobs: cert, cert-key, CA
* CLI: env to override the latter
* cmn: add helpers, other convenience
* part four

Signed-off-by: Alex Aizman <[email protected]>

ais/backend/http.go

@@ -9,7 +9,6 @@ import (
 	"fmt"
 	"io"
 	"net/http"
-	"strings"
 
 	"github.com/NVIDIA/aistore/api/apc"
 	"github.com/NVIDIA/aistore/cluster"
@@ -40,7 +39,7 @@ func NewHTTP(t cluster.TargetPut, config *cmn.Config) cluster.BackendProvider {
 			ReadBufferSize:  config.Net.HTTP.ReadBufferSize,
 		}
 		sargs = cmn.TLSArgs{
-			SkipVerify: config.Net.HTTP.SkipVerifyTLS,
+			SkipVerify: true, // TODO: may need more tls config to access remote URLs
 		}
 	)
 	hp.httpClient = cmn.NewClient(cargs)
@@ -51,7 +50,7 @@ func NewHTTP(t cluster.TargetPut, config *cmn.Config) cluster.BackendProvider {
 }
 
 func (hp *httpProvider) client(u string) *http.Client {
-	if strings.HasPrefix(u, "https") {
+	if cos.IsHTTPS(u) {
 		return hp.httpsClient
 	}
 	return hp.httpClient

ais/htcommon.go

@@ -6,13 +6,16 @@ package ais
 
 import (
 	"context"
+	"crypto/tls"
+	"crypto/x509"
 	"errors"
 	"fmt"
 	"io"
 	"log"
 	"net"
 	"net/http"
 	"net/url"
+	"os"
 	rdebug "runtime/debug"
 	"strings"
 	"sync"
@@ -576,7 +579,7 @@ func (server *netServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	go transfer(clientConn, destConn)
 }
 
-func (server *netServer) listen(addr string, logger *log.Logger) (err error) {
+func (server *netServer) listen(addr string, logger *log.Logger, tlsConf *tls.Config) (err error) {
 	var (
 		httpHandler = server.muxers
 		config      = cmn.GCO.Get()
@@ -592,11 +595,12 @@ func (server *netServer) listen(addr string, logger *log.Logger) (err error) {
 	if server.sndRcvBufSize > 0 && !config.Net.HTTP.UseHTTPS {
 		server.s.ConnState = server.connStateListener // setsockopt; see also cmn.NewTransport
 	}
+	server.s.TLSConfig = tlsConf
 	server.Unlock()
 retry:
 	if config.Net.HTTP.UseHTTPS {
 		tag = "HTTPS"
-		err = server.s.ListenAndServeTLS(config.Net.HTTP.Certificate, config.Net.HTTP.Key)
+		err = server.s.ListenAndServeTLS(config.Net.HTTP.Certificate, config.Net.HTTP.CertKey)
 	} else {
 		err = server.s.ListenAndServe()
 	}
@@ -613,6 +617,25 @@ retry:
 	return
 }
 
+func newTLS(conf *cmn.HTTPConf) (tlsConf *tls.Config, err error) {
+	var (
+		pool       *x509.CertPool
+		caCert     []byte
+		clientAuth = tls.ClientAuthType(conf.ClientAuthTLS)
+	)
+	if clientAuth > tls.RequestClientCert {
+		if caCert, err = os.ReadFile(conf.ClientCA); err != nil {
+			return
+		}
+		pool = x509.NewCertPool()
+		if ok := pool.AppendCertsFromPEM(caCert); !ok {
+			return nil, fmt.Errorf("tls: failed to append CA certs from PEM: %q", conf.ClientCA)
+		}
+	}
+	tlsConf = &tls.Config{ServerName: conf.ServerNameTLS, ClientAuth: clientAuth, ClientCAs: pool}
+	return
+}
+
 func (server *netServer) connStateListener(c net.Conn, cs http.ConnState) {
 	if cs != http.StateNew {
 		return

ais/htrun.go

@@ -7,6 +7,7 @@ package ais
 import (
 	"bytes"
 	"context"
+	"crypto/tls"
 	"errors"
 	"fmt"
 	"io"
@@ -298,19 +299,14 @@ func (h *htrun) initCtrlClient(config *cmn.Config) {
 		defaultControlWriteBufferSize = 16 * cos.KiB // for more defaults see cmn/network.go
 		defaultControlReadBufferSize  = 16 * cos.KiB
 	)
-	var (
-		cargs = cmn.TransportArgs{
-			Timeout:         config.Client.Timeout.D(),
-			WriteBufferSize: defaultControlWriteBufferSize,
-			ReadBufferSize:  defaultControlReadBufferSize,
-			UseHTTPS:        config.Net.HTTP.UseHTTPS,
-		}
-		sargs = cmn.TLSArgs{
-			SkipVerify: config.Net.HTTP.SkipVerifyTLS,
-		}
-	)
+	cargs := cmn.TransportArgs{
+		Timeout:         config.Client.Timeout.D(),
+		WriteBufferSize: defaultControlWriteBufferSize,
+		ReadBufferSize:  defaultControlReadBufferSize,
+		UseHTTPS:        config.Net.HTTP.UseHTTPS,
+	}
 	if config.Net.HTTP.UseHTTPS {
-		h.client.control = cmn.NewClientTLS(cargs, sargs)
+		h.client.control = cmn.NewIntraClientTLS(cargs, config)
 	} else {
 		h.client.control = cmn.NewClient(cargs)
 	}
@@ -325,19 +321,14 @@ func (h *htrun) initDataClient(config *cmn.Config) {
 	if rbuf == 0 {
 		rbuf = cmn.DefaultReadBufferSize
 	}
-	var (
-		cargs = cmn.TransportArgs{
-			Timeout:         config.Client.TimeoutLong.D(),
-			WriteBufferSize: wbuf,
-			ReadBufferSize:  rbuf,
-			UseHTTPS:        config.Net.HTTP.UseHTTPS,
-		}
-		sargs = cmn.TLSArgs{
-			SkipVerify: config.Net.HTTP.SkipVerifyTLS,
-		}
-	)
+	cargs := cmn.TransportArgs{
+		Timeout:         config.Client.TimeoutLong.D(),
+		WriteBufferSize: wbuf,
+		ReadBufferSize:  rbuf,
+		UseHTTPS:        config.Net.HTTP.UseHTTPS,
+	}
 	if config.Net.HTTP.UseHTTPS {
-		h.client.data = cmn.NewClientTLS(cargs, sargs)
+		h.client.data = cmn.NewIntraClientTLS(cargs, config)
 	} else {
 		h.client.data = cmn.NewClient(cargs)
 	}
@@ -505,18 +496,28 @@ func (h *htrun) setDaemonConfigQuery(w http.ResponseWriter, r *http.Request) {
 }
 
 func (h *htrun) run(config *cmn.Config) error {
-	logger := log.New(&nlogWriter{}, "net/http err: ", 0) // a wrapper to log http.Server errors
+	var (
+		tlsConf *tls.Config
+		logger  = log.New(&nlogWriter{}, "net/http err: ", 0) // a wrapper to log http.Server errors
+	)
+	if config.Net.HTTP.UseHTTPS {
+		c, err := newTLS(&config.Net.HTTP)
+		if err != nil {
+			cos.ExitLog(err)
+		}
+		tlsConf = c
+	}
 	if config.HostNet.UseIntraControl {
 		go func() {
-			_ = h.netServ.control.listen(h.si.ControlNet.TCPEndpoint(), logger)
+			_ = h.netServ.control.listen(h.si.ControlNet.TCPEndpoint(), logger, tlsConf)
 		}()
 	}
 	if config.HostNet.UseIntraData {
 		go func() {
-			_ = h.netServ.data.listen(h.si.DataNet.TCPEndpoint(), logger)
+			_ = h.netServ.data.listen(h.si.DataNet.TCPEndpoint(), logger, tlsConf)
 		}()
 	}
-	return h.netServ.pub.listen(h.pubListeningAddr(config), logger)
+	return h.netServ.pub.listen(h.pubListeningAddr(config), logger, tlsConf)
 }
 
 // testing environment excluding Kubernetes: listen on `host:port`

ais/proxy.go

@@ -2054,8 +2054,10 @@ func newRevProxyTransport(config *cmn.Config) *http.Transport {
 		transport = cmn.NewTransport(cmn.TransportArgs{UseHTTPS: config.Net.HTTP.UseHTTPS})
 	)
 	if config.Net.HTTP.UseHTTPS {
-		transport.TLSClientConfig, err = cmn.NewTLS(cmn.TLSArgs{SkipVerify: config.Net.HTTP.SkipVerifyTLS})
-		cos.AssertNoErr(err)
+		transport.TLSClientConfig, err = cmn.NewTLS(config.Net.HTTP.ToTLS())
+		if err != nil {
+			cos.ExitLog(err)
+		}
 	}
 	return transport
 }

api/env/ais.go

@@ -1,31 +1,36 @@
 // Package env contains environment variables
 /*
- * Copyright (c) 2018-2022, NVIDIA CORPORATION. All rights reserved.
+ * Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.
  */
 package env
 
 var (
 	AIS = struct {
-		Endpoint      string
-		IsPrimary     string
-		PrimaryID     string
-		SkipVerifyCrt string
-		UseHTTPS      string
-		DomainTLS     string
+		Endpoint  string
+		IsPrimary string
+		PrimaryID string
+		UseHTTPS  string
+		// TLS: client side
+		Certificate   string
+		CertKey       string
 		ClientCA      string
-		ClientAuthTLS string
-		NumTarget     string
-		NumProxy      string
-		K8sPod        string
+		SkipVerifyCrt string
+		// tests, CI
+		NumTarget string
+		NumProxy  string
+		// K8s
+		K8sPod string
 	}{
-		Endpoint:      "AIS_ENDPOINT",
-		IsPrimary:     "AIS_IS_PRIMARY",
-		PrimaryID:     "AIS_PRIMARY_ID",
+		Endpoint:  "AIS_ENDPOINT",
+		IsPrimary: "AIS_IS_PRIMARY",
+		PrimaryID: "AIS_PRIMARY_ID",
+		UseHTTPS:  "AIS_USE_HTTPS",
+
+		// TLS: client side
+		Certificate:   "AIS_CRT",
+		CertKey:       "AIS_CRT_KEY",
+		ClientCA:      "AIS_CLIENT_CA",
 		SkipVerifyCrt: "AIS_SKIP_VERIFY_CRT",
-		UseHTTPS:      "AIS_USE_HTTPS",
-		DomainTLS:     "AIS_DOMAIN_TLS",
-		ClientCA:      "AIS_CLIENT_CA_TLS",
-		ClientAuthTLS: "AIS_CLIENT_AUTH_TLS",
 
 		// Env variables used for tests or CI
 		NumTarget: "NUM_TARGET",

cmd/cli/cli/init.go

@@ -32,26 +32,28 @@ func Init() (err error) {
 	// auth
 	loggedUserToken = authn.LoadToken("")
 
-	//
-	// http transport and clients: the main one and auth, if enabled
-	//
+	// http clients: the main one and the auth, if enabled
 	clusterURL = _clusterURL(cfg)
 
-	useHTTPS := cos.IsHTTPS(clusterURL)
-	skipVerify := cfg.Cluster.SkipVerifyCrt
-	if useHTTPS {
-		if s := os.Getenv(env.AIS.SkipVerifyCrt); s != "" {
-			skipVerify = cos.IsParseBool(s)
+	var (
+		useHTTPS = cos.IsHTTPS(clusterURL)
+		cargs    = cmn.TransportArgs{
+			DialTimeout: cfg.Timeout.TCPTimeout,
+			Timeout:     cfg.Timeout.HTTPTimeout,
+			UseHTTPS:    useHTTPS,
 		}
-	}
-
-	cargs := cmn.TransportArgs{
-		DialTimeout: cfg.Timeout.TCPTimeout,
-		Timeout:     cfg.Timeout.HTTPTimeout,
-		UseHTTPS:    useHTTPS,
+		sargs = cmn.TLSArgs{
+			ClientCA:    cfg.Cluster.ClientCA,
+			Certificate: cfg.Cluster.Certificate,
+			Key:         cfg.Cluster.CertKey,
+			SkipVerify:  cfg.Cluster.SkipVerifyCrt,
+		}
+	)
+	if useHTTPS {
+		// environment to override client config
+		cmn.EnvToTLS(&sargs)
 	}
 	if useHTTPS {
-		sargs := cmn.TLSArgs{SkipVerify: skipVerify}
 		defaultHTTPClient = cmn.NewClientTLS(cargs, sargs)
 	} else {
 		defaultHTTPClient = cmn.NewClient(cargs)

cmd/cli/config/config.go

@@ -33,7 +33,11 @@ type (
 		URL               string `json:"url"`
 		DefaultAISHost    string `json:"default_ais_host"`
 		DefaultDockerHost string `json:"default_docker_host"`
-		SkipVerifyCrt     bool   `json:"skip_verify_crt"`
+		// TLS
+		Certificate   string `json:"client_crt"`     // X509 certificate
+		CertKey       string `json:"client_crt_key"` // X509 key
+		ClientCA      string `json:"client_ca_tls"`  // #6410
+		SkipVerifyCrt bool   `json:"skip_verify_crt"`
 	}
 	TimeoutConfig struct {
 		TCPTimeoutStr  string        `json:"tcp_timeout"`

cmd/cli/go.mod

@@ -4,7 +4,7 @@ go 1.21
 
 // direct
 require (
-	github.com/NVIDIA/aistore v1.3.21-0.20231024170343-fab6dc20181c
+	github.com/NVIDIA/aistore v1.3.21-0.20231025160105-6e1de1e2c2f0
 	github.com/fatih/color v1.15.0
 	github.com/json-iterator/go v1.1.12
 	github.com/onsi/ginkgo v1.16.5

cmd/cli/go.sum

@@ -1,7 +1,7 @@
 code.cloudfoundry.org/bytefmt v0.0.0-20190710193110-1eb035ffe2b6/go.mod h1:wN/zk7mhREp/oviagqUXY3EwuHhWyOvAdsn5Y4CzOrc=
 github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
-github.com/NVIDIA/aistore v1.3.21-0.20231024170343-fab6dc20181c h1:w6p12hr68uTfkS2tzagStTmjiKLoLTIvPCQLj53EGyA=
-github.com/NVIDIA/aistore v1.3.21-0.20231024170343-fab6dc20181c/go.mod h1:+iSnZg0ovMaLgaT9fLAs2WmYBP7IfeTW1WYkbKrwP4g=
+github.com/NVIDIA/aistore v1.3.21-0.20231025160105-6e1de1e2c2f0 h1:d2h1svmcqIY4DLRnQhuwuy1DgJsZ6FqjOiMFfpgnWS8=
+github.com/NVIDIA/aistore v1.3.21-0.20231025160105-6e1de1e2c2f0/go.mod h1:+iSnZg0ovMaLgaT9fLAs2WmYBP7IfeTW1WYkbKrwP4g=
 github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8=
 github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q=
 github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=