support TLS configuration

* docs
* part seven

Signed-off-by: Alex Aizman <[email protected]>

ais/test/tls-env/server.conf

@@ -5,3 +5,5 @@ export AIS_DOMAIN_TLS=localhost				# net.http.domain_tls
 export AIS_CLIENT_CA_TLS=/tmp/tls/MyFirst-CA.crt	# net.http.client_ca_tls
 export AIS_CLIENT_AUTH_TLS=4				# net.http.client_auth_tls
 export AIS_SKIP_VERIFY_CRT=false			# net.http.skip_verify
+
+# NOTE: if AIS_USE_HTTPS ("net.http.use_https") is false, all the settings above will be ignored

api/env/ais.go

@@ -21,18 +21,23 @@ var (
 		// K8s
 		K8sPod string
 	}{
+		// the way to designate primary when cluster's starting up
 		Endpoint:  "AIS_ENDPOINT",
 		IsPrimary: "AIS_IS_PRIMARY",
 		PrimaryID: "AIS_PRIMARY_ID",
-		UseHTTPS:  "AIS_USE_HTTPS",
+
+		// false: HTTP transport, with all the TLS config (below) ignored
+		// true:  HTTPS/TLS
+		UseHTTPS: "AIS_USE_HTTPS", // cluster config: "net.http.use_https"
 
 		// TLS: client side
-		Certificate:   "AIS_CRT",
-		CertKey:       "AIS_CRT_KEY",
-		ClientCA:      "AIS_CLIENT_CA",
-		SkipVerifyCrt: "AIS_SKIP_VERIFY_CRT",
+		Certificate: "AIS_CRT",
+		CertKey:     "AIS_CRT_KEY",
+		ClientCA:    "AIS_CLIENT_CA",
+		// TLS: common
+		SkipVerifyCrt: "AIS_SKIP_VERIFY_CRT", // cluster config: "net.http.skip_verify"
 
-		// Env variables used for tests or CI
+		// variables used in tests and CI
 		NumTarget: "NUM_TARGET",
 		NumProxy:  "NUM_PROXY",
 

docs/aisloader.md

@@ -107,11 +107,6 @@ For the most recently updated command-line options and examples, please run `ais
 | -usage | `bool` | Show command-line options, usage, and examples | `false` |
 | -verifyhash | `bool` | checksum-validate GET: recompute object checksums and validate it against the one received with the GET metadata | `true` |
 
-### Options via environment variables
-
-| Environment Variable | Type | Description |
-| `AIS_ENDPOINT` | `string` | Address of a cluster which aisloader will generate load on. Overrides `ip` and `port` flags. |
-
 ### Often used options explanation
 
 #### Duration
@@ -236,19 +231,29 @@ Note that this is entirely optional, and therefore an input such as `300` will b
 
 ## Environment variables
 
-Note that AIS cluster endpoint can be defined in two ways:
+| Environment Variable | Type | Description |
+| -- | -- | -- |
+| `AIS_ENDPOINT` | `string` | Cluster's endpoint: http or https address of any aistore gateway in this cluster. Overrides `ip` and `port` flags. |
 
-* as http://ip:port address whereby '--ip' and '--port' are command-line options
-* via `AIS_ENDPOINT` environment universally supported across all AIS clients.
+To state the same slightly differently, cluster endpoint can be defined in two ways:
 
-In addition, environment can be used to specify TLS (aka, HTTPS) configuration pertaining to client-side certificates.
+* as (plain) http://ip:port address, whereby '--ip' and '--port' are command-line options.
+* via `AIS_ENDPOINT` environment universally supported across all AIS clients, e.g.:
+
+```console
+$ export AIS_ENDPOINT=https://10.07.56.68:51080
+```
+
+In addition, environment can be used to specify client-side TLS (aka, HTTPS) configuration:
 
 | var name | description |
 | -- | -- |
-| AIS_CRT             | X509 certificate |
-| AIS_CRT_KEY         | X509 certificate's private key |
-| AIS_CLIENT_CA       | Cerificate authority that authorized (ie., signed) the certificate |
-| AIS_SKIP_VERIFY_CRT | true: skip X509 cert verification (usually enabled to circumvent limitations of self-signed certs) |
+| `AIS_CRT`             | X509 certificate |
+| `AIS_CRT_KEY`         | X509 certificate's private key |
+| `AIS_CLIENT_CA`       | Certificate authority that authorized (signed) the certificate |
+| `AIS_SKIP_VERIFY_CRT` | true: skip X509 cert verification (usually enabled to circumvent limitations of self-signed certs) |
+
+* See also: [TLS: testing with self-signed certificates](/docs/getting_started.md#tls-testing-with-self-signed-certificates)
 
 ## Examples
 

docs/authn.md

@@ -170,7 +170,7 @@ Environment variables used by the deployment script to setup AuthN server:
 | AIS_AUTHN_ENABLED | `false` | Set it to `true` to enable AuthN server and token-based access in AIStore proxy |
 | AIS_AUTHN_PORT | `52001` | Port on which AuthN listens to requests |
 | AIS_AUTHN_TTL | `24h` | A token expiration time. Can be set to 0 which means "no expiration time" |
-| AIS_AUTHN_USE_HTTPS | `false` | Enable secure HTTP for AuthN server. If `true`, AuthN server requires also `AIS_SERVER_CRT` and `AIS_SERVER_KEY` to be set |
+| AIS_AUTHN_USE_HTTPS | `false` | Enable HTTPS for AuthN server. If `true`, AuthN server requires also `AIS_SERVER_CRT` and `AIS_SERVER_KEY` to be set |
 | AIS_SERVER_CRT | ` ` | OpenSSL certificate. Optional: set it only when secure HTTP is enabled |
 | AIS_SERVER_KEY | ` ` | OpenSSL key. Optional: set it only when secure HTTP is enabled |
 

docs/cli.md

@@ -12,6 +12,7 @@ redirect_from:
 - [Getting Started](#getting-started)
 - [CLI Reference](#cli-reference)
 - [CLI Config](#cli-config)
+- [Environment variables](#environment-variables)
 - [First steps](#first-steps)
 - [Global options](#global-options)
 - [Backend Provider](#backend-provider)
@@ -127,11 +128,16 @@ Notice:
 When used the very first time, *or* if the `$HOME/.config/ais/cli/cli.json` does not exist, the latter will be created with default parameters:
 
 ```json
+$ ais config cli --json
+
 {
     "cluster": {
         "url": "http://127.0.0.1:8080",
         "default_ais_host": "http://127.0.0.1:8080",
         "default_docker_host": "http://172.50.0.2:8080",
+        "client_crt": "",
+        "client_crt_key": "",
+        "client_ca_tls": "",
         "skip_verify_crt": false
     },
     "timeout": {
@@ -142,23 +148,51 @@ When used the very first time, *or* if the `$HOME/.config/ais/cli/cli.json` does
         "url": "http://127.0.0.1:52001"
     },
     "aliases": {
+        "wait": "job wait",
         "cp": "bucket cp",
         "create": "bucket create",
-        "wait": "job wait",
-        "stop": "job stop",
+        "download": "job start download",
+        "evict": "bucket evict",
         "get": "object get",
-        "ls": "bucket ls",
         "put": "object put",
+        "dsort": "job start dsort",
+        "ls": "bucket ls",
         "rmb": "bucket rm",
-        "start": "job start"
+        "start": "job start",
+        "stop": "job stop"
     },
     "default_provider": "ais",
     "no_color": false,
     "verbose": false
 }
 ```
 
-If you update config via `ais config cli set` command (or even simply change the config file) the next time CLI will use updated values.
+CLI config can be updated using `ais config cli set` command or even simply by changing the config file.
+
+The next time you run it CLI will use the updated values.
+
+To get back to system defaults, run `ais config cli reset`.
+
+## Environment variables
+
+First and foremost, there's `AIS_ENDPOINT`. If defined, it'll take precedence over "cluster.url" (section [CLI Config](#cli-config) above).
+
+Example:
+
+```console
+$ export AIS_ENDPOINT=https://10.07.56.68:51080
+```
+
+In addition, environment can be used to **override** client-side TLS (aka, HTTPS) configuration - the knobs "client_crt", etc. also listed in the table below:
+
+| var name | description | the corresponding [CLI Config](#cli-config) |
+| -- | -- | -- |
+| `AIS_CRT`             | X509 certificate | "cluster.client_crt" |
+| `AIS_CRT_KEY`         | X509 certificate's private key | "cluster.client_crt_key"|
+| `AIS_CLIENT_CA`       | Certificate authority that authorized (signed) the certificate | "cluster.client_ca_tls" |
+| `AIS_SKIP_VERIFY_CRT` | true: skip X509 cert verification (usually enabled to circumvent limitations of self-signed certs) | "cluster.skip_verify_crt" |
+
+* See also: [TLS: testing with self-signed certificates](/docs/getting_started.md#tls-testing-with-self-signed-certificates)
 
 ## First steps
 

docs/getting_started.md

@@ -225,6 +225,10 @@ For developers, CLI `ais config cluster log.modules ec xs` (for instance) would
 
 > To list all log modules, type `ais config cluster` or `ais config node` and press `<TAB-TAB>`.
 
+Finally, there's also HTTPS configuration (including **X509** certificates and options), and the corresponding [environment](#tls-testing-with-self-signed-certificates).
+
+For details, please see section [TLS: testing with self-signed certificates](#tls-testing-with-self-signed-certificates) below.
+
 ## Multiple deployment options
 
 All [containerized deployments](/deploy/README.md) have their own separate `Makefiles`. With the exception of [local playground](#local-playground), each specific build-able development (`dev/`) and production (`prod/`) option under the `deploy` folder has a pair: {`Dockerfile`, `Makefile`}.
@@ -422,25 +426,39 @@ Further, given the container's cgroup/memory limitation, each AIS node adjusts t
 
 ## TLS: testing with self-signed certificates
 
-Here's a list of related environment variables, and a quick playground run: 6 gateways, 6 targets.
-
-| var name | description |
-| -- | -- |
-| AIS_CRT             | X509 certificate |
-| AIS_CRT_KEY         | X509 certificate's private key |
-| AIS_CLIENT_CA       | Cerificate authority that authorized (ie., signed) the certificate |
-| AIS_SKIP_VERIFY_CRT | true: skip X509 cert verification (usually enabled to circumvent limitations of self-signed certs) |
-
-Notice that `server.conf` environment (below) overrides aistore configuration which is also referenced inside the same `server.conf` file.
-
-On the other hand, `ais/test/tls-env/client.conf` contains environment variables to override CLI config. The correspondence between environment and config names is easy to see as well.
+Local playground run: 6 gateways, 6 targets:
 
 ```console
 $ source ais/test/tls-env/server.conf
 $ source ais/test/tls-env/client.conf
 $ AIS_USE_HTTPS=true make kill cli deploy <<< $'6\n6\n4\ny\ny\nn\nn\n'
 ```
 
+Notice that when the cluster is first time deployed `server.conf` environment (above) overrides aistore cluster configuration.
+
+> Environment is ignored upon cluster restarts and upgrades.
+
+Here's a quick summary of the corresponding configuration variables (that are also referenced inside `ais/test/tls-env/server.conf`).
+
+| var name | description | the corresponding cluster configuration |
+| -- | -- | -- |
+| `AIS_USE_HTTPS`       | when false, we use plain HTTP with all the TLS config (below) simply **ignored** | "net.http.use_https" |
+| -- | -- | -- |
+| `AIS_SERVER_CRT`         | aistore cluster X509 certificate | "net.http.server_crt" |
+| `AIS_SERVER_KEY`         | certificate's private key | "net.http.server_key"|
+| `AIS_DOMAIN_TLS`         | domain, hostname, or Subject Alternative Name (SAN) registered with the certificate | "net.http.domain_tls"|
+| `AIS_CLIENT_CA_TLS`       | Certificate authority that authorized (signed) the certificate | "net.http.client_ca_tls" |
+| `AIS_CLIENT_AUTH_TLS`       | Client authentication during TLS handshake: a range from 0 (no authentication) to 4 (request and validate client's certificate) | "net.http.client_auth_tls" |
+| `AIS_SKIP_VERIFY_CRT` | when true: skip X509 cert verification (usually enabled to circumvent limitations of self-signed certs) | "net.http.skip_verify" |
+
+
+On the other hand, `ais/test/tls-env/client.conf` contains environment variables to override CLI config. The correspondence between environment and config names is easy to see as well.
+
 Prerequisites:
 
-* `/tmp/tls` (or any other location of your choosing) must contain valid, possibly `openssl`-generated and self-signed, X509 certs for the CLI and aistore itself.
+* `/tmp/tls` (or any other location of your choosing) must contain valid X509 certs for both the CLI (and aisloader, other clients), and aistore itself.
+* testing-wise,`openssl`-generated and self-signed X509 certs will be perfectly fine.
+
+See also:
+
+* [Client-side TLS environment](/docs/cli.md#environment-variables)