Merge pull request #7 from triton-inference-server/feature/tls

Feature/tls

.github/workflows/ci.yml

@@ -0,0 +1,28 @@
+name: "CI"
+
+on:
+  push:
+    branches: ['main']
+  pull_request:
+    branches: ['main']
+jobs:
+  tls:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: install dependencies
+        run: sudo apt update && sudo apt install -y rapidjson-dev
+      - name: build
+        run: bash ./build.sh
+      - name: prepare models
+        run: cd examples && source fetch_model.sh && cd ..
+      - name: build test certs
+        run: |
+          source ./examples/tls/gen-certs.sh
+          chmod +r ./certs/ca.crt ./certs/redis.crt ./certs/redis.key
+      - name: mv certs
+        run:  mv certs examples/tls/
+      - name: docker-compose up
+        run: docker-compose -f ./examples/tls/docker-compose.yml up -d
+      - name: execute
+        run: docker-compose -f ./examples/tls/docker-compose.yml run client

.gitignore

@@ -3,4 +3,7 @@
 *.so
 .idea
 cmake-build-debug
-third-party
+third-party
+examples/tls/certs
+/examples/model_repository/densenet_onnx/1
+/examples/model_repository/inception_graphdef/1

CMakeLists.txt

@@ -74,6 +74,7 @@ execute_process(
 ## Add redis++ library to the project
 find_library(REDISPP redis++ PATHS build/install/lib NO_DEFAULT_PATH REQUIRED)
 find_library(HIREDIS hiredis PATHS build/install/lib NO_DEFAULT_PATH REQUIRED)
+find_library(HIREDIS_SSL hiredis_ssl PATHS build/install/lib NO_DEFAULT_PATH REQUIRED)
 
 #
 # Shared library implementing the Triton Cache API
@@ -111,6 +112,7 @@ target_link_libraries(
   triton-redis-cache
   PUBLIC
     ${HIREDIS}
+    ${HIREDIS_SSL}
     ${REDISPP}
   PRIVATE
     triton-core-serverapi   # from repo-core

Dockerfile

@@ -1,4 +1,3 @@
-FROM nvcr.io/nvidia/tritonserver:23.05-py3
+FROM nvcr.io/nvidia/tritonserver:23.06-py3
 
-RUN mkdir /opt/tritonserver/caches/redis
 COPY ./build/install/caches/redis/libtritoncache_redis.so /opt/tritonserver/caches/redis

README.md

@@ -108,6 +108,23 @@ tritonserver --cache-config redis,host=redis-host --cache-config redis,port=6379
 
 Optionally you may configure your `user`/`password` via environment variables. The corresponding `user` environment variable is `TRITONCACHE_REDIS_USERNAME` whereas the corresponding `password` environment variable is `TRITONCACHE_REDIS_PASSWORD`.
 
+### TLS
+
+Transport Layer Security (TLS) can be enabled in Redis and within the Triton Redis Cache, to do so you will need a TLS
+enabled version of Redis, e.g. [OSS Redis](https://redis.io/docs/management/security/encryption/) or
+[Redis Enterprise](https://docs.redis.com/latest/rs/security/tls/enable-tls/). You will also need to configure Triton Server to use TLS with Redis
+through the following `--cache-config` TLS options.
+
+#### Configuration Items for TLS
+
+| Configuration Option | Required | Description                                           |
+|----------------------|----------|-------------------------------------------------------|
+| tls_enabled          | Yes      | set to `true` to enable TLS                           |
+| cert                 | no       | The certificate to use for TLS.                       |
+| key                  | no       | The certificate key to use for TLS.                   |
+| cacert               | No       | The Certificate Authority certificate to use for TLS. |
+| sni                  | No       | Server name indication for TLS.                       |
+
 ## Monitoring and Observability
 
 There are many ways to go about monitoring what's going on in Redis. One popular mode is to export metrics data from Redis to Prometheus, and use Grafana to observe them.
@@ -120,6 +137,7 @@ There are many ways to go about monitoring what's going on in Redis. One popular
 You can try out the Redis Cache with Triton in docker:
 
 * clone this repo: `git clone https://github.com/triton-inference-server/redis_cache`
+* follow build instructions enumerated [above](https://github.com/triton-inference-server/redis_cache#build-the-cache)
 * clone the Triton server repo: `git clone https://github.com/triton-inference-server`
 * Add the following to: `docs/examples/model_repository/densenet_onnx/config.pbtxt`
 ```
@@ -140,7 +158,7 @@ Password: <MY API KEY>
 > NOTE: Username: $oauthtoken in this context means that your username is literally $oauthtoken - your API key serves as the unique part of your credentials
 * run `docker-compose build`
 * run `docker-compose up`
-* In a separate terminal run `docker run -it --rm --net=host nvcr.io/nvidia/tritonserver:23.03-py3-sdk`
+* In a separate terminal run `docker run -it --rm --net=host nvcr.io/nvidia/tritonserver:23.06-py3-sdk`
 * Run `/workspace/install/bin/image_client -m densenet_onnx -c 3 -s INCEPTION /workspace/images/mug.jpg`
   * on the first run - this will miss the cache
   * subsequent runs will pull the inference out of the cache

build.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+mkdir -p build
+cd build
+cmake -DCMAKE_INSTALL_PREFIX:PATH=`pwd`/install ..
+make install

build_deps.sh

@@ -22,8 +22,8 @@ else
     fi
     cd hiredis
 
-    LIBRARY_PATH=lib CC=gcc CXX=g++ make PREFIX="$(pwd)/../../build/install" static -j 4
-    LIBRARY_PATH=lib CC=gcc CXX=g++ make PREFIX="$(pwd)/../../build/install" install
+    LIBRARY_PATH=lib CC=gcc CXX=g++ make PREFIX="$(pwd)/../../build/install" USE_SSL=1 static -j 4
+    LIBRARY_PATH=lib CC=gcc CXX=g++ make PREFIX="$(pwd)/../../build/install" USE_SSL=1 install
     cd ../
     # delete shared libraries
     rm ../build/install/lib/*.so
@@ -44,7 +44,7 @@ else
     mkdir compile
     cd compile
 
-    $CMAKE -DCMAKE_BUILD_TYPE=Release -DREDIS_PLUS_PLUS_BUILD_TEST=OFF -DREDIS_PLUS_PLUS_BUILD_SHARED=OFF -DCMAKE_PREFIX_PATH="$(pwd)../../../build/install/lib/" -DCMAKE_INSTALL_PREFIX="$(pwd)/../../../build/install" -DCMAKE_CXX_STANDARD=17 ..
+    $CMAKE -DCMAKE_BUILD_TYPE=Release -DREDIS_PLUS_PLUS_BUILD_TEST=OFF -DREDIS_PLUS_PLUS_BUILD_SHARED=OFF -DCMAKE_PREFIX_PATH="$(pwd)../../../build/install/lib/" -DCMAKE_INSTALL_PREFIX="$(pwd)/../../../build/install" -DCMAKE_CXX_STANDARD=17 -DREDIS_PLUS_PLUS_USE_TLS=ON ..
     CC=gcc CXX=g++ make -j 4
     CC=gcc CXX=g++ make install
     cd ../../

examples/fetch_model.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+set -ex
+
+# ONNX densenet
+mkdir -p model_repository/densenet_onnx/1
+wget -O model_repository/densenet_onnx/1/model.onnx \
+     https://contentmamluswest001.blob.core.windows.net/content/14b2744cf8d6418c87ffddc3f3127242/9502630827244d60a1214f250e3bbca7/08aed7327d694b8dbaee2c97b8d0fcba/densenet121-1.2.onnx

examples/model_repository/densenet_onnx/config.pbtxt

@@ -0,0 +1,24 @@
+name: "densenet_onnx"
+platform: "onnxruntime_onnx"
+max_batch_size : 0
+response_cache {
+  enable: true
+}
+input [
+  {
+    name: "data_0"
+    data_type: TYPE_FP32
+    format: FORMAT_NCHW
+    dims: [ 3, 224, 224 ]
+    reshape { shape: [ 1, 3, 224, 224 ] }
+  }
+]
+output [
+  {
+    name: "fc6_1"
+    data_type: TYPE_FP32
+    dims: [ 1000 ]
+    reshape { shape: [ 1, 1000, 1, 1 ] }
+    label_filename: "densenet_labels.txt"
+  }
+]