diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..10aec9d --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,28 @@ +name: "CI" + +on: + push: + branches: ['main'] + pull_request: + branches: ['main'] +jobs: + tls: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: install dependencies + run: sudo apt update && sudo apt install -y rapidjson-dev + - name: build + run: bash ./build.sh + - name: prepare models + run: cd examples && source fetch_model.sh && cd .. + - name: build test certs + run: | + source ./examples/tls/gen-certs.sh + chmod +r ./certs/ca.crt ./certs/redis.crt ./certs/redis.key + - name: mv certs + run: mv certs examples/tls/ + - name: docker-compose up + run: docker-compose -f ./examples/tls/docker-compose.yml up -d + - name: execute + run: docker-compose -f ./examples/tls/docker-compose.yml run client \ No newline at end of file diff --git a/.gitignore b/.gitignore index 91c7319..4c9ccab 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,7 @@ *.so .idea cmake-build-debug -third-party \ No newline at end of file +third-party +examples/tls/certs +/examples/model_repository/densenet_onnx/1 +/examples/model_repository/inception_graphdef/1 \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index d6ab44c..42dbd6a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -74,6 +74,7 @@ execute_process( ## Add redis++ library to the project find_library(REDISPP redis++ PATHS build/install/lib NO_DEFAULT_PATH REQUIRED) find_library(HIREDIS hiredis PATHS build/install/lib NO_DEFAULT_PATH REQUIRED) +find_library(HIREDIS_SSL hiredis_ssl PATHS build/install/lib NO_DEFAULT_PATH REQUIRED) # # Shared library implementing the Triton Cache API @@ -111,6 +112,7 @@ target_link_libraries( triton-redis-cache PUBLIC ${HIREDIS} + ${HIREDIS_SSL} ${REDISPP} PRIVATE triton-core-serverapi # from repo-core diff --git a/Dockerfile b/Dockerfile index ddecdcc..ef7157d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,3 @@ -FROM nvcr.io/nvidia/tritonserver:23.05-py3 +FROM nvcr.io/nvidia/tritonserver:23.06-py3 -RUN mkdir /opt/tritonserver/caches/redis COPY ./build/install/caches/redis/libtritoncache_redis.so /opt/tritonserver/caches/redis diff --git a/README.md b/README.md index 6796346..5ca79e4 100644 --- a/README.md +++ b/README.md @@ -108,6 +108,23 @@ tritonserver --cache-config redis,host=redis-host --cache-config redis,port=6379 Optionally you may configure your `user`/`password` via environment variables. The corresponding `user` environment variable is `TRITONCACHE_REDIS_USERNAME` whereas the corresponding `password` environment variable is `TRITONCACHE_REDIS_PASSWORD`. +### TLS + +Transport Layer Security (TLS) can be enabled in Redis and within the Triton Redis Cache, to do so you will need a TLS +enabled version of Redis, e.g. [OSS Redis](https://redis.io/docs/management/security/encryption/) or +[Redis Enterprise](https://docs.redis.com/latest/rs/security/tls/enable-tls/). You will also need to configure Triton Server to use TLS with Redis +through the following `--cache-config` TLS options. + +#### Configuration Items for TLS + +| Configuration Option | Required | Description | +|----------------------|----------|-------------------------------------------------------| +| tls_enabled | Yes | set to `true` to enable TLS | +| cert | no | The certificate to use for TLS. | +| key | no | The certificate key to use for TLS. | +| cacert | No | The Certificate Authority certificate to use for TLS. | +| sni | No | Server name indication for TLS. | + ## Monitoring and Observability There are many ways to go about monitoring what's going on in Redis. One popular mode is to export metrics data from Redis to Prometheus, and use Grafana to observe them. @@ -120,6 +137,7 @@ There are many ways to go about monitoring what's going on in Redis. One popular You can try out the Redis Cache with Triton in docker: * clone this repo: `git clone https://github.com/triton-inference-server/redis_cache` +* follow build instructions enumerated [above](https://github.com/triton-inference-server/redis_cache#build-the-cache) * clone the Triton server repo: `git clone https://github.com/triton-inference-server` * Add the following to: `docs/examples/model_repository/densenet_onnx/config.pbtxt` ``` @@ -140,7 +158,7 @@ Password: > NOTE: Username: $oauthtoken in this context means that your username is literally $oauthtoken - your API key serves as the unique part of your credentials * run `docker-compose build` * run `docker-compose up` -* In a separate terminal run `docker run -it --rm --net=host nvcr.io/nvidia/tritonserver:23.03-py3-sdk` +* In a separate terminal run `docker run -it --rm --net=host nvcr.io/nvidia/tritonserver:23.06-py3-sdk` * Run `/workspace/install/bin/image_client -m densenet_onnx -c 3 -s INCEPTION /workspace/images/mug.jpg` * on the first run - this will miss the cache * subsequent runs will pull the inference out of the cache diff --git a/build.sh b/build.sh new file mode 100755 index 0000000..7f5a197 --- /dev/null +++ b/build.sh @@ -0,0 +1,5 @@ +#!/bin/bash +mkdir -p build +cd build +cmake -DCMAKE_INSTALL_PREFIX:PATH=`pwd`/install .. +make install diff --git a/build_deps.sh b/build_deps.sh index 6563e17..e80a04c 100755 --- a/build_deps.sh +++ b/build_deps.sh @@ -22,8 +22,8 @@ else fi cd hiredis - LIBRARY_PATH=lib CC=gcc CXX=g++ make PREFIX="$(pwd)/../../build/install" static -j 4 - LIBRARY_PATH=lib CC=gcc CXX=g++ make PREFIX="$(pwd)/../../build/install" install + LIBRARY_PATH=lib CC=gcc CXX=g++ make PREFIX="$(pwd)/../../build/install" USE_SSL=1 static -j 4 + LIBRARY_PATH=lib CC=gcc CXX=g++ make PREFIX="$(pwd)/../../build/install" USE_SSL=1 install cd ../ # delete shared libraries rm ../build/install/lib/*.so @@ -44,7 +44,7 @@ else mkdir compile cd compile - $CMAKE -DCMAKE_BUILD_TYPE=Release -DREDIS_PLUS_PLUS_BUILD_TEST=OFF -DREDIS_PLUS_PLUS_BUILD_SHARED=OFF -DCMAKE_PREFIX_PATH="$(pwd)../../../build/install/lib/" -DCMAKE_INSTALL_PREFIX="$(pwd)/../../../build/install" -DCMAKE_CXX_STANDARD=17 .. + $CMAKE -DCMAKE_BUILD_TYPE=Release -DREDIS_PLUS_PLUS_BUILD_TEST=OFF -DREDIS_PLUS_PLUS_BUILD_SHARED=OFF -DCMAKE_PREFIX_PATH="$(pwd)../../../build/install/lib/" -DCMAKE_INSTALL_PREFIX="$(pwd)/../../../build/install" -DCMAKE_CXX_STANDARD=17 -DREDIS_PLUS_PLUS_USE_TLS=ON .. CC=gcc CXX=g++ make -j 4 CC=gcc CXX=g++ make install cd ../../ diff --git a/examples/fetch_model.sh b/examples/fetch_model.sh new file mode 100755 index 0000000..67c8909 --- /dev/null +++ b/examples/fetch_model.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set -ex + +# ONNX densenet +mkdir -p model_repository/densenet_onnx/1 +wget -O model_repository/densenet_onnx/1/model.onnx \ + https://contentmamluswest001.blob.core.windows.net/content/14b2744cf8d6418c87ffddc3f3127242/9502630827244d60a1214f250e3bbca7/08aed7327d694b8dbaee2c97b8d0fcba/densenet121-1.2.onnx \ No newline at end of file diff --git a/examples/model_repository/densenet_onnx/config.pbtxt b/examples/model_repository/densenet_onnx/config.pbtxt new file mode 100644 index 0000000..c7974ca --- /dev/null +++ b/examples/model_repository/densenet_onnx/config.pbtxt @@ -0,0 +1,24 @@ +name: "densenet_onnx" +platform: "onnxruntime_onnx" +max_batch_size : 0 +response_cache { + enable: true +} +input [ + { + name: "data_0" + data_type: TYPE_FP32 + format: FORMAT_NCHW + dims: [ 3, 224, 224 ] + reshape { shape: [ 1, 3, 224, 224 ] } + } +] +output [ + { + name: "fc6_1" + data_type: TYPE_FP32 + dims: [ 1000 ] + reshape { shape: [ 1, 1000, 1, 1 ] } + label_filename: "densenet_labels.txt" + } +] \ No newline at end of file diff --git a/examples/model_repository/densenet_onnx/densenet_labels.txt b/examples/model_repository/densenet_onnx/densenet_labels.txt new file mode 100644 index 0000000..e59113f --- /dev/null +++ b/examples/model_repository/densenet_onnx/densenet_labels.txt @@ -0,0 +1,1000 @@ +TENCH +GOLDFISH +WHITE SHARK +TIGER SHARK +HAMMERHEAD SHARK +ELECTRIC RAY +STINGRAY +ROOSTER +HEN +OSTRICH +BRAMBLING +GOLDFINCH +HOUSE FINCH +SNOWBIRD +INDIGO FINCH +ROBIN +BULBUL +JAY +MAGPIE +CHICKADEE +WATER OUZEL +KITE +BALD EAGLE +VULTURE +GREAT GREY OWL +FIRE SALAMANDER +NEWT +EFT +SPOTTED SALAMANDER +AXOLOTL +BULL FROG +TREE FROG +TAILED FROG +LOGGERHEAD +LEATHERBACK TURTLE +MUD TURTLE +TERRAPIN +BOX TURTLE +BANDED GECKO +COMMON IGUANA +AMERICAN CHAMELEON +WHIPTAIL +AGAMA +FRILLED LIZARD +ALLIGATOR LIZARD +GILA MONSTER +GREEN LIZARD +AFRICAN CHAMELEON +KOMODO DRAGON +AFRICAN CROCODILE +AMERICAN ALLIGATOR +TRICERATOPS +THUNDER SNAKE +RINGNECK SNAKE +HOGNOSE SNAKE +GREEN SNAKE +KING SNAKE +GARTER SNAKE +WATER SNAKE +VINE SNAKE +NIGHT SNAKE +BOA +ROCK PYTHON +COBRA +GREEN MAMBA +SEA SNAKE +HORNED VIPER +DIAMONDBACK +SIDEWINDER +TRILOBITE +HARVESTMAN +SCORPION +GARDEN SPIDER +BARN SPIDER +GARDEN SPIDER +BLACK WIDOW +TARANTULA +WOLF SPIDER +TICK +CENTIPEDE +GROUSE +PTARMIGAN +RUFFED GROUSE +PRAIRIE CHICKEN +PEACOCK +QUAIL +PARTRIDGE +AFRICAN GREY +MACAW +COCKATOO +LORIKEET +COUCAL +BEE EATER +HORNBILL +HUMMINGBIRD +JACAMAR +TOUCAN +DRAKE +MERGANSER +GOOSE +BLACK SWAN +TUSKER +ECHIDNA +PLATYPUS +WALLABY +KOALA +WOMBAT +JELLYFISH +SEA ANEMONE +BRAIN CORAL +FLATWORM +NEMATODE +CONCH +SNAIL +SLUG +SEA SLUG +CHITON +CHAMBERED NAUTILUS +DUNGENESS CRAB +ROCK CRAB +FIDDLER CRAB +KING CRAB +AMERICAN LOBSTER +SPINY LOBSTER +CRAYFISH +HERMIT CRAB +ISOPOD +WHITE STORK +BLACK STORK +SPOONBILL +FLAMINGO +LITTLE BLUE HERON +AMERICAN EGRET +BITTERN +CRANE +LIMPKIN +EUROPEAN GALLINULE +AMERICAN COOT +BUSTARD +RUDDY TURNSTONE +RED-BACKED SANDPIPER +REDSHANK +DOWITCHER +OYSTERCATCHER +PELICAN +KING PENGUIN +ALBATROSS +GREY WHALE +KILLER WHALE +DUGONG +SEA LION +CHIHUAHUA +JAPANESE SPANIEL +MALTESE DOG +PEKINESE +SHIH-TZU +BLENHEIM SPANIEL +PAPILLON +TOY TERRIER +RHODESIAN RIDGEBACK +AFGHAN HOUND +BASSET +BEAGLE +BLOODHOUND +BLUETICK +COONHOUND +WALKER HOUND +ENGLISH FOXHOUND +REDBONE +BORZOI +IRISH WOLFHOUND +ITALIAN GREYHOUND +WHIPPET +IBIZAN HOUND +NORWEGIAN ELKHOUND +OTTERHOUND +SALUKI +SCOTTISH DEERHOUND +WEIMARANER +STAFFORDSHIRE BULLTERRIER +STAFFORDSHIRE TERRIER +BEDLINGTON TERRIER +BORDER TERRIER +KERRY BLUE TERRIER +IRISH TERRIER +NORFOLK TERRIER +NORWICH TERRIER +YORKSHIRE TERRIER +WIRE-HAIRED FOX TERRIER +LAKELAND TERRIER +SEALYHAM TERRIER +AIREDALE +CAIRN +AUSTRALIAN TERRIER +DANDIE DINMONT +BOSTON BULL +MINIATURE SCHNAUZER +GIANT SCHNAUZER +STANDARD SCHNAUZER +SCOTCH TERRIER +TIBETAN TERRIER +SILKY TERRIER +WHEATEN TERRIER +WHITE TERRIER +LHASA +RETRIEVER +CURLY-COATED RETRIEVER +GOLDEN RETRIEVER +LABRADOR RETRIEVER +CHESAPEAKE BAY RETRIEVER +SHORT-HAIRED POINTER +VISLA +ENGLISH SETTER +IRISH SETTER +GORDON SETTER +BRITTANY SPANIEL +CLUMBER +ENGLISH SPRINGER +WELSH SPRINGER SPANIEL +COCKER SPANIEL +SUSSEX SPANIEL +IRISH WATERSPANIEL +KUVASZ +SCHIPPERKE +GROENENDAEL +MALINOIS +BRIARD +KELPIE +KOMONDOR +OLD ENGLISH SHEEPDOG +SHETLAND SHEEPDOG +COLLIE +BORDER COLLIE +BOUVIER DES FLANDRES +ROTTWEILER +GERMAN SHEPHERD +DOBERMAN +MINIATURE PINSCHER +GREATER SWISS MOUNTAIN DOG +BERNESE MOUNTAIN DOG +APPENZELLER +ENTLEBUCHER +BOXER +BULL MASTIFF +TIBETAN MASTIFF +FRENCH BULLDOG +GREAT DANE +SAINT BERNARD +ESKIMO DOG +MALAMUTE +SIBERIAN HUSKY +DALMATIAN +AFFENPINSCHER +BASENJI +PUG +LEONBERG +NEWFOUNDLAND +GREAT PYRENEES +SAMOYED +POMERANIAN +CHOW +KEESHOND +BRABANCON GRIFFON +PEMBROKE +CARDIGAN +TOY POODLE +MINIATURE POODLE +STANDARD POODLE +MEXICAN HAIRLESS +TIMBER WOLF +WHITE WOLF +RED WOLF +COYOTE +DINGO +DHOLE +AFRICAN HUNTING DOG +HYENA +RED FOX +KIT FOX +ARCTIC FOX +GREY FOX +TABBY +TIGER CAT +PERSIAN CAT +SIAMESE CAT +EGYPTIAN CAT +COUGAR +LYNX +LEOPARD +SNOW LEOPARD +JAGUAR +LION +TIGER +CHEETAH +BROWN BEAR +AMERICAN BLACK BEAR +ICE BEAR +SLOTH BEAR +MONGOOSE +MEERKAT +TIGER BEETLE +LADYBUG +GROUND BEETLE +LONG-HORNED BEETLE +LEAF BEETLE +DUNG BEETLE +RHINOCEROS BEETLE +WEEVIL +FLY +BEE +ANT +GRASSHOPPER +CRICKET +WALKING STICK +COCKROACH +MANTIS +CICADA +LEAFHOPPER +LACEWING +DRAGONFLY +DAMSELFLY +ADMIRAL +RINGLET +MONARCH +CABBAGE BUTTERFLY +SULPHUR BUTTERFLY +LYCAENID +STARFISH +SEA URCHIN +SEA CUCUMBER +WOOD RABBIT +HARE +ANGORA +HAMSTER +PORCUPINE +FOX SQUIRREL +MARMOT +BEAVER +GUINEA PIG +SORREL +ZEBRA +HOG +WILD BOAR +WARTHOG +HIPPOPOTAMUS +OX +WATER BUFFALO +BISON +RAM +BIGHORN +IBEX +HARTEBEEST +IMPALA +GAZELLE +ARABIAN CAMEL +LLAMA +WEASEL +MINK +POLECAT +BLACK-FOOTED FERRET +OTTER +SKUNK +BADGER +ARMADILLO +THREE-TOED SLOTH +ORANGUTAN +GORILLA +CHIMPANZEE +GIBBON +SIAMANG +GUENON +PATAS +BABOON +MACAQUE +LANGUR +COLOBUS +PROBOSCIS MONKEY +MARMOSET +CAPUCHIN +HOWLER MONKEY +TITI +SPIDER MONKEY +SQUIRREL MONKEY +MADAGASCAR CAT +INDRI +INDIAN ELEPHANT +AFRICAN ELEPHANT +LESSER PANDA +GIANT PANDA +BARRACOUTA +EEL +COHO +ROCK BEAUTY +ANEMONE FISH +STURGEON +GAR +LIONFISH +PUFFER +ABACUS +ABAYA +ACADEMIC GOWN +ACCORDION +ACOUSTIC GUITAR +AIRCRAFT CARRIER +AIRLINER +AIRSHIP +ALTAR +AMBULANCE +AMPHIBIAN +ANALOG CLOCK +APIARY +APRON +ASHCAN +ASSAULT RIFLE +BACKPACK +BAKERY +BALANCE BEAM +BALLOON +BALLPOINT +BAND AID +BANJO +BANNISTER +BARBELL +BARBER CHAIR +BARBERSHOP +BARN +BAROMETER +BARREL +BARROW +BASEBALL +BASKETBALL +BASSINET +BASSOON +BATHING CAP +BATH TOWEL +BATHTUB +BEACH WAGON +BEACON +BEAKER +BEARSKIN +BEER BOTTLE +BEER GLASS +BELL COTE +BIB +BICYCLE-BUILT-FOR-TWO +BIKINI +BINDER +BINOCULARS +BIRDHOUSE +BOATHOUSE +BOBSLED +BOLO TIE +BONNET +BOOKCASE +BOOKSHOP +BOTTLECAP +BOW +BOW TIE +BRASS +BRASSIERE +BREAKWATER +BREASTPLATE +BROOM +BUCKET +BUCKLE +BULLETPROOF VEST +BULLET TRAIN +BUTCHER SHOP +CAB +CALDRON +CANDLE +CANNON +CANOE +CAN OPENER +CARDIGAN +CAR MIRROR +CAROUSEL +CARPENTERS KIT +CARTON +CAR WHEEL +CASH MACHINE +CASSETTE +CASSETTE PLAYER +CASTLE +CATAMARAN +CD PLAYER +CELLO +CELLULAR TELEPHONE +CHAIN +CHAINLINK FENCE +CHAIN MAIL +CHAIN SAW +CHEST +CHIFFONIER +CHIME +CHINA CABINET +CHRISTMAS STOCKING +CHURCH +CINEMA +CLEAVER +CLIFF DWELLING +CLOAK +CLOG +COCKTAIL SHAKER +COFFEE MUG +COFFEEPOT +COIL +COMBINATION LOCK +COMPUTER KEYBOARD +CONFECTIONERY +CONTAINER SHIP +CONVERTIBLE +CORKSCREW +CORNET +COWBOY BOOT +COWBOY HAT +CRADLE +CRANE +CRASH HELMET +CRATE +CRIB +CROCK POT +CROQUET BALL +CRUTCH +CUIRASS +DAM +DESK +DESKTOP COMPUTER +DIAL TELEPHONE +DIAPER +DIGITAL CLOCK +DIGITAL WATCH +DINING TABLE +DISHRAG +DISHWASHER +DISK BRAKE +DOCK +DOGSLED +DOME +DOORMAT +DRILLING PLATFORM +DRUM +DRUMSTICK +DUMBBELL +DUTCH OVEN +ELECTRIC FAN +ELECTRIC GUITAR +ELECTRIC LOCOMOTIVE +ENTERTAINMENT CENTER +ENVELOPE +ESPRESSO MAKER +FACE POWDER +FEATHER BOA +FILE +FIREBOAT +FIRE ENGINE +FIRE SCREEN +FLAGPOLE +FLUTE +FOLDING CHAIR +FOOTBALL HELMET +FORKLIFT +FOUNTAIN +FOUNTAIN PEN +FOUR-POSTER +FREIGHT CAR +FRENCH HORN +FRYING PAN +FUR COAT +GARBAGE TRUCK +GASMASK +GAS PUMP +GOBLET +GO-KART +GOLF BALL +GOLFCART +GONDOLA +GONG +GOWN +GRAND PIANO +GREENHOUSE +GRILLE +GROCERY STORE +GUILLOTINE +HAIR SLIDE +HAIR SPRAY +HALF TRACK +HAMMER +HAMPER +HAND BLOWER +HAND-HELD COMPUTER +HANDKERCHIEF +HARD DISC +HARMONICA +HARP +HARVESTER +HATCHET +HOLSTER +HOME THEATER +HONEYCOMB +HOOK +HOOPSKIRT +HORIZONTAL BAR +HORSE CART +HOURGLASS +IPOD +IRON +JACK-O-LANTERN +JEAN +JEEP +JERSEY +JIGSAW PUZZLE +JINRIKISHA +JOYSTICK +KIMONO +KNEE PAD +KNOT +LAB COAT +LADLE +LAMPSHADE +LAPTOP +LAWN MOWER +LENS CAP +LETTER OPENER +LIBRARY +LIFEBOAT +LIGHTER +LIMOUSINE +LINER +LIPSTICK +LOAFER +LOTION +LOUDSPEAKER +LOUPE +LUMBERMILL +MAGNETIC COMPASS +MAILBAG +MAILBOX +MAILLOT +MAILLOT +MANHOLE COVER +MARACA +MARIMBA +MASK +MATCHSTICK +MAYPOLE +MAZE +MEASURING CUP +MEDICINE CHEST +MEGALITH +MICROPHONE +MICROWAVE +MILITARY UNIFORM +MILK CAN +MINIBUS +MINISKIRT +MINIVAN +MISSILE +MITTEN +MIXING BOWL +MOBILE HOME +MODEL T +MODEM +MONASTERY +MONITOR +MOPED +MORTAR +MORTARBOARD +MOSQUE +MOSQUITO NET +MOTOR SCOOTER +MOUNTAIN BIKE +MOUNTAIN TENT +MOUSE +MOUSETRAP +MOVING VAN +MUZZLE +NAIL +NECK BRACE +NECKLACE +NIPPLE +NOTEBOOK +OBELISK +OBOE +OCARINA +ODOMETER +OIL FILTER +ORGAN +OSCILLOSCOPE +OVERSKIRT +OXCART +OXYGEN MASK +PACKET +PADDLE +PADDLEWHEEL +PADLOCK +PAINTBRUSH +PAJAMA +PALACE +PANPIPE +PAPER TOWEL +PARACHUTE +PARALLEL BARS +PARK BENCH +PARKING METER +PASSENGER CAR +PATIO +PAY-PHONE +PEDESTAL +PENCIL BOX +PENCIL SHARPENER +PERFUME +PETRI DISH +PHOTOCOPIER +PICK +PICKELHAUBE +PICKET FENCE +PICKUP +PIER +PIGGY BANK +PILL BOTTLE +PILLOW +PING-PONG BALL +PINWHEEL +PIRATE +PITCHER +PLANE +PLANETARIUM +PLASTIC BAG +PLATE RACK +PLOW +PLUNGER +POLAROID CAMERA +POLE +POLICE VAN +PONCHO +POOL TABLE +POP BOTTLE +POT +POTTERS WHEEL +POWER DRILL +PRAYER RUG +PRINTER +PRISON +PROJECTILE +PROJECTOR +PUCK +PUNCHING BAG +PURSE +QUILL +QUILT +RACER +RACKET +RADIATOR +RADIO +RADIO TELESCOPE +RAIN BARREL +RECREATIONAL VEHICLE +REEL +REFLEX CAMERA +REFRIGERATOR +REMOTE CONTROL +RESTAURANT +REVOLVER +RIFLE +ROCKING CHAIR +ROTISSERIE +RUBBER ERASER +RUGBY BALL +RULE +RUNNING SHOE +SAFE +SAFETY PIN +SALTSHAKER +SANDAL +SARONG +SAX +SCABBARD +SCALE +SCHOOL BUS +SCHOONER +SCOREBOARD +SCREEN +SCREW +SCREWDRIVER +SEAT BELT +SEWING MACHINE +SHIELD +SHOE SHOP +SHOJI +SHOPPING BASKET +SHOPPING CART +SHOVEL +SHOWER CAP +SHOWER CURTAIN +SKI +SKI MASK +SLEEPING BAG +SLIDE RULE +SLIDING DOOR +SLOT +SNORKEL +SNOWMOBILE +SNOWPLOW +SOAP DISPENSER +SOCCER BALL +SOCK +SOLAR DISH +SOMBRERO +SOUP BOWL +SPACE BAR +SPACE HEATER +SPACE SHUTTLE +SPATULA +SPEEDBOAT +SPIDER WEB +SPINDLE +SPORTS CAR +SPOTLIGHT +STAGE +STEAM LOCOMOTIVE +STEEL ARCH BRIDGE +STEEL DRUM +STETHOSCOPE +STOLE +STONE WALL +STOPWATCH +STOVE +STRAINER +STREETCAR +STRETCHER +STUDIO COUCH +STUPA +SUBMARINE +SUIT +SUNDIAL +SUNGLASS +SUNGLASSES +SUNSCREEN +SUSPENSION BRIDGE +SWAB +SWEATSHIRT +SWIMMING TRUNKS +SWING +SWITCH +SYRINGE +TABLE LAMP +TANK +TAPE PLAYER +TEAPOT +TEDDY +TELEVISION +TENNIS BALL +THATCH +THEATER CURTAIN +THIMBLE +THRESHER +THRONE +TILE ROOF +TOASTER +TOBACCO SHOP +TOILET SEAT +TORCH +TOTEM POLE +TOW TRUCK +TOYSHOP +TRACTOR +TRAILER TRUCK +TRAY +TRENCH COAT +TRICYCLE +TRIMARAN +TRIPOD +TRIUMPHAL ARCH +TROLLEYBUS +TROMBONE +TUB +TURNSTILE +TYPEWRITER KEYBOARD +UMBRELLA +UNICYCLE +UPRIGHT +VACUUM +VASE +VAULT +VELVET +VENDING MACHINE +VESTMENT +VIADUCT +VIOLIN +VOLLEYBALL +WAFFLE IRON +WALL CLOCK +WALLET +WARDROBE +WARPLANE +WASHBASIN +WASHER +WATER BOTTLE +WATER JUG +WATER TOWER +WHISKEY JUG +WHISTLE +WIG +WINDOW SCREEN +WINDOW SHADE +WINDSOR TIE +WINE BOTTLE +WING +WOK +WOODEN SPOON +WOOL +WORM FENCE +WRECK +YAWL +YURT +WEB SITE +COMIC BOOK +CROSSWORD PUZZLE +STREET SIGN +TRAFFIC LIGHT +BOOK JACKET +MENU +PLATE +GUACAMOLE +CONSOMME +HOT POT +TRIFLE +ICE CREAM +ICE LOLLY +FRENCH LOAF +BAGEL +PRETZEL +CHEESEBURGER +HOTDOG +MASHED POTATO +HEAD CABBAGE +BROCCOLI +CAULIFLOWER +ZUCCHINI +SPAGHETTI SQUASH +ACORN SQUASH +BUTTERNUT SQUASH +CUCUMBER +ARTICHOKE +BELL PEPPER +CARDOON +MUSHROOM +GRANNY SMITH +STRAWBERRY +ORANGE +LEMON +FIG +PINEAPPLE +BANANA +JACKFRUIT +CUSTARD APPLE +POMEGRANATE +HAY +CARBONARA +CHOCOLATE SAUCE +DOUGH +MEAT LOAF +PIZZA +POTPIE +BURRITO +RED WINE +ESPRESSO +CUP +EGGNOG +ALP +BUBBLE +CLIFF +CORAL REEF +GEYSER +LAKESIDE +PROMONTORY +SANDBAR +SEASHORE +VALLEY +VOLCANO +BALLPLAYER +GROOM +SCUBA DIVER +RAPESEED +DAISY +LADY SLIPPER +CORN +ACORN +HIP +BUCKEYE +CORAL FUNGUS +AGARIC +GYROMITRA +STINKHORN +EARTHSTAR +HEN-OF-THE-WOODS +BOLETE +EAR +TOILET TISSUE diff --git a/examples/tls/Dockerfile b/examples/tls/Dockerfile new file mode 100644 index 0000000..fb8b0b2 --- /dev/null +++ b/examples/tls/Dockerfile @@ -0,0 +1,9 @@ +FROM nvcr.io/nvidia/tritonserver:23.06-py3-sdk + +RUN apt install -y lsb-release curl gpg +RUN curl -fsSL https://packages.redis.io/gpg | gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg +RUN echo "deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main" | tee /etc/apt/sources.list.d/redis.list +RUN apt-get update +RUN apt-get install -y redis + +CMD "/app/test.sh" \ No newline at end of file diff --git a/examples/tls/README.md b/examples/tls/README.md new file mode 100644 index 0000000..d35635b --- /dev/null +++ b/examples/tls/README.md @@ -0,0 +1,16 @@ +# TLS Testing Environment + +This environment demonstrates how to run the Triton Redis Cache against a TLS enabled Redis instance. + +## How to Run + +* clone this repo: `git clone https://github.com/triton-inference-server/redis_cache` +* follow build instructions enumerated [in the README](https://github.com/triton-inference-server/redis_cache#build-the-cache) +* cd into `redis_cache/docker/tls` +* run `sh run-tls.sh` +* In a separate terminal run `docker run -it --rm --net=host nvcr.io/nvidia/tritonserver:23.06-py3-sdk` +* Run `/workspace/install/bin/image_client -m densenet_onnx -c 3 -s INCEPTION /workspace/images/mug.jpg` + * on the first run - this will miss the cache + * subsequent runs will pull the inference out of the cache + * you can validate this by checking what's in Redis with `docker exec -it tls_triton-redis_1 redis-cli --tls --cert /certs/redis.crt --key /certs/redis.key --cacert /certs/ca.crt SCAN 0` +* You can use the Redis CLI to talk to redis by running `docker exec -it tls_triton-redis_1 redis-cli --tls --cert /certs/redis.crt --key /certs/redis.key --cacert /certs/ca.crt` \ No newline at end of file diff --git a/examples/tls/conf/redis.conf b/examples/tls/conf/redis.conf new file mode 100644 index 0000000..a1046d7 --- /dev/null +++ b/examples/tls/conf/redis.conf @@ -0,0 +1,6 @@ +port 0 +tls-port 6379 + +tls-cert-file /certs/redis.crt +tls-key-file /certs/redis.key +tls-ca-cert-file /certs/ca.crt \ No newline at end of file diff --git a/examples/tls/docker-compose.yml b/examples/tls/docker-compose.yml new file mode 100644 index 0000000..bbb2c92 --- /dev/null +++ b/examples/tls/docker-compose.yml @@ -0,0 +1,33 @@ +version: "3.7" + +services: + client: + build: + context: . + command: bash -c "sh /app/test.sh" + links: + - triton-server + volumes: + - .:/app + - ./certs:/certs + triton-redis: + image: redis + command: "redis-server /conf/redis.conf" + ports: + - '6379:6379' + volumes: + - ./certs:/certs + - ./conf:/conf + triton-server: + links: + - triton-redis + build: + context: ../.. + command: tritonserver --cache-config redis,host=triton-redis --cache-config redis,port=6379 --cache-config redis,tls_enabled=true --cache-config redis,cert=/certs/redis.crt --cache-config redis,key=/certs/redis.key --cache-config redis,cacert=/certs/ca.crt --model-repository=/models + ports: + - '8000:8000' + - '8001:8001' + - '8002:8002' + volumes: + - ../model_repository:/models + - ./certs:/certs diff --git a/examples/tls/gen-certs.sh b/examples/tls/gen-certs.sh new file mode 100755 index 0000000..2ea4bb3 --- /dev/null +++ b/examples/tls/gen-certs.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +# lightly modified from redis' `gen-test-certs.sh` https://github.com/redis/redis/blob/93708c7f6a0e702657e4f296ea6fc299225eea8d/utils/gen-test-certs.sh +generate_cert() { + local name=$1 + local cn="$2" + local opts="$3" + + local keyfile=certs/${name}.key + local certfile=certs/${name}.crt + + [ -f $keyfile ] || openssl genrsa -out $keyfile 2048 + openssl req \ + -new -sha256 \ + -subj "/O=Redis Test/CN=$cn" \ + -key $keyfile | \ + openssl x509 \ + -req -sha256 \ + -CA certs/ca.crt \ + -CAkey certs/ca.key \ + -CAserial certs/ca.txt \ + -CAcreateserial \ + -days 365 \ + $opts \ + -out $certfile +} + +mkdir -p certs +[ -f certs/ca.key ] || openssl genrsa -out certs/ca.key 4096 +openssl req \ + -x509 -new -nodes -sha256 \ + -key certs/ca.key \ + -days 3650 \ + -subj '/O=Redis Test/CN=Certificate Authority' \ + -out certs/ca.crt + +cat > certs/openssl.cnf <<_END_ +[ server_cert ] +keyUsage = digitalSignature, keyEncipherment +nsCertType = server + +[ client_cert ] +keyUsage = digitalSignature, keyEncipherment +nsCertType = client +_END_ + +generate_cert server "Server-only" "-extfile certs/openssl.cnf -extensions server_cert" +generate_cert client "Client-only" "-extfile certs/openssl.cnf -extensions client_cert" +generate_cert redis "Generic-cert" + +[ -f certs/redis.dh ] || openssl dhparam -out certs/redis.dh 2048 \ No newline at end of file diff --git a/examples/tls/run-tls.sh b/examples/tls/run-tls.sh new file mode 100755 index 0000000..2512651 --- /dev/null +++ b/examples/tls/run-tls.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +bash gen-certs.sh +chmod +r ./certs/ca.crt ./certs/redis.crt ./certs/redis.key +cd .. +bash ./fetch_model.sh +cd tls +docker-compose build +docker-compose up \ No newline at end of file diff --git a/examples/tls/test.sh b/examples/tls/test.sh new file mode 100755 index 0000000..035fffb --- /dev/null +++ b/examples/tls/test.sh @@ -0,0 +1,17 @@ +#!/bin/bash +sleep 5 +classificationResult=$(/workspace/install/bin/image_client -u http://triton-server:8000 -m densenet_onnx -c 3 -s INCEPTION /workspace/images/mug.jpg | grep "COFFEE MUG") + +if [ -z "$classificationResult" ]; then + echo "Classification failed" + exit 1 +fi + +numKeys=$(redis-cli -h triton-redis --tls --cert /certs/redis.crt --key /certs/redis.key --cacert /certs/ca.crt DBSIZE) # check that there's only one key + +if [[ $numKeys -eq 1 ]]; then + exit 0 +else + echo "Redis did not have the expected number of keys." + exit 1 +fi \ No newline at end of file diff --git a/src/redis_cache.cc b/src/redis_cache.cc index 138939a..007435e 100644 --- a/src/redis_cache.cc +++ b/src/redis_cache.cc @@ -146,6 +146,17 @@ RedisCache::Create( poolOptions.wait_timeout = std::chrono::milliseconds(1000); } + // tls options + if (document.HasMember("tls_enabled")) { + options.tls.enabled = + strcmp(document["tls_enabled"].GetString(), "true") == 0; + setOption("cert", options.tls.cert, document); + setOption("key", options.tls.key, document); + setOption("cacert", options.tls.cacert, document); + setOption("cacert_dir", options.tls.cacertdir, document); + setOption("sni", options.tls.sni, document); + } + try { cache->reset(new RedisCache(options, poolOptions)); }