From 0bdd8db46b09357dba6d0ae859f479edf0c816bc Mon Sep 17 00:00:00 2001 From: Tom Stesco Date: Mon, 3 Feb 2025 14:38:55 -0500 Subject: [PATCH] add CONTAINER_APP_UID Docker ARG for cloud image and dev image with develoment instructions (#94) * remove git lfs from tt-metal build * add CONTAINER_APP_UID Docker ARG for cloud image and dev image with develoment instructions --- vllm-tt-metal-llama3/docs/development.md | 10 +- .../vllm.llama3.src.cloud.Dockerfile | 123 ++++++++++++++++++ .../vllm.llama3.src.dev.Dockerfile | 115 +--------------- 3 files changed, 135 insertions(+), 113 deletions(-) create mode 100644 vllm-tt-metal-llama3/vllm.llama3.src.cloud.Dockerfile diff --git a/vllm-tt-metal-llama3/docs/development.md b/vllm-tt-metal-llama3/docs/development.md index 0756e8f..80ff15a 100644 --- a/vllm-tt-metal-llama3/docs/development.md +++ b/vllm-tt-metal-llama3/docs/development.md @@ -19,12 +19,20 @@ export TT_METAL_COMMIT_SHA_OR_TAG=47fb1a2fb6e0b62ddfe3fc5fef95c18d4b857c20 export TT_METAL_COMMIT_DOCKER_TAG=${TT_METAL_COMMIT_SHA_OR_TAG:0:12} export TT_VLLM_COMMIT_SHA_OR_TAG=2f33504bad49a6202d3685155107a6126a5b5e6e export TT_VLLM_COMMIT_DOCKER_TAG=${TT_VLLM_COMMIT_SHA_OR_TAG:0:12} +export CONTAINER_APP_UID=1000 export IMAGE_VERSION=v0.0.1 +# build cloud deploy image docker build \ - -t ghcr.io/tenstorrent/tt-inference-server/vllm-llama3-src-dev-${OS_VERSION}:${IMAGE_VERSION}-${TT_METAL_COMMIT_DOCKER_TAG}-${TT_VLLM_COMMIT_DOCKER_TAG} \ + -t ghcr.io/tenstorrent/tt-inference-server/vllm-llama3-src-cloud-${OS_VERSION}:${IMAGE_VERSION}-${TT_METAL_COMMIT_DOCKER_TAG}-${TT_VLLM_COMMIT_DOCKER_TAG} \ --build-arg TT_METAL_DOCKERFILE_URL=${TT_METAL_DOCKERFILE_URL} \ --build-arg TT_METAL_COMMIT_SHA_OR_TAG=${TT_METAL_COMMIT_SHA_OR_TAG} \ --build-arg TT_VLLM_COMMIT_SHA_OR_TAG=${TT_VLLM_COMMIT_SHA_OR_TAG} \ + --build-arg CONTAINER_APP_UID=${CONTAINER_APP_UID} \ + . -f vllm-tt-metal-llama3/vllm.llama3.src.cloud.Dockerfile +# build dev image +docker build \ + -t ghcr.io/tenstorrent/tt-inference-server/vllm-llama3-src-dev-${OS_VERSION}:${IMAGE_VERSION}-${TT_METAL_COMMIT_DOCKER_TAG}-${TT_VLLM_COMMIT_DOCKER_TAG} \ + --build-arg CLOUD_DOCKERFILE_URL=ghcr.io/tenstorrent/tt-inference-server/vllm-llama3-src-cloud-${OS_VERSION}:${IMAGE_VERSION}-${TT_METAL_COMMIT_DOCKER_TAG}-${TT_VLLM_COMMIT_DOCKER_TAG} \ . -f vllm-tt-metal-llama3/vllm.llama3.src.dev.Dockerfile ``` diff --git a/vllm-tt-metal-llama3/vllm.llama3.src.cloud.Dockerfile b/vllm-tt-metal-llama3/vllm.llama3.src.cloud.Dockerfile new file mode 100644 index 0000000..8c0f123 --- /dev/null +++ b/vllm-tt-metal-llama3/vllm.llama3.src.cloud.Dockerfile @@ -0,0 +1,123 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# SPDX-FileCopyrightText: © 2024 Tenstorrent AI ULC + +# default base image, override with --build-arg TT_METAL_DOCKERFILE_URL= +# NOTE: tt-metal Ubuntu 22.04 Dockerfile must be built locally until release images are published +ARG TT_METAL_DOCKERFILE_URL + +FROM ${TT_METAL_DOCKERFILE_URL} + +# shared build stage, FROM is set by the OS specific Dockerfiles +LABEL maintainer="Tom Stesco " +# connect Github repo with package +LABEL org.opencontainers.image.source=https://github.com/tenstorrent/tt-inference-server + +# must set commit SHAs +ARG TT_METAL_COMMIT_SHA_OR_TAG +ARG TT_VLLM_COMMIT_SHA_OR_TAG + +# CONTAINER_APP_UID is a random ID, change this and rebuild if it collides with host +ARG CONTAINER_APP_UID=15863 +ARG DEBIAN_FRONTEND=noninteractive +# make build commit SHA available in the image for reference and debugging +ENV TT_METAL_COMMIT_SHA_OR_TAG=${TT_METAL_COMMIT_SHA_OR_TAG} +ENV SHELL=/bin/bash +ENV TZ=America/Los_Angeles +# tt-metal build vars +ENV ARCH_NAME=wormhole_b0 +ENV TT_METAL_HOME=/tt-metal +ENV CONFIG=Release +ENV TT_METAL_ENV=dev +ENV LOGURU_LEVEL=INFO +# derived vars +ENV PYTHONPATH=${TT_METAL_HOME} +# note: PYTHON_ENV_DIR is used by create_venv.sh +ENV PYTHON_ENV_DIR=${TT_METAL_HOME}/python_env +ENV LD_LIBRARY_PATH=${TT_METAL_HOME}/build/lib + +# apt-get might have an outdated keyring, preventing it to download packages, so we fetch the latest +RUN wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null \ + && echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ focal main' | tee /etc/apt/sources.list.d/kitware.list >/dev/null + +# extra system deps +RUN apt-get update && apt-get install -y \ + # required + gosu \ + # extra tt-metal TODO: remove as non longer needed + libsndfile1 \ + wget \ + nano \ + acl \ + jq \ + vim \ + # user deps + htop \ + screen \ + tmux \ + unzip \ + zip \ + curl \ + iputils-ping \ + rsync \ + && rm -rf /var/lib/apt/lists/* + +# build tt-metal +RUN git clone https://github.com/tenstorrent-metal/tt-metal.git ${TT_METAL_HOME} \ + && cd ${TT_METAL_HOME} \ + && git checkout ${TT_METAL_COMMIT_SHA_OR_TAG} \ + && git submodule update --init --recursive \ + && bash ./build_metal.sh \ + && bash ./create_venv.sh + +# user setup +ENV CONTAINER_APP_USERNAME=container_app_user +ARG HOME_DIR=/home/${CONTAINER_APP_USERNAME} +RUN useradd -u ${CONTAINER_APP_UID} -s /bin/bash -d ${HOME_DIR} ${CONTAINER_APP_USERNAME} \ + && mkdir -p ${HOME_DIR} \ + && chown -R ${CONTAINER_APP_USERNAME}:${CONTAINER_APP_USERNAME} ${HOME_DIR} \ + && chown -R ${CONTAINER_APP_USERNAME}:${CONTAINER_APP_USERNAME} ${TT_METAL_HOME} + +USER ${CONTAINER_APP_USERNAME} + +# tt-metal python env default +RUN echo "source ${PYTHON_ENV_DIR}/bin/activate" >> ${HOME_DIR}/.bashrc + +# install tt-smi +RUN /bin/bash -c "source ${PYTHON_ENV_DIR}/bin/activate \ + && pip3 install --upgrade pip \ + && pip3 install git+https://github.com/tenstorrent/tt-smi" + +# runtime required for tt-metal on WH +ENV WH_ARCH_YAML=wormhole_b0_80_arch_eth_dispatch.yaml + +WORKDIR ${HOME_DIR} +# vllm install, see: https://github.com/tenstorrent/vllm/blob/dev/tt_metal/README.md +ENV vllm_dir=${HOME_DIR}/vllm +ENV PYTHONPATH=${PYTHONPATH}:${vllm_dir} +ENV VLLM_TARGET_DEVICE="tt" +RUN git clone https://github.com/tenstorrent/vllm.git ${vllm_dir}\ + && cd ${vllm_dir} && git checkout ${TT_VLLM_COMMIT_SHA_OR_TAG} \ + && /bin/bash -c "source ${PYTHON_ENV_DIR}/bin/activate && pip install -e ." + +# extra vllm and model dependencies +RUN /bin/bash -c "source ${PYTHON_ENV_DIR}/bin/activate \ + && pip install compressed-tensors \ + && pip install -r /tt-metal/models/demos/llama3/requirements.txt" + +ARG APP_DIR="${HOME_DIR}/app" +WORKDIR ${APP_DIR} +ENV PYTHONPATH=${PYTHONPATH}:${APP_DIR} +COPY --chown=${CONTAINER_APP_USERNAME}:${CONTAINER_APP_USERNAME} "vllm-tt-metal-llama3/src" "${APP_DIR}/src" +COPY --chown=${CONTAINER_APP_USERNAME}:${CONTAINER_APP_USERNAME} "vllm-tt-metal-llama3/requirements.txt" "${APP_DIR}/requirements.txt" +COPY --chown=${CONTAINER_APP_USERNAME}:${CONTAINER_APP_USERNAME} "utils" "${APP_DIR}/utils" +COPY --chown=${CONTAINER_APP_USERNAME}:${CONTAINER_APP_USERNAME} "benchmarking" "${APP_DIR}/benchmarking" +COPY --chown=${CONTAINER_APP_USERNAME}:${CONTAINER_APP_USERNAME} "evals" "${APP_DIR}/evals" +COPY --chown=${CONTAINER_APP_USERNAME}:${CONTAINER_APP_USERNAME} "tests" "${APP_DIR}/tests" +COPY --chown=${CONTAINER_APP_USERNAME}:${CONTAINER_APP_USERNAME} "locust" "${APP_DIR}/locust" +RUN /bin/bash -c "source ${PYTHON_ENV_DIR}/bin/activate \ +&& pip install --default-timeout=240 --no-cache-dir -r requirements.txt" + +WORKDIR "${APP_DIR}/src" + +CMD ["/bin/bash", "-c", "source ${PYTHON_ENV_DIR}/bin/activate && python run_vllm_api_server.py"] diff --git a/vllm-tt-metal-llama3/vllm.llama3.src.dev.Dockerfile b/vllm-tt-metal-llama3/vllm.llama3.src.dev.Dockerfile index 005deb0..7d29cc9 100644 --- a/vllm-tt-metal-llama3/vllm.llama3.src.dev.Dockerfile +++ b/vllm-tt-metal-llama3/vllm.llama3.src.dev.Dockerfile @@ -2,125 +2,16 @@ # # SPDX-FileCopyrightText: © 2024 Tenstorrent AI ULC -# default base image, override with --build-arg TT_METAL_DOCKERFILE_URL= -# NOTE: tt-metal Ubuntu 22.04 Dockerfile must be built locally until release images are published -ARG TT_METAL_DOCKERFILE_URL +# default base image, override with --build-arg CLOUD_DOCKERFILE_URL= +ARG CLOUD_DOCKERFILE_URL -FROM ${TT_METAL_DOCKERFILE_URL} +FROM ${CLOUD_DOCKERFILE_URL} # shared build stage, FROM is set by the OS specific Dockerfiles LABEL maintainer="Tom Stesco " # connect Github repo with package LABEL org.opencontainers.image.source=https://github.com/tenstorrent/tt-inference-server -# must set commit SHAs -ARG TT_METAL_COMMIT_SHA_OR_TAG -ARG TT_VLLM_COMMIT_SHA_OR_TAG - -ARG DEBIAN_FRONTEND=noninteractive -# make build commit SHA available in the image for reference and debugging -ENV TT_METAL_COMMIT_SHA_OR_TAG=${TT_METAL_COMMIT_SHA_OR_TAG} -ENV SHELL=/bin/bash -ENV TZ=America/Los_Angeles -# tt-metal build vars -ENV ARCH_NAME=wormhole_b0 -ENV TT_METAL_HOME=/tt-metal -ENV CONFIG=Release -ENV TT_METAL_ENV=dev -ENV LOGURU_LEVEL=INFO -# derived vars -ENV PYTHONPATH=${TT_METAL_HOME} -# note: PYTHON_ENV_DIR is used by create_venv.sh -ENV PYTHON_ENV_DIR=${TT_METAL_HOME}/python_env -ENV LD_LIBRARY_PATH=${TT_METAL_HOME}/build/lib - -# apt-get might have an outdated keyring, preventing it to download packages, so we fetch the latest -RUN wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null \ - && echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ focal main' | tee /etc/apt/sources.list.d/kitware.list >/dev/null - -# extra system deps -RUN apt-get update && apt-get install -y \ - # required - gosu \ - # extra tt-metal TODO: remove as non longer needed - libsndfile1 \ - wget \ - nano \ - acl \ - jq \ - vim \ - # user deps - htop \ - screen \ - tmux \ - unzip \ - zip \ - curl \ - iputils-ping \ - rsync \ - && rm -rf /var/lib/apt/lists/* - -# build tt-metal -RUN git clone https://github.com/tenstorrent-metal/tt-metal.git ${TT_METAL_HOME} \ - && cd ${TT_METAL_HOME} \ - && git checkout ${TT_METAL_COMMIT_SHA_OR_TAG} \ - && git submodule update --init --recursive \ - && git submodule foreach 'git lfs fetch --all && git lfs pull' \ - && bash ./build_metal.sh \ - && bash ./create_venv.sh - -# user setup -# CONTAINER_APP_UID is a random ID, change this and rebuild if it collides with host -ENV CONTAINER_APP_UID=15863 -ENV CONTAINER_APP_USERNAME=container_app_user -ARG HOME_DIR=/home/${CONTAINER_APP_USERNAME} -RUN useradd -u ${CONTAINER_APP_UID} -s /bin/bash -d ${HOME_DIR} ${CONTAINER_APP_USERNAME} \ - && mkdir -p ${HOME_DIR} \ - && chown -R ${CONTAINER_APP_USERNAME}:${CONTAINER_APP_USERNAME} ${HOME_DIR} \ - && chown -R ${CONTAINER_APP_USERNAME}:${CONTAINER_APP_USERNAME} ${TT_METAL_HOME} - -USER ${CONTAINER_APP_USERNAME} - -# tt-metal python env default -RUN echo "source ${PYTHON_ENV_DIR}/bin/activate" >> ${HOME_DIR}/.bashrc - -# install tt-smi -RUN /bin/bash -c "source ${PYTHON_ENV_DIR}/bin/activate \ - && pip3 install --upgrade pip \ - && pip3 install git+https://github.com/tenstorrent/tt-smi" - -# runtime required for tt-metal on WH -ENV WH_ARCH_YAML=wormhole_b0_80_arch_eth_dispatch.yaml - -WORKDIR ${HOME_DIR} -# vllm install, see: https://github.com/tenstorrent/vllm/blob/dev/tt_metal/README.md -ENV vllm_dir=${HOME_DIR}/vllm -ENV PYTHONPATH=${PYTHONPATH}:${vllm_dir} -ENV VLLM_TARGET_DEVICE="tt" -RUN git clone https://github.com/tenstorrent/vllm.git ${vllm_dir}\ - && cd ${vllm_dir} && git checkout ${TT_VLLM_COMMIT_SHA_OR_TAG} \ - && /bin/bash -c "source ${PYTHON_ENV_DIR}/bin/activate && pip install -e ." - -# extra vllm and model dependencies -RUN /bin/bash -c "source ${PYTHON_ENV_DIR}/bin/activate \ - && pip install compressed-tensors \ - && pip install -r /tt-metal/models/demos/llama3/requirements.txt" - -ARG APP_DIR="${HOME_DIR}/app" -WORKDIR ${APP_DIR} -ENV PYTHONPATH=${PYTHONPATH}:${APP_DIR} -COPY --chown=${CONTAINER_APP_USERNAME}:${CONTAINER_APP_USERNAME} "vllm-tt-metal-llama3/src" "${APP_DIR}/src" -COPY --chown=${CONTAINER_APP_USERNAME}:${CONTAINER_APP_USERNAME} "vllm-tt-metal-llama3/requirements.txt" "${APP_DIR}/requirements.txt" -COPY --chown=${CONTAINER_APP_USERNAME}:${CONTAINER_APP_USERNAME} "utils" "${APP_DIR}/utils" -COPY --chown=${CONTAINER_APP_USERNAME}:${CONTAINER_APP_USERNAME} "benchmarking" "${APP_DIR}/benchmarking" -COPY --chown=${CONTAINER_APP_USERNAME}:${CONTAINER_APP_USERNAME} "evals" "${APP_DIR}/evals" -COPY --chown=${CONTAINER_APP_USERNAME}:${CONTAINER_APP_USERNAME} "tests" "${APP_DIR}/tests" -COPY --chown=${CONTAINER_APP_USERNAME}:${CONTAINER_APP_USERNAME} "locust" "${APP_DIR}/locust" -RUN /bin/bash -c "source ${PYTHON_ENV_DIR}/bin/activate \ -&& pip install --default-timeout=240 --no-cache-dir -r requirements.txt" - -WORKDIR "${APP_DIR}/src" - # Switch back to root for entrypoint USER root