Update docker

TIS/run.sh

@@ -62,7 +62,7 @@ echo "Maximum batch size: ${TRITON_MAX_BATCH_SIZE}"
 
 # Launch Triton Server
 # pip install SentencePiece  # already installed
-CMD="python3 ${TIS_DIR}/scripts/launch_triton_server.py --world_size ${WORLD_SIZE} --model_repo=${TRITON_MODEL_REPO} --http_port=8080 --grpc_port=9000 --metrics_port=8002"
+CMD="python3 ${TIS_DIR}/scripts/launch_triton_server.py --world_size ${WORLD_SIZE} --model_repo=${TRITON_MODEL_REPO} --http_port=8080 --grpc_port=8081 --metrics_port=8082 --log --log-file /code/triton_log.txt"
 echo "Call command: $CMD"
 
 eval $CMD

TIS/test_serve.sh

@@ -11,8 +11,8 @@ echo $INGRESS_HOST
 echo $INGRESS_PORT
 echo $SERVICE_HOSTNAME
 
-# Test in container
-curl "http://localhost:8080/v2/models/ensemble/generate" -d '{"text_input": "What is machine learning?", "max_tokens": 20, "bad_words": "", "stop_words": "", "pad_id": 2, "end_id": 2}'
+# Test in container or local server
+# curl "http://localhost:8080/v2/models/ensemble/generate" -d '{"text_input": "What is machine learning?", "max_tokens": 20, "bad_words": "", "stop_words": "", "pad_id": 2, "end_id": 2}'
 
 # Test in cluster
 curl -H "Host: ${SERVICE_HOSTNAME}" "http://${INGRESS_HOST}:${INGRESS_PORT}/v2/models/ensemble/generate" -d '{"text_input": "What is machine learning?", "max_tokens": 20, "bad_words": "", "stop_words": "", "pad_id": 2, "end_id": 2}'

docker/README.md

@@ -0,0 +1 @@
+Docker Developing and Debugging Reference: https://cloud.tencent.com/developer/article/2125942

docker/exec.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+docker exec -it reserve /bin/bash
+docker exec -it reserve-leader /bin/bash
+docker exec -it reserve-worker /bin/bash
+
+/code/REServe/Initializer/main.sh run --tp 2

docker/network.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker network create --driver bridge reserve-network

docker/prepare.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+apt-get install vim
+apt-get install openssh-server
+passwd
+# vim /etc/ssh/sshd_config

docker/run.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# Replace MODEL_PATH to your own model path
+MODEL_PATH="/home/LAB/zhangyh2/workspace/models"
+# under MODEL_PATH:
+# models/checkpoints
+# models/engines
+# models/models/Meta-Llama-3-8B-Instruct
+
+docker run -it -d --network=reserve-network -p 8888:22 -p 8080:8080 -p 8081:8081 -p 8082:8082 \
+                  --runtime=nvidia --cap-add=SYS_PTRACE --cap-add=SYS_ADMIN  --hostname reserve-leader \
+                  --security-opt seccomp=unconfined --shm-size=16g --privileged \
+                  --ulimit memlock=-1 --gpus=all --volume ${MODEL_PATH}:/mnt/models --workdir /code \
+                  --name=reserve-leader reserve-llm:latest
+
+docker run -it -d --network=reserve-network \
+                  --runtime=nvidia --cap-add=SYS_PTRACE --cap-add=SYS_ADMIN --hostname reserve-worker \
+                  --security-opt seccomp=unconfined --shm-size=16g --privileged \
+                  --ulimit memlock=-1 --gpus=all --volume ${MODEL_PATH}:/mnt/models --workdir /code \
+                  --name=reserve-worker reserve-llm:latest