modified

Dockerfile

@@ -66,5 +66,6 @@
 FROM vllm/vllm-openai:latest
 ENV PATH="/usr/local/bin:/usr/bin:/bin:/usr/local/sbin:/usr/sbin:/sbin:$PATH"
 EXPOSE 8000
-COPY entrypoint.sh /usr/local/bin/
-ENTRYPOINT [ "entrypoint.sh" ]
+WORKDIR /app
+COPY entrypoint.sh /app/entrypoint.sh
+ENTRYPOINT [ "/app/entrypoint.sh" ]

entrypoint.sh

@@ -2,40 +2,40 @@
 
 # Default values
 # MODEL=${MODEL:-"microsoft/Phi-3-mini-4k-instruct"}
-MODEL=${MODEL:-"EleutherAI/pythia-70m"}
-DTYPE=${DTYPE:-"half"}
-MAX_NUM_BATCHED_TOKENS=${MAX_NUM_BATCHED_TOKENS:-512}
-MAX_NUM_SEQS=${MAX_NUM_SEQS:-16}
-GPU_MEMORY_UTILIZATION=${GPU_MEMORY_UTILIZATION:-0.85}
-MAX_MODEL_LEN=${MAX_MODEL_LEN:-512}
-ENFORCE_EAGER=${ENFORCE_EAGER:-true}
+# MODEL=${MODEL:-"EleutherAI/pythia-70m"}
+# DTYPE=${DTYPE:-"half"}
+# MAX_NUM_BATCHED_TOKENS=${MAX_NUM_BATCHED_TOKENS:-512}
+# MAX_NUM_SEQS=${MAX_NUM_SEQS:-16}
+# GPU_MEMORY_UTILIZATION=${GPU_MEMORY_UTILIZATION:-0.85}
+# MAX_MODEL_LEN=${MAX_MODEL_LEN:-512}
+# ENFORCE_EAGER=${ENFORCE_EAGER:-true}
 
-# Disable usage stats via environment variable
-export VLLM_DISABLE_USAGE_STATS=true
+# # Disable usage stats via environment variable
+# export VLLM_DISABLE_USAGE_STATS=true
 
-# Print environment for debugging
-echo "Environment variables:"
-env
+# # Print environment for debugging
+# echo "Environment variables:"
+# env
 
-# Create and set permissions for the config directory
-CONFIG_DIR=${XDG_CONFIG_HOME:-"/tmp/config"}
+# # Create and set permissions for the config directory
+# CONFIG_DIR=${XDG_CONFIG_HOME:-"/tmp/config"}
 
-if [ ! -d "$CONFIG_DIR" ]; then
-    mkdir -p "$CONFIG_DIR"
-fi
-chmod -R 777 "$CONFIG_DIR"
-echo "Permissions for $CONFIG_DIR:"
-ls -la "$CONFIG_DIR"
+# if [ ! -d "$CONFIG_DIR" ]; then
+#     mkdir -p "$CONFIG_DIR"
+# fi
+# chmod -R 777 "$CONFIG_DIR"
+# echo "Permissions for $CONFIG_DIR:"
+# ls -la "$CONFIG_DIR"
 
-# Check and set permissions for directories
-for dir in /tmp/huggingface /tmp/cache /tmp/numba_cache /tmp/outlines_cache /.config; do
-    if [ ! -d "$dir" ]; then
-        mkdir -p "$dir"
-    fi
-    chmod -R 777 "$dir"
-    echo "Permissions for $dir:"
-    ls -la "$dir"
-done
+# # Check and set permissions for directories
+# for dir in /tmp/huggingface /tmp/cache /tmp/numba_cache /tmp/outlines_cache /.config; do
+#     if [ ! -d "$dir" ]; then
+#         mkdir -p "$dir"
+#     fi
+#     chmod -R 777 "$dir"
+#     echo "Permissions for $dir:"
+#     ls -la "$dir"
+# done
 
 # # Construct the command
 # CMD="vllm serve $MODEL \
@@ -53,12 +53,12 @@ done
 # fi
 
 
-CMD="python3 -m vllm.entrypoints.openai.api_server \
+python3 -m vllm.entrypoints.openai.api_server \
         --model EleutherAI/pythia-70m \
-        --gpu-memory-utilization 0.9 
-        --max-model-len 200"
+        --gpu-memory-utilization 0.9 \
+        --max-model-len 200
 
 
-# Execute the command
-echo "Running command: $CMD"
-exec $CMD
+# # Execute the command
+# echo "Running command: $CMD"
+# exec $CMD