# 1️⃣ Install Docker & NVIDIA Container Toolkit
sudo apt-get update && sudo apt-get install -y docker.io
distribution=$(. /etc/os-release;echo $ID$VERSION_ID)
curl -s -L https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add -
curl -s -L https://nvidia.github.io/nvidia-docker/$distribution/nvidia-docker.list | \
    sudo tee /etc/apt/sources.list.d/nvidia-docker.list
sudo apt-get update && sudo apt-get install -y nvidia-docker2
sudo systemctl restart docker

# 2️⃣ Pull the inference image (built on HuggingFace’s `transformers` + `vLLM`)
docker pull ghcr.io/yourorg/llm-inference:latest

# 3️⃣ Run the container, exposing a REST endpoint on port 8080
docker run -d --gpus all \
  -p 8080:8080 \
  -v $HOME/models:/models:ro \
  -e MODEL_NAME=llama2-7b \
  ghcr.io/yourorg/llm-inference:latest

{
  "prompt": "Explain quantum computing in two sentences.",
  "max_new_tokens": 64,
  "temperature": 0.7
}

# Test the endpoint
curl -X POST http://localhost:8080/generate \
  -H "Content-Type: application/json" \
  -d '{"prompt":"Explain quantum computing in two sentences.","max_new_tokens":64}'

# Inside the container (or on the host, then mount /models)
python - <<'PY'
from transformers import AutoModelForCausalLM, AutoTokenizer
import torch, bitsandbytes as bnb

model_name = "meta-llama/Llama-2-7b-hf"
tokenizer = AutoTokenizer.from_pretrained(model_name)

# Load in 4‑bit (GPTQ) – saves ~2× VRAM
model = AutoModelForCausalLM.from_pretrained(
    model_name,
    device_map="auto",
    load_in_4bit=True,
    quantization_config=bnb.nn.quantization.QuantizationConfig(
        bits=4, 
        dtype=torch.float16,
        quant_type="nf4"
    )
)

model.save_pretrained("/models/llama2-7b-4bit")
tokenizer.save_pretrained("/models/llama2-7b-4bit")
PY

# app.py – FastAPI wrapper (already baked into the Docker image)
from fastapi import FastAPI, Request
from transformers import AutoModelForCausalLM, AutoTokenizer
import torch

app = FastAPI()
model = AutoModelForCausalLM.from_pretrained(
    "/models/llama2-7b-4bit", device_map="auto", torch_dtype=torch.float16
)
tokenizer = AutoTokenizer.from_pretrained("/models/llama2-7b-4bit")

@app.post("/generate")
async def generate(req: Request):
    payload = await req.json()
    inputs = tokenizer(payload["prompt"], return_tensors="pt").to("cuda")
    output = model.generate(
        **inputs,
        max_new_tokens=payload.get("max_new_tokens", 64),
        temperature=payload.get("temperature", 0.7),
        do_sample=True,
    )
    text = tokenizer.decode(output[0], skip_special_tokens=True)
    return {"response": text}