yaml
qwen36-27b:
  predictor:
    image: "machinelearning-liftwing-inference-services-qwen36"
    image_version: "<ci-published-tag>"
    custom_env:
      - name: MODEL_NAME
        value: "qwen36-27b"
      - name: STORAGE_URI
        value: "s3://wmf-ml-models/qwen36-27b/"
      - name: TRUST_REMOTE_CODE
        value: "False"
      - name: GPU_MEMORY_UTILIZATION
        value: "0.90"
      - name: MAX_MODEL_LEN
        value: "65536"
      - name: TENSOR_PARALLEL_SIZE
        value: "2"
      - name: MAX_NUM_BATCHED_TOKENS
        value: "32768"
      - name: MAX_NUM_SEQS
        value: "256"
      - name: BLOCK_SIZE
        value: "64"
      - name: VLLM_ROCM_USE_AITER
        value: "1"
      - name: TORCH_BLAS_PREFER_HIPBLASLT
        value: "1"
      - name: HSA_FORCE_FINE_GRAIN_PCIE
        value: "1"
      - name: HSA_ENABLE_IPC_MODE_LEGACY
        value: "0"
      - name: NCCL_P2P_DISABLE
        value: "0"
      - name: NCCL_SOCKET_IFNAME
        value: lo
      - name: NCCL_SHM_DISABLE
        value: "0"
      - name: NCCL_DEBUG
        value: "INFO"
      - name: VLLM_SKIP_P2P_CHECK
        value: "0"
      - name: VLLM_HOST_IP
        value: "127.0.0.1"
      - name: RCCL_MSCCL_ENABLE
        value: "0"
      - name: MAX_JOBS
        value: "1"
      - name: AMDGCN_USE_BUFFER_OPS
        value: "0"
      - name: TORCH_SYMM_MEM_DISABLE_MULTICAST
        value: "1"
    container:
      resources:
        limits:
          cpu: "16"
          memory: 80Gi
          amd.com/gpu: "2"
        requests:
          cpu: "16"
          memory: 80Gi
          amd.com/gpu: "2"
      volumeMounts:
        - mountPath: /dev/shm
          name: dshm
    volumes:
      - name: dshm
        emptyDir:
          medium: Memory
          sizeLimit: "8Gi"
    config:
      maxReplicas: 1
      tolerations:
        - key: "dedicated"
          operator: "Equal"
          value: "mi300x-experiments"
          effect: "NoSchedule"
        - key: "dedicated"
          operator: "Equal"
          value: "mi300x-experiments"
          effect: "NoExecute"
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
              - matchExpressions:
                  - key: "kubernetes.io/hostname"
                    operator: "In"
                    values:
                      - "ml-serve1012.eqiad.wmnet"
                      - "ml-serve1013.eqiad.wmnet"
                      - "ml-serve1014.eqiad.wmnet"
                      - "ml-serve1015.eqiad.wmnet"

curl -N localhost:8080/openai/v1/completions -X POST \
  -H "Content-Type: application/json" \
  -d '{"model":"qwen3-0.6b","prompt":"What is 2+2?","max_tokens":50,"stream":true}'

data: {"id":"ae6d8ca612fb4e3c","object":"text_completion","created":1778748470,"model":"qwen3-0.6b","choices":[{"index":0,"text":" Is","logprobs":null,"finish_reason":null,"stop_reason":null,"prompt_token_ids":null,"token_ids":null}],"usage":null,"system_fingerprint":null}

data: {"id":"ae6d8ca612fb4e3c","object":"text_completion","created":1778748470,"model":"qwen3-0.6b","choices":[{"index":0,"text":" it","logprobs":null,"finish_reason":null,"stop_reason":null,"prompt_token_ids":null,"token_ids":null}],"usage":null,"system_fingerprint":null}

data: {"id":"ae6d8ca612fb4e3c","object":"text_completion","created":1778748470,"model":"qwen3-0.6b","choices":[{"index":0,"text":" possible","logprobs":null,"finish_reason":null,"stop_reason":null,"prompt_token_ids":null,"token_ids":null}],"usage":null,"system_fingerprint":null}
....

Traceback (most recent call last):
  File "/srv/app/model.py", line 404, in <module>
    model.load()
  File "/srv/app/model.py", line 93, in load
    raise kserve.errors.ModelMissingError(error_message)
kserve.errors.ModelMissingError: Failed to load model. Reason: 1 validation error for ModelConfig
  Value error, The checkpoint you are trying to load has model type `qwen3_5` but Transformers does not recognize this architecture. This could be because of an issue with the checkpoint, or because your version of Transformers is out of date.

You can update Transformers with the command `pip install --upgrade transformers`. If this does not work, and the checkpoint is very new, then there may not be a release version that supports this model yet. In this case, you can get the most up-to-date code by installing Transformers from source with the command `pip install git+https://github.com/huggingface/transformers.git` [type=value_error, input_value=ArgsKwargs((), {'model': ...rocessor_plugin': None}), input_type=ArgsKwargs]
    For further information visit https://errors.pydantic.dev/2.12/v/value_error

curl -Ns https://inference.svc.eqiad.wmnet:30443/openai/v1/chat/completions \
  -X POST \
  -H "Content-Type: application/json" \
  -H "Host: qwen3-14b.experimental.wikimedia.org" \
  -d '{
    "model": "qwen3-14b",
    "messages": [
      {"role": "user", "content": "Explain gravity in one sentence."}
    ],
    "max_tokens": 50,
    "stream": true
  }'

data: {"id":"8cecd046abf71d62","object":"chat.completion.chunk","created":1779278509,"model":"qwen3-14b","choices":[{"index":0,"delta":{"role":"assistant","content":"Gravity","reasoning":null,"reasoning_content":null,"tool_calls":[]},"logprobs":null,"finish_reason":null,"stop_reason":null,"token_ids":null}],"usage":null,"prompt_token_ids":null,"system_fingerprint":null}

data: {"id":"8cecd046abf71d62","object":"chat.completion.chunk","created":1779278509,"model":"qwen3-14b","choices":[{"index":0,"delta":{"role":"assistant","content":" is","reasoning":null,"reasoning_content":null,"tool_calls":[]},"logprobs":null,"finish_reason":null,"stop_reason":null,"token_ids":null}],"usage":null,"prompt_token_ids":null,"system_fingerprint":null}

data: {"id":"8cecd046abf71d62","object":"chat.completion.chunk","created":1779278509,"model":"qwen3-14b","choices":[{"index":0,"delta":{"role":"assistant","content":" the","reasoning":null,"reasoning_content":null,"tool_calls":[]},"logprobs":null,"finish_reason":null,"stop_reason":null,"token_ids":null}],"usage":null,"prompt_token_ids":null,"system_fingerprint":null}

data: {"id":"8cecd046abf71d62","object":"chat.completion.chunk","created":1779278509,"model":"qwen3-14b","choices":[{"index":0,"delta":{"role":"assistant","content":" fundamental","reasoning":null,"reasoning_content":null,"tool_calls":[]},"logprobs":null,"finish_reason":null,"stop_reason":null,"token_ids":null}],"usage":null,"prompt_token_ids":null,"system_fingerprint":null}

...

data: {"id":"8cecd046abf71d62","object":"chat.completion.chunk","created":1779278509,"model":"qwen3-14b","choices":[{"index":0,"delta":{"role":"assistant","content":" between","reasoning":null,"reasoning_content":null,"tool_calls":[]},"logprobs":null,"finish_reason":null,"stop_reason":null,"token_ids":null}],"usage":null,"prompt_token_ids":null,"system_fingerprint":null}

data: {"id":"8cecd046abf71d62","object":"chat.completion.chunk","created":1779278509,"model":"qwen3-14b","choices":[{"index":0,"delta":{"role":"assistant","content":" them","reasoning":null,"reasoning_content":null,"tool_calls":[]},"logprobs":null,"finish_reason":null,"stop_reason":null,"token_ids":null}],"usage":null,"prompt_token_ids":null,"system_fingerprint":null}

data: [DONE]

curl -s -i https://inference.svc.eqiad.wmnet:30443/v1/models/qwen3-14b:predict \
-X POST \
-H "Content-Type: application/json" \
-H "Host: qwen3-14b.experimental.wikimedia.org" \
-d '{"prompt": "What is the capital of France?", "max_tokens": 50}'

HTTP/2 200 
content-length: 117
content-type: application/json
date: Wed, 20 May 2026 12:04:46 GMT
server: istio-envoy
x-envoy-upstream-service-time: 75

{"model_name":"qwen3-14b","response":"The capital of France is **Paris**.","prompt_tokens":19,"completion_tokens":10}

curl -Ns https://inference.svc.eqiad.wmnet:30443/openai/v1/completions \
  -X POST \
  -H "Content-Type: application/json" \
  -H "Host: qwen3-14b.experimental.wikimedia.org" \
  -d '{
    "model": "qwen3-14b",
    "prompt": "Explain gravity in one sentence.",
    "max_tokens": 50,
    "stream": true
  }'

data: {"id":"88d30502576326c3","object":"text_completion","created":1779364754,"model":"qwen3-14b","choices":[{"index":0,"text":" Gravity","logprobs":null,"finish_reason":null,"stop_reason":null,"prompt_token_ids":null,"token_ids":null}],"usage":null,"system_fingerprint":null}

data: {"id":"88d30502576326c3","object":"text_completion","created":1779364754,"model":"qwen3-14b","choices":[{"index":0,"text":" is","logprobs":null,"finish_reason":null,"stop_reason":null,"prompt_token_ids":null,"token_ids":null}],"usage":null,"system_fingerprint":null}

data: {"id":"88d30502576326c3","object":"text_completion","created":1779364754,"model":"qwen3-14b","choices":[{"index":0,"text":" the","logprobs":null,"finish_reason":null,"stop_reason":null,"prompt_token_ids":null,"token_ids":null}],"usage":null,"system_fingerprint":null}

...

data: {"id":"86e68e07b53363e6","object":"text_completion","created":1779364817,"model":"qwen3-14b","choices":[{"index":0,"text":".","logprobs":null,"finish_reason":null,"stop_reason":null,"prompt_token_ids":null,"token_ids":null}],"usage":null,"system_fingerprint":null}

data: {"id":"86e68e07b53363e6","object":"text_completion","created":1779364817,"model":"qwen3-14b","choices":[{"index":0,"text":" Let","logprobs":null,"finish_reason":null,"stop_reason":null,"prompt_token_ids":null,"token_ids":null}],"usage":null,"system_fingerprint":null}

data: {"id":"86e68e07b53363e6","object":"text_completion","created":1779364817,"model":"qwen3-14b","choices":[{"index":0,"text":" me","logprobs":null,"finish_reason":"length","stop_reason":null,"prompt_token_ids":null,"token_ids":null}],"usage":{"prompt_tokens":7,"total_tokens":57,"completion_tokens":50,"prompt_tokens_details":null},"system_fingerprint":null}

data: [DONE]

cgoubert@deploy1003$ curl -s https://rest-gateway.discovery.wmnet:4113/service/lw/inference/v1/models/qwen3-14b/openai/v1/chat/completions   -X POST   -H "Content-Type: application/json"      -d '{                     
    "model": "qwen3-14b",                                                                                                                
    "messages": [                                                                                                                              {"role": "user", "content": "Explain gravity in one sentence."}                                                                        ],                                                                                                                                   
    "max_tokens": 50,                                                                                                                    
    "stream": true                                                                                                                         }'                                                                                                                                     data: {"id":"8d973f418089162a","object":"chat.completion.chunk","created":1779790003,"model":"qwen3-14b","choices":[{"index":0,"delta":{"
role":"assistant","content":"Gravity","reasoning":null,"reasoning_content":null,"tool_calls":[]},"logprobs":null,"finish_reason":null,"st
op_reason":null,"token_ids":null}],"usage":null,"prompt_token_ids":null,"system_fingerprint":null}
[...]
cgoubert@deploy2003$ curl  https://rest-gateway.discovery.wmnet:4113/service/lw/inference/v1/models/qwen3-14b:predict   -X POST   -H "Content-Type: application/json"      -d '{"prompt": "What is the capital of France?", "max_tokens": 50}'
{"model_name":"qwen3-14b","response":"The capital of France is **Paris**.","prompt_tokens":19,"completion_tokens":10}

❯ curl https://api.wikimedia.org/service/lw/inference/v1/models/qwen3-14b:predict   -X POST   -H "Content-Type: application/json"      -d '{"prompt": "What is the capital of France?", "max_tokens": 50}'
{"model_name":"qwen3-14b","response":"The capital of France is Paris.","prompt_tokens":19,"completion_tokens":8}
❯ curl https://api.wikimedia.org/service/lw/inference/v1/models/qwen3-14b/openai/v1/chat/completions   -X POST   -H "Content-Type: application/json"      -d '{
    "model": "qwen3-14b",
    "messages": [
      {"role": "user", "content": "Explain gravity in one sentence."}
    ],
    "max_tokens": 50,
    "stream": true
  }'
data: {"id":"879ea166c28ad33f","object":"chat.completion.chunk","created":1779790199,"model":"qwen3-14b","choices":[{"index":0,"delta":{"role":"assistant","content":"Gravity","reasoning":null,"reasoning_content":null,"tool_calls":[]},"logprobs":null,"finish_reason":null,"stop_reason":null,"token_ids":null}],"usage":null,"prompt_token_ids":null,"system_fingerprint":null}

curl -Ns https://api.wikimedia.org/service/lw/inference/v1/models/qwen3-14b/openai/v1/chat/completions \
  -X POST -H "Content-Type: application/json" \
  -d '{"model":"qwen3-14b","messages":[{"role":"user","content":"Write a 500-word essay on ML."}],"max_tokens":600,"stream":true}'

response_headers_to_add:
  - key: cache-control
    value: no-cache, no-transform