inference_plan.json

{
  "phases": {
    "prefill": {
      "description": "process full prompt, populate KV cache",
      "compute_bound": true,
      "attention": "flash-attention, causal"
    },
    "decode": {
      "description": "autoregressive single-token steps against cache",
      "memory_bandwidth_bound": true,
      "kv_read_bytes_per_step_per_seq": 12800
    }
  },
  "kv_cache": {
    "layout": "paged",
    "block_size_tokens": 16,
    "dtype": "bf16",
    "bytes_per_token": 12800,
    "per_seq_max": "52.43 MB"
  },
  "scheduling": {
    "policy": "continuous_batching",
    "preemption": "recompute",
    "max_model_len": 4096
  },
  "parallelism": {
    "tensor_parallel": 1,
    "pipeline_parallel": 1
  },
  "engine_hints": {
    "vllm": "vllm serve /model --dtype bfloat16 --max-model-len 4096 --tensor-parallel-size 1 --pipeline-parallel-size 1",
    "sglang": "python -m sglang.launch_server --model-path /model --tp 1 --context-length 4096"
  },
  "note": "UNTRAINED model: serving it yields noise by design; this plan becomes operative after training."
}