{
  "status": "MEASURED_CONFIGURATION_NOT_OPTIMAL_TUNING_CLAIM",
  "date": "2026-10-27",
  "hardware": "Single HGX B200 node,8 GPUs",
  "model_revisions": {
    "flash": "1bae0e1088993b9b82de08f59de5be5a34e42170",
    "glm": "ba32f8c19c8738e2c57c71798c601d477a7bf844"
  },
  "model_repositories": {
    "flash": "deepseek-ai/DeepSeek-V4.1-Flash-1017",
    "glm": "zai-org/GLM-5.3"
  },
  "weight_formats": {
    "flash": "official mixed FP4+FP8",
    "glm": "official FP8"
  },
  "image": "vllm/vllm-openai@sha256:a4a4c0437bf7240089da5f08aa370c4aee17ae5290f7a3b468825ee26c4c3a6b",
  "vllm": "0.31.0",
  "context": 1048576,
  "max_output_tokens": 262144,
  "temperature": 1.0,
  "top_p": 0.95,
  "seed_base": 20261026,
  "tp": 8,
  "max_num_seqs": 20,
  "max_num_batched_tokens": 4096,
  "prefix_cache": false,
  "speculative_decoding": false,
  "gpu_memory_utilization": {
    "flash": 0.8,
    "glm": 0.82
  },
  "kv_cache": {
    "flash": "fp8, indexer mxfp4",
    "glm": "fp8_e4m3"
  },
  "profiles": {
    "flash": {
      "short_science_qa": {
        "max_tokens": 262144,
        "timeout_s": 5400,
        "chat_template_kwargs": {
          "thinking": true,
          "reasoning_effort": 25
        },
        "temperature": 1.0,
        "top_p": 0.95
      },
      "long_document_grounded_qa": {
        "max_tokens": 262144,
        "timeout_s": 5400,
        "chat_template_kwargs": {
          "thinking": true,
          "reasoning_effort": 25
        },
        "temperature": 1.0,
        "top_p": 0.95
      },
      "scientific_reasoning": {
        "max_tokens": 262144,
        "timeout_s": 5400,
        "chat_template_kwargs": {
          "thinking": true,
          "reasoning_effort": 100
        },
        "temperature": 1.0,
        "top_p": 0.95
      },
      "scientific_code": {
        "max_tokens": 262144,
        "timeout_s": 5400,
        "chat_template_kwargs": {
          "thinking": true,
          "reasoning_effort": 100
        },
        "temperature": 1.0,
        "top_p": 0.95
      },
      "gpqa_diamond": {
        "max_tokens": 262144,
        "timeout_s": 5400,
        "chat_template_kwargs": {
          "thinking": true,
          "reasoning_effort": 100
        },
        "temperature": 1.0,
        "top_p": 0.95
      }
    },
    "glm": {
      "short_science_qa": {
        "max_tokens": 262144,
        "timeout_s": 5400,
        "chat_template_kwargs": {
          "reasoning_effort": "low"
        },
        "temperature": 1.0,
        "top_p": 0.95
      },
      "long_document_grounded_qa": {
        "max_tokens": 262144,
        "timeout_s": 5400,
        "chat_template_kwargs": {
          "reasoning_effort": "low"
        },
        "temperature": 1.0,
        "top_p": 0.95
      },
      "scientific_reasoning": {
        "max_tokens": 262144,
        "timeout_s": 5400,
        "chat_template_kwargs": {
          "reasoning_effort": "max"
        },
        "temperature": 1.0,
        "top_p": 0.95
      },
      "scientific_code": {
        "max_tokens": 262144,
        "timeout_s": 5400,
        "chat_template_kwargs": {
          "reasoning_effort": "max"
        },
        "temperature": 1.0,
        "top_p": 0.95
      },
      "gpqa_diamond": {
        "max_tokens": 262144,
        "timeout_s": 5400,
        "chat_template_kwargs": {
          "reasoning_effort": "max"
        },
        "temperature": 1.0,
        "top_p": 0.95
      }
    }
  },
  "dataset_sha256": "3759643da1c63d999ab5c7754db964a1e2d25e6700e85642a1b9265ba41e385d",
  "quality_policy": "One first result per question; retain incomplete responses unscored; no replacement with diagnostic trials.",
  "requested_request_timeout_s": 5400,
  "per_model_seconds_after_ready": 10800,
  "shared_capacity_stage_seconds": 1800,
  "load": "Closed-loop in-flight limit1/5/10/20. Same-host loopback; queue-inclusive client timing. Not20requests/sec.",
  "pending": [
    "GLM approximately786Kinput concurrency20 unmeasured",
    "Independent domain-expert explanation grading"
  ],
  "limits": [
    "Single run per frozen problem; no statistical superiority claim.",
    "Original capped results retained; diagnostic retries do not replace GPQA outcomes.",
    "Completed-only latency excludes censored requests; counts and failed durations retained in originals.",
    "Scientific explanation review is Codex-assisted and not blind independent expert grading.",
    "Long-context tests are synthetic retrieval/arithmetic, not evidence of full-document scientific understanding.",
    "GLM 786K C10 is stage-budget censored; C20 unmeasured. No claim of an absolute hardware limit.",
    "Model/tokenizer/attention/KV/effort and memory allocation differ despite shared prompts and limits.",
    "Measured on HGX B200 with 8 GPUs. Results are not guaranteed on other GPUs or configurations."
  ]
}