{
  "architectures_compared": 4,
  "results": {
    "stuffed_128k": {
      "name": "Standard OpenAI 128k (Stuffed)",
      "overall_accuracy_pct": 61.11,
      "early_constraint_retention_pct": 80.0,
      "stale_state_invalidation_pct": 0.0,
      "multihop_synthesis_pct": 0.0,
      "boundary_refusal_precision_pct": 0.0,
      "total_input_tokens": 42586,
      "total_output_tokens": 816,
      "cumulative_cost_usd": 0.114625,
      "mean_latency_ms": 1499.23,
      "mean_ttft_ms": 149.92,
      "stale_state_resurrections": 2,
      "boundary_refusal_failures": 2,
      "turn_accuracy_curve": [
        1,
        1,
        1,
        0,
        0,
        1,
        1,
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        1,
        0,
        1,
        0
      ],
      "turn_latency_curve": [
        1312.06,
        1158.4,
        1371.45,
        1650.41,
        1694.15,
        1823.52,
        1528.46,
        1681.49,
        1202.2,
        986.35,
        1488.43,
        1283.5,
        1401.38,
        1254.24,
        885.35,
        1646.58,
        641.31,
        3976.84
      ],
      "turn_cost_curve": [
        0.001453,
        0.00355,
        0.003602,
        0.00375,
        0.004375,
        0.004597,
        0.00571,
        0.005973,
        0.007102,
        0.00741,
        0.00773,
        0.007757,
        0.007925,
        0.00792,
        0.00815,
        0.008372,
        0.008332,
        0.010915
      ]
    },
    "vector_rag": {
      "name": "LangGraph + Pinecone Vector RAG",
      "overall_accuracy_pct": 77.78,
      "early_constraint_retention_pct": 100.0,
      "stale_state_invalidation_pct": 0.0,
      "multihop_synthesis_pct": 50.0,
      "boundary_refusal_precision_pct": 50.0,
      "total_input_tokens": 7249,
      "total_output_tokens": 642,
      "cumulative_cost_usd": 0.024543,
      "mean_latency_ms": 910.81,
      "mean_ttft_ms": 91.08,
      "stale_state_resurrections": 2,
      "boundary_refusal_failures": 1,
      "turn_accuracy_curve": [
        1,
        1,
        1,
        0,
        0,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        0,
        1,
        0
      ],
      "turn_latency_curve": [
        632.03,
        1043.1,
        712.55,
        546.15,
        1070.78,
        1001.7,
        1071.9,
        562.28,
        646.82,
        1012.79,
        1404.52,
        974.1,
        837.76,
        580.33,
        660.5,
        1057.46,
        888.18,
        1691.65
      ],
      "turn_cost_curve": [
        0.00125,
        0.001468,
        0.00099,
        0.001072,
        0.001433,
        0.001295,
        0.001027,
        0.001132,
        0.001195,
        0.00143,
        0.001822,
        0.001493,
        0.001603,
        0.001262,
        0.001235,
        0.001628,
        0.001215,
        0.001992
      ]
    },
    "memgpt_wrapper": {
      "name": "MemGPT / Letta Wrapper",
      "overall_accuracy_pct": 11.11,
      "early_constraint_retention_pct": 0.0,
      "stale_state_invalidation_pct": 0.0,
      "multihop_synthesis_pct": 0.0,
      "boundary_refusal_precision_pct": 50.0,
      "total_input_tokens": 19812,
      "total_output_tokens": 1129,
      "cumulative_cost_usd": 0.06082,
      "mean_latency_ms": 1444.92,
      "mean_ttft_ms": 144.49,
      "stale_state_resurrections": 3,
      "boundary_refusal_failures": 1,
      "turn_accuracy_curve": [
        1,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        1,
        0,
        0,
        0,
        0,
        0
      ],
      "turn_latency_curve": [
        977.35,
        1486.95,
        1283.92,
        855.12,
        1233.32,
        1751.75,
        758.81,
        1022.93,
        1470.22,
        895.36,
        483.15,
        1509.66,
        934.44,
        929.72,
        501.97,
        1435.85,
        673.51,
        7804.59
      ],
      "turn_cost_curve": [
        0.001775,
        0.002765,
        0.002757,
        0.002692,
        0.002783,
        0.00319,
        0.003077,
        0.002975,
        0.002988,
        0.003112,
        0.00319,
        0.003295,
        0.003608,
        0.003305,
        0.003373,
        0.003458,
        0.003385,
        0.009092
      ]
    },
    "icx_desk_warehouse": {
      "name": "Calera ICX Substrate (Desk-and-Warehouse)",
      "overall_accuracy_pct": 94.44,
      "early_constraint_retention_pct": 100.0,
      "stale_state_invalidation_pct": 100.0,
      "multihop_synthesis_pct": 50.0,
      "boundary_refusal_precision_pct": 100.0,
      "total_input_tokens": 1987,
      "total_output_tokens": 1876,
      "cumulative_cost_usd": 0.000456,
      "mean_latency_ms": 1610.44,
      "mean_ttft_ms": 161.14,
      "stale_state_resurrections": 0,
      "boundary_refusal_failures": 0,
      "turn_accuracy_curve": [
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        0,
        1,
        1,
        1,
        1,
        1,
        1,
        1
      ],
      "turn_latency_curve": [
        921.35,
        1271.89,
        1297.25,
        1.5,
        1153.01,
        1412.67,
        427.02,
        1372.57,
        1041.38,
        1378.59,
        4299.95,
        1423.32,
        1.5,
        1403.62,
        1205.08,
        1230.45,
        1150.63,
        7996.08
      ],
      "turn_cost_curve": [
        1e-05,
        1.8e-05,
        1.5e-05,
        0.0,
        1.6e-05,
        2.3e-05,
        7e-06,
        2.1e-05,
        1.6e-05,
        2.4e-05,
        6.7e-05,
        1.9e-05,
        0.0,
        2.1e-05,
        2.1e-05,
        2.2e-05,
        7e-06,
        0.000149
      ]
    }
  },
  "winner_matrix": {},
  "benchmark_mode": "empirical_live_execution",
  "model_under_test": "qwen2.5:1.5b",
  "provider": "local",
  "timestamp_utc": "2026-09-02T15:12:12Z",
  "traces_file": "/data/empirical_benchmark_traces.jsonl"
}