{ "schema_version": 3, "updated_at": "2026-10-12T16:56:16+00:00", "dataset_n": 2035, "dataset_sha256": "e22afaac1db009fb05ab0fef183ed1fc66e7999960ad26235b7c2d70e4f0913a", "results": [ { "name": "DeepSeek V4 Flash", "provider": "DeepSeek", "access": "open", "release_date": "2026-04-24", "release_note": "Preview release, matching the June evaluation. Later revisions of this API alias are not evaluated here.", "parameters_b": 284, "active_parameters_b": 13, "release_source": "https://api-docs.deepseek.com/news/news260424/", "parameters_source": "https://api-docs.deepseek.com/news/news260424/", "id": "deepseek-v4-flash-20260611", "model_key": "deepseek-v4-flash", "evaluation": "experiment_results/openrouter_full_20260611T064211Z/deepseek-v4-flash/evaluation_eval.json", "evaluated_on": "2026-06-11", "judge": "GPT-4.1 mini", "protocol_source": "experiment_results/openrouter_full_20260611T064211Z/deepseek-v4-flash/evaluate_eval.log", "notes": "", "score_aggregation": "Mean over evaluated examples", "complete": true, "n": 2035, "expected_n": 2035, "subset_counts": { "v_generation": 650, "v_conversion": 435, "t_generation": 250, "t_conversion": 700 }, "metrics": { "overall": 78.3, "renderable": 73.29, "non_renderable": 84.03, "render_rate": 93.66, "t_generation": 91.94, "t_conversion": 81.21, "v_generation": 71.9, "v_conversion": 75.38 }, "artifact_url": "./static/data/artifacts/deepseek-v4-flash-20260611.json" }, { "name": "Qwen3.7 Plus", "provider": "Qwen", "access": "closed", "release_date": "2026-06-01", "release_note": "Vendor-listed Global/International availability; May 26 is the snapshot date.", "parameters_b": null, "active_parameters_b": null, "release_source": "https://www.alibabacloud.com/help/en/model-studio/newly-released-models", "parameters_source": null, "id": "qwen3.7-plus-20260611", "model_key": "qwen3.7-plus", "evaluation": "experiment_results/openrouter_full_20260611T064211Z/qwen3.7-plus/evaluation_eval.json", "evaluated_on": "2026-06-11", "judge": "GPT-4.1 mini", "protocol_source": "experiment_results/openrouter_full_20260611T064211Z/qwen3.7-plus/evaluate_eval.log", "notes": "", "score_aggregation": "Mean over evaluated examples", "complete": true, "n": 2035, "expected_n": 2035, "subset_counts": { "v_generation": 650, "v_conversion": 435, "t_generation": 250, "t_conversion": 700 }, "metrics": { "overall": 76.8, "renderable": 70.22, "non_renderable": 84.31, "render_rate": 91.15, "t_generation": 92.21, "t_conversion": 81.49, "v_generation": 66.92, "v_conversion": 75.17 }, "artifact_url": "./static/data/artifacts/qwen3.7-plus-20260611.json" }, { "name": "MiMo V2.6 Flash", "provider": "Xiaomi MiMo", "access": "open", "release_date": "2026-09-22", "release_note": "API release date; size follows the published Flash-RL model card.", "parameters_b": 309, "active_parameters_b": 15, "release_source": "https://mimo.mi.com/docs/en-US/updates/model", "parameters_source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL", "id": "mimo-v2.6-flash-luna-20261010", "model_key": "mimo-v2.6-flash", "evaluation": "experiment_results/model_sweep_20261004/generations/xiaomi__mimo-v2.6-flash/evaluation_luna.json", "evaluated_on": "2026-10-10", "judge": "GPT-6 Luna", "protocol_source": "experiment_results/model_sweep_20261004/score_all.py", "notes": "Full 2,035-task run; VQA judge: GPT-6 Luna via OpenRouter. No successful renders recorded for Matplotlib, Typst. Recorded failures remain included in scores.", "generation_protocol_source": "experiment_results/model_sweep_20261004/infer_openrouter.py", "score_aggregation": "Mean over evaluated examples", "complete": true, "n": 2035, "expected_n": 2035, "subset_counts": { "v_generation": 650, "v_conversion": 435, "t_generation": 250, "t_conversion": 700 }, "metrics": { "overall": 76.15, "renderable": 69.45, "non_renderable": 83.82, "render_rate": 90.02, "t_generation": 92.3, "t_conversion": 80.79, "v_generation": 66.35, "v_conversion": 74.08 }, "artifact_url": "./static/data/artifacts/mimo-v2.6-flash-luna-20261010.json" }, { "name": "GPT-4.1 mini", "provider": "OpenAI", "access": "closed", "release_date": "2025-04-14", "release_note": "", "parameters_b": null, "active_parameters_b": null, "release_source": "https://developers.openai.com/api/docs/changelog", "parameters_source": null, "id": "gpt41mini-20260611", "model_key": "GPT-4.1-mini", "evaluation": "experiment_results/gpt41mini_openrouter_reproduce/full-rerender-fixed-20260611T054618Z/evaluation.json", "evaluated_on": "2026-06-11", "judge": "GPT-4.1 mini", "protocol_source": "experiment_results/gpt41mini_openrouter_reproduce/full-rerender-fixed-20260611T054618Z/evaluate.log", "notes": "Latest completed rerender run with renderer fixes.", "score_aggregation": "Mean over evaluated examples", "complete": true, "n": 2035, "expected_n": 2035, "subset_counts": { "v_generation": 650, "v_conversion": 435, "t_generation": 250, "t_conversion": 700 }, "metrics": { "overall": 76.06, "renderable": 72.2, "non_renderable": 80.47, "render_rate": 92.53, "t_generation": 93.43, "t_conversion": 75.84, "v_generation": 69.94, "v_conversion": 75.58 }, "artifact_url": "./static/data/artifacts/gpt41mini-20260611.json" }, { "name": "GPT-4o", "provider": "OpenAI", "access": "closed", "release_date": "2024-05-13", "release_note": "", "parameters_b": null, "active_parameters_b": null, "release_source": "https://developers.openai.com/api/docs/changelog", "parameters_source": null, "id": "paper-0", "model_key": "GPT-4o", "n": 2035, "expected_n": 2035, "complete": true, "score_aggregation": "Reported average of four subset scores", "evaluated_on": null, "judge": "Published paper protocol", "metrics": { "overall": 76.02, "t_generation": 91.52, "t_conversion": 73.95, "v_generation": 65.39, "v_conversion": 73.2 }, "artifact_url": "./static/data/paper-baseline.json", "notes": "Original website paper results; exact evaluation date and model snapshot not recorded in that table." }, { "name": "o1-mini", "provider": "OpenAI", "access": "closed", "release_date": "2024-09-12", "release_note": "", "parameters_b": null, "active_parameters_b": null, "release_source": "https://developers.openai.com/api/docs/changelog", "parameters_source": null, "id": "paper-1", "model_key": "o1-mini", "n": 2035, "expected_n": 2035, "complete": true, "score_aggregation": "Reported average of four subset scores", "evaluated_on": null, "judge": "Published paper protocol", "metrics": { "overall": 75.58, "t_generation": 88.12, "t_conversion": 81.82, "v_generation": 61.98, "v_conversion": 70.4 }, "artifact_url": "./static/data/paper-baseline.json", "notes": "Original website paper results; exact evaluation date and model snapshot not recorded in that table." }, { "name": "DeepSeek V3.1 Terminus", "provider": "DeepSeek", "access": "open", "release_date": "2025-09-22", "release_note": "Terminus update of DeepSeek V3.1; total model size follows the official checkpoint card.", "parameters_b": 685, "active_parameters_b": null, "release_source": "https://api-docs.deepseek.com/news/news250922/", "parameters_source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1-Terminus", "id": "deepseek-v3.1-terminus-luna-20261010", "model_key": "deepseek-v3.1-terminus", "evaluation": "experiment_results/model_sweep_20261004/generations/deepseek__deepseek-v3.1-terminus/evaluation_luna.json", "evaluated_on": "2026-10-10", "judge": "GPT-6 Luna", "protocol_source": "experiment_results/model_sweep_20261004/score_all.py", "notes": "Full 2,035-task run; VQA judge: GPT-6 Luna via OpenRouter. No successful renders recorded for Matplotlib, Typst. Recorded failures remain included in scores.", "generation_protocol_source": "experiment_results/model_sweep_20261004/infer_openrouter.py", "score_aggregation": "Mean over evaluated examples", "complete": true, "n": 2035, "expected_n": 2035, "subset_counts": { "v_generation": 650, "v_conversion": 435, "t_generation": 250, "t_conversion": 700 }, "metrics": { "overall": 74.54, "renderable": 68.67, "non_renderable": 81.26, "render_rate": 90.12, "t_generation": 92.97, "t_conversion": 77.07, "v_generation": 64.3, "v_conversion": 75.2 }, "artifact_url": "./static/data/artifacts/deepseek-v3.1-terminus-luna-20261010.json" }, { "name": "Nemotron 3 Ultra", "provider": "NVIDIA", "access": "open", "release_date": "2026-06-04", "release_note": "", "parameters_b": 550, "active_parameters_b": 55, "release_source": "https://developer.nvidia.com/blog/nvidia-nemotron-3-ultra-powers-faster-more-efficient-reasoning-for-long-running-agents/", "parameters_source": "https://developer.nvidia.com/blog/nvidia-nemotron-3-ultra-powers-faster-more-efficient-reasoning-for-long-running-agents/", "id": "nemotron-ultra-gpt41mini-20261001", "model_key": "nemotron-3-ultra", "evaluation": "experiment_results/nemotron_ultra_free_full_20261001/evaluation_gpt41mini.json", "evaluated_on": "2026-10-04", "judge": "GPT-4.1 mini", "protocol_source": "experiment_results/nemotron_ultra_free_full_20261001/run.json", "notes": "Full 2,035-task evaluation completed October 4. Two judges score the same outputs; the configured GPT-4.1 mini source remains the displayed result when date and coverage tie.", "score_aggregation": "Mean over evaluated examples", "complete": true, "n": 2035, "expected_n": 2035, "subset_counts": { "v_generation": 650, "v_conversion": 435, "t_generation": 250, "t_conversion": 700 }, "metrics": { "overall": 74.26, "renderable": 66.9, "non_renderable": 82.67, "render_rate": 89.29, "t_generation": 89.36, "t_conversion": 80.27, "v_generation": 61.35, "v_conversion": 75.2 }, "artifact_url": "./static/data/artifacts/nemotron-ultra-gpt41mini-20261001.json" }, { "name": "Gemini 3.8 Flash", "provider": "Google", "access": "closed", "release_date": "2026-09-02", "release_note": "", "parameters_b": null, "active_parameters_b": null, "release_source": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/3-8-flash-and-3-8-flash-cyber/", "parameters_source": null, "id": "gemini-3.8-flash-luna-20261010", "model_key": "gemini-3.8-flash", "evaluation": "experiment_results/model_sweep_20261004/generations/google__gemini-3.8-flash/evaluation_luna.json", "evaluated_on": "2026-10-10", "judge": "GPT-6 Luna", "protocol_source": "experiment_results/model_sweep_20261004/score_all.py", "notes": "Full 2,035-task run; VQA judge: GPT-6 Luna via OpenRouter. No successful renders recorded for Angular, Matplotlib, Typst. Recorded failures remain included in scores.", "generation_protocol_source": "experiment_results/model_sweep_20261004/infer_openrouter.py", "score_aggregation": "Mean over evaluated examples", "complete": true, "n": 2035, "expected_n": 2035, "subset_counts": { "v_generation": 650, "v_conversion": 435, "t_generation": 250, "t_conversion": 700 }, "metrics": { "overall": 73.59, "renderable": 65.92, "non_renderable": 82.35, "render_rate": 86.83, "t_generation": 93.87, "t_conversion": 78.23, "v_generation": 58.17, "v_conversion": 77.51 }, "artifact_url": "./static/data/artifacts/gemini-3.8-flash-luna-20261010.json" }, { "name": "Gemini 3.1 Flash-Lite", "provider": "Google", "access": "closed", "release_date": "2026-03-03", "release_note": "Initial preview availability.", "parameters_b": null, "active_parameters_b": null, "release_source": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-3-1-flash-lite/", "parameters_source": null, "id": "gemini-3.1-flash-lite-luna-20261010", "model_key": "gemini-3.1-flash-lite", "evaluation": "experiment_results/model_sweep_20261004/generations/google__gemini-3.1-flash-lite/evaluation_luna.json", "evaluated_on": "2026-10-10", "judge": "GPT-6 Luna", "protocol_source": "experiment_results/model_sweep_20261004/score_all.py", "notes": "Full 2,035-task run; VQA judge: GPT-6 Luna via OpenRouter. No successful renders recorded for Angular, Matplotlib, Typst. Recorded failures remain included in scores.", "generation_protocol_source": "experiment_results/model_sweep_20261004/infer_openrouter.py", "score_aggregation": "Mean over evaluated examples", "complete": true, "n": 2035, "expected_n": 2035, "subset_counts": { "v_generation": 650, "v_conversion": 435, "t_generation": 250, "t_conversion": 700 }, "metrics": { "overall": 73.37, "renderable": 66.22, "non_renderable": 81.54, "render_rate": 87.62, "t_generation": 93.14, "t_conversion": 77.39, "v_generation": 63.28, "v_conversion": 70.6 }, "artifact_url": "./static/data/artifacts/gemini-3.1-flash-lite-luna-20261010.json" }, { "name": "GPT-4o mini", "provider": "OpenAI", "access": "closed", "release_date": "2024-07-18", "release_note": "", "parameters_b": null, "active_parameters_b": null, "release_source": "https://developers.openai.com/api/docs/changelog", "parameters_source": null, "id": "paper-3", "model_key": "GPT-4o-mini", "n": 2035, "expected_n": 2035, "complete": true, "score_aggregation": "Reported average of four subset scores", "evaluated_on": null, "judge": "Published paper protocol", "metrics": { "overall": 73.19, "t_generation": 79.86, "t_conversion": 75.57, "v_generation": 60.77, "v_conversion": 76.54 }, "artifact_url": "./static/data/paper-baseline.json", "notes": "Original website paper results; exact evaluation date and model snapshot not recorded in that table." }, { "name": "Tencent Hy3", "provider": "Tencent", "access": "open", "release_date": "2026-07-06", "release_note": "Official release, not the April preview. The model card lists an additional 3.8B MTP layer separately.", "parameters_b": 295, "active_parameters_b": 21, "release_source": "https://www.tencent.com/tencent-hunyuan-officially-releases-hy3-advancing-agent-capabilities-and-deeper-product-integration/", "parameters_source": "https://huggingface.co/tencent/Hy3", "id": "tencent-hy3-luna-20261010", "model_key": "tencent-hy3", "evaluation": "experiment_results/model_sweep_20261004/generations/tencent__hy3/evaluation_luna.json", "evaluated_on": "2026-10-10", "judge": "GPT-6 Luna", "protocol_source": "experiment_results/model_sweep_20261004/score_all.py", "notes": "Full 2,035-task run; VQA judge: GPT-6 Luna via OpenRouter. No successful renders recorded for Matplotlib, Typst. Recorded failures remain included in scores.", "generation_protocol_source": "experiment_results/model_sweep_20261004/infer_openrouter.py", "score_aggregation": "Mean over evaluated examples", "complete": true, "n": 2035, "expected_n": 2035, "subset_counts": { "v_generation": 650, "v_conversion": 435, "t_generation": 250, "t_conversion": 700 }, "metrics": { "overall": 73.14, "renderable": 69.1, "non_renderable": 77.76, "render_rate": 88.8, "t_generation": 89.98, "t_conversion": 73.4, "v_generation": 65.36, "v_conversion": 74.7 }, "artifact_url": "./static/data/artifacts/tencent-hy3-luna-20261010.json" }, { "name": "DeepSeek V3.2", "provider": "DeepSeek", "access": "open", "release_date": "2025-12-01", "release_note": "Total size as reported for the published checkpoint; active count not recorded in the reviewed source.", "parameters_b": 685, "active_parameters_b": null, "release_source": "https://api-docs.deepseek.com/news/news251201/", "parameters_source": "https://huggingface.co/deepseek-ai/DeepSeek-V3.2", "id": "deepseek-v3.2-luna-20261010", "model_key": "deepseek-v3.2", "evaluation": "experiment_results/model_sweep_20261004/generations/deepseek__deepseek-v3.2/evaluation_luna.json", "evaluated_on": "2026-10-10", "judge": "GPT-6 Luna", "protocol_source": "experiment_results/model_sweep_20261004/score_all.py", "notes": "Full 2,035-task run; VQA judge: GPT-6 Luna via OpenRouter. No successful renders recorded for Matplotlib, Typst. Recorded failures remain included in scores.", "generation_protocol_source": "experiment_results/model_sweep_20261004/infer_openrouter.py", "score_aggregation": "Mean over evaluated examples", "complete": true, "n": 2035, "expected_n": 2035, "subset_counts": { "v_generation": 650, "v_conversion": 435, "t_generation": 250, "t_conversion": 700 }, "metrics": { "overall": 72.88, "renderable": 70.23, "non_renderable": 75.91, "render_rate": 88.89, "t_generation": 84.56, "t_conversion": 72.82, "v_generation": 64.48, "v_conversion": 78.82 }, "artifact_url": "./static/data/artifacts/deepseek-v3.2-luna-20261010.json" }, { "name": "Kimi K2.5", "provider": "Moonshot AI", "access": "open", "release_date": "2026-01-27", "release_note": "Total and active MoE parameter counts follow the official model card.", "parameters_b": 1000, "active_parameters_b": 32, "release_source": "https://www.kimi.ai/ai-models/kimi-k2-5", "parameters_source": "https://huggingface.co/moonshotai/Kimi-K2.5", "id": "kimi-k2.5-luna-20261010", "model_key": "kimi-k2.5", "evaluation": "experiment_results/model_sweep_20261004/generations/moonshotai__kimi-k2.5/evaluation_luna.json", "evaluated_on": "2026-10-10", "judge": "GPT-6 Luna", "protocol_source": "experiment_results/model_sweep_20261004/score_priority.py", "notes": "Full 2,035-task run; VQA judge: GPT-6 Luna via OpenRouter. No successful renders recorded for Matplotlib, Typst. Recorded failures remain included in scores.", "generation_protocol_source": "experiment_results/model_sweep_20261004/infer_openrouter.py", "score_aggregation": "Mean over evaluated examples", "complete": true, "n": 2035, "expected_n": 2035, "subset_counts": { "v_generation": 650, "v_conversion": 435, "t_generation": 250, "t_conversion": 700 }, "metrics": { "overall": 72.79, "renderable": 68.65, "non_renderable": 77.52, "render_rate": 89.43, "t_generation": 91.89, "t_conversion": 72.39, "v_generation": 62.42, "v_conversion": 77.97 }, "artifact_url": "./static/data/artifacts/kimi-k2.5-luna-20261010.json" }, { "name": "Gemma 4 31B IT", "provider": "Google", "access": "open", "release_date": "2026-03-31", "release_note": "Official model card reports 30.7B parameters for the dense model; the vision encoder is listed separately at approximately 0.55B.", "parameters_b": 30.7, "active_parameters_b": null, "release_source": "https://ai.google.dev/gemma/docs/releases", "parameters_source": "https://ai.google.dev/gemma/docs/core/model_card_4", "id": "gemma-4-31b-it-luna-20261010", "model_key": "gemma-4-31b-it", "evaluation": "experiment_results/model_sweep_20261004/generations/google__gemma-4-31b-it/evaluation_luna.json", "evaluated_on": "2026-10-10", "judge": "GPT-6 Luna", "protocol_source": "experiment_results/model_sweep_20261004/score_all.py", "notes": "Full 2,035-task run; VQA judge: GPT-6 Luna via OpenRouter. No successful renders recorded for Angular, Matplotlib, Typst. Recorded failures remain included in scores.", "generation_protocol_source": "experiment_results/model_sweep_20261004/infer_openrouter.py", "score_aggregation": "Mean over evaluated examples", "complete": true, "n": 2035, "expected_n": 2035, "subset_counts": { "v_generation": 650, "v_conversion": 435, "t_generation": 250, "t_conversion": 700 }, "metrics": { "overall": 71.78, "renderable": 63.74, "non_renderable": 80.96, "render_rate": 87.47, "t_generation": 94.33, "t_conversion": 76.19, "v_generation": 58.65, "v_conversion": 71.36 }, "artifact_url": "./static/data/artifacts/gemma-4-31b-it-luna-20261010.json" }, { "name": "Gemini 1.5 Pro", "provider": "Google", "access": "closed", "release_date": "2024-02-15", "release_note": "Initial preview availability; evaluated snapshot was not recorded in the original table.", "parameters_b": null, "active_parameters_b": null, "release_source": "https://blog.google/innovation-and-ai/products/google-gemini-next-generation-model-february-2024/", "parameters_source": null, "id": "paper-4", "model_key": "Gemini-1.5-pro", "n": 2035, "expected_n": 2035, "complete": true, "score_aggregation": "Reported average of four subset scores", "evaluated_on": null, "judge": "Published paper protocol", "metrics": { "overall": 71.75, "t_generation": 88.07, "t_conversion": 74.24, "v_generation": 58.11, "v_conversion": 66.59 }, "artifact_url": "./static/data/paper-baseline.json", "notes": "Original website paper results; exact evaluation date and model snapshot not recorded in that table." }, { "name": "Qwen3.5 9B (Ollama)", "provider": "Qwen", "access": "open", "release_date": "2026-03-02", "release_note": "The model card reports 9B language model parameters; a vision encoder is listed separately. Evaluated Ollama checkpoint quantization was not recorded.", "parameters_b": 9, "active_parameters_b": null, "release_source": "https://github.com/QwenLM/Qwen3.8/blob/main/README.md#news", "parameters_source": "https://huggingface.co/Qwen/Qwen3.5-9B", "id": "qwen3.5-9b-luna-20261011", "model_key": "qwen3.5-9b", "evaluation": "experiment_results/model_sweep_20261004/local_generations/qwen3.5-9b/evaluation_luna.json", "evaluated_on": "2026-10-11", "judge": "GPT-6 Luna", "protocol_source": "experiment_results/model_sweep_20261004/score_all.py", "notes": "Full 2,035-task run; VQA judge: GPT-6 Luna via OpenRouter. No successful renders recorded for Matplotlib, Typst. Recorded failures remain included in scores. Ollama local generation, temperature 0, 8,192-token context and output limits; checkpoint quantization was not recorded.", "generation_protocol_source": "experiment_results/model_sweep_20261004/infer_local.py", "score_aggregation": "Mean over evaluated examples", "complete": true, "n": 2035, "expected_n": 2035, "subset_counts": { "v_generation": 650, "v_conversion": 435, "t_generation": 250, "t_conversion": 700 }, "metrics": { "overall": 70.96, "renderable": 62.68, "non_renderable": 80.43, "render_rate": 86.04, "t_generation": 78.73, "t_conversion": 81.03, "v_generation": 58.76, "v_conversion": 68.53 }, "artifact_url": "./static/data/artifacts/qwen3.5-9b-luna-20261011.json" }, { "name": "Mistral Small 4", "provider": "Mistral", "access": "open", "release_date": "2026-03-16", "release_note": "", "parameters_b": 119, "active_parameters_b": 6.5, "release_source": "https://mistral.ai/news/mistral-small-4/", "parameters_source": "https://docs.mistral.ai/models/mistral-small-4-0-26-03", "id": "mistral-small-2603-luna-20261010", "model_key": "mistral-small-2603", "evaluation": "experiment_results/model_sweep_20261004/generations/mistralai__mistral-small-2603/evaluation_luna.json", "evaluated_on": "2026-10-10", "judge": "GPT-6 Luna", "protocol_source": "experiment_results/model_sweep_20261004/score_all.py", "notes": "Full 2,035-task run; VQA judge: GPT-6 Luna via OpenRouter. No successful renders recorded for Matplotlib, Typst. Recorded failures remain included in scores.", "generation_protocol_source": "experiment_results/model_sweep_20261004/infer_openrouter.py", "score_aggregation": "Mean over evaluated examples", "complete": true, "n": 2035, "expected_n": 2035, "subset_counts": { "v_generation": 650, "v_conversion": 435, "t_generation": 250, "t_conversion": 700 }, "metrics": { "overall": 70.07, "renderable": 62.26, "non_renderable": 79.01, "render_rate": 82.95, "t_generation": 80.6, "t_conversion": 78.44, "v_generation": 62.54, "v_conversion": 61.84 }, "artifact_url": "./static/data/artifacts/mistral-small-2603-luna-20261010.json" }, { "name": "Gemini 2.5 Flash-Lite", "provider": "Google", "access": "closed", "release_date": "2025-07-22", "release_note": "Stable general availability; preview was available earlier.", "parameters_b": null, "active_parameters_b": null, "release_source": "https://developers.googleblog.com/en/gemini-25-flash-lite-is-now-stable-and-generally-available/", "parameters_source": null, "id": "gemini-2.5-flash-lite-luna-20261004", "model_key": "gemini-2.5-flash-lite", "evaluation": "experiment_results/model_sweep_20261004/generations/google__gemini-2.5-flash-lite/evaluation_luna.json", "evaluated_on": "2026-10-04", "judge": "GPT-6 Luna", "protocol_source": "experiment_results/model_sweep_20261004/score_all.py", "notes": "Full 2,035-task run; VQA judge: GPT-6 Luna via OpenRouter. No successful renders recorded for Matplotlib, Typst. Recorded failures remain included in scores.", "generation_protocol_source": "experiment_results/model_sweep_20261004/infer_openrouter.py", "score_aggregation": "Mean over evaluated examples", "complete": true, "n": 2035, "expected_n": 2035, "subset_counts": { "v_generation": 650, "v_conversion": 435, "t_generation": 250, "t_conversion": 700 }, "metrics": { "overall": 69.85, "renderable": 65.24, "non_renderable": 75.11, "render_rate": 86.93, "t_generation": 81.6, "t_conversion": 72.79, "v_generation": 60.83, "v_conversion": 71.83 }, "artifact_url": "./static/data/artifacts/gemini-2.5-flash-lite-luna-20261004.json" }, { "name": "GLM 5.3 Flash", "provider": "Z.ai", "access": "open", "release_date": "2026-08-25", "release_note": "Weight release date documented by NVIDIA; parameter counts follow Z.ai documentation.", "parameters_b": 320, "active_parameters_b": 18, "release_source": "https://docs.api.nvidia.com/nim/reference/z-ai-glm-5-3-flash", "parameters_source": "https://docs.z.ai/guides/vlm/glm-5.3-flash", "id": "glm-5.3-flash-luna-20261010", "model_key": "glm-5.3-flash", "evaluation": "experiment_results/model_sweep_20261004/generations/z-ai__glm-5.3-flash/evaluation_luna.json", "evaluated_on": "2026-10-10", "judge": "GPT-6 Luna", "protocol_source": "experiment_results/model_sweep_20261004/score_all.py", "notes": "Full 2,035-task run; VQA judge: GPT-6 Luna via OpenRouter. No successful renders recorded for Matplotlib, Typst. Recorded failures remain included in scores.", "generation_protocol_source": "experiment_results/model_sweep_20261004/infer_openrouter.py", "score_aggregation": "Mean over evaluated examples", "complete": true, "n": 2035, "expected_n": 2035, "subset_counts": { "v_generation": 650, "v_conversion": 435, "t_generation": 250, "t_conversion": 700 }, "metrics": { "overall": 69.76, "renderable": 60.1, "non_renderable": 80.79, "render_rate": 83.19, "t_generation": 93.76, "t_conversion": 76.16, "v_generation": 62.76, "v_conversion": 56.13 }, "artifact_url": "./static/data/artifacts/glm-5.3-flash-luna-20261010.json" }, { "name": "GPT-5.4 nano", "provider": "OpenAI", "access": "closed", "release_date": "2026-03-17", "release_note": "", "parameters_b": null, "active_parameters_b": null, "release_source": "https://developers.openai.com/api/docs/changelog", "parameters_source": null, "id": "gpt-5.4-nano-20260611", "model_key": "gpt-5.4-nano", "evaluation": "experiment_results/openrouter_full_20260611T064211Z/gpt-5.4-nano/evaluation_eval.json", "evaluated_on": "2026-06-11", "judge": "GPT-4.1 mini", "protocol_source": "experiment_results/openrouter_full_20260611T064211Z/gpt-5.4-nano/evaluate_eval.log", "notes": "", "score_aggregation": "Mean over evaluated examples", "complete": true, "n": 2035, "expected_n": 2035, "subset_counts": { "v_generation": 650, "v_conversion": 435, "t_generation": 250, "t_conversion": 700 }, "metrics": { "overall": 69.75, "renderable": 62.12, "non_renderable": 78.46, "render_rate": 85.9, "t_generation": 81.79, "t_conversion": 77.28, "v_generation": 60.03, "v_conversion": 65.24 }, "artifact_url": "./static/data/artifacts/gpt-5.4-nano-20260611.json" }, { "name": "Qwen3 30B A3B Instruct 2507", "provider": "Qwen", "access": "open", "release_date": "2025-07-30", "release_note": "", "parameters_b": 30.5, "active_parameters_b": 3.3, "release_source": "https://github.com/QwenLM/Qwen3#news", "parameters_source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507", "id": "qwen3-30b-a3b-instruct-2507-luna-20261010", "model_key": "qwen3-30b-a3b-instruct-2507", "evaluation": "experiment_results/model_sweep_20261004/generations/qwen__qwen3-30b-a3b-instruct-2507/evaluation_luna.json", "evaluated_on": "2026-10-10", "judge": "GPT-6 Luna", "protocol_source": "experiment_results/model_sweep_20261004/score_all.py", "notes": "Full 2,035-task run; VQA judge: GPT-6 Luna via OpenRouter. No successful renders recorded for Matplotlib, Typst. Recorded failures remain included in scores.", "generation_protocol_source": "experiment_results/model_sweep_20261004/infer_openrouter.py", "score_aggregation": "Mean over evaluated examples", "complete": true, "n": 2035, "expected_n": 2035, "subset_counts": { "v_generation": 650, "v_conversion": 435, "t_generation": 250, "t_conversion": 700 }, "metrics": { "overall": 69.54, "renderable": 66.51, "non_renderable": 73.01, "render_rate": 86.58, "t_generation": 73.14, "t_conversion": 72.96, "v_generation": 62.72, "v_conversion": 72.17 }, "artifact_url": "./static/data/artifacts/qwen3-30b-a3b-instruct-2507-luna-20261010.json" }, { "name": "Qwen3 4B", "provider": "Qwen", "access": "open", "release_date": "2025-04-29", "release_note": "", "parameters_b": 4, "active_parameters_b": null, "release_source": "https://qwenlm.github.io/blog/qwen3/", "parameters_source": "https://huggingface.co/Qwen/Qwen3-4B", "id": "paper-5", "model_key": "Qwen3-4B", "n": 2035, "expected_n": 2035, "complete": true, "score_aggregation": "Reported average of four subset scores", "evaluated_on": null, "judge": "Published paper protocol", "metrics": { "overall": 67.04, "t_generation": 64.95, "t_conversion": 81.13, "v_generation": 57.0, "v_conversion": 65.08 }, "artifact_url": "./static/data/paper-baseline.json", "notes": "Original website paper results; exact evaluation date and model snapshot not recorded in that table." }, { "name": "Gemma 2 9B (Ollama)", "provider": "Google", "access": "open", "release_date": "2024-06-27", "release_note": "9B is the vendor-reported size class. Evaluated Ollama checkpoint quantization was not recorded.", "parameters_b": 9, "active_parameters_b": null, "release_source": "https://ai.google.dev/gemma/docs/releases", "parameters_source": "https://huggingface.co/google/gemma-2-9b-it", "id": "gemma2-9b-luna-20261011", "model_key": "gemma2-9b", "evaluation": "experiment_results/model_sweep_20261004/local_generations/gemma2-9b/evaluation_luna.json", "evaluated_on": "2026-10-11", "judge": "GPT-6 Luna", "protocol_source": "experiment_results/model_sweep_20261004/score_all.py", "notes": "Full 2,035-task run; VQA judge: GPT-6 Luna via OpenRouter. No successful renders recorded for Matplotlib, TOML, Typst. Recorded failures remain included in scores. Ollama local generation, temperature 0, 8,192-token context and output limits; checkpoint quantization was not recorded. Three generation requests failed with HTTP 500 after retries and are included as empty outputs in the 2,035-task score (at most 0.15 percentage points of the aggregate).", "generation_protocol_source": "experiment_results/model_sweep_20261004/infer_local.py", "score_aggregation": "Mean over evaluated examples", "complete": true, "n": 2035, "expected_n": 2035, "subset_counts": { "v_generation": 650, "v_conversion": 435, "t_generation": 250, "t_conversion": 700 }, "metrics": { "overall": 66.29, "renderable": 57.41, "non_renderable": 76.43, "render_rate": 83.0, "t_generation": 70.3, "t_conversion": 78.62, "v_generation": 53.1, "v_conversion": 63.84 }, "artifact_url": "./static/data/artifacts/gemma2-9b-luna-20261011.json" }, { "name": "Qwen3 8B (Ollama)", "provider": "Qwen", "access": "open", "release_date": "2025-04-29", "release_note": "The evaluated Ollama checkpoint quantization was not recorded.", "parameters_b": 8.2, "active_parameters_b": null, "release_source": "https://qwenlm.github.io/blog/qwen3/", "parameters_source": "https://huggingface.co/Qwen/Qwen3-8B", "id": "qwen3-8b-luna-20261010", "model_key": "Qwen3-8B", "evaluation": "experiment_results/model_sweep_20261004/local_generations/qwen3-8b/evaluation_luna.json", "evaluated_on": "2026-10-10", "judge": "GPT-6 Luna", "protocol_source": "experiment_results/model_sweep_20261004/score_all.py", "notes": "Full 2,035-task run; VQA judge: GPT-6 Luna via OpenRouter. No successful renders recorded for Matplotlib, Typst. Recorded failures remain included in scores. Ollama local generation, temperature 0, 8,192-token context and output limits; checkpoint quantization was not recorded.", "generation_protocol_source": "experiment_results/model_sweep_20261004/infer_local.py", "score_aggregation": "Mean over evaluated examples", "complete": true, "n": 2035, "expected_n": 2035, "subset_counts": { "v_generation": 650, "v_conversion": 435, "t_generation": 250, "t_conversion": 700 }, "metrics": { "overall": 64.71, "renderable": 57.14, "non_renderable": 73.36, "render_rate": 81.13, "t_generation": 66.54, "t_conversion": 75.79, "v_generation": 57.7, "v_conversion": 56.3 }, "artifact_url": "./static/data/artifacts/qwen3-8b-luna-20261010.json" }, { "name": "Gemma 4 26B A4B IT", "provider": "Google", "access": "open", "release_date": "2026-03-31", "release_note": "Official model-card MoE parameter counts; the vision encoder is listed separately at approximately 0.55B.", "parameters_b": 25.2, "active_parameters_b": 3.8, "release_source": "https://ai.google.dev/gemma/docs/releases", "parameters_source": "https://ai.google.dev/gemma/docs/core/model_card_4", "id": "gemma-4-26b-a4b-it-luna-20261010", "model_key": "gemma-4-26b-a4b-it", "evaluation": "experiment_results/model_sweep_20261004/generations/google__gemma-4-26b-a4b-it/evaluation_luna.json", "evaluated_on": "2026-10-10", "judge": "GPT-6 Luna", "protocol_source": "experiment_results/model_sweep_20261004/score_all.py", "notes": "Full 2,035-task run; VQA judge: GPT-6 Luna via OpenRouter. No successful renders recorded for Angular, Matplotlib, Typst. Recorded failures remain included in scores.", "generation_protocol_source": "experiment_results/model_sweep_20261004/infer_openrouter.py", "score_aggregation": "Mean over evaluated examples", "complete": true, "n": 2035, "expected_n": 2035, "subset_counts": { "v_generation": 650, "v_conversion": 435, "t_generation": 250, "t_conversion": 700 }, "metrics": { "overall": 64.24, "renderable": 56.86, "non_renderable": 72.67, "render_rate": 81.38, "t_generation": 65.37, "t_conversion": 75.28, "v_generation": 53.37, "v_conversion": 62.09 }, "artifact_url": "./static/data/artifacts/gemma-4-26b-a4b-it-luna-20261010.json" }, { "name": "Gemini 2.0 Flash", "provider": "Google", "access": "closed", "release_date": "2025-02-05", "release_note": "General availability of the stable model; experimental preview was released earlier.", "parameters_b": null, "active_parameters_b": null, "release_source": "https://blog.google/innovation-and-ai/models-and-research/google-deepmind/gemini-model-updates-february-2025/", "parameters_source": null, "id": "paper-7", "model_key": "Gemini-2.0-flash", "n": 2035, "expected_n": 2035, "complete": true, "score_aggregation": "Reported average of four subset scores", "evaluated_on": null, "judge": "Published paper protocol", "metrics": { "overall": 62.55, "t_generation": 72.42, "t_conversion": 72.2, "v_generation": 53.62, "v_conversion": 51.97 }, "artifact_url": "./static/data/paper-baseline.json", "notes": "Original website paper results; exact evaluation date and model snapshot not recorded in that table." }, { "name": "Llama 3.1 8B Instruct", "provider": "Meta", "access": "open", "release_date": "2024-07-23", "release_note": "Vendor-reported 8B size class.", "parameters_b": 8, "active_parameters_b": null, "release_source": "https://ai.meta.com/blog/meta-llama-3-1/", "parameters_source": "https://ai.meta.com/blog/meta-llama-3-1/", "id": "paper-6", "model_key": "Llama-3.1-8B-Instruct", "n": 2035, "expected_n": 2035, "complete": true, "score_aggregation": "Reported average of four subset scores", "evaluated_on": null, "judge": "Published paper protocol", "metrics": { "overall": 61.77, "t_generation": 60.22, "t_conversion": 71.26, "v_generation": 54.44, "v_conversion": 61.15 }, "artifact_url": "./static/data/paper-baseline.json", "notes": "Original website paper results; exact evaluation date and model snapshot not recorded in that table." }, { "name": "Phi-4 mini", "provider": "Microsoft", "access": "open", "release_date": "2025-02-26", "release_note": "", "parameters_b": 3.8, "active_parameters_b": null, "release_source": "https://azure.microsoft.com/en-us/blog/empowering-innovation-the-next-generation-of-the-phi-family/", "parameters_source": "https://azure.microsoft.com/en-us/blog/empowering-innovation-the-next-generation-of-the-phi-family/", "id": "paper-9", "model_key": "Phi-4-mini-instruct", "n": 2035, "expected_n": 2035, "complete": true, "score_aggregation": "Reported average of four subset scores", "evaluated_on": null, "judge": "Published paper protocol", "metrics": { "overall": 56.97, "t_generation": 51.38, "t_conversion": 72.39, "v_generation": 51.62, "v_conversion": 52.48 }, "artifact_url": "./static/data/paper-baseline.json", "notes": "Original website paper results; exact evaluation date and model snapshot not recorded in that table." }, { "name": "Gemma3 4B (Ollama)", "provider": "Google", "access": "open", "release_date": "2025-03-10", "release_note": "The evaluated Ollama checkpoint quantization was not recorded; 4B is the vendor-reported size class.", "parameters_b": 4, "active_parameters_b": null, "release_source": "https://ai.google.dev/gemma/docs/releases", "parameters_source": "https://huggingface.co/google/gemma-3-4b-it", "id": "gemma3-4b-luna-20261010", "model_key": "gemma3-4b", "evaluation": "experiment_results/model_sweep_20261004/local_generations/gemma3-4b/evaluation_luna.json", "evaluated_on": "2026-10-10", "judge": "GPT-6 Luna", "protocol_source": "experiment_results/model_sweep_20261004/score_all.py", "notes": "Full 2,035-task run; VQA judge: GPT-6 Luna via OpenRouter. No successful renders recorded for Matplotlib, TOML, Typst. Recorded failures remain included in scores. Ollama local generation, temperature 0, 8,192-token context and output limits; checkpoint quantization was not recorded.", "generation_protocol_source": "experiment_results/model_sweep_20261004/infer_local.py", "score_aggregation": "Mean over evaluated examples", "complete": true, "n": 2035, "expected_n": 2035, "subset_counts": { "v_generation": 650, "v_conversion": 435, "t_generation": 250, "t_conversion": 700 }, "metrics": { "overall": 54.14, "renderable": 46.7, "non_renderable": 62.63, "render_rate": 73.66, "t_generation": 60.89, "t_conversion": 63.25, "v_generation": 43.43, "v_conversion": 51.57 }, "artifact_url": "./static/data/artifacts/gemma3-4b-luna-20261010.json" }, { "name": "Llama 3 8B Instruct", "provider": "Meta", "access": "open", "release_date": "2024-04-18", "release_note": "Vendor-reported 8B size class.", "parameters_b": 8, "active_parameters_b": null, "release_source": "https://ai.meta.com/blog/meta-llama-3/", "parameters_source": "https://ai.meta.com/blog/meta-llama-3/", "id": "paper-10", "model_key": "Meta-Llama-3-8B-Instruct", "n": 2035, "expected_n": 2035, "complete": true, "score_aggregation": "Reported average of four subset scores", "evaluated_on": null, "judge": "Published paper protocol", "metrics": { "overall": 51.59, "t_generation": 49.18, "t_conversion": 53.65, "v_generation": 46.61, "v_conversion": 56.91 }, "artifact_url": "./static/data/paper-baseline.json", "notes": "Original website paper results; exact evaluation date and model snapshot not recorded in that table." }, { "name": "Qwen2.5 7B Instruct (Ollama)", "provider": "Qwen", "access": "open", "release_date": "2024-09-19", "release_note": "", "parameters_b": 7.61, "active_parameters_b": null, "release_source": "https://qwenlm.github.io/blog/qwen2.5/", "parameters_source": "https://huggingface.co/Qwen/Qwen2.5-7B-Instruct", "id": "qwen2.5-7b-instruct-luna-20261010", "model_key": "Qwen2.5-7B-Instruct", "evaluation": "experiment_results/model_sweep_20261004/local_generations/qwen2.5-7b/evaluation_luna.json", "evaluated_on": "2026-10-10", "judge": "GPT-6 Luna", "protocol_source": "experiment_results/model_sweep_20261004/score_all.py", "notes": "Full 2,035-task run; VQA judge: GPT-6 Luna via OpenRouter. No successful renders recorded for Matplotlib, Typst. Recorded failures remain included in scores. Ollama qwen2.5:7b, temperature 0, 8,192-token context and output limits; checkpoint quantization was not recorded. This is the latest complete run for this model.", "generation_protocol_source": "experiment_results/model_sweep_20261004/infer_local.py", "score_aggregation": "Mean over evaluated examples", "complete": true, "n": 2035, "expected_n": 2035, "subset_counts": { "v_generation": 650, "v_conversion": 435, "t_generation": 250, "t_conversion": 700 }, "metrics": { "overall": 50.1, "renderable": 49.9, "non_renderable": 50.34, "render_rate": 66.29, "t_generation": 63.76, "t_conversion": 45.54, "v_generation": 55.36, "v_conversion": 41.73 }, "artifact_url": "./static/data/artifacts/qwen2.5-7b-instruct-luna-20261010.json" }, { "name": "Phi-3 mini 128K", "provider": "Microsoft", "access": "open", "release_date": "2024-04-23", "release_note": "", "parameters_b": 3.8, "active_parameters_b": null, "release_source": "https://news.microsoft.com/source/features/ai/the-phi-3-small-language-models-with-big-potential/", "parameters_source": "https://news.microsoft.com/source/features/ai/the-phi-3-small-language-models-with-big-potential/", "id": "paper-11", "model_key": "Phi-3-mini-128k-instruct", "n": 2035, "expected_n": 2035, "complete": true, "score_aggregation": "Reported average of four subset scores", "evaluated_on": null, "judge": "Published paper protocol", "metrics": { "overall": 40.79, "t_generation": 47.39, "t_conversion": 29.78, "v_generation": 44.77, "v_conversion": 41.23 }, "artifact_url": "./static/data/paper-baseline.json", "notes": "Original website paper results; exact evaluation date and model snapshot not recorded in that table." } ] }