{"about":{"non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","site":"https://codewithpapers.app","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page","syntology":{"site":"https://syntology.ai","developers":"https://syntology.ai/developers","mcp":{"server":"https://syntology.ai/mcp","transport":"streamable-http","server_card":"https://syntology.ai/.well-known/mcp/server-card.json","auth":{"type":"trial token, no account","trial_token":"https://syntology.ai/api/oauth/trial/token","method":"POST","docs":"https://syntology.ai/developers"}},"have":"https://syntology.ai/api/graph/have?x=<method, arXiv id or title> (free, answers coverage only)","paper_base":"https://syntology.ai/paper/","atlas_base":"https://app.syntology.ai/?focus="},"machine_readable":[{"url":"https://codewithpapers.app/llms.txt","what":"the machine catalog: every machine-readable file, counted"},{"url":"https://codewithpapers.app/index/manifest.json","what":"paper-to-code index by arXiv id, with Syntology's counts"},{"url":"https://codewithpapers.app/search/manifest.json","what":"site search index (titles, authors) and its files"},{"url":"https://codewithpapers.app/download","what":"bulk files: Syntology's layer, described there"},{"url":"https://codewithpapers.app/build_manifest.json","what":"the build record: inputs, counts, exclusions, probes"}]},"url":"/task/language-model-evaluation","name":"Language Model Evaluation","slug":"language-model-evaluation","description_markdown":"The task of using LLMs as evaluators of large language and vision language models.","categories":[{"name":"Computer Vision","url":"/area/computer-vision"}],"source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","slug_source":"derived"},"counts":{"papers_tagged":69,"papers_with_code":31,"benchmarks":0,"benchmark_tables_in_archive":0,"benchmark_tables_shown":0,"benchmark_tables_withheld_as_spam":0,"benchmark_definition":"a leaderboard table with at least one row; benchmark_tables_shown also counts the zero-row tables; benchmark_tables_in_archive adds the tables withheld as spam","datasets":0,"subtasks":0,"parent_tasks":0},"benchmarks":[],"datasets":[],"subtasks":[],"parent_tasks":[],"papers":{"order":"repositories listed in the archive (most first), then date (newest first), then slug; the archive holds no stars","population":"every paper the archive tags with this task (distinct papers; spam titles not listed). 'of' counts the papers listed; until 2026-09-25 it counted only the papers with a code link, and that number is of_with_code","page":1,"pages_in_order":1,"rows_per_page":100,"shown":69,"of":69,"of_with_code":31,"tagged_in_all":69,"counts":{"archive_papers_tagged":69,"with_a_code_link":31,"where_syntology_ran_a_sample":12,"not_listed_spam_title":0,"listed":69,"listed_where_code_ran":12,"where_syntology_ran_a_sample_split":{"with_a_run_with_no_instrument_failure":11,"every_run_a_failure_of_syntologys_instrument":1,"listed_with_a_run_with_no_instrument_failure":11,"listed_every_run_a_failure_of_syntologys_instrument":1,"filter":{"states":["a run with no instrument failure","any run, instrument failures included"],"default":"a run with no instrument failure","note":"on the 'only where code ran' pages the default hides, in the browser, the rows where every run was a failure of Syntology's instrument; the second state shows them again. Rows are hidden, never re-ordered; these twins list every row"}},"definition":"distinct papers the archive tags; 'where Syntology ran a sample' counts papers with at least one harvested sample that ran, which is not a correctness claim"},"items":[{"url":"/paper/challenge-llms-to-reason-about-reasoning-a","slug":"challenge-llms-to-reason-about-reasoning-a","title":"MR-GSM8K: A Meta-Reasoning Benchmark for Large Language Model Evaluation","date":"2023-12-28","arxiv_id":"2312.17080","repositories_listed":2,"syntology":null},{"url":"/paper/scieval-a-multi-level-large-language-model","slug":"scieval-a-multi-level-large-language-model","title":"SciEval: A Multi-Level Large Language Model Evaluation Benchmark for Scientific Research","date":"2023-08-25","arxiv_id":"2308.13149","repositories_listed":2,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":2,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":2,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/scieval-a-multi-level-large-language-model#ran","syntology_url":"https://syntology.ai/paper/2308.13149","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2308.13149"}},"official":{"repos":["opendfm/bai-scieval","opendfm/scieval"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/bigbio-a-framework-for-data-centric","slug":"bigbio-a-framework-for-data-centric","title":"BigBIO: A Framework for Data-Centric Biomedical Natural Language Processing","date":"2022-06-30","arxiv_id":"2206.15076","repositories_listed":2,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/bigbio-a-framework-for-data-centric#ran","syntology_url":"https://syntology.ai/paper/2206.15076","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2206.15076"}},"official":{"repos":["bigscience-workshop/biomedical"],"state":"official: no sample here; runs from other or unrecorded repositories","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["found_in_text"]}}},{"url":"/paper/fable-a-novel-data-flow-analysis-benchmark-on","slug":"fable-a-novel-data-flow-analysis-benchmark-on","title":"FABLE: A Novel Data-Flow Analysis Benchmark on Procedural Text for Large Language Model Evaluation","date":"2025-05-30","arxiv_id":"2505.24258","repositories_listed":1,"syntology":null},{"url":"/paper/role-playing-evaluation-for-large-language","slug":"role-playing-evaluation-for-large-language","title":"Role-Playing Evaluation for Large Language Models","date":"2025-05-19","arxiv_id":"2505.13157","repositories_listed":1,"syntology":null},{"url":"/paper/m-absa-a-multilingual-dataset-for-aspect","slug":"m-absa-a-multilingual-dataset-for-aspect","title":"M-ABSA: A Multilingual Dataset for Aspect-Based Sentiment Analysis","date":"2025-02-17","arxiv_id":"2502.11824","repositories_listed":1,"syntology":null},{"url":"/paper/environmental-large-language-model-evaluation","slug":"environmental-large-language-model-evaluation","title":"Environmental large language model Evaluation (ELLE) dataset: A Benchmark for Evaluating Generative AI applications in Eco-environment Domain","date":"2025-01-10","arxiv_id":"2501.06277","repositories_listed":1,"syntology":null},{"url":"/paper/automated-generation-of-challenging-multiple","slug":"automated-generation-of-challenging-multiple","title":"Automated Generation of Challenging Multiple-Choice Questions for Vision Language Model Evaluation","date":"2025-01-06","arxiv_id":"2501.03225","repositories_listed":1,"syntology":null},{"url":"/paper/template-matters-understanding-the-role-of","slug":"template-matters-understanding-the-role-of","title":"Template Matters: Understanding the Role of Instruction Templates in Multimodal Language Model Evaluation and Training","date":"2024-12-11","arxiv_id":"2412.08307","repositories_listed":1,"syntology":null},{"url":"/paper/c-2-leva-toward-comprehensive-and","slug":"c-2-leva-toward-comprehensive-and","title":"C$^2$LEVA: Toward Comprehensive and Contamination-Free Language Model Evaluation","date":"2024-12-06","arxiv_id":"2412.04947","repositories_listed":1,"syntology":null},{"url":"/paper/dart-eval-a-comprehensive-dna-language-model","slug":"dart-eval-a-comprehensive-dna-language-model","title":"DART-Eval: A Comprehensive DNA Language Model Evaluation Benchmark on Regulatory DNA","date":"2024-12-06","arxiv_id":"2412.05430","repositories_listed":1,"syntology":{"n":16,"n_ran":11,"n_constructed":0,"n_ran_checked":6,"n_instrument":5,"n_unverified":5,"n_honours":1,"n_violates":0,"n_no_contract":5,"n_pointer_only":16,"phrase":"11 ran (of which 0 constructed an object rather than computing a result; 6 with no instrument failure: 1 honoured, 0 violated, 5 with no contract checked; 5 where Syntology's instrument failed) · 5 unverified","sample_list":"/paper/dart-eval-a-comprehensive-dna-language-model#ran","syntology_url":"https://syntology.ai/paper/2412.05430","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2412.05430"}},"official":{"repos":["kundajelab/dart-eval"],"state":"official (archive's flag): 11 ran","n_ran":11,"n_constructed":0,"n_ran_no_instrument_failure":6,"n_unverified":5,"ran_from_kinds":["official"]}}},{"url":"/paper/large-language-model-evaluation-via-matrix-1","slug":"large-language-model-evaluation-via-matrix-1","title":"Large Language Model Evaluation via Matrix Nuclear-Norm","date":"2024-10-14","arxiv_id":"2410.10672","repositories_listed":1,"syntology":null},{"url":"/paper/enterprise-benchmarks-for-large-language","slug":"enterprise-benchmarks-for-large-language","title":"Enterprise Benchmarks for Large Language Model Evaluation","date":"2024-10-11","arxiv_id":"2410.12857","repositories_listed":1,"syntology":null},{"url":"/paper/mitigating-the-bias-of-large-language-model","slug":"mitigating-the-bias-of-large-language-model","title":"Mitigating the Bias of Large Language Model Evaluation","date":"2024-09-25","arxiv_id":"2409.16788","repositories_listed":1,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":2,"n_instrument":0,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":1,"n_pointer_only":2,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 1 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/mitigating-the-bias-of-large-language-model#ran","syntology_url":"https://syntology.ai/paper/2409.16788","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2409.16788"}},"official":{"repos":["Joe-Hall-Lee/Debias"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/a-suite-for-acoustic-language-model","slug":"a-suite-for-acoustic-language-model","title":"Salmon: A Suite for Acoustic Language Model Evaluation","date":"2024-09-11","arxiv_id":"2409.07437","repositories_listed":1,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":1,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/a-suite-for-acoustic-language-model#ran","syntology_url":"https://syntology.ai/paper/2409.07437","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2409.07437"}},"official":{"repos":["slp-rl/salmon"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/inference-time-decontamination-reusing-leaked","slug":"inference-time-decontamination-reusing-leaked","title":"Inference-Time Decontamination: Reusing Leaked Benchmarks for Large Language Model Evaluation","date":"2024-06-20","arxiv_id":"2406.13990","repositories_listed":1,"syntology":null},{"url":"/paper/fennec-fine-grained-language-model-evaluation","slug":"fennec-fine-grained-language-model-evaluation","title":"Fennec: Fine-grained Language Model Evaluation and Correction Extended through Branching and Bridging","date":"2024-05-20","arxiv_id":"2405.12163","repositories_listed":1,"syntology":null},{"url":"/paper/paraphrase-and-solve-exploring-and-exploiting","slug":"paraphrase-and-solve-exploring-and-exploiting","title":"Paraphrase and Solve: Exploring and Exploiting the Impact of Surface Form on Mathematical Reasoning in Large Language Models","date":"2024-04-17","arxiv_id":"2404.11500","repositories_listed":1,"syntology":{"n":4,"n_ran":3,"n_constructed":0,"n_ran_checked":1,"n_instrument":2,"n_unverified":1,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/paraphrase-and-solve-exploring-and-exploiting#ran","syntology_url":"https://syntology.ai/paper/2404.11500","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2404.11500"}},"official":{"repos":["yue-llm-pit/scop"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/evalverse-unified-and-accessible-library-for","slug":"evalverse-unified-and-accessible-library-for","title":"Evalverse: Unified and Accessible Library for Large Language Model Evaluation","date":"2024-04-01","arxiv_id":"2404.00943","repositories_listed":1,"syntology":null},{"url":"/paper/towards-personalized-evaluation-of-large","slug":"towards-personalized-evaluation-of-large","title":"Towards Personalized Evaluation of Large Language Models with An Anonymous Crowd-Sourcing Platform","date":"2024-03-13","arxiv_id":"2403.08305","repositories_listed":1,"syntology":null},{"url":"/paper/arabicmmlu-assessing-massive-multitask","slug":"arabicmmlu-assessing-massive-multitask","title":"ArabicMMLU: Assessing Massive Multitask Language Understanding in Arabic","date":"2024-02-20","arxiv_id":"2402.12840","repositories_listed":1,"syntology":{"n":6,"n_ran":3,"n_constructed":0,"n_ran_checked":1,"n_instrument":2,"n_unverified":3,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":6,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 2 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/arabicmmlu-assessing-massive-multitask#ran","syntology_url":"https://syntology.ai/paper/2402.12840","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2402.12840"}},"official":{"repos":["mbzuai-nlp/arabicmmlu"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":3,"ran_from_kinds":["official"]}}},{"url":"/paper/avoiding-data-contamination-in-language-model","slug":"avoiding-data-contamination-in-language-model","title":"LatestEval: Addressing Data Contamination in Language Model Evaluation through Dynamic and Time-Sensitive Test Construction","date":"2023-12-19","arxiv_id":"2312.12343","repositories_listed":1,"syntology":{"n":6,"n_ran":2,"n_constructed":0,"n_ran_checked":1,"n_instrument":1,"n_unverified":4,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":6,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 4 unverified","sample_list":"/paper/avoiding-data-contamination-in-language-model#ran","syntology_url":"https://syntology.ai/paper/2312.12343","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2312.12343"}},"official":{"repos":["liyucheng09/latesteval"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":4,"ran_from_kinds":["official"]}}},{"url":"/paper/catwalk-a-unified-language-model-evaluation","slug":"catwalk-a-unified-language-model-evaluation","title":"Catwalk: A Unified Language Model Evaluation Framework for Many Datasets","date":"2023-12-15","arxiv_id":"2312.10253","repositories_listed":1,"syntology":null},{"url":"/paper/a-survey-on-language-models-for-code","slug":"a-survey-on-language-models-for-code","title":"Unifying the Perspectives of NLP and Software Engineering: A Survey on Language Models for Code","date":"2023-11-14","arxiv_id":"2311.07989","repositories_listed":1,"syntology":null},{"url":"/paper/estimating-contamination-via-perplexity","slug":"estimating-contamination-via-perplexity","title":"Estimating Contamination via Perplexity: Quantifying Memorisation in Language Model Evaluation","date":"2023-09-19","arxiv_id":"2309.10677","repositories_listed":1,"syntology":null},{"url":"/paper/agentsims-an-open-source-sandbox-for-large","slug":"agentsims-an-open-source-sandbox-for-large","title":"AgentSims: An Open-Source Sandbox for Large Language Model Evaluation","date":"2023-08-08","arxiv_id":"2308.04026","repositories_listed":1,"syntology":null},{"url":"/paper/flask-fine-grained-language-model-evaluation","slug":"flask-fine-grained-language-model-evaluation","title":"FLASK: Fine-grained Language Model Evaluation based on Alignment Skill Sets","date":"2023-07-20","arxiv_id":"2307.10928","repositories_listed":1,"syntology":{"n":5,"n_ran":5,"n_constructed":0,"n_ran_checked":1,"n_instrument":4,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":5,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 4 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/flask-fine-grained-language-model-evaluation#ran","syntology_url":"https://syntology.ai/paper/2307.10928","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2307.10928"}},"official":{"repos":["kaistai/flask"],"state":"official (archive's flag): 5 ran","n_ran":5,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/csts-conditional-semantic-textual-similarity","slug":"csts-conditional-semantic-textual-similarity","title":"C-STS: Conditional Semantic Textual Similarity","date":"2023-05-24","arxiv_id":"2305.15093","repositories_listed":1,"syntology":{"n":4,"n_ran":2,"n_constructed":2,"n_ran_checked":2,"n_instrument":0,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":4,"phrase":"2 ran (of which 2 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 0 where Syntology's instrument failed) · 2 unverified; every one of the 2 samples that ran constructed an object rather than computing a result","sample_list":"/paper/csts-conditional-semantic-textual-similarity#ran","syntology_url":"https://syntology.ai/paper/2305.15093","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2305.15093"}},"official":{"repos":["princeton-nlp/c-sts"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":2,"n_ran_no_instrument_failure":2,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/pronto-language-model-evaluations-for-859","slug":"pronto-language-model-evaluations-for-859","title":"PrOnto: Language Model Evaluations for 859 Languages","date":"2023-05-22","arxiv_id":"2305.12612","repositories_listed":1,"syntology":null},{"url":"/paper/zjuklab-at-semeval-2021-task-4-negative","slug":"zjuklab-at-semeval-2021-task-4-negative","title":"ZJUKLAB at SemEval-2021 Task 4: Negative Augmentation with Language Model for Reading Comprehension of Abstract Meaning","date":"2021-02-25","arxiv_id":"2102.12828","repositories_listed":1,"syntology":null},{"url":"/paper/pitfalls-of-static-language-modelling","slug":"pitfalls-of-static-language-modelling","title":"Mind the Gap: Assessing Temporal Generalization in Neural Language Models","date":"2021-02-03","arxiv_id":"2102.01951","repositories_listed":1,"syntology":null},{"url":null,"slug":"enterprise-large-language-model-evaluation","title":"Enterprise Large Language Model Evaluation Benchmark","date":"2025-06-25","arxiv_id":"2506.20274","repositories_listed":0,"syntology":null},{"url":null,"slug":"finance-language-model-evaluation-flame","title":"Finance Language Model Evaluation (FLaME)","date":"2025-06-18","arxiv_id":"2506.15846","repositories_listed":0,"syntology":null},{"url":null,"slug":"behaviorbox-automated-discovery-of-fine","title":"BehaviorBox: Automated Discovery of Fine-Grained Performance Differences Between Language Models","date":"2025-06-02","arxiv_id":"2506.02204","repositories_listed":0,"syntology":null},{"url":null,"slug":"r-bench-graduate-level-multi-disciplinary","title":"R-Bench: Graduate-level Multi-disciplinary Benchmarks for LLM & MLLM Complex Reasoning Evaluation","date":"2025-05-04","arxiv_id":"2505.02018","repositories_listed":0,"syntology":null},{"url":null,"slug":"confidence-in-large-language-model-evaluation","title":"Confidence in Large Language Model Evaluation: A Bayesian Approach to Limited-Sample Challenges","date":"2025-04-30","arxiv_id":"2504.21303","repositories_listed":0,"syntology":null},{"url":null,"slug":"coco-bench-a-comprehensive-code-benchmark-for","title":"CoCo-Bench: A Comprehensive Code Benchmark For Multi-task Large Language Model Evaluation","date":"2025-04-29","arxiv_id":"2504.20673","repositories_listed":0,"syntology":null},{"url":null,"slug":"upme-an-unsupervised-peer-review-framework","title":"UPME: An Unsupervised Peer Review Framework for Multimodal Large Language Model Evaluation","date":"2025-03-19","arxiv_id":"2503.14941","repositories_listed":0,"syntology":null},{"url":null,"slug":"mmlu-prox-a-multilingual-benchmark-for","title":"MMLU-ProX: A Multilingual Benchmark for Advanced Large Language Model Evaluation","date":"2025-03-13","arxiv_id":"2503.10497","repositories_listed":0,"syntology":null},{"url":null,"slug":"predicting-liquidity-aware-bond-yields-using","title":"Predicting Liquidity-Aware Bond Yields using Causal GANs and Deep Reinforcement Learning with LLM Evaluation","date":"2025-02-24","arxiv_id":"2502.17011","repositories_listed":0,"syntology":null},{"url":null,"slug":"setting-standards-in-turkish-nlp-tr-mmlu-for","title":"Setting Standards in Turkish NLP: TR-MMLU for Large Language Model Evaluation","date":"2024-12-31","arxiv_id":"2501.00593","repositories_listed":0,"syntology":null},{"url":null,"slug":"lmunit-fine-grained-evaluation-with-natural","title":"LMUnit: Fine-grained Evaluation with Natural Language Unit Tests","date":"2024-12-17","arxiv_id":"2412.13091","repositories_listed":0,"syntology":null},{"url":null,"slug":"benchmarking-harmonized-tariff-schedule","title":"Benchmarking Harmonized Tariff Schedule Classification Models","date":"2024-12-04","arxiv_id":"2412.14179","repositories_listed":0,"syntology":null},{"url":null,"slug":"vidas-vision-based-danger-assessment-and","title":"ViDAS: Vision-based Danger Assessment and Scoring","date":"2024-10-01","arxiv_id":"2410.00477","repositories_listed":0,"syntology":null},{"url":null,"slug":"beyond-metrics-a-critical-analysis-of-the","title":"Beyond Metrics: A Critical Analysis of the Variability in Large Language Model Evaluation Frameworks","date":"2024-07-29","arxiv_id":"2407.21072","repositories_listed":0,"syntology":null},{"url":"/paper/on-speeding-up-language-model-evaluation","slug":"on-speeding-up-language-model-evaluation","title":"On Speeding Up Language Model Evaluation","date":"2024-07-08","arxiv_id":"2407.06172","repositories_listed":0,"syntology":{"n":2,"n_ran":2,"n_constructed":1,"n_ran_checked":1,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"2 ran (of which 1 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/on-speeding-up-language-model-evaluation#ran","syntology_url":"https://syntology.ai/paper/2407.06172","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2407.06172"}},"official":null}},{"url":null,"slug":"stratified-prediction-powered-inference-for","title":"Stratified Prediction-Powered Inference for Hybrid Language Model Evaluation","date":"2024-06-06","arxiv_id":"2406.04291","repositories_listed":0,"syntology":null},{"url":null,"slug":"decompose-and-aggregate-a-step-by-step","title":"DnA-Eval: Enhancing Large Language Model Evaluation through Decomposition and Aggregation","date":"2024-05-24","arxiv_id":"2405.15329","repositories_listed":0,"syntology":null},{"url":null,"slug":"i-repo-i-mplicit-reward-pairwise-difference","title":"$i$REPO: $i$mplicit Reward Pairwise Difference based Empirical Preference Optimization","date":"2024-05-24","arxiv_id":"2405.15230","repositories_listed":0,"syntology":null},{"url":null,"slug":"lessons-from-the-trenches-on-reproducible","title":"Lessons from the Trenches on Reproducible Evaluation of Language Models","date":"2024-05-23","arxiv_id":"2405.14782","repositories_listed":0,"syntology":null},{"url":null,"slug":"generalization-measures-for-zero-shot-cross","title":"Generalization Measures for Zero-Shot Cross-Lingual Transfer","date":"2024-04-24","arxiv_id":"2404.15928","repositories_listed":0,"syntology":null},{"url":null,"slug":"rethinking-generative-large-language-model","title":"Rethinking Generative Large Language Model Evaluation for Semantic Comprehension","date":"2024-03-12","arxiv_id":"2403.07872","repositories_listed":0,"syntology":null},{"url":null,"slug":"advancing-biomedical-text-mining-with","title":"Advancing Chinese biomedical text mining with community challenges","date":"2024-03-07","arxiv_id":"2403.04261","repositories_listed":0,"syntology":null},{"url":null,"slug":"kmmlu-measuring-massive-multitask-language","title":"KMMLU: Measuring Massive Multitask Language Understanding in Korean","date":"2024-02-18","arxiv_id":"2402.11548","repositories_listed":0,"syntology":null},{"url":null,"slug":"cpsdbench-a-large-language-model-evaluation","title":"CPSDBench: A Large Language Model Evaluation Benchmark and Baseline for Chinese Public Security Domain","date":"2024-02-11","arxiv_id":"2402.07234","repositories_listed":0,"syntology":null},{"url":null,"slug":"elo-uncovered-robustness-and-best-practices","title":"Elo Uncovered: Robustness and Best Practices in Language Model Evaluation","date":"2023-11-29","arxiv_id":"2311.17295","repositories_listed":0,"syntology":null},{"url":null,"slug":"branch-solve-merge-improves-large-language","title":"Branch-Solve-Merge Improves Large Language Model Evaluation and Generation","date":"2023-10-23","arxiv_id":"2310.15123","repositories_listed":0,"syntology":null},{"url":"/paper/medeval-a-multi-level-multi-task-and-multi","slug":"medeval-a-multi-level-multi-task-and-multi","title":"MedEval: A Multi-Level, Multi-Task, and Multi-Domain Medical Benchmark for Language Model Evaluation","date":"2023-10-21","arxiv_id":"2310.14088","repositories_listed":0,"syntology":{"n":7,"n_ran":4,"n_constructed":0,"n_ran_checked":4,"n_instrument":0,"n_unverified":3,"n_honours":0,"n_violates":0,"n_no_contract":4,"n_pointer_only":7,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 4 with no instrument failure: 0 honoured, 0 violated, 4 with no contract checked; 0 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/medeval-a-multi-level-multi-task-and-multi#ran","syntology_url":"https://syntology.ai/paper/2310.14088","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2310.14088"}},"official":null}},{"url":null,"slug":"is-chatgpt-a-financial-expert-evaluating","title":"Is ChatGPT a Financial Expert? Evaluating Language Models on Financial Natural Language Processing","date":"2023-10-19","arxiv_id":"2310.12664","repositories_listed":0,"syntology":null},{"url":null,"slug":"pseudointelligence-a-unifying-framework-for","title":"Pseudointelligence: A Unifying Framework for Language Model Evaluation","date":"2023-10-18","arxiv_id":"2310.12135","repositories_listed":0,"syntology":null},{"url":null,"slug":"dialectical-language-model-evaluation-an","title":"Dialectical language model evaluation: An initial appraisal of the commonsense spatial reasoning abilities of LLMs","date":"2023-04-22","arxiv_id":"2304.11164","repositories_listed":0,"syntology":null},{"url":null,"slug":"controlling-for-stereotypes-in-multimodal","title":"Controlling for Stereotypes in Multimodal Language Model Evaluation","date":"2023-02-03","arxiv_id":"2302.01582","repositories_listed":0,"syntology":null},{"url":null,"slug":"a-dog-is-passing-over-the-jet-a-text","title":"A Dog Is Passing Over The Jet? A Text-Generation Dataset for Korean Commonsense Reasoning and Evaluation","date":"2022-07-01","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"bpomp-the-benchmark-of-poetic-minimal-pairs","title":"BPoMP: The Benchmark of Poetic Minimal Pairs – Limericks, Rhyme, and Narrative Coherence","date":"2021-09-01","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"language-model-evaluation-in-open-ended-text","title":"Language Model Evaluation in Open-ended Text Generation","date":"2021-08-08","arxiv_id":"2108.03578","repositories_listed":0,"syntology":null},{"url":null,"slug":"language-model-evaluation-beyond-perplexity","title":"Language Model Evaluation Beyond Perplexity","date":"2021-05-31","arxiv_id":"2106.00085","repositories_listed":0,"syntology":null},{"url":null,"slug":"climp-a-benchmark-for-chinese-language-model","title":"CLiMP: A Benchmark for Chinese Language Model Evaluation","date":"2021-01-26","arxiv_id":"2101.11131","repositories_listed":0,"syntology":null},{"url":null,"slug":"improving-explainable-recommendations-with","title":"Improving Explainable Recommendations with Synthetic Reviews","date":"2018-07-18","arxiv_id":"1807.06978","repositories_listed":0,"syntology":null},{"url":null,"slug":"contrastive-entropy-a-new-evaluation-metric","title":"Contrastive Entropy: A new evaluation metric for unnormalized language models","date":"2016-01-03","arxiv_id":"1601.00248","repositories_listed":0,"syntology":null}],"syntology_records":12,"syntology_note":"a paper without a record is not a recorded non-run: it may lack an arXiv id or simply be absent from the graph layer"},"key_notes":{"n_ran_checked":"legacy name, kept unchanged so existing readers do not break: it counts the samples that ran with no instrument failure (honoured, violated, and ran with no contract checked); it does not mean a contract was checked, and the pages print it as 'K with no instrument failure', not 'K checked'","n_constructed":"a sub-count of the samples that ran, never subtracted from them and never a failure: an executed sample whose run returned an instance of its own class (fixture_out_type equals the entry name): the run built an object and did not compute a result (Syntology's RAN record, counts.constructed)"},"since_archive_judged":{"papers":0,"papers_placed":0,"judged_by":{"model":"a model","models_as_recorded":[],"from":"lb_rows_run_verified.jsonl: each record's `model` field (the extractor's call that wrote decisions[])"},"note":"a model's judgement from the paper's own tables that they report on this leaderboard, not a result: no value from a paper that was not placed is printed or carried here; papers counts each paper once over these leaderboards","backfill":"8,886 of 18,637 newer papers checked so far (733 more had nothing to place; 41 could not be read)","backfill_complete":false,"leaderboards":[]},"papers_where_code_ran":12,"papers_where_code_ran_first_page":"/task/language-model-evaluation/papers/ran/1","papers_where_code_ran_caption":"We ran code from the paper's repository; we did not run it on this task or check it against the task's benchmarks.","pages":[{"order":"ran","page":1,"of":1,"url":"/task/language-model-evaluation/papers/ran/1","json":"/task/language-model-evaluation/papers/ran/1.json","rows":12}],"description_links":{"kept":0,"unwrapped_to_text":0,"bare_urls_linked":0,"relative_images_dropped":0,"rule":"internal links are kept only when the target slug exists in the catalog"},"syntology":{"read_at":"2026-10-05T00:36:11+00:00","claim":"Per-sample execution status on synthesized fixtures ('ran N of M samples'); not a correctness claim and not a ranking signal.","status_vocabulary":{"ran_honours":"ran, honoured the contract we drafted","ran_violates":"ran, violated the contract we drafted","ran_draft_wrong":"ran; our contract draft was wrong, not the code","ran_fixture":"ran; the contract check could not run (RAISES, DEP_MISSING or TIMEOUT)","ran":"ran on a synthesized input","unverified":"unverified (harvested, no recorded run)"},"row_route":"each syntology object on a benchmark's first row and on a paper carries sample_list (this site's paper page, #ran), syntology_url (syntology.ai's page for the paper, only when that route probed live) and mcp (the agent call that lists the paper's harvested samples; not a verdict on the paper)","developers":"https://syntology.ai/developers"},"not_shown":{"libraries":"the archive has no per-task library table","trend_sparklines":"the Trend column of the benchmarks table was a rendered image; it is not in the archive","social_and_latest_sorts":"stars and social signals are not in the archive"},"record_sha256":"3ddfd4ec9e80038ffb219e5516e7b55673addef6fdcf04420799dd8cea0c6ce7","record_changed_at":"2026-10-05","record_changed_at_basis":"changed"}