{"about":{"non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","site":"https://codewithpapers.app","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page","syntology":{"site":"https://syntology.ai","developers":"https://syntology.ai/developers","mcp":{"server":"https://syntology.ai/mcp","transport":"streamable-http","server_card":"https://syntology.ai/.well-known/mcp/server-card.json","auth":{"type":"trial token, no account","trial_token":"https://syntology.ai/api/oauth/trial/token","method":"POST","docs":"https://syntology.ai/developers"}},"have":"https://syntology.ai/api/graph/have?x=<method, arXiv id or title> (free, answers coverage only)","paper_base":"https://syntology.ai/paper/","atlas_base":"https://app.syntology.ai/?focus="},"machine_readable":[{"url":"https://codewithpapers.app/llms.txt","what":"the machine catalog: every machine-readable file, counted"},{"url":"https://codewithpapers.app/index/manifest.json","what":"paper-to-code index by arXiv id, with Syntology's counts"},{"url":"https://codewithpapers.app/search/manifest.json","what":"site search index (titles, authors) and its files"},{"url":"https://codewithpapers.app/download","what":"bulk files: Syntology's layer, described there"},{"url":"https://codewithpapers.app/build_manifest.json","what":"the build record: inputs, counts, exclusions, probes"}]},"url":"/task/multiple-choice","name":"Multiple-choice","slug":"multiple-choice","description_markdown":null,"categories":[],"source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","slug_source":"archive_url"},"counts":{"papers_tagged":1107,"papers_with_code":483,"benchmarks":0,"benchmark_tables_in_archive":0,"benchmark_tables_shown":0,"benchmark_tables_withheld_as_spam":0,"benchmark_definition":"a leaderboard table with at least one row; benchmark_tables_shown also counts the zero-row tables; benchmark_tables_in_archive adds the tables withheld as spam","datasets":12,"subtasks":0,"parent_tasks":0},"benchmarks":[],"datasets":[{"url":"/dataset/truthfulqa","name":"TruthfulQA","full_name":"","num_papers_in_archive":607},{"url":"/dataset/mathvista","name":"MathVista","full_name":"Mathematical Reasoning of in Visual Contexts","num_papers_in_archive":242},{"url":"/dataset/belebele","name":"Belebele","full_name":"","num_papers_in_archive":68},{"url":"/dataset/math-v","name":"MATH-V","full_name":"","num_papers_in_archive":12},{"url":"/dataset/neptune","name":"Neptune","full_name":"Neptune Long Video Understanding Benchmark","num_papers_in_archive":12},{"url":"/dataset/mindgames","name":"Mindgames","full_name":"","num_papers_in_archive":4},{"url":"/dataset/tasksource","name":"Tasksource","full_name":"","num_papers_in_archive":3},{"url":"/dataset/mminstruct-gpt4v","name":"MMInstruct-GPT4V","full_name":"MMInstruct: A High-Quality Multi-Modal Instruction Tuning Dataset with Extensive Diversity","num_papers_in_archive":1},{"url":"/dataset/ollabench-v-0-2","name":"OllaBench v.0.2","full_name":"OllaBench for Interdependent Cybersecurity v.0.2","num_papers_in_archive":1},{"url":"/dataset/probability-words-nli","name":"Probability words NLI","full_name":"Natural language inference with words estimative of probability (WEP)","num_papers_in_archive":1},{"url":"/dataset/tumtraffic-videoqa","name":"TUMTraffic-VideoQA","full_name":"","num_papers_in_archive":1},{"url":"/dataset/worldcuisines","name":"WorldCuisines","full_name":"","num_papers_in_archive":1}],"subtasks":[],"parent_tasks":[],"papers":{"order":"repositories listed in the archive (most first), then date (newest first), then slug; the archive holds no stars","population":"every paper the archive tags with this task (distinct papers; spam titles not listed). 'of' counts the papers listed; until 2026-09-25 it counted only the papers with a code link, and that number is of_with_code","page":1,"pages_in_order":12,"rows_per_page":100,"shown":100,"of":1107,"of_with_code":483,"tagged_in_all":1107,"counts":{"archive_papers_tagged":1107,"with_a_code_link":483,"where_syntology_ran_a_sample":161,"not_listed_spam_title":0,"listed":1107,"listed_where_code_ran":161,"where_syntology_ran_a_sample_split":{"with_a_run_with_no_instrument_failure":116,"every_run_a_failure_of_syntologys_instrument":45,"listed_with_a_run_with_no_instrument_failure":116,"listed_every_run_a_failure_of_syntologys_instrument":45,"filter":{"states":["a run with no instrument failure","any run, instrument failures included"],"default":"a run with no instrument failure","note":"on the 'only where code ran' pages the default hides, in the browser, the rows where every run was a failure of Syntology's instrument; the second state shows them again. Rows are hidden, never re-ordered; these twins list every row"}},"definition":"distinct papers the archive tags; 'where Syntology ran a sample' counts papers with at least one harvested sample that ran, which is not a correctness claim"},"items":[{"url":"/paper/vqa-visual-question-answering","slug":"vqa-visual-question-answering","title":"VQA: Visual Question Answering","date":"2015-05-03","arxiv_id":"1505.00468","repositories_listed":21,"syntology":{"n":7,"n_ran":6,"n_constructed":0,"n_ran_checked":2,"n_instrument":4,"n_unverified":1,"n_honours":2,"n_violates":0,"n_no_contract":0,"n_pointer_only":6,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 2 honoured, 0 violated, 0 with no contract checked; 4 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/vqa-visual-question-answering#ran","syntology_url":"https://syntology.ai/paper/1505.00468","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1505.00468"}},"official":null}},{"url":"/paper/blip-2-bootstrapping-language-image-pre","slug":"blip-2-bootstrapping-language-image-pre","title":"BLIP-2: Bootstrapping Language-Image Pre-training with Frozen Image Encoders and Large Language Models","date":"2023-01-30","arxiv_id":"2301.12597","repositories_listed":17,"syntology":{"n":8,"n_ran":4,"n_constructed":4,"n_ran_checked":4,"n_instrument":0,"n_unverified":4,"n_honours":0,"n_violates":0,"n_no_contract":4,"n_pointer_only":1,"phrase":"4 ran (of which 4 constructed an object rather than computing a result; 4 with no instrument failure: 0 honoured, 0 violated, 4 with no contract checked; 0 where Syntology's instrument failed) · 4 unverified; every one of the 4 samples that ran constructed an object rather than computing a result","sample_list":"/paper/blip-2-bootstrapping-language-image-pre#ran","syntology_url":"https://syntology.ai/paper/2301.12597","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2301.12597"}},"official":{"repos":["salesforce/lavis"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/video-llava-learning-united-visual-1","slug":"video-llava-learning-united-visual-1","title":"Video-LLaVA: Learning United Visual Representation by Alignment Before Projection","date":"2023-11-16","arxiv_id":"2311.10122","repositories_listed":6,"syntology":{"n":7,"n_ran":5,"n_constructed":0,"n_ran_checked":5,"n_instrument":0,"n_unverified":2,"n_honours":2,"n_violates":0,"n_no_contract":3,"n_pointer_only":2,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 2 honoured, 0 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/video-llava-learning-united-visual-1#ran","syntology_url":"https://syntology.ai/paper/2311.10122","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2311.10122"}},"official":{"repos":["PKU-YuanGroup/Video-LLaVA"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/gpt-takes-the-bar-exam","slug":"gpt-takes-the-bar-exam","title":"GPT Takes the Bar Exam","date":"2022-12-29","arxiv_id":"2212.14402","repositories_listed":5,"syntology":null},{"url":"/paper/flamingo-a-visual-language-model-for-few-shot-1","slug":"flamingo-a-visual-language-model-for-few-shot-1","title":"Flamingo: a Visual Language Model for Few-Shot Learning","date":"2022-04-29","arxiv_id":"2204.14198","repositories_listed":5,"syntology":{"n":24,"n_ran":18,"n_constructed":6,"n_ran_checked":11,"n_instrument":7,"n_unverified":6,"n_honours":0,"n_violates":1,"n_no_contract":10,"n_pointer_only":8,"phrase":"18 ran (of which 6 constructed an object rather than computing a result; 11 with no instrument failure: 0 honoured, 1 violated, 10 with no contract checked; 7 where Syntology's instrument failed) · 6 unverified","sample_list":"/paper/flamingo-a-visual-language-model-for-few-shot-1#ran","syntology_url":"https://syntology.ai/paper/2204.14198","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2204.14198"}},"official":null}},{"url":"/paper/automating-turkish-educational-quiz","slug":"automating-turkish-educational-quiz","title":"Automating Turkish Educational Quiz Generation Using Large Language Models","date":"2024-06-05","arxiv_id":"2406.03397","repositories_listed":4,"syntology":null},{"url":"/paper/tinybenchmarks-evaluating-llms-with-fewer","slug":"tinybenchmarks-evaluating-llms-with-fewer","title":"tinyBenchmarks: evaluating LLMs with fewer examples","date":"2024-02-22","arxiv_id":"2402.14992","repositories_listed":4,"syntology":{"n":39,"n_ran":25,"n_constructed":0,"n_ran_checked":22,"n_instrument":3,"n_unverified":14,"n_honours":4,"n_violates":1,"n_no_contract":17,"n_pointer_only":17,"phrase":"25 ran (of which 0 constructed an object rather than computing a result; 22 with no instrument failure: 4 honoured, 1 violated, 17 with no contract checked; 3 where Syntology's instrument failed) · 14 unverified","sample_list":"/paper/tinybenchmarks-evaluating-llms-with-fewer#ran","syntology_url":"https://syntology.ai/paper/2402.14992","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2402.14992"}},"official":{"repos":["felipemaiapolo/efficbench","felipemaiapolo/tinybenchmarks"],"state":"official (archive's flag): 10 ran","n_ran":10,"n_constructed":0,"n_ran_no_instrument_failure":7,"n_unverified":12,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/steering-llama-2-via-contrastive-activation","slug":"steering-llama-2-via-contrastive-activation","title":"Steering Llama 2 via Contrastive Activation Addition","date":"2023-12-09","arxiv_id":"2312.06681","repositories_listed":4,"syntology":{"n":22,"n_ran":21,"n_constructed":0,"n_ran_checked":16,"n_instrument":5,"n_unverified":1,"n_honours":4,"n_violates":0,"n_no_contract":12,"n_pointer_only":10,"phrase":"21 ran (of which 0 constructed an object rather than computing a result; 16 with no instrument failure: 4 honoured, 0 violated, 12 with no contract checked; 5 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/steering-llama-2-via-contrastive-activation#ran","syntology_url":"https://syntology.ai/paper/2312.06681","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2312.06681"}},"official":{"repos":["nrimsky/caa","nrimsky/sycophancysteering","wusche1/caa_hallucination"],"state":"official (archive's flag): 13 ran","n_ran":13,"n_constructed":0,"n_ran_no_instrument_failure":8,"n_unverified":1,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/case-commonsense-augmented-score-with-an","slug":"case-commonsense-augmented-score-with-an","title":"CASE: Commonsense-Augmented Score with an Expanded Answer Space","date":"2023-11-03","arxiv_id":"2311.01684","repositories_listed":4,"syntology":null},{"url":"/paper/from-recognition-to-cognition-visual","slug":"from-recognition-to-cognition-visual","title":"From Recognition to Cognition: Visual Commonsense Reasoning","date":"2018-11-27","arxiv_id":"1811.10830","repositories_listed":4,"syntology":{"n":6,"n_ran":5,"n_constructed":0,"n_ran_checked":5,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":5,"n_pointer_only":0,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 0 honoured, 0 violated, 5 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/from-recognition-to-cognition-visual#ran","syntology_url":"https://syntology.ai/paper/1811.10830","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1811.10830"}},"official":null}},{"url":"/paper/commonsenseqa-a-question-answering-challenge","slug":"commonsenseqa-a-question-answering-challenge","title":"CommonsenseQA: A Question Answering Challenge Targeting Commonsense Knowledge","date":"2018-11-02","arxiv_id":"1811.00937","repositories_listed":4,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":3,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/commonsenseqa-a-question-answering-challenge#ran","syntology_url":"https://syntology.ai/paper/1811.00937","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1811.00937"}},"official":{"repos":["jonathanherzig/commonsenseqa"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/mapeval-a-map-based-evaluation-of-geo-spatial","slug":"mapeval-a-map-based-evaluation-of-geo-spatial","title":"MapEval: A Map-Based Evaluation of Geo-Spatial Reasoning in Foundation Models","date":"2024-12-31","arxiv_id":"2501.00316","repositories_listed":3,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":0,"n_instrument":3,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":3,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 3 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/mapeval-a-map-based-evaluation-of-geo-spatial#ran","syntology_url":"https://syntology.ai/paper/2501.00316","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2501.00316"}},"official":{"repos":["MapEval/MapEval-API","MapEval/MapEval-Textual","MapEval/MapEval-Visual"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/videollama-2-advancing-spatial-temporal","slug":"videollama-2-advancing-spatial-temporal","title":"VideoLLaMA 2: Advancing Spatial-Temporal Modeling and Audio Understanding in Video-LLMs","date":"2024-06-11","arxiv_id":"2406.07476","repositories_listed":3,"syntology":{"n":17,"n_ran":12,"n_constructed":0,"n_ran_checked":7,"n_instrument":5,"n_unverified":5,"n_honours":1,"n_violates":1,"n_no_contract":5,"n_pointer_only":8,"phrase":"12 ran (of which 0 constructed an object rather than computing a result; 7 with no instrument failure: 1 honoured, 1 violated, 5 with no contract checked; 5 where Syntology's instrument failed) · 5 unverified","sample_list":"/paper/videollama-2-advancing-spatial-temporal#ran","syntology_url":"https://syntology.ai/paper/2406.07476","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.07476"}},"official":{"repos":["damo-nlp-sg/videollama2"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/mvbench-a-comprehensive-multi-modal-video","slug":"mvbench-a-comprehensive-multi-modal-video","title":"MVBench: A Comprehensive Multi-modal Video Understanding Benchmark","date":"2023-11-28","arxiv_id":"2311.17005","repositories_listed":3,"syntology":{"n":10,"n_ran":7,"n_constructed":0,"n_ran_checked":3,"n_instrument":4,"n_unverified":3,"n_honours":0,"n_violates":1,"n_no_contract":2,"n_pointer_only":2,"phrase":"7 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 1 violated, 2 with no contract checked; 4 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/mvbench-a-comprehensive-multi-modal-video#ran","syntology_url":"https://syntology.ai/paper/2311.17005","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2311.17005"}},"official":{"repos":["opengvlab/ask-anything"],"state":"official (archive's flag): 7 ran","n_ran":7,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":3,"ran_from_kinds":["official"]}}},{"url":"/paper/gpqa-a-graduate-level-google-proof-q-a","slug":"gpqa-a-graduate-level-google-proof-q-a","title":"GPQA: A Graduate-Level Google-Proof Q&A Benchmark","date":"2023-11-20","arxiv_id":"2311.12022","repositories_listed":3,"syntology":{"n":8,"n_ran":7,"n_constructed":0,"n_ran_checked":7,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":7,"n_pointer_only":0,"phrase":"7 ran (of which 0 constructed an object rather than computing a result; 7 with no instrument failure: 0 honoured, 0 violated, 7 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/gpqa-a-graduate-level-google-proof-q-a#ran","syntology_url":"https://syntology.ai/paper/2311.12022","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2311.12022"}},"official":{"repos":["idavidrein/gpqa"],"state":"official (archive's flag): 7 ran","n_ran":7,"n_constructed":0,"n_ran_no_instrument_failure":7,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/seed-bench-benchmarking-multimodal-llms-with","slug":"seed-bench-benchmarking-multimodal-llms-with","title":"SEED-Bench: Benchmarking Multimodal LLMs with Generative Comprehension","date":"2023-07-30","arxiv_id":"2307.16125","repositories_listed":3,"syntology":null},{"url":"/paper/mmbench-is-your-multi-modal-model-an-all","slug":"mmbench-is-your-multi-modal-model-an-all","title":"MMBench: Is Your Multi-modal Model an All-around Player?","date":"2023-07-12","arxiv_id":"2307.06281","repositories_listed":3,"syntology":null},{"url":"/paper/fine-tuning-language-models-with-just-forward-1","slug":"fine-tuning-language-models-with-just-forward-1","title":"Fine-Tuning Language Models with Just Forward Passes","date":"2023-05-27","arxiv_id":"2305.17333","repositories_listed":3,"syntology":{"n":17,"n_ran":10,"n_constructed":2,"n_ran_checked":6,"n_instrument":4,"n_unverified":7,"n_honours":1,"n_violates":0,"n_no_contract":5,"n_pointer_only":4,"phrase":"10 ran (of which 2 constructed an object rather than computing a result; 6 with no instrument failure: 1 honoured, 0 violated, 5 with no contract checked; 4 where Syntology's instrument failed) · 7 unverified","sample_list":"/paper/fine-tuning-language-models-with-just-forward-1#ran","syntology_url":"https://syntology.ai/paper/2305.17333","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2305.17333"}},"official":{"repos":["princeton-nlp/mezo"],"state":"official (archive's flag): 8 ran","n_ran":8,"n_constructed":0,"n_ran_no_instrument_failure":4,"n_unverified":6,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/quality-question-answering-with-long-input","slug":"quality-question-answering-with-long-input","title":"QuALITY: Question Answering with Long Input Texts, Yes!","date":"2021-12-16","arxiv_id":"2112.08608","repositories_listed":3,"syntology":null},{"url":"/paper/what-disease-does-this-patient-have-a-large","slug":"what-disease-does-this-patient-have-a-large","title":"What Disease does this Patient Have? A Large-scale Open Domain Question Answering Dataset from Medical Exams","date":"2020-09-28","arxiv_id":"2009.13081","repositories_listed":3,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/what-disease-does-this-patient-have-a-large#ran","syntology_url":"https://syntology.ai/paper/2009.13081","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2009.13081"}},"official":{"repos":["jind11/MedQA"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/learning-to-attend-on-essential-terms-an","slug":"learning-to-attend-on-essential-terms-an","title":"Learning to Attend On Essential Terms: An Enhanced Retriever-Reader Model for Open-domain Question Answering","date":"2018-08-28","arxiv_id":"1808.09492","repositories_listed":3,"syntology":null},{"url":"/paper/revisiting-visual-question-answering","slug":"revisiting-visual-question-answering","title":"Revisiting Visual Question Answering Baselines","date":"2016-06-27","arxiv_id":"1606.08390","repositories_listed":3,"syntology":null},{"url":"/paper/videoeval-pro-robust-and-realistic-long-video","slug":"videoeval-pro-robust-and-realistic-long-video","title":"VideoEval-Pro: Robust and Realistic Long Video Understanding Evaluation","date":"2025-05-20","arxiv_id":"2505.14640","repositories_listed":2,"syntology":{"n":14,"n_ran":12,"n_constructed":0,"n_ran_checked":6,"n_instrument":6,"n_unverified":2,"n_honours":1,"n_violates":0,"n_no_contract":5,"n_pointer_only":5,"phrase":"12 ran (of which 0 constructed an object rather than computing a result; 6 with no instrument failure: 1 honoured, 0 violated, 5 with no contract checked; 6 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/videoeval-pro-robust-and-realistic-long-video#ran","syntology_url":"https://syntology.ai/paper/2505.14640","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2505.14640"}},"official":null}},{"url":"/paper/2503-00096","slug":"2503-00096","title":"BixBench: a Comprehensive Benchmark for LLM-based Agents in Computational Biology","date":"2025-02-28","arxiv_id":"2503.00096","repositories_listed":2,"syntology":{"n":9,"n_ran":9,"n_constructed":0,"n_ran_checked":9,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":9,"n_pointer_only":0,"phrase":"9 ran (of which 0 constructed an object rather than computing a result; 9 with no instrument failure: 0 honoured, 0 violated, 9 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/2503-00096#ran","syntology_url":"https://syntology.ai/paper/2503.00096","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2503.00096"}},"official":{"repos":["Future-House/BixBench","Future-House/data-analysis-crow"],"state":"official (archive's flag): 9 ran","n_ran":9,"n_constructed":0,"n_ran_no_instrument_failure":9,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/vidcomposition-can-mllms-analyze-compositions","slug":"vidcomposition-can-mllms-analyze-compositions","title":"VidComposition: Can MLLMs Analyze Compositions in Compiled Videos?","date":"2024-11-17","arxiv_id":"2411.10979","repositories_listed":2,"syntology":null},{"url":"/paper/llava-onevision-easy-visual-task-transfer","slug":"llava-onevision-easy-visual-task-transfer","title":"LLaVA-OneVision: Easy Visual Task Transfer","date":"2024-08-06","arxiv_id":"2408.03326","repositories_listed":2,"syntology":null},{"url":"/paper/2408-01800","slug":"2408-01800","title":"MiniCPM-V: A GPT-4V Level MLLM on Your Phone","date":"2024-08-03","arxiv_id":"2408.01800","repositories_listed":2,"syntology":{"n":14,"n_ran":11,"n_constructed":0,"n_ran_checked":5,"n_instrument":6,"n_unverified":3,"n_honours":1,"n_violates":2,"n_no_contract":2,"n_pointer_only":6,"phrase":"11 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 1 honoured, 2 violated, 2 with no contract checked; 6 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/2408-01800#ran","syntology_url":"https://syntology.ai/paper/2408.01800","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2408.01800"}},"official":{"repos":["OpenBMB/MiniCPM-o"],"state":"official (archive's flag): 9 ran","n_ran":9,"n_constructed":0,"n_ran_no_instrument_failure":5,"n_unverified":3,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/2408-01337","slug":"2408-01337","title":"MuChoMusic: Evaluating Music Understanding in Multimodal Audio-Language Models","date":"2024-08-02","arxiv_id":"2408.01337","repositories_listed":2,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/2408-01337#ran","syntology_url":"https://syntology.ai/paper/2408.01337","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2408.01337"}},"official":{"repos":["mulab-mir/muchomusic"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/uncertainty-is-fragile-manipulating","slug":"uncertainty-is-fragile-manipulating","title":"Uncertainty is Fragile: Manipulating Uncertainty in Large Language Models","date":"2024-07-15","arxiv_id":"2407.11282","repositories_listed":2,"syntology":{"n":4,"n_ran":2,"n_constructed":0,"n_ran_checked":0,"n_instrument":2,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":4,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/uncertainty-is-fragile-manipulating#ran","syntology_url":"https://syntology.ai/paper/2407.11282","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2407.11282"}},"official":{"repos":["qcznlp/uncertainty_attack","qcznlp/uncertainty"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/intentionqa-a-benchmark-for-evaluating","slug":"intentionqa-a-benchmark-for-evaluating","title":"IntentionQA: A Benchmark for Evaluating Purchase Intention Comprehension Abilities of Language Models in E-commerce","date":"2024-06-14","arxiv_id":"2406.10173","repositories_listed":2,"syntology":null},{"url":"/paper/benchmarking-large-language-models-on-cflue-a","slug":"benchmarking-large-language-models-on-cflue-a","title":"Benchmarking Large Language Models on CFLUE -- A Chinese Financial Language Understanding Evaluation Dataset","date":"2024-05-17","arxiv_id":"2405.10542","repositories_listed":2,"syntology":null},{"url":"/paper/seed-bench-2-plus-benchmarking-multimodal","slug":"seed-bench-2-plus-benchmarking-multimodal","title":"SEED-Bench-2-Plus: Benchmarking Multimodal Large Language Models with Text-Rich Visual Comprehension","date":"2024-04-25","arxiv_id":"2404.16790","repositories_listed":2,"syntology":{"n":4,"n_ran":3,"n_constructed":0,"n_ran_checked":0,"n_instrument":3,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":4,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 3 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/seed-bench-2-plus-benchmarking-multimodal#ran","syntology_url":"https://syntology.ai/paper/2404.16790","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2404.16790"}},"official":{"repos":["ailab-cvc/seed-bench"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/automatic-generation-and-evaluation-of","slug":"automatic-generation-and-evaluation-of","title":"Automatic Generation and Evaluation of Reading Comprehension Test Items with Large Language Models","date":"2024-04-11","arxiv_id":"2404.07720","repositories_listed":2,"syntology":null},{"url":"/paper/minigpt4-video-advancing-multimodal-llms-for","slug":"minigpt4-video-advancing-multimodal-llms-for","title":"MiniGPT4-Video: Advancing Multimodal LLMs for Video Understanding with Interleaved Visual-Textual Tokens","date":"2024-04-04","arxiv_id":"2404.03413","repositories_listed":2,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":1,"n_instrument":1,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/minigpt4-video-advancing-multimodal-llms-for#ran","syntology_url":"https://syntology.ai/paper/2404.03413","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2404.03413"}},"official":{"repos":["Vision-CAIR/MiniGPT4-video"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/the-wmdp-benchmark-measuring-and-reducing","slug":"the-wmdp-benchmark-measuring-and-reducing","title":"The WMDP Benchmark: Measuring and Reducing Malicious Use With Unlearning","date":"2024-03-05","arxiv_id":"2403.03218","repositories_listed":2,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":1,"n_instrument":2,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":3,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 2 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/the-wmdp-benchmark-measuring-and-reducing#ran","syntology_url":"https://syntology.ai/paper/2403.03218","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2403.03218"}},"official":null}},{"url":"/paper/softmax-probabilities-mostly-predict-large","slug":"softmax-probabilities-mostly-predict-large","title":"Probabilities of Chat LLMs Are Miscalibrated but Still Predict Correctness on Multiple-Choice Q&A","date":"2024-02-20","arxiv_id":"2402.13213","repositories_listed":2,"syntology":{"n":14,"n_ran":10,"n_constructed":0,"n_ran_checked":4,"n_instrument":6,"n_unverified":4,"n_honours":1,"n_violates":0,"n_no_contract":3,"n_pointer_only":0,"phrase":"10 ran (of which 0 constructed an object rather than computing a result; 4 with no instrument failure: 1 honoured, 0 violated, 3 with no contract checked; 6 where Syntology's instrument failed) · 4 unverified","sample_list":"/paper/softmax-probabilities-mostly-predict-large#ran","syntology_url":"https://syntology.ai/paper/2402.13213","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2402.13213"}},"official":{"repos":["bplaut/softmax-probs-predict-llm-correctness","bplaut/llm-calibration-and-correctness-prediction"],"state":"official (archive's flag): 10 ran","n_ran":10,"n_constructed":0,"n_ran_no_instrument_failure":4,"n_unverified":4,"ran_from_kinds":["official"]}}},{"url":"/paper/de-cop-detecting-copyrighted-content-in","slug":"de-cop-detecting-copyrighted-content-in","title":"DE-COP: Detecting Copyrighted Content in Language Models Training Data","date":"2024-02-15","arxiv_id":"2402.09910","repositories_listed":2,"syntology":{"n":6,"n_ran":5,"n_constructed":0,"n_ran_checked":1,"n_instrument":4,"n_unverified":1,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 4 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/de-cop-detecting-copyrighted-content-in#ran","syntology_url":"https://syntology.ai/paper/2402.09910","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2402.09910"}},"official":{"repos":["avduarte333/de-cop_method","leililab/de-cop"],"state":"official (archive's flag): 5 ran","n_ran":5,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/shield-an-evaluation-benchmark-for-face","slug":"shield-an-evaluation-benchmark-for-face","title":"SHIELD : An Evaluation Benchmark for Face Spoofing and Forgery Detection with Multimodal Large Language Models","date":"2024-02-06","arxiv_id":"2402.04178","repositories_listed":2,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":1,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/shield-an-evaluation-benchmark-for-face#ran","syntology_url":"https://syntology.ai/paper/2402.04178","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2402.04178"}},"official":{"repos":["laiyingxin2/shield"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/i-think-therefore-i-am-awareness-in-large","slug":"i-think-therefore-i-am-awareness-in-large","title":"I Think, Therefore I am: Benchmarking Awareness of Large Language Models Using AwareBench","date":"2024-01-31","arxiv_id":"2401.17882","repositories_listed":2,"syntology":{"n":4,"n_ran":2,"n_constructed":0,"n_ran_checked":1,"n_instrument":1,"n_unverified":2,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":1,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/i-think-therefore-i-am-awareness-in-large#ran","syntology_url":"https://syntology.ai/paper/2401.17882","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2401.17882"}},"official":{"repos":["howiehwong/awareness-in-llm"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":2,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/seed-bench-2-benchmarking-multimodal-large","slug":"seed-bench-2-benchmarking-multimodal-large","title":"SEED-Bench-2: Benchmarking Multimodal Large Language Models","date":"2023-11-28","arxiv_id":"2311.17092","repositories_listed":2,"syntology":null},{"url":"/paper/data-contamination-quiz-a-tool-to-detect-and","slug":"data-contamination-quiz-a-tool-to-detect-and","title":"Data Contamination Quiz: A Tool to Detect and Estimate Contamination in Large Language Models","date":"2023-11-10","arxiv_id":"2311.06233","repositories_listed":2,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":3,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/data-contamination-quiz-a-tool-to-detect-and#ran","syntology_url":"https://syntology.ai/paper/2311.06233","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2311.06233"}},"official":{"repos":["shahriargolchin/dcq"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/brainteaser-lateral-thinking-puzzles-for","slug":"brainteaser-lateral-thinking-puzzles-for","title":"BRAINTEASER: Lateral Thinking Puzzles for Large Language Models","date":"2023-10-08","arxiv_id":"2310.05057","repositories_listed":2,"syntology":{"n":7,"n_ran":7,"n_constructed":0,"n_ran_checked":4,"n_instrument":3,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":4,"n_pointer_only":3,"phrase":"7 ran (of which 0 constructed an object rather than computing a result; 4 with no instrument failure: 0 honoured, 0 violated, 4 with no contract checked; 3 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/brainteaser-lateral-thinking-puzzles-for#ran","syntology_url":"https://syntology.ai/paper/2310.05057","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2310.05057"}},"official":null}},{"url":"/paper/the-belebele-benchmark-a-parallel-reading","slug":"the-belebele-benchmark-a-parallel-reading","title":"The Belebele Benchmark: a Parallel Reading Comprehension Dataset in 122 Language Variants","date":"2023-08-31","arxiv_id":"2308.16884","repositories_listed":2,"syntology":{"n":6,"n_ran":0,"n_constructed":0,"n_ran_checked":0,"n_instrument":0,"n_unverified":6,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":6,"phrase":"0 ran · 6 unverified","sample_list":"/paper/the-belebele-benchmark-a-parallel-reading#ran","syntology_url":"https://syntology.ai/paper/2308.16884","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2308.16884"}},"official":{"repos":["facebookresearch/belebele"],"state":"official: harvested, nothing ran","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":6,"ran_from_kinds":[]}}},{"url":"/paper/distractor-generation-for-multiple-choice-1","slug":"distractor-generation-for-multiple-choice-1","title":"Distractor generation for multiple-choice questions with predictive prompting and large language models","date":"2023-07-30","arxiv_id":"2307.16338","repositories_listed":2,"syntology":null},{"url":"/paper/data-driven-chance-constrained-multiple","slug":"data-driven-chance-constrained-multiple","title":"Chance-Constrained Multiple-Choice Knapsack Problem: Model, Algorithms, and Applications","date":"2023-06-26","arxiv_id":"2306.14690","repositories_listed":2,"syntology":null},{"url":"/paper/xiezhi-an-ever-updating-benchmark-for","slug":"xiezhi-an-ever-updating-benchmark-for","title":"Xiezhi: An Ever-Updating Benchmark for Holistic Domain Knowledge Evaluation","date":"2023-06-09","arxiv_id":"2306.05783","repositories_listed":2,"syntology":null},{"url":"/paper/language-models-don-t-always-say-what-they-1","slug":"language-models-don-t-always-say-what-they-1","title":"Language Models Don't Always Say What They Think: Unfaithful Explanations in Chain-of-Thought Prompting","date":"2023-05-07","arxiv_id":"2305.04388","repositories_listed":2,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":0,"n_instrument":3,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 3 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/language-models-don-t-always-say-what-they-1#ran","syntology_url":"https://syntology.ai/paper/2305.04388","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2305.04388"}},"official":{"repos":["milesaturpin/cot-unfaithfulness"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/mindgames-targeting-theory-of-mind-in-large","slug":"mindgames-targeting-theory-of-mind-in-large","title":"MindGames: Targeting Theory of Mind in Large Language Models with Dynamic Epistemic Modal Logic","date":"2023-05-05","arxiv_id":"2305.03353","repositories_listed":2,"syntology":null},{"url":"/paper/explicit-planning-helps-language-models-in","slug":"explicit-planning-helps-language-models-in","title":"Explicit Planning Helps Language Models in Logical Reasoning","date":"2023-03-28","arxiv_id":"2303.15714","repositories_listed":2,"syntology":{"n":1,"n_ran":1,"n_constructed":1,"n_ran_checked":1,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"1 ran (of which 1 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified; the one sample that ran constructed an object rather than computing a result","sample_list":"/paper/explicit-planning-helps-language-models-in#ran","syntology_url":"https://syntology.ai/paper/2303.15714","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2303.15714"}},"official":{"repos":["cindermond/explicit-planning-for-reasoning","cindermond/leap"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":1,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/mqag-multiple-choice-question-answering-and","slug":"mqag-multiple-choice-question-answering-and","title":"MQAG: Multiple-choice Question Answering and Generation for Assessing Information Consistency in Summarization","date":"2023-01-28","arxiv_id":"2301.12307","repositories_listed":2,"syntology":{"n":2,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":1,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/mqag-multiple-choice-question-answering-and#ran","syntology_url":"https://syntology.ai/paper/2301.12307","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2301.12307"}},"official":{"repos":["potsawee/mqag0"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/variational-open-domain-question-answering","slug":"variational-open-domain-question-answering","title":"Variational Open-Domain Question Answering","date":"2022-09-23","arxiv_id":"2210.06345","repositories_listed":2,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/variational-open-domain-question-answering#ran","syntology_url":"https://syntology.ai/paper/2210.06345","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2210.06345"}},"official":{"repos":["findzebra/fz-openqa","VodLM/vod"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/cc-riddle-a-question-answering-dataset-of","slug":"cc-riddle-a-question-answering-dataset-of","title":"CC-Riddle: A Question Answering Dataset of Chinese Character Riddles","date":"2022-06-28","arxiv_id":"2206.13778","repositories_listed":2,"syntology":null},{"url":"/paper/bridgeformer-bridging-video-text-retrieval","slug":"bridgeformer-bridging-video-text-retrieval","title":"Bridging Video-text Retrieval with Multiple Choice Questions","date":"2022-01-13","arxiv_id":"2201.04850","repositories_listed":2,"syntology":{"n":24,"n_ran":13,"n_constructed":8,"n_ran_checked":9,"n_instrument":4,"n_unverified":11,"n_honours":0,"n_violates":0,"n_no_contract":9,"n_pointer_only":6,"phrase":"13 ran (of which 8 constructed an object rather than computing a result; 9 with no instrument failure: 0 honoured, 0 violated, 9 with no contract checked; 4 where Syntology's instrument failed) · 11 unverified","sample_list":"/paper/bridgeformer-bridging-video-text-retrieval#ran","syntology_url":"https://syntology.ai/paper/2201.04850","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2201.04850"}},"official":{"repos":["tencentarc/mcq"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":4,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/general-purpose-question-answering-with-macaw","slug":"general-purpose-question-answering-with-macaw","title":"General-Purpose Question-Answering with Macaw","date":"2021-09-06","arxiv_id":"2109.02593","repositories_listed":2,"syntology":null},{"url":"/paper/option-tracing-beyond-correctness-analysis-in","slug":"option-tracing-beyond-correctness-analysis-in","title":"Option Tracing: Beyond Correctness Analysis in Knowledge Tracing","date":"2021-04-19","arxiv_id":"2104.09043","repositories_listed":2,"syntology":null},{"url":"/paper/when-does-pretraining-help-assessing-self","slug":"when-does-pretraining-help-assessing-self","title":"When Does Pretraining Help? Assessing Self-Supervised Learning for Law and the CaseHOLD Dataset","date":"2021-04-18","arxiv_id":"2104.08671","repositories_listed":2,"syntology":{"n":9,"n_ran":6,"n_constructed":0,"n_ran_checked":6,"n_instrument":0,"n_unverified":3,"n_honours":0,"n_violates":0,"n_no_contract":6,"n_pointer_only":0,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 6 with no instrument failure: 0 honoured, 0 violated, 6 with no contract checked; 0 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/when-does-pretraining-help-assessing-self#ran","syntology_url":"https://syntology.ai/paper/2104.08671","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2104.08671"}},"official":{"repos":["reglab/casehold"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/surface-form-competition-why-the-highest","slug":"surface-form-competition-why-the-highest","title":"Surface Form Competition: Why the Highest Probability Answer Isn't Always Right","date":"2021-04-16","arxiv_id":"2104.08315","repositories_listed":2,"syntology":{"n":5,"n_ran":5,"n_constructed":0,"n_ran_checked":0,"n_instrument":5,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 5 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/surface-form-competition-why-the-highest#ran","syntology_url":"https://syntology.ai/paper/2104.08315","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2104.08315"}},"official":{"repos":["peterwestuw/surface-form-competition"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/quiz-style-question-generation-for-news","slug":"quiz-style-question-generation-for-news","title":"Quiz-Style Question Generation for News Stories","date":"2021-02-18","arxiv_id":"2102.09094","repositories_listed":2,"syntology":null},{"url":"/paper/liveqa-a-question-answering-dataset-over","slug":"liveqa-a-question-answering-dataset-over","title":"LiveQA: A Question Answering Dataset over Sports Live","date":"2020-10-01","arxiv_id":"2010.00526","repositories_listed":2,"syntology":null},{"url":"/paper/unifiedqa-crossing-format-boundaries-with-a","slug":"unifiedqa-crossing-format-boundaries-with-a","title":"UnifiedQA: Crossing Format Boundaries With a Single QA System","date":"2020-05-02","arxiv_id":"2005.00700","repositories_listed":2,"syntology":{"n":7,"n_ran":4,"n_constructed":3,"n_ran_checked":4,"n_instrument":0,"n_unverified":3,"n_honours":0,"n_violates":1,"n_no_contract":3,"n_pointer_only":1,"phrase":"4 ran (of which 3 constructed an object rather than computing a result; 4 with no instrument failure: 0 honoured, 1 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/unifiedqa-crossing-format-boundaries-with-a#ran","syntology_url":"https://syntology.ai/paper/2005.00700","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2005.00700"}},"official":{"repos":["allenai/unifiedqa"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":3,"n_ran_no_instrument_failure":4,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/dmcl-distillation-multiple-choice-learning","slug":"dmcl-distillation-multiple-choice-learning","title":"DMCL: Distillation Multiple Choice Learning for Multimodal Action Recognition","date":"2019-12-23","arxiv_id":"1912.10982","repositories_listed":2,"syntology":{"n":4,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":3,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":4,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/dmcl-distillation-multiple-choice-learning#ran","syntology_url":"https://syntology.ai/paper/1912.10982","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1912.10982"}},"official":{"repos":["ncgarcia/DMCL"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":3,"ran_from_kinds":["official"]}}},{"url":"/paper/mmm-multi-stage-multi-task-learning-for-multi","slug":"mmm-multi-stage-multi-task-learning-for-multi","title":"MMM: Multi-stage Multi-task Learning for Multi-choice Reading Comprehension","date":"2019-10-01","arxiv_id":"1910.00458","repositories_listed":2,"syntology":null},{"url":"/paper/abductive-commonsense-reasoning","slug":"abductive-commonsense-reasoning","title":"Abductive Commonsense Reasoning","date":"2019-08-15","arxiv_id":"1908.05739","repositories_listed":2,"syntology":{"n":15,"n_ran":14,"n_constructed":0,"n_ran_checked":11,"n_instrument":3,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":11,"n_pointer_only":2,"phrase":"14 ran (of which 0 constructed an object rather than computing a result; 11 with no instrument failure: 0 honoured, 0 violated, 11 with no contract checked; 3 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/abductive-commonsense-reasoning#ran","syntology_url":"https://syntology.ai/paper/1908.05739","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1908.05739"}},"official":null}},{"url":"/paper/generating-distractors-for-reading","slug":"generating-distractors-for-reading","title":"Generating Distractors for Reading Comprehension Questions from Real Examinations","date":"2018-09-08","arxiv_id":"1809.02768","repositories_listed":2,"syntology":null},{"url":"/paper/a-joint-sequence-fusion-model-for-video","slug":"a-joint-sequence-fusion-model-for-video","title":"A Joint Sequence Fusion Model for Video Question Answering and Retrieval","date":"2018-08-07","arxiv_id":"1808.02559","repositories_listed":2,"syntology":{"n":2,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/a-joint-sequence-fusion-model-for-video#ran","syntology_url":"https://syntology.ai/paper/1808.02559","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1808.02559"}},"official":null}},{"url":"/paper/a-simple-method-for-commonsense-reasoning","slug":"a-simple-method-for-commonsense-reasoning","title":"A Simple Method for Commonsense Reasoning","date":"2018-06-07","arxiv_id":"1806.02847","repositories_listed":2,"syntology":{"n":7,"n_ran":4,"n_constructed":0,"n_ran_checked":4,"n_instrument":0,"n_unverified":3,"n_honours":0,"n_violates":0,"n_no_contract":4,"n_pointer_only":0,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 4 with no instrument failure: 0 honoured, 0 violated, 4 with no contract checked; 0 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/a-simple-method-for-commonsense-reasoning#ran","syntology_url":"https://syntology.ai/paper/1806.02847","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1806.02847"}},"official":null}},{"url":"/paper/confident-multiple-choice-learning","slug":"confident-multiple-choice-learning","title":"Confident Multiple Choice Learning","date":"2017-06-12","arxiv_id":"1706.03475","repositories_listed":2,"syntology":null},{"url":"/paper/training-free-llm-merging-for-multi-task","slug":"training-free-llm-merging-for-multi-task","title":"Training-free LLM Merging for Multi-task Learning","date":"2025-06-14","arxiv_id":"2506.12379","repositories_listed":1,"syntology":null},{"url":"/paper/stsbench-a-spatio-temporal-scenario-benchmark","slug":"stsbench-a-spatio-temporal-scenario-benchmark","title":"STSBench: A Spatio-temporal Scenario Benchmark for Multi-modal Large Language Models in Autonomous Driving","date":"2025-06-06","arxiv_id":"2506.06218","repositories_listed":1,"syntology":null},{"url":"/paper/llmeval-med-a-real-world-clinical-benchmark","slug":"llmeval-med-a-real-world-clinical-benchmark","title":"LLMEval-Med: A Real-world Clinical Benchmark for Medical LLMs with Physician Validation","date":"2025-06-04","arxiv_id":"2506.04078","repositories_listed":1,"syntology":null},{"url":"/paper/polishing-every-facet-of-the-gem-testing","slug":"polishing-every-facet-of-the-gem-testing","title":"Polishing Every Facet of the GEM: Testing Linguistic Competence of LLMs and Humans in Korean","date":"2025-06-02","arxiv_id":"2506.01237","repositories_listed":1,"syntology":null},{"url":"/paper/mixed-r1-unified-reward-perspective-for","slug":"mixed-r1-unified-reward-perspective-for","title":"Mixed-R1: Unified Reward Perspective For Reasoning Capability in Multimodal Large Language Models","date":"2025-05-30","arxiv_id":"2505.24164","repositories_listed":1,"syntology":null},{"url":"/paper/simulating-training-data-leakage-in-multiple","slug":"simulating-training-data-leakage-in-multiple","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","date":"2025-05-30","arxiv_id":"2505.24263","repositories_listed":1,"syntology":null},{"url":"/paper/dyepack-provably-flagging-test-set","slug":"dyepack-provably-flagging-test-set","title":"DyePack: Provably Flagging Test Set Contamination in LLMs Using Backdoors","date":"2025-05-29","arxiv_id":"2505.23001","repositories_listed":1,"syntology":null},{"url":"/paper/sns-bench-vl-benchmarking-multimodal-large","slug":"sns-bench-vl-benchmarking-multimodal-large","title":"SNS-Bench-VL: Benchmarking Multimodal Large Language Models in Social Networking Services","date":"2025-05-29","arxiv_id":"2505.23065","repositories_listed":1,"syntology":null},{"url":"/paper/vau-r1-advancing-video-anomaly-understanding","slug":"vau-r1-advancing-video-anomaly-understanding","title":"VAU-R1: Advancing Video Anomaly Understanding via Reinforcement Fine-Tuning","date":"2025-05-29","arxiv_id":"2505.23504","repositories_listed":1,"syntology":null},{"url":"/paper/bnmmlu-measuring-massive-multitask-language","slug":"bnmmlu-measuring-massive-multitask-language","title":"BnMMLU: Measuring Massive Multitask Language Understanding in Bengali","date":"2025-05-25","arxiv_id":"2505.18951","repositories_listed":1,"syntology":null},{"url":"/paper/daily-omni-towards-audio-visual-reasoning","slug":"daily-omni-towards-audio-visual-reasoning","title":"Daily-Omni: Towards Audio-Visual Reasoning with Temporal Alignment across Modalities","date":"2025-05-23","arxiv_id":"2505.17862","repositories_listed":1,"syntology":{"n":3,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":3,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/daily-omni-towards-audio-visual-reasoning#ran","syntology_url":"https://syntology.ai/paper/2505.17862","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2505.17862"}},"official":{"repos":["lliar-liar/daily-omni"],"state":"official: no sample here; runs from other or unrecorded repositories","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["unlocated"]}}},{"url":"/paper/let-androids-dream-of-electric-sheep-a-human","slug":"let-androids-dream-of-electric-sheep-a-human","title":"Let Androids Dream of Electric Sheep: A Human-like Image Implication Understanding and Reasoning Framework","date":"2025-05-22","arxiv_id":"2505.17019","repositories_listed":1,"syntology":null},{"url":"/paper/logicocr-do-your-large-multimodal-models","slug":"logicocr-do-your-large-multimodal-models","title":"LogicOCR: Do Your Large Multimodal Models Excel at Logical Reasoning on Text-Rich Images?","date":"2025-05-18","arxiv_id":"2505.12307","repositories_listed":1,"syntology":null},{"url":"/paper/teach2eval-an-indirect-evaluation-method-for","slug":"teach2eval-an-indirect-evaluation-method-for","title":"Teach2Eval: An Indirect Evaluation Method for LLM by Judging How It Teaches","date":"2025-05-18","arxiv_id":"2505.12259","repositories_listed":1,"syntology":null},{"url":"/paper/2505-10772","slug":"2505-10772","title":"Ranked Voting based Self-Consistency of Large Language Models","date":"2025-05-16","arxiv_id":"2505.10772","repositories_listed":1,"syntology":null},{"url":"/paper/2505-11404","slug":"2505-11404","title":"Patho-R1: A Multimodal Reinforcement Learning-Based Pathology Expert Reasoner","date":"2025-05-16","arxiv_id":"2505.11404","repositories_listed":1,"syntology":null},{"url":"/paper/2505-11493","slug":"2505-11493","title":"GIE-Bench: Towards Grounded Evaluation for Text-Guided Image Editing","date":"2025-05-16","arxiv_id":"2505.11493","repositories_listed":1,"syntology":null},{"url":"/paper/irlbench-a-multi-modal-culturally-grounded","slug":"irlbench-a-multi-modal-culturally-grounded","title":"IRLBench: A Multi-modal, Culturally Grounded, Parallel Irish-English Benchmark for Open-Ended LLM Reasoning Evaluation","date":"2025-05-16","arxiv_id":"2505.13498","repositories_listed":1,"syntology":null},{"url":"/paper/benchmarking-ai-scientists-in-omics-data","slug":"benchmarking-ai-scientists-in-omics-data","title":"Benchmarking AI scientists in omics data-driven biological research","date":"2025-05-13","arxiv_id":"2505.08341","repositories_listed":1,"syntology":null},{"url":"/paper/grounding-synthetic-data-evaluations-of","slug":"grounding-synthetic-data-evaluations-of","title":"Grounding Synthetic Data Evaluations of Language Models in Unsupervised Document Corpora","date":"2025-05-13","arxiv_id":"2505.08905","repositories_listed":1,"syntology":null},{"url":"/paper/healthbench-evaluating-large-language-models","slug":"healthbench-evaluating-large-language-models","title":"HealthBench: Evaluating Large Language Models Towards Improved Human Health","date":"2025-05-13","arxiv_id":"2505.08775","repositories_listed":1,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":2,"n_instrument":1,"n_unverified":0,"n_honours":2,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 2 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/healthbench-evaluating-large-language-models#ran","syntology_url":"https://syntology.ai/paper/2505.08775","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2505.08775"}},"official":{"repos":["openai/simple-evals"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/vcrbench-exploring-long-form-causal-reasoning","slug":"vcrbench-exploring-long-form-causal-reasoning","title":"VCRBench: Exploring Long-form Causal Reasoning Capabilities of Large Video Language Models","date":"2025-05-13","arxiv_id":"2505.08455","repositories_listed":1,"syntology":null},{"url":"/paper/assessing-the-chemical-intelligence-of-large","slug":"assessing-the-chemical-intelligence-of-large","title":"Assessing the Chemical Intelligence of Large Language Models","date":"2025-05-12","arxiv_id":"2505.07735","repositories_listed":1,"syntology":null},{"url":"/paper/echoink-r1-exploring-audio-visual-reasoning","slug":"echoink-r1-exploring-audio-visual-reasoning","title":"EchoInk-R1: Exploring Audio-Visual Reasoning in Multimodal LLMs via Reinforcement Learning","date":"2025-05-07","arxiv_id":"2505.04623","repositories_listed":1,"syntology":null},{"url":"/paper/medarabiq-benchmarking-large-language-models","slug":"medarabiq-benchmarking-large-language-models","title":"MedArabiQ: Benchmarking Large Language Models on Arabic Medical Tasks","date":"2025-05-06","arxiv_id":"2505.03427","repositories_listed":1,"syntology":null},{"url":"/paper/regrap-llava-reasoning-enabled-graph-based","slug":"regrap-llava-reasoning-enabled-graph-based","title":"ReGraP-LLaVA: Reasoning enabled Graph-based Personalized Large Language and Vision Assistant","date":"2025-05-06","arxiv_id":"2505.03654","repositories_listed":1,"syntology":null},{"url":"/paper/harnessing-structured-knowledge-a-concept-map","slug":"harnessing-structured-knowledge-a-concept-map","title":"Harnessing Structured Knowledge: A Concept Map-Based Approach for High-Quality Multiple Choice Question Generation with Effective Distractors","date":"2025-05-02","arxiv_id":"2505.02850","repositories_listed":1,"syntology":null},{"url":"/paper/are-vision-llms-road-ready-a-comprehensive","slug":"are-vision-llms-road-ready-a-comprehensive","title":"Are Vision LLMs Road-Ready? A Comprehensive Benchmark for Safety-Critical Driving Video Understanding","date":"2025-04-20","arxiv_id":"2504.14526","repositories_listed":1,"syntology":null},{"url":"/paper/kaleidoscope-in-language-exams-for-massively","slug":"kaleidoscope-in-language-exams-for-massively","title":"Kaleidoscope: In-language Exams for Massively Multilingual Vision Evaluation","date":"2025-04-09","arxiv_id":"2504.07072","repositories_listed":1,"syntology":null},{"url":"/paper/chartqapro-a-more-diverse-and-challenging","slug":"chartqapro-a-more-diverse-and-challenging","title":"ChartQAPro: A More Diverse and Challenging Benchmark for Chart Question Answering","date":"2025-04-07","arxiv_id":"2504.05506","repositories_listed":1,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":0,"n_instrument":3,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 3 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/chartqapro-a-more-diverse-and-challenging#ran","syntology_url":"https://syntology.ai/paper/2504.05506","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2504.05506"}},"official":{"repos":["vis-nlp/chartqapro"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/vegas-towards-visually-explainable-and","slug":"vegas-towards-visually-explainable-and","title":"VEGAS: Towards Visually Explainable and Grounded Artificial Social Intelligence","date":"2025-04-03","arxiv_id":"2504.02227","repositories_listed":1,"syntology":null},{"url":"/paper/exploring-the-effect-of-reinforcement","slug":"exploring-the-effect-of-reinforcement","title":"Exploring the Effect of Reinforcement Learning on Video Understanding: Insights from SEED-Bench-R1","date":"2025-03-31","arxiv_id":"2503.24376","repositories_listed":1,"syntology":{"n":9,"n_ran":7,"n_constructed":0,"n_ran_checked":7,"n_instrument":0,"n_unverified":2,"n_honours":3,"n_violates":0,"n_no_contract":4,"n_pointer_only":0,"phrase":"7 ran (of which 0 constructed an object rather than computing a result; 7 with no instrument failure: 3 honoured, 0 violated, 4 with no contract checked; 0 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/exploring-the-effect-of-reinforcement#ran","syntology_url":"https://syntology.ai/paper/2503.24376","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2503.24376"}},"official":{"repos":["tencentarc/seed-bench-r1"],"state":"official (archive's flag): 7 ran","n_ran":7,"n_constructed":0,"n_ran_no_instrument_failure":7,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/question-aware-knowledge-graph-prompting-for","slug":"question-aware-knowledge-graph-prompting-for","title":"Question-Aware Knowledge Graph Prompting for Enhancing Large Language Models","date":"2025-03-30","arxiv_id":"2503.23523","repositories_listed":1,"syntology":null}],"syntology_records":45,"syntology_note":"a paper without a record is not a recorded non-run: it may lack an arXiv id or simply be absent from the graph layer"},"key_notes":{"n_ran_checked":"legacy name, kept unchanged so existing readers do not break: it counts the samples that ran with no instrument failure (honoured, violated, and ran with no contract checked); it does not mean a contract was checked, and the pages print it as 'K with no instrument failure', not 'K checked'","n_constructed":"a sub-count of the samples that ran, never subtracted from them and never a failure: an executed sample whose run returned an instance of its own class (fixture_out_type equals the entry name): the run built an object and did not compute a result (Syntology's RAN record, counts.constructed)"},"since_archive_judged":{"papers":0,"papers_placed":0,"judged_by":{"model":"a model","models_as_recorded":[],"from":"lb_rows_run_verified.jsonl: each record's `model` field (the extractor's call that wrote decisions[])"},"note":"a model's judgement from the paper's own tables that they report on this leaderboard, not a result: no value from a paper that was not placed is printed or carried here; papers counts each paper once over these leaderboards","backfill":"8,886 of 18,637 newer papers checked so far (733 more had nothing to place; 41 could not be read)","backfill_complete":false,"leaderboards":[]},"papers_where_code_ran":161,"papers_where_code_ran_first_page":"/task/multiple-choice/papers/ran/1","papers_where_code_ran_caption":"We ran code from the paper's repository; we did not run it on this task or check it against the task's benchmarks.","pages":[{"order":"archive","page":2,"of":12,"url":"/task/multiple-choice/papers/2","json":"/task/multiple-choice/papers/2.json","rows":100},{"order":"archive","page":3,"of":12,"url":"/task/multiple-choice/papers/3","json":"/task/multiple-choice/papers/3.json","rows":100},{"order":"archive","page":4,"of":12,"url":"/task/multiple-choice/papers/4","json":"/task/multiple-choice/papers/4.json","rows":100},{"order":"archive","page":5,"of":12,"url":"/task/multiple-choice/papers/5","json":"/task/multiple-choice/papers/5.json","rows":100},{"order":"archive","page":6,"of":12,"url":"/task/multiple-choice/papers/6","json":"/task/multiple-choice/papers/6.json","rows":100},{"order":"archive","page":7,"of":12,"url":"/task/multiple-choice/papers/7","json":"/task/multiple-choice/papers/7.json","rows":100},{"order":"archive","page":8,"of":12,"url":"/task/multiple-choice/papers/8","json":"/task/multiple-choice/papers/8.json","rows":100},{"order":"archive","page":9,"of":12,"url":"/task/multiple-choice/papers/9","json":"/task/multiple-choice/papers/9.json","rows":100},{"order":"archive","page":10,"of":12,"url":"/task/multiple-choice/papers/10","json":"/task/multiple-choice/papers/10.json","rows":100},{"order":"archive","page":11,"of":12,"url":"/task/multiple-choice/papers/11","json":"/task/multiple-choice/papers/11.json","rows":100},{"order":"archive","page":12,"of":12,"url":"/task/multiple-choice/papers/12","json":"/task/multiple-choice/papers/12.json","rows":7},{"order":"ran","page":1,"of":2,"url":"/task/multiple-choice/papers/ran/1","json":"/task/multiple-choice/papers/ran/1.json","rows":100},{"order":"ran","page":2,"of":2,"url":"/task/multiple-choice/papers/ran/2","json":"/task/multiple-choice/papers/ran/2.json","rows":61}],"description_links":{"kept":0,"unwrapped_to_text":0,"bare_urls_linked":0,"relative_images_dropped":0,"rule":"internal links are kept only when the target slug exists in the catalog"},"syntology":{"read_at":"2026-10-05T00:36:11+00:00","claim":"Per-sample execution status on synthesized fixtures ('ran N of M samples'); not a correctness claim and not a ranking signal.","status_vocabulary":{"ran_honours":"ran, honoured the contract we drafted","ran_violates":"ran, violated the contract we drafted","ran_draft_wrong":"ran; our contract draft was wrong, not the code","ran_fixture":"ran; the contract check could not run (RAISES, DEP_MISSING or TIMEOUT)","ran":"ran on a synthesized input","unverified":"unverified (harvested, no recorded run)"},"row_route":"each syntology object on a benchmark's first row and on a paper carries sample_list (this site's paper page, #ran), syntology_url (syntology.ai's page for the paper, only when that route probed live) and mcp (the agent call that lists the paper's harvested samples; not a verdict on the paper)","developers":"https://syntology.ai/developers"},"not_shown":{"libraries":"the archive has no per-task library table","trend_sparklines":"the Trend column of the benchmarks table was a rendered image; it is not in the archive","social_and_latest_sorts":"stars and social signals are not in the archive"},"record_sha256":"d701c1307511c6f1e5aedfad305b899cdc954d086d2ebd68138f164d69fb92d3","record_changed_at":"2026-10-05","record_changed_at_basis":"changed"}