{"about":{"non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","site":"https://codewithpapers.app","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page","syntology":{"site":"https://syntology.ai","developers":"https://syntology.ai/developers","mcp":{"server":"https://syntology.ai/mcp","transport":"streamable-http","server_card":"https://syntology.ai/.well-known/mcp/server-card.json","auth":{"type":"trial token, no account","trial_token":"https://syntology.ai/api/oauth/trial/token","method":"POST","docs":"https://syntology.ai/developers"}},"have":"https://syntology.ai/api/graph/have?x=<method, arXiv id or title> (free, answers coverage only)","paper_base":"https://syntology.ai/paper/","atlas_base":"https://app.syntology.ai/?focus="},"machine_readable":[{"url":"https://codewithpapers.app/llms.txt","what":"the machine catalog: every machine-readable file, counted"},{"url":"https://codewithpapers.app/index/manifest.json","what":"paper-to-code index by arXiv id, with Syntology's counts"},{"url":"https://codewithpapers.app/search/manifest.json","what":"site search index (titles, authors) and its files"},{"url":"https://codewithpapers.app/download","what":"bulk files: Syntology's layer, described there"},{"url":"https://codewithpapers.app/build_manifest.json","what":"the build record: inputs, counts, exclusions, probes"}]},"url":"/task/benchmarking","name":"Benchmarking","slug":"benchmarking","description_markdown":null,"categories":[{"name":"Miscellaneous","url":"/area/miscellaneous"},{"name":"Robots","url":"/area/robots"}],"source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","slug_source":"archive_url"},"counts":{"papers_tagged":5548,"papers_with_code":2658,"benchmarks":2,"benchmark_tables_in_archive":2,"benchmark_tables_shown":2,"benchmark_tables_withheld_as_spam":0,"benchmark_definition":"a leaderboard table with at least one row; benchmark_tables_shown also counts the zero-row tables; benchmark_tables_in_archive adds the tables withheld as spam","datasets":10,"subtasks":0,"parent_tasks":1},"benchmarks":[{"leaderboard":"/sota/benchmarking-on-cloudeval-yaml","slug":"benchmarking-on-cloudeval-yaml","dataset":"CloudEval-YAML","dataset_url":null,"rows_in_archive":1,"metrics":["ACC"],"first_row_in_archive_order":{"model":"GPT-4 Turbo","paper_title":"CloudEval-YAML: A Practical Benchmark for Cloud Configuration Generation","paper_url":"/paper/cloudeval-yaml-a-practical-benchmark-for","paper_date":"2023-11-10","arxiv_id":"2401.06786","code_links":[{"title":"alibaba/cloudeval-yaml","url":"https://github.com/alibaba/cloudeval-yaml"}],"syntology":null}},{"leaderboard":"/sota/benchmarking-on-wiki-40b","slug":"benchmarking-on-wiki-40b","dataset":"Wiki-40B","dataset_url":"/dataset/wiki-40b","rows_in_archive":1,"metrics":["Perplexity"],"first_row_in_archive_order":{"model":"OutEffHop-Bert_base","paper_title":"Outlier-Efficient Hopfield Layers for Large Transformer-Based Models","paper_url":"/paper/outlier-efficient-hopfield-layers-for-large","paper_date":"2024-04-04","arxiv_id":"2404.03828","code_links":[{"title":"magics-lab/outeffhop","url":"https://github.com/magics-lab/outeffhop"}],"syntology":{"n":13,"n_ran":10,"n_constructed":5,"n_ran_checked":5,"n_instrument":5,"n_unverified":3,"n_honours":0,"n_violates":0,"n_no_contract":5,"n_pointer_only":0,"phrase":"10 ran (of which 5 constructed an object rather than computing a result; 5 with no instrument failure: 0 honoured, 0 violated, 5 with no contract checked; 5 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/outlier-efficient-hopfield-layers-for-large#ran","syntology_url":"https://syntology.ai/paper/2404.03828","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2404.03828"}},"official":{"repos":["magics-lab/outeffhop"],"state":"official (archive's flag): 10 ran","n_ran":10,"n_constructed":5,"n_ran_no_instrument_failure":5,"n_unverified":3,"ran_from_kinds":["official"]}}}}],"datasets":[{"url":"/dataset/wiki-40b","name":"Wiki-40B","full_name":"","num_papers_in_archive":30},{"url":"/dataset/cropandweed-dataset","name":"CropAndWeed","full_name":"","num_papers_in_archive":10},{"url":"/dataset/europarl-asr","name":"Europarl-ASR","full_name":"","num_papers_in_archive":8},{"url":"/dataset/fluidlab","name":"FluidLab","full_name":"","num_papers_in_archive":5},{"url":"/dataset/apron-dataset","name":"Apron Dataset","full_name":"","num_papers_in_archive":1},{"url":"/dataset/coco-n-medium","name":"COCO-N Medium","full_name":"","num_papers_in_archive":1},{"url":"/dataset/csts","name":"CSTS","full_name":"Correlation Structures in Time Series","num_papers_in_archive":1},{"url":"/dataset/genotex","name":"GenoTEX","full_name":"An LLM Agent Benchmark for Automated Gene Expression Data Analysis","num_papers_in_archive":1},{"url":"/dataset/psocr","name":"PsOCR","full_name":"Pashto OCR Dataset","num_papers_in_archive":1},{"url":"/dataset/semanticsugarbeets","name":"SemanticSugarBeets","full_name":"","num_papers_in_archive":1}],"subtasks":[],"parent_tasks":[{"url":"/task/data-free-knowledge-distillation","name":"Data-free Knowledge Distillation"}],"papers":{"order":"repositories listed in the archive (most first), then date (newest first), then slug; the archive holds no stars","population":"every paper the archive tags with this task (distinct papers; spam titles not listed). 'of' counts the papers listed; until 2026-09-25 it counted only the papers with a code link, and that number is of_with_code","page":1,"pages_in_order":56,"rows_per_page":100,"shown":100,"of":5548,"of_with_code":2658,"tagged_in_all":5548,"counts":{"archive_papers_tagged":5548,"with_a_code_link":2658,"where_syntology_ran_a_sample":749,"not_listed_spam_title":0,"listed":5548,"listed_where_code_ran":749,"where_syntology_ran_a_sample_split":{"with_a_run_with_no_instrument_failure":603,"every_run_a_failure_of_syntologys_instrument":146,"listed_with_a_run_with_no_instrument_failure":603,"listed_every_run_a_failure_of_syntologys_instrument":146,"filter":{"states":["a run with no instrument failure","any run, instrument failures included"],"default":"a run with no instrument failure","note":"on the 'only where code ran' pages the default hides, in the browser, the rows where every run was a failure of Syntology's instrument; the second state shows them again. Rows are hidden, never re-ordered; these twins list every row"}},"definition":"distinct papers the archive tags; 'where Syntology ran a sample' counts papers with at least one harvested sample that ran, which is not a correctness claim"},"items":[{"url":"/paper/mmdetection-open-mmlab-detection-toolbox-and","slug":"mmdetection-open-mmlab-detection-toolbox-and","title":"MMDetection: Open MMLab Detection Toolbox and Benchmark","date":"2019-06-17","arxiv_id":"1906.07155","repositories_listed":142,"syntology":{"n":82,"n_ran":62,"n_constructed":0,"n_ran_checked":53,"n_instrument":9,"n_unverified":20,"n_honours":1,"n_violates":2,"n_no_contract":50,"n_pointer_only":12,"phrase":"62 ran (of which 0 constructed an object rather than computing a result; 53 with no instrument failure: 1 honoured, 2 violated, 50 with no contract checked; 9 where Syntology's instrument failed) · 20 unverified","sample_list":"/paper/mmdetection-open-mmlab-detection-toolbox-and#ran","syntology_url":"https://syntology.ai/paper/1906.07155","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1906.07155"}},"official":{"repos":["open-mmlab/mmdetection"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/learning-transferable-visual-models-from","slug":"learning-transferable-visual-models-from","title":"Learning Transferable Visual Models From Natural Language Supervision","date":"2021-02-26","arxiv_id":"2103.00020","repositories_listed":82,"syntology":{"n":20,"n_ran":16,"n_constructed":0,"n_ran_checked":2,"n_instrument":14,"n_unverified":4,"n_honours":1,"n_violates":1,"n_no_contract":0,"n_pointer_only":16,"phrase":"16 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 1 honoured, 1 violated, 0 with no contract checked; 14 where Syntology's instrument failed) · 4 unverified","sample_list":"/paper/learning-transferable-visual-models-from#ran","syntology_url":"https://syntology.ai/paper/2103.00020","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2103.00020"}},"official":{"repos":["openai/CLIP"],"state":"official: no sample here; runs from other or unrecorded repositories","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed","unlocated"]}}},{"url":"/paper/fashion-mnist-a-novel-image-dataset-for","slug":"fashion-mnist-a-novel-image-dataset-for","title":"Fashion-MNIST: a Novel Image Dataset for Benchmarking Machine Learning Algorithms","date":"2017-08-25","arxiv_id":"1708.07747","repositories_listed":37,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/fashion-mnist-a-novel-image-dataset-for#ran","syntology_url":"https://syntology.ai/paper/1708.07747","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1708.07747"}},"official":{"repos":["zalandoresearch/fashion-mnist"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/cider-consensus-based-image-description","slug":"cider-consensus-based-image-description","title":"CIDEr: Consensus-based Image Description Evaluation","date":"2014-11-20","arxiv_id":"1411.5726","repositories_listed":24,"syntology":{"n":32,"n_ran":13,"n_constructed":7,"n_ran_checked":9,"n_instrument":4,"n_unverified":19,"n_honours":1,"n_violates":0,"n_no_contract":8,"n_pointer_only":27,"phrase":"13 ran (of which 7 constructed an object rather than computing a result; 9 with no instrument failure: 1 honoured, 0 violated, 8 with no contract checked; 4 where Syntology's instrument failed) · 19 unverified","sample_list":"/paper/cider-consensus-based-image-description#ran","syntology_url":"https://syntology.ai/paper/1411.5726","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1411.5726"}},"official":null}},{"url":"/paper/the-starcraft-multi-agent-challenge","slug":"the-starcraft-multi-agent-challenge","title":"The StarCraft Multi-Agent Challenge","date":"2019-02-11","arxiv_id":"1902.04043","repositories_listed":23,"syntology":{"n":15,"n_ran":10,"n_constructed":0,"n_ran_checked":4,"n_instrument":6,"n_unverified":5,"n_honours":0,"n_violates":0,"n_no_contract":4,"n_pointer_only":14,"phrase":"10 ran (of which 0 constructed an object rather than computing a result; 4 with no instrument failure: 0 honoured, 0 violated, 4 with no contract checked; 6 where Syntology's instrument failed) · 5 unverified","sample_list":"/paper/the-starcraft-multi-agent-challenge#ran","syntology_url":"https://syntology.ai/paper/1902.04043","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1902.04043"}},"official":{"repos":["oxwhirl/pymarl","oxwhirl/smac"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/benchmarking-graph-neural-networks","slug":"benchmarking-graph-neural-networks","title":"Benchmarking Graph Neural Networks","date":"2020-03-02","arxiv_id":"2003.00982","repositories_listed":15,"syntology":{"n":23,"n_ran":10,"n_constructed":0,"n_ran_checked":9,"n_instrument":1,"n_unverified":13,"n_honours":0,"n_violates":0,"n_no_contract":9,"n_pointer_only":1,"phrase":"10 ran (of which 0 constructed an object rather than computing a result; 9 with no instrument failure: 0 honoured, 0 violated, 9 with no contract checked; 1 where Syntology's instrument failed) · 13 unverified","sample_list":"/paper/benchmarking-graph-neural-networks#ran","syntology_url":"https://syntology.ai/paper/2003.00982","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2003.00982"}},"official":{"repos":["graphdeeplearning/benchmarking-gnns","karl-zhao/benchmarking-gnns-pyg"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/benchmarking-deep-reinforcement-learning-for","slug":"benchmarking-deep-reinforcement-learning-for","title":"Benchmarking Deep Reinforcement Learning for Continuous Control","date":"2016-04-22","arxiv_id":"1604.06778","repositories_listed":15,"syntology":{"n":4,"n_ran":3,"n_constructed":0,"n_ran_checked":3,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/benchmarking-deep-reinforcement-learning-for#ran","syntology_url":"https://syntology.ai/paper/1604.06778","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1604.06778"}},"official":{"repos":["rllab/rllab"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/benchmarking-neural-network-robustness-to-2","slug":"benchmarking-neural-network-robustness-to-2","title":"Benchmarking Neural Network Robustness to Common Corruptions and Perturbations","date":"2019-03-28","arxiv_id":"1903.12261","repositories_listed":14,"syntology":{"n":3,"n_ran":2,"n_constructed":0,"n_ran_checked":0,"n_instrument":2,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/benchmarking-neural-network-robustness-to-2#ran","syntology_url":"https://syntology.ai/paper/1903.12261","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1903.12261"}},"official":{"repos":["hendrycks/robustness"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/ms-marco-a-human-generated-machine-reading","slug":"ms-marco-a-human-generated-machine-reading","title":"MS MARCO: A Human Generated MAchine Reading COmprehension Dataset","date":"2016-11-28","arxiv_id":"1611.09268","repositories_listed":14,"syntology":{"n":33,"n_ran":28,"n_constructed":0,"n_ran_checked":23,"n_instrument":5,"n_unverified":5,"n_honours":0,"n_violates":0,"n_no_contract":23,"n_pointer_only":0,"phrase":"28 ran (of which 0 constructed an object rather than computing a result; 23 with no instrument failure: 0 honoured, 0 violated, 23 with no contract checked; 5 where Syntology's instrument failed) · 5 unverified","sample_list":"/paper/ms-marco-a-human-generated-machine-reading#ran","syntology_url":"https://syntology.ai/paper/1611.09268","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1611.09268"}},"official":null}},{"url":"/paper/habitat-a-platform-for-embodied-ai-research","slug":"habitat-a-platform-for-embodied-ai-research","title":"Habitat: A Platform for Embodied AI Research","date":"2019-04-02","arxiv_id":"1904.01201","repositories_listed":13,"syntology":{"n":15,"n_ran":13,"n_constructed":0,"n_ran_checked":11,"n_instrument":2,"n_unverified":2,"n_honours":0,"n_violates":1,"n_no_contract":10,"n_pointer_only":15,"phrase":"13 ran (of which 0 constructed an object rather than computing a result; 11 with no instrument failure: 0 honoured, 1 violated, 10 with no contract checked; 2 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/habitat-a-platform-for-embodied-ai-research#ran","syntology_url":"https://syntology.ai/paper/1904.01201","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1904.01201"}},"official":{"repos":["facebookresearch/habitat-sim"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/ai-fairness-360-an-extensible-toolkit-for","slug":"ai-fairness-360-an-extensible-toolkit-for","title":"AI Fairness 360: An Extensible Toolkit for Detecting, Understanding, and Mitigating Unwanted Algorithmic Bias","date":"2018-10-03","arxiv_id":"1810.01943","repositories_listed":13,"syntology":{"n":11,"n_ran":11,"n_constructed":0,"n_ran_checked":10,"n_instrument":1,"n_unverified":0,"n_honours":2,"n_violates":0,"n_no_contract":8,"n_pointer_only":1,"phrase":"11 ran (of which 0 constructed an object rather than computing a result; 10 with no instrument failure: 2 honoured, 0 violated, 8 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/ai-fairness-360-an-extensible-toolkit-for#ran","syntology_url":"https://syntology.ai/paper/1810.01943","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1810.01943"}},"official":{"repos":["IBM/AIF360"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":4,"n_unverified":0,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/technical-report-on-the-cleverhans-v210","slug":"technical-report-on-the-cleverhans-v210","title":"Technical Report on the CleverHans v2.1.0 Adversarial Examples Library","date":"2016-10-03","arxiv_id":"1610.00768","repositories_listed":13,"syntology":{"n":26,"n_ran":12,"n_constructed":0,"n_ran_checked":11,"n_instrument":1,"n_unverified":14,"n_honours":1,"n_violates":0,"n_no_contract":10,"n_pointer_only":26,"phrase":"12 ran (of which 0 constructed an object rather than computing a result; 11 with no instrument failure: 1 honoured, 0 violated, 10 with no contract checked; 1 where Syntology's instrument failed) · 14 unverified","sample_list":"/paper/technical-report-on-the-cleverhans-v210#ran","syntology_url":"https://syntology.ai/paper/1610.00768","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1610.00768"}},"official":{"repos":["tensorflow/cleverhans"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/a-large-annotated-medical-image-dataset-for","slug":"a-large-annotated-medical-image-dataset-for","title":"A large annotated medical image dataset for the development and evaluation of segmentation algorithms","date":"2019-02-25","arxiv_id":"1902.09063","repositories_listed":12,"syntology":{"n":8,"n_ran":7,"n_constructed":0,"n_ran_checked":7,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":7,"n_pointer_only":0,"phrase":"7 ran (of which 0 constructed an object rather than computing a result; 7 with no instrument failure: 0 honoured, 0 violated, 7 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/a-large-annotated-medical-image-dataset-for#ran","syntology_url":"https://syntology.ai/paper/1902.09063","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1902.09063"}},"official":null}},{"url":"/paper/coco-a-platform-for-comparing-continuous","slug":"coco-a-platform-for-comparing-continuous","title":"COCO: A Platform for Comparing Continuous Optimizers in a Black-Box Setting","date":"2016-03-29","arxiv_id":"1603.08785","repositories_listed":12,"syntology":null},{"url":"/paper/multitask-learning-and-benchmarking-with","slug":"multitask-learning-and-benchmarking-with","title":"Multitask learning and benchmarking with clinical time series data","date":"2017-03-22","arxiv_id":"1703.07771","repositories_listed":11,"syntology":{"n":40,"n_ran":25,"n_constructed":0,"n_ran_checked":18,"n_instrument":7,"n_unverified":15,"n_honours":1,"n_violates":0,"n_no_contract":17,"n_pointer_only":5,"phrase":"25 ran (of which 0 constructed an object rather than computing a result; 18 with no instrument failure: 1 honoured, 0 violated, 17 with no contract checked; 7 where Syntology's instrument failed) · 15 unverified","sample_list":"/paper/multitask-learning-and-benchmarking-with#ran","syntology_url":"https://syntology.ai/paper/1703.07771","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1703.07771"}},"official":{"repos":["yerevann/mimic3-benchmarks"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":1,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/benchmarking-generalization-via-in-context","slug":"benchmarking-generalization-via-in-context","title":"Super-NaturalInstructions: Generalization via Declarative Instructions on 1600+ NLP Tasks","date":"2022-04-16","arxiv_id":"2204.07705","repositories_listed":10,"syntology":{"n":28,"n_ran":17,"n_constructed":1,"n_ran_checked":15,"n_instrument":2,"n_unverified":11,"n_honours":0,"n_violates":0,"n_no_contract":15,"n_pointer_only":4,"phrase":"17 ran (of which 1 constructed an object rather than computing a result; 15 with no instrument failure: 0 honoured, 0 violated, 15 with no contract checked; 2 where Syntology's instrument failed) · 11 unverified","sample_list":"/paper/benchmarking-generalization-via-in-context#ran","syntology_url":"https://syntology.ai/paper/2204.07705","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2204.07705"}},"official":{"repos":["allenai/natural-instructions"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/on-evaluation-of-embodied-navigation-agents","slug":"on-evaluation-of-embodied-navigation-agents","title":"On Evaluation of Embodied Navigation Agents","date":"2018-07-18","arxiv_id":"1807.06757","repositories_listed":10,"syntology":null},{"url":"/paper/comparative-evaluation-of-multi-agent-deep","slug":"comparative-evaluation-of-multi-agent-deep","title":"Benchmarking Multi-Agent Deep Reinforcement Learning Algorithms in Cooperative Tasks","date":"2020-06-14","arxiv_id":"2006.07869","repositories_listed":9,"syntology":{"n":12,"n_ran":11,"n_constructed":0,"n_ran_checked":6,"n_instrument":5,"n_unverified":1,"n_honours":1,"n_violates":0,"n_no_contract":5,"n_pointer_only":7,"phrase":"11 ran (of which 0 constructed an object rather than computing a result; 6 with no instrument failure: 1 honoured, 0 violated, 5 with no contract checked; 5 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/comparative-evaluation-of-multi-agent-deep#ran","syntology_url":"https://syntology.ai/paper/2006.07869","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2006.07869"}},"official":{"repos":["uoe-agents/epymarl","uoe-agents/robotic-warehouse","uoe-agents/lb-foraging"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/torchreid-a-library-for-deep-learning-person","slug":"torchreid-a-library-for-deep-learning-person","title":"Torchreid: A Library for Deep Learning Person Re-Identification in Pytorch","date":"2019-10-22","arxiv_id":"1910.10093","repositories_listed":9,"syntology":{"n":24,"n_ran":11,"n_constructed":0,"n_ran_checked":10,"n_instrument":1,"n_unverified":13,"n_honours":0,"n_violates":0,"n_no_contract":10,"n_pointer_only":3,"phrase":"11 ran (of which 0 constructed an object rather than computing a result; 10 with no instrument failure: 0 honoured, 0 violated, 10 with no contract checked; 1 where Syntology's instrument failed) · 13 unverified","sample_list":"/paper/torchreid-a-library-for-deep-learning-person#ran","syntology_url":"https://syntology.ai/paper/1910.10093","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1910.10093"}},"official":{"repos":["KaiyangZhou/deep-person-reid"],"state":"official (archive's flag): 6 ran","n_ran":6,"n_constructed":0,"n_ran_no_instrument_failure":6,"n_unverified":9,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/benchmarking-natural-language-understanding","slug":"benchmarking-natural-language-understanding","title":"Benchmarking Natural Language Understanding Services for building Conversational Agents","date":"2019-03-13","arxiv_id":"1903.05566","repositories_listed":9,"syntology":null},{"url":"/paper/data-splits-and-metrics-for-method","slug":"data-splits-and-metrics-for-method","title":"Data Splits and Metrics for Method Benchmarking on Surgical Action Triplet Datasets","date":"2022-04-11","arxiv_id":"2204.05235","repositories_listed":8,"syntology":null},{"url":"/paper/multitask-prompted-training-enables-zero-shot-1","slug":"multitask-prompted-training-enables-zero-shot-1","title":"Multitask Prompted Training Enables Zero-Shot Task Generalization","date":"2021-10-15","arxiv_id":"2110.08207","repositories_listed":8,"syntology":{"n":15,"n_ran":13,"n_constructed":0,"n_ran_checked":9,"n_instrument":4,"n_unverified":2,"n_honours":0,"n_violates":1,"n_no_contract":8,"n_pointer_only":3,"phrase":"13 ran (of which 0 constructed an object rather than computing a result; 9 with no instrument failure: 0 honoured, 1 violated, 8 with no contract checked; 4 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/multitask-prompted-training-enables-zero-shot-1#ran","syntology_url":"https://syntology.ai/paper/2110.08207","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2110.08207"}},"official":{"repos":["bigscience-workshop/promptsource","bigscience-workshop/t-zero"],"state":"official (archive's flag): 6 ran","n_ran":6,"n_constructed":0,"n_ran_no_instrument_failure":4,"n_unverified":0,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/capsule-vision-2024-challenge-multi-class","slug":"capsule-vision-2024-challenge-multi-class","title":"Capsule Vision 2024 Challenge: Multi-Class Abnormality Classification for Video Capsule Endoscopy","date":"2024-08-09","arxiv_id":"2408.04940","repositories_listed":7,"syntology":null},{"url":"/paper/the-kits19-challenge-data-300-kidney-tumor","slug":"the-kits19-challenge-data-300-kidney-tumor","title":"The KiTS19 Challenge Data: 300 Kidney Tumor Cases with Clinical Context, CT Semantic Segmentations, and Surgical Outcomes","date":"2019-03-31","arxiv_id":"1904.00445","repositories_listed":7,"syntology":{"n":4,"n_ran":3,"n_constructed":0,"n_ran_checked":1,"n_instrument":2,"n_unverified":1,"n_honours":0,"n_violates":1,"n_no_contract":0,"n_pointer_only":4,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 1 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/the-kits19-challenge-data-300-kidney-tumor#ran","syntology_url":"https://syntology.ai/paper/1904.00445","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1904.00445"}},"official":{"repos":["neheller/kits19"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/leaf-a-benchmark-for-federated-settings","slug":"leaf-a-benchmark-for-federated-settings","title":"LEAF: A Benchmark for Federated Settings","date":"2018-12-03","arxiv_id":"1812.01097","repositories_listed":7,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":2,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/leaf-a-benchmark-for-federated-settings#ran","syntology_url":"https://syntology.ai/paper/1812.01097","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1812.01097"}},"official":{"repos":["TalwalkarLab/leaf"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/benchmarking-robustness-of-3d-point-cloud","slug":"benchmarking-robustness-of-3d-point-cloud","title":"Benchmarking Robustness of 3D Point Cloud Recognition Against Common Corruptions","date":"2022-01-28","arxiv_id":"2201.12296","repositories_listed":6,"syntology":{"n":29,"n_ran":24,"n_constructed":0,"n_ran_checked":12,"n_instrument":12,"n_unverified":5,"n_honours":2,"n_violates":0,"n_no_contract":10,"n_pointer_only":7,"phrase":"24 ran (of which 0 constructed an object rather than computing a result; 12 with no instrument failure: 2 honoured, 0 violated, 10 with no contract checked; 12 where Syntology's instrument failed) · 5 unverified","sample_list":"/paper/benchmarking-robustness-of-3d-point-cloud#ran","syntology_url":"https://syntology.ai/paper/2201.12296","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2201.12296"}},"official":{"repos":["jiachens/ModelNet40-C"],"state":"official (archive's flag): 11 ran","n_ran":11,"n_constructed":0,"n_ran_no_instrument_failure":4,"n_unverified":2,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/fuxictr-an-open-benchmark-for-click-through","slug":"fuxictr-an-open-benchmark-for-click-through","title":"BARS-CTR: Open Benchmarking for Click-Through Rate Prediction","date":"2020-09-12","arxiv_id":"2009.05794","repositories_listed":6,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/fuxictr-an-open-benchmark-for-click-through#ran","syntology_url":"https://syntology.ai/paper/2009.05794","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2009.05794"}},"official":{"repos":["reczoo/BARS"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/the-measure-of-intelligence","slug":"the-measure-of-intelligence","title":"On the Measure of Intelligence","date":"2019-11-05","arxiv_id":"1911.01547","repositories_listed":6,"syntology":{"n":11,"n_ran":10,"n_constructed":0,"n_ran_checked":10,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":10,"n_pointer_only":0,"phrase":"10 ran (of which 0 constructed an object rather than computing a result; 10 with no instrument failure: 0 honoured, 0 violated, 10 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/the-measure-of-intelligence#ran","syntology_url":"https://syntology.ai/paper/1911.01547","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1911.01547"}},"official":{"repos":["fchollet/ARC"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/the-liver-tumor-segmentation-benchmark-lits","slug":"the-liver-tumor-segmentation-benchmark-lits","title":"The Liver Tumor Segmentation Benchmark (LiTS)","date":"2019-01-13","arxiv_id":"1901.04056","repositories_listed":6,"syntology":{"n":8,"n_ran":7,"n_constructed":0,"n_ran_checked":7,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":7,"n_pointer_only":0,"phrase":"7 ran (of which 0 constructed an object rather than computing a result; 7 with no instrument failure: 0 honoured, 0 violated, 7 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/the-liver-tumor-segmentation-benchmark-lits#ran","syntology_url":"https://syntology.ai/paper/1901.04056","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1901.04056"}},"official":null}},{"url":"/paper/retrieve-merge-predict-augmenting-tables-with","slug":"retrieve-merge-predict-augmenting-tables-with","title":"Retrieve, Merge, Predict: Augmenting Tables with Data Lakes","date":"2024-02-09","arxiv_id":"2402.06282","repositories_listed":5,"syntology":null},{"url":"/paper/exploring-progress-in-multivariate-time","slug":"exploring-progress-in-multivariate-time","title":"Exploring Progress in Multivariate Time Series Forecasting: Comprehensive Benchmarking and Heterogeneity Analysis","date":"2023-10-09","arxiv_id":"2310.06119","repositories_listed":5,"syntology":{"n":12,"n_ran":11,"n_constructed":0,"n_ran_checked":8,"n_instrument":3,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":8,"n_pointer_only":6,"phrase":"11 ran (of which 0 constructed an object rather than computing a result; 8 with no instrument failure: 0 honoured, 0 violated, 8 with no contract checked; 3 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/exploring-progress-in-multivariate-time#ran","syntology_url":"https://syntology.ai/paper/2310.06119","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2310.06119"}},"official":{"repos":["gestaltcogteam/basicts","zezhishao/basicts"],"state":"official (archive's flag): 5 ran","n_ran":5,"n_constructed":0,"n_ran_no_instrument_failure":4,"n_unverified":0,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/corl-research-oriented-deep-offline-1","slug":"corl-research-oriented-deep-offline-1","title":"CORL: Research-oriented Deep Offline Reinforcement Learning Library","date":"2022-10-13","arxiv_id":"2210.07105","repositories_listed":5,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":1,"n_instrument":1,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/corl-research-oriented-deep-offline-1#ran","syntology_url":"https://syntology.ai/paper/2210.07105","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2210.07105"}},"official":{"repos":["corl-team/CORL","hanjuku-kaso/awesome-offline-rl","tinkoff-ai/CORL"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/mteb-massive-text-embedding-benchmark","slug":"mteb-massive-text-embedding-benchmark","title":"MTEB: Massive Text Embedding Benchmark","date":"2022-10-13","arxiv_id":"2210.07316","repositories_listed":5,"syntology":{"n":13,"n_ran":10,"n_constructed":0,"n_ran_checked":7,"n_instrument":3,"n_unverified":3,"n_honours":0,"n_violates":0,"n_no_contract":7,"n_pointer_only":0,"phrase":"10 ran (of which 0 constructed an object rather than computing a result; 7 with no instrument failure: 0 honoured, 0 violated, 7 with no contract checked; 3 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/mteb-massive-text-embedding-benchmark#ran","syntology_url":"https://syntology.ai/paper/2210.07316","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2210.07316"}},"official":{"repos":["embeddings-benchmark/mteb"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/fast-vision-transformers-with-hilo-attention","slug":"fast-vision-transformers-with-hilo-attention","title":"Fast Vision Transformers with HiLo Attention","date":"2022-05-26","arxiv_id":"2205.13213","repositories_listed":5,"syntology":{"n":13,"n_ran":11,"n_constructed":1,"n_ran_checked":9,"n_instrument":2,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":9,"n_pointer_only":2,"phrase":"11 ran (of which 1 constructed an object rather than computing a result; 9 with no instrument failure: 0 honoured, 0 violated, 9 with no contract checked; 2 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/fast-vision-transformers-with-hilo-attention#ran","syntology_url":"https://syntology.ai/paper/2205.13213","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2205.13213"}},"official":{"repos":["zip-group/litv2","ziplab/litv2"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/bars-towards-open-benchmarking-for","slug":"bars-towards-open-benchmarking-for","title":"BARS: Towards Open Benchmarking for Recommender Systems","date":"2022-05-19","arxiv_id":"2205.09626","repositories_listed":5,"syntology":{"n":8,"n_ran":5,"n_constructed":0,"n_ran_checked":3,"n_instrument":2,"n_unverified":3,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":1,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 2 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/bars-towards-open-benchmarking-for#ran","syntology_url":"https://syntology.ai/paper/2205.09626","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2205.09626"}},"official":{"repos":["openbenchmark/BARS"],"state":"official (archive's flag): 5 ran","n_ran":5,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":3,"ran_from_kinds":["official"]}}},{"url":"/paper/benchmarking-graphormer-on-large-scale","slug":"benchmarking-graphormer-on-large-scale","title":"Benchmarking Graphormer on Large-Scale Molecular Modeling Datasets","date":"2022-03-09","arxiv_id":"2203.04810","repositories_listed":5,"syntology":{"n":9,"n_ran":8,"n_constructed":0,"n_ran_checked":5,"n_instrument":3,"n_unverified":1,"n_honours":0,"n_violates":1,"n_no_contract":4,"n_pointer_only":2,"phrase":"8 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 0 honoured, 1 violated, 4 with no contract checked; 3 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/benchmarking-graphormer-on-large-scale#ran","syntology_url":"https://syntology.ai/paper/2203.04810","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2203.04810"}},"official":{"repos":["Microsoft/Graphormer"],"state":"official (archive's flag): 5 ran","n_ran":5,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/contemporary-symbolic-regression-methods-and","slug":"contemporary-symbolic-regression-methods-and","title":"Contemporary Symbolic Regression Methods and their Relative Performance","date":"2021-07-29","arxiv_id":"2107.14351","repositories_listed":5,"syntology":{"n":8,"n_ran":8,"n_constructed":0,"n_ran_checked":8,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":8,"n_pointer_only":0,"phrase":"8 ran (of which 0 constructed an object rather than computing a result; 8 with no instrument failure: 0 honoured, 0 violated, 8 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/contemporary-symbolic-regression-methods-and#ran","syntology_url":"https://syntology.ai/paper/2107.14351","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2107.14351"}},"official":{"repos":["EpistasisLab/srbench","heal-research/operon","lacava/ode-strogatz","marcovirgolin/GP-GOMEA"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/long-range-arena-a-benchmark-for-efficient-1","slug":"long-range-arena-a-benchmark-for-efficient-1","title":"Long Range Arena: A Benchmark for Efficient Transformers","date":"2020-11-08","arxiv_id":"2011.04006","repositories_listed":5,"syntology":{"n":3,"n_ran":2,"n_constructed":0,"n_ran_checked":2,"n_instrument":0,"n_unverified":1,"n_honours":2,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 2 honoured, 0 violated, 0 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/long-range-arena-a-benchmark-for-efficient-1#ran","syntology_url":"https://syntology.ai/paper/2011.04006","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2011.04006"}},"official":{"repos":["google-research/long-range-arena"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/tadgan-time-series-anomaly-detection-using","slug":"tadgan-time-series-anomaly-detection-using","title":"TadGAN: Time Series Anomaly Detection Using Generative Adversarial Networks","date":"2020-09-16","arxiv_id":"2009.07769","repositories_listed":5,"syntology":null},{"url":"/paper/ugc-vqa-benchmarking-blind-video-quality","slug":"ugc-vqa-benchmarking-blind-video-quality","title":"UGC-VQA: Benchmarking Blind Video Quality Assessment for User Generated Content","date":"2020-05-29","arxiv_id":"2005.14354","repositories_listed":5,"syntology":null},{"url":"/paper/ntire-2020-challenge-on-real-world-image","slug":"ntire-2020-challenge-on-real-world-image","title":"NTIRE 2020 Challenge on Real-World Image Super-Resolution: Methods and Results","date":"2020-05-05","arxiv_id":"2005.01996","repositories_listed":5,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":1,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":2,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/ntire-2020-challenge-on-real-world-image#ran","syntology_url":"https://syntology.ai/paper/2005.01996","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2005.01996"}},"official":null}},{"url":"/paper/image-matching-across-wide-baselines-from","slug":"image-matching-across-wide-baselines-from","title":"Image Matching across Wide Baselines: From Paper to Practice","date":"2020-03-03","arxiv_id":"2003.01587","repositories_listed":5,"syntology":{"n":15,"n_ran":14,"n_constructed":0,"n_ran_checked":14,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":1,"n_no_contract":13,"n_pointer_only":1,"phrase":"14 ran (of which 0 constructed an object rather than computing a result; 14 with no instrument failure: 0 honoured, 1 violated, 13 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/image-matching-across-wide-baselines-from#ran","syntology_url":"https://syntology.ai/paper/2003.01587","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2003.01587"}},"official":{"repos":["etrulls/slurm-gcp","vcg-uvic/image-matching-benchmark"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":4,"n_unverified":0,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/benchmarking-batch-deep-reinforcement","slug":"benchmarking-batch-deep-reinforcement","title":"Benchmarking Batch Deep Reinforcement Learning Algorithms","date":"2019-10-03","arxiv_id":"1910.01708","repositories_listed":5,"syntology":null},{"url":"/paper/an-evaluation-dataset-for-intent","slug":"an-evaluation-dataset-for-intent","title":"An Evaluation Dataset for Intent Classification and Out-of-Scope Prediction","date":"2019-09-04","arxiv_id":"1909.02027","repositories_listed":5,"syntology":null},{"url":"/paper/new-loss-functions-for-fast-maximum-inner","slug":"new-loss-functions-for-fast-maximum-inner","title":"Accelerating Large-Scale Inference with Anisotropic Vector Quantization","date":"2019-08-27","arxiv_id":"1908.10396","repositories_listed":5,"syntology":{"n":8,"n_ran":7,"n_constructed":0,"n_ran_checked":7,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":7,"n_pointer_only":0,"phrase":"7 ran (of which 0 constructed an object rather than computing a result; 7 with no instrument failure: 0 honoured, 0 violated, 7 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/new-loss-functions-for-fast-maximum-inner#ran","syntology_url":"https://syntology.ai/paper/1908.10396","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1908.10396"}},"official":{"repos":["google-research/google-research"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/gratis-generating-time-series-with-diverse","slug":"gratis-generating-time-series-with-diverse","title":"GRATIS: GeneRAting TIme Series with diverse and controllable characteristics","date":"2019-03-07","arxiv_id":"1903.02787","repositories_listed":5,"syntology":null},{"url":"/paper/partnet-a-large-scale-benchmark-for-fine","slug":"partnet-a-large-scale-benchmark-for-fine","title":"PartNet: A Large-scale Benchmark for Fine-grained and Hierarchical Part-level 3D Object Understanding","date":"2018-12-06","arxiv_id":"1812.02713","repositories_listed":5,"syntology":null},{"url":"/paper/iohprofiler-a-benchmarking-and-profiling-tool","slug":"iohprofiler-a-benchmarking-and-profiling-tool","title":"IOHprofiler: A Benchmarking and Profiling Tool for Iterative Optimization Heuristics","date":"2018-10-11","arxiv_id":"1810.05281","repositories_listed":5,"syntology":null},{"url":"/paper/evaluating-shallow-and-deep-neural-networks","slug":"evaluating-shallow-and-deep-neural-networks","title":"Evaluating Shallow and Deep Neural Networks for Network Intrusion Detection Systems in Cyber Security","date":"2018-10-08","arxiv_id":null,"repositories_listed":5,"syntology":null},{"url":"/paper/fast-and-accurate-alignment-of-long-bisulfite","slug":"fast-and-accurate-alignment-of-long-bisulfite","title":"Fast and accurate alignment of long bisulfite-seq reads","date":"2014-01-06","arxiv_id":"1401.1129","repositories_listed":5,"syntology":null},{"url":"/paper/nninteractive-redefining-3d-promptable","slug":"nninteractive-redefining-3d-promptable","title":"nnInteractive: Redefining 3D Promptable Segmentation","date":"2025-03-11","arxiv_id":"2503.08373","repositories_listed":4,"syntology":{"n":9,"n_ran":8,"n_constructed":0,"n_ran_checked":7,"n_instrument":1,"n_unverified":1,"n_honours":2,"n_violates":0,"n_no_contract":5,"n_pointer_only":0,"phrase":"8 ran (of which 0 constructed an object rather than computing a result; 7 with no instrument failure: 2 honoured, 0 violated, 5 with no contract checked; 1 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/nninteractive-redefining-3d-promptable#ran","syntology_url":"https://syntology.ai/paper/2503.08373","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2503.08373"}},"official":{"repos":["mic-dkfz/napari-nninteractive","mic-dkfz/nninteractive"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":0,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/limesoda-a-dataset-collection-for","slug":"limesoda-a-dataset-collection-for","title":"LimeSoDa: A Dataset Collection for Benchmarking of Machine Learning Regressors in Digital Soil Mapping","date":"2025-02-27","arxiv_id":"2502.20139","repositories_listed":4,"syntology":null},{"url":"/paper/enabling-more-efficient-and-cost-effective-ai","slug":"enabling-more-efficient-and-cost-effective-ai","title":"Enabling more efficient and cost-effective AI/ML systems with Collective Mind, virtualized MLOps, MLPerf, Collective Knowledge Playground and reproducible optimization tournaments","date":"2024-06-24","arxiv_id":"2406.16791","repositories_listed":4,"syntology":null},{"url":"/paper/bigcodebench-benchmarking-code-generation","slug":"bigcodebench-benchmarking-code-generation","title":"BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions","date":"2024-06-22","arxiv_id":"2406.15877","repositories_listed":4,"syntology":{"n":6,"n_ran":5,"n_constructed":0,"n_ran_checked":1,"n_instrument":4,"n_unverified":1,"n_honours":0,"n_violates":1,"n_no_contract":0,"n_pointer_only":1,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 1 violated, 0 with no contract checked; 4 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/bigcodebench-benchmarking-code-generation#ran","syntology_url":"https://syntology.ai/paper/2406.15877","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.15877"}},"official":{"repos":["bigcode-project/bigcodebench","bigcode-project/bigcodebench-annotation"],"state":"official (archive's flag): 5 ran","n_ran":5,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/tsi-bench-benchmarking-time-series-imputation-1","slug":"tsi-bench-benchmarking-time-series-imputation-1","title":"TSI-Bench: Benchmarking Time Series Imputation","date":"2024-06-18","arxiv_id":"2406.12747","repositories_listed":4,"syntology":null},{"url":"/paper/bench2drive-towards-multi-ability","slug":"bench2drive-towards-multi-ability","title":"Bench2Drive: Towards Multi-Ability Benchmarking of Closed-Loop End-To-End Autonomous Driving","date":"2024-06-06","arxiv_id":"2406.03877","repositories_listed":4,"syntology":{"n":17,"n_ran":13,"n_constructed":0,"n_ran_checked":3,"n_instrument":10,"n_unverified":4,"n_honours":1,"n_violates":0,"n_no_contract":2,"n_pointer_only":17,"phrase":"13 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 1 honoured, 0 violated, 2 with no contract checked; 10 where Syntology's instrument failed) · 4 unverified","sample_list":"/paper/bench2drive-towards-multi-ability#ran","syntology_url":"https://syntology.ai/paper/2406.03877","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.03877"}},"official":{"repos":["Thinklab-SJTU/Bench2Drive","autonomousvision/carla_garage","thinklab-sjtu/bench2drivezoo"],"state":"official (archive's flag): 13 ran","n_ran":13,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":4,"ran_from_kinds":["official"]}}},{"url":"/paper/openfactcheck-a-unified-framework-for","slug":"openfactcheck-a-unified-framework-for","title":"OpenFactCheck: Building, Benchmarking Customized Fact-Checking Systems and Evaluating the Factuality of Claims and LLMs","date":"2024-05-09","arxiv_id":"2405.05583","repositories_listed":4,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":0,"n_instrument":3,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":3,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 3 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/openfactcheck-a-unified-framework-for#ran","syntology_url":"https://syntology.ai/paper/2405.05583","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2405.05583"}},"official":{"repos":["yuxiaw/openfactcheck"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/nsina-a-news-corpus-for-sinhala","slug":"nsina-a-news-corpus-for-sinhala","title":"NSINA: A News Corpus for Sinhala","date":"2024-03-25","arxiv_id":"2403.16571","repositories_listed":4,"syntology":null},{"url":"/paper/stabletoolbench-towards-stable-large-scale","slug":"stabletoolbench-towards-stable-large-scale","title":"StableToolBench: Towards Stable Large-Scale Benchmarking on Tool Learning of Large Language Models","date":"2024-03-12","arxiv_id":"2403.07714","repositories_listed":4,"syntology":{"n":15,"n_ran":11,"n_constructed":0,"n_ran_checked":8,"n_instrument":3,"n_unverified":4,"n_honours":0,"n_violates":2,"n_no_contract":6,"n_pointer_only":5,"phrase":"11 ran (of which 0 constructed an object rather than computing a result; 8 with no instrument failure: 0 honoured, 2 violated, 6 with no contract checked; 3 where Syntology's instrument failed) · 4 unverified","sample_list":"/paper/stabletoolbench-towards-stable-large-scale#ran","syntology_url":"https://syntology.ai/paper/2403.07714","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2403.07714"}},"official":{"repos":["thunlp-mt/stabletoolbench","zhichengg/stabletoolbench"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/mme-a-comprehensive-evaluation-benchmark-for","slug":"mme-a-comprehensive-evaluation-benchmark-for","title":"MME: A Comprehensive Evaluation Benchmark for Multimodal Large Language Models","date":"2023-06-23","arxiv_id":"2306.13394","repositories_listed":4,"syntology":null},{"url":"/paper/openp5-benchmarking-foundation-models-for","slug":"openp5-benchmarking-foundation-models-for","title":"OpenP5: An Open-Source Platform for Developing, Training, and Evaluating LLM-based Recommender Systems","date":"2023-06-19","arxiv_id":"2306.11134","repositories_listed":4,"syntology":{"n":25,"n_ran":23,"n_constructed":0,"n_ran_checked":16,"n_instrument":7,"n_unverified":2,"n_honours":1,"n_violates":1,"n_no_contract":14,"n_pointer_only":9,"phrase":"23 ran (of which 0 constructed an object rather than computing a result; 16 with no instrument failure: 1 honoured, 1 violated, 14 with no contract checked; 7 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/openp5-benchmarking-foundation-models-for#ran","syntology_url":"https://syntology.ai/paper/2306.11134","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2306.11134"}},"official":{"repos":["agiresearch/openp5"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/benchmarking-neural-network-training","slug":"benchmarking-neural-network-training","title":"Benchmarking Neural Network Training Algorithms","date":"2023-06-12","arxiv_id":"2306.07179","repositories_listed":4,"syntology":{"n":22,"n_ran":21,"n_constructed":0,"n_ran_checked":21,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":21,"n_pointer_only":0,"phrase":"21 ran (of which 0 constructed an object rather than computing a result; 21 with no instrument failure: 0 honoured, 0 violated, 21 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/benchmarking-neural-network-training#ran","syntology_url":"https://syntology.ai/paper/2306.07179","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2306.07179"}},"official":{"repos":["mlcommons/algorithmic-efficiency"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/yet-another-icu-benchmark-a-flexible-multi","slug":"yet-another-icu-benchmark-a-flexible-multi","title":"Yet Another ICU Benchmark: A Flexible Multi-Center Framework for Clinical ML","date":"2023-06-08","arxiv_id":"2306.05109","repositories_listed":4,"syntology":{"n":25,"n_ran":21,"n_constructed":0,"n_ran_checked":15,"n_instrument":6,"n_unverified":4,"n_honours":1,"n_violates":0,"n_no_contract":14,"n_pointer_only":1,"phrase":"21 ran (of which 0 constructed an object rather than computing a result; 15 with no instrument failure: 1 honoured, 0 violated, 14 with no contract checked; 6 where Syntology's instrument failed) · 4 unverified","sample_list":"/paper/yet-another-icu-benchmark-a-flexible-multi#ran","syntology_url":"https://syntology.ai/paper/2306.05109","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2306.05109"}},"official":{"repos":["rvandewater/recipys","rvandewater/yaib","rvandewater/yaib-cohorts","rvandewater/yaib-models"],"state":"official (archive's flag): 21 ran","n_ran":21,"n_constructed":0,"n_ran_no_instrument_failure":15,"n_unverified":4,"ran_from_kinds":["official"]}}},{"url":"/paper/mgtbench-benchmarking-machine-generated-text","slug":"mgtbench-benchmarking-machine-generated-text","title":"MGTBench: Benchmarking Machine-Generated Text Detection","date":"2023-03-26","arxiv_id":"2303.14822","repositories_listed":4,"syntology":{"n":26,"n_ran":19,"n_constructed":0,"n_ran_checked":12,"n_instrument":7,"n_unverified":7,"n_honours":0,"n_violates":0,"n_no_contract":12,"n_pointer_only":5,"phrase":"19 ran (of which 0 constructed an object rather than computing a result; 12 with no instrument failure: 0 honoured, 0 violated, 12 with no contract checked; 7 where Syntology's instrument failed) · 7 unverified","sample_list":"/paper/mgtbench-benchmarking-machine-generated-text#ran","syntology_url":"https://syntology.ai/paper/2303.14822","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2303.14822"}},"official":{"repos":["xinleihe/mgtbench"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/openood-benchmarking-generalized-out-of","slug":"openood-benchmarking-generalized-out-of","title":"OpenOOD: Benchmarking Generalized Out-of-Distribution Detection","date":"2022-10-13","arxiv_id":"2210.07242","repositories_listed":4,"syntology":{"n":17,"n_ran":15,"n_constructed":0,"n_ran_checked":10,"n_instrument":5,"n_unverified":2,"n_honours":3,"n_violates":2,"n_no_contract":5,"n_pointer_only":2,"phrase":"15 ran (of which 0 constructed an object rather than computing a result; 10 with no instrument failure: 3 honoured, 2 violated, 5 with no contract checked; 5 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/openood-benchmarking-generalized-out-of#ran","syntology_url":"https://syntology.ai/paper/2210.07242","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2210.07242"}},"official":{"repos":["jingkang50/openood"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/a-framework-for-benchmarking-clustering","slug":"a-framework-for-benchmarking-clustering","title":"A framework for benchmarking clustering algorithms","date":"2022-09-20","arxiv_id":"2209.09493","repositories_listed":4,"syntology":null},{"url":"/paper/benchmarking-and-analyzing-point-cloud","slug":"benchmarking-and-analyzing-point-cloud","title":"Benchmarking and Analyzing Point Cloud Classification under Corruptions","date":"2022-02-07","arxiv_id":"2202.03377","repositories_listed":4,"syntology":{"n":13,"n_ran":11,"n_constructed":0,"n_ran_checked":6,"n_instrument":5,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":6,"n_pointer_only":2,"phrase":"11 ran (of which 0 constructed an object rather than computing a result; 6 with no instrument failure: 0 honoured, 0 violated, 6 with no contract checked; 5 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/benchmarking-and-analyzing-point-cloud#ran","syntology_url":"https://syntology.ai/paper/2202.03377","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2202.03377"}},"official":{"repos":["jiawei-ren/modelnetc"],"state":"official (archive's flag): 11 ran","n_ran":11,"n_constructed":0,"n_ran_no_instrument_failure":6,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/learning-with-noisy-labels-revisited-a-study-1","slug":"learning-with-noisy-labels-revisited-a-study-1","title":"Learning with Noisy Labels Revisited: A Study Using Real-World Human Annotations","date":"2021-10-22","arxiv_id":"2110.12088","repositories_listed":4,"syntology":null},{"url":"/paper/carla-a-python-library-to-benchmark","slug":"carla-a-python-library-to-benchmark","title":"CARLA: A Python Library to Benchmark Algorithmic Recourse and Counterfactual Explanation Algorithms","date":"2021-08-02","arxiv_id":"2108.00783","repositories_listed":4,"syntology":{"n":8,"n_ran":4,"n_constructed":0,"n_ran_checked":4,"n_instrument":0,"n_unverified":4,"n_honours":0,"n_violates":0,"n_no_contract":4,"n_pointer_only":0,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 4 with no instrument failure: 0 honoured, 0 violated, 4 with no contract checked; 0 where Syntology's instrument failed) · 4 unverified","sample_list":"/paper/carla-a-python-library-to-benchmark#ran","syntology_url":"https://syntology.ai/paper/2108.00783","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2108.00783"}},"official":{"repos":["indyfree/CARLA"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":4,"n_unverified":4,"ran_from_kinds":["official"]}}},{"url":"/paper/nats-bench-benchmarking-nas-algorithms-for","slug":"nats-bench-benchmarking-nas-algorithms-for","title":"NATS-Bench: Benchmarking NAS Algorithms for Architecture Topology and Size","date":"2020-08-28","arxiv_id":"2009.00437","repositories_listed":4,"syntology":{"n":4,"n_ran":4,"n_constructed":0,"n_ran_checked":0,"n_instrument":4,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":2,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 4 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/nats-bench-benchmarking-nas-algorithms-for#ran","syntology_url":"https://syntology.ai/paper/2009.00437","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2009.00437"}},"official":{"repos":["D-X-Y/AutoDL-Projects"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/global-wheat-head-detection-gwhd-dataset-a","slug":"global-wheat-head-detection-gwhd-dataset-a","title":"Global Wheat Head Detection (GWHD) dataset: a large and diverse dataset of high resolution RGB labelled images to develop and benchmark wheat head detection methods","date":"2020-04-25","arxiv_id":"2005.02162","repositories_listed":4,"syntology":null},{"url":"/paper/mlperf-inference-benchmark","slug":"mlperf-inference-benchmark","title":"MLPerf Inference Benchmark","date":"2019-11-06","arxiv_id":"1911.02549","repositories_listed":4,"syntology":{"n":21,"n_ran":18,"n_constructed":0,"n_ran_checked":18,"n_instrument":0,"n_unverified":3,"n_honours":0,"n_violates":0,"n_no_contract":18,"n_pointer_only":0,"phrase":"18 ran (of which 0 constructed an object rather than computing a result; 18 with no instrument failure: 0 honoured, 0 violated, 18 with no contract checked; 0 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/mlperf-inference-benchmark#ran","syntology_url":"https://syntology.ai/paper/1911.02549","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1911.02549"}},"official":null}},{"url":"/paper/benchmarking-zero-shot-text-classification","slug":"benchmarking-zero-shot-text-classification","title":"Benchmarking Zero-shot Text Classification: Datasets, Evaluation and Entailment Approach","date":"2019-08-31","arxiv_id":"1909.00161","repositories_listed":4,"syntology":{"n":6,"n_ran":3,"n_constructed":0,"n_ran_checked":1,"n_instrument":2,"n_unverified":3,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":3,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 2 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/benchmarking-zero-shot-text-classification#ran","syntology_url":"https://syntology.ai/paper/1909.00161","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1909.00161"}},"official":{"repos":["yinwenpeng/BenchmarkingZeroShot"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed","official","unlocated"]}}},{"url":"/paper/benchmarking-robustness-in-object-detection","slug":"benchmarking-robustness-in-object-detection","title":"Benchmarking Robustness in Object Detection: Autonomous Driving when Winter is Coming","date":"2019-07-17","arxiv_id":"1907.07484","repositories_listed":4,"syntology":{"n":5,"n_ran":5,"n_constructed":0,"n_ran_checked":4,"n_instrument":1,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":3,"n_pointer_only":0,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 4 with no instrument failure: 1 honoured, 0 violated, 3 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/benchmarking-robustness-in-object-detection#ran","syntology_url":"https://syntology.ai/paper/1907.07484","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1907.07484"}},"official":{"repos":["bethgelab/mmdetection","bethgelab/imagecorruptions","bethgelab/robust-detection-benchmark","bethgelab/stylize-datasets"],"state":"official (archive's flag): 5 ran","n_ran":5,"n_constructed":0,"n_ran_no_instrument_failure":4,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/transfer-learning-in-biomedical-natural","slug":"transfer-learning-in-biomedical-natural","title":"Transfer Learning in Biomedical Natural Language Processing: An Evaluation of BERT and ELMo on Ten Benchmarking Datasets","date":"2019-06-13","arxiv_id":"1906.05474","repositories_listed":4,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":2,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/transfer-learning-in-biomedical-natural#ran","syntology_url":"https://syntology.ai/paper/1906.05474","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1906.05474"}},"official":{"repos":["ncbi-nlp/BLUE_Benchmark","ncbi-nlp/NCBI_BERT"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/nas-bench-101-towards-reproducible-neural","slug":"nas-bench-101-towards-reproducible-neural","title":"NAS-Bench-101: Towards Reproducible Neural Architecture Search","date":"2019-02-25","arxiv_id":"1902.09635","repositories_listed":4,"syntology":{"n":10,"n_ran":9,"n_constructed":0,"n_ran_checked":9,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":9,"n_pointer_only":0,"phrase":"9 ran (of which 0 constructed an object rather than computing a result; 9 with no instrument failure: 0 honoured, 0 violated, 9 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/nas-bench-101-towards-reproducible-neural#ran","syntology_url":"https://syntology.ai/paper/1902.09635","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1902.09635"}},"official":{"repos":["google-research/nasbench","automl/nas_benchmarks"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/ml-net-multi-label-classification-of","slug":"ml-net-multi-label-classification-of","title":"ML-Net: multi-label classification of biomedical texts with deep neural networks","date":"2018-11-13","arxiv_id":"1811.05475","repositories_listed":4,"syntology":null},{"url":"/paper/biosentvec-creating-sentence-embeddings-for","slug":"biosentvec-creating-sentence-embeddings-for","title":"BioSentVec: creating sentence embeddings for biomedical texts","date":"2018-10-22","arxiv_id":"1810.09302","repositories_listed":4,"syntology":null},{"url":"/paper/signsgd-with-majority-vote-is-communication","slug":"signsgd-with-majority-vote-is-communication","title":"signSGD with Majority Vote is Communication Efficient And Fault Tolerant","date":"2018-10-11","arxiv_id":"1810.05291","repositories_listed":4,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":1,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/signsgd-with-majority-vote-is-communication#ran","syntology_url":"https://syntology.ai/paper/1810.05291","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1810.05291"}},"official":null}},{"url":"/paper/benchmarking-automatic-machine-learning","slug":"benchmarking-automatic-machine-learning","title":"Benchmarking Automatic Machine Learning Frameworks","date":"2018-08-17","arxiv_id":"1808.06492","repositories_listed":4,"syntology":null},{"url":"/paper/openml-benchmarking-suites-and-the-openml100","slug":"openml-benchmarking-suites-and-the-openml100","title":"OpenML Benchmarking Suites","date":"2017-08-11","arxiv_id":"1708.03731","repositories_listed":4,"syntology":null},{"url":"/paper/chartgalaxy-a-dataset-for-infographic-chart","slug":"chartgalaxy-a-dataset-for-infographic-chart","title":"ChartGalaxy: A Dataset for Infographic Chart Understanding and Generation","date":"2025-05-24","arxiv_id":"2505.18668","repositories_listed":3,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":0,"n_instrument":3,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 3 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/chartgalaxy-a-dataset-for-infographic-chart#ran","syntology_url":"https://syntology.ai/paper/2505.18668","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2505.18668"}},"official":{"repos":["chartgalaxy/chartgalaxy"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/omnigenbench-a-modular-platform-for","slug":"omnigenbench-a-modular-platform-for","title":"OmniGenBench: A Modular Platform for Reproducible Genomic Foundation Models Benchmarking","date":"2025-05-20","arxiv_id":"2505.14402","repositories_listed":3,"syntology":null},{"url":"/paper/the-ml-energy-benchmark-toward-automated","slug":"the-ml-energy-benchmark-toward-automated","title":"The ML.ENERGY Benchmark: Toward Automated Inference Energy Measurement and Optimization","date":"2025-05-09","arxiv_id":"2505.06371","repositories_listed":3,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/the-ml-energy-benchmark-toward-automated#ran","syntology_url":"https://syntology.ai/paper/2505.06371","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2505.06371"}},"official":{"repos":["ml-energy/leaderboard"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/lemur-neural-network-dataset-towards-seamless","slug":"lemur-neural-network-dataset-towards-seamless","title":"LEMUR Neural Network Dataset: Towards Seamless AutoML","date":"2025-04-14","arxiv_id":"2504.10552","repositories_listed":3,"syntology":null},{"url":"/paper/smplest-x-ultimate-scaling-for-expressive","slug":"smplest-x-ultimate-scaling-for-expressive","title":"SMPLest-X: Ultimate Scaling for Expressive Human Pose and Shape Estimation","date":"2025-01-16","arxiv_id":"2501.09782","repositories_listed":3,"syntology":null},{"url":"/paper/the-browsergym-ecosystem-for-web-agent","slug":"the-browsergym-ecosystem-for-web-agent","title":"The BrowserGym Ecosystem for Web Agent Research","date":"2024-12-06","arxiv_id":"2412.05467","repositories_listed":3,"syntology":{"n":24,"n_ran":17,"n_constructed":0,"n_ran_checked":7,"n_instrument":10,"n_unverified":7,"n_honours":1,"n_violates":1,"n_no_contract":5,"n_pointer_only":24,"phrase":"17 ran (of which 0 constructed an object rather than computing a result; 7 with no instrument failure: 1 honoured, 1 violated, 5 with no contract checked; 10 where Syntology's instrument failed) · 7 unverified","sample_list":"/paper/the-browsergym-ecosystem-for-web-agent#ran","syntology_url":"https://syntology.ai/paper/2412.05467","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2412.05467"}},"official":{"repos":["servicenow/agentlab","servicenow/browsergym"],"state":"official (archive's flag): 15 ran","n_ran":15,"n_constructed":0,"n_ran_no_instrument_failure":5,"n_unverified":2,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/embodied-agent-interface-benchmarking-llms","slug":"embodied-agent-interface-benchmarking-llms","title":"Embodied Agent Interface: Benchmarking LLMs for Embodied Decision Making","date":"2024-10-09","arxiv_id":"2410.07166","repositories_listed":3,"syntology":{"n":7,"n_ran":7,"n_constructed":0,"n_ran_checked":1,"n_instrument":6,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"7 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 6 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/embodied-agent-interface-benchmarking-llms#ran","syntology_url":"https://syntology.ai/paper/2410.07166","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2410.07166"}},"official":{"repos":["embodied-agent-eval/embodied-agent-eval","embodied-agent-interface/embodied-agent-interface","embodied-agent-eval/embodied-agent-eval.github.io"],"state":"official (archive's flag): 7 ran","n_ran":7,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/omnigenbench-automating-large-scale-in-silico","slug":"omnigenbench-automating-large-scale-in-silico","title":"OmniGenBench: Automating Large-scale in-silico Benchmarking for Genomic Foundation Models","date":"2024-10-02","arxiv_id":"2410.01784","repositories_listed":3,"syntology":null},{"url":"/paper/mathodyssey-benchmarking-mathematical-problem","slug":"mathodyssey-benchmarking-mathematical-problem","title":"MathOdyssey: Benchmarking Mathematical Problem-Solving Skills in Large Language Models Using Odyssey Math Data","date":"2024-06-26","arxiv_id":"2406.18321","repositories_listed":3,"syntology":{"n":5,"n_ran":5,"n_constructed":0,"n_ran_checked":4,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":4,"n_pointer_only":0,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 4 with no instrument failure: 0 honoured, 0 violated, 4 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/mathodyssey-benchmarking-mathematical-problem#ran","syntology_url":"https://syntology.ai/paper/2406.18321","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.18321"}},"official":null}},{"url":"/paper/benchmarking-uncertainty-quantification","slug":"benchmarking-uncertainty-quantification","title":"Benchmarking Uncertainty Quantification Methods for Large Language Models with LM-Polygraph","date":"2024-06-21","arxiv_id":"2406.15627","repositories_listed":3,"syntology":{"n":4,"n_ran":2,"n_constructed":0,"n_ran_checked":2,"n_instrument":0,"n_unverified":2,"n_honours":2,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 2 honoured, 0 violated, 0 with no contract checked; 0 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/benchmarking-uncertainty-quantification#ran","syntology_url":"https://syntology.ai/paper/2406.15627","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.15627"}},"official":{"repos":["iinemo/lm-polygraph"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/tgb-2-0-a-benchmark-for-learning-on-temporal","slug":"tgb-2-0-a-benchmark-for-learning-on-temporal","title":"TGB 2.0: A Benchmark for Learning on Temporal Knowledge Graphs and Heterogeneous Graphs","date":"2024-06-14","arxiv_id":"2406.09639","repositories_listed":3,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/tgb-2-0-a-benchmark-for-learning-on-temporal#ran","syntology_url":"https://syntology.ai/paper/2406.09639","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.09639"}},"official":{"repos":["erfanloghmani/myket-android-application-market-dataset","juliagast/tgb2","shenyanghuang/tgb"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/topobenchmarkx-a-framework-for-benchmarking","slug":"topobenchmarkx-a-framework-for-benchmarking","title":"TopoBench: A Framework for Benchmarking Topological Deep Learning","date":"2024-06-09","arxiv_id":"2406.06642","repositories_listed":3,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/topobenchmarkx-a-framework-for-benchmarking#ran","syntology_url":"https://syntology.ai/paper/2406.06642","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.06642"}},"official":{"repos":["geometric-intelligence/TopoBench","pyt-team/TopoBenchmarkX"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/mlvu-a-comprehensive-benchmark-for-multi-task","slug":"mlvu-a-comprehensive-benchmark-for-multi-task","title":"MLVU: Benchmarking Multi-task Long Video Understanding","date":"2024-06-06","arxiv_id":"2406.04264","repositories_listed":3,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":1,"n_instrument":2,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":3,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/mlvu-a-comprehensive-benchmark-for-multi-task#ran","syntology_url":"https://syntology.ai/paper/2406.04264","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.04264"}},"official":{"repos":["junjie99/mlvu","FlagOpen/FlagEmbedding"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official","unlocated"]}}},{"url":"/paper/an-empirical-study-into-clustering-of-unseen","slug":"an-empirical-study-into-clustering-of-unseen","title":"An Empirical Study into Clustering of Unseen Datasets with Self-Supervised Encoders","date":"2024-06-04","arxiv_id":"2406.02465","repositories_listed":3,"syntology":{"n":18,"n_ran":15,"n_constructed":0,"n_ran_checked":8,"n_instrument":7,"n_unverified":3,"n_honours":1,"n_violates":0,"n_no_contract":7,"n_pointer_only":0,"phrase":"15 ran (of which 0 constructed an object rather than computing a result; 8 with no instrument failure: 1 honoured, 0 violated, 7 with no contract checked; 7 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/an-empirical-study-into-clustering-of-unseen#ran","syntology_url":"https://syntology.ai/paper/2406.02465","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.02465"}},"official":{"repos":["scottclowe/zs-ssl-clustering"],"state":"official (archive's flag): 15 ran","n_ran":15,"n_constructed":0,"n_ran_no_instrument_failure":8,"n_unverified":3,"ran_from_kinds":["official"]}}},{"url":"/paper/are-eeg-to-text-models-working","slug":"are-eeg-to-text-models-working","title":"Are EEG-to-Text Models Working?","date":"2024-05-10","arxiv_id":"2405.06459","repositories_listed":3,"syntology":{"n":11,"n_ran":6,"n_constructed":0,"n_ran_checked":3,"n_instrument":3,"n_unverified":5,"n_honours":0,"n_violates":1,"n_no_contract":2,"n_pointer_only":11,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 1 violated, 2 with no contract checked; 3 where Syntology's instrument failed) · 5 unverified","sample_list":"/paper/are-eeg-to-text-models-working#ran","syntology_url":"https://syntology.ai/paper/2405.06459","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2405.06459"}},"official":{"repos":["mikewangwzhl/eeg-to-text","neuspeech/eeg-to-text"],"state":"official (archive's flag): 6 ran","n_ran":6,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":5,"ran_from_kinds":["official"]}}},{"url":"/paper/mmcode-evaluating-multi-modal-code-large","slug":"mmcode-evaluating-multi-modal-code-large","title":"MMCode: Benchmarking Multimodal Large Language Models for Code Generation with Visually Rich Programming Problems","date":"2024-04-15","arxiv_id":"2404.09486","repositories_listed":3,"syntology":{"n":16,"n_ran":13,"n_constructed":0,"n_ran_checked":3,"n_instrument":10,"n_unverified":3,"n_honours":1,"n_violates":0,"n_no_contract":2,"n_pointer_only":16,"phrase":"13 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 1 honoured, 0 violated, 2 with no contract checked; 10 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/mmcode-evaluating-multi-modal-code-large#ran","syntology_url":"https://syntology.ai/paper/2404.09486","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2404.09486"}},"official":{"repos":["happylkx/mmcode","likaixin2000/mmcode"],"state":"official (archive's flag): 13 ran","n_ran":13,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":3,"ran_from_kinds":["official"]}}},{"url":"/paper/nnu-net-revisited-a-call-for-rigorous","slug":"nnu-net-revisited-a-call-for-rigorous","title":"nnU-Net Revisited: A Call for Rigorous Validation in 3D Medical Image Segmentation","date":"2024-04-15","arxiv_id":"2404.09556","repositories_listed":3,"syntology":{"n":7,"n_ran":7,"n_constructed":0,"n_ran_checked":6,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":1,"n_no_contract":5,"n_pointer_only":2,"phrase":"7 ran (of which 0 constructed an object rather than computing a result; 6 with no instrument failure: 0 honoured, 1 violated, 5 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/nnu-net-revisited-a-call-for-rigorous#ran","syntology_url":"https://syntology.ai/paper/2404.09556","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2404.09556"}},"official":{"repos":["MIC-DKFZ/nnunet"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/are-large-language-models-superhuman-chemists","slug":"are-large-language-models-superhuman-chemists","title":"Are large language models superhuman chemists?","date":"2024-04-01","arxiv_id":"2404.01475","repositories_listed":3,"syntology":null},{"url":"/paper/tfb-towards-comprehensive-and-fair","slug":"tfb-towards-comprehensive-and-fair","title":"TFB: Towards Comprehensive and Fair Benchmarking of Time Series Forecasting Methods","date":"2024-03-29","arxiv_id":"2403.20150","repositories_listed":3,"syntology":{"n":4,"n_ran":4,"n_constructed":0,"n_ran_checked":1,"n_instrument":3,"n_unverified":0,"n_honours":0,"n_violates":1,"n_no_contract":0,"n_pointer_only":0,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 1 violated, 0 with no contract checked; 3 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/tfb-towards-comprehensive-and-fair#ran","syntology_url":"https://syntology.ai/paper/2403.20150","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2403.20150"}},"official":{"repos":["decisionintelligence/tfb"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}}],"syntology_records":68,"syntology_note":"a paper without a record is not a recorded non-run: it may lack an arXiv id or simply be absent from the graph layer"},"key_notes":{"n_ran_checked":"legacy name, kept unchanged so existing readers do not break: it counts the samples that ran with no instrument failure (honoured, violated, and ran with no contract checked); it does not mean a contract was checked, and the pages print it as 'K with no instrument failure', not 'K checked'","n_constructed":"a sub-count of the samples that ran, never subtracted from them and never a failure: an executed sample whose run returned an instance of its own class (fixture_out_type equals the entry name): the run built an object and did not compute a result (Syntology's RAN record, counts.constructed)"},"since_archive_judged":{"papers":0,"papers_placed":0,"judged_by":{"model":"a model","models_as_recorded":[],"from":"lb_rows_run_verified.jsonl: each record's `model` field (the extractor's call that wrote decisions[])"},"note":"a model's judgement from the paper's own tables that they report on this leaderboard, not a result: no value from a paper that was not placed is printed or carried here; papers counts each paper once over these leaderboards","backfill":"8,886 of 18,637 newer papers checked so far (733 more had nothing to place; 41 could not be read)","backfill_complete":false,"leaderboards":[]},"papers_where_code_ran":749,"papers_where_code_ran_first_page":"/task/benchmarking/papers/ran/1","papers_where_code_ran_caption":"We ran code from the paper's repository; we did not run it on this task or check it against the task's benchmarks.","pages":[{"order":"archive","page":2,"of":56,"url":"/task/benchmarking/papers/2","json":"/task/benchmarking/papers/2.json","rows":100},{"order":"archive","page":3,"of":56,"url":"/task/benchmarking/papers/3","json":"/task/benchmarking/papers/3.json","rows":100},{"order":"archive","page":4,"of":56,"url":"/task/benchmarking/papers/4","json":"/task/benchmarking/papers/4.json","rows":100},{"order":"archive","page":5,"of":56,"url":"/task/benchmarking/papers/5","json":"/task/benchmarking/papers/5.json","rows":100},{"order":"archive","page":6,"of":56,"url":"/task/benchmarking/papers/6","json":"/task/benchmarking/papers/6.json","rows":100},{"order":"archive","page":7,"of":56,"url":"/task/benchmarking/papers/7","json":"/task/benchmarking/papers/7.json","rows":100},{"order":"archive","page":8,"of":56,"url":"/task/benchmarking/papers/8","json":"/task/benchmarking/papers/8.json","rows":100},{"order":"archive","page":9,"of":56,"url":"/task/benchmarking/papers/9","json":"/task/benchmarking/papers/9.json","rows":100},{"order":"archive","page":10,"of":56,"url":"/task/benchmarking/papers/10","json":"/task/benchmarking/papers/10.json","rows":100},{"order":"archive","page":11,"of":56,"url":"/task/benchmarking/papers/11","json":"/task/benchmarking/papers/11.json","rows":100},{"order":"archive","page":12,"of":56,"url":"/task/benchmarking/papers/12","json":"/task/benchmarking/papers/12.json","rows":100},{"order":"archive","page":13,"of":56,"url":"/task/benchmarking/papers/13","json":"/task/benchmarking/papers/13.json","rows":100},{"order":"archive","page":14,"of":56,"url":"/task/benchmarking/papers/14","json":"/task/benchmarking/papers/14.json","rows":100},{"order":"archive","page":15,"of":56,"url":"/task/benchmarking/papers/15","json":"/task/benchmarking/papers/15.json","rows":100},{"order":"archive","page":16,"of":56,"url":"/task/benchmarking/papers/16","json":"/task/benchmarking/papers/16.json","rows":100},{"order":"archive","page":17,"of":56,"url":"/task/benchmarking/papers/17","json":"/task/benchmarking/papers/17.json","rows":100},{"order":"archive","page":18,"of":56,"url":"/task/benchmarking/papers/18","json":"/task/benchmarking/papers/18.json","rows":100},{"order":"archive","page":19,"of":56,"url":"/task/benchmarking/papers/19","json":"/task/benchmarking/papers/19.json","rows":100},{"order":"archive","page":20,"of":56,"url":"/task/benchmarking/papers/20","json":"/task/benchmarking/papers/20.json","rows":100},{"order":"archive","page":21,"of":56,"url":"/task/benchmarking/papers/21","json":"/task/benchmarking/papers/21.json","rows":100},{"order":"archive","page":22,"of":56,"url":"/task/benchmarking/papers/22","json":"/task/benchmarking/papers/22.json","rows":100},{"order":"archive","page":23,"of":56,"url":"/task/benchmarking/papers/23","json":"/task/benchmarking/papers/23.json","rows":100},{"order":"archive","page":24,"of":56,"url":"/task/benchmarking/papers/24","json":"/task/benchmarking/papers/24.json","rows":100},{"order":"archive","page":25,"of":56,"url":"/task/benchmarking/papers/25","json":"/task/benchmarking/papers/25.json","rows":100},{"order":"archive","page":26,"of":56,"url":"/task/benchmarking/papers/26","json":"/task/benchmarking/papers/26.json","rows":100},{"order":"archive","page":27,"of":56,"url":"/task/benchmarking/papers/27","json":"/task/benchmarking/papers/27.json","rows":100},{"order":"archive","page":28,"of":56,"url":"/task/benchmarking/papers/28","json":"/task/benchmarking/papers/28.json","rows":100},{"order":"archive","page":29,"of":56,"url":"/task/benchmarking/papers/29","json":"/task/benchmarking/papers/29.json","rows":100},{"order":"archive","page":30,"of":56,"url":"/task/benchmarking/papers/30","json":"/task/benchmarking/papers/30.json","rows":100},{"order":"archive","page":31,"of":56,"url":"/task/benchmarking/papers/31","json":"/task/benchmarking/papers/31.json","rows":100},{"order":"archive","page":32,"of":56,"url":"/task/benchmarking/papers/32","json":"/task/benchmarking/papers/32.json","rows":100},{"order":"archive","page":33,"of":56,"url":"/task/benchmarking/papers/33","json":"/task/benchmarking/papers/33.json","rows":100},{"order":"archive","page":34,"of":56,"url":"/task/benchmarking/papers/34","json":"/task/benchmarking/papers/34.json","rows":100},{"order":"archive","page":35,"of":56,"url":"/task/benchmarking/papers/35","json":"/task/benchmarking/papers/35.json","rows":100},{"order":"archive","page":36,"of":56,"url":"/task/benchmarking/papers/36","json":"/task/benchmarking/papers/36.json","rows":100},{"order":"archive","page":37,"of":56,"url":"/task/benchmarking/papers/37","json":"/task/benchmarking/papers/37.json","rows":100},{"order":"archive","page":38,"of":56,"url":"/task/benchmarking/papers/38","json":"/task/benchmarking/papers/38.json","rows":100},{"order":"archive","page":39,"of":56,"url":"/task/benchmarking/papers/39","json":"/task/benchmarking/papers/39.json","rows":100},{"order":"archive","page":40,"of":56,"url":"/task/benchmarking/papers/40","json":"/task/benchmarking/papers/40.json","rows":100},{"order":"archive","page":41,"of":56,"url":"/task/benchmarking/papers/41","json":"/task/benchmarking/papers/41.json","rows":100},{"order":"archive","page":42,"of":56,"url":"/task/benchmarking/papers/42","json":"/task/benchmarking/papers/42.json","rows":100},{"order":"archive","page":43,"of":56,"url":"/task/benchmarking/papers/43","json":"/task/benchmarking/papers/43.json","rows":100},{"order":"archive","page":44,"of":56,"url":"/task/benchmarking/papers/44","json":"/task/benchmarking/papers/44.json","rows":100},{"order":"archive","page":45,"of":56,"url":"/task/benchmarking/papers/45","json":"/task/benchmarking/papers/45.json","rows":100},{"order":"archive","page":46,"of":56,"url":"/task/benchmarking/papers/46","json":"/task/benchmarking/papers/46.json","rows":100},{"order":"archive","page":47,"of":56,"url":"/task/benchmarking/papers/47","json":"/task/benchmarking/papers/47.json","rows":100},{"order":"archive","page":48,"of":56,"url":"/task/benchmarking/papers/48","json":"/task/benchmarking/papers/48.json","rows":100},{"order":"archive","page":49,"of":56,"url":"/task/benchmarking/papers/49","json":"/task/benchmarking/papers/49.json","rows":100},{"order":"archive","page":50,"of":56,"url":"/task/benchmarking/papers/50","json":"/task/benchmarking/papers/50.json","rows":100},{"order":"archive","page":51,"of":56,"url":"/task/benchmarking/papers/51","json":"/task/benchmarking/papers/51.json","rows":100},{"order":"archive","page":52,"of":56,"url":"/task/benchmarking/papers/52","json":"/task/benchmarking/papers/52.json","rows":100},{"order":"archive","page":53,"of":56,"url":"/task/benchmarking/papers/53","json":"/task/benchmarking/papers/53.json","rows":100},{"order":"archive","page":54,"of":56,"url":"/task/benchmarking/papers/54","json":"/task/benchmarking/papers/54.json","rows":100},{"order":"archive","page":55,"of":56,"url":"/task/benchmarking/papers/55","json":"/task/benchmarking/papers/55.json","rows":100},{"order":"archive","page":56,"of":56,"url":"/task/benchmarking/papers/56","json":"/task/benchmarking/papers/56.json","rows":48},{"order":"ran","page":1,"of":8,"url":"/task/benchmarking/papers/ran/1","json":"/task/benchmarking/papers/ran/1.json","rows":100},{"order":"ran","page":2,"of":8,"url":"/task/benchmarking/papers/ran/2","json":"/task/benchmarking/papers/ran/2.json","rows":100},{"order":"ran","page":3,"of":8,"url":"/task/benchmarking/papers/ran/3","json":"/task/benchmarking/papers/ran/3.json","rows":100},{"order":"ran","page":4,"of":8,"url":"/task/benchmarking/papers/ran/4","json":"/task/benchmarking/papers/ran/4.json","rows":100},{"order":"ran","page":5,"of":8,"url":"/task/benchmarking/papers/ran/5","json":"/task/benchmarking/papers/ran/5.json","rows":100},{"order":"ran","page":6,"of":8,"url":"/task/benchmarking/papers/ran/6","json":"/task/benchmarking/papers/ran/6.json","rows":100},{"order":"ran","page":7,"of":8,"url":"/task/benchmarking/papers/ran/7","json":"/task/benchmarking/papers/ran/7.json","rows":100},{"order":"ran","page":8,"of":8,"url":"/task/benchmarking/papers/ran/8","json":"/task/benchmarking/papers/ran/8.json","rows":49}],"description_links":{"kept":0,"unwrapped_to_text":0,"bare_urls_linked":0,"relative_images_dropped":0,"rule":"internal links are kept only when the target slug exists in the catalog"},"syntology":{"read_at":"2026-10-05T00:36:11+00:00","claim":"Per-sample execution status on synthesized fixtures ('ran N of M samples'); not a correctness claim and not a ranking signal.","status_vocabulary":{"ran_honours":"ran, honoured the contract we drafted","ran_violates":"ran, violated the contract we drafted","ran_draft_wrong":"ran; our contract draft was wrong, not the code","ran_fixture":"ran; the contract check could not run (RAISES, DEP_MISSING or TIMEOUT)","ran":"ran on a synthesized input","unverified":"unverified (harvested, no recorded run)"},"row_route":"each syntology object on a benchmark's first row and on a paper carries sample_list (this site's paper page, #ran), syntology_url (syntology.ai's page for the paper, only when that route probed live) and mcp (the agent call that lists the paper's harvested samples; not a verdict on the paper)","developers":"https://syntology.ai/developers"},"not_shown":{"libraries":"the archive has no per-task library table","trend_sparklines":"the Trend column of the benchmarks table was a rendered image; it is not in the archive","social_and_latest_sorts":"stars and social signals are not in the archive"},"record_sha256":"ef288a4795cb39204589f7f33d5f0738edea2a7329f48c63e7a942b93ce69b90","record_changed_at":"2026-10-05","record_changed_at_basis":"changed"}