{"about":{"non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","site":"https://codewithpapers.app","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page","syntology":{"site":"https://syntology.ai","developers":"https://syntology.ai/developers","mcp":{"server":"https://syntology.ai/mcp","transport":"streamable-http","server_card":"https://syntology.ai/.well-known/mcp/server-card.json","auth":{"type":"trial token, no account","trial_token":"https://syntology.ai/api/oauth/trial/token","method":"POST","docs":"https://syntology.ai/developers"}},"have":"https://syntology.ai/api/graph/have?x=<method, arXiv id or title> (free, answers coverage only)","paper_base":"https://syntology.ai/paper/","atlas_base":"https://app.syntology.ai/?focus="},"machine_readable":[{"url":"https://codewithpapers.app/llms.txt","what":"the machine catalog: every machine-readable file, counted"},{"url":"https://codewithpapers.app/index/manifest.json","what":"paper-to-code index by arXiv id, with Syntology's counts"},{"url":"https://codewithpapers.app/search/manifest.json","what":"site search index (titles, authors) and its files"},{"url":"https://codewithpapers.app/download","what":"bulk files: Syntology's layer, described there"},{"url":"https://codewithpapers.app/build_manifest.json","what":"the build record: inputs, counts, exclusions, probes"}]},"url":"/task/multiple-choice-qa","name":"Multiple Choice Question Answering (MCQA)","slug":"multiple-choice-qa","description_markdown":"A multiple-choice question (MCQ) is composed of two parts: a stem that identifies the question or problem, and a set of alternatives or possible answers that contain a key that is the best answer to the question, and a number of distractors that are plausible but incorrect answers to the question.\r\n\r\nIn a k-way MCQA task, a model is provided with a question q, a set of candidate options O = {O1, . . . , Ok}, and a supporting context for each option C = {C1, . . . , Ck}. The model needs to predict the correct answer option that is best supported by the given contexts.","categories":[{"name":"Miscellaneous","url":"/area/miscellaneous"},{"name":"Natural Language Processing","url":"/area/natural-language-processing"},{"name":"Reasoning","url":"/area/reasoning"}],"source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","slug_source":"archive_url"},"counts":{"papers_tagged":65,"papers_with_code":37,"benchmarks":31,"benchmark_tables_in_archive":31,"benchmark_tables_shown":31,"benchmark_tables_withheld_as_spam":0,"benchmark_definition":"a leaderboard table with at least one row; benchmark_tables_shown also counts the zero-row tables; benchmark_tables_in_archive adds the tables withheld as spam","datasets":9,"subtasks":0,"parent_tasks":1},"benchmarks":[{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-21","slug":"multiple-choice-question-answering-mcqa-on-21","dataset":"MedMCQA","dataset_url":"/dataset/medmcqa","rows_in_archive":22,"metrics":["Test Set (Acc-%)","Dev Set (Acc-%)"],"first_row_in_archive_order":{"model":"Med-PaLM 2 (ER)","paper_title":"Towards Expert-Level Medical Question Answering with Large Language Models","paper_url":"/paper/towards-expert-level-medical-question","paper_date":"2023-05-16","arxiv_id":"2305.09617","code_links":[{"title":"m42-health/med42","url":"https://github.com/m42-health/med42"}],"syntology":null}},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-27","slug":"multiple-choice-question-answering-mcqa-on-27","dataset":"BIG-bench (Hyperbaton)","dataset_url":"/dataset/big-bench","rows_in_archive":9,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"Bloomberg GPT (few-shot, k=3)","paper_title":"BloombergGPT: A Large Language Model for Finance","paper_url":"/paper/bloomberggpt-a-large-language-model-for","paper_date":"2023-03-30","arxiv_id":"2303.17564","code_links":[{"title":"yangletliu/finlora","url":"https://github.com/yangletliu/finlora"},{"title":"open-finance-lab/finlora","url":"https://github.com/open-finance-lab/finlora"}],"syntology":null}},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-28","slug":"multiple-choice-question-answering-mcqa-on-28","dataset":"BIG-bench (Movie Recommendation)","dataset_url":"/dataset/big-bench","rows_in_archive":9,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"PaLM 2 (few-shot, k=3, CoT)","paper_title":"PaLM 2 Technical Report","paper_url":"/paper/palm-2-technical-report-1","paper_date":"2023-05-17","arxiv_id":"2305.10403","code_links":[{"title":"eternityyw/tram-benchmark","url":"https://github.com/eternityyw/tram-benchmark"}],"syntology":null}},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-29","slug":"multiple-choice-question-answering-mcqa-on-29","dataset":"BIG-bench (Navigate)","dataset_url":"/dataset/big-bench","rows_in_archive":9,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"PaLM 2 (few-shot, k=3, CoT)","paper_title":"PaLM 2 Technical Report","paper_url":"/paper/palm-2-technical-report-1","paper_date":"2023-05-17","arxiv_id":"2305.10403","code_links":[{"title":"eternityyw/tram-benchmark","url":"https://github.com/eternityyw/tram-benchmark"}],"syntology":null}},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-30","slug":"multiple-choice-question-answering-mcqa-on-30","dataset":"BIG-bench (Ruin Names)","dataset_url":"/dataset/big-bench","rows_in_archive":9,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"PaLM 2 (few-shot, k=3, Direct)","paper_title":"PaLM 2 Technical Report","paper_url":"/paper/palm-2-technical-report-1","paper_date":"2023-05-17","arxiv_id":"2305.10403","code_links":[{"title":"eternityyw/tram-benchmark","url":"https://github.com/eternityyw/tram-benchmark"}],"syntology":null}},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-11","slug":"multiple-choice-question-answering-mcqa-on-11","dataset":"MMLU (College Biology)","dataset_url":"/dataset/mmlu","rows_in_archive":8,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"Med-PaLM 2 (ER)","paper_title":"Towards Expert-Level Medical Question Answering with Large Language Models","paper_url":"/paper/towards-expert-level-medical-question","paper_date":"2023-05-16","arxiv_id":"2305.09617","code_links":[{"title":"m42-health/med42","url":"https://github.com/m42-health/med42"}],"syntology":null}},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-8","slug":"multiple-choice-question-answering-mcqa-on-8","dataset":"MMLU (Medical Genetics)","dataset_url":"/dataset/mmlu","rows_in_archive":8,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"Med-PaLM 2 (ER)","paper_title":"Towards Expert-Level Medical Question Answering with Large Language Models","paper_url":"/paper/towards-expert-level-medical-question","paper_date":"2023-05-16","arxiv_id":"2305.09617","code_links":[{"title":"m42-health/med42","url":"https://github.com/m42-health/med42"}],"syntology":null}},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-25","slug":"multiple-choice-question-answering-mcqa-on-25","dataset":"MMLU (Professional medicine)","dataset_url":"/dataset/mmlu","rows_in_archive":6,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"Med-PaLM 2 (5-shot)","paper_title":"Towards Expert-Level Medical Question Answering with Large Language Models","paper_url":"/paper/towards-expert-level-medical-question","paper_date":"2023-05-16","arxiv_id":"2305.09617","code_links":[{"title":"m42-health/med42","url":"https://github.com/m42-health/med42"}],"syntology":null}},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-10","slug":"multiple-choice-question-answering-mcqa-on-10","dataset":"MMLU (Elementary Mathematics)","dataset_url":"/dataset/mmlu","rows_in_archive":5,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"Chinchilla (few-shot, k=5)","paper_title":"Galactica: A Large Language Model for Science","paper_url":"/paper/galactica-a-large-language-model-for-science-1","paper_date":"2022-11-16","arxiv_id":"2211.09085","code_links":[{"title":"paperswithcode/galai","url":"https://github.com/paperswithcode/galai"}],"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":2,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/galactica-a-large-language-model-for-science-1#ran","syntology_url":"https://syntology.ai/paper/2211.09085","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2211.09085"}},"official":{"repos":["paperswithcode/galai"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["official"]}}}},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-12","slug":"multiple-choice-question-answering-mcqa-on-12","dataset":"MMLU (High School Biology)","dataset_url":"/dataset/mmlu","rows_in_archive":5,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"Chinchilla (few-shot, k=5)","paper_title":"Galactica: A Large Language Model for Science","paper_url":"/paper/galactica-a-large-language-model-for-science-1","paper_date":"2022-11-16","arxiv_id":"2211.09085","code_links":[{"title":"paperswithcode/galai","url":"https://github.com/paperswithcode/galai"}],"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":2,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/galactica-a-large-language-model-for-science-1#ran","syntology_url":"https://syntology.ai/paper/2211.09085","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2211.09085"}},"official":{"repos":["paperswithcode/galai"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["official"]}}}},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-13","slug":"multiple-choice-question-answering-mcqa-on-13","dataset":"MMLU (College Chemistry)","dataset_url":"/dataset/mmlu","rows_in_archive":5,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"Chinchilla (few-shot, k=5)","paper_title":"Galactica: A Large Language Model for Science","paper_url":"/paper/galactica-a-large-language-model-for-science-1","paper_date":"2022-11-16","arxiv_id":"2211.09085","code_links":[{"title":"paperswithcode/galai","url":"https://github.com/paperswithcode/galai"}],"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":2,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/galactica-a-large-language-model-for-science-1#ran","syntology_url":"https://syntology.ai/paper/2211.09085","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2211.09085"}},"official":{"repos":["paperswithcode/galai"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["official"]}}}},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-16","slug":"multiple-choice-question-answering-mcqa-on-16","dataset":"MMLU (High School Mathematics)","dataset_url":"/dataset/mmlu","rows_in_archive":5,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"GAL 120B (zero-shot)","paper_title":"Galactica: A Large Language Model for Science","paper_url":"/paper/galactica-a-large-language-model-for-science-1","paper_date":"2022-11-16","arxiv_id":"2211.09085","code_links":[{"title":"paperswithcode/galai","url":"https://github.com/paperswithcode/galai"}],"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":2,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/galactica-a-large-language-model-for-science-1#ran","syntology_url":"https://syntology.ai/paper/2211.09085","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2211.09085"}},"official":{"repos":["paperswithcode/galai"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["official"]}}}},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-17","slug":"multiple-choice-question-answering-mcqa-on-17","dataset":"MMLU (Electrical Engineer)","dataset_url":"/dataset/mmlu","rows_in_archive":5,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"GAL 120B (zero-shot)","paper_title":"Galactica: A Large Language Model for Science","paper_url":"/paper/galactica-a-large-language-model-for-science-1","paper_date":"2022-11-16","arxiv_id":"2211.09085","code_links":[{"title":"paperswithcode/galai","url":"https://github.com/paperswithcode/galai"}],"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":2,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/galactica-a-large-language-model-for-science-1#ran","syntology_url":"https://syntology.ai/paper/2211.09085","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2211.09085"}},"official":{"repos":["paperswithcode/galai"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["official"]}}}},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-18","slug":"multiple-choice-question-answering-mcqa-on-18","dataset":"MMLU (College Physics)","dataset_url":"/dataset/mmlu","rows_in_archive":5,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"Chinchilla (few-shot, k=5)","paper_title":"Galactica: A Large Language Model for Science","paper_url":"/paper/galactica-a-large-language-model-for-science-1","paper_date":"2022-11-16","arxiv_id":"2211.09085","code_links":[{"title":"paperswithcode/galai","url":"https://github.com/paperswithcode/galai"}],"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":2,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/galactica-a-large-language-model-for-science-1#ran","syntology_url":"https://syntology.ai/paper/2211.09085","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2211.09085"}},"official":{"repos":["paperswithcode/galai"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["official"]}}}},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-2","slug":"multiple-choice-question-answering-mcqa-on-2","dataset":"MMLU (Formal Logic)","dataset_url":"/dataset/mmlu","rows_in_archive":5,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"Gopher (few-shot, k=5)","paper_title":"Galactica: A Large Language Model for Science","paper_url":"/paper/galactica-a-large-language-model-for-science-1","paper_date":"2022-11-16","arxiv_id":"2211.09085","code_links":[{"title":"paperswithcode/galai","url":"https://github.com/paperswithcode/galai"}],"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":2,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/galactica-a-large-language-model-for-science-1#ran","syntology_url":"https://syntology.ai/paper/2211.09085","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2211.09085"}},"official":{"repos":["paperswithcode/galai"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["official"]}}}},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-20","slug":"multiple-choice-question-answering-mcqa-on-20","dataset":"MMLU (High School Statistics)","dataset_url":"/dataset/mmlu","rows_in_archive":5,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"Chinchilla (few-shot, k=5)","paper_title":"Galactica: A Large Language Model for Science","paper_url":"/paper/galactica-a-large-language-model-for-science-1","paper_date":"2022-11-16","arxiv_id":"2211.09085","code_links":[{"title":"paperswithcode/galai","url":"https://github.com/paperswithcode/galai"}],"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":2,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/galactica-a-large-language-model-for-science-1#ran","syntology_url":"https://syntology.ai/paper/2211.09085","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2211.09085"}},"official":{"repos":["paperswithcode/galai"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["official"]}}}},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-3","slug":"multiple-choice-question-answering-mcqa-on-3","dataset":"MMLU (Abstract Algebra)","dataset_url":"/dataset/mmlu","rows_in_archive":5,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"GAL 30B (zero-shot)","paper_title":"Galactica: A Large Language Model for Science","paper_url":"/paper/galactica-a-large-language-model-for-science-1","paper_date":"2022-11-16","arxiv_id":"2211.09085","code_links":[{"title":"paperswithcode/galai","url":"https://github.com/paperswithcode/galai"}],"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":2,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/galactica-a-large-language-model-for-science-1#ran","syntology_url":"https://syntology.ai/paper/2211.09085","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2211.09085"}},"official":{"repos":["paperswithcode/galai"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["official"]}}}},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-4","slug":"multiple-choice-question-answering-mcqa-on-4","dataset":"MMLU (Econometrics)","dataset_url":"/dataset/mmlu","rows_in_archive":5,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"Gopher (few-shot, k=5)","paper_title":"Galactica: A Large Language Model for Science","paper_url":"/paper/galactica-a-large-language-model-for-science-1","paper_date":"2022-11-16","arxiv_id":"2211.09085","code_links":[{"title":"paperswithcode/galai","url":"https://github.com/paperswithcode/galai"}],"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":2,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/galactica-a-large-language-model-for-science-1#ran","syntology_url":"https://syntology.ai/paper/2211.09085","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2211.09085"}},"official":{"repos":["paperswithcode/galai"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["official"]}}}},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-5","slug":"multiple-choice-question-answering-mcqa-on-5","dataset":"MMLU (High School Computer Science)","dataset_url":"/dataset/mmlu","rows_in_archive":5,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"GAL 120B (zero-shot)","paper_title":"Galactica: A Large Language Model for Science","paper_url":"/paper/galactica-a-large-language-model-for-science-1","paper_date":"2022-11-16","arxiv_id":"2211.09085","code_links":[{"title":"paperswithcode/galai","url":"https://github.com/paperswithcode/galai"}],"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":2,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/galactica-a-large-language-model-for-science-1#ran","syntology_url":"https://syntology.ai/paper/2211.09085","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2211.09085"}},"official":{"repos":["paperswithcode/galai"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["official"]}}}},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-7","slug":"multiple-choice-question-answering-mcqa-on-7","dataset":"MMLU (College Mathematics)","dataset_url":"/dataset/mmlu","rows_in_archive":5,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"GAL 120B (zero-shot)","paper_title":"Galactica: A Large Language Model for Science","paper_url":"/paper/galactica-a-large-language-model-for-science-1","paper_date":"2022-11-16","arxiv_id":"2211.09085","code_links":[{"title":"paperswithcode/galai","url":"https://github.com/paperswithcode/galai"}],"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":2,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/galactica-a-large-language-model-for-science-1#ran","syntology_url":"https://syntology.ai/paper/2211.09085","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2211.09085"}},"official":{"repos":["paperswithcode/galai"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["official"]}}}},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-9","slug":"multiple-choice-question-answering-mcqa-on-9","dataset":"MMLU (Astronomy)","dataset_url":"/dataset/mmlu","rows_in_archive":5,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"Chinchilla (few-shot, k=5)","paper_title":"Galactica: A Large Language Model for Science","paper_url":"/paper/galactica-a-large-language-model-for-science-1","paper_date":"2022-11-16","arxiv_id":"2211.09085","code_links":[{"title":"paperswithcode/galai","url":"https://github.com/paperswithcode/galai"}],"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":2,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/galactica-a-large-language-model-for-science-1#ran","syntology_url":"https://syntology.ai/paper/2211.09085","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2211.09085"}},"official":{"repos":["paperswithcode/galai"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["official"]}}}},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-14","slug":"multiple-choice-question-answering-mcqa-on-14","dataset":"MMLU (High School Chemistry)","dataset_url":"/dataset/mmlu","rows_in_archive":4,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"Chinchilla (few-shot, k=5)","paper_title":"Galactica: A Large Language Model for Science","paper_url":"/paper/galactica-a-large-language-model-for-science-1","paper_date":"2022-11-16","arxiv_id":"2211.09085","code_links":[{"title":"paperswithcode/galai","url":"https://github.com/paperswithcode/galai"}],"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":2,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/galactica-a-large-language-model-for-science-1#ran","syntology_url":"https://syntology.ai/paper/2211.09085","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2211.09085"}},"official":{"repos":["paperswithcode/galai"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["official"]}}}},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-15","slug":"multiple-choice-question-answering-mcqa-on-15","dataset":"MMLU (College Computer Science)","dataset_url":"/dataset/mmlu","rows_in_archive":4,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"Chinchilla (few-shot, k=5)","paper_title":"Galactica: A Large Language Model for Science","paper_url":"/paper/galactica-a-large-language-model-for-science-1","paper_date":"2022-11-16","arxiv_id":"2211.09085","code_links":[{"title":"paperswithcode/galai","url":"https://github.com/paperswithcode/galai"}],"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":2,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/galactica-a-large-language-model-for-science-1#ran","syntology_url":"https://syntology.ai/paper/2211.09085","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2211.09085"}},"official":{"repos":["paperswithcode/galai"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["official"]}}}},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-19","slug":"multiple-choice-question-answering-mcqa-on-19","dataset":"MMLU (High School Physics)","dataset_url":"/dataset/mmlu","rows_in_archive":4,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"Chinchilla (few-shot, k=5)","paper_title":"Galactica: A Large Language Model for Science","paper_url":"/paper/galactica-a-large-language-model-for-science-1","paper_date":"2022-11-16","arxiv_id":"2211.09085","code_links":[{"title":"paperswithcode/galai","url":"https://github.com/paperswithcode/galai"}],"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":2,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/galactica-a-large-language-model-for-science-1#ran","syntology_url":"https://syntology.ai/paper/2211.09085","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2211.09085"}},"official":{"repos":["paperswithcode/galai"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["official"]}}}},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-31","slug":"multiple-choice-question-answering-mcqa-on-31","dataset":"BIG-bench (Novel Concepts)","dataset_url":"/dataset/big-bench","rows_in_archive":4,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"PaLM-540B (few-shot, k=5)","paper_title":"PaLM: Scaling Language Modeling with Pathways","paper_url":"/paper/palm-scaling-language-modeling-with-pathways-1","paper_date":"2022-04-05","arxiv_id":"2204.02311","code_links":[{"title":"lucidrains/CoCa-pytorch","url":"https://github.com/lucidrains/CoCa-pytorch"},{"title":"lucidrains/PaLM-pytorch","url":"https://github.com/lucidrains/PaLM-pytorch"},{"title":"google/paxml","url":"https://github.com/google/paxml"},{"title":"foundation-model-stack/fms-fsdp","url":"https://github.com/foundation-model-stack/fms-fsdp"},{"title":"lucidrains/PaLM-jax","url":"https://github.com/lucidrains/PaLM-jax"},{"title":"chrisociepa/allamo","url":"https://github.com/chrisociepa/allamo"},{"title":"conceptofmind/PaLM-flax","url":"https://github.com/conceptofmind/PaLM-flax"}],"syntology":{"n":37,"n_ran":32,"n_constructed":16,"n_ran_checked":24,"n_instrument":8,"n_unverified":5,"n_honours":2,"n_violates":1,"n_no_contract":21,"n_pointer_only":0,"phrase":"32 ran (of which 16 constructed an object rather than computing a result; 24 with no instrument failure: 2 honoured, 1 violated, 21 with no contract checked; 8 where Syntology's instrument failed) · 5 unverified","sample_list":"/paper/palm-scaling-language-modeling-with-pathways-1#ran","syntology_url":"https://syntology.ai/paper/2204.02311","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2204.02311"}},"official":null}}},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-6","slug":"multiple-choice-question-answering-mcqa-on-6","dataset":"MMLU (Machine Learning)","dataset_url":"/dataset/mmlu","rows_in_archive":4,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"Chinchilla (few-shot, k=5)","paper_title":"Galactica: A Large Language Model for Science","paper_url":"/paper/galactica-a-large-language-model-for-science-1","paper_date":"2022-11-16","arxiv_id":"2211.09085","code_links":[{"title":"paperswithcode/galai","url":"https://github.com/paperswithcode/galai"}],"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":2,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/galactica-a-large-language-model-for-science-1#ran","syntology_url":"https://syntology.ai/paper/2211.09085","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2211.09085"}},"official":{"repos":["paperswithcode/galai"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["official"]}}}},{"leaderboard":"/sota/multiple-choice-qa-on-indicglue-wstp-pa","slug":"multiple-choice-qa-on-indicglue-wstp-pa","dataset":"IndicGLUE WSTP Pa","dataset_url":"/dataset/indicglue","rows_in_archive":3,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"xlmindic-base-uniscript","paper_title":"Does Transliteration Help Multilingual Language Modeling?","paper_url":"/paper/does-transliteration-help-multilingual","paper_date":"2022-01-29","arxiv_id":"2201.12501","code_links":[{"title":"ibraheem-moosa/xlm-indic","url":"https://github.com/ibraheem-moosa/xlm-indic"}],"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/does-transliteration-help-multilingual#ran","syntology_url":"https://syntology.ai/paper/2201.12501","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2201.12501"}},"official":{"repos":["ibraheem-moosa/xlm-indic"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}}},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-23","slug":"multiple-choice-question-answering-mcqa-on-23","dataset":"MMLU (Clinical Knowledge)","dataset_url":"/dataset/mmlu","rows_in_archive":3,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"Med-PaLM 2 (ER)","paper_title":"Towards Expert-Level Medical Question Answering with Large Language Models","paper_url":"/paper/towards-expert-level-medical-question","paper_date":"2023-05-16","arxiv_id":"2305.09617","code_links":[{"title":"m42-health/med42","url":"https://github.com/m42-health/med42"}],"syntology":null}},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-24","slug":"multiple-choice-question-answering-mcqa-on-24","dataset":"MMLU (Anatomy)","dataset_url":"/dataset/mmlu","rows_in_archive":3,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"Med-PaLM 2 (ER)","paper_title":"Towards Expert-Level Medical Question Answering with Large Language Models","paper_url":"/paper/towards-expert-level-medical-question","paper_date":"2023-05-16","arxiv_id":"2305.09617","code_links":[{"title":"m42-health/med42","url":"https://github.com/m42-health/med42"}],"syntology":null}},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-26","slug":"multiple-choice-question-answering-mcqa-on-26","dataset":"MMLU (College Medicine)","dataset_url":"/dataset/mmlu","rows_in_archive":3,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"Med-PaLM (ER)","paper_title":"Towards Expert-Level Medical Question Answering with Large Language Models","paper_url":"/paper/towards-expert-level-medical-question","paper_date":"2023-05-16","arxiv_id":"2305.09617","code_links":[{"title":"m42-health/med42","url":"https://github.com/m42-health/med42"}],"syntology":null}},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-22","slug":"multiple-choice-question-answering-mcqa-on-22","dataset":"FrenchMedMCQA","dataset_url":"/dataset/frenchmedmcqa","rows_in_archive":2,"metrics":["Exact Match Accuracy","Hamming Score"],"first_row_in_archive_order":{"model":"CamemBERT","paper_title":"FrenchMedMCQA: A French Multiple-Choice Question Answering Dataset for Medical domain","paper_url":"/paper/frenchmedmcqa-a-french-multiple-choice-1","paper_date":"2023-04-09","arxiv_id":"2304.04280","code_links":[{"title":"qanastek/FrenchMedMCQA","url":"https://github.com/qanastek/FrenchMedMCQA"}],"syntology":null}}],"datasets":[{"url":"/dataset/mmlu","name":"MML","full_name":"Massive Multitask Language Understanding","num_papers_in_archive":1922},{"url":"/dataset/big-bench","name":"BIG-bench","full_name":"Beyond the Imitation Game Benchmark","num_papers_in_archive":349},{"url":"/dataset/medmcqa","name":"MedMCQA","full_name":"","num_papers_in_archive":144},{"url":"/dataset/indicglue","name":"IndicGLUE","full_name":"Indic General Language Understanding Evaluation Benchmark","num_papers_in_archive":16},{"url":"/dataset/m3ke","name":"M3KE","full_name":"Massive Multi-Level Multi-Subject Knowledge Evaluation Benchmark","num_papers_in_archive":14},{"url":"/dataset/frenchmedmcqa","name":"FrenchMedMCQA","full_name":"FrenchMedMCQA: A French Multiple-Choice Question Answering Dataset for Medical domain","num_papers_in_archive":6},{"url":"/dataset/secqa","name":"SecQA","full_name":"","num_papers_in_archive":4},{"url":"/dataset/sfd","name":"SF20K","full_name":"Short-Films 20K","num_papers_in_archive":1},{"url":"/dataset/lmcqa","name":"LMCQA","full_name":"Legal Multiple Choice Question Answering","num_papers_in_archive":0}],"subtasks":[],"parent_tasks":[{"url":"/task/question-answering","name":"Question Answering"}],"papers":{"order":"repositories listed in the archive (most first), then date (newest first), then slug; the archive holds no stars","population":"every paper the archive tags with this task (distinct papers; spam titles not listed). 'of' counts the papers listed; until 2026-09-25 it counted only the papers with a code link, and that number is of_with_code","page":1,"pages_in_order":1,"rows_per_page":100,"shown":65,"of":65,"of_with_code":37,"tagged_in_all":65,"counts":{"archive_papers_tagged":65,"with_a_code_link":37,"where_syntology_ran_a_sample":16,"not_listed_spam_title":0,"listed":65,"listed_where_code_ran":16,"where_syntology_ran_a_sample_split":{"with_a_run_with_no_instrument_failure":13,"every_run_a_failure_of_syntologys_instrument":3,"listed_with_a_run_with_no_instrument_failure":13,"listed_every_run_a_failure_of_syntologys_instrument":3,"filter":{"states":["a run with no instrument failure","any run, instrument failures included"],"default":"a run with no instrument failure","note":"on the 'only where code ran' pages the default hides, in the browser, the rows where every run was a failure of Syntology's instrument; the second state shows them again. Rows are hidden, never re-ordered; these twins list every row"}},"definition":"distinct papers the archive tags; 'where Syntology ran a sample' counts papers with at least one harvested sample that ran, which is not a correctness claim"},"items":[{"url":"/paper/llama-2-open-foundation-and-fine-tuned-chat","slug":"llama-2-open-foundation-and-fine-tuned-chat","title":"Llama 2: Open Foundation and Fine-Tuned Chat Models","date":"2023-07-18","arxiv_id":"2307.09288","repositories_listed":19,"syntology":{"n":52,"n_ran":33,"n_constructed":7,"n_ran_checked":20,"n_instrument":13,"n_unverified":19,"n_honours":1,"n_violates":1,"n_no_contract":18,"n_pointer_only":20,"phrase":"33 ran (of which 7 constructed an object rather than computing a result; 20 with no instrument failure: 1 honoured, 1 violated, 18 with no contract checked; 13 where Syntology's instrument failed) · 19 unverified","sample_list":"/paper/llama-2-open-foundation-and-fine-tuned-chat#ran","syntology_url":"https://syntology.ai/paper/2307.09288","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2307.09288"}},"official":{"repos":["facebookresearch/llama"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/palm-scaling-language-modeling-with-pathways-1","slug":"palm-scaling-language-modeling-with-pathways-1","title":"PaLM: Scaling Language Modeling with Pathways","date":"2022-04-05","arxiv_id":"2204.02311","repositories_listed":7,"syntology":{"n":37,"n_ran":32,"n_constructed":16,"n_ran_checked":24,"n_instrument":8,"n_unverified":5,"n_honours":2,"n_violates":1,"n_no_contract":21,"n_pointer_only":0,"phrase":"32 ran (of which 16 constructed an object rather than computing a result; 24 with no instrument failure: 2 honoured, 1 violated, 21 with no contract checked; 8 where Syntology's instrument failed) · 5 unverified","sample_list":"/paper/palm-scaling-language-modeling-with-pathways-1#ran","syntology_url":"https://syntology.ai/paper/2204.02311","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2204.02311"}},"official":null}},{"url":"/paper/from-recognition-to-cognition-visual","slug":"from-recognition-to-cognition-visual","title":"From Recognition to Cognition: Visual Commonsense Reasoning","date":"2018-11-27","arxiv_id":"1811.10830","repositories_listed":4,"syntology":{"n":6,"n_ran":5,"n_constructed":0,"n_ran_checked":5,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":5,"n_pointer_only":0,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 0 honoured, 0 violated, 5 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/from-recognition-to-cognition-visual#ran","syntology_url":"https://syntology.ai/paper/1811.10830","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1811.10830"}},"official":null}},{"url":"/paper/quality-question-answering-with-long-input","slug":"quality-question-answering-with-long-input","title":"QuALITY: Question Answering with Long Input Texts, Yes!","date":"2021-12-16","arxiv_id":"2112.08608","repositories_listed":3,"syntology":null},{"url":"/paper/scaling-language-models-methods-analysis-1","slug":"scaling-language-models-methods-analysis-1","title":"Scaling Language Models: Methods, Analysis & Insights from Training Gopher","date":"2021-12-08","arxiv_id":"2112.11446","repositories_listed":3,"syntology":null},{"url":"/paper/learning-to-attend-on-essential-terms-an","slug":"learning-to-attend-on-essential-terms-an","title":"Learning to Attend On Essential Terms: An Enhanced Retriever-Reader Model for Open-domain Question Answering","date":"2018-08-28","arxiv_id":"1808.09492","repositories_listed":3,"syntology":null},{"url":"/paper/bloomberggpt-a-large-language-model-for","slug":"bloomberggpt-a-large-language-model-for","title":"BloombergGPT: A Large Language Model for Finance","date":"2023-03-30","arxiv_id":"2303.17564","repositories_listed":2,"syntology":null},{"url":"/paper/variational-open-domain-question-answering","slug":"variational-open-domain-question-answering","title":"Variational Open-Domain Question Answering","date":"2022-09-23","arxiv_id":"2210.06345","repositories_listed":2,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/variational-open-domain-question-answering#ran","syntology_url":"https://syntology.ai/paper/2210.06345","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2210.06345"}},"official":{"repos":["findzebra/fz-openqa","VodLM/vod"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/training-compute-optimal-large-language","slug":"training-compute-optimal-large-language","title":"Training Compute-Optimal Large Language Models","date":"2022-03-29","arxiv_id":"2203.15556","repositories_listed":2,"syntology":{"n":11,"n_ran":8,"n_constructed":3,"n_ran_checked":5,"n_instrument":3,"n_unverified":3,"n_honours":2,"n_violates":0,"n_no_contract":3,"n_pointer_only":4,"phrase":"8 ran (of which 3 constructed an object rather than computing a result; 5 with no instrument failure: 2 honoured, 0 violated, 3 with no contract checked; 3 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/training-compute-optimal-large-language#ran","syntology_url":"https://syntology.ai/paper/2203.15556","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2203.15556"}},"official":null}},{"url":"/paper/mmm-multi-stage-multi-task-learning-for-multi","slug":"mmm-multi-stage-multi-task-learning-for-multi","title":"MMM: Multi-stage Multi-task Learning for Multi-choice Reading Comprehension","date":"2019-10-01","arxiv_id":"1910.00458","repositories_listed":2,"syntology":null},{"url":"/paper/question-aware-knowledge-graph-prompting-for","slug":"question-aware-knowledge-graph-prompting-for","title":"Question-Aware Knowledge Graph Prompting for Enhancing Large Language Models","date":"2025-03-30","arxiv_id":"2503.23523","repositories_listed":1,"syntology":null},{"url":"/paper/wrong-answers-can-also-be-useful-plausibleqa","slug":"wrong-answers-can-also-be-useful-plausibleqa","title":"Wrong Answers Can Also Be Useful: PlausibleQA -- A Large-Scale QA Dataset with Answer Plausibility Scores","date":"2025-02-22","arxiv_id":"2502.16358","repositories_listed":1,"syntology":null},{"url":"/paper/investigating-the-shortcomings-of-llms-in","slug":"investigating-the-shortcomings-of-llms-in","title":"Investigating the Shortcomings of LLMs in Step-by-Step Legal Reasoning","date":"2025-02-08","arxiv_id":"2502.05675","repositories_listed":1,"syntology":null},{"url":"/paper/medg-krp-medical-graph-knowledge","slug":"medg-krp-medical-graph-knowledge","title":"MedG-KRP: Medical Graph Knowledge Representation Probing","date":"2024-12-14","arxiv_id":"2412.10982","repositories_listed":1,"syntology":null},{"url":"/paper/exploring-the-abilities-of-large-language","slug":"exploring-the-abilities-of-large-language","title":"KnowledgePrompts: Exploring the Abilities of Large Language Models to Solve Proportional Analogies via Knowledge-Enhanced Prompting","date":"2024-12-01","arxiv_id":"2412.00869","repositories_listed":1,"syntology":null},{"url":"/paper/differentiating-choices-via-commonality-for","slug":"differentiating-choices-via-commonality-for","title":"Differentiating Choices via Commonality for Multiple-Choice Question Answering","date":"2024-08-21","arxiv_id":"2408.11554","repositories_listed":1,"syntology":null},{"url":"/paper/econlogicqa-a-question-answering-benchmark","slug":"econlogicqa-a-question-answering-benchmark","title":"EconLogicQA: A Question-Answering Benchmark for Evaluating Large Language Models in Economic Sequential Reasoning","date":"2024-05-13","arxiv_id":"2405.07938","repositories_listed":1,"syntology":{"n":10,"n_ran":7,"n_constructed":0,"n_ran_checked":3,"n_instrument":4,"n_unverified":3,"n_honours":1,"n_violates":0,"n_no_contract":2,"n_pointer_only":2,"phrase":"7 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 1 honoured, 0 violated, 2 with no contract checked; 4 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/econlogicqa-a-question-answering-benchmark#ran","syntology_url":"https://syntology.ai/paper/2405.07938","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2405.07938"}},"official":{"repos":["yinzhu-quan/lm-evaluation-harness"],"state":"official (archive's flag): 7 ran","n_ran":7,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":3,"ran_from_kinds":["official"]}}},{"url":"/paper/adamole-fine-tuning-large-language-models","slug":"adamole-fine-tuning-large-language-models","title":"AdaMoLE: Fine-Tuning Large Language Models with Adaptive Mixture of Low-Rank Adaptation Experts","date":"2024-05-01","arxiv_id":"2405.00361","repositories_listed":1,"syntology":{"n":2,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/adamole-fine-tuning-large-language-models#ran","syntology_url":"https://syntology.ai/paper/2405.00361","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2405.00361"}},"official":{"repos":["zefang-liu/adamole"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/can-a-multichoice-dataset-be-repurposed-for","slug":"can-a-multichoice-dataset-be-repurposed-for","title":"From Multiple-Choice to Extractive QA: A Case Study for English and Arabic","date":"2024-04-26","arxiv_id":"2404.17342","repositories_listed":1,"syntology":null},{"url":"/paper/artifacts-or-abduction-how-do-llms-answer","slug":"artifacts-or-abduction-how-do-llms-answer","title":"Artifacts or Abduction: How Do LLMs Answer Multiple-Choice Questions Without the Question?","date":"2024-02-19","arxiv_id":"2402.12483","repositories_listed":1,"syntology":{"n":10,"n_ran":4,"n_constructed":0,"n_ran_checked":0,"n_instrument":4,"n_unverified":6,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":10,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 4 where Syntology's instrument failed) · 6 unverified","sample_list":"/paper/artifacts-or-abduction-how-do-llms-answer#ran","syntology_url":"https://syntology.ai/paper/2402.12483","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2402.12483"}},"official":{"repos":["nbalepur/mcqa-artifacts"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":6,"ran_from_kinds":["official"]}}},{"url":"/paper/meditron-70b-scaling-medical-pretraining-for","slug":"meditron-70b-scaling-medical-pretraining-for","title":"MEDITRON-70B: Scaling Medical Pretraining for Large Language Models","date":"2023-11-27","arxiv_id":"2311.16079","repositories_listed":1,"syntology":{"n":14,"n_ran":9,"n_constructed":0,"n_ran_checked":9,"n_instrument":0,"n_unverified":5,"n_honours":0,"n_violates":0,"n_no_contract":9,"n_pointer_only":0,"phrase":"9 ran (of which 0 constructed an object rather than computing a result; 9 with no instrument failure: 0 honoured, 0 violated, 9 with no contract checked; 0 where Syntology's instrument failed) · 5 unverified","sample_list":"/paper/meditron-70b-scaling-medical-pretraining-for#ran","syntology_url":"https://syntology.ai/paper/2311.16079","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2311.16079"}},"official":{"repos":["epfllm/meditron"],"state":"official (archive's flag): 9 ran","n_ran":9,"n_constructed":0,"n_ran_no_instrument_failure":9,"n_unverified":5,"ran_from_kinds":["official"]}}},{"url":"/paper/fool-your-vision-and-language-model-with","slug":"fool-your-vision-and-language-model-with","title":"Fool Your (Vision and) Language Model With Embarrassingly Simple Permutations","date":"2023-10-02","arxiv_id":"2310.01651","repositories_listed":1,"syntology":{"n":5,"n_ran":4,"n_constructed":0,"n_ran_checked":1,"n_instrument":3,"n_unverified":1,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":5,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 3 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/fool-your-vision-and-language-model-with#ran","syntology_url":"https://syntology.ai/paper/2310.01651","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2310.01651"}},"official":{"repos":["ys-zong/foolyourvllms"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":1,"ran_from_kinds":["official","unlocated"]}}},{"url":"/paper/biomedgpt-open-multimodal-generative-pre","slug":"biomedgpt-open-multimodal-generative-pre","title":"BioMedGPT: Open Multimodal Generative Pre-trained Transformer for BioMedicine","date":"2023-08-18","arxiv_id":"2308.09442","repositories_listed":1,"syntology":null},{"url":"/paper/m3ke-a-massive-multi-level-multi-subject","slug":"m3ke-a-massive-multi-level-multi-subject","title":"M3KE: A Massive Multi-Level Multi-Subject Knowledge Evaluation Benchmark for Chinese Large Language Models","date":"2023-05-17","arxiv_id":"2305.10263","repositories_listed":1,"syntology":null},{"url":"/paper/towards-expert-level-medical-question","slug":"towards-expert-level-medical-question","title":"Towards Expert-Level Medical Question Answering with Large Language Models","date":"2023-05-16","arxiv_id":"2305.09617","repositories_listed":1,"syntology":null},{"url":"/paper/frenchmedmcqa-a-french-multiple-choice-1","slug":"frenchmedmcqa-a-french-multiple-choice-1","title":"FrenchMedMCQA: A French Multiple-Choice Question Answering Dataset for Medical domain","date":"2023-04-09","arxiv_id":"2304.04280","repositories_listed":1,"syntology":null},{"url":"/paper/large-language-models-encode-clinical","slug":"large-language-models-encode-clinical","title":"Large Language Models Encode Clinical Knowledge","date":"2022-12-26","arxiv_id":"2212.13138","repositories_listed":1,"syntology":null},{"url":"/paper/galactica-a-large-language-model-for-science-1","slug":"galactica-a-large-language-model-for-science-1","title":"Galactica: A Large Language Model for Science","date":"2022-11-16","arxiv_id":"2211.09085","repositories_listed":1,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":2,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/galactica-a-large-language-model-for-science-1#ran","syntology_url":"https://syntology.ai/paper/2211.09085","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2211.09085"}},"official":{"repos":["paperswithcode/galai"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/leveraging-large-language-models-for-multiple","slug":"leveraging-large-language-models-for-multiple","title":"Leveraging Large Language Models for Multiple Choice Question Answering","date":"2022-10-22","arxiv_id":"2210.12353","repositories_listed":1,"syntology":{"n":7,"n_ran":6,"n_constructed":3,"n_ran_checked":3,"n_instrument":3,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":0,"phrase":"6 ran (of which 3 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 3 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/leveraging-large-language-models-for-multiple#ran","syntology_url":"https://syntology.ai/paper/2210.12353","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2210.12353"}},"official":{"repos":["byu-pccl/leveraging-llms-for-mcqa"],"state":"official (archive's flag): 6 ran","n_ran":6,"n_constructed":3,"n_ran_no_instrument_failure":3,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/can-large-language-models-reason-about","slug":"can-large-language-models-reason-about","title":"Can large language models reason about medical questions?","date":"2022-07-17","arxiv_id":"2207.08143","repositories_listed":1,"syntology":{"n":10,"n_ran":10,"n_constructed":0,"n_ran_checked":10,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":10,"n_pointer_only":0,"phrase":"10 ran (of which 0 constructed an object rather than computing a result; 10 with no instrument failure: 0 honoured, 0 violated, 10 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/can-large-language-models-reason-about#ran","syntology_url":"https://syntology.ai/paper/2207.08143","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2207.08143"}},"official":{"repos":["vlievin/medical-reasoning"],"state":"official (archive's flag): 10 ran","n_ran":10,"n_constructed":0,"n_ran_no_instrument_failure":10,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/clues-before-answers-generation-enhanced","slug":"clues-before-answers-generation-enhanced","title":"Clues Before Answers: Generation-Enhanced Multiple-Choice QA","date":"2022-04-30","arxiv_id":"2205.00274","repositories_listed":1,"syntology":{"n":11,"n_ran":5,"n_constructed":0,"n_ran_checked":5,"n_instrument":0,"n_unverified":6,"n_honours":1,"n_violates":0,"n_no_contract":4,"n_pointer_only":0,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 1 honoured, 0 violated, 4 with no contract checked; 0 where Syntology's instrument failed) · 6 unverified","sample_list":"/paper/clues-before-answers-generation-enhanced#ran","syntology_url":"https://syntology.ai/paper/2205.00274","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2205.00274"}},"official":{"repos":["nju-websoft/genmc"],"state":"official (archive's flag): 5 ran","n_ran":5,"n_constructed":0,"n_ran_no_instrument_failure":5,"n_unverified":6,"ran_from_kinds":["official"]}}},{"url":"/paper/medmcqa-a-large-scale-multi-subject-multi","slug":"medmcqa-a-large-scale-multi-subject-multi","title":"MedMCQA : A Large-scale Multi-Subject Multi-Choice Dataset for Medical domain Question Answering","date":"2022-03-27","arxiv_id":"2203.14371","repositories_listed":1,"syntology":{"n":1,"n_ran":0,"n_constructed":0,"n_ran_checked":0,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"0 ran · 1 unverified","sample_list":"/paper/medmcqa-a-large-scale-multi-subject-multi#ran","syntology_url":"https://syntology.ai/paper/2203.14371","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2203.14371"}},"official":{"repos":["MedMCQA/MedMCQA"],"state":"official: harvested, nothing ran","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":1,"ran_from_kinds":[]}}},{"url":"/paper/does-transliteration-help-multilingual","slug":"does-transliteration-help-multilingual","title":"Does Transliteration Help Multilingual Language Modeling?","date":"2022-01-29","arxiv_id":"2201.12501","repositories_listed":1,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/does-transliteration-help-multilingual#ran","syntology_url":"https://syntology.ai/paper/2201.12501","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2201.12501"}},"official":{"repos":["ibraheem-moosa/xlm-indic"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/lexglue-a-benchmark-dataset-for-legal","slug":"lexglue-a-benchmark-dataset-for-legal","title":"LexGLUE: A Benchmark Dataset for Legal Language Understanding in English","date":"2021-10-03","arxiv_id":"2110.00976","repositories_listed":1,"syntology":{"n":1,"n_ran":0,"n_constructed":0,"n_ran_checked":0,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":1,"phrase":"0 ran · 1 unverified","sample_list":"/paper/lexglue-a-benchmark-dataset-for-legal#ran","syntology_url":"https://syntology.ai/paper/2110.00976","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2110.00976"}},"official":{"repos":["coastalcph/lex-glue"],"state":"official: not harvested","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":[]}}},{"url":"/paper/role-of-language-relatedness-in-multilingual","slug":"role-of-language-relatedness-in-multilingual","title":"Role of Language Relatedness in Multilingual Fine-tuning of Language Models: A Case Study in Indo-Aryan Languages","date":"2021-09-22","arxiv_id":"2109.10534","repositories_listed":1,"syntology":null},{"url":"/paper/indicnlpsuite-monolingual-corpora-evaluation","slug":"indicnlpsuite-monolingual-corpora-evaluation","title":"IndicNLPSuite: Monolingual Corpora, Evaluation Benchmarks and Pre-trained Multilingual Language Models for Indian Languages","date":"2020-11-08","arxiv_id":null,"repositories_listed":1,"syntology":null},{"url":"/paper/counterfactual-variable-control-for-robust","slug":"counterfactual-variable-control-for-robust","title":"Counterfactual Variable Control for Robust and Interpretable Question Answering","date":"2020-10-12","arxiv_id":"2010.05581","repositories_listed":1,"syntology":null},{"url":null,"slug":"cp-router-an-uncertainty-aware-router-between","title":"CP-Router: An Uncertainty-Aware Router Between LLM and LRM","date":"2025-05-26","arxiv_id":"2505.19970","repositories_listed":0,"syntology":null},{"url":null,"slug":"improving-llm-first-token-predictions-in","title":"Improving LLM First-Token Predictions in Multiple-Choice Question Answering via Prefilling Attack","date":"2025-05-21","arxiv_id":"2505.15323","repositories_listed":0,"syntology":null},{"url":null,"slug":"healthy-llms-benchmarking-llm-knowledge-of-uk","title":"Healthy LLMs? Benchmarking LLM Knowledge of UK Government Public Health Information","date":"2025-05-09","arxiv_id":"2505.06046","repositories_listed":0,"syntology":null},{"url":null,"slug":"statistical-guarantees-of-correctness","title":"Correctness Coverage Evaluation for Medical Multiple-Choice Question Answering Based on the Enhanced Conformal Prediction Framework","date":"2025-03-07","arxiv_id":"2503.05505","repositories_listed":0,"syntology":null},{"url":null,"slug":"med-rlvr-emerging-medical-reasoning-from-a-3b","title":"Med-RLVR: Emerging Medical Reasoning from a 3B base model via reinforcement Learning","date":"2025-02-27","arxiv_id":"2502.19655","repositories_listed":0,"syntology":null},{"url":null,"slug":"which-of-these-best-describes-multiple-choice","title":"Which of These Best Describes Multiple Choice Evaluation with LLMs? A) Forced B) Flawed C) Fixable D) All of the Above","date":"2025-02-19","arxiv_id":"2502.14127","repositories_listed":0,"syntology":null},{"url":null,"slug":"first-token-probability-guided-rag-for","title":"First Token Probability Guided RAG for Telecom Question Answering","date":"2025-01-11","arxiv_id":"2501.06468","repositories_listed":0,"syntology":null},{"url":null,"slug":"llm-distillation-for-efficient-few-shot","title":"LLM Distillation for Efficient Few-Shot Multiple Choice Question Answering","date":"2024-12-13","arxiv_id":"2412.09807","repositories_listed":0,"syntology":null},{"url":null,"slug":"sandboxaq-s-submission-to-mrl-2024-shared","title":"SandboxAQ's submission to MRL 2024 Shared Task on Multi-lingual Multi-task Information Retrieval","date":"2024-10-28","arxiv_id":"2410.21501","repositories_listed":0,"syntology":null},{"url":null,"slug":"addressing-blind-guessing-calibration-of","title":"Addressing Blind Guessing: Calibration of Selection Bias in Multiple-Choice Question Answering by Video Language Models","date":"2024-10-18","arxiv_id":"2410.14248","repositories_listed":0,"syntology":null},{"url":"/paper/answer-assemble-ace-understanding-how","slug":"answer-assemble-ace-understanding-how","title":"Answer, Assemble, Ace: Understanding How Transformers Answer Multiple Choice Questions","date":"2024-07-21","arxiv_id":"2407.15018","repositories_listed":0,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/answer-assemble-ace-understanding-how#ran","syntology_url":"https://syntology.ai/paper/2407.15018","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2407.15018"}},"official":null}},{"url":"/paper/short-film-dataset-sfd-a-benchmark-for-story","slug":"short-film-dataset-sfd-a-benchmark-for-story","title":"Long Story Short: Story-level Video Understanding from 20K Short Films","date":"2024-06-14","arxiv_id":"2406.10221","repositories_listed":0,"syntology":null},{"url":null,"slug":"rethinking-generative-large-language-model","title":"Rethinking Generative Large Language Model Evaluation for Semantic Comprehension","date":"2024-03-12","arxiv_id":"2403.07872","repositories_listed":0,"syntology":null},{"url":null,"slug":"kormedmcqa-multi-choice-question-answering","title":"KorMedMCQA: Multi-Choice Question Answering Benchmark for Korean Healthcare Professional Licensing Examinations","date":"2024-03-03","arxiv_id":"2403.01469","repositories_listed":0,"syntology":null},{"url":null,"slug":"unsupervised-multiple-choices-question","title":"Unsupervised multiple choices question answering via universal corpus","date":"2024-02-27","arxiv_id":"2402.17333","repositories_listed":0,"syntology":null},{"url":null,"slug":"beyond-the-answers-reviewing-the-rationality","title":"LLMs May Perform MCQA by Selecting the Least Incorrect Option","date":"2024-02-02","arxiv_id":"2402.01349","repositories_listed":0,"syntology":null},{"url":null,"slug":"evaluating-the-symbol-binding-ability-of","title":"Evaluating the Symbol Binding Ability of Large Language Models for Multiple-Choice Questions in Vietnamese General Education","date":"2023-10-18","arxiv_id":"2310.12059","repositories_listed":0,"syntology":null},{"url":null,"slug":"generating-multiple-choice-questions-for","title":"Generating multiple-choice questions for medical question answering with distractors and cue-masking","date":"2023-03-13","arxiv_id":"2303.07069","repositories_listed":0,"syntology":null},{"url":null,"slug":"hrca-advanced-multiple-choice-machine-reading","title":"HRCA+: Advanced Multiple-choice Machine Reading Comprehension Method","date":"2022-06-01","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"context-guided-triple-matching-for-multiple-1","title":"Context-guided Triple Matching for Multiple Choice Question Answering","date":"2022-01-16","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"disaggregating-hops-can-we-guide-a-multi-hop","title":"Disaggregating Hops: Can We Guide a Multi-Hop Reasoning Language Model to Incrementally Learn at each Hop?","date":"2022-01-16","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"transliteration-a-simple-technique-for","title":"Transliteration: A Simple Technique For Improving Multilingual Language Modeling","date":"2021-09-29","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"context-guided-triple-matching-for-multiple","title":"Context-guided Triple Matching for Multiple Choice Question Answering","date":"2021-09-27","arxiv_id":"2109.12996","repositories_listed":0,"syntology":null},{"url":null,"slug":"fine-tuning-bert-with-focus-words-for","title":"Fine-tuning BERT with Focus Words for Explanation Regeneration","date":"2020-12-01","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"what-do-we-expect-from-multiple-choice-qa","title":"What do we expect from Multiple-choice QA Systems?","date":"2020-11-20","arxiv_id":"2011.10647","repositories_listed":0,"syntology":null},{"url":null,"slug":"context-modeling-with-evidence-filter-for","title":"Context Modeling with Evidence Filter for Multiple Choice Question Answering","date":"2020-10-06","arxiv_id":"2010.02649","repositories_listed":0,"syntology":null},{"url":null,"slug":"multi-source-meta-transfer-for-low-resource","title":"Multi-source Meta Transfer for Low Resource Multiple-Choice Question Answering","date":"2020-07-01","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":"/paper/visual7w-grounded-question-answering-in","slug":"visual7w-grounded-question-answering-in","title":"Visual7W: Grounded Question Answering in Images","date":"2015-11-11","arxiv_id":"1511.03416","repositories_listed":0,"syntology":null}],"syntology_records":18,"syntology_note":"a paper without a record is not a recorded non-run: it may lack an arXiv id or simply be absent from the graph layer"},"key_notes":{"n_ran_checked":"legacy name, kept unchanged so existing readers do not break: it counts the samples that ran with no instrument failure (honoured, violated, and ran with no contract checked); it does not mean a contract was checked, and the pages print it as 'K with no instrument failure', not 'K checked'","n_constructed":"a sub-count of the samples that ran, never subtracted from them and never a failure: an executed sample whose run returned an instance of its own class (fixture_out_type equals the entry name): the run built an object and did not compute a result (Syntology's RAN record, counts.constructed)"},"since_archive_judged":{"papers":12,"papers_placed":0,"judged_by":{"model":"Claude Sonnet 4.5","models_as_recorded":["global.anthropic.claude-sonnet-4-5-20250929-v1:0"],"from":"lb_rows_run_verified.jsonl: each record's `model` field (the extractor's call that wrote decisions[])"},"note":"a model's judgement from the paper's own tables that they report on this leaderboard, not a result: no value from a paper that was not placed is printed or carried here; papers counts each paper once over these leaderboards","backfill":"8,886 of 18,637 newer papers checked so far (733 more had nothing to place; 41 could not be read)","backfill_complete":false,"leaderboards":[{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-21","since_archive":"/sota/multiple-choice-question-answering-mcqa-on-21#since-judged","papers_judged":2,"placed":0},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-27","since_archive":"/sota/multiple-choice-question-answering-mcqa-on-27#since-judged","papers_judged":1,"placed":0},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-28","since_archive":"/sota/multiple-choice-question-answering-mcqa-on-28#since-judged","papers_judged":1,"placed":0},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-11","since_archive":"/sota/multiple-choice-question-answering-mcqa-on-11#since-judged","papers_judged":5,"placed":0},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-8","since_archive":"/sota/multiple-choice-question-answering-mcqa-on-8#since-judged","papers_judged":3,"placed":0},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-25","since_archive":"/sota/multiple-choice-question-answering-mcqa-on-25#since-judged","papers_judged":1,"placed":0},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-2","since_archive":"/sota/multiple-choice-question-answering-mcqa-on-2#since-judged","papers_judged":1,"placed":0},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-3","since_archive":"/sota/multiple-choice-question-answering-mcqa-on-3#since-judged","papers_judged":2,"placed":0},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-4","since_archive":"/sota/multiple-choice-question-answering-mcqa-on-4#since-judged","papers_judged":1,"placed":0},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-5","since_archive":"/sota/multiple-choice-question-answering-mcqa-on-5#since-judged","papers_judged":1,"placed":0},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-7","since_archive":"/sota/multiple-choice-question-answering-mcqa-on-7#since-judged","papers_judged":4,"placed":0},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-9","since_archive":"/sota/multiple-choice-question-answering-mcqa-on-9#since-judged","papers_judged":1,"placed":0}]},"papers_where_code_ran":16,"papers_where_code_ran_first_page":"/task/multiple-choice-qa/papers/ran/1","papers_where_code_ran_caption":"We ran code from the paper's repository; we did not run it on this task or check it against the task's benchmarks.","pages":[{"order":"ran","page":1,"of":1,"url":"/task/multiple-choice-qa/papers/ran/1","json":"/task/multiple-choice-qa/papers/ran/1.json","rows":16}],"description_links":{"kept":0,"unwrapped_to_text":0,"bare_urls_linked":0,"relative_images_dropped":0,"rule":"internal links are kept only when the target slug exists in the catalog"},"syntology":{"read_at":"2026-10-05T00:36:11+00:00","claim":"Per-sample execution status on synthesized fixtures ('ran N of M samples'); not a correctness claim and not a ranking signal.","status_vocabulary":{"ran_honours":"ran, honoured the contract we drafted","ran_violates":"ran, violated the contract we drafted","ran_draft_wrong":"ran; our contract draft was wrong, not the code","ran_fixture":"ran; the contract check could not run (RAISES, DEP_MISSING or TIMEOUT)","ran":"ran on a synthesized input","unverified":"unverified (harvested, no recorded run)"},"row_route":"each syntology object on a benchmark's first row and on a paper carries sample_list (this site's paper page, #ran), syntology_url (syntology.ai's page for the paper, only when that route probed live) and mcp (the agent call that lists the paper's harvested samples; not a verdict on the paper)","developers":"https://syntology.ai/developers"},"not_shown":{"libraries":"the archive has no per-task library table","trend_sparklines":"the Trend column of the benchmarks table was a rendered image; it is not in the archive","social_and_latest_sorts":"stars and social signals are not in the archive"},"record_sha256":"e92776fcae8a02bf1abfb2e89d66da9368774ae0ad92188ce8de66a1e3ff6f0d","record_changed_at":"2026-10-05","record_changed_at_basis":"changed"}