{"about":{"non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","site":"https://codewithpapers.app","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page","syntology":{"site":"https://syntology.ai","developers":"https://syntology.ai/developers","mcp":{"server":"https://syntology.ai/mcp","transport":"streamable-http","server_card":"https://syntology.ai/.well-known/mcp/server-card.json","auth":{"type":"trial token, no account","trial_token":"https://syntology.ai/api/oauth/trial/token","method":"POST","docs":"https://syntology.ai/developers"}},"have":"https://syntology.ai/api/graph/have?x=<method, arXiv id or title> (free, answers coverage only)","paper_base":"https://syntology.ai/paper/","atlas_base":"https://app.syntology.ai/?focus="},"machine_readable":[{"url":"https://codewithpapers.app/llms.txt","what":"the machine catalog: every machine-readable file, counted"},{"url":"https://codewithpapers.app/index/manifest.json","what":"paper-to-code index by arXiv id, with Syntology's counts"},{"url":"https://codewithpapers.app/search/manifest.json","what":"site search index (titles, authors) and its files"},{"url":"https://codewithpapers.app/download","what":"bulk files: Syntology's layer, described there"},{"url":"https://codewithpapers.app/build_manifest.json","what":"the build record: inputs, counts, exclusions, probes"}]},"url":"/dataset/ok-vqa","name":"OK-VQA","full_name":"Outside Knowledge Visual Question Answering","description_markdown":"Outside Knowledge Visual Question Answering (OK-VQA) includes more than 14,000 questions that require external knowledge to answer. \r\n\r\nSource: [OK-VQA: A Visual Question Answering Benchmark Requiring External Knowledge](https://arxiv.org/pdf/1906.00067v2.pdf)\r\nImage Source: [https://okvqa.allenai.org/](https://okvqa.allenai.org/)","description_withheld":null,"homepage":"https://okvqa.allenai.org/","introduced_date":null,"introduced_date_note":null,"introduced_by":{"paper":"/paper/190600067","title":"OK-VQA: A Visual Question Answering Benchmark Requiring External Knowledge","first_author":"Kenneth Marino","url":null},"license":{"name":"Unknown","url":null},"modalities":[{"name":"Images","url":"/datasets/modality/images"},{"name":"Texts","url":"/datasets/modality/texts"}],"tasks":[{"name":"Question Answering","url":"/task/question-answering","datasets_with_task":"/datasets/task/question-answering"},{"name":"Visual Question Answering (VQA)","url":"/task/visual-question-answering","datasets_with_task":"/datasets/task/visual-question-answering"},{"name":"Retrieval","url":"/task/retrieval","datasets_with_task":"/datasets/task/retrieval"},{"name":"Question Generation","url":"/task/question-generation","datasets_with_task":"/datasets/task/question-generation"}],"languages":[{"name":"English","url":"/datasets/language/english"}],"variants":["OK-VQA"],"data_loaders":[],"num_papers_in_archive":368,"source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28"},"benchmarks":[{"leaderboard":"/sota/visual-question-answering-on-ok-vqa","task":"Visual Question Answering (VQA)","dataset_variant":"OK-VQA","rows":37,"metrics":["Accuracy","Exact Match (EM)","Recall@5"],"first_row_in_archive_order":{"model":"PaLI-X-VPD","paper":"/paper/visual-program-distillation-distilling-tools","metrics":{"Accuracy":"66.8"},"code_links":[]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"},{"leaderboard":"/sota/retrieval-on-ok-vqa","task":"Retrieval","dataset_variant":"OK-VQA","rows":2,"metrics":["Recall@5"],"first_row_in_archive_order":{"model":"FLMR","paper":"/paper/fine-grained-late-interaction-multi-modal-1","metrics":{"Recall@5":"89.32"},"code_links":[{"title":"linweizhedragon/retrieval-augmented-visual-question-answering","url":"https://github.com/linweizhedragon/retrieval-augmented-visual-question-answering"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"}],"papers":{"order":"date (newest first), then slug","population":"every paper with a leaderboard row on this dataset's benchmarks (the benchmark-backed subset): the archive's own papers-using-this-dataset list was never published, so this is not that list; num_papers_in_archive is the archive's own count","page":1,"pages_in_order":1,"rows_per_page":100,"shown":26,"of":26,"counts":{"papers_with_a_benchmark_row":26,"with_a_code_link":20,"where_syntology_ran_a_sample":13,"not_listed_spam_title":0,"listed":26,"listed_where_code_ran":13,"where_syntology_ran_a_sample_split":{"with_a_run_with_no_instrument_failure":9,"every_run_a_failure_of_syntologys_instrument":4,"listed_with_a_run_with_no_instrument_failure":9,"listed_every_run_a_failure_of_syntologys_instrument":4,"filter":{"states":["a run with no instrument failure","any run, instrument failures included"],"default":"a run with no instrument failure","note":"on the 'only where code ran' pages the default hides, in the browser, the rows where every run was a failure of Syntology's instrument; the second state shows them again. Rows are hidden, never re-ordered; these twins list every row"}},"definition":"distinct papers with at least one leaderboard row on this dataset's benchmarks; 'where Syntology ran a sample' counts papers with at least one harvested sample that ran, which is not a correctness claim"},"items":[{"paper":"/paper/hydra-a-hyper-agent-for-dynamic-compositional","slug":"hydra-a-hyper-agent-for-dynamic-compositional","title":"HYDRA: A Hyper Agent for Dynamic Compositional Visual Reasoning","date":"2024-03-19","arxiv_id":"2403.12884","rows_on_this_dataset":1,"code_links":1,"syntology":{"read_at":"2026-10-05T00:36:11+00:00","samples_harvested":8,"samples_ran":5,"samples_constructed":0,"samples_ran_checked":2,"samples_ran_instrument_failed":3,"samples_unverified":3,"pointer_only_for_licence":0,"official":{"repos":["ControlNet/HYDRA"],"state":"official (archive's flag): 5 ran","n_ran":5,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":3,"ran_from_kinds":["official"]},"claim":"Per-sample execution on synthesized fixtures; not a correctness claim.","sample_list":"/paper/hydra-a-hyper-agent-for-dynamic-compositional#ran","syntology_url":"https://syntology.ai/paper/2403.12884","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2403.12884"}}}},{"paper":"/paper/lyrics-boosting-fine-grained-language-vision","slug":"lyrics-boosting-fine-grained-language-vision","title":"Lyrics: Boosting Fine-grained Language-Vision Alignment and Comprehension via Semantic-aware Visual Objects","date":"2023-12-08","arxiv_id":"2312.05278","rows_on_this_dataset":1,"code_links":0,"syntology":null},{"paper":"/paper/visual-program-distillation-distilling-tools","slug":"visual-program-distillation-distilling-tools","title":"Visual Program Distillation: Distilling Tools and Programmatic Reasoning into Vision-Language Models","date":"2023-12-05","arxiv_id":"2312.03052","rows_on_this_dataset":1,"code_links":0,"syntology":null},{"paper":"/paper/a-simple-baseline-for-knowledge-based-visual","slug":"a-simple-baseline-for-knowledge-based-visual","title":"A Simple Baseline for Knowledge-Based Visual Question Answering","date":"2023-10-20","arxiv_id":"2310.13570","rows_on_this_dataset":1,"code_links":0,"syntology":{"read_at":"2026-10-05T00:36:11+00:00","samples_harvested":3,"samples_ran":2,"samples_constructed":0,"samples_ran_checked":0,"samples_ran_instrument_failed":2,"samples_unverified":1,"pointer_only_for_licence":3,"official":null,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim.","sample_list":"/paper/a-simple-baseline-for-knowledge-based-visual#ran","syntology_url":"https://syntology.ai/paper/2310.13570","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2310.13570"}}}},{"paper":"/paper/fine-grained-late-interaction-multi-modal-1","slug":"fine-grained-late-interaction-multi-modal-1","title":"Fine-grained Late-interaction Multi-modal Retrieval for Retrieval Augmented Visual Question Answering","date":"2023-09-29","arxiv_id":"2309.17133","rows_on_this_dataset":3,"code_links":1,"syntology":{"read_at":"2026-10-05T00:36:11+00:00","samples_harvested":8,"samples_ran":8,"samples_constructed":0,"samples_ran_checked":2,"samples_ran_instrument_failed":6,"samples_unverified":0,"pointer_only_for_licence":8,"official":{"repos":["linweizhedragon/retrieval-augmented-visual-question-answering"],"state":"official (archive's flag): 8 ran","n_ran":8,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["official"]},"claim":"Per-sample execution on synthesized fixtures; not a correctness claim.","sample_list":"/paper/fine-grained-late-interaction-multi-modal-1#ran","syntology_url":"https://syntology.ai/paper/2309.17133","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2309.17133"}}}},{"paper":"/paper/implicit-differentiable-outlier-detection","slug":"implicit-differentiable-outlier-detection","title":"Implicit Differentiable Outlier Detection Enable Robust Deep Multimodal Analysis","date":"2023-09-21","arxiv_id":null,"rows_on_this_dataset":1,"code_links":1,"syntology":null},{"paper":"/paper/pali-x-on-scaling-up-a-multilingual-vision","slug":"pali-x-on-scaling-up-a-multilingual-vision","title":"PaLI-X: On Scaling up a Multilingual Vision and Language Model","date":"2023-05-29","arxiv_id":"2305.18565","rows_on_this_dataset":1,"code_links":2,"syntology":{"read_at":"2026-10-05T00:36:11+00:00","samples_harvested":7,"samples_ran":7,"samples_constructed":0,"samples_ran_checked":6,"samples_ran_instrument_failed":1,"samples_unverified":0,"pointer_only_for_licence":2,"official":null,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim.","sample_list":"/paper/pali-x-on-scaling-up-a-multilingual-vision#ran","syntology_url":"https://syntology.ai/paper/2305.18565","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2305.18565"}}}},{"paper":"/paper/palm-e-an-embodied-multimodal-language-model","slug":"palm-e-an-embodied-multimodal-language-model","title":"PaLM-E: An Embodied Multimodal Language Model","date":"2023-03-06","arxiv_id":"2303.03378","rows_on_this_dataset":1,"code_links":2,"syntology":null},{"paper":"/paper/prompting-large-language-models-with-answer","slug":"prompting-large-language-models-with-answer","title":"Prophet: Prompting Large Language Models with Complementary Answer Heuristics for Knowledge-based Visual Question Answering","date":"2023-03-03","arxiv_id":"2303.01903","rows_on_this_dataset":1,"code_links":1,"syntology":{"read_at":"2026-10-05T00:36:11+00:00","samples_harvested":8,"samples_ran":6,"samples_constructed":0,"samples_ran_checked":6,"samples_ran_instrument_failed":0,"samples_unverified":2,"pointer_only_for_licence":0,"official":{"repos":["milvlg/prophet"],"state":"official (archive's flag): 6 ran","n_ran":6,"n_constructed":0,"n_ran_no_instrument_failure":6,"n_unverified":2,"ran_from_kinds":["official"]},"claim":"Per-sample execution on synthesized fixtures; not a correctness claim.","sample_list":"/paper/prompting-large-language-models-with-answer#ran","syntology_url":"https://syntology.ai/paper/2303.01903","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2303.01903"}}}},{"paper":"/paper/differentiable-outlier-detection-enable","slug":"differentiable-outlier-detection-enable","title":"Differentiable Outlier Detection Enable Robust Deep Multimodal Analysis","date":"2023-02-11","arxiv_id":"2302.05608","rows_on_this_dataset":1,"code_links":1,"syntology":null},{"paper":"/paper/blip-2-bootstrapping-language-image-pre","slug":"blip-2-bootstrapping-language-image-pre","title":"BLIP-2: Bootstrapping Language-Image Pre-training with Frozen Image Encoders and Large Language Models","date":"2023-01-30","arxiv_id":"2301.12597","rows_on_this_dataset":6,"code_links":17,"syntology":{"read_at":"2026-10-05T00:36:11+00:00","samples_harvested":8,"samples_ran":4,"samples_constructed":4,"samples_ran_checked":4,"samples_ran_instrument_failed":0,"samples_unverified":4,"pointer_only_for_licence":1,"official":{"repos":["salesforce/lavis"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]},"claim":"Per-sample execution on synthesized fixtures; not a correctness claim.","sample_list":"/paper/blip-2-bootstrapping-language-image-pre#ran","syntology_url":"https://syntology.ai/paper/2301.12597","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2301.12597"}}}},{"paper":"/paper/reveal-retrieval-augmented-visual-language","slug":"reveal-retrieval-augmented-visual-language","title":"REVEAL: Retrieval-Augmented Visual-Language Pre-Training with Multi-Source Multimodal Knowledge Memory","date":"2022-12-10","arxiv_id":"2212.05221","rows_on_this_dataset":1,"code_links":1,"syntology":null},{"paper":"/paper/promptcap-prompt-guided-task-aware-image","slug":"promptcap-prompt-guided-task-aware-image","title":"PromptCap: Prompt-Guided Task-Aware Image Captioning","date":"2022-11-15","arxiv_id":"2211.09699","rows_on_this_dataset":1,"code_links":1,"syntology":null},{"paper":"/paper/vlc-bert-visual-question-answering-with","slug":"vlc-bert-visual-question-answering-with","title":"VLC-BERT: Visual Question Answering with Contextualized Commonsense Knowledge","date":"2022-10-24","arxiv_id":"2210.13626","rows_on_this_dataset":1,"code_links":1,"syntology":{"read_at":"2026-10-05T00:36:11+00:00","samples_harvested":2,"samples_ran":2,"samples_constructed":0,"samples_ran_checked":2,"samples_ran_instrument_failed":0,"samples_unverified":0,"pointer_only_for_licence":0,"official":{"repos":["aditya10/vlc-bert"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["official"]},"claim":"Per-sample execution on synthesized fixtures; not a correctness claim.","sample_list":"/paper/vlc-bert-visual-question-answering-with#ran","syntology_url":"https://syntology.ai/paper/2210.13626","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2210.13626"}}}},{"paper":"/paper/plug-and-play-vqa-zero-shot-vqa-by-conjoining","slug":"plug-and-play-vqa-zero-shot-vqa-by-conjoining","title":"Plug-and-Play VQA: Zero-shot VQA by Conjoining Large Pretrained Models with Zero Training","date":"2022-10-17","arxiv_id":"2210.08773","rows_on_this_dataset":1,"code_links":3,"syntology":{"read_at":"2026-10-05T00:36:11+00:00","samples_harvested":3,"samples_ran":2,"samples_constructed":0,"samples_ran_checked":0,"samples_ran_instrument_failed":2,"samples_unverified":1,"pointer_only_for_licence":3,"official":{"repos":["salesforce/lavis"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]},"claim":"Per-sample execution on synthesized fixtures; not a correctness claim.","sample_list":"/paper/plug-and-play-vqa-zero-shot-vqa-by-conjoining#ran","syntology_url":"https://syntology.ai/paper/2210.08773","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2210.08773"}}}},{"paper":"/paper/retrieval-augmented-visual-question-answering","slug":"retrieval-augmented-visual-question-answering","title":"Retrieval Augmented Visual Question Answering with Outside Knowledge","date":"2022-10-07","arxiv_id":"2210.03809","rows_on_this_dataset":3,"code_links":1,"syntology":null},{"paper":"/paper/pali-a-jointly-scaled-multilingual-language","slug":"pali-a-jointly-scaled-multilingual-language","title":"PaLI: A Jointly-Scaled Multilingual Language-Image Model","date":"2022-09-14","arxiv_id":"2209.06794","rows_on_this_dataset":1,"code_links":1,"syntology":{"read_at":"2026-10-05T00:36:11+00:00","samples_harvested":4,"samples_ran":2,"samples_constructed":1,"samples_ran_checked":2,"samples_ran_instrument_failed":0,"samples_unverified":2,"pointer_only_for_licence":0,"official":{"repos":["google-research/big_vision"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":1,"n_ran_no_instrument_failure":2,"n_unverified":2,"ran_from_kinds":["official"]},"claim":"Per-sample execution on synthesized fixtures; not a correctness claim.","sample_list":"/paper/pali-a-jointly-scaled-multilingual-language#ran","syntology_url":"https://syntology.ai/paper/2209.06794","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2209.06794"}}}},{"paper":"/paper/lako-knowledge-driven-visual-question","slug":"lako-knowledge-driven-visual-question","title":"LaKo: Knowledge-driven Visual Question Answering via Late Knowledge-to-Text Injection","date":"2022-07-26","arxiv_id":"2207.12888","rows_on_this_dataset":2,"code_links":1,"syntology":{"read_at":"2026-10-05T00:36:11+00:00","samples_harvested":1,"samples_ran":1,"samples_constructed":0,"samples_ran_checked":0,"samples_ran_instrument_failed":1,"samples_unverified":0,"pointer_only_for_licence":1,"official":{"repos":["hackerchenzhuo/LaKo"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]},"claim":"Per-sample execution on synthesized fixtures; not a correctness claim.","sample_list":"/paper/lako-knowledge-driven-visual-question#ran","syntology_url":"https://syntology.ai/paper/2207.12888","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2207.12888"}}}},{"paper":"/paper/language-models-are-general-purpose","slug":"language-models-are-general-purpose","title":"Language Models are General-Purpose Interfaces","date":"2022-06-13","arxiv_id":"2206.06336","rows_on_this_dataset":1,"code_links":1,"syntology":null},{"paper":"/paper/revive-regional-visual-representation-matters","slug":"revive-regional-visual-representation-matters","title":"REVIVE: Regional Visual Representation Matters in Knowledge-Based Visual Question Answering","date":"2022-06-02","arxiv_id":"2206.01201","rows_on_this_dataset":2,"code_links":1,"syntology":{"read_at":"2026-10-05T00:36:11+00:00","samples_harvested":2,"samples_ran":2,"samples_constructed":2,"samples_ran_checked":2,"samples_ran_instrument_failed":0,"samples_unverified":0,"pointer_only_for_licence":2,"official":{"repos":["yzleroy/revive"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":2,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["official"]},"claim":"Per-sample execution on synthesized fixtures; not a correctness claim.","sample_list":"/paper/revive-regional-visual-representation-matters#ran","syntology_url":"https://syntology.ai/paper/2206.01201","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2206.01201"}}}},{"paper":"/paper/flamingo-a-visual-language-model-for-few-shot-1","slug":"flamingo-a-visual-language-model-for-few-shot-1","title":"Flamingo: a Visual Language Model for Few-Shot Learning","date":"2022-04-29","arxiv_id":"2204.14198","rows_on_this_dataset":3,"code_links":5,"syntology":{"read_at":"2026-10-05T00:36:11+00:00","samples_harvested":24,"samples_ran":18,"samples_constructed":6,"samples_ran_checked":11,"samples_ran_instrument_failed":7,"samples_unverified":6,"pointer_only_for_licence":8,"official":null,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim.","sample_list":"/paper/flamingo-a-visual-language-model-for-few-shot-1#ran","syntology_url":"https://syntology.ai/paper/2204.14198","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2204.14198"}}}},{"paper":"/paper/transform-retrieve-generate-natural-language","slug":"transform-retrieve-generate-natural-language","title":"Transform-Retrieve-Generate: Natural Language-Centric Outside-Knowledge Visual Question Answering","date":"2022-01-01","arxiv_id":null,"rows_on_this_dataset":1,"code_links":0,"syntology":null},{"paper":"/paper/enabling-multimodal-generation-on-clip-via","slug":"enabling-multimodal-generation-on-clip-via","title":"Enabling Multimodal Generation on CLIP via Vision-Language Knowledge Distillation","date":"2021-11-16","arxiv_id":null,"rows_on_this_dataset":1,"code_links":0,"syntology":null},{"paper":"/paper/a-good-prompt-is-worth-millions-of-parameters","slug":"a-good-prompt-is-worth-millions-of-parameters","title":"A Good Prompt Is Worth Millions of Parameters: Low-resource Prompt-based Learning for Vision-Language Models","date":"2021-10-16","arxiv_id":"2110.08484","rows_on_this_dataset":1,"code_links":1,"syntology":null},{"paper":"/paper/an-empirical-study-of-gpt-3-for-few-shot","slug":"an-empirical-study-of-gpt-3-for-few-shot","title":"An Empirical Study of GPT-3 for Few-Shot Knowledge-Based VQA","date":"2021-09-10","arxiv_id":"2109.05014","rows_on_this_dataset":1,"code_links":1,"syntology":{"read_at":"2026-10-05T00:36:11+00:00","samples_harvested":2,"samples_ran":2,"samples_constructed":0,"samples_ran_checked":0,"samples_ran_instrument_failed":2,"samples_unverified":0,"pointer_only_for_licence":0,"official":{"repos":["microsoft/PICa"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]},"claim":"Per-sample execution on synthesized fixtures; not a correctness claim.","sample_list":"/paper/an-empirical-study-of-gpt-3-for-few-shot#ran","syntology_url":"https://syntology.ai/paper/2109.05014","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2109.05014"}}}},{"paper":"/paper/multimodal-few-shot-learning-with-frozen","slug":"multimodal-few-shot-learning-with-frozen","title":"Multimodal Few-Shot Learning with Frozen Language Models","date":"2021-06-25","arxiv_id":"2106.13884","rows_on_this_dataset":1,"code_links":0,"syntology":null}],"full_list":"papers_with_a_benchmark_row: every listed paper, in this order"},"papers_with_a_benchmark_row":[{"paper":"/paper/hydra-a-hyper-agent-for-dynamic-compositional","slug":"hydra-a-hyper-agent-for-dynamic-compositional","title":"HYDRA: A Hyper Agent for Dynamic Compositional Visual Reasoning","date":"2024-03-19","arxiv_id":"2403.12884","rows_on_this_dataset":1,"code_links":1,"syntology":{"read_at":"2026-10-05T00:36:11+00:00","samples_harvested":8,"samples_ran":5,"samples_constructed":0,"samples_ran_checked":2,"samples_ran_instrument_failed":3,"samples_unverified":3,"pointer_only_for_licence":0,"official":{"repos":["ControlNet/HYDRA"],"state":"official (archive's flag): 5 ran","n_ran":5,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":3,"ran_from_kinds":["official"]},"claim":"Per-sample execution on synthesized fixtures; not a correctness claim.","sample_list":"/paper/hydra-a-hyper-agent-for-dynamic-compositional#ran","syntology_url":"https://syntology.ai/paper/2403.12884","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2403.12884"}}}},{"paper":"/paper/lyrics-boosting-fine-grained-language-vision","slug":"lyrics-boosting-fine-grained-language-vision","title":"Lyrics: Boosting Fine-grained Language-Vision Alignment and Comprehension via Semantic-aware Visual Objects","date":"2023-12-08","arxiv_id":"2312.05278","rows_on_this_dataset":1,"code_links":0,"syntology":null},{"paper":"/paper/visual-program-distillation-distilling-tools","slug":"visual-program-distillation-distilling-tools","title":"Visual Program Distillation: Distilling Tools and Programmatic Reasoning into Vision-Language Models","date":"2023-12-05","arxiv_id":"2312.03052","rows_on_this_dataset":1,"code_links":0,"syntology":null},{"paper":"/paper/a-simple-baseline-for-knowledge-based-visual","slug":"a-simple-baseline-for-knowledge-based-visual","title":"A Simple Baseline for Knowledge-Based Visual Question Answering","date":"2023-10-20","arxiv_id":"2310.13570","rows_on_this_dataset":1,"code_links":0,"syntology":{"read_at":"2026-10-05T00:36:11+00:00","samples_harvested":3,"samples_ran":2,"samples_constructed":0,"samples_ran_checked":0,"samples_ran_instrument_failed":2,"samples_unverified":1,"pointer_only_for_licence":3,"official":null,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim.","sample_list":"/paper/a-simple-baseline-for-knowledge-based-visual#ran","syntology_url":"https://syntology.ai/paper/2310.13570","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2310.13570"}}}},{"paper":"/paper/fine-grained-late-interaction-multi-modal-1","slug":"fine-grained-late-interaction-multi-modal-1","title":"Fine-grained Late-interaction Multi-modal Retrieval for Retrieval Augmented Visual Question Answering","date":"2023-09-29","arxiv_id":"2309.17133","rows_on_this_dataset":3,"code_links":1,"syntology":{"read_at":"2026-10-05T00:36:11+00:00","samples_harvested":8,"samples_ran":8,"samples_constructed":0,"samples_ran_checked":2,"samples_ran_instrument_failed":6,"samples_unverified":0,"pointer_only_for_licence":8,"official":{"repos":["linweizhedragon/retrieval-augmented-visual-question-answering"],"state":"official (archive's flag): 8 ran","n_ran":8,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["official"]},"claim":"Per-sample execution on synthesized fixtures; not a correctness claim.","sample_list":"/paper/fine-grained-late-interaction-multi-modal-1#ran","syntology_url":"https://syntology.ai/paper/2309.17133","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2309.17133"}}}},{"paper":"/paper/implicit-differentiable-outlier-detection","slug":"implicit-differentiable-outlier-detection","title":"Implicit Differentiable Outlier Detection Enable Robust Deep Multimodal Analysis","date":"2023-09-21","arxiv_id":null,"rows_on_this_dataset":1,"code_links":1,"syntology":null},{"paper":"/paper/pali-x-on-scaling-up-a-multilingual-vision","slug":"pali-x-on-scaling-up-a-multilingual-vision","title":"PaLI-X: On Scaling up a Multilingual Vision and Language Model","date":"2023-05-29","arxiv_id":"2305.18565","rows_on_this_dataset":1,"code_links":2,"syntology":{"read_at":"2026-10-05T00:36:11+00:00","samples_harvested":7,"samples_ran":7,"samples_constructed":0,"samples_ran_checked":6,"samples_ran_instrument_failed":1,"samples_unverified":0,"pointer_only_for_licence":2,"official":null,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim.","sample_list":"/paper/pali-x-on-scaling-up-a-multilingual-vision#ran","syntology_url":"https://syntology.ai/paper/2305.18565","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2305.18565"}}}},{"paper":"/paper/palm-e-an-embodied-multimodal-language-model","slug":"palm-e-an-embodied-multimodal-language-model","title":"PaLM-E: An Embodied Multimodal Language Model","date":"2023-03-06","arxiv_id":"2303.03378","rows_on_this_dataset":1,"code_links":2,"syntology":null},{"paper":"/paper/prompting-large-language-models-with-answer","slug":"prompting-large-language-models-with-answer","title":"Prophet: Prompting Large Language Models with Complementary Answer Heuristics for Knowledge-based Visual Question Answering","date":"2023-03-03","arxiv_id":"2303.01903","rows_on_this_dataset":1,"code_links":1,"syntology":{"read_at":"2026-10-05T00:36:11+00:00","samples_harvested":8,"samples_ran":6,"samples_constructed":0,"samples_ran_checked":6,"samples_ran_instrument_failed":0,"samples_unverified":2,"pointer_only_for_licence":0,"official":{"repos":["milvlg/prophet"],"state":"official (archive's flag): 6 ran","n_ran":6,"n_constructed":0,"n_ran_no_instrument_failure":6,"n_unverified":2,"ran_from_kinds":["official"]},"claim":"Per-sample execution on synthesized fixtures; not a correctness claim.","sample_list":"/paper/prompting-large-language-models-with-answer#ran","syntology_url":"https://syntology.ai/paper/2303.01903","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2303.01903"}}}},{"paper":"/paper/differentiable-outlier-detection-enable","slug":"differentiable-outlier-detection-enable","title":"Differentiable Outlier Detection Enable Robust Deep Multimodal Analysis","date":"2023-02-11","arxiv_id":"2302.05608","rows_on_this_dataset":1,"code_links":1,"syntology":null},{"paper":"/paper/blip-2-bootstrapping-language-image-pre","slug":"blip-2-bootstrapping-language-image-pre","title":"BLIP-2: Bootstrapping Language-Image Pre-training with Frozen Image Encoders and Large Language Models","date":"2023-01-30","arxiv_id":"2301.12597","rows_on_this_dataset":6,"code_links":17,"syntology":{"read_at":"2026-10-05T00:36:11+00:00","samples_harvested":8,"samples_ran":4,"samples_constructed":4,"samples_ran_checked":4,"samples_ran_instrument_failed":0,"samples_unverified":4,"pointer_only_for_licence":1,"official":{"repos":["salesforce/lavis"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]},"claim":"Per-sample execution on synthesized fixtures; not a correctness claim.","sample_list":"/paper/blip-2-bootstrapping-language-image-pre#ran","syntology_url":"https://syntology.ai/paper/2301.12597","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2301.12597"}}}},{"paper":"/paper/reveal-retrieval-augmented-visual-language","slug":"reveal-retrieval-augmented-visual-language","title":"REVEAL: Retrieval-Augmented Visual-Language Pre-Training with Multi-Source Multimodal Knowledge Memory","date":"2022-12-10","arxiv_id":"2212.05221","rows_on_this_dataset":1,"code_links":1,"syntology":null},{"paper":"/paper/promptcap-prompt-guided-task-aware-image","slug":"promptcap-prompt-guided-task-aware-image","title":"PromptCap: Prompt-Guided Task-Aware Image Captioning","date":"2022-11-15","arxiv_id":"2211.09699","rows_on_this_dataset":1,"code_links":1,"syntology":null},{"paper":"/paper/vlc-bert-visual-question-answering-with","slug":"vlc-bert-visual-question-answering-with","title":"VLC-BERT: Visual Question Answering with Contextualized Commonsense Knowledge","date":"2022-10-24","arxiv_id":"2210.13626","rows_on_this_dataset":1,"code_links":1,"syntology":{"read_at":"2026-10-05T00:36:11+00:00","samples_harvested":2,"samples_ran":2,"samples_constructed":0,"samples_ran_checked":2,"samples_ran_instrument_failed":0,"samples_unverified":0,"pointer_only_for_licence":0,"official":{"repos":["aditya10/vlc-bert"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["official"]},"claim":"Per-sample execution on synthesized fixtures; not a correctness claim.","sample_list":"/paper/vlc-bert-visual-question-answering-with#ran","syntology_url":"https://syntology.ai/paper/2210.13626","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2210.13626"}}}},{"paper":"/paper/plug-and-play-vqa-zero-shot-vqa-by-conjoining","slug":"plug-and-play-vqa-zero-shot-vqa-by-conjoining","title":"Plug-and-Play VQA: Zero-shot VQA by Conjoining Large Pretrained Models with Zero Training","date":"2022-10-17","arxiv_id":"2210.08773","rows_on_this_dataset":1,"code_links":3,"syntology":{"read_at":"2026-10-05T00:36:11+00:00","samples_harvested":3,"samples_ran":2,"samples_constructed":0,"samples_ran_checked":0,"samples_ran_instrument_failed":2,"samples_unverified":1,"pointer_only_for_licence":3,"official":{"repos":["salesforce/lavis"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]},"claim":"Per-sample execution on synthesized fixtures; not a correctness claim.","sample_list":"/paper/plug-and-play-vqa-zero-shot-vqa-by-conjoining#ran","syntology_url":"https://syntology.ai/paper/2210.08773","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2210.08773"}}}},{"paper":"/paper/retrieval-augmented-visual-question-answering","slug":"retrieval-augmented-visual-question-answering","title":"Retrieval Augmented Visual Question Answering with Outside Knowledge","date":"2022-10-07","arxiv_id":"2210.03809","rows_on_this_dataset":3,"code_links":1,"syntology":null},{"paper":"/paper/pali-a-jointly-scaled-multilingual-language","slug":"pali-a-jointly-scaled-multilingual-language","title":"PaLI: A Jointly-Scaled Multilingual Language-Image Model","date":"2022-09-14","arxiv_id":"2209.06794","rows_on_this_dataset":1,"code_links":1,"syntology":{"read_at":"2026-10-05T00:36:11+00:00","samples_harvested":4,"samples_ran":2,"samples_constructed":1,"samples_ran_checked":2,"samples_ran_instrument_failed":0,"samples_unverified":2,"pointer_only_for_licence":0,"official":{"repos":["google-research/big_vision"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":1,"n_ran_no_instrument_failure":2,"n_unverified":2,"ran_from_kinds":["official"]},"claim":"Per-sample execution on synthesized fixtures; not a correctness claim.","sample_list":"/paper/pali-a-jointly-scaled-multilingual-language#ran","syntology_url":"https://syntology.ai/paper/2209.06794","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2209.06794"}}}},{"paper":"/paper/lako-knowledge-driven-visual-question","slug":"lako-knowledge-driven-visual-question","title":"LaKo: Knowledge-driven Visual Question Answering via Late Knowledge-to-Text Injection","date":"2022-07-26","arxiv_id":"2207.12888","rows_on_this_dataset":2,"code_links":1,"syntology":{"read_at":"2026-10-05T00:36:11+00:00","samples_harvested":1,"samples_ran":1,"samples_constructed":0,"samples_ran_checked":0,"samples_ran_instrument_failed":1,"samples_unverified":0,"pointer_only_for_licence":1,"official":{"repos":["hackerchenzhuo/LaKo"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]},"claim":"Per-sample execution on synthesized fixtures; not a correctness claim.","sample_list":"/paper/lako-knowledge-driven-visual-question#ran","syntology_url":"https://syntology.ai/paper/2207.12888","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2207.12888"}}}},{"paper":"/paper/language-models-are-general-purpose","slug":"language-models-are-general-purpose","title":"Language Models are General-Purpose Interfaces","date":"2022-06-13","arxiv_id":"2206.06336","rows_on_this_dataset":1,"code_links":1,"syntology":null},{"paper":"/paper/revive-regional-visual-representation-matters","slug":"revive-regional-visual-representation-matters","title":"REVIVE: Regional Visual Representation Matters in Knowledge-Based Visual Question Answering","date":"2022-06-02","arxiv_id":"2206.01201","rows_on_this_dataset":2,"code_links":1,"syntology":{"read_at":"2026-10-05T00:36:11+00:00","samples_harvested":2,"samples_ran":2,"samples_constructed":2,"samples_ran_checked":2,"samples_ran_instrument_failed":0,"samples_unverified":0,"pointer_only_for_licence":2,"official":{"repos":["yzleroy/revive"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":2,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["official"]},"claim":"Per-sample execution on synthesized fixtures; not a correctness claim.","sample_list":"/paper/revive-regional-visual-representation-matters#ran","syntology_url":"https://syntology.ai/paper/2206.01201","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2206.01201"}}}},{"paper":"/paper/flamingo-a-visual-language-model-for-few-shot-1","slug":"flamingo-a-visual-language-model-for-few-shot-1","title":"Flamingo: a Visual Language Model for Few-Shot Learning","date":"2022-04-29","arxiv_id":"2204.14198","rows_on_this_dataset":3,"code_links":5,"syntology":{"read_at":"2026-10-05T00:36:11+00:00","samples_harvested":24,"samples_ran":18,"samples_constructed":6,"samples_ran_checked":11,"samples_ran_instrument_failed":7,"samples_unverified":6,"pointer_only_for_licence":8,"official":null,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim.","sample_list":"/paper/flamingo-a-visual-language-model-for-few-shot-1#ran","syntology_url":"https://syntology.ai/paper/2204.14198","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2204.14198"}}}},{"paper":"/paper/transform-retrieve-generate-natural-language","slug":"transform-retrieve-generate-natural-language","title":"Transform-Retrieve-Generate: Natural Language-Centric Outside-Knowledge Visual Question Answering","date":"2022-01-01","arxiv_id":null,"rows_on_this_dataset":1,"code_links":0,"syntology":null},{"paper":"/paper/enabling-multimodal-generation-on-clip-via","slug":"enabling-multimodal-generation-on-clip-via","title":"Enabling Multimodal Generation on CLIP via Vision-Language Knowledge Distillation","date":"2021-11-16","arxiv_id":null,"rows_on_this_dataset":1,"code_links":0,"syntology":null},{"paper":"/paper/a-good-prompt-is-worth-millions-of-parameters","slug":"a-good-prompt-is-worth-millions-of-parameters","title":"A Good Prompt Is Worth Millions of Parameters: Low-resource Prompt-based Learning for Vision-Language Models","date":"2021-10-16","arxiv_id":"2110.08484","rows_on_this_dataset":1,"code_links":1,"syntology":null},{"paper":"/paper/an-empirical-study-of-gpt-3-for-few-shot","slug":"an-empirical-study-of-gpt-3-for-few-shot","title":"An Empirical Study of GPT-3 for Few-Shot Knowledge-Based VQA","date":"2021-09-10","arxiv_id":"2109.05014","rows_on_this_dataset":1,"code_links":1,"syntology":{"read_at":"2026-10-05T00:36:11+00:00","samples_harvested":2,"samples_ran":2,"samples_constructed":0,"samples_ran_checked":0,"samples_ran_instrument_failed":2,"samples_unverified":0,"pointer_only_for_licence":0,"official":{"repos":["microsoft/PICa"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]},"claim":"Per-sample execution on synthesized fixtures; not a correctness claim.","sample_list":"/paper/an-empirical-study-of-gpt-3-for-few-shot#ran","syntology_url":"https://syntology.ai/paper/2109.05014","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2109.05014"}}}},{"paper":"/paper/multimodal-few-shot-learning-with-frozen","slug":"multimodal-few-shot-learning-with-frozen","title":"Multimodal Few-Shot Learning with Frozen Language Models","date":"2021-06-25","arxiv_id":"2106.13884","rows_on_this_dataset":1,"code_links":0,"syntology":null}],"papers_where_code_ran":13,"papers_where_code_ran_first_page":"/dataset/ok-vqa/papers/ran/1","papers_where_code_ran_caption":"We ran code from the paper's repository; we did not run it on this dataset or check it against this dataset's benchmarks.","pages":[{"order":"ran","page":1,"of":1,"url":"/dataset/ok-vqa/papers/ran/1","json":"/dataset/ok-vqa/papers/ran/1.json","rows":13}],"row_route":"each paper's syntology object carries sample_list (this site's paper page, #ran), syntology_url (syntology.ai's page for the paper, only when that route probed live) and mcp (the agent call that lists the paper's harvested samples; not a verdict on the paper)","developers":"https://syntology.ai/developers","syntology_totals":{"read_at":"2026-10-05T00:36:11+00:00","papers_with_samples":13,"samples_harvested":80,"samples_ran":61,"samples_constructed":13,"samples_ran_checked":37,"samples_ran_instrument_failed":24,"samples_unverified":19,"pointer_only_for_licence":28,"papers_with_no_sample_that_ran":0,"papers_with_a_sample_that_ran":13,"papers_with_a_run_with_no_instrument_failure":9,"papers_where_every_run_was_an_instrument_failure":4,"note":"the per-paper counts of every listed paper (all pages), summed; not a rate"},"key_notes":{"samples_ran_checked":"legacy name, kept unchanged so existing readers do not break: it counts the samples that ran with no instrument failure (honoured, violated, and ran with no contract checked); it does not mean a contract was checked, and the pages print it as 'K with no instrument failure', not 'K checked'","samples_constructed":"a sub-count of the samples that ran, never subtracted from them and never a failure: an executed sample whose run returned an instance of its own class (fixture_out_type equals the entry name): the run built an object and did not compute a result (Syntology's RAN record, counts.constructed)"},"since_archive_judged":{"papers":2,"papers_placed":0,"judged_by":{"model":"Claude Sonnet 4.5","models_as_recorded":["global.anthropic.claude-sonnet-4-5-20250929-v1:0"],"from":"lb_rows_run_verified.jsonl: each record's `model` field (the extractor's call that wrote decisions[])"},"note":"a model's judgement from the paper's own tables that they report on this leaderboard, not a result: no value from a paper that was not placed is printed or carried here; papers counts each paper once over these leaderboards","backfill":"8,886 of 18,637 newer papers checked so far (733 more had nothing to place; 41 could not be read)","backfill_complete":false,"leaderboards":[{"leaderboard":"/sota/visual-question-answering-on-ok-vqa","since_archive":"/sota/visual-question-answering-on-ok-vqa#since-judged","papers_judged":2,"placed":0}]},"papers_note":"The archive never published its papers-using-dataset list; these are papers with a leaderboard row on this dataset's benchmarks.","record_sha256":"102b0e155e8438f61f8584c4bc7a845f53c307e59e8241c78ed6b85cf224c5ba","record_changed_at":"2026-10-05","record_changed_at_basis":"changed"}