{"about":{"non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","site":"https://codewithpapers.app","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page","syntology":{"site":"https://syntology.ai","developers":"https://syntology.ai/developers","mcp":{"server":"https://syntology.ai/mcp","transport":"streamable-http","server_card":"https://syntology.ai/.well-known/mcp/server-card.json","auth":{"type":"trial token, no account","trial_token":"https://syntology.ai/api/oauth/trial/token","method":"POST","docs":"https://syntology.ai/developers"}},"have":"https://syntology.ai/api/graph/have?x=<method, arXiv id or title> (free, answers coverage only)","paper_base":"https://syntology.ai/paper/","atlas_base":"https://app.syntology.ai/?focus="},"machine_readable":[{"url":"https://codewithpapers.app/llms.txt","what":"the machine catalog: every machine-readable file, counted"},{"url":"https://codewithpapers.app/index/manifest.json","what":"paper-to-code index by arXiv id, with Syntology's counts"},{"url":"https://codewithpapers.app/search/manifest.json","what":"site search index (titles, authors) and its files"},{"url":"https://codewithpapers.app/download","what":"bulk files: Syntology's layer, described there"},{"url":"https://codewithpapers.app/build_manifest.json","what":"the build record: inputs, counts, exclusions, probes"}]},"url":"/task/visual-question-answering-1","name":"Visual Question Answering","slug":"visual-question-answering-1","description_markdown":"MLLM Leaderboard","categories":[{"name":"Computer Vision","url":"/area/computer-vision"},{"name":"Natural Language Processing","url":"/area/natural-language-processing"}],"source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","slug_source":"archive_url"},"counts":{"papers_tagged":2177,"papers_with_code":1042,"benchmarks":29,"benchmark_tables_in_archive":29,"benchmark_tables_shown":29,"benchmark_tables_withheld_as_spam":0,"benchmark_definition":"a leaderboard table with at least one row; benchmark_tables_shown also counts the zero-row tables; benchmark_tables_in_archive adds the tables withheld as spam","datasets":32,"subtasks":5,"parent_tasks":1},"benchmarks":[{"leaderboard":"/sota/visual-question-answering-on-mm-vet","slug":"visual-question-answering-on-mm-vet","dataset":"MM-Vet","dataset_url":"/dataset/mm-vet","rows_in_archive":231,"metrics":["GPT-4 score","Params"],"first_row_in_archive_order":{"model":"gemini-2.0-flash-exp","paper_title":null,"paper_url":null,"paper_date":"","arxiv_id":null,"code_links":[],"syntology":null}},{"leaderboard":"/sota/visual-question-answering-on-mm-vet-v2","slug":"visual-question-answering-on-mm-vet-v2","dataset":"MM-Vet v2","dataset_url":"/dataset/mm-vet-v2","rows_in_archive":24,"metrics":["GPT-4 score","Params"],"first_row_in_archive_order":{"model":"gemini-2.0-flash-exp","paper_title":null,"paper_url":null,"paper_date":"","arxiv_id":null,"code_links":[],"syntology":null}},{"leaderboard":"/sota/visual-question-answering-on-vip-bench","slug":"visual-question-answering-on-vip-bench","dataset":"ViP-Bench","dataset_url":"/dataset/vip-bench","rows_in_archive":13,"metrics":["GPT-4 score (bbox)","GPT-4 score (human)"],"first_row_in_archive_order":{"model":"GPT-4V-turbo-detail:high (Visual Prompt)","paper_title":"GPT-4 Technical Report","paper_url":"/paper/gpt-4-technical-report-1","paper_date":"2023-03-15","arxiv_id":"2303.08774","code_links":[{"title":"openai/evals","url":"https://github.com/openai/evals"},{"title":"shmsw25/factscore","url":"https://github.com/shmsw25/factscore"},{"title":"unispac/visual-adversarial-examples-jailbreak-large-language-models","url":"https://github.com/unispac/visual-adversarial-examples-jailbreak-large-language-models"},{"title":"gpt4life/alpagasus","url":"https://github.com/gpt4life/alpagasus"},{"title":"emrgnt-cmplxty/zero-shot-replication","url":"https://github.com/emrgnt-cmplxty/zero-shot-replication"},{"title":"ethz-privsec/superhuman-ai-consistency","url":"https://github.com/ethz-privsec/superhuman-ai-consistency"},{"title":"ethz-spylab/superhuman-ai-consistency","url":"https://github.com/ethz-spylab/superhuman-ai-consistency"},{"title":"eternityyw/tram-benchmark","url":"https://github.com/eternityyw/tram-benchmark"},{"title":"AUCOHL/RTL-Repo","url":"https://github.com/AUCOHL/RTL-Repo"},{"title":"zach-zhiling-zheng/reticular_chemist","url":"https://github.com/zach-zhiling-zheng/reticular_chemist"},{"title":"lflage/openfactscore","url":"https://github.com/lflage/openfactscore"}],"syntology":{"n":5,"n_ran":5,"n_constructed":0,"n_ran_checked":5,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":2,"n_no_contract":3,"n_pointer_only":1,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 0 honoured, 2 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/gpt-4-technical-report-1#ran","syntology_url":"https://syntology.ai/paper/2303.08774","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2303.08774"}},"official":{"repos":["openai/evals"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}}},{"leaderboard":"/sota/visual-question-answering-on-vqa-v2-test-dev-1","slug":"visual-question-answering-on-vqa-v2-test-dev-1","dataset":"VQA v2 test-dev","dataset_url":"/dataset/visual-question-answering-v2-0","rows_in_archive":11,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"BLIP-2 ViT-G OPT 6.7B (fine-tuned)","paper_title":"BLIP-2: Bootstrapping Language-Image Pre-training with Frozen Image Encoders and Large Language Models","paper_url":"/paper/blip-2-bootstrapping-language-image-pre","paper_date":"2023-01-30","arxiv_id":"2301.12597","code_links":[{"title":"huggingface/transformers","url":"https://github.com/huggingface/transformers"},{"title":"salesforce/lavis","url":"https://github.com/salesforce/lavis"},{"title":"thudm/visualglm-6b","url":"https://github.com/thudm/visualglm-6b"},{"title":"baaivision/eva","url":"https://github.com/baaivision/eva"},{"title":"junshutang/Make-It-3D","url":"https://github.com/junshutang/Make-It-3D"},{"title":"facebookresearch/multimodal","url":"https://github.com/facebookresearch/multimodal"},{"title":"unispac/visual-adversarial-examples-jailbreak-large-language-models","url":"https://github.com/unispac/visual-adversarial-examples-jailbreak-large-language-models"},{"title":"yukw777/videoblip","url":"https://github.com/yukw777/videoblip"},{"title":"alibaba/graphtranslator","url":"https://github.com/alibaba/graphtranslator"},{"title":"gregor-ge/mblip","url":"https://github.com/gregor-ge/mblip"},{"title":"linzhiqiu/clip-flant5","url":"https://github.com/linzhiqiu/clip-flant5"},{"title":"kdr/videorag-mrr2024","url":"https://github.com/kdr/videorag-mrr2024"},{"title":"rabiulcste/vqazero","url":"https://github.com/rabiulcste/vqazero"},{"title":"jiwanchung/vlis","url":"https://github.com/jiwanchung/vlis"},{"title":"yangyucheng000/University","url":"https://github.com/yangyucheng000/University/tree/main/model-2/blip_2"},{"title":"2024-MindSpore-1/Code2","url":"https://github.com/2024-MindSpore-1/Code2/tree/main/model-1/blip_2"},{"title":"albertotestoni/ndq_visual_objects","url":"https://github.com/albertotestoni/ndq_visual_objects"}],"syntology":{"n":8,"n_ran":4,"n_constructed":4,"n_ran_checked":4,"n_instrument":0,"n_unverified":4,"n_honours":0,"n_violates":0,"n_no_contract":4,"n_pointer_only":1,"phrase":"4 ran (of which 4 constructed an object rather than computing a result; 4 with no instrument failure: 0 honoured, 0 violated, 4 with no contract checked; 0 where Syntology's instrument failed) · 4 unverified; every one of the 4 samples that ran constructed an object rather than computing a result","sample_list":"/paper/blip-2-bootstrapping-language-image-pre#ran","syntology_url":"https://syntology.ai/paper/2301.12597","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2301.12597"}},"official":{"repos":["salesforce/lavis"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}}},{"leaderboard":"/sota/visual-question-answering-on-benchlmm","slug":"visual-question-answering-on-benchlmm","dataset":"BenchLMM","dataset_url":"/dataset/benchlmm","rows_in_archive":10,"metrics":["GPT-3.5 score"],"first_row_in_archive_order":{"model":"GPT-4V","paper_title":"GPT-4 Technical Report","paper_url":"/paper/gpt-4-technical-report-1","paper_date":"2023-03-15","arxiv_id":"2303.08774","code_links":[{"title":"openai/evals","url":"https://github.com/openai/evals"},{"title":"shmsw25/factscore","url":"https://github.com/shmsw25/factscore"},{"title":"unispac/visual-adversarial-examples-jailbreak-large-language-models","url":"https://github.com/unispac/visual-adversarial-examples-jailbreak-large-language-models"},{"title":"gpt4life/alpagasus","url":"https://github.com/gpt4life/alpagasus"},{"title":"emrgnt-cmplxty/zero-shot-replication","url":"https://github.com/emrgnt-cmplxty/zero-shot-replication"},{"title":"ethz-privsec/superhuman-ai-consistency","url":"https://github.com/ethz-privsec/superhuman-ai-consistency"},{"title":"ethz-spylab/superhuman-ai-consistency","url":"https://github.com/ethz-spylab/superhuman-ai-consistency"},{"title":"eternityyw/tram-benchmark","url":"https://github.com/eternityyw/tram-benchmark"},{"title":"AUCOHL/RTL-Repo","url":"https://github.com/AUCOHL/RTL-Repo"},{"title":"zach-zhiling-zheng/reticular_chemist","url":"https://github.com/zach-zhiling-zheng/reticular_chemist"},{"title":"lflage/openfactscore","url":"https://github.com/lflage/openfactscore"}],"syntology":{"n":5,"n_ran":5,"n_constructed":0,"n_ran_checked":5,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":2,"n_no_contract":3,"n_pointer_only":1,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 0 honoured, 2 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/gpt-4-technical-report-1#ran","syntology_url":"https://syntology.ai/paper/2303.08774","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2303.08774"}},"official":{"repos":["openai/evals"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}}},{"leaderboard":"/sota/visual-question-answering-on-mmbench","slug":"visual-question-answering-on-mmbench","dataset":"MMBench","dataset_url":"/dataset/mmbench","rows_in_archive":5,"metrics":["GPT-3.5 score"],"first_row_in_archive_order":{"model":"LLaVA-InternLM2-ViT + MoSLoRA","paper_title":"Mixture-of-Subspaces in Low-Rank Adaptation","paper_url":"/paper/mixture-of-subspaces-in-low-rank-adaptation","paper_date":"2024-06-16","arxiv_id":"2406.11909","code_links":[{"title":"wutaiqiang/moslora","url":"https://github.com/wutaiqiang/moslora"}],"syntology":{"n":6,"n_ran":4,"n_constructed":0,"n_ran_checked":1,"n_instrument":3,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":6,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 3 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/mixture-of-subspaces-in-low-rank-adaptation#ran","syntology_url":"https://syntology.ai/paper/2406.11909","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.11909"}},"official":{"repos":["wutaiqiang/moslora"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":2,"ran_from_kinds":["official"]}}}},{"leaderboard":"/sota/visual-question-answering-on-v-bench","slug":"visual-question-answering-on-v-bench","dataset":"V*bench","dataset_url":null,"rows_in_archive":5,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"LLaVA-OneVision7B w. FOCUS","paper_title":"FOCUS: Internal MLLM Representations for Efficient Fine-Grained Visual Question Answering","paper_url":"/paper/focus-internal-mllm-representations-for","paper_date":"2025-06-25","arxiv_id":"2506.21710","code_links":[],"syntology":null}},{"leaderboard":"/sota/visual-question-answering-on-msrvtt-qa-2","slug":"visual-question-answering-on-msrvtt-qa-2","dataset":"MSRVTT-QA","dataset_url":"/dataset/msrvtt-qa","rows_in_archive":4,"metrics":["Test Accuracy","Accuracy"],"first_row_in_archive_order":{"model":"Aurora (ours, r=64) Aurora (ours, r=64)","paper_title":null,"paper_url":null,"paper_date":"","arxiv_id":null,"code_links":[],"syntology":null}},{"leaderboard":"/sota/visual-question-answering-on-vqa-v2-val-1","slug":"visual-question-answering-on-vqa-v2-val-1","dataset":"VQA v2 val","dataset_url":"/dataset/visual-question-answering-v2-0","rows_in_archive":4,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"BLIP-2 ViT-G OPT 6.7B (fine-tuned)","paper_title":"BLIP-2: Bootstrapping Language-Image Pre-training with Frozen Image Encoders and Large Language Models","paper_url":"/paper/blip-2-bootstrapping-language-image-pre","paper_date":"2023-01-30","arxiv_id":"2301.12597","code_links":[{"title":"huggingface/transformers","url":"https://github.com/huggingface/transformers"},{"title":"salesforce/lavis","url":"https://github.com/salesforce/lavis"},{"title":"thudm/visualglm-6b","url":"https://github.com/thudm/visualglm-6b"},{"title":"baaivision/eva","url":"https://github.com/baaivision/eva"},{"title":"junshutang/Make-It-3D","url":"https://github.com/junshutang/Make-It-3D"},{"title":"facebookresearch/multimodal","url":"https://github.com/facebookresearch/multimodal"},{"title":"unispac/visual-adversarial-examples-jailbreak-large-language-models","url":"https://github.com/unispac/visual-adversarial-examples-jailbreak-large-language-models"},{"title":"yukw777/videoblip","url":"https://github.com/yukw777/videoblip"},{"title":"alibaba/graphtranslator","url":"https://github.com/alibaba/graphtranslator"},{"title":"gregor-ge/mblip","url":"https://github.com/gregor-ge/mblip"},{"title":"linzhiqiu/clip-flant5","url":"https://github.com/linzhiqiu/clip-flant5"},{"title":"kdr/videorag-mrr2024","url":"https://github.com/kdr/videorag-mrr2024"},{"title":"rabiulcste/vqazero","url":"https://github.com/rabiulcste/vqazero"},{"title":"jiwanchung/vlis","url":"https://github.com/jiwanchung/vlis"},{"title":"yangyucheng000/University","url":"https://github.com/yangyucheng000/University/tree/main/model-2/blip_2"},{"title":"2024-MindSpore-1/Code2","url":"https://github.com/2024-MindSpore-1/Code2/tree/main/model-1/blip_2"},{"title":"albertotestoni/ndq_visual_objects","url":"https://github.com/albertotestoni/ndq_visual_objects"}],"syntology":{"n":8,"n_ran":4,"n_constructed":4,"n_ran_checked":4,"n_instrument":0,"n_unverified":4,"n_honours":0,"n_violates":0,"n_no_contract":4,"n_pointer_only":1,"phrase":"4 ran (of which 4 constructed an object rather than computing a result; 4 with no instrument failure: 0 honoured, 0 violated, 4 with no contract checked; 0 where Syntology's instrument failed) · 4 unverified; every one of the 4 samples that ran constructed an object rather than computing a result","sample_list":"/paper/blip-2-bootstrapping-language-image-pre#ran","syntology_url":"https://syntology.ai/paper/2301.12597","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2301.12597"}},"official":{"repos":["salesforce/lavis"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}}},{"leaderboard":"/sota/visual-question-answering-on-vqa-v2-test-std-1","slug":"visual-question-answering-on-vqa-v2-test-std-1","dataset":"VQA v2 test-std","dataset_url":"/dataset/visual-question-answering-v2-0","rows_in_archive":3,"metrics":["Accuracy","number","other","overall","yes/no"],"first_row_in_archive_order":{"model":"LXMERT (low-magnitude pruning)","paper_title":"LXMERT Model Compression for Visual Question Answering","paper_url":"/paper/lxmert-model-compression-for-visual-question","paper_date":"2023-10-23","arxiv_id":"2310.15325","code_links":[{"title":"ghazaleh-mahmoodi/lxmert_compression","url":"https://github.com/ghazaleh-mahmoodi/lxmert_compression"},{"title":"pwc-1/Paper-9","url":"https://github.com/pwc-1/Paper-9/tree/main/lxmert"}],"syntology":null}},{"leaderboard":"/sota/visual-question-answering-on-mmhal-bench","slug":"visual-question-answering-on-mmhal-bench","dataset":"MMHal-Bench","dataset_url":null,"rows_in_archive":2,"metrics":["Hallucination Rate","Score"],"first_row_in_archive_order":{"model":"RLAIF-V 12B","paper_title":"RLAIF-V: Open-Source AI Feedback Leads to Super GPT-4V Trustworthiness","paper_url":"/paper/rlaif-v-aligning-mllms-through-open-source-ai","paper_date":"2024-05-27","arxiv_id":"2405.17220","code_links":[{"title":"openbmb/minicpm-v","url":"https://github.com/openbmb/minicpm-v"},{"title":"openbmb/omnilmm","url":"https://github.com/openbmb/omnilmm"},{"title":"OpenBMB/MiniCPM-o","url":"https://github.com/OpenBMB/MiniCPM-o"},{"title":"rlhf-v/rlaif-v","url":"https://github.com/rlhf-v/rlaif-v"},{"title":"rlhf-v/rlhf-v","url":"https://github.com/rlhf-v/rlhf-v"}],"syntology":{"n":22,"n_ran":17,"n_constructed":0,"n_ran_checked":7,"n_instrument":10,"n_unverified":5,"n_honours":2,"n_violates":3,"n_no_contract":2,"n_pointer_only":17,"phrase":"17 ran (of which 0 constructed an object rather than computing a result; 7 with no instrument failure: 2 honoured, 3 violated, 2 with no contract checked; 10 where Syntology's instrument failed) · 5 unverified","sample_list":"/paper/rlaif-v-aligning-mllms-through-open-source-ai#ran","syntology_url":"https://syntology.ai/paper/2405.17220","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2405.17220"}},"official":{"repos":["openbmb/omnilmm","rlhf-v/rlaif-v"],"state":"official (archive's flag): 11 ran","n_ran":11,"n_constructed":0,"n_ran_no_instrument_failure":5,"n_unverified":4,"ran_from_kinds":["listed","official"]}}}},{"leaderboard":"/sota/visual-question-answering-on-msvd-qa-2","slug":"visual-question-answering-on-msvd-qa-2","dataset":"MSVD-QA","dataset_url":"/dataset/msvd-qa","rows_in_archive":2,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"FrozenBiLM","paper_title":"Zero-Shot Video Question Answering via Frozen Bidirectional Language Models","paper_url":"/paper/zero-shot-video-question-answering-via-frozen","paper_date":"2022-06-16","arxiv_id":"2206.08155","code_links":[{"title":"antoyang/FrozenBiLM","url":"https://github.com/antoyang/FrozenBiLM"},{"title":"klauscc/dam","url":"https://github.com/klauscc/dam"},{"title":"sts-vlcc/sts-vlcc","url":"https://github.com/sts-vlcc/sts-vlcc"}],"syntology":{"n":34,"n_ran":14,"n_constructed":9,"n_ran_checked":14,"n_instrument":0,"n_unverified":20,"n_honours":1,"n_violates":1,"n_no_contract":12,"n_pointer_only":1,"phrase":"14 ran (of which 9 constructed an object rather than computing a result; 14 with no instrument failure: 1 honoured, 1 violated, 12 with no contract checked; 0 where Syntology's instrument failed) · 20 unverified","sample_list":"/paper/zero-shot-video-question-answering-via-frozen#ran","syntology_url":"https://syntology.ai/paper/2206.08155","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2206.08155"}},"official":{"repos":["antoyang/FrozenBiLM"],"state":"official: harvested, nothing ran","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":7,"ran_from_kinds":["listed"]}}}},{"leaderboard":"/sota/visual-question-answering-on-plotqa-d1-1","slug":"visual-question-answering-on-plotqa-d1-1","dataset":"PlotQA-D1","dataset_url":"/dataset/plotqa","rows_in_archive":2,"metrics":["1:1 Accuracy"],"first_row_in_archive_order":{"model":"MatCha4096 + LaMenDa","paper_title":"Synthesize Step-by-Step: Tools Templates and LLMs as Data Generators for Reasoning-Based Chart VQA","paper_url":"/paper/synthesize-step-by-step-tools-templates-and-1","paper_date":"2024-01-01","arxiv_id":null,"code_links":[],"syntology":null}},{"leaderboard":"/sota/visual-question-answering-on-plotqa-d2-1","slug":"visual-question-answering-on-plotqa-d2-1","dataset":"PlotQA-D2","dataset_url":"/dataset/plotqa","rows_in_archive":2,"metrics":["1:1 Accuracy"],"first_row_in_archive_order":{"model":"MatCha4096 + LaMenDa","paper_title":"Synthesize Step-by-Step: Tools Templates and LLMs as Data Generators for Reasoning-Based Chart VQA","paper_url":"/paper/synthesize-step-by-step-tools-templates-and-1","paper_date":"2024-01-01","arxiv_id":null,"code_links":[],"syntology":null}},{"leaderboard":"/sota/visual-question-answering-on-vqa-v2-1","slug":"visual-question-answering-on-vqa-v2-1","dataset":"VQA v2","dataset_url":"/dataset/visual-question-answering-v2-0","rows_in_archive":2,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"RLHF-V","paper_title":"RLHF-V: Towards Trustworthy MLLMs via Behavior Alignment from Fine-grained Correctional Human Feedback","paper_url":"/paper/rlhf-v-towards-trustworthy-mllms-via-behavior","paper_date":"2023-12-01","arxiv_id":"2312.00849","code_links":[{"title":"openbmb/minicpm-v","url":"https://github.com/openbmb/minicpm-v"},{"title":"rlhf-v/rlhf-v","url":"https://github.com/rlhf-v/rlhf-v"},{"title":"tidedra/vl-rlhf","url":"https://github.com/tidedra/vl-rlhf"},{"title":"exgc/r1v-free","url":"https://github.com/exgc/r1v-free"}],"syntology":null}},{"leaderboard":"/sota/visual-question-answering-on-aid-vqa","slug":"visual-question-answering-on-aid-vqa","dataset":"AID-VQA","dataset_url":null,"rows_in_archive":1,"metrics":["Acc. (test)"],"first_row_in_archive_order":{"model":"SkySense-O","paper_title":"SkySense: A Multi-Modal Remote Sensing Foundation Model Towards Universal Interpretation for Earth Observation Imagery","paper_url":"/paper/skysense-a-multi-modal-remote-sensing","paper_date":"2023-12-15","arxiv_id":"2312.10115","code_links":[{"title":"jack-bo1220/awesome-remote-sensing-foundation-models","url":"https://github.com/jack-bo1220/awesome-remote-sensing-foundation-models"}],"syntology":null}},{"leaderboard":"/sota/visual-question-answering-on-amber","slug":"visual-question-answering-on-amber","dataset":"AMBER","dataset_url":null,"rows_in_archive":1,"metrics":["Accuracy","F1"],"first_row_in_archive_order":{"model":"RLAIF-V 12B","paper_title":"RLAIF-V: Open-Source AI Feedback Leads to Super GPT-4V Trustworthiness","paper_url":"/paper/rlaif-v-aligning-mllms-through-open-source-ai","paper_date":"2024-05-27","arxiv_id":"2405.17220","code_links":[{"title":"openbmb/minicpm-v","url":"https://github.com/openbmb/minicpm-v"},{"title":"openbmb/omnilmm","url":"https://github.com/openbmb/omnilmm"},{"title":"OpenBMB/MiniCPM-o","url":"https://github.com/OpenBMB/MiniCPM-o"},{"title":"rlhf-v/rlaif-v","url":"https://github.com/rlhf-v/rlaif-v"},{"title":"rlhf-v/rlhf-v","url":"https://github.com/rlhf-v/rlhf-v"}],"syntology":{"n":22,"n_ran":17,"n_constructed":0,"n_ran_checked":7,"n_instrument":10,"n_unverified":5,"n_honours":2,"n_violates":3,"n_no_contract":2,"n_pointer_only":17,"phrase":"17 ran (of which 0 constructed an object rather than computing a result; 7 with no instrument failure: 2 honoured, 3 violated, 2 with no contract checked; 10 where Syntology's instrument failed) · 5 unverified","sample_list":"/paper/rlaif-v-aligning-mllms-through-open-source-ai#ran","syntology_url":"https://syntology.ai/paper/2405.17220","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2405.17220"}},"official":{"repos":["openbmb/omnilmm","rlhf-v/rlaif-v"],"state":"official (archive's flag): 11 ran","n_ran":11,"n_constructed":0,"n_ran_no_instrument_failure":5,"n_unverified":4,"ran_from_kinds":["listed","official"]}}}},{"leaderboard":"/sota/visual-question-answering-on-clevr-1","slug":"visual-question-answering-on-clevr-1","dataset":"CLEVR","dataset_url":"/dataset/clevr","rows_in_archive":1,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"NeSyCoCo Neuro-Symbolic","paper_title":"NeSyCoCo: A Neuro-Symbolic Concept Composer for Compositional Generalization","paper_url":"/paper/nesycoco-a-neuro-symbolic-concept-composer","paper_date":"2024-12-20","arxiv_id":"2412.15588","code_links":[{"title":"hlr/nesycoco","url":"https://github.com/hlr/nesycoco"}],"syntology":null}},{"leaderboard":"/sota/visual-question-answering-on-coco-visual-5","slug":"visual-question-answering-on-coco-visual-5","dataset":"COCO Visual Question Answering (VQA) real images 2.0 open ended","dataset_url":"/dataset/coco","rows_in_archive":1,"metrics":["Percentage correct"],"first_row_in_archive_order":{"model":"MaMMUT (2B)","paper_title":"MaMMUT: A Simple Architecture for Joint Learning for MultiModal Tasks","paper_url":"/paper/mammut-a-simple-architecture-for-joint","paper_date":"2023-03-29","arxiv_id":"2303.16839","code_links":[{"title":"lucidrains/mammut-pytorch","url":"https://github.com/lucidrains/mammut-pytorch"}],"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":3,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":3,"n_no_contract":0,"n_pointer_only":2,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 3 violated, 0 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/mammut-a-simple-architecture-for-joint#ran","syntology_url":"https://syntology.ai/paper/2303.16839","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2303.16839"}},"official":null}}},{"leaderboard":"/sota/visual-question-answering-on-earthvqa","slug":"visual-question-answering-on-earthvqa","dataset":"EarthVQA","dataset_url":"/dataset/earthvqa","rows_in_archive":1,"metrics":["Overall Accuracy"],"first_row_in_archive_order":{"model":"SOBA","paper_title":"EarthVQA: Towards Queryable Earth via Relational Reasoning-Based Remote Sensing Visual Question Answering","paper_url":"/paper/earthvqa-towards-queryable-earth-via","paper_date":"2023-12-19","arxiv_id":"2312.12222","code_links":[{"title":"Junjue-Wang/EarthVQA","url":"https://github.com/Junjue-Wang/EarthVQA"}],"syntology":{"n":6,"n_ran":6,"n_constructed":0,"n_ran_checked":3,"n_instrument":3,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":2,"n_pointer_only":6,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 1 honoured, 0 violated, 2 with no contract checked; 3 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/earthvqa-towards-queryable-earth-via#ran","syntology_url":"https://syntology.ai/paper/2312.12222","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2312.12222"}},"official":{"repos":["Junjue-Wang/EarthVQA"],"state":"official (archive's flag): 6 ran","n_ran":6,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":0,"ran_from_kinds":["official"]}}}},{"leaderboard":"/sota/visual-question-answering-on-gqa-1","slug":"visual-question-answering-on-gqa-1","dataset":"GQA","dataset_url":"/dataset/gqa","rows_in_archive":1,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"LocVLM-L","paper_title":"Learning to Localize Objects Improves Spatial Reasoning in Visual-LLMs","paper_url":"/paper/learning-to-localize-objects-improves-spatial","paper_date":"2024-04-11","arxiv_id":"2404.07449","code_links":[{"title":"kahnchana/locvlm","url":"https://github.com/kahnchana/locvlm"}],"syntology":null}},{"leaderboard":"/sota/visual-question-answering-on-grit-1","slug":"visual-question-answering-on-grit-1","dataset":"GRIT","dataset_url":"/dataset/grit","rows_in_archive":1,"metrics":["VQA (ablation)"],"first_row_in_archive_order":{"model":"OFA","paper_title":"OFA: Unifying Architectures, Tasks, and Modalities Through a Simple Sequence-to-Sequence Learning Framework","paper_url":"/paper/unifying-architectures-tasks-and-modalities","paper_date":"2022-02-07","arxiv_id":"2202.03052","code_links":[{"title":"modelscope/modelscope","url":"https://github.com/modelscope/modelscope"},{"title":"ofa-sys/ofa","url":"https://github.com/ofa-sys/ofa"},{"title":"JHKim-snu/GVCCI","url":"https://github.com/JHKim-snu/GVCCI"},{"title":"JHKim-snu/PGA","url":"https://github.com/JHKim-snu/PGA"}],"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/unifying-architectures-tasks-and-modalities#ran","syntology_url":"https://syntology.ai/paper/2202.03052","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2202.03052"}},"official":{"repos":["ofa-sys/ofa"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}}},{"leaderboard":"/sota/visual-question-answering-on-mapeval-visual","slug":"visual-question-answering-on-mapeval-visual","dataset":"MapEval-Visual","dataset_url":"/dataset/mapeval-visual","rows_in_archive":1,"metrics":["Accuracy (% )"],"first_row_in_archive_order":{"model":"Claude-3.5-Sonnet","paper_title":"MapEval: A Map-Based Evaluation of Geo-Spatial Reasoning in Foundation Models","paper_url":"/paper/mapeval-a-map-based-evaluation-of-geo-spatial","paper_date":"2024-12-31","arxiv_id":"2501.00316","code_links":[{"title":"MapEval/MapEval-Visual","url":"https://github.com/MapEval/MapEval-Visual"},{"title":"MapEval/MapEval-API","url":"https://github.com/MapEval/MapEval-API"},{"title":"MapEval/MapEval-Textual","url":"https://github.com/MapEval/MapEval-Textual"}],"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":0,"n_instrument":3,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":3,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 3 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/mapeval-a-map-based-evaluation-of-geo-spatial#ran","syntology_url":"https://syntology.ai/paper/2501.00316","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2501.00316"}},"official":{"repos":["MapEval/MapEval-API","MapEval/MapEval-Textual","MapEval/MapEval-Visual"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}}},{"leaderboard":"/sota/visual-question-answering-on-mm-vet-w-o","slug":"visual-question-answering-on-mm-vet-w-o","dataset":"MM-Vet (w/o External Tools)","dataset_url":null,"rows_in_archive":1,"metrics":["GPT-4 score"],"first_row_in_archive_order":{"model":"Emu-14B","paper_title":"Emu: Generative Pretraining in Multimodality","paper_url":"/paper/generative-pretraining-in-multimodality","paper_date":"2023-07-11","arxiv_id":"2307.05222","code_links":[{"title":"baaivision/emu","url":"https://github.com/baaivision/emu"},{"title":"doc-doc/NExT-OE","url":"https://github.com/doc-doc/NExT-OE"}],"syntology":{"n":2,"n_ran":0,"n_constructed":0,"n_ran_checked":0,"n_instrument":0,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"0 ran · 2 unverified","sample_list":"/paper/generative-pretraining-in-multimodality#ran","syntology_url":"https://syntology.ai/paper/2307.05222","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2307.05222"}},"official":{"repos":["baaivision/emu"],"state":"official: harvested for another paper","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":[]}}}},{"leaderboard":"/sota/visual-question-answering-on-rsvqa-hr","slug":"visual-question-answering-on-rsvqa-hr","dataset":"RSVQA-HR","dataset_url":null,"rows_in_archive":1,"metrics":["zero-shot Acc"],"first_row_in_archive_order":{"model":"SkySense-O","paper_title":"SkySense: A Multi-Modal Remote Sensing Foundation Model Towards Universal Interpretation for Earth Observation Imagery","paper_url":"/paper/skysense-a-multi-modal-remote-sensing","paper_date":"2023-12-15","arxiv_id":"2312.10115","code_links":[{"title":"jack-bo1220/awesome-remote-sensing-foundation-models","url":"https://github.com/jack-bo1220/awesome-remote-sensing-foundation-models"}],"syntology":null}},{"leaderboard":"/sota/visual-question-answering-on-siri-whu","slug":"visual-question-answering-on-siri-whu","dataset":"SIRI-WHU","dataset_url":null,"rows_in_archive":1,"metrics":["Acc. (test)"],"first_row_in_archive_order":{"model":"SkySense-O","paper_title":"SkySense: A Multi-Modal Remote Sensing Foundation Model Towards Universal Interpretation for Earth Observation Imagery","paper_url":"/paper/skysense-a-multi-modal-remote-sensing","paper_date":"2023-12-15","arxiv_id":"2312.10115","code_links":[{"title":"jack-bo1220/awesome-remote-sensing-foundation-models","url":"https://github.com/jack-bo1220/awesome-remote-sensing-foundation-models"}],"syntology":null}},{"leaderboard":"/sota/visual-question-answering-on-textvqa-test-2","slug":"visual-question-answering-on-textvqa-test-2","dataset":"TextVQA test-standard","dataset_url":"/dataset/textvqa","rows_in_archive":1,"metrics":["overall"],"first_row_in_archive_order":{"model":"PromptCap","paper_title":"PromptCap: Prompt-Guided Task-Aware Image Captioning","paper_url":"/paper/promptcap-prompt-guided-task-aware-image","paper_date":"2022-11-15","arxiv_id":"2211.09699","code_links":[{"title":"Yushi-Hu/PromptCap","url":"https://github.com/Yushi-Hu/PromptCap"}],"syntology":null}},{"leaderboard":"/sota/visual-question-answering-on-visualmrc","slug":"visual-question-answering-on-visualmrc","dataset":"VisualMRC","dataset_url":"/dataset/visualmrc","rows_in_archive":1,"metrics":["CIDEr"],"first_row_in_archive_order":{"model":"LayoutT5 (Large)","paper_title":"VisualMRC: Machine Reading Comprehension on Document Images","paper_url":"/paper/visualmrc-machine-reading-comprehension-on","paper_date":"2021-01-27","arxiv_id":"2101.11272","code_links":[{"title":"nttmdlab-nlp/VisualMRC","url":"https://github.com/nttmdlab-nlp/VisualMRC"}],"syntology":null}},{"leaderboard":"/sota/visual-question-answering-on-vizwiz-1","slug":"visual-question-answering-on-vizwiz-1","dataset":"VizWiz","dataset_url":"/dataset/vizwiz","rows_in_archive":1,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"Emu-I *","paper_title":"Emu: Generative Pretraining in Multimodality","paper_url":"/paper/generative-pretraining-in-multimodality","paper_date":"2023-07-11","arxiv_id":"2307.05222","code_links":[{"title":"baaivision/emu","url":"https://github.com/baaivision/emu"},{"title":"doc-doc/NExT-OE","url":"https://github.com/doc-doc/NExT-OE"}],"syntology":{"n":2,"n_ran":0,"n_constructed":0,"n_ran_checked":0,"n_instrument":0,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"0 ran · 2 unverified","sample_list":"/paper/generative-pretraining-in-multimodality#ran","syntology_url":"https://syntology.ai/paper/2307.05222","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2307.05222"}},"official":{"repos":["baaivision/emu"],"state":"official: harvested for another paper","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":[]}}}}],"datasets":[{"url":"/dataset/coco","name":"COCO (Common Objects in Context)","full_name":"Common Objects in Context","num_papers_in_archive":11922},{"url":"/dataset/gqa","name":"GQA","full_name":"GQA","num_papers_in_archive":749},{"url":"/dataset/clevr","name":"CLEVR","full_name":"Compositional Language and Elementary Visual Reasoning","num_papers_in_archive":657},{"url":"/dataset/textvqa","name":"TextVQA","full_name":"","num_papers_in_archive":476},{"url":"/dataset/mmbench","name":"MMBench","full_name":"","num_papers_in_archive":384},{"url":"/dataset/visual-question-answering-v2-0","name":"Visual Question Answering v2.0","full_name":"VQA v2.0","num_papers_in_archive":366},{"url":"/dataset/mm-vet","name":"MM-Vet","full_name":"","num_papers_in_archive":339},{"url":"/dataset/vizwiz","name":"VizWiz","full_name":"VizWiz-VQA","num_papers_in_archive":260},{"url":"/dataset/mathvista","name":"MathVista","full_name":"Mathematical Reasoning of in Visual Contexts","num_papers_in_archive":242},{"url":"/dataset/msrvtt-qa","name":"MSRVTT-QA","full_name":"","num_papers_in_archive":66},{"url":"/dataset/msvd-qa","name":"MSVD-QA","full_name":"","num_papers_in_archive":61},{"url":"/dataset/plotqa","name":"PlotQA","full_name":"","num_papers_in_archive":50},{"url":"/dataset/visualmrc","name":"VisualMRC","full_name":"VisualMRC: Machine Reading Comprehension on Document Images","num_papers_in_archive":36},{"url":"/dataset/mm-vet-v2","name":"MM-Vet v2","full_name":"","num_papers_in_archive":17},{"url":"/dataset/grit","name":"GRIT","full_name":"General Robust Image Task Benchmark","num_papers_in_archive":16},{"url":"/dataset/benchlmm","name":"BenchLMM","full_name":"BenchLMM: Benchmarking Cross-style Visual Capability of Large Multimodal Models","num_papers_in_archive":12},{"url":"/dataset/vip-bench","name":"ViP-Bench","full_name":"Making Large Multimodal Models Understand Arbitrary Visual Prompts","num_papers_in_archive":10},{"url":"/dataset/scigraphqa","name":"SciGraphQA","full_name":"","num_papers_in_archive":8},{"url":"/dataset/vnhsge","name":"VNHSGE","full_name":"VietNamese High School Graduation Examination Dataset for Large Language Models","num_papers_in_archive":7},{"url":"/dataset/earthvqa","name":"EarthVQA","full_name":"A multi-modal multi-task VQA dataset for remote sensing","num_papers_in_archive":6},{"url":"/dataset/skyeye-968k","name":"SkyEye-968k","full_name":"","num_papers_in_archive":5},{"url":"/dataset/uit-evjvqa","name":"EVJVQA","full_name":"English-Japanese-Vietnamese Visual Question Answering","num_papers_in_archive":4},{"url":"/dataset/openvivqa","name":"OpenViVQA","full_name":"Open-domain Visual Question Answering in Vietnamese","num_papers_in_archive":4},{"url":"/dataset/kvasir-vqa","name":"Kvasir-VQA","full_name":"A Text-Image Pair GI Tract Dataset","num_papers_in_archive":3},{"url":"/dataset/cii-bench","name":"CII-Bench","full_name":"Chinese Image Implication understanding Benchmark","num_papers_in_archive":2},{"url":"/dataset/climateiqa","name":"ClimateIQA","full_name":"","num_papers_in_archive":2},{"url":"/dataset/gqa-ood","name":"GQA-OOD","full_name":"","num_papers_in_archive":2},{"url":"/dataset/mapeval-visual","name":"MapEval-Visual","full_name":"","num_papers_in_archive":1},{"url":"/dataset/mminstruct-gpt4v","name":"MMInstruct-GPT4V","full_name":"MMInstruct: A High-Quality Multi-Modal Instruction Tuning Dataset with Extensive Diversity","num_papers_in_archive":1},{"url":"/dataset/ubench","name":"uBench","full_name":"MicroBench","num_papers_in_archive":1},{"url":"/dataset/vilco","name":"ViLCo","full_name":"ViLCo-Bench","num_papers_in_archive":1},{"url":"/dataset/worldcuisines","name":"WorldCuisines","full_name":"","num_papers_in_archive":1}],"subtasks":[{"url":"/task/explanatory-visual-question-answering","name":"Explanatory Visual Question Answering"},{"url":"/task/mm-vet-v2","name":"MM-Vet v2"},{"url":"/task/object-hallucination","name":"Object Hallucination"},{"url":"/task/spatial-reasoning","name":"Spatial Reasoning"},{"url":"/task/vietnamese-visual-question-answering","name":"Vietnamese Visual Question Answering"}],"parent_tasks":[{"url":"/task/visual-question-answering","name":"Visual Question Answering (VQA)"}],"papers":{"order":"repositories listed in the archive (most first), then date (newest first), then slug; the archive holds no stars","population":"every paper the archive tags with this task (distinct papers; spam titles not listed). 'of' counts the papers listed; until 2026-09-25 it counted only the papers with a code link, and that number is of_with_code","page":1,"pages_in_order":22,"rows_per_page":100,"shown":100,"of":2177,"of_with_code":1042,"tagged_in_all":2177,"counts":{"archive_papers_tagged":2177,"with_a_code_link":1042,"where_syntology_ran_a_sample":378,"not_listed_spam_title":0,"listed":2177,"listed_where_code_ran":378,"where_syntology_ran_a_sample_split":{"with_a_run_with_no_instrument_failure":292,"every_run_a_failure_of_syntologys_instrument":86,"listed_with_a_run_with_no_instrument_failure":292,"listed_every_run_a_failure_of_syntologys_instrument":86,"filter":{"states":["a run with no instrument failure","any run, instrument failures included"],"default":"a run with no instrument failure","note":"on the 'only where code ran' pages the default hides, in the browser, the rows where every run was a failure of Syntology's instrument; the second state shows them again. Rows are hidden, never re-ordered; these twins list every row"}},"definition":"distinct papers the archive tags; 'where Syntology ran a sample' counts papers with at least one harvested sample that ran, which is not a correctness claim"},"items":[{"url":"/paper/grad-cam-visual-explanations-from-deep","slug":"grad-cam-visual-explanations-from-deep","title":"Grad-CAM: Visual Explanations from Deep Networks via Gradient-based Localization","date":"2016-10-07","arxiv_id":"1610.02391","repositories_listed":126,"syntology":{"n":141,"n_ran":90,"n_constructed":32,"n_ran_checked":62,"n_instrument":28,"n_unverified":51,"n_honours":6,"n_violates":4,"n_no_contract":52,"n_pointer_only":69,"phrase":"90 ran (of which 32 constructed an object rather than computing a result; 62 with no instrument failure: 6 honoured, 4 violated, 52 with no contract checked; 28 where Syntology's instrument failed) · 51 unverified","sample_list":"/paper/grad-cam-visual-explanations-from-deep#ran","syntology_url":"https://syntology.ai/paper/1610.02391","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1610.02391"}},"official":{"repos":["ramprs/grad-cam"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/bottom-up-and-top-down-attention-for-image","slug":"bottom-up-and-top-down-attention-for-image","title":"Bottom-Up and Top-Down Attention for Image Captioning and Visual Question Answering","date":"2017-07-25","arxiv_id":"1707.07998","repositories_listed":65,"syntology":{"n":9,"n_ran":9,"n_constructed":0,"n_ran_checked":1,"n_instrument":8,"n_unverified":0,"n_honours":0,"n_violates":1,"n_no_contract":0,"n_pointer_only":6,"phrase":"9 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 1 violated, 0 with no contract checked; 8 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/bottom-up-and-top-down-attention-for-image#ran","syntology_url":"https://syntology.ai/paper/1707.07998","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1707.07998"}},"official":{"repos":["peteanderson80/bottom-up-attention"],"state":"official: no sample here; runs from other or unrecorded repositories","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed","unlocated"]}}},{"url":"/paper/vqa-visual-question-answering","slug":"vqa-visual-question-answering","title":"VQA: Visual Question Answering","date":"2015-05-03","arxiv_id":"1505.00468","repositories_listed":21,"syntology":{"n":7,"n_ran":6,"n_constructed":0,"n_ran_checked":2,"n_instrument":4,"n_unverified":1,"n_honours":2,"n_violates":0,"n_no_contract":0,"n_pointer_only":6,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 2 honoured, 0 violated, 0 with no contract checked; 4 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/vqa-visual-question-answering#ran","syntology_url":"https://syntology.ai/paper/1505.00468","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1505.00468"}},"official":null}},{"url":"/paper/a-simple-neural-network-module-for-relational","slug":"a-simple-neural-network-module-for-relational","title":"A simple neural network module for relational reasoning","date":"2017-06-05","arxiv_id":"1706.01427","repositories_listed":20,"syntology":{"n":7,"n_ran":3,"n_constructed":0,"n_ran_checked":3,"n_instrument":0,"n_unverified":4,"n_honours":2,"n_violates":1,"n_no_contract":0,"n_pointer_only":1,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 2 honoured, 1 violated, 0 with no contract checked; 0 where Syntology's instrument failed) · 4 unverified","sample_list":"/paper/a-simple-neural-network-module-for-relational#ran","syntology_url":"https://syntology.ai/paper/1706.01427","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1706.01427"}},"official":null}},{"url":"/paper/blip-2-bootstrapping-language-image-pre","slug":"blip-2-bootstrapping-language-image-pre","title":"BLIP-2: Bootstrapping Language-Image Pre-training with Frozen Image Encoders and Large Language Models","date":"2023-01-30","arxiv_id":"2301.12597","repositories_listed":17,"syntology":{"n":8,"n_ran":4,"n_constructed":4,"n_ran_checked":4,"n_instrument":0,"n_unverified":4,"n_honours":0,"n_violates":0,"n_no_contract":4,"n_pointer_only":1,"phrase":"4 ran (of which 4 constructed an object rather than computing a result; 4 with no instrument failure: 0 honoured, 0 violated, 4 with no contract checked; 0 where Syntology's instrument failed) · 4 unverified; every one of the 4 samples that ran constructed an object rather than computing a result","sample_list":"/paper/blip-2-bootstrapping-language-image-pre#ran","syntology_url":"https://syntology.ai/paper/2301.12597","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2301.12597"}},"official":{"repos":["salesforce/lavis"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/visual-instruction-tuning-1","slug":"visual-instruction-tuning-1","title":"Visual Instruction Tuning","date":"2023-04-17","arxiv_id":"2304.08485","repositories_listed":13,"syntology":{"n":51,"n_ran":16,"n_constructed":6,"n_ran_checked":8,"n_instrument":8,"n_unverified":35,"n_honours":0,"n_violates":1,"n_no_contract":7,"n_pointer_only":0,"phrase":"16 ran (of which 6 constructed an object rather than computing a result; 8 with no instrument failure: 0 honoured, 1 violated, 7 with no contract checked; 8 where Syntology's instrument failed) · 35 unverified","sample_list":"/paper/visual-instruction-tuning-1#ran","syntology_url":"https://syntology.ai/paper/2304.08485","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2304.08485"}},"official":{"repos":["haotian-liu/LLaVA"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":1,"n_ran_no_instrument_failure":1,"n_unverified":8,"ran_from_kinds":["community","listed","named_in_paper","official"]}}},{"url":"/paper/show-ask-attend-and-answer-a-strong-baseline","slug":"show-ask-attend-and-answer-a-strong-baseline","title":"Show, Ask, Attend, and Answer: A Strong Baseline For Visual Question Answering","date":"2017-04-11","arxiv_id":"1704.03162","repositories_listed":13,"syntology":{"n":9,"n_ran":9,"n_constructed":0,"n_ran_checked":2,"n_instrument":7,"n_unverified":0,"n_honours":1,"n_violates":1,"n_no_contract":0,"n_pointer_only":9,"phrase":"9 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 1 honoured, 1 violated, 0 with no contract checked; 7 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/show-ask-attend-and-answer-a-strong-baseline#ran","syntology_url":"https://syntology.ai/paper/1704.03162","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1704.03162"}},"official":null}},{"url":"/paper/gpt-4-technical-report-1","slug":"gpt-4-technical-report-1","title":"GPT-4 Technical Report","date":"2023-03-15","arxiv_id":"2303.08774","repositories_listed":11,"syntology":{"n":5,"n_ran":5,"n_constructed":0,"n_ran_checked":5,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":2,"n_no_contract":3,"n_pointer_only":1,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 0 honoured, 2 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/gpt-4-technical-report-1#ran","syntology_url":"https://syntology.ai/paper/2303.08774","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2303.08774"}},"official":{"repos":["openai/evals"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/vilbert-pretraining-task-agnostic","slug":"vilbert-pretraining-task-agnostic","title":"ViLBERT: Pretraining Task-Agnostic Visiolinguistic Representations for Vision-and-Language Tasks","date":"2019-08-06","arxiv_id":"1908.02265","repositories_listed":11,"syntology":{"n":34,"n_ran":10,"n_constructed":6,"n_ran_checked":8,"n_instrument":2,"n_unverified":24,"n_honours":0,"n_violates":0,"n_no_contract":8,"n_pointer_only":34,"phrase":"10 ran (of which 6 constructed an object rather than computing a result; 8 with no instrument failure: 0 honoured, 0 violated, 8 with no contract checked; 2 where Syntology's instrument failed) · 24 unverified","sample_list":"/paper/vilbert-pretraining-task-agnostic#ran","syntology_url":"https://syntology.ai/paper/1908.02265","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1908.02265"}},"official":null}},{"url":"/paper/hadamard-product-for-low-rank-bilinear","slug":"hadamard-product-for-low-rank-bilinear","title":"Hadamard Product for Low-rank Bilinear Pooling","date":"2016-10-14","arxiv_id":"1610.04325","repositories_listed":11,"syntology":null},{"url":"/paper/tips-and-tricks-for-visual-question-answering","slug":"tips-and-tricks-for-visual-question-answering","title":"Tips and Tricks for Visual Question Answering: Learnings from the 2017 Challenge","date":"2017-08-09","arxiv_id":"1708.02711","repositories_listed":10,"syntology":null},{"url":"/paper/multimodal-compact-bilinear-pooling-for","slug":"multimodal-compact-bilinear-pooling-for","title":"Multimodal Compact Bilinear Pooling for Visual Question Answering and Visual Grounding","date":"2016-06-06","arxiv_id":"1606.01847","repositories_listed":10,"syntology":null},{"url":"/paper/dynamic-memory-networks-for-visual-and","slug":"dynamic-memory-networks-for-visual-and","title":"Dynamic Memory Networks for Visual and Textual Question Answering","date":"2016-03-04","arxiv_id":"1603.01417","repositories_listed":10,"syntology":{"n":7,"n_ran":7,"n_constructed":0,"n_ran_checked":0,"n_instrument":7,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":7,"phrase":"7 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 7 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/dynamic-memory-networks-for-visual-and#ran","syntology_url":"https://syntology.ai/paper/1603.01417","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1603.01417"}},"official":null}},{"url":"/paper/improved-baselines-with-visual-instruction","slug":"improved-baselines-with-visual-instruction","title":"Improved Baselines with Visual Instruction Tuning","date":"2023-10-05","arxiv_id":"2310.03744","repositories_listed":9,"syntology":{"n":9,"n_ran":6,"n_constructed":0,"n_ran_checked":3,"n_instrument":3,"n_unverified":3,"n_honours":0,"n_violates":3,"n_no_contract":0,"n_pointer_only":8,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 3 violated, 0 with no contract checked; 3 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/improved-baselines-with-visual-instruction#ran","syntology_url":"https://syntology.ai/paper/2310.03744","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2310.03744"}},"official":null}},{"url":"/paper/layoutlmv2-multi-modal-pre-training-for","slug":"layoutlmv2-multi-modal-pre-training-for","title":"LayoutLMv2: Multi-modal Pre-training for Visually-Rich Document Understanding","date":"2020-12-29","arxiv_id":"2012.14740","repositories_listed":9,"syntology":null},{"url":"/paper/lxmert-learning-cross-modality-encoder","slug":"lxmert-learning-cross-modality-encoder","title":"LXMERT: Learning Cross-Modality Encoder Representations from Transformers","date":"2019-08-20","arxiv_id":"1908.07490","repositories_listed":9,"syntology":{"n":15,"n_ran":4,"n_constructed":4,"n_ran_checked":4,"n_instrument":0,"n_unverified":11,"n_honours":0,"n_violates":0,"n_no_contract":4,"n_pointer_only":3,"phrase":"4 ran (of which 4 constructed an object rather than computing a result; 4 with no instrument failure: 0 honoured, 0 violated, 4 with no contract checked; 0 where Syntology's instrument failed) · 11 unverified; every one of the 4 samples that ran constructed an object rather than computing a result","sample_list":"/paper/lxmert-learning-cross-modality-encoder#ran","syntology_url":"https://syntology.ai/paper/1908.07490","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1908.07490"}},"official":{"repos":["airsplay/lxmert"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/hierarchical-question-image-co-attention-for","slug":"hierarchical-question-image-co-attention-for","title":"Hierarchical Question-Image Co-Attention for Visual Question Answering","date":"2016-05-31","arxiv_id":"1606.00061","repositories_listed":9,"syntology":{"n":7,"n_ran":4,"n_constructed":0,"n_ran_checked":3,"n_instrument":1,"n_unverified":3,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":1,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 1 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/hierarchical-question-image-co-attention-for#ran","syntology_url":"https://syntology.ai/paper/1606.00061","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1606.00061"}},"official":{"repos":["jiasenlu/HieCoAttenVQA"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/qwen2-vl-enhancing-vision-language-model-s","slug":"qwen2-vl-enhancing-vision-language-model-s","title":"Qwen2-VL: Enhancing Vision-Language Model's Perception of the World at Any Resolution","date":"2024-09-18","arxiv_id":"2409.12191","repositories_listed":8,"syntology":{"n":12,"n_ran":12,"n_constructed":1,"n_ran_checked":10,"n_instrument":2,"n_unverified":0,"n_honours":4,"n_violates":1,"n_no_contract":5,"n_pointer_only":1,"phrase":"12 ran (of which 1 constructed an object rather than computing a result; 10 with no instrument failure: 4 honoured, 1 violated, 5 with no contract checked; 2 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/qwen2-vl-enhancing-vision-language-model-s#ran","syntology_url":"https://syntology.ai/paper/2409.12191","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2409.12191"}},"official":{"repos":["qwenlm/qwen2-vl"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":4,"n_unverified":0,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/bilinear-attention-networks","slug":"bilinear-attention-networks","title":"Bilinear Attention Networks","date":"2018-05-21","arxiv_id":"1805.07932","repositories_listed":8,"syntology":{"n":13,"n_ran":13,"n_constructed":0,"n_ran_checked":13,"n_instrument":0,"n_unverified":0,"n_honours":1,"n_violates":3,"n_no_contract":9,"n_pointer_only":3,"phrase":"13 ran (of which 0 constructed an object rather than computing a result; 13 with no instrument failure: 1 honoured, 3 violated, 9 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/bilinear-attention-networks#ran","syntology_url":"https://syntology.ai/paper/1805.07932","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1805.07932"}},"official":{"repos":["jnhwkim/ban-vqa"],"state":"official (archive's flag): 7 ran","n_ran":7,"n_constructed":0,"n_ran_no_instrument_failure":7,"n_unverified":0,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/vision-and-language-navigation-interpreting","slug":"vision-and-language-navigation-interpreting","title":"Vision-and-Language Navigation: Interpreting visually-grounded navigation instructions in real environments","date":"2017-11-20","arxiv_id":"1711.07280","repositories_listed":8,"syntology":null},{"url":"/paper/floodnet-a-high-resolution-aerial-imagery","slug":"floodnet-a-high-resolution-aerial-imagery","title":"FloodNet: A High Resolution Aerial Imagery Dataset for Post Flood Scene Understanding","date":"2020-12-05","arxiv_id":"2012.02951","repositories_listed":7,"syntology":null},{"url":"/paper/uniter-learning-universal-image-text-1","slug":"uniter-learning-universal-image-text-1","title":"UNITER: UNiversal Image-TExt Representation Learning","date":"2019-09-25","arxiv_id":"1909.11740","repositories_listed":7,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":0,"n_instrument":3,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":2,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 3 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/uniter-learning-universal-image-text-1#ran","syntology_url":"https://syntology.ai/paper/1909.11740","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1909.11740"}},"official":{"repos":["ChenRocks/UNITER"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/deep-modular-co-attention-networks-for-visual-1","slug":"deep-modular-co-attention-networks-for-visual-1","title":"Deep Modular Co-Attention Networks for Visual Question Answering","date":"2019-06-25","arxiv_id":"1906.10770","repositories_listed":7,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/deep-modular-co-attention-networks-for-visual-1#ran","syntology_url":"https://syntology.ai/paper/1906.10770","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1906.10770"}},"official":{"repos":["MILVLG/mcan-vqa"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/learning-cooperative-visual-dialog-agents","slug":"learning-cooperative-visual-dialog-agents","title":"Learning Cooperative Visual Dialog Agents with Deep Reinforcement Learning","date":"2017-03-20","arxiv_id":"1703.06585","repositories_listed":7,"syntology":{"n":4,"n_ran":4,"n_constructed":0,"n_ran_checked":2,"n_instrument":2,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":1,"n_pointer_only":1,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 1 honoured, 0 violated, 1 with no contract checked; 2 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/learning-cooperative-visual-dialog-agents#ran","syntology_url":"https://syntology.ai/paper/1703.06585","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1703.06585"}},"official":null}},{"url":"/paper/making-the-v-in-vqa-matter-elevating-the-role","slug":"making-the-v-in-vqa-matter-elevating-the-role","title":"Making the V in VQA Matter: Elevating the Role of Image Understanding in Visual Question Answering","date":"2016-12-02","arxiv_id":"1612.00837","repositories_listed":7,"syntology":null},{"url":"/paper/simple-baseline-for-visual-question-answering","slug":"simple-baseline-for-visual-question-answering","title":"Simple Baseline for Visual Question Answering","date":"2015-12-07","arxiv_id":"1512.02167","repositories_listed":7,"syntology":null},{"url":"/paper/knowledge-graphs-meet-multi-modal-learning-a","slug":"knowledge-graphs-meet-multi-modal-learning-a","title":"Knowledge Graphs Meet Multi-Modal Learning: A Comprehensive Survey","date":"2024-02-08","arxiv_id":"2402.05391","repositories_listed":6,"syntology":{"n":18,"n_ran":15,"n_constructed":0,"n_ran_checked":10,"n_instrument":5,"n_unverified":3,"n_honours":0,"n_violates":2,"n_no_contract":8,"n_pointer_only":4,"phrase":"15 ran (of which 0 constructed an object rather than computing a result; 10 with no instrument failure: 0 honoured, 2 violated, 8 with no contract checked; 5 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/knowledge-graphs-meet-multi-modal-learning-a#ran","syntology_url":"https://syntology.ai/paper/2402.05391","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2402.05391"}},"official":{"repos":["zjukg/kg-mm-survey"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/video-llava-learning-united-visual-1","slug":"video-llava-learning-united-visual-1","title":"Video-LLaVA: Learning United Visual Representation by Alignment Before Projection","date":"2023-11-16","arxiv_id":"2311.10122","repositories_listed":6,"syntology":{"n":7,"n_ran":5,"n_constructed":0,"n_ran_checked":5,"n_instrument":0,"n_unverified":2,"n_honours":2,"n_violates":0,"n_no_contract":3,"n_pointer_only":2,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 2 honoured, 0 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/video-llava-learning-united-visual-1#ran","syntology_url":"https://syntology.ai/paper/2311.10122","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2311.10122"}},"official":{"repos":["PKU-YuanGroup/Video-LLaVA"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/minigpt-4-enhancing-vision-language","slug":"minigpt-4-enhancing-vision-language","title":"MiniGPT-4: Enhancing Vision-Language Understanding with Advanced Large Language Models","date":"2023-04-20","arxiv_id":"2304.10592","repositories_listed":6,"syntology":null},{"url":"/paper/coca-contrastive-captioners-are-image-text","slug":"coca-contrastive-captioners-are-image-text","title":"CoCa: Contrastive Captioners are Image-Text Foundation Models","date":"2022-05-04","arxiv_id":"2205.01917","repositories_listed":6,"syntology":{"n":17,"n_ran":10,"n_constructed":5,"n_ran_checked":9,"n_instrument":1,"n_unverified":7,"n_honours":2,"n_violates":0,"n_no_contract":7,"n_pointer_only":2,"phrase":"10 ran (of which 5 constructed an object rather than computing a result; 9 with no instrument failure: 2 honoured, 0 violated, 7 with no contract checked; 1 where Syntology's instrument failed) · 7 unverified","sample_list":"/paper/coca-contrastive-captioners-are-image-text#ran","syntology_url":"https://syntology.ai/paper/2205.01917","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2205.01917"}},"official":null}},{"url":"/paper/multi-modal-factorized-bilinear-pooling-with","slug":"multi-modal-factorized-bilinear-pooling-with","title":"Multi-modal Factorized Bilinear Pooling with Co-Attention Learning for Visual Question Answering","date":"2017-08-04","arxiv_id":"1708.01471","repositories_listed":6,"syntology":null},{"url":"/paper/mutan-multimodal-tucker-fusion-for-visual","slug":"mutan-multimodal-tucker-fusion-for-visual","title":"MUTAN: Multimodal Tucker Fusion for Visual Question Answering","date":"2017-05-18","arxiv_id":"1705.06676","repositories_listed":6,"syntology":null},{"url":"/paper/rlaif-v-aligning-mllms-through-open-source-ai","slug":"rlaif-v-aligning-mllms-through-open-source-ai","title":"RLAIF-V: Open-Source AI Feedback Leads to Super GPT-4V Trustworthiness","date":"2024-05-27","arxiv_id":"2405.17220","repositories_listed":5,"syntology":{"n":22,"n_ran":17,"n_constructed":0,"n_ran_checked":7,"n_instrument":10,"n_unverified":5,"n_honours":2,"n_violates":3,"n_no_contract":2,"n_pointer_only":17,"phrase":"17 ran (of which 0 constructed an object rather than computing a result; 7 with no instrument failure: 2 honoured, 3 violated, 2 with no contract checked; 10 where Syntology's instrument failed) · 5 unverified","sample_list":"/paper/rlaif-v-aligning-mllms-through-open-source-ai#ran","syntology_url":"https://syntology.ai/paper/2405.17220","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2405.17220"}},"official":{"repos":["openbmb/omnilmm","rlhf-v/rlaif-v"],"state":"official (archive's flag): 11 ran","n_ran":11,"n_constructed":0,"n_ran_no_instrument_failure":5,"n_unverified":4,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/large-scale-domain-specific-pretraining-for","slug":"large-scale-domain-specific-pretraining-for","title":"BiomedCLIP: a multimodal biomedical foundation model pretrained from fifteen million scientific image-text pairs","date":"2023-03-02","arxiv_id":"2303.00915","repositories_listed":5,"syntology":null},{"url":"/paper/flamingo-a-visual-language-model-for-few-shot-1","slug":"flamingo-a-visual-language-model-for-few-shot-1","title":"Flamingo: a Visual Language Model for Few-Shot Learning","date":"2022-04-29","arxiv_id":"2204.14198","repositories_listed":5,"syntology":{"n":24,"n_ran":18,"n_constructed":6,"n_ran_checked":11,"n_instrument":7,"n_unverified":6,"n_honours":0,"n_violates":1,"n_no_contract":10,"n_pointer_only":8,"phrase":"18 ran (of which 6 constructed an object rather than computing a result; 11 with no instrument failure: 0 honoured, 1 violated, 10 with no contract checked; 7 where Syntology's instrument failed) · 6 unverified","sample_list":"/paper/flamingo-a-visual-language-model-for-few-shot-1#ran","syntology_url":"https://syntology.ai/paper/2204.14198","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2204.14198"}},"official":null}},{"url":"/paper/mdetr-modulated-detection-for-end-to-end","slug":"mdetr-modulated-detection-for-end-to-end","title":"MDETR -- Modulated Detection for End-to-End Multi-Modal Understanding","date":"2021-04-26","arxiv_id":"2104.12763","repositories_listed":5,"syntology":{"n":11,"n_ran":7,"n_constructed":4,"n_ran_checked":6,"n_instrument":1,"n_unverified":4,"n_honours":0,"n_violates":1,"n_no_contract":5,"n_pointer_only":0,"phrase":"7 ran (of which 4 constructed an object rather than computing a result; 6 with no instrument failure: 0 honoured, 1 violated, 5 with no contract checked; 1 where Syntology's instrument failed) · 4 unverified","sample_list":"/paper/mdetr-modulated-detection-for-end-to-end#ran","syntology_url":"https://syntology.ai/paper/2104.12763","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2104.12763"}},"official":{"repos":["ashkamath/mdetr"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/pathvqa-30000-questions-for-medical-visual","slug":"pathvqa-30000-questions-for-medical-visual","title":"PathVQA: 30000+ Questions for Medical Visual Question Answering","date":"2020-03-07","arxiv_id":"2003.10286","repositories_listed":5,"syntology":{"n":4,"n_ran":4,"n_constructed":0,"n_ran_checked":3,"n_instrument":1,"n_unverified":0,"n_honours":1,"n_violates":2,"n_no_contract":0,"n_pointer_only":4,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 1 honoured, 2 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/pathvqa-30000-questions-for-medical-visual#ran","syntology_url":"https://syntology.ai/paper/2003.10286","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2003.10286"}},"official":null}},{"url":"/paper/12-in-1-multi-task-vision-and-language","slug":"12-in-1-multi-task-vision-and-language","title":"12-in-1: Multi-Task Vision and Language Representation Learning","date":"2019-12-05","arxiv_id":"1912.02315","repositories_listed":5,"syntology":{"n":20,"n_ran":15,"n_constructed":0,"n_ran_checked":13,"n_instrument":2,"n_unverified":5,"n_honours":1,"n_violates":0,"n_no_contract":12,"n_pointer_only":20,"phrase":"15 ran (of which 0 constructed an object rather than computing a result; 13 with no instrument failure: 1 honoured, 0 violated, 12 with no contract checked; 2 where Syntology's instrument failed) · 5 unverified","sample_list":"/paper/12-in-1-multi-task-vision-and-language#ran","syntology_url":"https://syntology.ai/paper/1912.02315","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1912.02315"}},"official":{"repos":["facebookresearch/vilbert-multi-task"],"state":"official (archive's flag): 11 ran","n_ran":11,"n_constructed":0,"n_ran_no_instrument_failure":9,"n_unverified":3,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/clevr-a-diagnostic-dataset-for-compositional","slug":"clevr-a-diagnostic-dataset-for-compositional","title":"CLEVR: A Diagnostic Dataset for Compositional Language and Elementary Visual Reasoning","date":"2016-12-20","arxiv_id":"1612.06890","repositories_listed":5,"syntology":{"n":5,"n_ran":2,"n_constructed":0,"n_ran_checked":0,"n_instrument":2,"n_unverified":3,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":4,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/clevr-a-diagnostic-dataset-for-compositional#ran","syntology_url":"https://syntology.ai/paper/1612.06890","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1612.06890"}},"official":null}},{"url":"/paper/xmodel-vlm-a-simple-baseline-for-multimodal","slug":"xmodel-vlm-a-simple-baseline-for-multimodal","title":"Xmodel-VLM: A Simple Baseline for Multimodal Vision Language Model","date":"2024-05-15","arxiv_id":"2405.09215","repositories_listed":4,"syntology":null},{"url":"/paper/rlhf-v-towards-trustworthy-mllms-via-behavior","slug":"rlhf-v-towards-trustworthy-mllms-via-behavior","title":"RLHF-V: Towards Trustworthy MLLMs via Behavior Alignment from Fine-grained Correctional Human Feedback","date":"2023-12-01","arxiv_id":"2312.00849","repositories_listed":4,"syntology":null},{"url":"/paper/cogvlm-visual-expert-for-pretrained-language","slug":"cogvlm-visual-expert-for-pretrained-language","title":"CogVLM: Visual Expert for Pretrained Language Models","date":"2023-11-06","arxiv_id":"2311.03079","repositories_listed":4,"syntology":null},{"url":"/paper/med-flamingo-a-multimodal-medical-few-shot","slug":"med-flamingo-a-multimodal-medical-few-shot","title":"Med-Flamingo: a Multimodal Medical Few-shot Learner","date":"2023-07-27","arxiv_id":"2307.15189","repositories_listed":4,"syntology":null},{"url":"/paper/aligning-large-multi-modal-model-with-robust","slug":"aligning-large-multi-modal-model-with-robust","title":"Mitigating Hallucination in Large Multi-Modal Models via Robust Instruction Tuning","date":"2023-06-26","arxiv_id":"2306.14565","repositories_listed":4,"syntology":null},{"url":"/paper/instructblip-towards-general-purpose-vision","slug":"instructblip-towards-general-purpose-vision","title":"InstructBLIP: Towards General-purpose Vision-Language Models with Instruction Tuning","date":"2023-05-11","arxiv_id":"2305.06500","repositories_listed":4,"syntology":null},{"url":"/paper/mplug-2-a-modularized-multi-modal-foundation","slug":"mplug-2-a-modularized-multi-modal-foundation","title":"mPLUG-2: A Modularized Multi-modal Foundation Model Across Text, Image and Video","date":"2023-02-01","arxiv_id":"2302.00402","repositories_listed":4,"syntology":{"n":19,"n_ran":17,"n_constructed":0,"n_ran_checked":8,"n_instrument":9,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":8,"n_pointer_only":2,"phrase":"17 ran (of which 0 constructed an object rather than computing a result; 8 with no instrument failure: 0 honoured, 0 violated, 8 with no contract checked; 9 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/mplug-2-a-modularized-multi-modal-foundation#ran","syntology_url":"https://syntology.ai/paper/2302.00402","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2302.00402"}},"official":{"repos":["alibaba/AliceMind"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/layoutlmv3-pre-training-for-document-ai-with","slug":"layoutlmv3-pre-training-for-document-ai-with","title":"LayoutLMv3: Pre-training for Document AI with Unified Text and Image Masking","date":"2022-04-18","arxiv_id":"2204.08387","repositories_listed":4,"syntology":null},{"url":"/paper/unifying-architectures-tasks-and-modalities","slug":"unifying-architectures-tasks-and-modalities","title":"OFA: Unifying Architectures, Tasks, and Modalities Through a Simple Sequence-to-Sequence Learning Framework","date":"2022-02-07","arxiv_id":"2202.03052","repositories_listed":4,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/unifying-architectures-tasks-and-modalities#ran","syntology_url":"https://syntology.ai/paper/2202.03052","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2202.03052"}},"official":{"repos":["ofa-sys/ofa"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/distilled-dual-encoder-model-for-vision","slug":"distilled-dual-encoder-model-for-vision","title":"Distilled Dual-Encoder Model for Vision-Language Understanding","date":"2021-12-16","arxiv_id":"2112.08723","repositories_listed":4,"syntology":null},{"url":"/paper/how-much-can-clip-benefit-vision-and-language","slug":"how-much-can-clip-benefit-vision-and-language","title":"How Much Can CLIP Benefit Vision-and-Language Tasks?","date":"2021-07-13","arxiv_id":"2107.06383","repositories_listed":4,"syntology":{"n":10,"n_ran":7,"n_constructed":0,"n_ran_checked":1,"n_instrument":6,"n_unverified":3,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":9,"phrase":"7 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 6 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/how-much-can-clip-benefit-vision-and-language#ran","syntology_url":"https://syntology.ai/paper/2107.06383","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2107.06383"}},"official":{"repos":["clip-vil/CLIP-ViL"],"state":"official: harvested, nothing ran","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":1,"ran_from_kinds":["listed"]}}},{"url":"/paper/scene-text-visual-question-answering","slug":"scene-text-visual-question-answering","title":"Scene Text Visual Question Answering","date":"2019-05-31","arxiv_id":"1905.13648","repositories_listed":4,"syntology":null},{"url":"/paper/did-the-model-understand-the-question","slug":"did-the-model-understand-the-question","title":"Did the Model Understand the Question?","date":"2018-05-14","arxiv_id":"1805.05492","repositories_listed":4,"syntology":{"n":10,"n_ran":9,"n_constructed":0,"n_ran_checked":9,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":9,"n_pointer_only":0,"phrase":"9 ran (of which 0 constructed an object rather than computing a result; 9 with no instrument failure: 0 honoured, 0 violated, 9 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/did-the-model-understand-the-question#ran","syntology_url":"https://syntology.ai/paper/1805.05492","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1805.05492"}},"official":{"repos":["pramodkaushik/acl18_results"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/infochartqa-a-benchmark-for-multimodal","slug":"infochartqa-a-benchmark-for-multimodal","title":"InfoChartQA: A Benchmark for Multimodal Question Answering on Infographic Charts","date":"2025-05-25","arxiv_id":"2505.19028","repositories_listed":3,"syntology":{"n":20,"n_ran":19,"n_constructed":0,"n_ran_checked":15,"n_instrument":4,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":15,"n_pointer_only":3,"phrase":"19 ran (of which 0 constructed an object rather than computing a result; 15 with no instrument failure: 0 honoured, 0 violated, 15 with no contract checked; 4 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/infochartqa-a-benchmark-for-multimodal#ran","syntology_url":"https://syntology.ai/paper/2505.19028","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2505.19028"}},"official":{"repos":["cooldawnant/infochartqa"],"state":"official (archive's flag): 12 ran","n_ran":12,"n_constructed":0,"n_ran_no_instrument_failure":12,"n_unverified":0,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/mapeval-a-map-based-evaluation-of-geo-spatial","slug":"mapeval-a-map-based-evaluation-of-geo-spatial","title":"MapEval: A Map-Based Evaluation of Geo-Spatial Reasoning in Foundation Models","date":"2024-12-31","arxiv_id":"2501.00316","repositories_listed":3,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":0,"n_instrument":3,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":3,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 3 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/mapeval-a-map-based-evaluation-of-geo-spatial#ran","syntology_url":"https://syntology.ai/paper/2501.00316","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2501.00316"}},"official":{"repos":["MapEval/MapEval-API","MapEval/MapEval-Textual","MapEval/MapEval-Visual"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/cogvlm2-visual-language-models-for-image-and","slug":"cogvlm2-visual-language-models-for-image-and","title":"CogVLM2: Visual Language Models for Image and Video Understanding","date":"2024-08-29","arxiv_id":"2408.16500","repositories_listed":3,"syntology":{"n":17,"n_ran":12,"n_constructed":0,"n_ran_checked":8,"n_instrument":4,"n_unverified":5,"n_honours":2,"n_violates":0,"n_no_contract":6,"n_pointer_only":0,"phrase":"12 ran (of which 0 constructed an object rather than computing a result; 8 with no instrument failure: 2 honoured, 0 violated, 6 with no contract checked; 4 where Syntology's instrument failed) · 5 unverified","sample_list":"/paper/cogvlm2-visual-language-models-for-image-and#ran","syntology_url":"https://syntology.ai/paper/2408.16500","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2408.16500"}},"official":{"repos":["thudm/cogvlm2","thudm/glm-4"],"state":"official (archive's flag): 12 ran","n_ran":12,"n_constructed":0,"n_ran_no_instrument_failure":8,"n_unverified":5,"ran_from_kinds":["official"]}}},{"url":"/paper/swift-a-scalable-lightweight-infrastructure","slug":"swift-a-scalable-lightweight-infrastructure","title":"SWIFT:A Scalable lightWeight Infrastructure for Fine-Tuning","date":"2024-08-10","arxiv_id":"2408.05517","repositories_listed":3,"syntology":{"n":1,"n_ran":0,"n_constructed":0,"n_ran_checked":0,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":1,"phrase":"0 ran · 1 unverified","sample_list":"/paper/swift-a-scalable-lightweight-infrastructure#ran","syntology_url":"https://syntology.ai/paper/2408.05517","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2408.05517"}},"official":{"repos":["modelscope/ms-swift","modelscope/swift"],"state":"official: harvested, nothing ran","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":1,"ran_from_kinds":[]}}},{"url":"/paper/vrsbench-a-versatile-vision-language","slug":"vrsbench-a-versatile-vision-language","title":"VRSBench: A Versatile Vision-Language Benchmark Dataset for Remote Sensing Image Understanding","date":"2024-06-18","arxiv_id":"2406.12384","repositories_listed":3,"syntology":{"n":13,"n_ran":11,"n_constructed":0,"n_ran_checked":4,"n_instrument":7,"n_unverified":2,"n_honours":0,"n_violates":1,"n_no_contract":3,"n_pointer_only":13,"phrase":"11 ran (of which 0 constructed an object rather than computing a result; 4 with no instrument failure: 0 honoured, 1 violated, 3 with no contract checked; 7 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/vrsbench-a-versatile-vision-language#ran","syntology_url":"https://syntology.ai/paper/2406.12384","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.12384"}},"official":{"repos":["lx709/vrsbench"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":0,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/chameleon-mixed-modal-early-fusion-foundation","slug":"chameleon-mixed-modal-early-fusion-foundation","title":"Chameleon: Mixed-Modal Early-Fusion Foundation Models","date":"2024-05-16","arxiv_id":"2405.09818","repositories_listed":3,"syntology":null},{"url":"/paper/evaluating-text-to-visual-generation-with","slug":"evaluating-text-to-visual-generation-with","title":"Evaluating Text-to-Visual Generation with Image-to-Text Generation","date":"2024-04-01","arxiv_id":"2404.01291","repositories_listed":3,"syntology":null},{"url":"/paper/prismatic-vlms-investigating-the-design-space","slug":"prismatic-vlms-investigating-the-design-space","title":"Prismatic VLMs: Investigating the Design Space of Visually-Conditioned Language Models","date":"2024-02-12","arxiv_id":"2402.07865","repositories_listed":3,"syntology":{"n":5,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":4,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":4,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 4 unverified","sample_list":"/paper/prismatic-vlms-investigating-the-design-space#ran","syntology_url":"https://syntology.ai/paper/2402.07865","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2402.07865"}},"official":{"repos":["tri-ml/prismatic-vlms","tri-ml/vlm-evaluation"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":4,"ran_from_kinds":["official"]}}},{"url":"/paper/moe-llava-mixture-of-experts-for-large-vision","slug":"moe-llava-mixture-of-experts-for-large-vision","title":"MoE-LLaVA: Mixture of Experts for Large Vision-Language Models","date":"2024-01-29","arxiv_id":"2401.15947","repositories_listed":3,"syntology":{"n":2,"n_ran":0,"n_constructed":0,"n_ran_checked":0,"n_instrument":0,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":2,"phrase":"0 ran · 2 unverified","sample_list":"/paper/moe-llava-mixture-of-experts-for-large-vision#ran","syntology_url":"https://syntology.ai/paper/2401.15947","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2401.15947"}},"official":{"repos":["PKU-YuanGroup/MoE-LLaVA"],"state":"official: harvested, nothing ran","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":2,"ran_from_kinds":[]}}},{"url":"/paper/drivelm-driving-with-graph-visual-question","slug":"drivelm-driving-with-graph-visual-question","title":"DriveLM: Driving with Graph Visual Question Answering","date":"2023-12-21","arxiv_id":"2312.14150","repositories_listed":3,"syntology":null},{"url":"/paper/cogagent-a-visual-language-model-for-gui","slug":"cogagent-a-visual-language-model-for-gui","title":"CogAgent: A Visual Language Model for GUI Agents","date":"2023-12-14","arxiv_id":"2312.08914","repositories_listed":3,"syntology":{"n":18,"n_ran":12,"n_constructed":0,"n_ran_checked":4,"n_instrument":8,"n_unverified":6,"n_honours":0,"n_violates":0,"n_no_contract":4,"n_pointer_only":1,"phrase":"12 ran (of which 0 constructed an object rather than computing a result; 4 with no instrument failure: 0 honoured, 0 violated, 4 with no contract checked; 8 where Syntology's instrument failed) · 6 unverified","sample_list":"/paper/cogagent-a-visual-language-model-for-gui#ran","syntology_url":"https://syntology.ai/paper/2312.08914","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2312.08914"}},"official":{"repos":["THUDM/CogAgent","thudm/cogvlm"],"state":"official (archive's flag): 12 ran","n_ran":12,"n_constructed":0,"n_ran_no_instrument_failure":4,"n_unverified":6,"ran_from_kinds":["official"]}}},{"url":"/paper/vila-on-pre-training-for-visual-language","slug":"vila-on-pre-training-for-visual-language","title":"VILA: On Pre-training for Visual Language Models","date":"2023-12-12","arxiv_id":"2312.07533","repositories_listed":3,"syntology":null},{"url":"/paper/ehrxqa-a-multi-modal-question-answering-1","slug":"ehrxqa-a-multi-modal-question-answering-1","title":"EHRXQA: A Multi-Modal Question Answering Dataset for Electronic Health Records with Chest X-ray Images","date":"2023-10-28","arxiv_id":"2310.18652","repositories_listed":3,"syntology":{"n":20,"n_ran":20,"n_constructed":0,"n_ran_checked":17,"n_instrument":3,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":17,"n_pointer_only":0,"phrase":"20 ran (of which 0 constructed an object rather than computing a result; 17 with no instrument failure: 0 honoured, 0 violated, 17 with no contract checked; 3 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/ehrxqa-a-multi-modal-question-answering-1#ran","syntology_url":"https://syntology.ai/paper/2310.18652","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2310.18652"}},"official":{"repos":["baeseongsu/ehrxqa","baeseongsu/mimic-cxr-vqa"],"state":"official (archive's flag): 12 ran","n_ran":12,"n_constructed":0,"n_ran_no_instrument_failure":9,"n_unverified":0,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/instructiongpt-4-a-200-instruction-paradigm","slug":"instructiongpt-4-a-200-instruction-paradigm","title":"InstructionGPT-4: A 200-Instruction Paradigm for Fine-Tuning MiniGPT-4","date":"2023-08-23","arxiv_id":"2308.12067","repositories_listed":3,"syntology":{"n":2,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/instructiongpt-4-a-200-instruction-paradigm#ran","syntology_url":"https://syntology.ai/paper/2308.12067","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2308.12067"}},"official":{"repos":["waltonfuture/InstructionGPT-4"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/mmbench-is-your-multi-modal-model-an-all","slug":"mmbench-is-your-multi-modal-model-an-all","title":"MMBench: Is Your Multi-modal Model an All-around Player?","date":"2023-07-12","arxiv_id":"2307.06281","repositories_listed":3,"syntology":null},{"url":"/paper/gpt4roi-instruction-tuning-large-language","slug":"gpt4roi-instruction-tuning-large-language","title":"GPT4RoI: Instruction Tuning Large Language Model on Region-of-Interest","date":"2023-07-07","arxiv_id":"2307.03601","repositories_listed":3,"syntology":{"n":5,"n_ran":3,"n_constructed":0,"n_ran_checked":3,"n_instrument":0,"n_unverified":2,"n_honours":2,"n_violates":0,"n_no_contract":1,"n_pointer_only":5,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 2 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/gpt4roi-instruction-tuning-large-language#ran","syntology_url":"https://syntology.ai/paper/2307.03601","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2307.03601"}},"official":{"repos":["jshilong/gpt4roi"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/vision-language-models-in-remote-sensing","slug":"vision-language-models-in-remote-sensing","title":"Vision-Language Models in Remote Sensing: Current Progress and Future Trends","date":"2023-05-09","arxiv_id":"2305.05726","repositories_listed":3,"syntology":null},{"url":"/paper/llama-adapter-v2-parameter-efficient-visual","slug":"llama-adapter-v2-parameter-efficient-visual","title":"LLaMA-Adapter V2: Parameter-Efficient Visual Instruction Model","date":"2023-04-28","arxiv_id":"2304.15010","repositories_listed":3,"syntology":{"n":1,"n_ran":0,"n_constructed":0,"n_ran_checked":0,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":1,"phrase":"0 ran · 1 unverified","sample_list":"/paper/llama-adapter-v2-parameter-efficient-visual#ran","syntology_url":"https://syntology.ai/paper/2304.15010","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2304.15010"}},"official":{"repos":["zrrskywalker/llama-adapter"],"state":"official: not harvested","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":[]}}},{"url":"/paper/from-images-to-textual-prompts-zero-shot-vqa","slug":"from-images-to-textual-prompts-zero-shot-vqa","title":"From Images to Textual Prompts: Zero-shot VQA with Frozen Large Language Models","date":"2022-12-21","arxiv_id":"2212.10846","repositories_listed":3,"syntology":null},{"url":"/paper/plug-and-play-vqa-zero-shot-vqa-by-conjoining","slug":"plug-and-play-vqa-zero-shot-vqa-by-conjoining","title":"Plug-and-Play VQA: Zero-shot VQA by Conjoining Large Pretrained Models with Zero Training","date":"2022-10-17","arxiv_id":"2210.08773","repositories_listed":3,"syntology":{"n":3,"n_ran":2,"n_constructed":0,"n_ran_checked":0,"n_instrument":2,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":3,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/plug-and-play-vqa-zero-shot-vqa-by-conjoining#ran","syntology_url":"https://syntology.ai/paper/2210.08773","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2210.08773"}},"official":{"repos":["salesforce/lavis"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/tvlt-textless-vision-language-transformer","slug":"tvlt-textless-vision-language-transformer","title":"TVLT: Textless Vision-Language Transformer","date":"2022-09-28","arxiv_id":"2209.14156","repositories_listed":3,"syntology":{"n":4,"n_ran":3,"n_constructed":2,"n_ran_checked":3,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":0,"phrase":"3 ran (of which 2 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/tvlt-textless-vision-language-transformer#ran","syntology_url":"https://syntology.ai/paper/2209.14156","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2209.14156"}},"official":{"repos":["zinengtang/tvlt"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":2,"n_ran_no_instrument_failure":3,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/surgical-vqa-visual-question-answering-in","slug":"surgical-vqa-visual-question-answering-in","title":"Surgical-VQA: Visual Question Answering in Surgical Scenes using Transformer","date":"2022-06-22","arxiv_id":"2206.11053","repositories_listed":3,"syntology":{"n":2,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":1,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":2,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/surgical-vqa-visual-question-answering-in#ran","syntology_url":"https://syntology.ai/paper/2206.11053","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2206.11053"}},"official":{"repos":["lalithjets/surgical_vqa"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/zero-shot-video-question-answering-via-frozen","slug":"zero-shot-video-question-answering-via-frozen","title":"Zero-Shot Video Question Answering via Frozen Bidirectional Language Models","date":"2022-06-16","arxiv_id":"2206.08155","repositories_listed":3,"syntology":{"n":34,"n_ran":14,"n_constructed":9,"n_ran_checked":14,"n_instrument":0,"n_unverified":20,"n_honours":1,"n_violates":1,"n_no_contract":12,"n_pointer_only":1,"phrase":"14 ran (of which 9 constructed an object rather than computing a result; 14 with no instrument failure: 1 honoured, 1 violated, 12 with no contract checked; 0 where Syntology's instrument failed) · 20 unverified","sample_list":"/paper/zero-shot-video-question-answering-via-frozen#ran","syntology_url":"https://syntology.ai/paper/2206.08155","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2206.08155"}},"official":{"repos":["antoyang/FrozenBiLM"],"state":"official: harvested, nothing ran","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":7,"ran_from_kinds":["listed"]}}},{"url":"/paper/mplug-effective-and-efficient-vision-language","slug":"mplug-effective-and-efficient-vision-language","title":"mPLUG: Effective and Efficient Vision-Language Learning by Cross-modal Skip-connections","date":"2022-05-24","arxiv_id":"2205.12005","repositories_listed":3,"syntology":null},{"url":"/paper/iglue-a-benchmark-for-transfer-learning","slug":"iglue-a-benchmark-for-transfer-learning","title":"IGLUE: A Benchmark for Transfer Learning across Modalities, Tasks, and Languages","date":"2022-01-27","arxiv_id":"2201.11732","repositories_listed":3,"syntology":{"n":21,"n_ran":17,"n_constructed":2,"n_ran_checked":11,"n_instrument":6,"n_unverified":4,"n_honours":2,"n_violates":0,"n_no_contract":9,"n_pointer_only":3,"phrase":"17 ran (of which 2 constructed an object rather than computing a result; 11 with no instrument failure: 2 honoured, 0 violated, 9 with no contract checked; 6 where Syntology's instrument failed) · 4 unverified","sample_list":"/paper/iglue-a-benchmark-for-transfer-learning#ran","syntology_url":"https://syntology.ai/paper/2201.11732","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2201.11732"}},"official":{"repos":["e-bug/iglue","e-bug/volta"],"state":"official (archive's flag): 17 ran","n_ran":17,"n_constructed":2,"n_ran_no_instrument_failure":11,"n_unverified":4,"ran_from_kinds":["official"]}}},{"url":"/paper/perceptual-score-what-data-modalities-does","slug":"perceptual-score-what-data-modalities-does","title":"Perceptual Score: What Data Modalities Does Your Model Perceive?","date":"2021-10-27","arxiv_id":"2110.14375","repositories_listed":3,"syntology":{"n":1,"n_ran":1,"n_constructed":1,"n_ran_checked":1,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":1,"phrase":"1 ran (of which 1 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified; the one sample that ran constructed an object rather than computing a result","sample_list":"/paper/perceptual-score-what-data-modalities-does#ran","syntology_url":"https://syntology.ai/paper/2110.14375","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2110.14375"}},"official":{"repos":["itaigat/perceptual-score"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":1,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/webqa-multihop-and-multimodal-qa","slug":"webqa-multihop-and-multimodal-qa","title":"WebQA: Multihop and Multimodal QA","date":"2021-09-01","arxiv_id":"2109.00590","repositories_listed":3,"syntology":{"n":13,"n_ran":10,"n_constructed":0,"n_ran_checked":7,"n_instrument":3,"n_unverified":3,"n_honours":2,"n_violates":0,"n_no_contract":5,"n_pointer_only":12,"phrase":"10 ran (of which 0 constructed an object rather than computing a result; 7 with no instrument failure: 2 honoured, 0 violated, 5 with no contract checked; 3 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/webqa-multihop-and-multimodal-qa#ran","syntology_url":"https://syntology.ai/paper/2109.00590","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2109.00590"}},"official":null}},{"url":"/paper/conceptual-12m-pushing-web-scale-image-text","slug":"conceptual-12m-pushing-web-scale-image-text","title":"Conceptual 12M: Pushing Web-Scale Image-Text Pre-Training To Recognize Long-Tail Visual Concepts","date":"2021-02-17","arxiv_id":"2102.08981","repositories_listed":3,"syntology":null},{"url":"/paper/lrta-a-transparent-neural-symbolic-reasoning","slug":"lrta-a-transparent-neural-symbolic-reasoning","title":"LRTA: A Transparent Neural-Symbolic Reasoning Framework with Modular Supervision for Visual Question Answering","date":"2020-11-21","arxiv_id":"2011.10731","repositories_listed":3,"syntology":null},{"url":"/paper/docvqa-a-dataset-for-vqa-on-document-images","slug":"docvqa-a-dataset-for-vqa-on-document-images","title":"DocVQA: A Dataset for VQA on Document Images","date":"2020-07-01","arxiv_id":"2007.00398","repositories_listed":3,"syntology":{"n":6,"n_ran":5,"n_constructed":0,"n_ran_checked":2,"n_instrument":3,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":2,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 3 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/docvqa-a-dataset-for-vqa-on-document-images#ran","syntology_url":"https://syntology.ai/paper/2007.00398","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2007.00398"}},"official":null}},{"url":"/paper/unified-vision-language-pre-training-for","slug":"unified-vision-language-pre-training-for","title":"Unified Vision-Language Pre-Training for Image Captioning and VQA","date":"2019-09-24","arxiv_id":"1909.11059","repositories_listed":3,"syntology":{"n":14,"n_ran":11,"n_constructed":0,"n_ran_checked":6,"n_instrument":5,"n_unverified":3,"n_honours":1,"n_violates":0,"n_no_contract":5,"n_pointer_only":14,"phrase":"11 ran (of which 0 constructed an object rather than computing a result; 6 with no instrument failure: 1 honoured, 0 violated, 5 with no contract checked; 5 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/unified-vision-language-pre-training-for#ran","syntology_url":"https://syntology.ai/paper/1909.11059","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1909.11059"}},"official":{"repos":["LuoweiZhou/VLP"],"state":"official (archive's flag): 11 ran","n_ran":11,"n_constructed":0,"n_ran_no_instrument_failure":6,"n_unverified":3,"ran_from_kinds":["official"]}}},{"url":"/paper/dont-take-the-easy-way-out-ensemble-based","slug":"dont-take-the-easy-way-out-ensemble-based","title":"Don't Take the Easy Way Out: Ensemble Based Methods for Avoiding Known Dataset Biases","date":"2019-09-09","arxiv_id":"1909.03683","repositories_listed":3,"syntology":{"n":6,"n_ran":4,"n_constructed":0,"n_ran_checked":1,"n_instrument":3,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":3,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 3 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/dont-take-the-easy-way-out-ensemble-based#ran","syntology_url":"https://syntology.ai/paper/1909.03683","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1909.03683"}},"official":{"repos":["chrisc36/debias"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":2,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/vl-bert-pre-training-of-generic-visual","slug":"vl-bert-pre-training-of-generic-visual","title":"VL-BERT: Pre-training of Generic Visual-Linguistic Representations","date":"2019-08-22","arxiv_id":"1908.08530","repositories_listed":3,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/vl-bert-pre-training-of-generic-visual#ran","syntology_url":"https://syntology.ai/paper/1908.08530","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1908.08530"}},"official":{"repos":["jackroos/VL-BERT"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/clevr-ref-diagnosing-visual-reasoning-with","slug":"clevr-ref-diagnosing-visual-reasoning-with","title":"CLEVR-Ref+: Diagnosing Visual Reasoning with Referring Expressions","date":"2019-01-03","arxiv_id":"1901.00850","repositories_listed":3,"syntology":{"n":5,"n_ran":4,"n_constructed":0,"n_ran_checked":1,"n_instrument":3,"n_unverified":1,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":5,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 3 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/clevr-ref-diagnosing-visual-reasoning-with#ran","syntology_url":"https://syntology.ai/paper/1901.00850","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1901.00850"}},"official":null}},{"url":"/paper/attention-on-attention-architectures-for","slug":"attention-on-attention-architectures-for","title":"Attention on Attention: Architectures for Visual Question Answering (VQA)","date":"2018-03-21","arxiv_id":"1803.07724","repositories_listed":3,"syntology":null},{"url":"/paper/a-simple-loss-function-for-improving-the","slug":"a-simple-loss-function-for-improving-the","title":"A Simple Loss Function for Improving the Convergence and Accuracy of Visual Question Answering Models","date":"2017-08-02","arxiv_id":"1708.00584","repositories_listed":3,"syntology":null},{"url":"/paper/shapeworld-a-new-test-methodology-for","slug":"shapeworld-a-new-test-methodology-for","title":"ShapeWorld - A new test methodology for multimodal language understanding","date":"2017-04-14","arxiv_id":"1704.04517","repositories_listed":3,"syntology":null},{"url":"/paper/revisiting-visual-question-answering","slug":"revisiting-visual-question-answering","title":"Revisiting Visual Question Answering Baselines","date":"2016-06-27","arxiv_id":"1606.08390","repositories_listed":3,"syntology":null},{"url":"/paper/sofar-language-grounded-orientation-bridges","slug":"sofar-language-grounded-orientation-bridges","title":"SoFar: Language-Grounded Orientation Bridges Spatial Reasoning and Object Manipulation","date":"2025-02-18","arxiv_id":"2502.13143","repositories_listed":2,"syntology":null},{"url":"/paper/vargpt-unified-understanding-and-generation","slug":"vargpt-unified-understanding-and-generation","title":"VARGPT: Unified Understanding and Generation in a Visual Autoregressive Multimodal Large Language Model","date":"2025-01-21","arxiv_id":"2501.12327","repositories_listed":2,"syntology":null},{"url":"/paper/learn-from-downstream-and-be-yourself-in","slug":"learn-from-downstream-and-be-yourself-in","title":"Learn from Downstream and Be Yourself in Multimodal Large Language Model Fine-Tuning","date":"2024-11-17","arxiv_id":"2411.10928","repositories_listed":2,"syntology":null},{"url":"/paper/llava-o1-let-vision-language-models-reason","slug":"llava-o1-let-vision-language-models-reason","title":"LLaVA-CoT: Let Vision Language Models Reason Step-by-Step","date":"2024-11-15","arxiv_id":"2411.10440","repositories_listed":2,"syntology":{"n":6,"n_ran":6,"n_constructed":0,"n_ran_checked":4,"n_instrument":2,"n_unverified":0,"n_honours":0,"n_violates":1,"n_no_contract":3,"n_pointer_only":0,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 4 with no instrument failure: 0 honoured, 1 violated, 3 with no contract checked; 2 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/llava-o1-let-vision-language-models-reason#ran","syntology_url":"https://syntology.ai/paper/2411.10440","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2411.10440"}},"official":{"repos":["PKU-YuanGroup/LLaVA-CoT"],"state":"official (archive's flag): 6 ran","n_ran":6,"n_constructed":0,"n_ran_no_instrument_failure":4,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/towards-foundation-models-for-3d-vision-how","slug":"towards-foundation-models-for-3d-vision-how","title":"Towards Foundation Models for 3D Vision: How Close Are We?","date":"2024-10-14","arxiv_id":"2410.10799","repositories_listed":2,"syntology":{"n":29,"n_ran":20,"n_constructed":0,"n_ran_checked":15,"n_instrument":5,"n_unverified":9,"n_honours":1,"n_violates":0,"n_no_contract":14,"n_pointer_only":1,"phrase":"20 ran (of which 0 constructed an object rather than computing a result; 15 with no instrument failure: 1 honoured, 0 violated, 14 with no contract checked; 5 where Syntology's instrument failed) · 9 unverified","sample_list":"/paper/towards-foundation-models-for-3d-vision-how#ran","syntology_url":"https://syntology.ai/paper/2410.10799","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2410.10799"}},"official":{"repos":["princeton-vl/uniqa-3d"],"state":"official (archive's flag): 10 ran","n_ran":10,"n_constructed":0,"n_ran_no_instrument_failure":5,"n_unverified":3,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/baichuan-omni-technical-report","slug":"baichuan-omni-technical-report","title":"Baichuan-Omni Technical Report","date":"2024-10-11","arxiv_id":"2410.08565","repositories_listed":2,"syntology":null},{"url":"/paper/emu3-next-token-prediction-is-all-you-need","slug":"emu3-next-token-prediction-is-all-you-need","title":"Emu3: Next-Token Prediction is All You Need","date":"2024-09-27","arxiv_id":"2409.18869","repositories_listed":2,"syntology":{"n":6,"n_ran":6,"n_constructed":0,"n_ran_checked":3,"n_instrument":3,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":0,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 3 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/emu3-next-token-prediction-is-all-you-need#ran","syntology_url":"https://syntology.ai/paper/2409.18869","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2409.18869"}},"official":null}},{"url":"/paper/mediconfusion-can-you-trust-your-ai","slug":"mediconfusion-can-you-trust-your-ai","title":"MediConfusion: Can you trust your AI radiologist? Probing the reliability of multimodal medical foundation models","date":"2024-09-23","arxiv_id":"2409.15477","repositories_listed":2,"syntology":{"n":28,"n_ran":12,"n_constructed":0,"n_ran_checked":8,"n_instrument":4,"n_unverified":16,"n_honours":0,"n_violates":0,"n_no_contract":8,"n_pointer_only":28,"phrase":"12 ran (of which 0 constructed an object rather than computing a result; 8 with no instrument failure: 0 honoured, 0 violated, 8 with no contract checked; 4 where Syntology's instrument failed) · 16 unverified","sample_list":"/paper/mediconfusion-can-you-trust-your-ai#ran","syntology_url":"https://syntology.ai/paper/2409.15477","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2409.15477"}},"official":{"repos":["mshahabsepehri/mediconfusion","AIF4S/MediConfusion"],"state":"official (archive's flag): 12 ran","n_ran":12,"n_constructed":0,"n_ran_no_instrument_failure":8,"n_unverified":16,"ran_from_kinds":["official"]}}},{"url":"/paper/lime-m-less-is-more-for-evaluation-of-mllms","slug":"lime-m-less-is-more-for-evaluation-of-mllms","title":"LIME: Less Is More for MLLM Evaluation","date":"2024-09-10","arxiv_id":"2409.06851","repositories_listed":2,"syntology":{"n":14,"n_ran":13,"n_constructed":0,"n_ran_checked":0,"n_instrument":13,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":14,"phrase":"13 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 13 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/lime-m-less-is-more-for-evaluation-of-mllms#ran","syntology_url":"https://syntology.ai/paper/2409.06851","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2409.06851"}},"official":{"repos":["kangreen0210/lime","kangreen0210/lime-m"],"state":"official (archive's flag): 13 ran","n_ran":13,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/llava-onevision-easy-visual-task-transfer","slug":"llava-onevision-easy-visual-task-transfer","title":"LLaVA-OneVision: Easy Visual Task Transfer","date":"2024-08-06","arxiv_id":"2408.03326","repositories_listed":2,"syntology":null}],"syntology_records":60,"syntology_note":"a paper without a record is not a recorded non-run: it may lack an arXiv id or simply be absent from the graph layer"},"key_notes":{"n_ran_checked":"legacy name, kept unchanged so existing readers do not break: it counts the samples that ran with no instrument failure (honoured, violated, and ran with no contract checked); it does not mean a contract was checked, and the pages print it as 'K with no instrument failure', not 'K checked'","n_constructed":"a sub-count of the samples that ran, never subtracted from them and never a failure: an executed sample whose run returned an instance of its own class (fixture_out_type equals the entry name): the run built an object and did not compute a result (Syntology's RAN record, counts.constructed)"},"since_archive_judged":{"papers":30,"papers_placed":1,"judged_by":{"model":"Claude Sonnet 4.5","models_as_recorded":["global.anthropic.claude-sonnet-4-5-20250929-v1:0"],"from":"lb_rows_run_verified.jsonl: each record's `model` field (the extractor's call that wrote decisions[])"},"note":"a model's judgement from the paper's own tables that they report on this leaderboard, not a result: no value from a paper that was not placed is printed or carried here; papers counts each paper once over these leaderboards","backfill":"8,886 of 18,637 newer papers checked so far (733 more had nothing to place; 41 could not be read)","backfill_complete":false,"leaderboards":[{"leaderboard":"/sota/visual-question-answering-on-mm-vet","since_archive":"/sota/visual-question-answering-on-mm-vet#since-judged","papers_judged":3,"placed":0},{"leaderboard":"/sota/visual-question-answering-on-mmbench","since_archive":"/sota/visual-question-answering-on-mmbench#since-judged","papers_judged":8,"placed":0},{"leaderboard":"/sota/visual-question-answering-on-v-bench","since_archive":"/sota/visual-question-answering-on-v-bench#since-judged","papers_judged":4,"placed":0},{"leaderboard":"/sota/visual-question-answering-on-vqa-v2-test-std-1","since_archive":"/sota/visual-question-answering-on-vqa-v2-test-std-1#since-judged","papers_judged":1,"placed":0},{"leaderboard":"/sota/visual-question-answering-on-mmhal-bench","since_archive":"/sota/visual-question-answering-on-mmhal-bench#since-judged","papers_judged":1,"placed":0},{"leaderboard":"/sota/visual-question-answering-on-amber","since_archive":"/sota/visual-question-answering-on-amber#since-judged","papers_judged":2,"placed":1},{"leaderboard":"/sota/visual-question-answering-on-clevr-1","since_archive":"/sota/visual-question-answering-on-clevr-1#since-judged","papers_judged":3,"placed":0},{"leaderboard":"/sota/visual-question-answering-on-gqa-1","since_archive":"/sota/visual-question-answering-on-gqa-1#since-judged","papers_judged":7,"placed":0},{"leaderboard":"/sota/visual-question-answering-on-textvqa-test-2","since_archive":"/sota/visual-question-answering-on-textvqa-test-2#since-judged","papers_judged":4,"placed":0},{"leaderboard":"/sota/visual-question-answering-on-vizwiz-1","since_archive":"/sota/visual-question-answering-on-vizwiz-1#since-judged","papers_judged":5,"placed":0}]},"papers_where_code_ran":378,"papers_where_code_ran_first_page":"/task/visual-question-answering-1/papers/ran/1","papers_where_code_ran_caption":"We ran code from the paper's repository; we did not run it on this task or check it against the task's benchmarks.","pages":[{"order":"archive","page":2,"of":22,"url":"/task/visual-question-answering-1/papers/2","json":"/task/visual-question-answering-1/papers/2.json","rows":100},{"order":"archive","page":3,"of":22,"url":"/task/visual-question-answering-1/papers/3","json":"/task/visual-question-answering-1/papers/3.json","rows":100},{"order":"archive","page":4,"of":22,"url":"/task/visual-question-answering-1/papers/4","json":"/task/visual-question-answering-1/papers/4.json","rows":100},{"order":"archive","page":5,"of":22,"url":"/task/visual-question-answering-1/papers/5","json":"/task/visual-question-answering-1/papers/5.json","rows":100},{"order":"archive","page":6,"of":22,"url":"/task/visual-question-answering-1/papers/6","json":"/task/visual-question-answering-1/papers/6.json","rows":100},{"order":"archive","page":7,"of":22,"url":"/task/visual-question-answering-1/papers/7","json":"/task/visual-question-answering-1/papers/7.json","rows":100},{"order":"archive","page":8,"of":22,"url":"/task/visual-question-answering-1/papers/8","json":"/task/visual-question-answering-1/papers/8.json","rows":100},{"order":"archive","page":9,"of":22,"url":"/task/visual-question-answering-1/papers/9","json":"/task/visual-question-answering-1/papers/9.json","rows":100},{"order":"archive","page":10,"of":22,"url":"/task/visual-question-answering-1/papers/10","json":"/task/visual-question-answering-1/papers/10.json","rows":100},{"order":"archive","page":11,"of":22,"url":"/task/visual-question-answering-1/papers/11","json":"/task/visual-question-answering-1/papers/11.json","rows":100},{"order":"archive","page":12,"of":22,"url":"/task/visual-question-answering-1/papers/12","json":"/task/visual-question-answering-1/papers/12.json","rows":100},{"order":"archive","page":13,"of":22,"url":"/task/visual-question-answering-1/papers/13","json":"/task/visual-question-answering-1/papers/13.json","rows":100},{"order":"archive","page":14,"of":22,"url":"/task/visual-question-answering-1/papers/14","json":"/task/visual-question-answering-1/papers/14.json","rows":100},{"order":"archive","page":15,"of":22,"url":"/task/visual-question-answering-1/papers/15","json":"/task/visual-question-answering-1/papers/15.json","rows":100},{"order":"archive","page":16,"of":22,"url":"/task/visual-question-answering-1/papers/16","json":"/task/visual-question-answering-1/papers/16.json","rows":100},{"order":"archive","page":17,"of":22,"url":"/task/visual-question-answering-1/papers/17","json":"/task/visual-question-answering-1/papers/17.json","rows":100},{"order":"archive","page":18,"of":22,"url":"/task/visual-question-answering-1/papers/18","json":"/task/visual-question-answering-1/papers/18.json","rows":100},{"order":"archive","page":19,"of":22,"url":"/task/visual-question-answering-1/papers/19","json":"/task/visual-question-answering-1/papers/19.json","rows":100},{"order":"archive","page":20,"of":22,"url":"/task/visual-question-answering-1/papers/20","json":"/task/visual-question-answering-1/papers/20.json","rows":100},{"order":"archive","page":21,"of":22,"url":"/task/visual-question-answering-1/papers/21","json":"/task/visual-question-answering-1/papers/21.json","rows":100},{"order":"archive","page":22,"of":22,"url":"/task/visual-question-answering-1/papers/22","json":"/task/visual-question-answering-1/papers/22.json","rows":77},{"order":"ran","page":1,"of":4,"url":"/task/visual-question-answering-1/papers/ran/1","json":"/task/visual-question-answering-1/papers/ran/1.json","rows":100},{"order":"ran","page":2,"of":4,"url":"/task/visual-question-answering-1/papers/ran/2","json":"/task/visual-question-answering-1/papers/ran/2.json","rows":100},{"order":"ran","page":3,"of":4,"url":"/task/visual-question-answering-1/papers/ran/3","json":"/task/visual-question-answering-1/papers/ran/3.json","rows":100},{"order":"ran","page":4,"of":4,"url":"/task/visual-question-answering-1/papers/ran/4","json":"/task/visual-question-answering-1/papers/ran/4.json","rows":78}],"description_links":{"kept":0,"unwrapped_to_text":0,"bare_urls_linked":0,"relative_images_dropped":0,"rule":"internal links are kept only when the target slug exists in the catalog"},"syntology":{"read_at":"2026-10-05T00:36:11+00:00","claim":"Per-sample execution status on synthesized fixtures ('ran N of M samples'); not a correctness claim and not a ranking signal.","status_vocabulary":{"ran_honours":"ran, honoured the contract we drafted","ran_violates":"ran, violated the contract we drafted","ran_draft_wrong":"ran; our contract draft was wrong, not the code","ran_fixture":"ran; the contract check could not run (RAISES, DEP_MISSING or TIMEOUT)","ran":"ran on a synthesized input","unverified":"unverified (harvested, no recorded run)"},"row_route":"each syntology object on a benchmark's first row and on a paper carries sample_list (this site's paper page, #ran), syntology_url (syntology.ai's page for the paper, only when that route probed live) and mcp (the agent call that lists the paper's harvested samples; not a verdict on the paper)","developers":"https://syntology.ai/developers"},"not_shown":{"libraries":"the archive has no per-task library table","trend_sparklines":"the Trend column of the benchmarks table was a rendered image; it is not in the archive","social_and_latest_sorts":"stars and social signals are not in the archive"},"record_sha256":"102719949023bc2fdb35b46400352fd2c20d569115a78cf6a808517aff785a67","record_changed_at":"2026-10-05","record_changed_at_basis":"changed"}