{"about":{"non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","site":"https://codewithpapers.app","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page","syntology":{"site":"https://syntology.ai","developers":"https://syntology.ai/developers","mcp":{"server":"https://syntology.ai/mcp","transport":"streamable-http","server_card":"https://syntology.ai/.well-known/mcp/server-card.json","auth":{"type":"trial token, no account","trial_token":"https://syntology.ai/api/oauth/trial/token","method":"POST","docs":"https://syntology.ai/developers"}},"have":"https://syntology.ai/api/graph/have?x=<method, arXiv id or title> (free, answers coverage only)","paper_base":"https://syntology.ai/paper/","atlas_base":"https://app.syntology.ai/?focus="},"machine_readable":[{"url":"https://codewithpapers.app/llms.txt","what":"the machine catalog: every machine-readable file, counted"},{"url":"https://codewithpapers.app/index/manifest.json","what":"paper-to-code index by arXiv id, with Syntology's counts"},{"url":"https://codewithpapers.app/search/manifest.json","what":"site search index (titles, authors) and its files"},{"url":"https://codewithpapers.app/download","what":"bulk files: Syntology's layer, described there"},{"url":"https://codewithpapers.app/build_manifest.json","what":"the build record: inputs, counts, exclusions, probes"}]},"url":"/task/multimodal-interaction","name":"multimodal interaction","slug":"multimodal-interaction","description_markdown":null,"categories":[{"name":"Robots","url":"/area/robots"}],"source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","slug_source":"derived"},"counts":{"papers_tagged":106,"papers_with_code":41,"benchmarks":0,"benchmark_tables_in_archive":0,"benchmark_tables_shown":0,"benchmark_tables_withheld_as_spam":0,"benchmark_definition":"a leaderboard table with at least one row; benchmark_tables_shown also counts the zero-row tables; benchmark_tables_in_archive adds the tables withheld as spam","datasets":0,"subtasks":0,"parent_tasks":0},"benchmarks":[],"datasets":[],"subtasks":[],"parent_tasks":[],"papers":{"order":"repositories listed in the archive (most first), then date (newest first), then slug; the archive holds no stars","population":"every paper the archive tags with this task (distinct papers; spam titles not listed). 'of' counts the papers listed; until 2026-09-25 it counted only the papers with a code link, and that number is of_with_code","page":1,"pages_in_order":2,"rows_per_page":100,"shown":100,"of":106,"of_with_code":41,"tagged_in_all":106,"counts":{"archive_papers_tagged":106,"with_a_code_link":41,"where_syntology_ran_a_sample":9,"not_listed_spam_title":0,"listed":106,"listed_where_code_ran":9,"where_syntology_ran_a_sample_split":{"with_a_run_with_no_instrument_failure":8,"every_run_a_failure_of_syntologys_instrument":1,"listed_with_a_run_with_no_instrument_failure":8,"listed_every_run_a_failure_of_syntologys_instrument":1,"filter":{"states":["a run with no instrument failure","any run, instrument failures included"],"default":"a run with no instrument failure","note":"on the 'only where code ran' pages the default hides, in the browser, the rows where every run was a failure of Syntology's instrument; the second state shows them again. Rows are hidden, never re-ordered; these twins list every row"}},"definition":"distinct papers the archive tags; 'where Syntology ran a sample' counts papers with at least one harvested sample that ran, which is not a correctness claim"},"items":[{"url":"/paper/vilt-vision-and-language-transformer-without","slug":"vilt-vision-and-language-transformer-without","title":"ViLT: Vision-and-Language Transformer Without Convolution or Region Supervision","date":"2021-02-05","arxiv_id":"2102.03334","repositories_listed":6,"syntology":{"n":4,"n_ran":1,"n_constructed":1,"n_ran_checked":1,"n_instrument":0,"n_unverified":3,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":1,"phrase":"1 ran (of which 1 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 3 unverified; the one sample that ran constructed an object rather than computing a result","sample_list":"/paper/vilt-vision-and-language-transformer-without#ran","syntology_url":"https://syntology.ai/paper/2102.03334","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2102.03334"}},"official":{"repos":["dandelin/vilt"],"state":"official: harvested, nothing ran","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":1,"ran_from_kinds":["listed"]}}},{"url":"/paper/stream-omni-simultaneous-multimodal","slug":"stream-omni-simultaneous-multimodal","title":"Stream-Omni: Simultaneous Multimodal Interactions with Large Language-Vision-Speech Model","date":"2025-06-16","arxiv_id":"2506.13642","repositories_listed":2,"syntology":null},{"url":"/paper/mm-bigbench-evaluating-multimodal-models-on","slug":"mm-bigbench-evaluating-multimodal-models-on","title":"MM-BigBench: Evaluating Multimodal Models on Multimodal Content Comprehension Tasks","date":"2023-10-13","arxiv_id":"2310.09036","repositories_listed":2,"syntology":null},{"url":"/paper/chartsketcher-reasoning-with-multimodal","slug":"chartsketcher-reasoning-with-multimodal","title":"ChartSketcher: Reasoning with Multimodal Feedback and Reflection for Chart Understanding","date":"2025-05-25","arxiv_id":"2505.19076","repositories_listed":1,"syntology":null},{"url":"/paper/i2moe-interpretable-multimodal-interaction","slug":"i2moe-interpretable-multimodal-interaction","title":"I2MoE: Interpretable Multimodal Interaction-aware Mixture-of-Experts","date":"2025-05-25","arxiv_id":"2505.19190","repositories_listed":1,"syntology":{"n":4,"n_ran":4,"n_constructed":4,"n_ran_checked":4,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":4,"n_pointer_only":4,"phrase":"4 ran (of which 4 constructed an object rather than computing a result; 4 with no instrument failure: 0 honoured, 0 violated, 4 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified; every one of the 4 samples that ran constructed an object rather than computing a result","sample_list":"/paper/i2moe-interpretable-multimodal-interaction#ran","syntology_url":"https://syntology.ai/paper/2505.19190","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2505.19190"}},"official":{"repos":["raina-xin/i2moe"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":4,"n_ran_no_instrument_failure":4,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/ming-lite-uni-advancements-in-unified","slug":"ming-lite-uni-advancements-in-unified","title":"Ming-Lite-Uni: Advancements in Unified Architecture for Natural Multimodal Interaction","date":"2025-05-05","arxiv_id":"2505.02471","repositories_listed":1,"syntology":null},{"url":"/paper/ovtr-end-to-end-open-vocabulary-multiple","slug":"ovtr-end-to-end-open-vocabulary-multiple","title":"OVTR: End-to-End Open-Vocabulary Multiple Object Tracking with Transformer","date":"2025-03-13","arxiv_id":"2503.10616","repositories_listed":1,"syntology":{"n":12,"n_ran":11,"n_constructed":0,"n_ran_checked":6,"n_instrument":5,"n_unverified":1,"n_honours":0,"n_violates":1,"n_no_contract":5,"n_pointer_only":2,"phrase":"11 ran (of which 0 constructed an object rather than computing a result; 6 with no instrument failure: 0 honoured, 1 violated, 5 with no contract checked; 5 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/ovtr-end-to-end-open-vocabulary-multiple#ran","syntology_url":"https://syntology.ai/paper/2503.10616","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2503.10616"}},"official":{"repos":["jinyanglii/ovtr"],"state":"official (archive's flag): 11 ran","n_ran":11,"n_constructed":0,"n_ran_no_instrument_failure":6,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/towards-explainable-multimodal-depression","slug":"towards-explainable-multimodal-depression","title":"Towards Explainable Multimodal Depression Recognition for Clinical Interviews","date":"2025-01-27","arxiv_id":"2501.16106","repositories_listed":1,"syntology":null},{"url":"/paper/spider-any-to-many-multimodal-llm","slug":"spider-any-to-many-multimodal-llm","title":"Spider: Any-to-Many Multimodal LLM","date":"2024-11-14","arxiv_id":"2411.09439","repositories_listed":1,"syntology":null},{"url":"/paper/enhancing-multimodal-query-representation-via","slug":"enhancing-multimodal-query-representation-via","title":"MIRe: Enhancing Multimodal Queries Representation via Fusion-Free Modality Interaction for Multimodal Retrieval","date":"2024-11-13","arxiv_id":"2411.08334","repositories_listed":1,"syntology":null},{"url":"/paper/foundations-and-recent-trends-in-multimodal-1","slug":"foundations-and-recent-trends-in-multimodal-1","title":"Foundations and Recent Trends in Multimodal Mobile Agents: A Survey","date":"2024-11-04","arxiv_id":"2411.02006","repositories_listed":1,"syntology":null},{"url":"/paper/llms-can-evolve-continually-on-modality-for-x","slug":"llms-can-evolve-continually-on-modality-for-x","title":"LLMs Can Evolve Continually on Modality for X-Modal Reasoning","date":"2024-10-26","arxiv_id":"2410.20178","repositories_listed":1,"syntology":null},{"url":"/paper/spatio-temporal-3d-point-clouds-from-wifi-csi","slug":"spatio-temporal-3d-point-clouds-from-wifi-csi","title":"Spatio-Temporal 3D Point Clouds from WiFi-CSI Data via Transformer Networks","date":"2024-10-07","arxiv_id":"2410.16303","repositories_listed":1,"syntology":null},{"url":"/paper/mamba-enhanced-text-audio-video-alignment","slug":"mamba-enhanced-text-audio-video-alignment","title":"Mamba-Enhanced Text-Audio-Video Alignment Network for Emotion Recognition in Conversations","date":"2024-09-08","arxiv_id":"2409.05243","repositories_listed":1,"syntology":null},{"url":"/paper/dissecting-dissonance-benchmarking-large","slug":"dissecting-dissonance-benchmarking-large","title":"Dissecting Dissonance: Benchmarking Large Multimodal Models Against Self-Contradictory Instructions","date":"2024-08-02","arxiv_id":"2408.01091","repositories_listed":1,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":1,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":1,"n_no_contract":0,"n_pointer_only":2,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 1 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/dissecting-dissonance-benchmarking-large#ran","syntology_url":"https://syntology.ai/paper/2408.01091","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2408.01091"}},"official":{"repos":["shiyegao/Self-Contradictory-Instructions-SCI"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/unimel-a-unified-framework-for-multimodal","slug":"unimel-a-unified-framework-for-multimodal","title":"UniMEL: A Unified Framework for Multimodal Entity Linking with Large Language Models","date":"2024-07-23","arxiv_id":"2407.16160","repositories_listed":1,"syntology":null},{"url":"/paper/tip-tabular-image-pre-training-for-multimodal","slug":"tip-tabular-image-pre-training-for-multimodal","title":"TIP: Tabular-Image Pre-training for Multimodal Classification with Incomplete Data","date":"2024-07-10","arxiv_id":"2407.07582","repositories_listed":1,"syntology":{"n":4,"n_ran":2,"n_constructed":0,"n_ran_checked":1,"n_instrument":1,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":1,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 1 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/tip-tabular-image-pre-training-for-multimodal#ran","syntology_url":"https://syntology.ai/paper/2407.07582","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2407.07582"}},"official":{"repos":["siyi-wind/tip"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/empathic-grounding-explorations-using","slug":"empathic-grounding-explorations-using","title":"Empathic Grounding: Explorations using Multimodal Interaction and Large Language Models with Conversational Agents","date":"2024-07-01","arxiv_id":"2407.01824","repositories_listed":1,"syntology":null},{"url":"/paper/investigating-and-mitigating-the-multimodal","slug":"investigating-and-mitigating-the-multimodal","title":"Investigating and Mitigating the Multimodal Hallucination Snowballing in Large Vision-Language Models","date":"2024-06-30","arxiv_id":"2407.00569","repositories_listed":1,"syntology":{"n":2,"n_ran":0,"n_constructed":0,"n_ran_checked":0,"n_instrument":0,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":2,"phrase":"0 ran · 2 unverified","sample_list":"/paper/investigating-and-mitigating-the-multimodal#ran","syntology_url":"https://syntology.ai/paper/2407.00569","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2407.00569"}},"official":{"repos":["whongzhong/MMHalSnowball"],"state":"official: harvested, nothing ran","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":2,"ran_from_kinds":[]}}},{"url":"/paper/look-m-look-once-optimization-in-kv-cache-for","slug":"look-m-look-once-optimization-in-kv-cache-for","title":"LOOK-M: Look-Once Optimization in KV Cache for Efficient Multimodal Long-Context Inference","date":"2024-06-26","arxiv_id":"2406.18139","repositories_listed":1,"syntology":{"n":5,"n_ran":4,"n_constructed":0,"n_ran_checked":0,"n_instrument":4,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 4 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/look-m-look-once-optimization-in-kv-cache-for#ran","syntology_url":"https://syntology.ai/paper/2406.18139","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.18139"}},"official":{"repos":["sustechbruce/look-m"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/narrative-action-evaluation-with-prompt","slug":"narrative-action-evaluation-with-prompt","title":"Narrative Action Evaluation with Prompt-Guided Multimodal Interaction","date":"2024-04-22","arxiv_id":"2404.14471","repositories_listed":1,"syntology":null},{"url":"/paper/cooperative-sentiment-agents-for-multimodal","slug":"cooperative-sentiment-agents-for-multimodal","title":"Cooperative Sentiment Agents for Multimodal Sentiment Analysis","date":"2024-04-19","arxiv_id":"2404.12642","repositories_listed":1,"syntology":null},{"url":"/paper/draw-and-understand-leveraging-visual-prompts","slug":"draw-and-understand-leveraging-visual-prompts","title":"Draw-and-Understand: Leveraging Visual Prompts to Enable MLLMs to Comprehend What You Want","date":"2024-03-29","arxiv_id":"2403.20271","repositories_listed":1,"syntology":null},{"url":"/paper/agent-ai-surveying-the-horizons-of-multimodal","slug":"agent-ai-surveying-the-horizons-of-multimodal","title":"Agent AI: Surveying the Horizons of Multimodal Interaction","date":"2024-01-07","arxiv_id":"2401.03568","repositories_listed":1,"syntology":{"n":7,"n_ran":6,"n_constructed":0,"n_ran_checked":4,"n_instrument":2,"n_unverified":1,"n_honours":0,"n_violates":2,"n_no_contract":2,"n_pointer_only":7,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 4 with no instrument failure: 0 honoured, 2 violated, 2 with no contract checked; 2 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/agent-ai-surveying-the-horizons-of-multimodal#ran","syntology_url":"https://syntology.ai/paper/2401.03568","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2401.03568"}},"official":{"repos":["guilk/kat"],"state":"official (archive's flag): 6 ran","n_ran":6,"n_constructed":0,"n_ran_no_instrument_failure":4,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/mmoe-mixture-of-multimodal-interaction","slug":"mmoe-mixture-of-multimodal-interaction","title":"MMoE: Enhancing Multimodal Models with Mixtures of Multimodal Interaction Experts","date":"2023-11-16","arxiv_id":"2311.09580","repositories_listed":1,"syntology":{"n":5,"n_ran":4,"n_constructed":0,"n_ran_checked":4,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":4,"n_pointer_only":5,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 4 with no instrument failure: 0 honoured, 0 violated, 4 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/mmoe-mixture-of-multimodal-interaction#ran","syntology_url":"https://syntology.ai/paper/2311.09580","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2311.09580"}},"official":{"repos":["lwaekfjlk/mmoe"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":4,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/dialogue-based-generation-of-self-driving","slug":"dialogue-based-generation-of-self-driving","title":"Dialogue-based generation of self-driving simulation scenarios using Large Language Models","date":"2023-10-26","arxiv_id":"2310.17372","repositories_listed":1,"syntology":null},{"url":"/paper/cfn-esa-a-cross-modal-fusion-network-with","slug":"cfn-esa-a-cross-modal-fusion-network-with","title":"CFN-ESA: A Cross-Modal Fusion Network with Emotion-Shift Awareness for Dialogue Emotion Recognition","date":"2023-07-28","arxiv_id":"2307.15432","repositories_listed":1,"syntology":null},{"url":"/paper/multi-grained-multimodal-interaction-network","slug":"multi-grained-multimodal-interaction-network","title":"Multi-Grained Multimodal Interaction Network for Entity Linking","date":"2023-07-19","arxiv_id":"2307.09721","repositories_listed":1,"syntology":null},{"url":"/paper/a-facial-expression-aware-multimodal-multi","slug":"a-facial-expression-aware-multimodal-multi","title":"A Facial Expression-Aware Multimodal Multi-task Learning Framework for Emotion Recognition in Multi-party Conversations","date":"2023-07-01","arxiv_id":null,"repositories_listed":1,"syntology":null},{"url":"/paper/generative-multimodal-entity-linking","slug":"generative-multimodal-entity-linking","title":"Generative Multimodal Entity Linking","date":"2023-06-22","arxiv_id":"2306.12725","repositories_listed":1,"syntology":null},{"url":"/paper/segment-and-track-anything","slug":"segment-and-track-anything","title":"Segment and Track Anything","date":"2023-05-11","arxiv_id":"2305.06558","repositories_listed":1,"syntology":null},{"url":"/paper/cross-modal-implicit-relation-reasoning-and","slug":"cross-modal-implicit-relation-reasoning-and","title":"Cross-Modal Implicit Relation Reasoning and Aligning for Text-to-Image Person Retrieval","date":"2023-03-22","arxiv_id":"2303.12501","repositories_listed":1,"syntology":{"n":9,"n_ran":5,"n_constructed":3,"n_ran_checked":4,"n_instrument":1,"n_unverified":4,"n_honours":0,"n_violates":0,"n_no_contract":4,"n_pointer_only":0,"phrase":"5 ran (of which 3 constructed an object rather than computing a result; 4 with no instrument failure: 0 honoured, 0 violated, 4 with no contract checked; 1 where Syntology's instrument failed) · 4 unverified","sample_list":"/paper/cross-modal-implicit-relation-reasoning-and#ran","syntology_url":"https://syntology.ai/paper/2303.12501","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2303.12501"}},"official":{"repos":["anosorae/irra"],"state":"official (archive's flag): 5 ran","n_ran":5,"n_constructed":3,"n_ran_no_instrument_failure":4,"n_unverified":4,"ran_from_kinds":["official"]}}},{"url":"/paper/a-comprehensive-survey-of-ai-generated","slug":"a-comprehensive-survey-of-ai-generated","title":"A Comprehensive Survey of AI-Generated Content (AIGC): A History of Generative AI from GAN to ChatGPT","date":"2023-03-07","arxiv_id":"2303.04226","repositories_listed":1,"syntology":null},{"url":"/paper/adaptive-contrastive-learning-on-multimodal","slug":"adaptive-contrastive-learning-on-multimodal","title":"Adaptive Contrastive Learning on Multimodal Transformer for Review Helpfulness Predictions","date":"2022-11-07","arxiv_id":"2211.03524","repositories_listed":1,"syntology":null},{"url":"/paper/multilevel-hierarchical-network-with","slug":"multilevel-hierarchical-network-with","title":"Multilevel Hierarchical Network with Multiscale Sampling for Video Question Answering","date":"2022-05-09","arxiv_id":"2205.04061","repositories_listed":1,"syntology":null},{"url":"/paper/ml-persref-a-machine-learning-based","slug":"ml-persref-a-machine-learning-based","title":"ML-PersRef: A Machine Learning-based Personalized Multimodal Fusion Approach for Referencing Outside Objects From a Moving Vehicle","date":"2021-11-03","arxiv_id":"2111.02327","repositories_listed":1,"syntology":null},{"url":"/paper/mirtt-learning-multimodal-interaction","slug":"mirtt-learning-multimodal-interaction","title":"MIRTT: Learning Multimodal Interaction Representations from Trilinear Transformers for Visual Question Answering","date":"2021-11-01","arxiv_id":null,"repositories_listed":1,"syntology":null},{"url":"/paper/temporal-pyramid-transformer-with-multimodal","slug":"temporal-pyramid-transformer-with-multimodal","title":"Temporal Pyramid Transformer with Multimodal Interaction for Video Question Answering","date":"2021-09-10","arxiv_id":"2109.04735","repositories_listed":1,"syntology":null},{"url":"/paper/dynamic-modality-interaction-modeling-for","slug":"dynamic-modality-interaction-modeling-for","title":"Dynamic Modality Interaction Modeling for Image-Text Retrieval","date":"2021-07-11","arxiv_id":null,"repositories_listed":1,"syntology":null},{"url":"/paper/improving-multimodal-named-entity-recognition","slug":"improving-multimodal-named-entity-recognition","title":"Improving Multimodal Named Entity Recognition via Entity Span Detection with Unified Multimodal Transformer","date":"2020-07-01","arxiv_id":null,"repositories_listed":1,"syntology":null},{"url":"/paper/recurrent-multimodal-interaction-for","slug":"recurrent-multimodal-interaction-for","title":"Recurrent Multimodal Interaction for Referring Image Segmentation","date":"2017-03-23","arxiv_id":"1703.07939","repositories_listed":1,"syntology":null},{"url":null,"slug":"a-multi-stage-augmented-multimodal","title":"A multi-stage augmented multimodal interaction network for fish feeding intensity quantification","date":"2025-06-17","arxiv_id":"2506.14170","repositories_listed":0,"syntology":null},{"url":null,"slug":"intermt-multi-turn-interleaved-preference","title":"InterMT: Multi-Turn Interleaved Preference Alignment with Human Feedback","date":"2025-05-29","arxiv_id":"2505.23950","repositories_listed":0,"syntology":null},{"url":null,"slug":"deepsort-driven-visual-tracking-approach-for","title":"DeepSORT-Driven Visual Tracking Approach for Gesture Recognition in Interactive Systems","date":"2025-05-11","arxiv_id":"2505.07110","repositories_listed":0,"syntology":null},{"url":null,"slug":"a-survey-of-interactive-generative-video","title":"A Survey of Interactive Generative Video","date":"2025-04-30","arxiv_id":"2504.21853","repositories_listed":0,"syntology":null},{"url":null,"slug":"2506-10004","title":"Immersive Multimedia Communication: State-of-the-Art on eXtended Reality Streaming","date":"2025-03-27","arxiv_id":"2506.10004","repositories_listed":0,"syntology":null},{"url":null,"slug":"revision-a-dataset-and-baseline-vlm-for","title":"ReVision: A Dataset and Baseline VLM for Privacy-Preserving Task-Oriented Visual Instruction Rewriting","date":"2025-02-20","arxiv_id":"2502.14780","repositories_listed":0,"syntology":null},{"url":null,"slug":"interactive-sketchpad-a-multimodal-tutoring","title":"Interactive Sketchpad: A Multimodal Tutoring System for Collaborative, Visual Problem-Solving","date":"2025-02-12","arxiv_id":"2503.16434","repositories_listed":0,"syntology":null},{"url":null,"slug":"fgu3r-fine-grained-fusion-via-unified-3d","title":"FGU3R: Fine-Grained Fusion via Unified 3D Representation for Multimodal 3D Object Detection","date":"2025-01-08","arxiv_id":"2501.04373","repositories_listed":0,"syntology":null},{"url":null,"slug":"modfinity-unsupervised-domain-adaptation-with","title":"MODfinity: Unsupervised Domain Adaptation with Multimodal Information Flow Intertwining","date":"2025-01-01","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"computer-vision-driven-gesture-recognition","title":"Computer Vision-Driven Gesture Recognition: Toward Natural and Intuitive Human-Computer","date":"2024-12-24","arxiv_id":"2412.18321","repositories_listed":0,"syntology":null},{"url":null,"slug":"cmath-cross-modality-augmented-transformer","title":"CMATH: Cross-Modality Augmented Transformer with Hierarchical Variational Distillation for Multimodal Emotion Recognition in Conversation","date":"2024-11-15","arxiv_id":"2411.10060","repositories_listed":0,"syntology":null},{"url":null,"slug":"generative-ai-in-multimodal-user-interfaces","title":"Generative AI in Multimodal User Interfaces: Trends, Challenges, and Cross-Platform Adaptability","date":"2024-11-15","arxiv_id":"2411.10234","repositories_listed":0,"syntology":null},{"url":null,"slug":"phase-diagram-of-vision-large-language-models","title":"Phase Diagram of Vision Large Language Models Inference: A Perspective from Interaction across Image and Instruction","date":"2024-11-01","arxiv_id":"2411.00646","repositories_listed":0,"syntology":null},{"url":null,"slug":"analyzing-multimodal-interaction-strategies","title":"Analyzing Multimodal Interaction Strategies for LLM-Assisted Manipulation of 3D Scenes","date":"2024-10-29","arxiv_id":"2410.22177","repositories_listed":0,"syntology":null},{"url":null,"slug":"retrospective-learning-from-interactions","title":"Retrospective Learning from Interactions","date":"2024-10-17","arxiv_id":"2410.13852","repositories_listed":0,"syntology":null},{"url":null,"slug":"robi-butler-remote-multimodal-interactions","title":"Robi Butler: Multimodal Remote Interaction with a Household Robot Assistant","date":"2024-09-30","arxiv_id":"2409.20548","repositories_listed":0,"syntology":null},{"url":null,"slug":"llm-assisted-visual-analytics-opportunities","title":"LLM-Assisted Visual Analytics: Opportunities and Challenges","date":"2024-09-04","arxiv_id":"2409.02691","repositories_listed":0,"syntology":null},{"url":"/paper/rgbt-tracking-via-all-layer-multimodal","slug":"rgbt-tracking-via-all-layer-multimodal","title":"RGBT Tracking via All-layer Multimodal Interactions with Progressive Fusion Mamba","date":"2024-08-16","arxiv_id":"2408.08827","repositories_listed":0,"syntology":null},{"url":null,"slug":"a-unified-understanding-of-adversarial","title":"A Unified Understanding of Adversarial Vulnerability Regarding Unimodal Models and Vision-Language Pre-training Models","date":"2024-07-25","arxiv_id":"2407.17797","repositories_listed":0,"syntology":null},{"url":null,"slug":"dallah-a-dialect-aware-multimodal-large","title":"Dallah: A Dialect-Aware Multimodal Large Language Model for Arabic","date":"2024-07-25","arxiv_id":"2407.18129","repositories_listed":0,"syntology":null},{"url":null,"slug":"hgnet-a-hierarchical-feature-guided-network","title":"HGNET: A Hierarchical Feature Guided Network for Occupancy Flow Field Prediction","date":"2024-07-01","arxiv_id":"2407.01097","repositories_listed":0,"syntology":null},{"url":null,"slug":"a-look-under-the-hood-of-the-interactive-deep","title":"A look under the hood of the Interactive Deep Learning Enterprise (No-IDLE)","date":"2024-06-27","arxiv_id":"2406.19054","repositories_listed":0,"syntology":null},{"url":null,"slug":"omnijarvis-unified-vision-language-action","title":"OmniJARVIS: Unified Vision-Language-Action Tokenization Enables Open-World Instruction Following Agents","date":"2024-06-27","arxiv_id":"2407.00114","repositories_listed":0,"syntology":null},{"url":null,"slug":"emmi-empathic-multimodal-motivational","title":"EMMI -- Empathic Multimodal Motivational Interviews Dataset: Analyses and Annotations","date":"2024-06-24","arxiv_id":"2406.16478","repositories_listed":0,"syntology":null},{"url":"/paper/revisiting-multimodal-emotion-recognition-in","slug":"revisiting-multimodal-emotion-recognition-in","title":"Revisiting Multimodal Emotion Recognition in Conversation from the Perspective of Graph Spectrum","date":"2024-04-27","arxiv_id":"2404.17862","repositories_listed":0,"syntology":null},{"url":null,"slug":"blendscape-enabling-unified-and-personalized","title":"BlendScape: Enabling End-User Customization of Video-Conferencing Environments through Generative AI","date":"2024-03-20","arxiv_id":"2403.13947","repositories_listed":0,"syntology":null},{"url":null,"slug":"improving-adversarial-transferability-of","title":"Improving Adversarial Transferability of Vision-Language Pre-training Models through Collaborative Multimodal Interaction","date":"2024-03-16","arxiv_id":"2403.10883","repositories_listed":0,"syntology":null},{"url":null,"slug":"on-organizational-principles-of-neural","title":"On the Arrow of Inference","date":"2024-02-22","arxiv_id":"2402.14186","repositories_listed":0,"syntology":null},{"url":null,"slug":"memory-inspired-temporal-prompt-interaction","title":"Memory-Inspired Temporal Prompt Interaction for Text-Image Classification","date":"2024-01-26","arxiv_id":"2401.14856","repositories_listed":0,"syntology":null},{"url":null,"slug":"dynamic-hand-gesture-featured-human-motor","title":"Dynamic Hand Gesture-Featured Human Motor Adaptation in Tool Delivery using Voice Recognition","date":"2023-09-20","arxiv_id":"2309.11368","repositories_listed":0,"syntology":null},{"url":null,"slug":"adaptive-user-centered-neuro-symbolic","title":"Adaptive User-centered Neuro-symbolic Learning for Multimodal Interaction with Autonomous Systems","date":"2023-09-11","arxiv_id":"2309.05787","repositories_listed":0,"syntology":null},{"url":null,"slug":"expanding-the-role-of-affective-phenomena-in","title":"Expanding the Role of Affective Phenomena in Multimodal Interaction Research","date":"2023-05-18","arxiv_id":"2305.10827","repositories_listed":0,"syntology":null},{"url":"/paper/html-hybrid-temporal-scale-multimodal","slug":"html-hybrid-temporal-scale-multimodal","title":"HTML: Hybrid Temporal-scale Multimodal Learning Framework for Referring Video Object Segmentation","date":"2023-01-01","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"intermulti-multi-view-multimodal-interactions","title":"InterMulti:Multi-view Multimodal Interactions with Text-dominated Hierarchical High-order Fusion for Emotion Analysis","date":"2022-12-20","arxiv_id":"2212.10030","repositories_listed":0,"syntology":null},{"url":null,"slug":"a-novel-multimodal-dynamic-fusion-network-for","title":"A novel multimodal dynamic fusion network for disfluency detection in spoken utterances","date":"2022-11-27","arxiv_id":"2211.14700","repositories_listed":0,"syntology":null},{"url":null,"slug":"adaptive-user-centered-multimodal-interaction","title":"Adaptive User-Centered Multimodal Interaction towards Reliable and Trusted Automotive Interfaces","date":"2022-11-07","arxiv_id":"2211.03539","repositories_listed":0,"syntology":null},{"url":null,"slug":"semantics-consistent-cross-domain","title":"Semantics-Consistent Cross-domain Summarization via Optimal Transport Alignment","date":"2022-10-10","arxiv_id":"2210.04722","repositories_listed":0,"syntology":null},{"url":null,"slug":"mamo-masked-multimodal-modeling-for-fine","title":"MAMO: Masked Multimodal Modeling for Fine-Grained Vision-Language Representation Learning","date":"2022-10-09","arxiv_id":"2210.04183","repositories_listed":0,"syntology":null},{"url":null,"slug":"on-the-horizon-interactive-and-compositional","title":"On the Horizon: Interactive and Compositional Deepfakes","date":"2022-09-05","arxiv_id":"2209.01714","repositories_listed":0,"syntology":null},{"url":null,"slug":"chat-to-design-ai-assisted-personalized","title":"Chat-to-Design: AI Assisted Personalized Fashion Design","date":"2022-07-03","arxiv_id":"2207.01058","repositories_listed":0,"syntology":null},{"url":null,"slug":"roomreader-a-multimodal-corpus-of-online","title":"RoomReader: A Multimodal Corpus of Online Multiparty Conversational Interactions","date":"2022-06-01","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"the-voxworld-platform-for-multimodal-embodied","title":"The VoxWorld Platform for Multimodal Embodied Agents","date":"2022-06-01","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"graph-based-fine-grained-multimodal-attention","title":"Graph-based Fine-grained Multimodal Attention Mechanism for Sentiment Analysis","date":"2021-11-16","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"neural-dsca-demixing-multimodal-interaction","title":"Neural dSCA: demixing multimodal interaction among brain areas during naturalistic experiments","date":"2021-06-05","arxiv_id":"2106.02948","repositories_listed":0,"syntology":null},{"url":null,"slug":"socialinteractiongan-multi-person-interaction","title":"SocialInteractionGAN: Multi-person Interaction Sequence Generation","date":"2021-03-10","arxiv_id":"2103.05916","repositories_listed":0,"syntology":null},{"url":null,"slug":"dyadic-speech-based-affect-recognition-using","title":"Dyadic Speech-based Affect Recognition using DAMI-P2C Parent-child Multimodal Interaction Dataset","date":"2020-08-20","arxiv_id":"2008.09207","repositories_listed":0,"syntology":null},{"url":null,"slug":"sbat-video-captioning-with-sparse-boundary","title":"SBAT: Video Captioning with Sparse Boundary-Aware Transformer","date":"2020-07-23","arxiv_id":"2007.11888","repositories_listed":0,"syntology":null},{"url":null,"slug":"dual-convolutional-lstm-network-for-referring","title":"Dual Convolutional LSTM Network for Referring Image Segmentation","date":"2020-01-30","arxiv_id":"2001.11561","repositories_listed":0,"syntology":null},{"url":null,"slug":"scene-graph-based-image-retrieval-a-case","title":"Scene Graph based Image Retrieval -- A case study on the CLEVR Dataset","date":"2019-11-03","arxiv_id":"1911.00850","repositories_listed":0,"syntology":null},{"url":null,"slug":"corpus-of-multimodal-interaction-for","title":"Corpus of Multimodal Interaction for Collaborative Planning","date":"2019-06-01","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"bridging-dialogue-generation-and-facial","title":"HUMBO: Bridging Response Generation and Facial Expression Synthesis","date":"2019-05-24","arxiv_id":"1905.11240","repositories_listed":0,"syntology":null},{"url":null,"slug":"guidelines-for-creating-man-machine","title":"Guidelines for creating man-machine multimodal interfaces","date":"2019-01-29","arxiv_id":"1901.10408","repositories_listed":0,"syntology":null},{"url":null,"slug":"shaping-a-social-robots-humor-with-natural","title":"Shaping a social robot's humor with Natural Language Generation and socially-aware reinforcement learning","date":"2018-11-01","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"emotiw-2018-audio-video-student-engagement","title":"EmotiW 2018: Audio-Video, Student Engagement and Group-Level Affect Prediction","date":"2018-08-23","arxiv_id":"1808.07773","repositories_listed":0,"syntology":null},{"url":"/paper/multimodal-interaction-aware-motion","slug":"multimodal-interaction-aware-motion","title":"Multimodal Interaction-aware Motion Prediction for Autonomous Street Crossing","date":"2018-08-21","arxiv_id":"1808.06887","repositories_listed":0,"syntology":null},{"url":null,"slug":"toward-multimodal-interaction-in-scalable","title":"Toward Multimodal Interaction in Scalable Visual Digital Evidence Visualization Using Computer Vision Techniques and ISS","date":"2018-08-01","arxiv_id":"1808.00118","repositories_listed":0,"syntology":null},{"url":null,"slug":"an-evaluation-framework-for-multimodal","title":"An Evaluation Framework for Multimodal Interaction","date":"2018-05-01","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"automatized-generation-of-alphabets-of","title":"Automatized Generation of Alphabets of Symbols","date":"2017-07-16","arxiv_id":"1707.04935","repositories_listed":0,"syntology":null},{"url":null,"slug":"from-modal-to-multimodal-ambiguities-a","title":"From Modal to Multimodal Ambiguities: a Classification Approach","date":"2017-04-04","arxiv_id":"1704.02841","repositories_listed":0,"syntology":null}],"syntology_records":10,"syntology_note":"a paper without a record is not a recorded non-run: it may lack an arXiv id or simply be absent from the graph layer"},"key_notes":{"n_ran_checked":"legacy name, kept unchanged so existing readers do not break: it counts the samples that ran with no instrument failure (honoured, violated, and ran with no contract checked); it does not mean a contract was checked, and the pages print it as 'K with no instrument failure', not 'K checked'","n_constructed":"a sub-count of the samples that ran, never subtracted from them and never a failure: an executed sample whose run returned an instance of its own class (fixture_out_type equals the entry name): the run built an object and did not compute a result (Syntology's RAN record, counts.constructed)"},"since_archive_judged":{"papers":0,"papers_placed":0,"judged_by":{"model":"a model","models_as_recorded":[],"from":"lb_rows_run_verified.jsonl: each record's `model` field (the extractor's call that wrote decisions[])"},"note":"a model's judgement from the paper's own tables that they report on this leaderboard, not a result: no value from a paper that was not placed is printed or carried here; papers counts each paper once over these leaderboards","backfill":"8,886 of 18,637 newer papers checked so far (733 more had nothing to place; 41 could not be read)","backfill_complete":false,"leaderboards":[]},"papers_where_code_ran":9,"papers_where_code_ran_first_page":"/task/multimodal-interaction/papers/ran/1","papers_where_code_ran_caption":"We ran code from the paper's repository; we did not run it on this task or check it against the task's benchmarks.","pages":[{"order":"archive","page":2,"of":2,"url":"/task/multimodal-interaction/papers/2","json":"/task/multimodal-interaction/papers/2.json","rows":6},{"order":"ran","page":1,"of":1,"url":"/task/multimodal-interaction/papers/ran/1","json":"/task/multimodal-interaction/papers/ran/1.json","rows":9}],"description_links":{"kept":0,"unwrapped_to_text":0,"bare_urls_linked":0,"relative_images_dropped":0,"rule":"internal links are kept only when the target slug exists in the catalog"},"syntology":{"read_at":"2026-10-05T00:36:11+00:00","claim":"Per-sample execution status on synthesized fixtures ('ran N of M samples'); not a correctness claim and not a ranking signal.","status_vocabulary":{"ran_honours":"ran, honoured the contract we drafted","ran_violates":"ran, violated the contract we drafted","ran_draft_wrong":"ran; our contract draft was wrong, not the code","ran_fixture":"ran; the contract check could not run (RAISES, DEP_MISSING or TIMEOUT)","ran":"ran on a synthesized input","unverified":"unverified (harvested, no recorded run)"},"row_route":"each syntology object on a benchmark's first row and on a paper carries sample_list (this site's paper page, #ran), syntology_url (syntology.ai's page for the paper, only when that route probed live) and mcp (the agent call that lists the paper's harvested samples; not a verdict on the paper)","developers":"https://syntology.ai/developers"},"not_shown":{"libraries":"the archive has no per-task library table","trend_sparklines":"the Trend column of the benchmarks table was a rendered image; it is not in the archive","social_and_latest_sorts":"stars and social signals are not in the archive"},"record_sha256":"9fe3b6624d63e3a8d1e506aabfd15ce97a492d3f62f721c39008c5b8ec5236cb","record_changed_at":"2026-10-05","record_changed_at_basis":"changed"}