{"about":{"non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","site":"https://codewithpapers.app","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page","syntology":{"site":"https://syntology.ai","developers":"https://syntology.ai/developers","mcp":{"server":"https://syntology.ai/mcp","transport":"streamable-http","server_card":"https://syntology.ai/.well-known/mcp/server-card.json","auth":{"type":"trial token, no account","trial_token":"https://syntology.ai/api/oauth/trial/token","method":"POST","docs":"https://syntology.ai/developers"}},"have":"https://syntology.ai/api/graph/have?x=<method, arXiv id or title> (free, answers coverage only)","paper_base":"https://syntology.ai/paper/","atlas_base":"https://app.syntology.ai/?focus="},"machine_readable":[{"url":"https://codewithpapers.app/llms.txt","what":"the machine catalog: every machine-readable file, counted"},{"url":"https://codewithpapers.app/index/manifest.json","what":"paper-to-code index by arXiv id, with Syntology's counts"},{"url":"https://codewithpapers.app/search/manifest.json","what":"site search index (titles, authors) and its files"},{"url":"https://codewithpapers.app/download","what":"bulk files: Syntology's layer, described there"},{"url":"https://codewithpapers.app/build_manifest.json","what":"the build record: inputs, counts, exclusions, probes"}]},"url":"/task/speech-to-speech-translation","name":"Speech-to-Speech Translation","slug":"speech-to-speech-translation","description_markdown":"Speech-to-speech translation (S2ST) consists on translating speech from one language to speech in another language. This can be done with a cascade of automatic speech recognition (ASR), text-to-text machine translation (MT), and text-to-speech (TTS) synthesis\r\nsub-systems, which is text-centric. Recently, works on S2ST without relying on intermediate text representation is emerging.","categories":[{"name":"Speech","url":"/area/speech"}],"source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","slug_source":"archive_url"},"counts":{"papers_tagged":117,"papers_with_code":39,"benchmarks":3,"benchmark_tables_in_archive":3,"benchmark_tables_shown":3,"benchmark_tables_withheld_as_spam":0,"benchmark_definition":"a leaderboard table with at least one row; benchmark_tables_shown also counts the zero-row tables; benchmark_tables_in_archive adds the tables withheld as spam","datasets":5,"subtasks":0,"parent_tasks":0},"benchmarks":[{"leaderboard":"/sota/speech-to-speech-translation-on-tat","slug":"speech-to-speech-translation-on-tat","dataset":"TAT","dataset_url":"/dataset/tat","rows_in_archive":8,"metrics":["ASR-BLEU (Dev)","ASR-BLEU (Test)"],"first_row_in_archive_order":{"model":"Hokkien→En (Two-pass decoding)","paper_title":"Speech-to-speech translation for a real-world unwritten language","paper_url":"/paper/speech-to-speech-translation-for-a-real-world","paper_date":"2022-10-19","arxiv_id":null,"code_links":[{"title":"facebookresearch/fairseq","url":"https://github.com/facebookresearch/fairseq/tree/ust/examples/hokkien"}],"syntology":null}},{"leaderboard":"/sota/speech-to-speech-translation-on-fleurs-x-eng","slug":"speech-to-speech-translation-on-fleurs-x-eng","dataset":"FLEURS X-eng","dataset_url":null,"rows_in_archive":7,"metrics":["ASR-BLEU"],"first_row_in_archive_order":{"model":"GenTranslateV2","paper_title":"GenTranslate: Large Language Models are Generative Multilingual Speech and Machine Translators","paper_url":"/paper/gentranslate-large-language-models-are","paper_date":"2024-02-10","arxiv_id":"2402.06894","code_links":[{"title":"yuchen005/gentranslate","url":"https://github.com/yuchen005/gentranslate"}],"syntology":null}},{"leaderboard":"/sota/speech-to-speech-translation-on-cvss","slug":"speech-to-speech-translation-on-cvss","dataset":"CVSS","dataset_url":"/dataset/cvss","rows_in_archive":2,"metrics":["ASR-BLEU","Parameters"],"first_row_in_archive_order":{"model":"SeamlessM4T Large","paper_title":"SeamlessM4T: Massively Multilingual & Multimodal Machine Translation","paper_url":"/paper/seamlessm4t-massively-multilingual-multimodal","paper_date":"2023-08-22","arxiv_id":"2308.11596","code_links":[{"title":"facebookresearch/seamless_communication","url":"https://github.com/facebookresearch/seamless_communication"},{"title":"facebookresearch/sonar","url":"https://github.com/facebookresearch/sonar"},{"title":"yangyucheng000/University","url":"https://github.com/yangyucheng000/University/tree/main/model-3/seamless_m4t"},{"title":"pwc-1/Paper-10","url":"https://github.com/pwc-1/Paper-10/tree/main/seamless_m4t"}],"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":0,"n_instrument":2,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":2,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/seamlessm4t-massively-multilingual-multimodal#ran","syntology_url":"https://syntology.ai/paper/2308.11596","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2308.11596"}},"official":{"repos":["facebookresearch/seamless_communication"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}}}],"datasets":[{"url":"/dataset/cvss","name":"CVSS","full_name":"","num_papers_in_archive":26},{"url":"/dataset/speechmatrix","name":"SpeechMatrix","full_name":"","num_papers_in_archive":6},{"url":"/dataset/heroes-corpus","name":"Heroes Corpus","full_name":"","num_papers_in_archive":1},{"url":"/dataset/libris2s","name":"LibriS2S","full_name":"","num_papers_in_archive":1},{"url":"/dataset/tat","name":"TAT","full_name":"Taiwanese Across Taiwan","num_papers_in_archive":1}],"subtasks":[],"parent_tasks":[],"papers":{"order":"repositories listed in the archive (most first), then date (newest first), then slug; the archive holds no stars","population":"every paper the archive tags with this task (distinct papers; spam titles not listed). 'of' counts the papers listed; until 2026-09-25 it counted only the papers with a code link, and that number is of_with_code","page":1,"pages_in_order":2,"rows_per_page":100,"shown":100,"of":117,"of_with_code":39,"tagged_in_all":117,"counts":{"archive_papers_tagged":117,"with_a_code_link":39,"where_syntology_ran_a_sample":12,"not_listed_spam_title":0,"listed":117,"listed_where_code_ran":12,"where_syntology_ran_a_sample_split":{"with_a_run_with_no_instrument_failure":10,"every_run_a_failure_of_syntologys_instrument":2,"listed_with_a_run_with_no_instrument_failure":10,"listed_every_run_a_failure_of_syntologys_instrument":2,"filter":{"states":["a run with no instrument failure","any run, instrument failures included"],"default":"a run with no instrument failure","note":"on the 'only where code ran' pages the default hides, in the browser, the rows where every run was a failure of Syntology's instrument; the second state shows them again. Rows are hidden, never re-ordered; these twins list every row"}},"definition":"distinct papers the archive tags; 'where Syntology ran a sample' counts papers with at least one harvested sample that ran, which is not a correctness claim"},"items":[{"url":"/paper/robust-speech-recognition-via-large-scale-1","slug":"robust-speech-recognition-via-large-scale-1","title":"Robust Speech Recognition via Large-Scale Weak Supervision","date":"2022-12-06","arxiv_id":"2212.04356","repositories_listed":15,"syntology":{"n":59,"n_ran":49,"n_constructed":0,"n_ran_checked":36,"n_instrument":13,"n_unverified":10,"n_honours":4,"n_violates":1,"n_no_contract":31,"n_pointer_only":21,"phrase":"49 ran (of which 0 constructed an object rather than computing a result; 36 with no instrument failure: 4 honoured, 1 violated, 31 with no contract checked; 13 where Syntology's instrument failed) · 10 unverified","sample_list":"/paper/robust-speech-recognition-via-large-scale-1#ran","syntology_url":"https://syntology.ai/paper/2212.04356","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2212.04356"}},"official":{"repos":["openai/whisper"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/audiolm-a-language-modeling-approach-to-audio","slug":"audiolm-a-language-modeling-approach-to-audio","title":"AudioLM: a Language Modeling Approach to Audio Generation","date":"2022-09-07","arxiv_id":"2209.03143","repositories_listed":6,"syntology":{"n":15,"n_ran":12,"n_constructed":0,"n_ran_checked":10,"n_instrument":2,"n_unverified":3,"n_honours":0,"n_violates":2,"n_no_contract":8,"n_pointer_only":4,"phrase":"12 ran (of which 0 constructed an object rather than computing a result; 10 with no instrument failure: 0 honoured, 2 violated, 8 with no contract checked; 2 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/audiolm-a-language-modeling-approach-to-audio#ran","syntology_url":"https://syntology.ai/paper/2209.03143","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2209.03143"}},"official":null}},{"url":"/paper/seamlessm4t-massively-multilingual-multimodal","slug":"seamlessm4t-massively-multilingual-multimodal","title":"SeamlessM4T: Massively Multilingual & Multimodal Machine Translation","date":"2023-08-22","arxiv_id":"2308.11596","repositories_listed":4,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":0,"n_instrument":2,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":2,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/seamlessm4t-massively-multilingual-multimodal#ran","syntology_url":"https://syntology.ai/paper/2308.11596","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2308.11596"}},"official":{"repos":["facebookresearch/seamless_communication"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/funaudiollm-voice-understanding-and","slug":"funaudiollm-voice-understanding-and","title":"FunAudioLLM: Voice Understanding and Generation Foundation Models for Natural Interaction Between Humans and LLMs","date":"2024-07-04","arxiv_id":"2407.04051","repositories_listed":3,"syntology":{"n":10,"n_ran":10,"n_constructed":0,"n_ran_checked":4,"n_instrument":6,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":3,"n_pointer_only":0,"phrase":"10 ran (of which 0 constructed an object rather than computing a result; 4 with no instrument failure: 1 honoured, 0 violated, 3 with no contract checked; 6 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/funaudiollm-voice-understanding-and#ran","syntology_url":"https://syntology.ai/paper/2407.04051","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2407.04051"}},"official":{"repos":["FunAudioLLM/SenseVoice","funaudiollm/cosyvoice"],"state":"official (archive's flag): 9 ran","n_ran":9,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":0,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/unit-based-speech-to-speech-translation","slug":"unit-based-speech-to-speech-translation","title":"Textless Speech-to-Speech Translation With Limited Parallel Data","date":"2023-05-24","arxiv_id":"2305.15405","repositories_listed":2,"syntology":null},{"url":"/paper/leveraging-unit-language-guidance-to-advance","slug":"leveraging-unit-language-guidance-to-advance","title":"Leveraging Unit Language Guidance to Advance Speech Modeling in Textless Speech-to-Speech Translation","date":"2025-05-21","arxiv_id":"2505.15333","repositories_listed":1,"syntology":null},{"url":"/paper/using-phonemes-in-cascaded-s2s-translation","slug":"using-phonemes-in-cascaded-s2s-translation","title":"Using Phonemes in cascaded S2S translation pipeline","date":"2025-04-22","arxiv_id":"2504.16234","repositories_listed":1,"syntology":null},{"url":"/paper/high-fidelity-simultaneous-speech-to-speech","slug":"high-fidelity-simultaneous-speech-to-speech","title":"High-Fidelity Simultaneous Speech-To-Speech Translation","date":"2025-02-05","arxiv_id":"2502.03382","repositories_listed":1,"syntology":null},{"url":"/paper/improving-speech-emotion-recognition-in-under","slug":"improving-speech-emotion-recognition-in-under","title":"Improving Speech Emotion Recognition in Under-Resourced Languages via Speech-to-Speech Translation with Bootstrapping Data Selection","date":"2024-09-17","arxiv_id":"2409.10985","repositories_listed":1,"syntology":null},{"url":"/paper/laugh-now-cry-later-controlling-time-varying","slug":"laugh-now-cry-later-controlling-time-varying","title":"Laugh Now Cry Later: Controlling Time-Varying Emotional States of Flow-Matching-Based Zero-Shot Text-to-Speech","date":"2024-07-17","arxiv_id":"2407.12229","repositories_listed":1,"syntology":null},{"url":"/paper/a-non-autoregressive-generation-framework-for","slug":"a-non-autoregressive-generation-framework-for","title":"A Non-autoregressive Generation Framework for End-to-End Simultaneous Speech-to-Speech Translation","date":"2024-06-11","arxiv_id":"2406.06937","repositories_listed":1,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":1,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":2,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/a-non-autoregressive-generation-framework-for#ran","syntology_url":"https://syntology.ai/paper/2406.06937","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.06937"}},"official":{"repos":["ictnlp/nast-s2x"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/ctc-based-non-autoregressive-textless-speech","slug":"ctc-based-non-autoregressive-textless-speech","title":"CTC-based Non-autoregressive Textless Speech-to-Speech Translation","date":"2024-06-11","arxiv_id":"2406.07330","repositories_listed":1,"syntology":null},{"url":"/paper/streamspeech-simultaneous-speech-to-speech","slug":"streamspeech-simultaneous-speech-to-speech","title":"StreamSpeech: Simultaneous Speech-to-Speech Translation with Multi-task Learning","date":"2024-06-05","arxiv_id":"2406.03049","repositories_listed":1,"syntology":{"n":4,"n_ran":4,"n_constructed":0,"n_ran_checked":2,"n_instrument":2,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":0,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 2 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/streamspeech-simultaneous-speech-to-speech#ran","syntology_url":"https://syntology.ai/paper/2406.03049","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.03049"}},"official":{"repos":["ictnlp/streamspeech"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/transvip-speech-to-speech-translation-system","slug":"transvip-speech-to-speech-translation-system","title":"TransVIP: Speech to Speech Translation System with Voice and Isochrony Preservation","date":"2024-05-28","arxiv_id":"2405.17809","repositories_listed":1,"syntology":null},{"url":"/paper/diffnorm-self-supervised-normalization-for","slug":"diffnorm-self-supervised-normalization-for","title":"DiffNorm: Self-Supervised Normalization for Non-autoregressive Speech-to-speech Translation","date":"2024-05-22","arxiv_id":"2405.13274","repositories_listed":1,"syntology":null},{"url":"/paper/gentranslate-large-language-models-are","slug":"gentranslate-large-language-models-are","title":"GenTranslate: Large Language Models are Generative Multilingual Speech and Machine Translators","date":"2024-02-10","arxiv_id":"2402.06894","repositories_listed":1,"syntology":null},{"url":"/paper/emphassess-a-prosodic-benchmark-on-assessing","slug":"emphassess-a-prosodic-benchmark-on-assessing","title":"EmphAssess : a Prosodic Benchmark on Assessing Emphasis Transfer in Speech-to-Speech Models","date":"2023-12-21","arxiv_id":"2312.14069","repositories_listed":1,"syntology":null},{"url":"/paper/av2av-direct-audio-visual-speech-to-audio","slug":"av2av-direct-audio-visual-speech-to-audio","title":"AV2AV: Direct Audio-Visual Speech to Audio-Visual Speech Translation with Unified Audio-Visual Speech Representation","date":"2023-12-05","arxiv_id":"2312.02512","repositories_listed":1,"syntology":{"n":7,"n_ran":6,"n_constructed":0,"n_ran_checked":4,"n_instrument":2,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":4,"n_pointer_only":6,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 4 with no instrument failure: 0 honoured, 0 violated, 4 with no contract checked; 2 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/av2av-direct-audio-visual-speech-to-audio#ran","syntology_url":"https://syntology.ai/paper/2312.02512","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2312.02512"}},"official":{"repos":["choijeongsoo/av2av"],"state":"official (archive's flag): 6 ran","n_ran":6,"n_constructed":0,"n_ran_no_instrument_failure":4,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/daspeech-directed-acyclic-transformer-for-1","slug":"daspeech-directed-acyclic-transformer-for-1","title":"DASpeech: Directed Acyclic Transformer for Fast and High-quality Speech-to-Speech Translation","date":"2023-10-11","arxiv_id":"2310.07403","repositories_listed":1,"syntology":{"n":7,"n_ran":5,"n_constructed":0,"n_ran_checked":5,"n_instrument":0,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":5,"n_pointer_only":7,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 0 honoured, 0 violated, 5 with no contract checked; 0 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/daspeech-directed-acyclic-transformer-for-1#ran","syntology_url":"https://syntology.ai/paper/2310.07403","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2310.07403"}},"official":{"repos":["ictnlp/daspeech"],"state":"official (archive's flag): 5 ran","n_ran":5,"n_constructed":0,"n_ran_no_instrument_failure":5,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/many-to-many-spoken-language-translation-via","slug":"many-to-many-spoken-language-translation-via","title":"Textless Unit-to-Unit training for Many-to-Many Multilingual Speech-to-Speech Translation","date":"2023-08-03","arxiv_id":"2308.01831","repositories_listed":1,"syntology":null},{"url":"/paper/towards-cross-language-prosody-transfer-for","slug":"towards-cross-language-prosody-transfer-for","title":"Towards cross-language prosody transfer for dialog","date":"2023-07-09","arxiv_id":"2307.04123","repositories_listed":1,"syntology":null},{"url":"/paper/learning-when-to-speak-latency-and-quality","slug":"learning-when-to-speak-latency-and-quality","title":"Learning When to Speak: Latency and Quality Trade-offs for Simultaneous Speech-to-Speech Translation with Offline Models","date":"2023-06-01","arxiv_id":"2306.01201","repositories_listed":1,"syntology":null},{"url":"/paper/espnet-st-v2-multipurpose-spoken-language","slug":"espnet-st-v2-multipurpose-spoken-language","title":"ESPnet-ST-v2: Multipurpose Spoken Language Translation Toolkit","date":"2023-04-10","arxiv_id":"2304.04596","repositories_listed":1,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":0,"n_instrument":3,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":1,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 3 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/espnet-st-v2-multipurpose-spoken-language#ran","syntology_url":"https://syntology.ai/paper/2304.04596","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2304.04596"}},"official":{"repos":["espnet/espnet"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/speak-foreign-languages-with-your-own-voice","slug":"speak-foreign-languages-with-your-own-voice","title":"Speak Foreign Languages with Your Own Voice: Cross-Lingual Neural Codec Language Modeling","date":"2023-03-07","arxiv_id":"2303.03926","repositories_listed":1,"syntology":{"n":6,"n_ran":6,"n_constructed":0,"n_ran_checked":3,"n_instrument":3,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":0,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 3 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/speak-foreign-languages-with-your-own-voice#ran","syntology_url":"https://syntology.ai/paper/2303.03926","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2303.03926"}},"official":null}},{"url":"/paper/blaser-a-text-free-speech-to-speech","slug":"blaser-a-text-free-speech-to-speech","title":"BLASER: A Text-Free Speech-to-Speech Translation Evaluation Metric","date":"2022-12-16","arxiv_id":"2212.08486","repositories_listed":1,"syntology":null},{"url":"/paper/unity-two-pass-direct-speech-to-speech","slug":"unity-two-pass-direct-speech-to-speech","title":"UnitY: Two-pass Direct Speech-to-speech Translation with Discrete Units","date":"2022-12-15","arxiv_id":"2212.08055","repositories_listed":1,"syntology":null},{"url":"/paper/dialogs-re-enacted-across-languages","slug":"dialogs-re-enacted-across-languages","title":"Dialogs Re-enacted Across Languages","date":"2022-11-18","arxiv_id":"2211.11584","repositories_listed":1,"syntology":null},{"url":"/paper/joint-pre-training-with-speech-and-bilingual","slug":"joint-pre-training-with-speech-and-bilingual","title":"Joint Pre-Training with Speech and Bilingual Text for Direct Speech to Speech Translation","date":"2022-10-31","arxiv_id":"2210.17027","repositories_listed":1,"syntology":null},{"url":"/paper/a-textless-metric-for-speech-to-speech","slug":"a-textless-metric-for-speech-to-speech","title":"A Textless Metric for Speech-to-Speech Comparison","date":"2022-10-21","arxiv_id":"2210.11835","repositories_listed":1,"syntology":null},{"url":"/paper/transpeech-speech-to-speech-translation-with","slug":"transpeech-speech-to-speech-translation-with","title":"TranSpeech: Speech-to-Speech Translation With Bilateral Perturbation","date":"2022-05-25","arxiv_id":"2205.12523","repositories_listed":1,"syntology":null},{"url":"/paper/leveraging-pseudo-labeled-data-to-improve","slug":"leveraging-pseudo-labeled-data-to-improve","title":"Leveraging Pseudo-labeled Data to Improve Direct Speech-to-Speech Translation","date":"2022-05-18","arxiv_id":"2205.08993","repositories_listed":1,"syntology":{"n":5,"n_ran":5,"n_constructed":1,"n_ran_checked":5,"n_instrument":0,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":4,"n_pointer_only":5,"phrase":"5 ran (of which 1 constructed an object rather than computing a result; 5 with no instrument failure: 1 honoured, 0 violated, 4 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/leveraging-pseudo-labeled-data-to-improve#ran","syntology_url":"https://syntology.ai/paper/2205.08993","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2205.08993"}},"official":{"repos":["fengpeng-yue/speech-to-speech-translation"],"state":"official (archive's flag): 5 ran","n_ran":5,"n_constructed":1,"n_ran_no_instrument_failure":5,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/pretrained-speech-encoders-and-efficient-fine","slug":"pretrained-speech-encoders-and-efficient-fine","title":"Pretrained Speech Encoders and Efficient Fine-tuning Methods for Speech Translation: UPC at IWSLT 2022","date":"2022-05-01","arxiv_id":null,"repositories_listed":1,"syntology":null},{"url":"/paper/libris2s-a-german-english-speech-to-speech","slug":"libris2s-a-german-english-speech-to-speech","title":"LibriS2S: A German-English Speech-to-Speech Translation Corpus","date":"2022-04-22","arxiv_id":"2204.10593","repositories_listed":1,"syntology":null},{"url":"/paper/cvss-corpus-and-massively-multilingual-speech","slug":"cvss-corpus-and-massively-multilingual-speech","title":"CVSS Corpus and Massively Multilingual Speech-to-Speech Translation","date":"2022-01-11","arxiv_id":"2201.03713","repositories_listed":1,"syntology":null},{"url":"/paper/multimodal-and-multilingual-embeddings-for","slug":"multimodal-and-multilingual-embeddings-for","title":"Multimodal and Multilingual Embeddings for Large-Scale Speech Mining","date":"2021-12-01","arxiv_id":null,"repositories_listed":1,"syntology":null},{"url":"/paper/direct-speech-to-speech-translation-with","slug":"direct-speech-to-speech-translation-with","title":"Direct speech-to-speech translation with discrete units","date":"2021-07-12","arxiv_id":"2107.05604","repositories_listed":1,"syntology":null},{"url":"/paper/espnet-st-all-in-one-speech-translation","slug":"espnet-st-all-in-one-speech-translation","title":"ESPnet-ST: All-in-One Speech Translation Toolkit","date":"2020-04-21","arxiv_id":"2004.10234","repositories_listed":1,"syntology":null},{"url":"/paper/towards-automatic-face-to-face-translation-1","slug":"towards-automatic-face-to-face-translation-1","title":"Towards Automatic Face-to-Face Translation","date":"2020-03-01","arxiv_id":"2003.00418","repositories_listed":1,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":1,"n_instrument":1,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/towards-automatic-face-to-face-translation-1#ran","syntology_url":"https://syntology.ai/paper/2003.00418","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2003.00418"}},"official":{"repos":["Rudrabha/LipGAN"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/direct-speech-to-speech-translation-with-a","slug":"direct-speech-to-speech-translation-with-a","title":"Direct speech-to-speech translation with a sequence-to-sequence model","date":"2019-04-12","arxiv_id":"1904.06037","repositories_listed":1,"syntology":null},{"url":null,"slug":"2506-10299","title":"Scheduled Interleaved Speech-Text Training for Speech-to-Speech Translation with LLMs","date":"2025-06-12","arxiv_id":"2506.10299","repositories_listed":0,"syntology":null},{"url":null,"slug":"s2st-omni-an-efficient-and-scalable","title":"S2ST-Omni: An Efficient and Scalable Multilingual Speech-to-Speech Translation Framework via Seamless Speech-Text Alignment and Streaming Speech Generation","date":"2025-06-11","arxiv_id":"2506.11160","repositories_listed":0,"syntology":null},{"url":null,"slug":"phi-omni-st-a-multimodal-language-model-for","title":"Phi-Omni-ST: A multimodal language model for direct speech-to-speech translation","date":"2025-06-04","arxiv_id":"2506.04392","repositories_listed":0,"syntology":null},{"url":null,"slug":"dub-s2st-textless-speech-to-speech","title":"Dub-S2ST: Textless Speech-to-Speech Translation for Seamless Dubbing","date":"2025-05-27","arxiv_id":"2505.20899","repositories_listed":0,"syntology":null},{"url":null,"slug":"language-translation-and-change-of-accent-for","title":"Language translation, and change of accent for speech-to-speech task using diffusion model","date":"2025-05-04","arxiv_id":"2505.04639","repositories_listed":0,"syntology":null},{"url":null,"slug":"simuls2s-llm-unlocking-simultaneous-inference","title":"SimulS2S-LLM: Unlocking Simultaneous Inference of Speech LLMs for Speech-to-Speech Translation","date":"2025-04-22","arxiv_id":"2504.15509","repositories_listed":0,"syntology":null},{"url":null,"slug":"direct-speech-to-speech-translation-a-review","title":"Direct Speech to Speech Translation: A Review","date":"2025-03-03","arxiv_id":"2503.04799","repositories_listed":0,"syntology":null},{"url":null,"slug":"connecting-voices-lorespeech-as-a-low","title":"Connecting Voices: LoReSpeech as a Low-Resource Speech Parallel Corpus","date":"2025-02-25","arxiv_id":"2502.18215","repositories_listed":0,"syntology":null},{"url":null,"slug":"balancing-speech-understanding-and-generation","title":"Balancing Speech Understanding and Generation Using Continual Pre-training for Codec-based Speech LLM","date":"2025-02-24","arxiv_id":"2502.16897","repositories_listed":0,"syntology":null},{"url":null,"slug":"speech-to-speech-translation-with","title":"Speech to Speech Translation with Translatotron: A State of the Art Review","date":"2025-02-09","arxiv_id":"2502.05980","repositories_listed":0,"syntology":null},{"url":null,"slug":"a-unit-based-system-and-dataset-for","title":"A Unit-based System and Dataset for Expressive Direct Speech-to-Speech Translation","date":"2025-02-01","arxiv_id":"2502.00374","repositories_listed":0,"syntology":null},{"url":null,"slug":"improving-lip-synchrony-in-direct-audio","title":"Improving Lip-synchrony in Direct Audio-Visual Speech-to-Speech Translation","date":"2024-12-21","arxiv_id":"2412.16530","repositories_listed":0,"syntology":null},{"url":null,"slug":"direct-speech-to-speech-neural-machine","title":"Direct Speech-to-Speech Neural Machine Translation: A Survey","date":"2024-11-13","arxiv_id":"2411.14453","repositories_listed":0,"syntology":null},{"url":null,"slug":"findings-of-the-iwslt-2024-evaluation","title":"Findings of the IWSLT 2024 Evaluation Campaign","date":"2024-11-07","arxiv_id":"2411.05088","repositories_listed":0,"syntology":null},{"url":null,"slug":"exploiting-phonological-similarities-between","title":"Phonology-Guided Speech-to-Speech Translation for African Languages","date":"2024-10-30","arxiv_id":"2410.23323","repositories_listed":0,"syntology":null},{"url":null,"slug":"textless-streaming-speech-to-speech","title":"Textless Streaming Speech-to-Speech Translation using Semantic Speech Tokens","date":"2024-10-04","arxiv_id":"2410.03298","repositories_listed":0,"syntology":null},{"url":null,"slug":"what-does-it-take-to-get-state-of-the-art-in","title":"What does it take to get state of the art in simultaneous speech-to-speech translation?","date":"2024-09-02","arxiv_id":"2409.00965","repositories_listed":0,"syntology":null},{"url":null,"slug":"polysinger-singing-voice-to-singing-voice","title":"PolySinger: Singing-Voice to Singing-Voice Translation from English to Japanese","date":"2024-07-19","arxiv_id":"2407.14399","repositories_listed":0,"syntology":null},{"url":null,"slug":"preset-voice-matching-for-privacy-regulated","title":"Preset-Voice Matching for Privacy Regulated Speech-to-Speech Translation Systems","date":"2024-07-18","arxiv_id":"2407.13153","repositories_listed":0,"syntology":null},{"url":null,"slug":"analyzing-speech-unit-selection-for-textless","title":"Analyzing Speech Unit Selection for Textless Speech-to-Speech Translation","date":"2024-07-08","arxiv_id":"2407.18332","repositories_listed":0,"syntology":null},{"url":null,"slug":"naist-simultaneous-speech-translation-system","title":"NAIST Simultaneous Speech Translation System for IWSLT 2024","date":"2024-06-30","arxiv_id":"2407.00826","repositories_listed":0,"syntology":null},{"url":null,"slug":"diffusion-synthesizer-for-efficient","title":"Diffusion Synthesizer for Efficient Multilingual Speech to Speech Translation","date":"2024-06-14","arxiv_id":"2406.10223","repositories_listed":0,"syntology":null},{"url":null,"slug":"can-we-achieve-high-quality-direct-speech-to","title":"Can We Achieve High-quality Direct Speech-to-Speech Translation without Parallel Speech Data?","date":"2024-06-11","arxiv_id":"2406.07289","repositories_listed":0,"syntology":null},{"url":null,"slug":"discrete-multimodal-transformers-with-a","title":"Discrete Multimodal Transformers with a Pretrained Large Language Model for Mixed-Supervision Speech Processing","date":"2024-06-04","arxiv_id":"2406.06582","repositories_listed":0,"syntology":null},{"url":null,"slug":"simultron-on-device-simultaneous-speech-to","title":"SimulTron: On-Device Simultaneous Speech to Speech Translation","date":"2024-06-04","arxiv_id":"2406.02133","repositories_listed":0,"syntology":null},{"url":null,"slug":"textless-acoustic-model-with-self-supervised","title":"Textless Acoustic Model with Self-Supervised Distillation for Noise-Robust Expressive Speech-to-Speech Translation","date":"2024-06-04","arxiv_id":"2406.02733","repositories_listed":0,"syntology":null},{"url":null,"slug":"seamlessexpressivelm-speech-language-model","title":"SeamlessExpressiveLM: Speech Language Model for Expressive Speech-to-Speech Translation with Chain-of-Thought","date":"2024-05-30","arxiv_id":"2405.20410","repositories_listed":0,"syntology":null},{"url":null,"slug":"crossvoice-crosslingual-prosody-preserving","title":"CrossVoice: Crosslingual Prosody Preserving Cascade-S2ST using Transfer Learning","date":"2024-05-23","arxiv_id":"2406.00021","repositories_listed":0,"syntology":null},{"url":null,"slug":"mslm-s2st-a-multitask-speech-language-model","title":"MSLM-S2ST: A Multitask Speech Language Model for Textless Speech-to-Speech Translation with Speaker Style Preservation","date":"2024-03-19","arxiv_id":"2403.12408","repositories_listed":0,"syntology":null},{"url":null,"slug":"direct-punjabi-to-english-speech-translation","title":"Direct Punjabi to English speech translation using discrete units","date":"2024-02-25","arxiv_id":"2402.15967","repositories_listed":0,"syntology":null},{"url":null,"slug":"a-case-study-on-filtering-for-end-to-end","title":"A Case Study on Filtering for End-to-End Speech Translation","date":"2024-02-02","arxiv_id":"2402.01945","repositories_listed":0,"syntology":null},{"url":null,"slug":"transentence-speech-to-speech-translation-via","title":"TranSentence: Speech-to-speech Translation via Language-agnostic Sentence-level Speech Encoding without Language-parallel Data","date":"2024-01-17","arxiv_id":"2401.12992","repositories_listed":0,"syntology":null},{"url":null,"slug":"transface-unit-based-audio-visual-speech","title":"TransFace: Unit-Based Audio-Visual Speech Synthesizer for Talking Head Translation","date":"2023-12-23","arxiv_id":"2312.15197","repositories_listed":0,"syntology":null},{"url":null,"slug":"diffs2ut-a-semantic-preserving-diffusion","title":"DiffS2UT: A Semantic Preserving Diffusion Model for Textless Direct Speech-to-Speech Translation","date":"2023-10-26","arxiv_id":"2310.17570","repositories_listed":0,"syntology":null},{"url":null,"slug":"enhancing-expressivity-transfer-in-textless","title":"Enhancing expressivity transfer in textless speech-to-speech translation","date":"2023-10-11","arxiv_id":"2310.07279","repositories_listed":0,"syntology":null},{"url":null,"slug":"direct-text-to-speech-translation-system","title":"Direct Text to Speech Translation System using Acoustic Units","date":"2023-09-14","arxiv_id":"2309.07478","repositories_listed":0,"syntology":null},{"url":null,"slug":"speech-to-speech-translation-with-discrete","title":"Speech-to-Speech Translation with Discrete-Unit-Based Style Transfer","date":"2023-09-14","arxiv_id":"2309.07566","repositories_listed":0,"syntology":null},{"url":null,"slug":"multilingual-speech-to-speech-translation","title":"Multilingual Speech-to-Speech Translation into Multiple Target Languages","date":"2023-07-17","arxiv_id":"2307.08655","repositories_listed":0,"syntology":null},{"url":null,"slug":"audiopalm-a-large-language-model-that-can","title":"AudioPaLM: A Large Language Model That Can Speak and Listen","date":"2023-06-22","arxiv_id":"2306.12925","repositories_listed":0,"syntology":null},{"url":null,"slug":"polyvoice-language-models-for-speech-to","title":"PolyVoice: Language Models for Speech to Speech Translation","date":"2023-06-05","arxiv_id":"2306.02982","repositories_listed":0,"syntology":null},{"url":null,"slug":"translatotron-3-speech-to-speech-translation","title":"Translatotron 3: Speech to Speech Translation with Monolingual Data","date":"2023-05-27","arxiv_id":"2305.17547","repositories_listed":0,"syntology":null},{"url":null,"slug":"av-transpeech-audio-visual-robust-speech-to","title":"AV-TranSpeech: Audio-Visual Robust Speech-to-Speech Translation","date":"2023-05-24","arxiv_id":"2305.15403","repositories_listed":0,"syntology":null},{"url":null,"slug":"i-code-studio-a-configurable-and-composable","title":"i-Code Studio: A Configurable and Composable Framework for Integrative AI","date":"2023-05-23","arxiv_id":"2305.13738","repositories_listed":0,"syntology":null},{"url":null,"slug":"duplex-diffusion-models-improve-speech-to","title":"Duplex Diffusion Models Improve Speech-to-Speech Translation","date":"2023-05-22","arxiv_id":"2305.12628","repositories_listed":0,"syntology":null},{"url":null,"slug":"enhancing-speech-to-speech-translation-with","title":"Enhancing Speech-to-Speech Translation with Multiple TTS Targets","date":"2023-04-10","arxiv_id":"2304.04618","repositories_listed":0,"syntology":null},{"url":null,"slug":"a-holistic-cascade-system-benchmark-and-human","title":"A Holistic Cascade System, benchmark, and Human Evaluation Protocol for Expressive Speech-to-Speech Translation","date":"2023-01-25","arxiv_id":"2301.10606","repositories_listed":0,"syntology":null},{"url":null,"slug":"direct-speech-to-speech-translation-without","title":"Direct Speech-to-speech Translation without Textual Annotation using Bottleneck Features","date":"2022-12-12","arxiv_id":"2212.05805","repositories_listed":0,"syntology":null},{"url":null,"slug":"speech-to-speech-translation-for-a-real-world-1","title":"Speech-to-Speech Translation For A Real-world Unwritten Language","date":"2022-11-11","arxiv_id":"2211.06474","repositories_listed":0,"syntology":null},{"url":null,"slug":"speechmatrix-a-large-scale-mined-corpus-of-1","title":"SpeechMatrix: A Large-Scale Mined Corpus of Multilingual Speech-to-Speech Translations","date":"2022-11-08","arxiv_id":"2211.04508","repositories_listed":0,"syntology":null},{"url":null,"slug":"textless-direct-speech-to-speech-translation","title":"Textless Direct Speech-to-Speech Translation with Discrete Speech Representation","date":"2022-10-31","arxiv_id":"2211.00115","repositories_listed":0,"syntology":null},{"url":null,"slug":"improving-speech-to-speech-translation","title":"Improving Speech-to-Speech Translation Through Unlabeled Text","date":"2022-10-26","arxiv_id":"2210.14514","repositories_listed":0,"syntology":null},{"url":null,"slug":"on-the-robustness-of-self-supervised-1","title":"Augmentation Invariant Discrete Representation for Generative Spoken Language Modeling","date":"2022-09-30","arxiv_id":"2209.15483","repositories_listed":0,"syntology":null},{"url":null,"slug":"samu-xlsr-semantically-aligned-multimodal","title":"SAMU-XLSR: Semantically-Aligned Multimodal Utterance-level Cross-Lingual Speech Representation","date":"2022-05-17","arxiv_id":"2205.08180","repositories_listed":0,"syntology":null},{"url":null,"slug":"findings-of-the-iwslt-2022-evaluation","title":"Findings of the IWSLT 2022 Evaluation Campaign","date":"2022-05-01","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"mllp-vrain-upv-systems-for-the-iwslt-2022","title":"MLLP-VRAIN UPV systems for the IWSLT 2022 Simultaneous Speech Translation and Speech-to-Speech Translation tasks","date":"2022-05-01","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"the-hw-tscs-speech-to-speech-translation","title":"The HW-TSC’s Speech to Speech Translation System for IWSLT 2022 Evaluation","date":"2022-05-01","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"enhanced-direct-speech-to-speech-translation","title":"Enhanced Direct Speech-to-Speech Translation Using Self-supervised Pre-training and Data Augmentation","date":"2022-04-06","arxiv_id":"2204.02967","repositories_listed":0,"syntology":null},{"url":null,"slug":"prosodic-alignment-for-off-screen-automatic","title":"Prosodic Alignment for off-screen automatic dubbing","date":"2022-04-06","arxiv_id":"2204.02530","repositories_listed":0,"syntology":null},{"url":null,"slug":"leveraging-unsupervised-and-weakly-supervised","title":"Leveraging unsupervised and weakly-supervised data to improve direct speech-to-speech translation","date":"2022-03-24","arxiv_id":"2203.13339","repositories_listed":0,"syntology":null},{"url":null,"slug":"evaluating-mt-systems-a-theoretical-framework","title":"Evaluating MT Systems: A Theoretical Framework","date":"2022-02-11","arxiv_id":"2202.05806","repositories_listed":0,"syntology":null},{"url":null,"slug":"textless-speech-to-speech-translation-on-real","title":"Textless Speech-to-Speech Translation on Real Data","date":"2021-12-15","arxiv_id":"2112.08352","repositories_listed":0,"syntology":null}],"syntology_records":12,"syntology_note":"a paper without a record is not a recorded non-run: it may lack an arXiv id or simply be absent from the graph layer"},"key_notes":{"n_ran_checked":"legacy name, kept unchanged so existing readers do not break: it counts the samples that ran with no instrument failure (honoured, violated, and ran with no contract checked); it does not mean a contract was checked, and the pages print it as 'K with no instrument failure', not 'K checked'","n_constructed":"a sub-count of the samples that ran, never subtracted from them and never a failure: an executed sample whose run returned an instance of its own class (fixture_out_type equals the entry name): the run built an object and did not compute a result (Syntology's RAN record, counts.constructed)"},"since_archive_judged":{"papers":0,"papers_placed":0,"judged_by":{"model":"a model","models_as_recorded":[],"from":"lb_rows_run_verified.jsonl: each record's `model` field (the extractor's call that wrote decisions[])"},"note":"a model's judgement from the paper's own tables that they report on this leaderboard, not a result: no value from a paper that was not placed is printed or carried here; papers counts each paper once over these leaderboards","backfill":"8,886 of 18,637 newer papers checked so far (733 more had nothing to place; 41 could not be read)","backfill_complete":false,"leaderboards":[]},"papers_where_code_ran":12,"papers_where_code_ran_first_page":"/task/speech-to-speech-translation/papers/ran/1","papers_where_code_ran_caption":"We ran code from the paper's repository; we did not run it on this task or check it against the task's benchmarks.","pages":[{"order":"archive","page":2,"of":2,"url":"/task/speech-to-speech-translation/papers/2","json":"/task/speech-to-speech-translation/papers/2.json","rows":17},{"order":"ran","page":1,"of":1,"url":"/task/speech-to-speech-translation/papers/ran/1","json":"/task/speech-to-speech-translation/papers/ran/1.json","rows":12}],"description_links":{"kept":0,"unwrapped_to_text":0,"bare_urls_linked":0,"relative_images_dropped":0,"rule":"internal links are kept only when the target slug exists in the catalog"},"syntology":{"read_at":"2026-10-05T00:36:11+00:00","claim":"Per-sample execution status on synthesized fixtures ('ran N of M samples'); not a correctness claim and not a ranking signal.","status_vocabulary":{"ran_honours":"ran, honoured the contract we drafted","ran_violates":"ran, violated the contract we drafted","ran_draft_wrong":"ran; our contract draft was wrong, not the code","ran_fixture":"ran; the contract check could not run (RAISES, DEP_MISSING or TIMEOUT)","ran":"ran on a synthesized input","unverified":"unverified (harvested, no recorded run)"},"row_route":"each syntology object on a benchmark's first row and on a paper carries sample_list (this site's paper page, #ran), syntology_url (syntology.ai's page for the paper, only when that route probed live) and mcp (the agent call that lists the paper's harvested samples; not a verdict on the paper)","developers":"https://syntology.ai/developers"},"not_shown":{"libraries":"the archive has no per-task library table","trend_sparklines":"the Trend column of the benchmarks table was a rendered image; it is not in the archive","social_and_latest_sorts":"stars and social signals are not in the archive"},"record_sha256":"ba5cd1fbce550171c954a2f01e2881d6f402f49ee4f9645c4ffaa88e368a15d1","record_changed_at":"2026-10-05","record_changed_at_basis":"changed"}