{"about":{"non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","site":"https://codewithpapers.app","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page","syntology":{"site":"https://syntology.ai","developers":"https://syntology.ai/developers","mcp":{"server":"https://syntology.ai/mcp","transport":"streamable-http","server_card":"https://syntology.ai/.well-known/mcp/server-card.json","auth":{"type":"trial token, no account","trial_token":"https://syntology.ai/api/oauth/trial/token","method":"POST","docs":"https://syntology.ai/developers"}},"have":"https://syntology.ai/api/graph/have?x=<method, arXiv id or title> (free, answers coverage only)","paper_base":"https://syntology.ai/paper/","atlas_base":"https://app.syntology.ai/?focus="},"machine_readable":[{"url":"https://codewithpapers.app/llms.txt","what":"the machine catalog: every machine-readable file, counted"},{"url":"https://codewithpapers.app/index/manifest.json","what":"paper-to-code index by arXiv id, with Syntology's counts"},{"url":"https://codewithpapers.app/search/manifest.json","what":"site search index (titles, authors) and its files"},{"url":"https://codewithpapers.app/download","what":"bulk files: Syntology's layer, described there"},{"url":"https://codewithpapers.app/build_manifest.json","what":"the build record: inputs, counts, exclusions, probes"}]},"url":"/method/switch-ffn","slug":"switch-ffn","name":"Switch FFN","full_name":"Switch FFN","full_name_withheld":false,"description_markdown":"A **Switch FFN** is a sparse layer that operates independently on tokens within an input sequence. It is shown in the blue block in the figure. We diagram two tokens ($x\\_{1}$ = “More” and $x\\_{2}$ = “Parameters” below) being routed (solid lines) across four FFN experts, where the router independently routes each token. The switch FFN layer returns the output of the selected FFN multiplied by the router gate value (dotted-line).","description_state":"present","introduced_year":null,"introduced_by":{"title":"Switch Transformers: Scaling to Trillion Parameter Models with Simple and Efficient Sparsity","paper":"/paper/switch-transformers-scaling-to-trillion","first_author":"William Fedus","n_authors":3,"url_abs":null,"archive_paper_url":"https://paperswithcode.com/paper/switch-transformers-scaling-to-trillion"},"source":{"url":"https://arxiv.org/abs/2101.03961v3","title":"Switch Transformers: Scaling to Trillion Parameter Models with Simple and Efficient Sparsity","url_on_a_paper_host":true},"code_snippet_url":null,"code_snippet_url_on_a_code_host":false,"categories":[{"area":"General","area_id":"general","collection":"Feedforward Networks","url":"/methods/category/feedforward-networks","pwc_aliases":[]}],"n_papers_tagged":20,"archive_num_papers":20,"papers_newest_first":[{"paper":null,"slug":"qos-efficient-serving-of-multiple-mixture-of","title":"QoS-Efficient Serving of Multiple Mixture-of-Expert LLMs Using Partial Runtime Reconfiguration","date":"2025-05-10","arxiv_id":"2505.06481","n_code_links":0,"syntology":null},{"paper":null,"slug":"expertrag-efficient-rag-with-mixture-of","title":"ExpertRAG: Efficient RAG with Mixture of Experts -- Optimizing Context Retrieval for Adaptive LLM Responses","date":"2025-03-23","arxiv_id":"2504.08744","n_code_links":0,"syntology":null},{"paper":"/paper/resmoe-space-efficient-compression-of-mixture","slug":"resmoe-space-efficient-compression-of-mixture","title":"ResMoE: Space-efficient Compression of Mixture of Experts LLMs via Residual Restoration","date":"2025-03-10","arxiv_id":"2503.06881","n_code_links":1,"syntology":{"ran":10,"of":15,"n_ran_checked":7,"n_instrument":3,"unverified":5,"pointer_only":1,"phrase":"10 ran (of which 0 constructed an object rather than computing a result; 7 with no instrument failure: 0 honoured, 1 violated, 6 with no contract checked; 3 where Syntology's instrument failed) · 5 unverified","official":{"repos":["idea-isail-lab-uiuc/resmoe"],"state":"official (archive's flag): 10 ran","n_ran":10,"n_constructed":0,"n_ran_no_instrument_failure":7,"n_unverified":5,"ran_from_kinds":["official"]}}},{"paper":null,"slug":"sparse-backpropagation-for-moe-training","title":"Sparse Backpropagation for MoE Training","date":"2023-10-01","arxiv_id":"2310.00811","n_code_links":0,"syntology":null},{"paper":null,"slug":"serving-moe-models-on-resource-constrained","title":"SwapMoE: Serving Off-the-shelf MoE-based Large Language Models with Tunable Memory Budget","date":"2023-08-29","arxiv_id":"2308.15030","n_code_links":0,"syntology":null},{"paper":"/paper/condensing-multilingual-knowledge-with","slug":"condensing-multilingual-knowledge-with","title":"Condensing Multilingual Knowledge with Lightweight Language-Specific Modules","date":"2023-05-23","arxiv_id":"2305.13993","n_code_links":1,"syntology":{"ran":2,"of":2,"n_ran_checked":1,"n_instrument":1,"unverified":0,"pointer_only":0,"phrase":"2 ran (of which 2 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified; every one of the 2 samples that ran constructed an object rather than computing a result","official":{"repos":["fe1ixxu/lms_fd"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":2,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"paper":null,"slug":"towards-a-unified-view-of-sparse-feed-forward","title":"Towards A Unified View of Sparse Feed-Forward Network in Pretraining Large Language Model","date":"2023-05-23","arxiv_id":"2305.13999","n_code_links":0,"syntology":null},{"paper":null,"slug":"a-small-scale-switch-transformer-and-nlp","title":"Improving Transformer Performance for French Clinical Notes Classification Using Mixture of Experts on a Limited Dataset","date":"2023-03-22","arxiv_id":"2303.12892","n_code_links":0,"syntology":null},{"paper":null,"slug":"smile-scaling-mixture-of-experts-with","title":"SMILE: Scaling Mixture-of-Experts with Efficient Bi-level Routing","date":"2022-12-10","arxiv_id":"2212.05191","n_code_links":0,"syntology":null},{"paper":"/paper/argumentative-text-generation-in-economic","slug":"argumentative-text-generation-in-economic","title":"Argumentative Text Generation in Economic Domain","date":"2022-06-18","arxiv_id":"2206.09251","n_code_links":1,"syntology":null},{"paper":null,"slug":"automatic-summarization-of-russian-texts","title":"Automatic Summarization of Russian Texts: Comparison of Extractive and Abstractive Methods","date":"2022-06-18","arxiv_id":"2206.09253","n_code_links":0,"syntology":null},{"paper":"/paper/build-a-robust-qa-system-with-transformer","slug":"build-a-robust-qa-system-with-transformer","title":"Build a Robust QA System with Transformer-based Mixture of Experts","date":"2022-03-20","arxiv_id":"2204.09598","n_code_links":1,"syntology":null},{"paper":"/paper/efficient-language-modeling-with-sparse-all","slug":"efficient-language-modeling-with-sparse-all","title":"Efficient Language Modeling with Sparse all-MLP","date":"2022-03-14","arxiv_id":"2203.06850","n_code_links":0,"syntology":null},{"paper":null,"slug":"switch-trajectory-transformer-with","title":"Switch Trajectory Transformer with Distributional Value Approximation for Multi-Task Reinforcement Learning","date":"2022-03-14","arxiv_id":"2203.07413","n_code_links":0,"syntology":null},{"paper":null,"slug":"mixture-of-experts-with-expert-choice-routing","title":"Mixture-of-Experts with Expert Choice Routing","date":"2022-02-18","arxiv_id":"2202.09368","n_code_links":0,"syntology":null},{"paper":null,"slug":"m6-10t-a-sharing-delinking-paradigm-for","title":"M6-10T: A Sharing-Delinking Paradigm for Efficient Multi-Trillion Parameter Pretraining","date":"2021-10-08","arxiv_id":"2110.03888","n_code_links":0,"syntology":null},{"paper":"/paper/taming-sparsely-activated-transformer-with","slug":"taming-sparsely-activated-transformer-with","title":"Taming Sparsely Activated Transformer with Stochastic Experts","date":"2021-10-08","arxiv_id":"2110.04260","n_code_links":1,"syntology":{"ran":2,"of":2,"n_ran_checked":2,"n_instrument":0,"unverified":0,"pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","official":{"repos":["microsoft/stochastic-mixture-of-experts"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["official"]}}},{"paper":null,"slug":"random-offset-block-embedding-array-robe-for","title":"Random Offset Block Embedding Array (ROBE) for CriteoTB Benchmark MLPerf DLRM Model : 1000$\\times$ Compression and 3.1$\\times$ Faster Inference","date":"2021-08-04","arxiv_id":"2108.02191","n_code_links":0,"syntology":null},{"paper":null,"slug":"carbon-emissions-and-large-neural-network","title":"Carbon Emissions and Large Neural Network Training","date":"2021-04-21","arxiv_id":"2104.10350","n_code_links":0,"syntology":null},{"paper":"/paper/switch-transformers-scaling-to-trillion","slug":"switch-transformers-scaling-to-trillion","title":"Switch Transformers: Scaling to Trillion Parameter Models with Simple and Efficient Sparsity","date":"2021-01-11","arxiv_id":"2101.03961","n_code_links":8,"syntology":{"ran":7,"of":13,"n_ran_checked":3,"n_instrument":4,"unverified":6,"pointer_only":8,"phrase":"7 ran (of which 2 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 4 where Syntology's instrument failed) · 6 unverified","official":{"repos":["tensorflow/mesh"],"state":"official: harvested, nothing ran","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":4,"ran_from_kinds":["listed","unlocated"]}}}],"papers_shown":20,"papers_order":"date (newest first), then slug","paper_counts":{"archive_papers_tagged":20,"with_a_code_link":6,"where_syntology_ran_a_sample":4,"not_listed_spam_title":0,"listed":20,"listed_where_code_ran":4,"where_syntology_ran_a_sample_split":{"with_a_run_with_no_instrument_failure":4,"every_run_a_failure_of_syntologys_instrument":0,"listed_with_a_run_with_no_instrument_failure":4,"listed_every_run_a_failure_of_syntologys_instrument":0,"filter":{"states":["a run with no instrument failure","any run, instrument failures included"],"default":"a run with no instrument failure","note":"on the 'only where code ran' pages the default hides, in the browser, the rows where every run was a failure of Syntology's instrument; the second state shows them again. Rows are hidden, never re-ordered; these twins list every row"}},"definition":"distinct papers the archive tags; 'where Syntology ran a sample' counts papers with at least one harvested sample that ran, which is not a correctness claim"},"papers_where_code_ran":4,"papers_where_code_ran_first_page":"/method/switch-ffn/papers/ran/1","papers_where_code_ran_caption":"We ran code from the paper's repository; we did not isolate this method inside it.","pages":[{"order":"ran","page":1,"of":1,"url":"/method/switch-ffn/papers/ran/1","json":"/method/switch-ffn/papers/ran/1.json","rows":4}],"tasks":[{"task":"/task/mixture-of-experts","name":"Mixture-of-Experts","papers":13},{"task":"/task/language-modelling","name":"Language Modelling","papers":4},{"task":null,"name":"GPU","papers":3},{"task":"/task/language-modeling","name":"Language Modeling","papers":3},{"task":"/task/machine-translation","name":"Machine Translation","papers":3},{"task":"/task/question-answering","name":"Question Answering","papers":3},{"task":"/task/text-generation","name":"Text Generation","papers":2},{"task":"/task/translation","name":"Translation","papers":2},{"task":"/task/all","name":"All","papers":1},{"task":null,"name":"Avg","papers":1},{"task":null,"name":"CPU","papers":1},{"task":"/task/common-sense-reasoning","name":"Common Sense Reasoning","papers":1},{"task":"/task/data-augmentation","name":"Data Augmentation","papers":1},{"task":"/task/in-context-learning","name":"In-Context Learning","papers":1},{"task":"/task/large-language-model","name":"Large Language Model","papers":1},{"task":"/task/model-compression","name":"Model Compression","papers":1},{"task":"/task/architecture-search","name":"Neural Architecture Search","papers":1},{"task":"/task/object-detection","name":"Object Detection","papers":1},{"task":"/task/rag","name":"RAG","papers":1},{"task":"/task/reinforcement-learning-1","name":"Reinforcement Learning (RL)","papers":1}],"tasks_shown":20,"n_tasks":30,"usage_by_year":[{"year":"2021","papers":5},{"year":"2022","papers":7},{"year":"2023","papers":5},{"year":"2025","papers":3}],"row_source":"methods_table","archive":{"source":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","archive_url":"https://paperswithcode.com/method/switch-ffn"},"syntology_read_at":"2026-10-05T00:36:11+00:00","syntology":{"source":"Syntology's have check for this name (the free coverage lookup), read from Syntology's graph","have_url":"https://syntology.ai/api/graph/have?x=Switch%20FFN","read_at":"2026-10-05T01:02:48+00:00","query":"Switch FFN","kind":"method","have":"unknown_name","tier":null,"verified_implementation":null,"level":null,"level_meaning":"the served verification level, per answer; never summed across methods","resolved_to":null,"resolution":null,"resolved_name_differs":false,"absence_asserted":false,"absence_unchecked":null,"no_code_is_about_this_method":false,"subject":null,"archive_paper_arxiv_id":"2101.03961","repositories":null,"retrieve_with":null,"name_competition":{"verdict":"none","may_be_wrong_sense":false,"bearers_found":0,"served":null,"leading_other_bearer":null,"n_other_bearers_listed":0,"note":"Nothing in this corpus is named 'Switch FFN' -- no method node, no paper announcing it in its title, no expanded form declaring it as an acronym."},"name_competition_text":null,"near_matches":["NC-FFN","W-Switch","T-Switch","Switch-KD","FB π-Switch","FFN-SkipLLM","Auto-Switch","TACIT-SWITCH"],"code_nearby":[],"code_nearby_caption":null,"origin_paper":null,"line":"name not resolved (unknown_name). The lookup matched no method, paper or venue by this name; that asserts nothing about whether code exists.","level_legend":null},"syntology_have_state":"loaded","record_sha256":"00f3cc0598cf1e77570474619bb40e3b5c5d8531172657d61a6624dce703f493","record_changed_at":"2026-10-05","record_changed_at_basis":"changed"}