{"about":{"non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","site":"https://codewithpapers.app","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page","syntology":{"site":"https://syntology.ai","developers":"https://syntology.ai/developers","mcp":{"server":"https://syntology.ai/mcp","transport":"streamable-http","server_card":"https://syntology.ai/.well-known/mcp/server-card.json","auth":{"type":"trial token, no account","trial_token":"https://syntology.ai/api/oauth/trial/token","method":"POST","docs":"https://syntology.ai/developers"}},"have":"https://syntology.ai/api/graph/have?x=<method, arXiv id or title> (free, answers coverage only)","paper_base":"https://syntology.ai/paper/","atlas_base":"https://app.syntology.ai/?focus="},"machine_readable":[{"url":"https://codewithpapers.app/llms.txt","what":"the machine catalog: every machine-readable file, counted"},{"url":"https://codewithpapers.app/index/manifest.json","what":"paper-to-code index by arXiv id, with Syntology's counts"},{"url":"https://codewithpapers.app/search/manifest.json","what":"site search index (titles, authors) and its files"},{"url":"https://codewithpapers.app/download","what":"bulk files: Syntology's layer, described there"},{"url":"https://codewithpapers.app/build_manifest.json","what":"the build record: inputs, counts, exclusions, probes"}]},"url":"/method/gradient-sparsification","slug":"gradient-sparsification","name":"Gradient Sparsification","full_name":"Gradient Sparsification","full_name_withheld":false,"description_markdown":"**Gradient Sparsification** is a technique for distributed training that sparsifies stochastic gradients to reduce the communication cost, with minor increase in the number of iterations. The key idea behind our sparsification technique is to drop some coordinates of the stochastic gradient and appropriately amplify the remaining coordinates to ensure the unbiasedness of the sparsified stochastic gradient. The sparsification approach can significantly reduce the coding length of the stochastic gradient and only slightly increase the variance of the stochastic gradient.","description_state":"present","introduced_year":null,"introduced_by":{"title":"Gradient Sparsification for Communication-Efficient Distributed Optimization","paper":"/paper/gradient-sparsification-for-communication","first_author":"Jianqiao Wangni","n_authors":4,"url_abs":null,"archive_paper_url":"https://paperswithcode.com/paper/gradient-sparsification-for-communication"},"source":{"url":"http://arxiv.org/abs/1710.09854v1","title":"Gradient Sparsification for Communication-Efficient Distributed Optimization","url_on_a_paper_host":true},"code_snippet_url":null,"code_snippet_url_on_a_code_host":false,"categories":[{"area":"General","area_id":"general","collection":"Data Parallel Methods","url":"/methods/category/data-parallel-methods","pwc_aliases":[]},{"area":"General","area_id":"general","collection":"Stochastic Optimization","url":"/methods/category/stochastic-optimization","pwc_aliases":[]},{"area":"General","area_id":"general","collection":"Optimization","url":"/methods/category/optimization","pwc_aliases":[]},{"area":"General","area_id":"general","collection":"Distributed Methods","url":"/methods/category/distributed-methods","pwc_aliases":[]}],"n_papers_tagged":38,"archive_num_papers":38,"papers_newest_first":[{"paper":null,"slug":"mobility-aware-asynchronous-federated","title":"Mobility-Aware Asynchronous Federated Learning with Dynamic Sparsification","date":"2025-06-08","arxiv_id":"2506.07328","n_code_links":0,"syntology":null},{"paper":"/paper/dynamic-gradient-sparsification-training-for","slug":"dynamic-gradient-sparsification-training-for","title":"Dynamic Gradient Sparsification Training for Few-Shot Fine-tuning of CT Lymph Node Segmentation Foundation Model","date":"2025-03-02","arxiv_id":"2503.00748","n_code_links":1,"syntology":null},{"paper":null,"slug":"sparse-incremental-aggregation-in-satellite","title":"Sparse Incremental Aggregation in Satellite Federated Learning","date":"2025-01-20","arxiv_id":"2501.11385","n_code_links":0,"syntology":null},{"paper":null,"slug":"regularized-top-k-a-bayesian-framework-for","title":"Regularized Top-$k$: A Bayesian Framework for Gradient Sparsification","date":"2025-01-10","arxiv_id":"2501.05633","n_code_links":0,"syntology":null},{"paper":"/paper/dqrm-deep-quantized-recommendation-models","slug":"dqrm-deep-quantized-recommendation-models","title":"DQRM: Deep Quantized Recommendation Models","date":"2024-10-26","arxiv_id":"2410.20046","n_code_links":1,"syntology":null},{"paper":null,"slug":"age-of-gradient-updates-for-federated","title":"Age-of-Gradient Updates for Federated Learning over Random Access Channels","date":"2024-10-15","arxiv_id":"2410.11986","n_code_links":0,"syntology":null},{"paper":null,"slug":"novel-gradient-sparsification-algorithm-via","title":"Novel Gradient Sparsification Algorithm via Bayesian Inference","date":"2024-09-23","arxiv_id":"2409.14893","n_code_links":0,"syntology":null},{"paper":"/paper/preserving-near-optimal-gradient","slug":"preserving-near-optimal-gradient","title":"Preserving Near-Optimal Gradient Sparsification Cost for Scalable Distributed Deep Learning","date":"2024-02-21","arxiv_id":"2402.13781","n_code_links":1,"syntology":null},{"paper":"/paper/jointsq-joint-sparsification-quantization-for","slug":"jointsq-joint-sparsification-quantization-for","title":"JointSQ: Joint Sparsification-Quantization for Distributed Learning","date":"2024-01-01","arxiv_id":null,"n_code_links":1,"syntology":null},{"paper":null,"slug":"rs-dgc-exploring-neighborhood-statistics-for","title":"RS-DGC: Exploring Neighborhood Statistics for Dynamic Gradient Compression on Remote Sensing Image Interpretation","date":"2023-12-29","arxiv_id":"2312.17530","n_code_links":0,"syntology":null},{"paper":"/paper/micro-near-zero-cost-gradient-sparsification","slug":"micro-near-zero-cost-gradient-sparsification","title":"MiCRO: Near-Zero Cost Gradient Sparsification for Scaling and Accelerating Distributed DNN Training","date":"2023-10-02","arxiv_id":"2310.00967","n_code_links":1,"syntology":null},{"paper":null,"slug":"gradient-sparsification-for-masked-fine","title":"Gradient Sparsification For Masked Fine-Tuning of Transformers","date":"2023-07-19","arxiv_id":"2307.10098","n_code_links":0,"syntology":null},{"paper":"/paper/deft-exploiting-gradient-norm-difference","slug":"deft-exploiting-gradient-norm-difference","title":"DEFT: Exploiting Gradient Norm Difference between Model Layers for Scalable Gradient Sparsification","date":"2023-07-07","arxiv_id":"2307.03500","n_code_links":1,"syntology":null},{"paper":null,"slug":"gradient-sparsification-for-efficient","title":"Gradient Sparsification for Efficient Wireless Federated Learning with Differential Privacy","date":"2023-04-09","arxiv_id":"2304.04164","n_code_links":0,"syntology":null},{"paper":null,"slug":"efficient-and-secure-federated-learning-for","title":"Efficient and Secure Federated Learning for Financial Applications","date":"2023-03-15","arxiv_id":"2303.08355","n_code_links":0,"syntology":null},{"paper":null,"slug":"on-the-interaction-between-differential","title":"On the Interaction Between Differential Privacy and Gradient Compression in Deep Learning","date":"2022-11-01","arxiv_id":"2211.00734","n_code_links":0,"syntology":null},{"paper":null,"slug":"downlink-compression-improves-topk","title":"Downlink Compression Improves TopK Sparsification","date":"2022-09-30","arxiv_id":"2209.15203","n_code_links":0,"syntology":null},{"paper":null,"slug":"empirical-analysis-on-top-k-gradient","title":"Empirical Analysis on Top-k Gradient Sparsification for Distributed Deep Learning in a Supercomputing Environment","date":"2022-09-18","arxiv_id":"2209.08497","n_code_links":0,"syntology":null},{"paper":"/paper/near-optimal-sparse-allreduce-for-distributed","slug":"near-optimal-sparse-allreduce-for-distributed","title":"Near-Optimal Sparse Allreduce for Distributed Deep Learning","date":"2022-01-19","arxiv_id":"2201.07598","n_code_links":1,"syntology":null},{"paper":null,"slug":"sparsified-secure-aggregation-for-privacy","title":"Sparsified Secure Aggregation for Privacy-Preserving Federated Learning","date":"2021-12-23","arxiv_id":"2112.12872","n_code_links":0,"syntology":null},{"paper":null,"slug":"communication-efficient-federated-learning-10","title":"Communication-Efficient Federated Learning via Quantized Compressed Sensing","date":"2021-11-30","arxiv_id":"2111.15071","n_code_links":0,"syntology":null},{"paper":null,"slug":"federated-dynamic-neural-network-for-deep","title":"Federated Dynamic Neural Network for Deep MIMO Detection","date":"2021-11-24","arxiv_id":"2111.12260","n_code_links":0,"syntology":null},{"paper":null,"slug":"gradient-sparsification-for-emph-masked-fine","title":"Gradient Sparsification For \\emph{Masked Fine-Tuning} of Transformers","date":"2021-11-16","arxiv_id":null,"n_code_links":0,"syntology":null},{"paper":null,"slug":"achieving-linear-convergence-in-federated","title":"Linear Convergence in Federated Learning: Tackling Client Heterogeneity and Sparse Gradients","date":"2021-02-14","arxiv_id":"2102.07053","n_code_links":0,"syntology":null},{"paper":null,"slug":"large-scale-training-system-for-100-million","title":"Large-Scale Training System for 100-Million Classification at Alibaba","date":"2021-02-09","arxiv_id":"2102.06025","n_code_links":0,"syntology":null},{"paper":null,"slug":"gradient-sparsification-can-improve","title":"Gradient Sparsification Can Improve Performance of Differentially-Private Convex Machine Learning","date":"2020-11-30","arxiv_id":"2011.14572","n_code_links":0,"syntology":null},{"paper":"/paper/towards-building-a-robust-and-fair-federated","slug":"towards-building-a-robust-and-fair-federated","title":"A Reputation Mechanism Is All You Need: Collaborative Fairness and Adversarial Robustness in Federated Learning","date":"2020-11-20","arxiv_id":"2011.10464","n_code_links":2,"syntology":null},{"paper":"/paper/flame-differentially-private-federated","slug":"flame-differentially-private-federated","title":"FLAME: Differentially Private Federated Learning in the Shuffle Model","date":"2020-09-17","arxiv_id":"2009.08063","n_code_links":1,"syntology":{"ran":1,"of":1,"n_ran_checked":1,"n_instrument":0,"unverified":0,"pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","official":null}},{"paper":null,"slug":"rtop-k-a-statistical-estimation-approach-to","title":"rTop-k: A Statistical Estimation Approach to Distributed SGD","date":"2020-05-21","arxiv_id":"2005.10761","n_code_links":0,"syntology":null},{"paper":null,"slug":"adaptive-gradient-sparsification-for","title":"Adaptive Gradient Sparsification for Efficient Federated Learning: An Online Learning Approach","date":"2020-01-14","arxiv_id":"2001.04756","n_code_links":0,"syntology":null},{"paper":null,"slug":"layer-wise-adaptive-gradient-sparsification","title":"Layer-wise Adaptive Gradient Sparsification for Distributed Deep Learning with Convergence Guarantees","date":"2019-11-20","arxiv_id":"1911.08727","n_code_links":0,"syntology":null},{"paper":"/paper/understanding-top-k-sparsification-in-1","slug":"understanding-top-k-sparsification-in-1","title":"Understanding Top-k Sparsification in Distributed Deep Learning","date":"2019-11-20","arxiv_id":"1911.08772","n_code_links":1,"syntology":null},{"paper":null,"slug":"gradient-sparification-for-asynchronous","title":"Gradient Sparification for Asynchronous Distributed Training","date":"2019-10-24","arxiv_id":"1910.10929","n_code_links":0,"syntology":null},{"paper":null,"slug":"hierarchical-federated-learning-across","title":"Hierarchical Federated Learning Across Heterogeneous Cellular Networks","date":"2019-09-05","arxiv_id":"1909.02362","n_code_links":0,"syntology":null},{"paper":null,"slug":"a-linear-speedup-analysis-of-distributed-deep","title":"A Linear Speedup Analysis of Distributed Deep Learning with Sparse and Quantized Communication","date":"2018-12-01","arxiv_id":null,"n_code_links":0,"syntology":null},{"paper":null,"slug":"double-quantization-for-communication","title":"Double Quantization for Communication-Efficient Distributed Optimization","date":"2018-05-25","arxiv_id":"1805.10111","n_code_links":0,"syntology":null},{"paper":null,"slug":"sparse-binary-compression-towards-distributed","title":"Sparse Binary Compression: Towards Distributed Deep Learning with minimal Communication","date":"2018-05-22","arxiv_id":"1805.08768","n_code_links":0,"syntology":null},{"paper":"/paper/gradient-sparsification-for-communication","slug":"gradient-sparsification-for-communication","title":"Gradient Sparsification for Communication-Efficient Distributed Optimization","date":"2017-10-26","arxiv_id":"1710.09854","n_code_links":0,"syntology":null}],"papers_shown":38,"papers_order":"date (newest first), then slug","paper_counts":{"archive_papers_tagged":38,"with_a_code_link":10,"where_syntology_ran_a_sample":1,"not_listed_spam_title":0,"listed":38,"listed_where_code_ran":1,"where_syntology_ran_a_sample_split":{"with_a_run_with_no_instrument_failure":1,"every_run_a_failure_of_syntologys_instrument":0,"listed_with_a_run_with_no_instrument_failure":1,"listed_every_run_a_failure_of_syntologys_instrument":0,"filter":{"states":["a run with no instrument failure","any run, instrument failures included"],"default":"a run with no instrument failure","note":"on the 'only where code ran' pages the default hides, in the browser, the rows where every run was a failure of Syntology's instrument; the second state shows them again. Rows are hidden, never re-ordered; these twins list every row"}},"definition":"distinct papers the archive tags; 'where Syntology ran a sample' counts papers with at least one harvested sample that ran, which is not a correctness claim"},"papers_where_code_ran":1,"papers_where_code_ran_first_page":"/method/gradient-sparsification/papers/ran/1","papers_where_code_ran_caption":"We ran code from the paper's repository; we did not isolate this method inside it.","pages":[{"order":"ran","page":1,"of":1,"url":"/method/gradient-sparsification/papers/ran/1","json":"/method/gradient-sparsification/papers/ran/1.json","rows":1}],"tasks":[{"task":"/task/federated-learning","name":"Federated Learning","papers":13},{"task":"/task/quantization","name":"Quantization","papers":5},{"task":"/task/deep-learning","name":"Deep Learning","papers":3},{"task":"/task/distributed-optimization","name":"Distributed Optimization","papers":3},{"task":"/task/stochastic-optimization","name":"Stochastic Optimization","papers":3},{"task":"/task/machine-learning","name":"BIG-bench Machine Learning","papers":2},{"task":"/task/fairness","name":"Fairness","papers":2},{"task":null,"name":"GPU","papers":2},{"task":"/task/privacy-preserving","name":"Privacy Preserving","papers":2},{"task":"/task/transfer-learning","name":"Transfer Learning","papers":2},{"task":"/task/compressed-sensing","name":"compressed sensing","papers":2},{"task":"/task/adversarial-defense","name":"Adversarial Defense","papers":1},{"task":"/task/adversarial-robustness","name":"Adversarial Robustness","papers":1},{"task":"/task/all","name":"All","papers":1},{"task":"/task/bayesian-inference","name":"Bayesian Inference","papers":1},{"task":"/task/binarization","name":"Binarization","papers":1},{"task":"/task/classification-1","name":"Classification","papers":1},{"task":"/task/collaborative-fairness","name":"Collaborative Fairness","papers":1},{"task":"/task/earth-observation","name":"Earth Observation","papers":1},{"task":"/task/classification","name":"General Classification","papers":1}],"tasks_shown":20,"n_tasks":28,"usage_by_year":[{"year":"2017","papers":1},{"year":"2018","papers":3},{"year":"2019","papers":4},{"year":"2020","papers":5},{"year":"2021","papers":6},{"year":"2022","papers":4},{"year":"2023","papers":6},{"year":"2024","papers":5},{"year":"2025","papers":4}],"row_source":"methods_table","archive":{"source":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","archive_url":"https://paperswithcode.com/method/gradient-sparsification"},"syntology_read_at":"2026-10-05T00:36:11+00:00","syntology":{"source":"Syntology's have check for this name (the free coverage lookup), read from Syntology's graph","have_url":"https://syntology.ai/api/graph/have?x=Gradient%20Sparsification","read_at":"2026-10-05T00:57:12+00:00","query":"Gradient Sparsification","kind":"method","have":"unknown_name","tier":null,"verified_implementation":null,"level":null,"level_meaning":"the served verification level, per answer; never summed across methods","resolved_to":null,"resolution":null,"resolved_name_differs":false,"absence_asserted":false,"absence_unchecked":null,"no_code_is_about_this_method":false,"subject":null,"archive_paper_arxiv_id":"1710.09854","repositories":null,"retrieve_with":null,"name_competition":{"verdict":"none","may_be_wrong_sense":false,"bearers_found":0,"served":null,"leading_other_bearer":null,"n_other_bearers_listed":0,"note":"Nothing in this corpus is named 'Gradient Sparsification' -- no method node, no paper announcing it in its title, no expanded form declaring it as an acronym."},"name_competition_text":null,"near_matches":["gradient EM","Gradient Cuff","Gradient Mirage","pathway gradient","Gradient Surgery","Gradient Sealing","Gradient Surgery","phantom gradient"],"code_nearby":[],"code_nearby_caption":null,"origin_paper":null,"line":"name not resolved (unknown_name). The lookup matched no method, paper or venue by this name; that asserts nothing about whether code exists.","level_legend":null},"syntology_have_state":"loaded","record_sha256":"7aa3858f939d47d00fc22e4d021744a19cf635e7bc4165944543aa9f277deea0","record_changed_at":"2026-10-05","record_changed_at_basis":"changed"}