{"about":{"non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","site":"https://codewithpapers.app","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page","syntology":{"site":"https://syntology.ai","developers":"https://syntology.ai/developers","mcp":{"server":"https://syntology.ai/mcp","transport":"streamable-http","server_card":"https://syntology.ai/.well-known/mcp/server-card.json","auth":{"type":"trial token, no account","trial_token":"https://syntology.ai/api/oauth/trial/token","method":"POST","docs":"https://syntology.ai/developers"}},"have":"https://syntology.ai/api/graph/have?x=<method, arXiv id or title> (free, answers coverage only)","paper_base":"https://syntology.ai/paper/","atlas_base":"https://app.syntology.ai/?focus="},"machine_readable":[{"url":"https://codewithpapers.app/llms.txt","what":"the machine catalog: every machine-readable file, counted"},{"url":"https://codewithpapers.app/index/manifest.json","what":"paper-to-code index by arXiv id, with Syntology's counts"},{"url":"https://codewithpapers.app/search/manifest.json","what":"site search index (titles, authors) and its files"},{"url":"https://codewithpapers.app/download","what":"bulk files: Syntology's layer, described there"},{"url":"https://codewithpapers.app/build_manifest.json","what":"the build record: inputs, counts, exclusions, probes"}]},"url":"/method/chinchilla","slug":"chinchilla","name":"Chinchilla","full_name":"Chinchilla","full_name_withheld":false,"description_markdown":"Chinchilla is a 70B parameters model trained as a compute-optimal model with 1.4 trillion tokens. Findings suggest that these types of models are trained optimally by equally scaling both model size and training tokens. It uses the same compute budget as Gopher but with 4x more training data. Chinchilla and Gopher are trained for the same number of FLOPs. It is trained using [MassiveText](/dataset/massivetext) using a slightly modified SentencePiece tokenizer. More architectural details in the paper.","description_state":"present","introduced_year":null,"introduced_by":{"title":null,"paper":null,"first_author":null,"n_authors":0,"url_abs":null,"archive_paper_url":null},"source":{"url":"https://arxiv.org/abs/2203.15556v1","title":"Training Compute-Optimal Large Language Models","url_on_a_paper_host":true},"code_snippet_url":null,"code_snippet_url_on_a_code_host":false,"categories":[{"area":"Natural Language Processing","area_id":"natural-language-processing","collection":"Language Models","url":"/methods/category/language-models","pwc_aliases":[]}],"n_papers_tagged":25,"archive_num_papers":null,"papers_newest_first":[{"paper":"/paper/superposition-yields-robust-neural-scaling","slug":"superposition-yields-robust-neural-scaling","title":"Superposition Yields Robust Neural Scaling","date":"2025-05-15","arxiv_id":"2505.10465","n_code_links":1,"syntology":null},{"paper":null,"slug":"compute-optimal-llms-provably-generalize","title":"Compute-Optimal LLMs Provably Generalize Better With Scale","date":"2025-04-21","arxiv_id":"2504.15208","n_code_links":0,"syntology":null},{"paper":null,"slug":"scaling-inference-efficient-language-models","title":"Scaling Inference-Efficient Language Models","date":"2025-01-30","arxiv_id":"2501.18107","n_code_links":0,"syntology":null},{"paper":"/paper/physics-of-skill-learning","slug":"physics-of-skill-learning","title":"Physics of Skill Learning","date":"2025-01-21","arxiv_id":"2501.12391","n_code_links":1,"syntology":null},{"paper":null,"slug":"the-journey-matters-average-parameter-count","title":"The Journey Matters: Average Parameter Count over Pre-training Unifies Sparse and Dense Scaling Laws","date":"2025-01-21","arxiv_id":"2501.12486","n_code_links":0,"syntology":null},{"paper":"/paper/how-much-can-we-forget-about-data","slug":"how-much-can-we-forget-about-data","title":"How Much Can We Forget about Data Contamination?","date":"2024-10-04","arxiv_id":"2410.03249","n_code_links":1,"syntology":{"ran":1,"of":2,"n_ran_checked":1,"n_instrument":0,"unverified":1,"pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 1 violated, 0 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","official":{"repos":["tml-tuebingen/forgetting-contamination"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":1,"ran_from_kinds":["official"]}}},{"paper":null,"slug":"energy-estimation-of-last-mile-electric","title":"Energy Estimation of Last Mile Electric Vehicle Routes","date":"2024-08-21","arxiv_id":"2408.12006","n_code_links":0,"syntology":null},{"paper":null,"slug":"scaling-law-with-learning-rate-annealing","title":"Scaling Law with Learning Rate Annealing","date":"2024-08-20","arxiv_id":"2408.11029","n_code_links":0,"syntology":null},{"paper":null,"slug":"time-matters-scaling-laws-for-any-budget","title":"Time Matters: Scaling Laws for Any Budget","date":"2024-06-27","arxiv_id":"2406.18922","n_code_links":0,"syntology":null},{"paper":"/paper/reconciling-kaplan-and-chinchilla-scaling","slug":"reconciling-kaplan-and-chinchilla-scaling","title":"Reconciling Kaplan and Chinchilla Scaling Laws","date":"2024-06-12","arxiv_id":"2406.12907","n_code_links":1,"syntology":{"ran":2,"of":2,"n_ran_checked":2,"n_instrument":0,"unverified":0,"pointer_only":2,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","official":{"repos":["teapearce/reconciling_kaplan_chinchilla_scaling_laws"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["official"]}}},{"paper":"/paper/revisiting-moe-and-dense-speed-accuracy","slug":"revisiting-moe-and-dense-speed-accuracy","title":"Revisiting MoE and Dense Speed-Accuracy Comparisons for LLM Training","date":"2024-05-23","arxiv_id":"2405.15052","n_code_links":1,"syntology":null},{"paper":null,"slug":"more-compute-is-what-you-need","title":"More Compute Is What You Need","date":"2024-04-30","arxiv_id":"2404.19484","n_code_links":0,"syntology":null},{"paper":"/paper/minicpm-unveiling-the-potential-of-small","slug":"minicpm-unveiling-the-potential-of-small","title":"MiniCPM: Unveiling the Potential of Small Language Models with Scalable Training Strategies","date":"2024-04-09","arxiv_id":"2404.06395","n_code_links":3,"syntology":{"ran":1,"of":1,"n_ran_checked":1,"n_instrument":0,"unverified":0,"pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","official":{"repos":["openbmb/minicpm"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"paper":null,"slug":"vbart-the-turkish-llm","title":"VBART: The Turkish LLM","date":"2024-03-02","arxiv_id":"2403.01308","n_code_links":0,"syntology":null},{"paper":null,"slug":"a-resource-model-for-neural-scaling-law","title":"A Resource Model For Neural Scaling Law","date":"2024-02-07","arxiv_id":"2402.05164","n_code_links":0,"syntology":null},{"paper":null,"slug":"beyond-chinchilla-optimal-accounting-for","title":"Beyond Chinchilla-Optimal: Accounting for Inference in Language Model Scaling Laws","date":"2023-12-31","arxiv_id":"2401.00448","n_code_links":0,"syntology":null},{"paper":"/paper/the-falcon-series-of-open-language-models","slug":"the-falcon-series-of-open-language-models","title":"The Falcon Series of Open Language Models","date":"2023-11-28","arxiv_id":"2311.16867","n_code_links":0,"syntology":null},{"paper":"/paper/language-modeling-is-compression","slug":"language-modeling-is-compression","title":"Language Modeling Is Compression","date":"2023-09-19","arxiv_id":"2309.10668","n_code_links":1,"syntology":{"ran":6,"of":8,"n_ran_checked":1,"n_instrument":5,"unverified":2,"pointer_only":0,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 5 where Syntology's instrument failed) · 2 unverified","official":{"repos":["google-deepmind/language_modeling_is_compression"],"state":"official (archive's flag): 6 ran","n_ran":6,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":2,"ran_from_kinds":["official"]}}},{"paper":null,"slug":"does-circuit-analysis-interpretability-scale","title":"Does Circuit Analysis Interpretability Scale? Evidence from Multiple Choice Capabilities in Chinchilla","date":"2023-07-18","arxiv_id":"2307.09458","n_code_links":0,"syntology":null},{"paper":null,"slug":"tune-as-you-scale-hyperparameter-optimization","title":"Tune As You Scale: Hyperparameter Optimization For Compute Efficient Training","date":"2023-06-13","arxiv_id":"2306.08055","n_code_links":0,"syntology":null},{"paper":"/paper/cerebras-gpt-open-compute-optimal-language","slug":"cerebras-gpt-open-compute-optimal-language","title":"Cerebras-GPT: Open Compute-Optimal Language Models Trained on the Cerebras Wafer-Scale Cluster","date":"2023-04-06","arxiv_id":"2304.03208","n_code_links":2,"syntology":null},{"paper":"/paper/accelerating-large-language-model-decoding","slug":"accelerating-large-language-model-decoding","title":"Accelerating Large Language Model Decoding with Speculative Sampling","date":"2023-02-02","arxiv_id":"2302.01318","n_code_links":5,"syntology":null},{"paper":null,"slug":"an-information-theoretic-analysis-of-compute","title":"An Information-Theoretic Analysis of Compute-Optimal Neural Scaling Laws","date":"2022-12-02","arxiv_id":"2212.01365","n_code_links":0,"syntology":null},{"paper":"/paper/galactica-a-large-language-model-for-science-1","slug":"galactica-a-large-language-model-for-science-1","title":"Galactica: A Large Language Model for Science","date":"2022-11-16","arxiv_id":"2211.09085","n_code_links":1,"syntology":{"ran":2,"of":2,"n_ran_checked":2,"n_instrument":0,"unverified":0,"pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","official":{"repos":["paperswithcode/galai"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["official"]}}},{"paper":"/paper/training-compute-optimal-large-language","slug":"training-compute-optimal-large-language","title":"Training Compute-Optimal Large Language Models","date":"2022-03-29","arxiv_id":"2203.15556","n_code_links":2,"syntology":{"ran":8,"of":11,"n_ran_checked":5,"n_instrument":3,"unverified":3,"pointer_only":4,"phrase":"8 ran (of which 3 constructed an object rather than computing a result; 5 with no instrument failure: 2 honoured, 0 violated, 3 with no contract checked; 3 where Syntology's instrument failed) · 3 unverified","official":null}}],"papers_shown":25,"papers_order":"date (newest first), then slug","paper_counts":{"archive_papers_tagged":25,"with_a_code_link":11,"where_syntology_ran_a_sample":6,"not_listed_spam_title":0,"listed":25,"listed_where_code_ran":6,"where_syntology_ran_a_sample_split":{"with_a_run_with_no_instrument_failure":6,"every_run_a_failure_of_syntologys_instrument":0,"listed_with_a_run_with_no_instrument_failure":6,"listed_every_run_a_failure_of_syntologys_instrument":0,"filter":{"states":["a run with no instrument failure","any run, instrument failures included"],"default":"a run with no instrument failure","note":"on the 'only where code ran' pages the default hides, in the browser, the rows where every run was a failure of Syntology's instrument; the second state shows them again. Rows are hidden, never re-ordered; these twins list every row"}},"definition":"distinct papers the archive tags; 'where Syntology ran a sample' counts papers with at least one harvested sample that ran, which is not a correctness claim"},"papers_where_code_ran":6,"papers_where_code_ran_first_page":"/method/chinchilla/papers/ran/1","papers_where_code_ran_caption":"We ran code from the paper's repository; we did not isolate this method inside it.","pages":[{"order":"ran","page":1,"of":1,"url":"/method/chinchilla/papers/ran/1","json":"/method/chinchilla/papers/ran/1.json","rows":6}],"tasks":[{"task":"/task/language-modelling","name":"Language Modelling","papers":8},{"task":"/task/language-modeling","name":"Language Modeling","papers":5},{"task":"/task/large-language-model","name":"Large Language Model","papers":4},{"task":"/task/question-answering","name":"Question Answering","papers":4},{"task":"/task/mmlu","name":"MMLU","papers":3},{"task":"/task/multi-task-language-understanding","name":"Multi-task Language Understanding","papers":3},{"task":"/task/model","name":"model","papers":3},{"task":"/task/anachronisms","name":"Anachronisms","papers":2},{"task":"/task/common-sense-reasoning","name":"Common Sense Reasoning","papers":2},{"task":"/task/decoder","name":"Decoder","papers":2},{"task":"/task/mathematical-reasoning","name":"Mathematical Reasoning","papers":2},{"task":"/task/multiple-choice-qa","name":"Multiple Choice Question Answering (MCQA)","papers":2},{"task":"/task/sentence-completion","name":"Sentence Completion","papers":2},{"task":"/task/word-sense-disambiguation","name":"Word Sense Disambiguation","papers":2},{"task":"/task/abstractive-text-summarization","name":"Abstractive Text Summarization","papers":1},{"task":"/task/analogical-similarity","name":"Analogical Similarity","papers":1},{"task":"/task/analytic-entailment","name":"Analytic Entailment","papers":1},{"task":"/task/bayesian-optimization","name":"Bayesian Optimization","papers":1},{"task":"/task/bias-detection","name":"Bias Detection","papers":1},{"task":"/task/causal-judgment","name":"Causal Judgment","papers":1}],"tasks_shown":20,"n_tasks":106,"usage_by_year":[{"year":"2022","papers":3},{"year":"2023","papers":7},{"year":"2024","papers":10},{"year":"2025","papers":5}],"row_source":"embedded","archive":{"source":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","archive_url":"https://paperswithcode.com/method/chinchilla"},"syntology_read_at":"2026-10-05T00:36:11+00:00","syntology":{"source":"Syntology's have check for this name (the free coverage lookup), read from Syntology's graph","have_url":"https://syntology.ai/api/graph/have?x=Chinchilla","read_at":"2026-10-05T00:53:39+00:00","query":"Chinchilla","kind":"method","have":"declared_code","tier":"repo_known","verified_implementation":false,"level":null,"level_meaning":"the served verification level, per answer; never summed across methods","resolved_to":{"method":"Chinchilla","origin_arxiv_id":"2203.15556"},"resolution":"exact","resolved_name_differs":false,"absence_asserted":false,"absence_unchecked":null,"no_code_is_about_this_method":false,"subject":"unknown","archive_paper_arxiv_id":null,"repositories":{"listed":2,"in_graph":2,"list_truncated":false,"strongest_link":"mentioned_in_github"},"retrieve_with":null,"name_competition":{"verdict":"sole_bearer","may_be_wrong_sense":false,"bearers_found":1,"served":{"arxiv_id":"2203.15556","citations":3665},"leading_other_bearer":null,"n_other_bearers_listed":0,"note":"Nothing else in this corpus is named 'Chinchilla'. That is a statement about this corpus and not about the field: short names are reused, and a sense we never collected cannot show up here. Confirm `served.arxiv_id` if you had a specific paper in mind."},"name_competition_text":null,"near_matches":null,"code_nearby":[],"code_nearby_caption":null,"origin_paper":{"arxiv_id":"2203.15556","page":"/paper/training-compute-optimal-large-language","syntology_url":"https://syntology.ai/paper/2203.15556","in_graph_layer":true,"samples":11,"ran":8,"n_ran_checked":5,"n_instrument":3,"unverified":3,"phrase":"8 ran (of which 3 constructed an object rather than computing a result; 5 with no instrument failure: 2 honoured, 0 violated, 3 with no contract checked; 3 where Syntology's instrument failed) · 3 unverified","instrument_means":"failures of Syntology's instrument, not of the code"},"line":"repository known, no verified implementation (declared_code; repositories: 2 listed of 2 in the graph, strongest link: mentioned on GitHub). The archive credits no paper with an arXiv id for this method, so whether this answer (about arXiv:2203.15556) is about this method is not checked.","level_legend":null},"syntology_have_state":"loaded","record_sha256":"d8ed16e1bc0efa88a8568edb2a9f18acaaf6272d184f68c9bfdc18120ceb7b69","record_changed_at":"2026-10-05","record_changed_at_basis":"changed"}