{"about":{"non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","site":"https://codewithpapers.app","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page","syntology":{"site":"https://syntology.ai","developers":"https://syntology.ai/developers","mcp":{"server":"https://syntology.ai/mcp","transport":"streamable-http","server_card":"https://syntology.ai/.well-known/mcp/server-card.json","auth":{"type":"trial token, no account","trial_token":"https://syntology.ai/api/oauth/trial/token","method":"POST","docs":"https://syntology.ai/developers"}},"have":"https://syntology.ai/api/graph/have?x=<method, arXiv id or title> (free, answers coverage only)","paper_base":"https://syntology.ai/paper/","atlas_base":"https://app.syntology.ai/?focus="},"machine_readable":[{"url":"https://codewithpapers.app/llms.txt","what":"the machine catalog: every machine-readable file, counted"},{"url":"https://codewithpapers.app/index/manifest.json","what":"paper-to-code index by arXiv id, with Syntology's counts"},{"url":"https://codewithpapers.app/search/manifest.json","what":"site search index (titles, authors) and its files"},{"url":"https://codewithpapers.app/download","what":"bulk files: Syntology's layer, described there"},{"url":"https://codewithpapers.app/build_manifest.json","what":"the build record: inputs, counts, exclusions, probes"}]},"url":"/method/pythia","slug":"pythia","name":"Pythia","full_name":"Pythia","full_name_withheld":false,"description_markdown":"**Pythia** is a suite of decoder-only autoregressive language models all trained on public data seen in the exact same order and ranging in size from 70M to 12B parameters. The model architecture and hyperparameters largely follow GPT-3, with a few notable deviations based on recent advances in best practices for large scale language modeling.","description_state":"present","introduced_year":null,"introduced_by":{"title":"Pythia: A Suite for Analyzing Large Language Models Across Training and Scaling","paper":"/paper/pythia-a-suite-for-analyzing-large-language","first_author":"Stella Biderman","n_authors":13,"url_abs":null,"archive_paper_url":"https://paperswithcode.com/paper/pythia-a-suite-for-analyzing-large-language"},"source":{"url":"https://arxiv.org/abs/2304.01373v2","title":"Pythia: A Suite for Analyzing Large Language Models Across Training and Scaling","url_on_a_paper_host":true},"code_snippet_url":null,"code_snippet_url_on_a_code_host":false,"categories":[{"area":"Natural Language Processing","area_id":"natural-language-processing","collection":"Language Models","url":"/methods/category/language-models","pwc_aliases":[]}],"n_papers_tagged":60,"archive_num_papers":60,"papers_newest_first":[{"paper":"/paper/leximark-robust-watermarking-via-lexical","slug":"leximark-robust-watermarking-via-lexical","title":"LexiMark: Robust Watermarking via Lexical Substitutions to Enhance Membership Verification of an LLM's Textual Training Data","date":"2025-06-17","arxiv_id":"2506.14474","n_code_links":1,"syntology":null},{"paper":"/paper/what-happens-during-the-loss-plateau","slug":"what-happens-during-the-loss-plateau","title":"What Happens During the Loss Plateau? Understanding Abrupt Learning in Transformers","date":"2025-06-16","arxiv_id":"2506.13688","n_code_links":1,"syntology":{"ran":2,"of":2,"n_ran_checked":0,"n_instrument":2,"unverified":0,"pointer_only":2,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 0 unverified","official":{"repos":["pulkitgopalani/tf-loss-plateau"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"paper":null,"slug":"stochastic-chameleons-irrelevant-context","title":"Stochastic Chameleons: Irrelevant Context Hallucinations Reveal Class-Based (Mis)Generalization in LLMs","date":"2025-05-28","arxiv_id":"2505.22630","n_code_links":0,"syntology":null},{"paper":"/paper/pretraining-language-models-to-ponder-in","slug":"pretraining-language-models-to-ponder-in","title":"Pretraining Language Models to Ponder in Continuous Space","date":"2025-05-27","arxiv_id":"2505.20674","n_code_links":1,"syntology":{"ran":6,"of":6,"n_ran_checked":1,"n_instrument":5,"unverified":0,"pointer_only":4,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 5 where Syntology's instrument failed) · 0 unverified","official":{"repos":["lumia-group/ponderinglm"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official","unlocated"]}}},{"paper":"/paper/2505-11004","slug":"2505-11004","title":"Illusion or Algorithm? Investigating Memorization, Emergence, and Symbolic Processing in In-Context Learning","date":"2025-05-16","arxiv_id":"2505.11004","n_code_links":1,"syntology":null},{"paper":null,"slug":"memorization-or-interpolation-detecting-llm","title":"Memorization or Interpolation ? Detecting LLM Memorization through Input Perturbation Analysis","date":"2025-05-05","arxiv_id":"2505.03019","n_code_links":0,"syntology":null},{"paper":null,"slug":"an-empirical-study-of-the-role-of","title":"An Empirical Study of the Role of Incompleteness and Ambiguity in Interactions with Large Language Models","date":"2025-03-23","arxiv_id":"2503.17936","n_code_links":0,"syntology":null},{"paper":null,"slug":"i-predict-therefore-i-am-is-next-token","title":"I Predict Therefore I Am: Is Next Token Prediction Enough to Learn Human-Interpretable Concepts from Data?","date":"2025-03-12","arxiv_id":"2503.08980","n_code_links":0,"syntology":null},{"paper":"/paper/polypythias-stability-and-outliers-across","slug":"polypythias-stability-and-outliers-across","title":"PolyPythias: Stability and Outliers across Fifty Language Model Pre-Training Runs","date":"2025-03-12","arxiv_id":"2503.09543","n_code_links":1,"syntology":null},{"paper":null,"slug":"interrogating-llm-design-under-a-fair","title":"Interrogating LLM design under a fair learning doctrine","date":"2025-02-22","arxiv_id":"2502.16290","n_code_links":0,"syntology":null},{"paper":null,"slug":"revisiting-privacy-utility-and-efficiency","title":"Revisiting Privacy, Utility, and Efficiency Trade-offs when Fine-Tuning Large Language Models","date":"2025-02-18","arxiv_id":"2502.13313","n_code_links":0,"syntology":null},{"paper":"/paper/roste-an-efficient-quantization-aware","slug":"roste-an-efficient-quantization-aware","title":"RoSTE: An Efficient Quantization-Aware Supervised Fine-Tuning Approach for Large Language Models","date":"2025-02-13","arxiv_id":"2502.09003","n_code_links":0,"syntology":{"ran":5,"of":5,"n_ran_checked":5,"n_instrument":0,"unverified":0,"pointer_only":5,"phrase":"5 ran (of which 5 constructed an object rather than computing a result; 5 with no instrument failure: 0 honoured, 0 violated, 5 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified; every one of the 5 samples that ran constructed an object rather than computing a result","official":null}},{"paper":null,"slug":"memhunter-automated-and-verifiable","title":"MemHunter: Automated and Verifiable Memorization Detection at Dataset-scale in LLMs","date":"2024-12-10","arxiv_id":"2412.07261","n_code_links":0,"syntology":null},{"paper":"/paper/star-agents-automatic-data-optimization-with","slug":"star-agents-automatic-data-optimization-with","title":"Star-Agents: Automatic Data Optimization with LLM Agents for Instruction Tuning","date":"2024-11-21","arxiv_id":"2411.14497","n_code_links":1,"syntology":null},{"paper":"/paper/explaining-and-improving-contrastive-decoding","slug":"explaining-and-improving-contrastive-decoding","title":"Explaining and Improving Contrastive Decoding by Extrapolating the Probabilities of a Huge and Hypothetical LM","date":"2024-11-03","arxiv_id":"2411.01610","n_code_links":1,"syntology":null},{"paper":null,"slug":"efficient-training-of-sparse-autoencoders-for","title":"Efficient Training of Sparse Autoencoders for Large Language Models via Layer Groups","date":"2024-10-28","arxiv_id":"2410.21508","n_code_links":0,"syntology":null},{"paper":null,"slug":"relaxed-recursive-transformers-effective","title":"Relaxed Recursive Transformers: Effective Parameter Sharing with Layer-wise LoRA","date":"2024-10-28","arxiv_id":"2410.20672","n_code_links":0,"syntology":null},{"paper":null,"slug":"hallucination-detox-sensitive-neuron-dropout","title":"Hallucination Detox: Sensitivity Dropout (SenD) for Large Language Model Training","date":"2024-10-20","arxiv_id":"2410.15460","n_code_links":0,"syntology":null},{"paper":"/paper/tending-towards-stability-convergence","slug":"tending-towards-stability-convergence","title":"Tending Towards Stability: Convergence Challenges in Small Language Models","date":"2024-10-15","arxiv_id":"2410.11451","n_code_links":1,"syntology":null},{"paper":null,"slug":"context-parametric-inversion-why-instruction","title":"Context-Parametric Inversion: Why Instruction Finetuning Can Worsen Context Reliance","date":"2024-10-14","arxiv_id":"2410.10796","n_code_links":0,"syntology":null},{"paper":"/paper/large-language-model-evaluation-via-matrix-1","slug":"large-language-model-evaluation-via-matrix-1","title":"Large Language Model Evaluation via Matrix Nuclear-Norm","date":"2024-10-14","arxiv_id":"2410.10672","n_code_links":1,"syntology":null},{"paper":"/paper/local-and-global-decoding-in-text-generation","slug":"local-and-global-decoding-in-text-generation","title":"Local and Global Decoding in Text Generation","date":"2024-10-14","arxiv_id":"2410.10810","n_code_links":1,"syntology":{"ran":11,"of":12,"n_ran_checked":5,"n_instrument":6,"unverified":1,"pointer_only":12,"phrase":"11 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 1 honoured, 1 violated, 3 with no contract checked; 6 where Syntology's instrument failed) · 1 unverified","official":{"repos":["lowlypalace/global-decoding"],"state":"official (archive's flag): 11 ran","n_ran":11,"n_constructed":0,"n_ran_no_instrument_failure":5,"n_unverified":1,"ran_from_kinds":["official"]}}},{"paper":"/paper/reconstruction-of-particle-flow-energy","slug":"reconstruction-of-particle-flow-energy","title":"Lightweight Deep Learning Framework for Accurate Particle Flow Energy Reconstruction","date":"2024-10-08","arxiv_id":"2410.07250","n_code_links":1,"syntology":null},{"paper":null,"slug":"order-of-magnitude-speedups-for-llm","title":"Order of Magnitude Speedups for LLM Membership Inference","date":"2024-09-22","arxiv_id":"2409.14513","n_code_links":0,"syntology":null},{"paper":null,"slug":"generated-data-with-fake-privacy-hidden","title":"Generated Data with Fake Privacy: Hidden Dangers of Fine-tuning Large Language Models on Generated Data","date":"2024-09-12","arxiv_id":"2409.11423","n_code_links":0,"syntology":null},{"paper":null,"slug":"accelerating-large-language-model-pretraining","title":"Accelerating Large Language Model Pretraining via LFR Pedagogy: Learn, Focus, and Review","date":"2024-09-10","arxiv_id":"2409.06131","n_code_links":0,"syntology":null},{"paper":"/paper/demystifying-verbatim-memorization-in-large","slug":"demystifying-verbatim-memorization-in-large","title":"Demystifying Verbatim Memorization in Large Language Models","date":"2024-07-25","arxiv_id":"2407.17817","n_code_links":1,"syntology":{"ran":6,"of":6,"n_ran_checked":5,"n_instrument":1,"unverified":0,"pointer_only":0,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 2 honoured, 0 violated, 3 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","official":{"repos":["explanare/verbatim-memorization"],"state":"official (archive's flag): 6 ran","n_ran":6,"n_constructed":0,"n_ran_no_instrument_failure":5,"n_unverified":0,"ran_from_kinds":["official"]}}},{"paper":"/paper/generalization-v-s-memorization-tracing","slug":"generalization-v-s-memorization-tracing","title":"Generalization v.s. Memorization: Tracing Language Models' Capabilities Back to Pretraining Data","date":"2024-07-20","arxiv_id":"2407.14985","n_code_links":0,"syntology":{"ran":11,"of":15,"n_ran_checked":4,"n_instrument":7,"unverified":4,"pointer_only":0,"phrase":"11 ran (of which 0 constructed an object rather than computing a result; 4 with no instrument failure: 3 honoured, 1 violated, 0 with no contract checked; 7 where Syntology's instrument failed) · 4 unverified","official":null}},{"paper":"/paper/mathcamps-fine-grained-synthesis-of","slug":"mathcamps-fine-grained-synthesis-of","title":"MathCAMPS: Fine-grained Synthesis of Mathematical Problems From Human Curricula","date":"2024-07-01","arxiv_id":"2407.00900","n_code_links":1,"syntology":{"ran":8,"of":8,"n_ran_checked":1,"n_instrument":7,"unverified":0,"pointer_only":0,"phrase":"8 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 7 where Syntology's instrument failed) · 0 unverified","official":{"repos":["gpoesia/mathcamps"],"state":"official (archive's flag): 8 ran","n_ran":8,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"paper":"/paper/evaluating-n-gram-novelty-of-language-models","slug":"evaluating-n-gram-novelty-of-language-models","title":"Evaluating $n$-Gram Novelty of Language Models Using Rusty-DAWG","date":"2024-06-18","arxiv_id":"2406.13069","n_code_links":1,"syntology":null},{"paper":null,"slug":"semantic-membership-inference-attack-against","title":"Semantic Membership Inference Attack against Large Language Models","date":"2024-06-14","arxiv_id":"2406.10218","n_code_links":0,"syntology":null},{"paper":"/paper/mates-model-aware-data-selection-for","slug":"mates-model-aware-data-selection-for","title":"MATES: Model-Aware Data Selection for Efficient Pretraining with Data Influence Models","date":"2024-06-10","arxiv_id":"2406.06046","n_code_links":1,"syntology":{"ran":11,"of":12,"n_ran_checked":8,"n_instrument":3,"unverified":1,"pointer_only":4,"phrase":"11 ran (of which 0 constructed an object rather than computing a result; 8 with no instrument failure: 5 honoured, 2 violated, 1 with no contract checked; 3 where Syntology's instrument failed) · 1 unverified","official":{"repos":["cxcscmu/mates"],"state":"official (archive's flag): 11 ran","n_ran":11,"n_constructed":0,"n_ran_no_instrument_failure":8,"n_unverified":1,"ran_from_kinds":["official"]}}},{"paper":"/paper/causal-estimation-of-memorisation-profiles","slug":"causal-estimation-of-memorisation-profiles","title":"Causal Estimation of Memorisation Profiles","date":"2024-06-06","arxiv_id":"2406.04327","n_code_links":1,"syntology":{"ran":3,"of":3,"n_ran_checked":2,"n_instrument":1,"unverified":0,"pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 1 honoured, 0 violated, 1 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","official":{"repos":["pietrolesci/memorisation-profiles"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["official"]}}},{"paper":null,"slug":"how-truncating-weights-improves-reasoning-in","title":"How Truncating Weights Improves Reasoning in Language Models","date":"2024-06-05","arxiv_id":"2406.03068","n_code_links":0,"syntology":null},{"paper":"/paper/zyda-a-1-3t-dataset-for-open-language","slug":"zyda-a-1-3t-dataset-for-open-language","title":"Zyda: A 1.3T Dataset for Open Language Modeling","date":"2024-06-04","arxiv_id":"2406.01981","n_code_links":1,"syntology":null},{"paper":"/paper/map-neo-highly-capable-and-transparent","slug":"map-neo-highly-capable-and-transparent","title":"MAP-Neo: Highly Capable and Transparent Bilingual Large Language Model Series","date":"2024-05-29","arxiv_id":"2405.19327","n_code_links":1,"syntology":{"ran":12,"of":14,"n_ran_checked":4,"n_instrument":8,"unverified":2,"pointer_only":14,"phrase":"12 ran (of which 0 constructed an object rather than computing a result; 4 with no instrument failure: 0 honoured, 0 violated, 4 with no contract checked; 8 where Syntology's instrument failed) · 2 unverified","official":{"repos":["multimodal-art-projection/map-neo"],"state":"official (archive's flag): 12 ran","n_ran":12,"n_constructed":0,"n_ran_no_instrument_failure":4,"n_unverified":2,"ran_from_kinds":["official"]}}},{"paper":"/paper/towards-reliable-latent-knowledge-estimation","slug":"towards-reliable-latent-knowledge-estimation","title":"Towards Reliable Latent Knowledge Estimation in LLMs: Zero-Prompt Many-Shot Based Factual Knowledge Extraction","date":"2024-04-19","arxiv_id":"2404.12957","n_code_links":1,"syntology":null},{"paper":null,"slug":"learn-your-reference-model-for-real-good","title":"Learn Your Reference Model for Real Good Alignment","date":"2024-04-15","arxiv_id":"2404.09656","n_code_links":0,"syntology":null},{"paper":"/paper/mechanisms-of-non-factual-hallucinations-in","slug":"mechanisms-of-non-factual-hallucinations-in","title":"Mechanistic Understanding and Mitigation of Language Model Non-Factual Hallucinations","date":"2024-03-27","arxiv_id":"2403.18167","n_code_links":1,"syntology":{"ran":4,"of":5,"n_ran_checked":2,"n_instrument":2,"unverified":1,"pointer_only":5,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 1 honoured, 1 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 1 unverified","official":{"repos":["jadeleiyu/lm_hallucination_mechanisms"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":1,"ran_from_kinds":["official"]}}},{"paper":"/paper/the-n-implementation-details-of-rlhf-with-ppo","slug":"the-n-implementation-details-of-rlhf-with-ppo","title":"The N+ Implementation Details of RLHF with PPO: A Case Study on TL;DR Summarization","date":"2024-03-24","arxiv_id":"2403.17031","n_code_links":1,"syntology":{"ran":8,"of":13,"n_ran_checked":7,"n_instrument":1,"unverified":5,"pointer_only":0,"phrase":"8 ran (of which 0 constructed an object rather than computing a result; 7 with no instrument failure: 1 honoured, 0 violated, 6 with no contract checked; 1 where Syntology's instrument failed) · 5 unverified","official":{"repos":["vwxyzjn/summarize_from_feedback_details"],"state":"official (archive's flag): 8 ran","n_ran":8,"n_constructed":0,"n_ran_no_instrument_failure":7,"n_unverified":5,"ran_from_kinds":["official"]}}},{"paper":"/paper/pandora-s-white-box-increased-training-data","slug":"pandora-s-white-box-increased-training-data","title":"Pandora's White-Box: Precise Training Data Detection and Extraction in Large Language Models","date":"2024-02-26","arxiv_id":"2402.17012","n_code_links":1,"syntology":{"ran":7,"of":9,"n_ran_checked":4,"n_instrument":3,"unverified":2,"pointer_only":0,"phrase":"7 ran (of which 0 constructed an object rather than computing a result; 4 with no instrument failure: 0 honoured, 0 violated, 4 with no contract checked; 3 where Syntology's instrument failed) · 2 unverified","official":null}},{"paper":"/paper/causalgym-benchmarking-causal","slug":"causalgym-benchmarking-causal","title":"CausalGym: Benchmarking causal interpretability methods on linguistic tasks","date":"2024-02-19","arxiv_id":"2402.12560","n_code_links":2,"syntology":{"ran":2,"of":2,"n_ran_checked":0,"n_instrument":2,"unverified":0,"pointer_only":2,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 0 unverified","official":{"repos":["aryamanarora/causalgym"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"paper":"/paper/model-editing-with-canonical-examples","slug":"model-editing-with-canonical-examples","title":"Model Editing with Canonical Examples","date":"2024-02-09","arxiv_id":"2402.06155","n_code_links":1,"syntology":{"ran":2,"of":4,"n_ran_checked":2,"n_instrument":0,"unverified":2,"pointer_only":4,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 0 where Syntology's instrument failed) · 2 unverified","official":{"repos":["john-hewitt/model-editing-canonical-examples"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":2,"ran_from_kinds":["official"]}}},{"paper":null,"slug":"successor-heads-recurring-interpretable","title":"Successor Heads: Recurring, Interpretable Attention Heads In The Wild","date":"2023-12-14","arxiv_id":"2312.09230","n_code_links":0,"syntology":null},{"paper":"/paper/dyad-a-descriptive-yet-abjuring-density","slug":"dyad-a-descriptive-yet-abjuring-density","title":"DYAD: A Descriptive Yet Abjuring Density efficient approximation to linear neural network layers","date":"2023-12-11","arxiv_id":"2312.06881","n_code_links":1,"syntology":{"ran":1,"of":1,"n_ran_checked":1,"n_instrument":0,"unverified":0,"pointer_only":1,"phrase":"1 ran (of which 1 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified; the one sample that ran constructed an object rather than computing a result","official":{"repos":["asappresearch/dyad"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":1,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"paper":"/paper/sparq-attention-bandwidth-efficient-llm","slug":"sparq-attention-bandwidth-efficient-llm","title":"SparQ Attention: Bandwidth-Efficient LLM Inference","date":"2023-12-08","arxiv_id":"2312.04985","n_code_links":1,"syntology":{"ran":3,"of":3,"n_ran_checked":2,"n_instrument":1,"unverified":0,"pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 2 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","official":{"repos":["graphcore-research/llm-inference-research"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["official"]}}},{"paper":null,"slug":"scalable-extraction-of-training-data-from","title":"Scalable Extraction of Training Data from (Production) Language Models","date":"2023-11-28","arxiv_id":"2311.17035","n_code_links":0,"syntology":null},{"paper":"/paper/how-do-language-models-bind-entities-in","slug":"how-do-language-models-bind-entities-in","title":"How do Language Models Bind Entities in Context?","date":"2023-10-26","arxiv_id":"2310.17191","n_code_links":1,"syntology":{"ran":1,"of":1,"n_ran_checked":1,"n_instrument":0,"unverified":0,"pointer_only":1,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","official":{"repos":["jiahai-feng/binding-iclr"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"paper":null,"slug":"characterizing-mechanisms-for-factual-recall","title":"Characterizing Mechanisms for Factual Recall in Language Models","date":"2023-10-24","arxiv_id":"2310.15910","n_code_links":0,"syntology":null},{"paper":"/paper/sheared-llama-accelerating-language-model-pre","slug":"sheared-llama-accelerating-language-model-pre","title":"Sheared LLaMA: Accelerating Language Model Pre-training via Structured Pruning","date":"2023-10-10","arxiv_id":"2310.06694","n_code_links":2,"syntology":{"ran":3,"of":3,"n_ran_checked":1,"n_instrument":2,"unverified":0,"pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 0 unverified","official":{"repos":["princeton-nlp/llm-shearing"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"paper":"/paper/efficient-streaming-language-models-with","slug":"efficient-streaming-language-models-with","title":"Efficient Streaming Language Models with Attention Sinks","date":"2023-09-29","arxiv_id":"2309.17453","n_code_links":6,"syntology":{"ran":8,"of":11,"n_ran_checked":4,"n_instrument":4,"unverified":3,"pointer_only":2,"phrase":"8 ran (of which 4 constructed an object rather than computing a result; 4 with no instrument failure: 0 honoured, 0 violated, 4 with no contract checked; 4 where Syntology's instrument failed) · 3 unverified","official":{"repos":["intel/intel-extension-for-transformers","mit-han-lab/streaming-llm"],"state":"official (archive's flag): 6 ran","n_ran":6,"n_constructed":2,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["listed","official"]}}},{"paper":"/paper/tart-a-plug-and-play-transformer-module-for","slug":"tart-a-plug-and-play-transformer-module-for","title":"TART: A plug-and-play Transformer module for task-agnostic reasoning","date":"2023-09-21","arxiv_id":null,"n_code_links":1,"syntology":null},{"paper":"/paper/continual-pre-training-of-large-language","slug":"continual-pre-training-of-large-language","title":"Continual Pre-Training of Large Language Models: How to (re)warm your model?","date":"2023-08-08","arxiv_id":"2308.04014","n_code_links":2,"syntology":null},{"paper":"/paper/universal-and-transferable-adversarial","slug":"universal-and-transferable-adversarial","title":"Universal and Transferable Adversarial Attacks on Aligned Language Models","date":"2023-07-27","arxiv_id":"2307.15043","n_code_links":25,"syntology":{"ran":39,"of":61,"n_ran_checked":19,"n_instrument":20,"unverified":22,"pointer_only":4,"phrase":"39 ran (of which 7 constructed an object rather than computing a result; 19 with no instrument failure: 4 honoured, 1 violated, 14 with no contract checked; 20 where Syntology's instrument failed) · 22 unverified","official":{"repos":["llm-attacks/llm-attacks"],"state":"official: no sample here; runs from other or unrecorded repositories","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed","unlocated"]}}},{"paper":"/paper/stay-on-topic-with-classifier-free-guidance","slug":"stay-on-topic-with-classifier-free-guidance","title":"Stay on topic with Classifier-Free Guidance","date":"2023-06-30","arxiv_id":"2306.17806","n_code_links":0,"syntology":null},{"paper":"/paper/easyguide-esg-issue-identification-framework","slug":"easyguide-esg-issue-identification-framework","title":"EaSyGuide : ESG Issue Identification Framework leveraging Abilities of Generative Large Language Models","date":"2023-06-11","arxiv_id":"2306.06662","n_code_links":1,"syntology":null},{"paper":"/paper/understanding-the-effectiveness-of-early","slug":"understanding-the-effectiveness-of-early","title":"Early Weight Averaging meets High Learning Rates for LLM Pre-training","date":"2023-06-05","arxiv_id":"2306.03241","n_code_links":1,"syntology":{"ran":3,"of":3,"n_ran_checked":2,"n_instrument":1,"unverified":0,"pointer_only":3,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 1 honoured, 1 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","official":{"repos":["sanyalsunny111/early_weight_avg"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["official"]}}},{"paper":null,"slug":"emergent-inabilities-inverse-scaling-over-the","title":"Emergent inabilities? Inverse scaling over the course of pretraining","date":"2023-05-24","arxiv_id":"2305.14681","n_code_links":0,"syntology":null},{"paper":"/paper/emergent-and-predictable-memorization-in-1","slug":"emergent-and-predictable-memorization-in-1","title":"Emergent and Predictable Memorization in Large Language Models","date":"2023-04-21","arxiv_id":"2304.11158","n_code_links":2,"syntology":{"ran":1,"of":1,"n_ran_checked":0,"n_instrument":1,"unverified":0,"pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","official":{"repos":["eleutherai/gpt-neox","eleutherai/pythia"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"paper":"/paper/pythia-a-suite-for-analyzing-large-language","slug":"pythia-a-suite-for-analyzing-large-language","title":"Pythia: A Suite for Analyzing Large Language Models Across Training and Scaling","date":"2023-04-03","arxiv_id":"2304.01373","n_code_links":4,"syntology":null}],"papers_shown":60,"papers_order":"date (newest first), then slug","paper_counts":{"archive_papers_tagged":60,"with_a_code_link":36,"where_syntology_ran_a_sample":23,"not_listed_spam_title":0,"listed":60,"listed_where_code_ran":23,"where_syntology_ran_a_sample_split":{"with_a_run_with_no_instrument_failure":20,"every_run_a_failure_of_syntologys_instrument":3,"listed_with_a_run_with_no_instrument_failure":20,"listed_every_run_a_failure_of_syntologys_instrument":3,"filter":{"states":["a run with no instrument failure","any run, instrument failures included"],"default":"a run with no instrument failure","note":"on the 'only where code ran' pages the default hides, in the browser, the rows where every run was a failure of Syntology's instrument; the second state shows them again. Rows are hidden, never re-ordered; these twins list every row"}},"definition":"distinct papers the archive tags; 'where Syntology ran a sample' counts papers with at least one harvested sample that ran, which is not a correctness claim"},"papers_where_code_ran":23,"papers_where_code_ran_first_page":"/method/pythia/papers/ran/1","papers_where_code_ran_caption":"We ran code from the paper's repository; we did not isolate this method inside it.","pages":[{"order":"ran","page":1,"of":1,"url":"/method/pythia/papers/ran/1","json":"/method/pythia/papers/ran/1.json","rows":23}],"tasks":[{"task":"/task/language-modelling","name":"Language Modelling","papers":23},{"task":"/task/language-modeling","name":"Language Modeling","papers":15},{"task":"/task/memorization","name":"Memorization","papers":10},{"task":"/task/large-language-model","name":"Large Language Model","papers":5},{"task":"/task/question-answering","name":"Question Answering","papers":4},{"task":"/task/text-generation","name":"Text Generation","papers":3},{"task":"/task/articles","name":"Articles","papers":2},{"task":"/task/common-sense-reasoning","name":"Common Sense Reasoning","papers":2},{"task":"/task/computational-efficiency","name":"Computational Efficiency","papers":2},{"task":"/task/hallucination","name":"Hallucination","papers":2},{"task":"/task/in-context-learning","name":"In-Context Learning","papers":2},{"task":"/task/knowledge-distillation","name":"Knowledge Distillation","papers":2},{"task":"/task/lambada","name":"LAMBADA","papers":2},{"task":"/task/machine-translation","name":"Machine Translation","papers":2},{"task":"/task/math","name":"Math","papers":2},{"task":"/task/sentence-completion","name":"Sentence Completion","papers":2},{"task":"/task/world-knowledge","name":"World Knowledge","papers":2},{"task":null,"name":"counterfactual","papers":2},{"task":"/task/model","name":"model","papers":2},{"task":"/task/adversarial-attack","name":"Adversarial Attack","papers":1}],"tasks_shown":20,"n_tasks":58,"usage_by_year":[{"year":"2023","papers":17},{"year":"2024","papers":31},{"year":"2025","papers":12}],"row_source":"methods_table","archive":{"source":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","archive_url":"https://paperswithcode.com/method/pythia"},"syntology_read_at":"2026-10-05T00:36:11+00:00","syntology":{"source":"Syntology's have check for this name (the free coverage lookup), read from Syntology's graph","have_url":"https://syntology.ai/api/graph/have?x=Pythia","read_at":"2026-10-05T01:00:47+00:00","query":"Pythia","kind":"paper","have":"declared_code","tier":"repo_known","verified_implementation":false,"level":null,"level_meaning":"the served verification level, per answer; never summed across methods","resolved_to":{"arxiv_id":"2304.01373"},"resolution":"paper","resolved_name_differs":false,"absence_asserted":false,"absence_unchecked":null,"no_code_is_about_this_method":false,"subject":"same_paper","archive_paper_arxiv_id":"2304.01373","repositories":{"listed":5,"in_graph":5,"list_truncated":false,"strongest_link":"official"},"retrieve_with":null,"name_competition":{"verdict":"contested","may_be_wrong_sense":false,"bearers_found":4,"served":{"arxiv_id":"2304.01373","citations":2091},"leading_other_bearer":{"arxiv_id":"1912.00742","title":"Pythia: AI-assisted Code Completion System","citations":182,"named_by":["title_prefix"],"bearer_name":"Pythia: AI-assisted Code Completion System","ask_with":"have('1912.00742')"},"n_other_bearers_listed":3,"note":"3 other thing(s) here are also called 'Pythia', and none of them is cited more than the paper this answer is about. They are listed in `also_named` -- check `served.arxiv_id` is the paper you meant."},"name_competition_text":null,"near_matches":null,"code_nearby":[],"code_nearby_caption":null,"origin_paper":{"arxiv_id":"2304.01373","page":"/paper/pythia-a-suite-for-analyzing-large-language","syntology_url":"https://syntology.ai/paper/2304.01373","in_graph_layer":false,"samples":0},"line":"repository known, no verified implementation (declared_code; repositories: 5 listed of 5 in the graph, strongest link: official). The lookup matched a paper by its title, arXiv:2304.01373, not a method entry. This answer is about arXiv:2304.01373, the paper the archive credits with this method.","level_legend":null},"syntology_have_state":"loaded","record_sha256":"88bb80b246923dece85f42d7c177cfbc1de28070fa021125393a15ffa0d9ed9e","record_changed_at":"2026-10-05","record_changed_at_basis":"changed"}