{"about":{"non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","site":"https://codewithpapers.app","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page","syntology":{"site":"https://syntology.ai","developers":"https://syntology.ai/developers","mcp":{"server":"https://syntology.ai/mcp","transport":"streamable-http","server_card":"https://syntology.ai/.well-known/mcp/server-card.json","auth":{"type":"trial token, no account","trial_token":"https://syntology.ai/api/oauth/trial/token","method":"POST","docs":"https://syntology.ai/developers"}},"have":"https://syntology.ai/api/graph/have?x=<method, arXiv id or title> (free, answers coverage only)","paper_base":"https://syntology.ai/paper/","atlas_base":"https://app.syntology.ai/?focus="},"machine_readable":[{"url":"https://codewithpapers.app/llms.txt","what":"the machine catalog: every machine-readable file, counted"},{"url":"https://codewithpapers.app/index/manifest.json","what":"paper-to-code index by arXiv id, with Syntology's counts"},{"url":"https://codewithpapers.app/search/manifest.json","what":"site search index (titles, authors) and its files"},{"url":"https://codewithpapers.app/download","what":"bulk files: Syntology's layer, described there"},{"url":"https://codewithpapers.app/build_manifest.json","what":"the build record: inputs, counts, exclusions, probes"}]},"url":"/task/memorization","name":"Memorization","slug":"memorization","description_markdown":null,"categories":[{"name":"Natural Language Processing","url":"/area/natural-language-processing"}],"source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","slug_source":"archive_url"},"counts":{"papers_tagged":1088,"papers_with_code":438,"benchmarks":1,"benchmark_tables_in_archive":1,"benchmark_tables_shown":1,"benchmark_tables_withheld_as_spam":0,"benchmark_definition":"a leaderboard table with at least one row; benchmark_tables_shown also counts the zero-row tables; benchmark_tables_in_archive adds the tables withheld as spam","datasets":4,"subtasks":0,"parent_tasks":0},"benchmarks":[{"leaderboard":"/sota/memorization-on-big-bench-hindu-knowledge","slug":"memorization-on-big-bench-hindu-knowledge","dataset":"BIG-bench (Hindu Knowledge)","dataset_url":"/dataset/big-bench","rows_in_archive":3,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"PaLM-540B (few-shot, k=5)","paper_title":"PaLM: Scaling Language Modeling with Pathways","paper_url":"/paper/palm-scaling-language-modeling-with-pathways-1","paper_date":"2022-04-05","arxiv_id":"2204.02311","code_links":[{"title":"lucidrains/CoCa-pytorch","url":"https://github.com/lucidrains/CoCa-pytorch"},{"title":"lucidrains/PaLM-pytorch","url":"https://github.com/lucidrains/PaLM-pytorch"},{"title":"google/paxml","url":"https://github.com/google/paxml"},{"title":"foundation-model-stack/fms-fsdp","url":"https://github.com/foundation-model-stack/fms-fsdp"},{"title":"lucidrains/PaLM-jax","url":"https://github.com/lucidrains/PaLM-jax"},{"title":"chrisociepa/allamo","url":"https://github.com/chrisociepa/allamo"},{"title":"conceptofmind/PaLM-flax","url":"https://github.com/conceptofmind/PaLM-flax"}],"syntology":{"n":37,"n_ran":32,"n_constructed":16,"n_ran_checked":24,"n_instrument":8,"n_unverified":5,"n_honours":2,"n_violates":1,"n_no_contract":21,"n_pointer_only":0,"phrase":"32 ran (of which 16 constructed an object rather than computing a result; 24 with no instrument failure: 2 honoured, 1 violated, 21 with no contract checked; 8 where Syntology's instrument failed) · 5 unverified","sample_list":"/paper/palm-scaling-language-modeling-with-pathways-1#ran","syntology_url":"https://syntology.ai/paper/2204.02311","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2204.02311"}},"official":null}}}],"datasets":[{"url":"/dataset/big-bench","name":"BIG-bench","full_name":"Beyond the Imitation Game Benchmark","num_papers_in_archive":349},{"url":"/dataset/popqa","name":"PopQA","full_name":"","num_papers_in_archive":91},{"url":"/dataset/ds-1000","name":"DS-1000","full_name":"","num_papers_in_archive":72},{"url":"/dataset/lm-email-address-leakage","name":"LM Email Address Leakage","full_name":"","num_papers_in_archive":1}],"subtasks":[],"parent_tasks":[],"papers":{"order":"repositories listed in the archive (most first), then date (newest first), then slug; the archive holds no stars","population":"every paper the archive tags with this task (distinct papers; spam titles not listed). 'of' counts the papers listed; until 2026-09-25 it counted only the papers with a code link, and that number is of_with_code","page":1,"pages_in_order":11,"rows_per_page":100,"shown":100,"of":1088,"of_with_code":438,"tagged_in_all":1088,"counts":{"archive_papers_tagged":1088,"with_a_code_link":438,"where_syntology_ran_a_sample":177,"not_listed_spam_title":0,"listed":1088,"listed_where_code_ran":177,"where_syntology_ran_a_sample_split":{"with_a_run_with_no_instrument_failure":141,"every_run_a_failure_of_syntologys_instrument":36,"listed_with_a_run_with_no_instrument_failure":141,"listed_every_run_a_failure_of_syntologys_instrument":36,"filter":{"states":["a run with no instrument failure","any run, instrument failures included"],"default":"a run with no instrument failure","note":"on the 'only where code ran' pages the default hides, in the browser, the rows where every run was a failure of Syntology's instrument; the second state shows them again. Rows are hidden, never re-ordered; these twins list every row"}},"definition":"distinct papers the archive tags; 'where Syntology ran a sample' counts papers with at least one harvested sample that ran, which is not a correctness claim"},"items":[{"url":"/paper/mixup-beyond-empirical-risk-minimization","slug":"mixup-beyond-empirical-risk-minimization","title":"mixup: Beyond Empirical Risk Minimization","date":"2017-10-25","arxiv_id":"1710.09412","repositories_listed":71,"syntology":{"n":47,"n_ran":36,"n_constructed":5,"n_ran_checked":14,"n_instrument":22,"n_unverified":11,"n_honours":0,"n_violates":0,"n_no_contract":14,"n_pointer_only":15,"phrase":"36 ran (of which 5 constructed an object rather than computing a result; 14 with no instrument failure: 0 honoured, 0 violated, 14 with no contract checked; 22 where Syntology's instrument failed) · 11 unverified","sample_list":"/paper/mixup-beyond-empirical-risk-minimization#ran","syntology_url":"https://syntology.ai/paper/1710.09412","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1710.09412"}},"official":{"repos":["facebookresearch/mixup-cifar10"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/wide-deep-learning-for-recommender-systems","slug":"wide-deep-learning-for-recommender-systems","title":"Wide & Deep Learning for Recommender Systems","date":"2016-06-24","arxiv_id":"1606.07792","repositories_listed":39,"syntology":{"n":5,"n_ran":3,"n_constructed":0,"n_ran_checked":3,"n_instrument":0,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":5,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/wide-deep-learning-for-recommender-systems#ran","syntology_url":"https://syntology.ai/paper/1606.07792","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1606.07792"}},"official":null}},{"url":"/paper/neural-machine-translation-in-linear-time","slug":"neural-machine-translation-in-linear-time","title":"Neural Machine Translation in Linear Time","date":"2016-10-31","arxiv_id":"1610.10099","repositories_listed":11,"syntology":null},{"url":"/paper/palm-scaling-language-modeling-with-pathways-1","slug":"palm-scaling-language-modeling-with-pathways-1","title":"PaLM: Scaling Language Modeling with Pathways","date":"2022-04-05","arxiv_id":"2204.02311","repositories_listed":7,"syntology":{"n":37,"n_ran":32,"n_constructed":16,"n_ran_checked":24,"n_instrument":8,"n_unverified":5,"n_honours":2,"n_violates":1,"n_no_contract":21,"n_pointer_only":0,"phrase":"32 ran (of which 16 constructed an object rather than computing a result; 24 with no instrument failure: 2 honoured, 1 violated, 21 with no contract checked; 8 where Syntology's instrument failed) · 5 unverified","sample_list":"/paper/palm-scaling-language-modeling-with-pathways-1#ran","syntology_url":"https://syntology.ai/paper/2204.02311","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2204.02311"}},"official":null}},{"url":"/paper/beyond-the-imitation-game-quantifying-and","slug":"beyond-the-imitation-game-quantifying-and","title":"Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models","date":"2022-06-09","arxiv_id":"2206.04615","repositories_listed":6,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":3,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/beyond-the-imitation-game-quantifying-and#ran","syntology_url":"https://syntology.ai/paper/2206.04615","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2206.04615"}},"official":{"repos":["google/BIG-bench"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/grokking-generalization-beyond-overfitting-on","slug":"grokking-generalization-beyond-overfitting-on","title":"Grokking: Generalization Beyond Overfitting on Small Algorithmic Datasets","date":"2022-01-06","arxiv_id":"2201.02177","repositories_listed":6,"syntology":{"n":14,"n_ran":11,"n_constructed":0,"n_ran_checked":9,"n_instrument":2,"n_unverified":3,"n_honours":0,"n_violates":0,"n_no_contract":9,"n_pointer_only":0,"phrase":"11 ran (of which 0 constructed an object rather than computing a result; 9 with no instrument failure: 0 honoured, 0 violated, 9 with no contract checked; 2 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/grokking-generalization-beyond-overfitting-on#ran","syntology_url":"https://syntology.ai/paper/2201.02177","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2201.02177"}},"official":{"repos":["openai/grok"],"state":"official (archive's flag): 8 ran","n_ran":8,"n_constructed":0,"n_ran_no_instrument_failure":8,"n_unverified":1,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/generalization-through-memorization-nearest","slug":"generalization-through-memorization-nearest","title":"Generalization through Memorization: Nearest Neighbor Language Models","date":"2019-11-01","arxiv_id":"1911.00172","repositories_listed":5,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":0,"n_instrument":3,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 3 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/generalization-through-memorization-nearest#ran","syntology_url":"https://syntology.ai/paper/1911.00172","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1911.00172"}},"official":{"repos":["urvashik/knnlm"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/co-teaching-robust-training-of-deep-neural","slug":"co-teaching-robust-training-of-deep-neural","title":"Co-teaching: Robust Training of Deep Neural Networks with Extremely Noisy Labels","date":"2018-04-18","arxiv_id":"1804.06872","repositories_listed":5,"syntology":{"n":7,"n_ran":7,"n_constructed":0,"n_ran_checked":1,"n_instrument":6,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":7,"phrase":"7 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 6 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/co-teaching-robust-training-of-deep-neural#ran","syntology_url":"https://syntology.ai/paper/1804.06872","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1804.06872"}},"official":{"repos":["bhanML/Co-teaching"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/exposing-flaws-of-generative-model-evaluation-1","slug":"exposing-flaws-of-generative-model-evaluation-1","title":"Exposing flaws of generative model evaluation metrics and their unfair treatment of diffusion models","date":"2023-06-07","arxiv_id":"2306.04675","repositories_listed":4,"syntology":{"n":63,"n_ran":53,"n_constructed":0,"n_ran_checked":30,"n_instrument":23,"n_unverified":10,"n_honours":6,"n_violates":1,"n_no_contract":23,"n_pointer_only":12,"phrase":"53 ran (of which 0 constructed an object rather than computing a result; 30 with no instrument failure: 6 honoured, 1 violated, 23 with no contract checked; 23 where Syntology's instrument failed) · 10 unverified","sample_list":"/paper/exposing-flaws-of-generative-model-evaluation-1#ran","syntology_url":"https://syntology.ai/paper/2306.04675","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2306.04675"}},"official":{"repos":["layer6ai-labs/dgm-eval"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":1,"ran_from_kinds":["found_in_text","official","unlocated"]}}},{"url":"/paper/pythia-a-suite-for-analyzing-large-language","slug":"pythia-a-suite-for-analyzing-large-language","title":"Pythia: A Suite for Analyzing Large Language Models Across Training and Scaling","date":"2023-04-03","arxiv_id":"2304.01373","repositories_listed":4,"syntology":null},{"url":"/paper/learning-with-noisy-labels-revisited-a-study-1","slug":"learning-with-noisy-labels-revisited-a-study-1","title":"Learning with Noisy Labels Revisited: A Study Using Real-World Human Annotations","date":"2021-10-22","arxiv_id":"2110.12088","repositories_listed":4,"syntology":null},{"url":"/paper/learning-explanations-that-are-hard-to-vary","slug":"learning-explanations-that-are-hard-to-vary","title":"Learning explanations that are hard to vary","date":"2020-09-01","arxiv_id":"2009.00329","repositories_listed":4,"syntology":{"n":5,"n_ran":4,"n_constructed":0,"n_ran_checked":4,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":4,"n_pointer_only":0,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 4 with no instrument failure: 0 honoured, 0 violated, 4 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/learning-explanations-that-are-hard-to-vary#ran","syntology_url":"https://syntology.ai/paper/2009.00329","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2009.00329"}},"official":{"repos":["gibipara92/learning-explanations-hard-to-vary","pytorch/ignite"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":4,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/das3h-modeling-student-learning-and","slug":"das3h-modeling-student-learning-and","title":"DAS3H: Modeling Student Learning and Forgetting for Optimally Scheduling Distributed Practice of Skills","date":"2019-05-14","arxiv_id":"1905.06873","repositories_listed":4,"syntology":{"n":22,"n_ran":15,"n_constructed":0,"n_ran_checked":13,"n_instrument":2,"n_unverified":7,"n_honours":1,"n_violates":2,"n_no_contract":10,"n_pointer_only":10,"phrase":"15 ran (of which 0 constructed an object rather than computing a result; 13 with no instrument failure: 1 honoured, 2 violated, 10 with no contract checked; 2 where Syntology's instrument failed) · 7 unverified","sample_list":"/paper/das3h-modeling-student-learning-and#ran","syntology_url":"https://syntology.ai/paper/1905.06873","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1905.06873"}},"official":{"repos":["BenoitChoffin/das3h"],"state":"official: harvested, nothing ran","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":3,"ran_from_kinds":["listed"]}}},{"url":"/paper/r1-searcher-incentivizing-the-dynamic","slug":"r1-searcher-incentivizing-the-dynamic","title":"R1-Searcher++: Incentivizing the Dynamic Knowledge Acquisition of LLMs via Reinforcement Learning","date":"2025-05-22","arxiv_id":"2505.17005","repositories_listed":3,"syntology":{"n":10,"n_ran":9,"n_constructed":0,"n_ran_checked":8,"n_instrument":1,"n_unverified":1,"n_honours":1,"n_violates":0,"n_no_contract":7,"n_pointer_only":0,"phrase":"9 ran (of which 0 constructed an object rather than computing a result; 8 with no instrument failure: 1 honoured, 0 violated, 7 with no contract checked; 1 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/r1-searcher-incentivizing-the-dynamic#ran","syntology_url":"https://syntology.ai/paper/2505.17005","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2505.17005"}},"official":{"repos":["rucaibox/r1-searcher-plus"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/limo-less-is-more-for-reasoning","slug":"limo-less-is-more-for-reasoning","title":"LIMO: Less is More for Reasoning","date":"2025-02-05","arxiv_id":"2502.03387","repositories_listed":3,"syntology":null},{"url":"/paper/muse-machine-unlearning-six-way-evaluation","slug":"muse-machine-unlearning-six-way-evaluation","title":"MUSE: Machine Unlearning Six-Way Evaluation for Language Models","date":"2024-07-08","arxiv_id":"2407.06460","repositories_listed":3,"syntology":null},{"url":"/paper/evaluating-inexact-unlearning-requires","slug":"evaluating-inexact-unlearning-requires","title":"Towards Adversarial Evaluations for Inexact Machine Unlearning","date":"2022-01-17","arxiv_id":"2201.06640","repositories_listed":3,"syntology":null},{"url":"/paper/scaling-language-models-methods-analysis-1","slug":"scaling-language-models-methods-analysis-1","title":"Scaling Language Models: Methods, Analysis & Insights from Training Gopher","date":"2021-12-08","arxiv_id":"2112.11446","repositories_listed":3,"syntology":null},{"url":"/paper/consensual-collaborative-training-and","slug":"consensual-collaborative-training-and","title":"Consensual Collaborative Training And Knowledge Distillation Based Facial Expression Recognition Under Noisy Annotations","date":"2021-07-10","arxiv_id":"2107.04746","repositories_listed":3,"syntology":null},{"url":"/paper/searching-to-exploit-memorization-effect-in","slug":"searching-to-exploit-memorization-effect-in","title":"Searching to Exploit Memorization Effect in Learning from Corrupted Labels","date":"2019-11-06","arxiv_id":"1911.02377","repositories_listed":3,"syntology":{"n":1,"n_ran":0,"n_constructed":0,"n_ran_checked":0,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":1,"phrase":"0 ran · 1 unverified","sample_list":"/paper/searching-to-exploit-memorization-effect-in#ran","syntology_url":"https://syntology.ai/paper/1911.02377","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1911.02377"}},"official":null}},{"url":"/paper/how-does-disagreement-help-generalization","slug":"how-does-disagreement-help-generalization","title":"How does Disagreement Help Generalization against Label Corruption?","date":"2019-01-14","arxiv_id":"1901.04215","repositories_listed":3,"syntology":null},{"url":"/paper/associative-long-short-term-memory","slug":"associative-long-short-term-memory","title":"Associative Long Short-Term Memory","date":"2016-02-09","arxiv_id":"1602.03032","repositories_listed":3,"syntology":null},{"url":"/paper/llm-srbench-a-new-benchmark-for-scientific","slug":"llm-srbench-a-new-benchmark-for-scientific","title":"LLM-SRBench: A New Benchmark for Scientific Equation Discovery with Large Language Models","date":"2025-04-14","arxiv_id":"2504.10415","repositories_listed":2,"syntology":{"n":4,"n_ran":4,"n_constructed":0,"n_ran_checked":0,"n_instrument":4,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":4,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 4 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/llm-srbench-a-new-benchmark-for-scientific#ran","syntology_url":"https://syntology.ai/paper/2504.10415","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2504.10415"}},"official":{"repos":["deep-symbolic-mathematics/llm-sr","deep-symbolic-mathematics/llm-srbench"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/videochat-flash-hierarchical-compression-for","slug":"videochat-flash-hierarchical-compression-for","title":"VideoChat-Flash: Hierarchical Compression for Long-Context Video Modeling","date":"2024-12-31","arxiv_id":"2501.00574","repositories_listed":2,"syntology":null},{"url":"/paper/dash-warm-starting-neural-network-training-in","slug":"dash-warm-starting-neural-network-training-in","title":"DASH: Warm-Starting Neural Network Training in Stationary Settings without Loss of Plasticity","date":"2024-10-30","arxiv_id":"2410.23495","repositories_listed":2,"syntology":{"n":14,"n_ran":11,"n_constructed":0,"n_ran_checked":6,"n_instrument":5,"n_unverified":3,"n_honours":2,"n_violates":1,"n_no_contract":3,"n_pointer_only":0,"phrase":"11 ran (of which 0 constructed an object rather than computing a result; 6 with no instrument failure: 2 honoured, 1 violated, 3 with no contract checked; 5 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/dash-warm-starting-neural-network-training-in#ran","syntology_url":"https://syntology.ai/paper/2410.23495","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2410.23495"}},"official":{"repos":["NAVER-INTEL-Co-Lab/gaudi-dash","baekrok/DASH-Direction-Aware-SHrinking"],"state":"official (archive's flag): 11 ran","n_ran":11,"n_constructed":0,"n_ran_no_instrument_failure":6,"n_unverified":3,"ran_from_kinds":["official"]}}},{"url":"/paper/how-do-large-language-models-acquire-factual","slug":"how-do-large-language-models-acquire-factual","title":"How Do Large Language Models Acquire Factual Knowledge During Pretraining?","date":"2024-06-17","arxiv_id":"2406.11813","repositories_listed":2,"syntology":{"n":7,"n_ran":7,"n_constructed":0,"n_ran_checked":1,"n_instrument":6,"n_unverified":0,"n_honours":0,"n_violates":1,"n_no_contract":0,"n_pointer_only":3,"phrase":"7 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 1 violated, 0 with no contract checked; 6 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/how-do-large-language-models-acquire-factual#ran","syntology_url":"https://syntology.ai/paper/2406.11813","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.11813"}},"official":{"repos":["kaistai/factual-knowledge-acquisition"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/disentangled-continual-learning-separating","slug":"disentangled-continual-learning-separating","title":"Infinite dSprites for Disentangled Continual Learning: Separating Memory Edits from Generalization","date":"2023-12-27","arxiv_id":"2312.16731","repositories_listed":2,"syntology":{"n":6,"n_ran":5,"n_constructed":0,"n_ran_checked":5,"n_instrument":0,"n_unverified":1,"n_honours":2,"n_violates":0,"n_no_contract":3,"n_pointer_only":0,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 2 honoured, 0 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/disentangled-continual-learning-separating#ran","syntology_url":"https://syntology.ai/paper/2312.16731","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2312.16731"}},"official":{"repos":["sbdzdz/disco","sbdzdz/idsprites"],"state":"official (archive's flag): 5 ran","n_ran":5,"n_constructed":0,"n_ran_no_instrument_failure":5,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/negative-pre-aware-for-noisy-cross-modal","slug":"negative-pre-aware-for-noisy-cross-modal","title":"Negative Pre-aware for Noisy Cross-modal Matching","date":"2023-12-10","arxiv_id":"2312.05777","repositories_listed":2,"syntology":null},{"url":"/paper/data-contamination-quiz-a-tool-to-detect-and","slug":"data-contamination-quiz-a-tool-to-detect-and","title":"Data Contamination Quiz: A Tool to Detect and Estimate Contamination in Large Language Models","date":"2023-11-10","arxiv_id":"2311.06233","repositories_listed":2,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":3,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/data-contamination-quiz-a-tool-to-detect-and#ran","syntology_url":"https://syntology.ai/paper/2311.06233","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2311.06233"}},"official":{"repos":["shahriargolchin/dcq"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/practical-membership-inference-attacks","slug":"practical-membership-inference-attacks","title":"Practical Membership Inference Attacks against Fine-tuned Large Language Models via Self-prompt Calibration","date":"2023-11-10","arxiv_id":"2311.06062","repositories_listed":2,"syntology":{"n":6,"n_ran":5,"n_constructed":0,"n_ran_checked":4,"n_instrument":1,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":4,"n_pointer_only":6,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 4 with no instrument failure: 0 honoured, 0 violated, 4 with no contract checked; 1 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/practical-membership-inference-attacks#ran","syntology_url":"https://syntology.ai/paper/2311.06062","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2311.06062"}},"official":{"repos":["tsinghua-fib-lab/neurips2024_spv-mia","wjfu99/MIA-LLMs"],"state":"official (archive's flag): 5 ran","n_ran":5,"n_constructed":0,"n_ran_no_instrument_failure":4,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/cousins-of-the-vendi-score-a-family-of","slug":"cousins-of-the-vendi-score-a-family-of","title":"Cousins Of The Vendi Score: A Family Of Similarity-Based Diversity Metrics For Science And Machine Learning","date":"2023-10-19","arxiv_id":"2310.12952","repositories_listed":2,"syntology":{"n":16,"n_ran":15,"n_constructed":0,"n_ran_checked":11,"n_instrument":4,"n_unverified":1,"n_honours":2,"n_violates":0,"n_no_contract":9,"n_pointer_only":6,"phrase":"15 ran (of which 0 constructed an object rather than computing a result; 11 with no instrument failure: 2 honoured, 0 violated, 9 with no contract checked; 4 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/cousins-of-the-vendi-score-a-family-of#ran","syntology_url":"https://syntology.ai/paper/2310.12952","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2310.12952"}},"official":{"repos":["vertaix/vendi-sampling","vertaix/vendi-score"],"state":"official (archive's flag): 15 ran","n_ran":15,"n_constructed":0,"n_ran_no_instrument_failure":11,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/beyond-memorization-violating-privacy-via","slug":"beyond-memorization-violating-privacy-via","title":"Beyond Memorization: Violating Privacy Via Inference with Large Language Models","date":"2023-10-11","arxiv_id":"2310.07298","repositories_listed":2,"syntology":{"n":13,"n_ran":13,"n_constructed":0,"n_ran_checked":11,"n_instrument":2,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":10,"n_pointer_only":2,"phrase":"13 ran (of which 0 constructed an object rather than computing a result; 11 with no instrument failure: 1 honoured, 0 violated, 10 with no contract checked; 2 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/beyond-memorization-violating-privacy-via#ran","syntology_url":"https://syntology.ai/paper/2310.07298","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2310.07298"}},"official":{"repos":["eth-sri/llmprivacy"],"state":"official (archive's flag): 11 ran","n_ran":11,"n_constructed":0,"n_ran_no_instrument_failure":10,"n_unverified":0,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/on-memorization-in-diffusion-models","slug":"on-memorization-in-diffusion-models","title":"On Memorization in Diffusion Models","date":"2023-10-04","arxiv_id":"2310.02664","repositories_listed":2,"syntology":{"n":14,"n_ran":10,"n_constructed":0,"n_ran_checked":5,"n_instrument":5,"n_unverified":4,"n_honours":3,"n_violates":0,"n_no_contract":2,"n_pointer_only":5,"phrase":"10 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 3 honoured, 0 violated, 2 with no contract checked; 5 where Syntology's instrument failed) · 4 unverified","sample_list":"/paper/on-memorization-in-diffusion-models#ran","syntology_url":"https://syntology.ai/paper/2310.02664","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2310.02664"}},"official":{"repos":["sail-sg/DiffMemorize"],"state":"official (archive's flag): 10 ran","n_ran":10,"n_constructed":0,"n_ran_no_instrument_failure":5,"n_unverified":4,"ran_from_kinds":["official"]}}},{"url":"/paper/leave-one-out-distinguishability-in-machine","slug":"leave-one-out-distinguishability-in-machine","title":"Leave-one-out Distinguishability in Machine Learning","date":"2023-09-29","arxiv_id":"2309.17310","repositories_listed":2,"syntology":{"n":16,"n_ran":11,"n_constructed":1,"n_ran_checked":7,"n_instrument":4,"n_unverified":5,"n_honours":3,"n_violates":3,"n_no_contract":1,"n_pointer_only":0,"phrase":"11 ran (of which 1 constructed an object rather than computing a result; 7 with no instrument failure: 3 honoured, 3 violated, 1 with no contract checked; 4 where Syntology's instrument failed) · 5 unverified","sample_list":"/paper/leave-one-out-distinguishability-in-machine#ran","syntology_url":"https://syntology.ai/paper/2309.17310","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2309.17310"}},"official":{"repos":["privacytrustlab/lood_in_ml"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/in-context-autoencoder-for-context","slug":"in-context-autoencoder-for-context","title":"In-context Autoencoder for Context Compression in a Large Language Model","date":"2023-07-13","arxiv_id":"2307.06945","repositories_listed":2,"syntology":{"n":6,"n_ran":6,"n_constructed":0,"n_ran_checked":3,"n_instrument":3,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":0,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 3 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/in-context-autoencoder-for-context#ran","syntology_url":"https://syntology.ai/paper/2307.06945","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2307.06945"}},"official":{"repos":["getao/icae"],"state":"official (archive's flag): 6 ran","n_ran":6,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/emergent-and-predictable-memorization-in-1","slug":"emergent-and-predictable-memorization-in-1","title":"Emergent and Predictable Memorization in Large Language Models","date":"2023-04-21","arxiv_id":"2304.11158","repositories_listed":2,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/emergent-and-predictable-memorization-in-1#ran","syntology_url":"https://syntology.ai/paper/2304.11158","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2304.11158"}},"official":{"repos":["eleutherai/gpt-neox","eleutherai/pythia"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/the-ab-use-of-open-source-code-to-train-large","slug":"the-ab-use-of-open-source-code-to-train-large","title":"The (ab)use of Open Source Code to Train Large Language Models","date":"2023-02-27","arxiv_id":"2302.13681","repositories_listed":2,"syntology":null},{"url":"/paper/ds-1000-a-natural-and-reliable-benchmark-for","slug":"ds-1000-a-natural-and-reliable-benchmark-for","title":"DS-1000: A Natural and Reliable Benchmark for Data Science Code Generation","date":"2022-11-18","arxiv_id":"2211.11501","repositories_listed":2,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":0,"n_instrument":2,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":2,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/ds-1000-a-natural-and-reliable-benchmark-for#ran","syntology_url":"https://syntology.ai/paper/2211.11501","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2211.11501"}},"official":{"repos":["HKUNLP/DS-1000"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/decoupling-knowledge-from-memorization","slug":"decoupling-knowledge-from-memorization","title":"Decoupling Knowledge from Memorization: Retrieval-augmented Prompt Learning","date":"2022-05-29","arxiv_id":"2205.14704","repositories_listed":2,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/decoupling-knowledge-from-memorization#ran","syntology_url":"https://syntology.ai/paper/2205.14704","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2205.14704"}},"official":{"repos":["zjunlp/promptkg"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/self-guided-learning-to-denoise-for-robust","slug":"self-guided-learning-to-denoise-for-robust","title":"Self-Guided Learning to Denoise for Robust Recommendation","date":"2022-04-14","arxiv_id":"2204.06832","repositories_listed":2,"syntology":null},{"url":"/paper/quantifying-memorization-across-neural","slug":"quantifying-memorization-across-neural","title":"Quantifying Memorization Across Neural Language Models","date":"2022-02-15","arxiv_id":"2202.07646","repositories_listed":2,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/quantifying-memorization-across-neural#ran","syntology_url":"https://syntology.ai/paper/2202.07646","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2202.07646"}},"official":null}},{"url":"/paper/reconstructing-training-data-with-informed","slug":"reconstructing-training-data-with-informed","title":"Reconstructing Training Data with Informed Adversaries","date":"2022-01-13","arxiv_id":"2201.04845","repositories_listed":2,"syntology":null},{"url":"/paper/quantifying-adaptability-in-pre-trained","slug":"quantifying-adaptability-in-pre-trained","title":"Quantifying Adaptability in Pre-trained Language Models with 500 Tasks","date":"2021-12-06","arxiv_id":"2112.03204","repositories_listed":2,"syntology":{"n":8,"n_ran":7,"n_constructed":0,"n_ran_checked":6,"n_instrument":1,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":6,"n_pointer_only":1,"phrase":"7 ran (of which 0 constructed an object rather than computing a result; 6 with no instrument failure: 0 honoured, 0 violated, 6 with no contract checked; 1 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/quantifying-adaptability-in-pre-trained#ran","syntology_url":"https://syntology.ai/paper/2112.03204","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2112.03204"}},"official":{"repos":["belindal/taskbench500","facebookresearch/task_bench"],"state":"official (archive's flag): 7 ran","n_ran":7,"n_constructed":0,"n_ran_no_instrument_failure":6,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/personalized-federated-learning-through-local","slug":"personalized-federated-learning-through-local","title":"Personalized Federated Learning through Local Memorization","date":"2021-11-17","arxiv_id":"2111.09360","repositories_listed":2,"syntology":{"n":2,"n_ran":0,"n_constructed":0,"n_ran_checked":0,"n_instrument":0,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":2,"phrase":"0 ran · 2 unverified","sample_list":"/paper/personalized-federated-learning-through-local#ran","syntology_url":"https://syntology.ai/paper/2111.09360","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2111.09360"}},"official":{"repos":["omarfoq/knn-per"],"state":"official: harvested, nothing ran","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":2,"ran_from_kinds":[]}}},{"url":"/paper/combining-recurrent-convolutional-and","slug":"combining-recurrent-convolutional-and","title":"Combining Recurrent, Convolutional, and Continuous-time Models with Linear State-Space Layers","date":"2021-10-26","arxiv_id":"2110.13985","repositories_listed":2,"syntology":{"n":53,"n_ran":19,"n_constructed":0,"n_ran_checked":5,"n_instrument":14,"n_unverified":34,"n_honours":2,"n_violates":0,"n_no_contract":3,"n_pointer_only":0,"phrase":"19 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 2 honoured, 0 violated, 3 with no contract checked; 14 where Syntology's instrument failed) · 34 unverified","sample_list":"/paper/combining-recurrent-convolutional-and#ran","syntology_url":"https://syntology.ai/paper/2110.13985","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2110.13985"}},"official":{"repos":["hazyresearch/state-spaces"],"state":"official (archive's flag): 5 ran","n_ran":5,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":16,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/adaptive-early-learning-correction-for","slug":"adaptive-early-learning-correction-for","title":"Adaptive Early-Learning Correction for Segmentation from Noisy Annotations","date":"2021-10-07","arxiv_id":"2110.03740","repositories_listed":2,"syntology":{"n":4,"n_ran":4,"n_constructed":0,"n_ran_checked":3,"n_instrument":1,"n_unverified":0,"n_honours":1,"n_violates":1,"n_no_contract":1,"n_pointer_only":3,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 1 honoured, 1 violated, 1 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/adaptive-early-learning-correction-for#ran","syntology_url":"https://syntology.ai/paper/2110.03740","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2110.03740"}},"official":{"repos":["kangningthu/adele"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/do-what-nature-did-to-us-evolving-plastic","slug":"do-what-nature-did-to-us-evolving-plastic","title":"Evolving Decomposed Plasticity Rules for Information-Bottlenecked Meta-Learning","date":"2021-09-08","arxiv_id":"2109.03554","repositories_listed":2,"syntology":null},{"url":"/paper/pointer-value-retrieval-a-new-benchmark-for","slug":"pointer-value-retrieval-a-new-benchmark-for","title":"Pointer Value Retrieval: A new benchmark for understanding the limits of neural network generalization","date":"2021-07-27","arxiv_id":"2107.12580","repositories_listed":2,"syntology":null},{"url":"/paper/memorization-and-generalization-in-neural","slug":"memorization-and-generalization-in-neural","title":"Memorization and Generalization in Neural Code Intelligence Models","date":"2021-06-16","arxiv_id":"2106.08704","repositories_listed":2,"syntology":null},{"url":"/paper/hyperspectral-image-super-resolution-with-1","slug":"hyperspectral-image-super-resolution-with-1","title":"Hyperspectral Image Super-Resolution with Spectral Mixup and Heterogeneous Datasets","date":"2021-01-19","arxiv_id":"2101.07589","repositories_listed":2,"syntology":null},{"url":"/paper/superloss-a-generic-loss-for-robust","slug":"superloss-a-generic-loss-for-robust","title":"SuperLoss: A Generic Loss for Robust Curriculum Learning","date":"2020-12-01","arxiv_id":null,"repositories_listed":2,"syntology":null},{"url":"/paper/noisy-concurrent-training-for-efficient","slug":"noisy-concurrent-training-for-efficient","title":"Noisy Concurrent Training for Efficient Learning under Label Noise","date":"2020-09-17","arxiv_id":"2009.08325","repositories_listed":2,"syntology":{"n":6,"n_ran":5,"n_constructed":0,"n_ran_checked":2,"n_instrument":3,"n_unverified":1,"n_honours":1,"n_violates":1,"n_no_contract":0,"n_pointer_only":0,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 1 honoured, 1 violated, 0 with no contract checked; 3 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/noisy-concurrent-training-for-efficient#ran","syntology_url":"https://syntology.ai/paper/2009.08325","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2009.08325"}},"official":{"repos":["NeurAI-Lab/NCT"],"state":"official (archive's flag): 5 ran","n_ran":5,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/jointly-non-sampling-learning-for-knowledge","slug":"jointly-non-sampling-learning-for-knowledge","title":"Jointly Non-Sampling Learning for Knowledge Graph Enhanced Recommendation","date":"2020-07-01","arxiv_id":null,"repositories_listed":2,"syntology":null},{"url":"/paper/early-learning-regularization-prevents","slug":"early-learning-regularization-prevents","title":"Early-Learning Regularization Prevents Memorization of Noisy Labels","date":"2020-06-30","arxiv_id":"2007.00151","repositories_listed":2,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":1,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 1 violated, 0 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/early-learning-regularization-prevents#ran","syntology_url":"https://syntology.ai/paper/2007.00151","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2007.00151"}},"official":{"repos":["shengliu66/ELR"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/eicient-non-sampling-factorization-machines","slug":"eicient-non-sampling-factorization-machines","title":"Eicient Non-Sampling Factorization Machines for Optimal Context-Aware Recommendation","date":"2020-05-19","arxiv_id":null,"repositories_listed":2,"syntology":null},{"url":"/paper/the-natural-auditor-how-to-tell-if-someone","slug":"the-natural-auditor-how-to-tell-if-someone","title":"Auditing Data Provenance in Text-Generation Models","date":"2018-11-01","arxiv_id":"1811.00513","repositories_listed":2,"syntology":null},{"url":"/paper/split-and-rephrase-better-evaluation-and-a","slug":"split-and-rephrase-better-evaluation-and-a","title":"Split and Rephrase: Better Evaluation and a Stronger Baseline","date":"2018-05-02","arxiv_id":"1805.01035","repositories_listed":2,"syntology":null},{"url":"/paper/bidirectional-deep-readout-echo-state","slug":"bidirectional-deep-readout-echo-state","title":"Bidirectional deep-readout echo state networks","date":"2017-11-17","arxiv_id":"1711.06509","repositories_listed":2,"syntology":null},{"url":"/paper/rotational-unit-of-memory","slug":"rotational-unit-of-memory","title":"Rotational Unit of Memory","date":"2017-10-26","arxiv_id":"1710.09537","repositories_listed":2,"syntology":null},{"url":"/paper/a-closer-look-at-memorization-in-deep","slug":"a-closer-look-at-memorization-in-deep","title":"A Closer Look at Memorization in Deep Networks","date":"2017-06-16","arxiv_id":"1706.05394","repositories_listed":2,"syntology":null},{"url":"/paper/reasoning-or-memorization-unreliable-results","slug":"reasoning-or-memorization-unreliable-results","title":"Reasoning or Memorization? Unreliable Results of Reinforcement Learning Due to Data Contamination","date":"2025-07-14","arxiv_id":"2507.10532","repositories_listed":1,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":1,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/reasoning-or-memorization-unreliable-results#ran","syntology_url":"https://syntology.ai/paper/2507.10532","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2507.10532"}},"official":{"repos":["wumingqi/LLM-Math-Evaluation"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/capacity-matters-a-proof-of-concept-for","slug":"capacity-matters-a-proof-of-concept-for","title":"Capacity Matters: a Proof-of-Concept for Transformer Memorization on Real-World Data","date":"2025-06-17","arxiv_id":"2506.14704","repositories_listed":1,"syntology":null},{"url":"/paper/dataset-distillation-for-memorized-data-soft","slug":"dataset-distillation-for-memorized-data-soft","title":"Dataset distillation for memorized data: Soft labels can leak held-out teacher knowledge","date":"2025-06-17","arxiv_id":"2506.14457","repositories_listed":1,"syntology":{"n":7,"n_ran":7,"n_constructed":0,"n_ran_checked":2,"n_instrument":5,"n_unverified":0,"n_honours":2,"n_violates":0,"n_no_contract":0,"n_pointer_only":7,"phrase":"7 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 2 honoured, 0 violated, 0 with no contract checked; 5 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/dataset-distillation-for-memorized-data-soft#ran","syntology_url":"https://syntology.ai/paper/2506.14457","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2506.14457"}},"official":{"repos":["spoc-group/dataset-distillation-memorization"],"state":"official (archive's flag): 7 ran","n_ran":7,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/leximark-robust-watermarking-via-lexical","slug":"leximark-robust-watermarking-via-lexical","title":"LexiMark: Robust Watermarking via Lexical Substitutions to Enhance Membership Verification of an LLM's Textual Training Data","date":"2025-06-17","arxiv_id":"2506.14474","repositories_listed":1,"syntology":null},{"url":"/paper/2506-10024","slug":"2506-10024","title":"Private Memorization Editing: Turning Memorization into a Defense to Strengthen Data Privacy in Large Language Models","date":"2025-06-09","arxiv_id":"2506.10024","repositories_listed":1,"syntology":null},{"url":"/paper/diffusion-models-under-low-noise-regime","slug":"diffusion-models-under-low-noise-regime","title":"Diffusion models under low-noise regime","date":"2025-06-09","arxiv_id":"2506.07841","repositories_listed":1,"syntology":{"n":9,"n_ran":9,"n_constructed":0,"n_ran_checked":5,"n_instrument":4,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":5,"n_pointer_only":4,"phrase":"9 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 0 honoured, 0 violated, 5 with no contract checked; 4 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/diffusion-models-under-low-noise-regime#ran","syntology_url":"https://syntology.ai/paper/2506.07841","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2506.07841"}},"official":{"repos":["lizardp1/diffusion_low_noise_regime"],"state":"official (archive's flag): 9 ran","n_ran":9,"n_constructed":0,"n_ran_no_instrument_failure":5,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/generative-modeling-of-weights-generalization","slug":"generative-modeling-of-weights-generalization","title":"Generative Modeling of Weights: Generalization or Memorization?","date":"2025-06-09","arxiv_id":"2506.07998","repositories_listed":1,"syntology":null},{"url":"/paper/heurigym-an-agentic-benchmark-for-llm-crafted","slug":"heurigym-an-agentic-benchmark-for-llm-crafted","title":"HeuriGym: An Agentic Benchmark for LLM-Crafted Heuristics in Combinatorial Optimization","date":"2025-06-09","arxiv_id":"2506.07972","repositories_listed":1,"syntology":{"n":12,"n_ran":7,"n_constructed":0,"n_ran_checked":7,"n_instrument":0,"n_unverified":5,"n_honours":0,"n_violates":0,"n_no_contract":7,"n_pointer_only":0,"phrase":"7 ran (of which 0 constructed an object rather than computing a result; 7 with no instrument failure: 0 honoured, 0 violated, 7 with no contract checked; 0 where Syntology's instrument failed) · 5 unverified","sample_list":"/paper/heurigym-an-agentic-benchmark-for-llm-crafted#ran","syntology_url":"https://syntology.ai/paper/2506.07972","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2506.07972"}},"official":{"repos":["cornell-zhang/heurigym"],"state":"official (archive's flag): 7 ran","n_ran":7,"n_constructed":0,"n_ran_no_instrument_failure":7,"n_unverified":5,"ran_from_kinds":["official"]}}},{"url":"/paper/beyond-memorization-a-rigorous-evaluation","slug":"beyond-memorization-a-rigorous-evaluation","title":"Beyond Memorization: A Rigorous Evaluation Framework for Medical Knowledge Editing","date":"2025-06-04","arxiv_id":"2506.03490","repositories_listed":1,"syntology":null},{"url":"/paper/how-does-transformer-learn-implicit-reasoning","slug":"how-does-transformer-learn-implicit-reasoning","title":"How does Transformer Learn Implicit Reasoning?","date":"2025-05-29","arxiv_id":"2505.23653","repositories_listed":1,"syntology":null},{"url":"/paper/matharena-evaluating-llms-on-uncontaminated","slug":"matharena-evaluating-llms-on-uncontaminated","title":"MathArena: Evaluating LLMs on Uncontaminated Math Competitions","date":"2025-05-29","arxiv_id":"2505.23281","repositories_listed":1,"syntology":{"n":19,"n_ran":15,"n_constructed":0,"n_ran_checked":13,"n_instrument":2,"n_unverified":4,"n_honours":0,"n_violates":0,"n_no_contract":13,"n_pointer_only":0,"phrase":"15 ran (of which 0 constructed an object rather than computing a result; 13 with no instrument failure: 0 honoured, 0 violated, 13 with no contract checked; 2 where Syntology's instrument failed) · 4 unverified","sample_list":"/paper/matharena-evaluating-llms-on-uncontaminated#ran","syntology_url":"https://syntology.ai/paper/2505.23281","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2505.23281"}},"official":{"repos":["eth-sri/matharena"],"state":"official (archive's flag): 15 ran","n_ran":15,"n_constructed":0,"n_ran_no_instrument_failure":13,"n_unverified":4,"ran_from_kinds":["official"]}}},{"url":"/paper/benchmarking-abstract-and-reasoning-abilities","slug":"benchmarking-abstract-and-reasoning-abilities","title":"Benchmarking Abstract and Reasoning Abilities Through A Theoretical Perspective","date":"2025-05-28","arxiv_id":"2505.23833","repositories_listed":1,"syntology":null},{"url":"/paper/owl-probing-cross-lingual-recall-of-memorized","slug":"owl-probing-cross-lingual-recall-of-memorized","title":"OWL: Probing Cross-Lingual Recall of Memorized Texts via World Literature","date":"2025-05-28","arxiv_id":"2505.22945","repositories_listed":1,"syntology":null},{"url":"/paper/emergent-llm-behaviors-are-observationally","slug":"emergent-llm-behaviors-are-observationally","title":"Emergent LLM behaviors are observationally equivalent to data leakage","date":"2025-05-26","arxiv_id":"2505.23796","repositories_listed":1,"syntology":null},{"url":"/paper/grokking-explaind-unifying-model-data-and","slug":"grokking-explaind-unifying-model-data-and","title":"Grokking ExPLAIND: Unifying Model, Data, and Training Attribution to Study Model Behavior","date":"2025-05-26","arxiv_id":"2505.20076","repositories_listed":1,"syntology":{"n":5,"n_ran":3,"n_constructed":3,"n_ran_checked":3,"n_instrument":0,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":0,"phrase":"3 ran (of which 3 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 2 unverified; every one of the 3 samples that ran constructed an object rather than computing a result","sample_list":"/paper/grokking-explaind-unifying-model-data-and#ran","syntology_url":"https://syntology.ai/paper/2505.20076","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2505.20076"}},"official":{"repos":["mainlp/explaind"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":3,"n_ran_no_instrument_failure":3,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/querying-kernel-methods-suffices-for","slug":"querying-kernel-methods-suffices-for","title":"Querying Kernel Methods Suffices for Reconstructing their Training Data","date":"2025-05-25","arxiv_id":"2505.19019","repositories_listed":1,"syntology":null},{"url":"/paper/sudoku-bench-evaluating-creative-reasoning","slug":"sudoku-bench-evaluating-creative-reasoning","title":"Sudoku-Bench: Evaluating creative reasoning with Sudoku variants","date":"2025-05-22","arxiv_id":"2505.16135","repositories_listed":1,"syntology":null},{"url":"/paper/pre-training-large-memory-language-models","slug":"pre-training-large-memory-language-models","title":"Pre-training Large Memory Language Models with Internal and External Knowledge","date":"2025-05-21","arxiv_id":"2505.15962","repositories_listed":1,"syntology":{"n":8,"n_ran":7,"n_constructed":0,"n_ran_checked":7,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":7,"n_pointer_only":0,"phrase":"7 ran (of which 0 constructed an object rather than computing a result; 7 with no instrument failure: 0 honoured, 0 violated, 7 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/pre-training-large-memory-language-models#ran","syntology_url":"https://syntology.ai/paper/2505.15962","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2505.15962"}},"official":{"repos":["kilian-group/lmlm"],"state":"official (archive's flag): 7 ran","n_ran":7,"n_constructed":0,"n_ran_no_instrument_failure":7,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/causal-cartographer-from-mapping-to-reasoning","slug":"causal-cartographer-from-mapping-to-reasoning","title":"Causal Cartographer: From Mapping to Reasoning Over Counterfactual Worlds","date":"2025-05-20","arxiv_id":"2505.14396","repositories_listed":1,"syntology":null},{"url":"/paper/fragments-to-facts-partial-information","slug":"fragments-to-facts-partial-information","title":"Fragments to Facts: Partial-Information Fragment Inference from LLMs","date":"2025-05-20","arxiv_id":"2505.13819","repositories_listed":1,"syntology":null},{"url":"/paper/bidirectional-lms-are-better-knowledge","slug":"bidirectional-lms-are-better-knowledge","title":"Bidirectional LMs are Better Knowledge Memorizers? A Benchmark for Real-world Knowledge Injection","date":"2025-05-18","arxiv_id":"2505.12306","repositories_listed":1,"syntology":null},{"url":"/paper/panorama-a-synthetic-pii-laced-dataset-for","slug":"panorama-a-synthetic-pii-laced-dataset-for","title":"PANORAMA: A synthetic PII-laced dataset for studying sensitive data memorization in LLMs","date":"2025-05-18","arxiv_id":"2505.12238","repositories_listed":1,"syntology":null},{"url":"/paper/teach2eval-an-indirect-evaluation-method-for","slug":"teach2eval-an-indirect-evaluation-method-for","title":"Teach2Eval: An Indirect Evaluation Method for LLM by Judging How It Teaches","date":"2025-05-18","arxiv_id":"2505.12259","repositories_listed":1,"syntology":null},{"url":"/paper/2505-11004","slug":"2505-11004","title":"Illusion or Algorithm? Investigating Memorization, Emergence, and Symbolic Processing in In-Context Learning","date":"2025-05-16","arxiv_id":"2505.11004","repositories_listed":1,"syntology":null},{"url":"/paper/do-llms-memorize-recommendation-datasets-a","slug":"do-llms-memorize-recommendation-datasets-a","title":"Do LLMs Memorize Recommendation Datasets? A Preliminary Study on MovieLens-1M","date":"2025-05-15","arxiv_id":"2505.10212","repositories_listed":1,"syntology":null},{"url":"/paper/identifying-memorization-of-diffusion-models","slug":"identifying-memorization-of-diffusion-models","title":"Identifying Memorization of Diffusion Models through p-Laplace Analysis","date":"2025-05-13","arxiv_id":"2505.08246","repositories_listed":1,"syntology":null},{"url":"/paper/identifying-legal-holdings-with-llms-a","slug":"identifying-legal-holdings-with-llms-a","title":"Identifying Legal Holdings with LLMs: A Systematic Study of Performance, Scale, and Memorization","date":"2025-05-04","arxiv_id":"2505.02172","repositories_listed":1,"syntology":null},{"url":"/paper/wide-deep-learning-for-node-classification","slug":"wide-deep-learning-for-node-classification","title":"Wide & Deep Learning for Node Classification","date":"2025-05-04","arxiv_id":"2505.02020","repositories_listed":1,"syntology":null},{"url":"/paper/memorization-and-knowledge-injection-in-gated","slug":"memorization-and-knowledge-injection-in-gated","title":"Memorization and Knowledge Injection in Gated LLMs","date":"2025-04-30","arxiv_id":"2504.21239","repositories_listed":1,"syntology":null},{"url":"/paper/a-mean-teacher-algorithm-for-unlearning-of","slug":"a-mean-teacher-algorithm-for-unlearning-of","title":"A mean teacher algorithm for unlearning of language models","date":"2025-04-18","arxiv_id":"2504.13388","repositories_listed":1,"syntology":null},{"url":"/paper/memory-modular-classification-learning-to","slug":"memory-modular-classification-learning-to","title":"Memory-Modular Classification: Learning to Generalize with Memory Replacement","date":"2025-04-08","arxiv_id":"2504.06021","repositories_listed":1,"syntology":null},{"url":"/paper/agitb-a-signal-level-benchmark-for-evaluating","slug":"agitb-a-signal-level-benchmark-for-evaluating","title":"AGITB: A Signal-Level Benchmark for Evaluating Artificial General Intelligence","date":"2025-04-06","arxiv_id":"2504.04430","repositories_listed":1,"syntology":null},{"url":"/paper/generative-evaluation-of-complex-reasoning-in","slug":"generative-evaluation-of-complex-reasoning-in","title":"Generative Evaluation of Complex Reasoning in Large Language Models","date":"2025-04-03","arxiv_id":"2504.02810","repositories_listed":1,"syntology":null},{"url":"/paper/cascade-your-datasets-for-cross-mode","slug":"cascade-your-datasets-for-cross-mode","title":"CASCADE Your Datasets for Cross-Mode Knowledge Retrieval of Language Models","date":"2025-04-02","arxiv_id":"2504.01450","repositories_listed":1,"syntology":null},{"url":"/paper/gmai-vl-r1-harnessing-reinforcement-learning","slug":"gmai-vl-r1-harnessing-reinforcement-learning","title":"GMAI-VL-R1: Harnessing Reinforcement Learning for Multimodal Medical Reasoning","date":"2025-04-02","arxiv_id":"2504.01886","repositories_listed":1,"syntology":null},{"url":"/paper/rare-retrieval-augmented-reasoning-modeling","slug":"rare-retrieval-augmented-reasoning-modeling","title":"RARE: Retrieval-Augmented Reasoning Modeling","date":"2025-03-30","arxiv_id":"2503.23513","repositories_listed":1,"syntology":{"n":14,"n_ran":13,"n_constructed":0,"n_ran_checked":13,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":13,"n_pointer_only":0,"phrase":"13 ran (of which 0 constructed an object rather than computing a result; 13 with no instrument failure: 0 honoured, 0 violated, 13 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/rare-retrieval-augmented-reasoning-modeling#ran","syntology_url":"https://syntology.ai/paper/2503.23513","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2503.23513"}},"official":{"repos":["open-dataflow/rare"],"state":"official (archive's flag): 13 ran","n_ran":13,"n_constructed":0,"n_ran_no_instrument_failure":13,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/lotus-large-scale-machine-unlearning-with-a","slug":"lotus-large-scale-machine-unlearning-with-a","title":"LoTUS: Large-Scale Machine Unlearning with a Taste of Uncertainty","date":"2025-03-24","arxiv_id":"2503.18314","repositories_listed":1,"syntology":null},{"url":"/paper/palate-peculiar-application-of-the-law-of","slug":"palate-peculiar-application-of-the-law-of","title":"PALATE: Peculiar Application of the Law of Total Expectation to Enhance the Evaluation of Deep Generative Models","date":"2025-03-24","arxiv_id":"2503.18462","repositories_listed":1,"syntology":null},{"url":"/paper/can-language-models-follow-multiple-turns-of","slug":"can-language-models-follow-multiple-turns-of","title":"Can Language Models Follow Multiple Turns of Entangled Instructions?","date":"2025-03-17","arxiv_id":"2503.13222","repositories_listed":1,"syntology":null},{"url":"/paper/pre-training-meta-rule-selection-policy-for","slug":"pre-training-meta-rule-selection-policy-for","title":"Pre-Training Meta-Rule Selection Policy for Visual Generative Abductive Learning","date":"2025-03-09","arxiv_id":"2503.06427","repositories_listed":1,"syntology":null}],"syntology_records":41,"syntology_note":"a paper without a record is not a recorded non-run: it may lack an arXiv id or simply be absent from the graph layer"},"key_notes":{"n_ran_checked":"legacy name, kept unchanged so existing readers do not break: it counts the samples that ran with no instrument failure (honoured, violated, and ran with no contract checked); it does not mean a contract was checked, and the pages print it as 'K with no instrument failure', not 'K checked'","n_constructed":"a sub-count of the samples that ran, never subtracted from them and never a failure: an executed sample whose run returned an instance of its own class (fixture_out_type equals the entry name): the run built an object and did not compute a result (Syntology's RAN record, counts.constructed)"},"since_archive_judged":{"papers":0,"papers_placed":0,"judged_by":{"model":"a model","models_as_recorded":[],"from":"lb_rows_run_verified.jsonl: each record's `model` field (the extractor's call that wrote decisions[])"},"note":"a model's judgement from the paper's own tables that they report on this leaderboard, not a result: no value from a paper that was not placed is printed or carried here; papers counts each paper once over these leaderboards","backfill":"8,886 of 18,637 newer papers checked so far (733 more had nothing to place; 41 could not be read)","backfill_complete":false,"leaderboards":[]},"papers_where_code_ran":177,"papers_where_code_ran_first_page":"/task/memorization/papers/ran/1","papers_where_code_ran_caption":"We ran code from the paper's repository; we did not run it on this task or check it against the task's benchmarks.","pages":[{"order":"archive","page":2,"of":11,"url":"/task/memorization/papers/2","json":"/task/memorization/papers/2.json","rows":100},{"order":"archive","page":3,"of":11,"url":"/task/memorization/papers/3","json":"/task/memorization/papers/3.json","rows":100},{"order":"archive","page":4,"of":11,"url":"/task/memorization/papers/4","json":"/task/memorization/papers/4.json","rows":100},{"order":"archive","page":5,"of":11,"url":"/task/memorization/papers/5","json":"/task/memorization/papers/5.json","rows":100},{"order":"archive","page":6,"of":11,"url":"/task/memorization/papers/6","json":"/task/memorization/papers/6.json","rows":100},{"order":"archive","page":7,"of":11,"url":"/task/memorization/papers/7","json":"/task/memorization/papers/7.json","rows":100},{"order":"archive","page":8,"of":11,"url":"/task/memorization/papers/8","json":"/task/memorization/papers/8.json","rows":100},{"order":"archive","page":9,"of":11,"url":"/task/memorization/papers/9","json":"/task/memorization/papers/9.json","rows":100},{"order":"archive","page":10,"of":11,"url":"/task/memorization/papers/10","json":"/task/memorization/papers/10.json","rows":100},{"order":"archive","page":11,"of":11,"url":"/task/memorization/papers/11","json":"/task/memorization/papers/11.json","rows":88},{"order":"ran","page":1,"of":2,"url":"/task/memorization/papers/ran/1","json":"/task/memorization/papers/ran/1.json","rows":100},{"order":"ran","page":2,"of":2,"url":"/task/memorization/papers/ran/2","json":"/task/memorization/papers/ran/2.json","rows":77}],"description_links":{"kept":0,"unwrapped_to_text":0,"bare_urls_linked":0,"relative_images_dropped":0,"rule":"internal links are kept only when the target slug exists in the catalog"},"syntology":{"read_at":"2026-10-05T00:36:11+00:00","claim":"Per-sample execution status on synthesized fixtures ('ran N of M samples'); not a correctness claim and not a ranking signal.","status_vocabulary":{"ran_honours":"ran, honoured the contract we drafted","ran_violates":"ran, violated the contract we drafted","ran_draft_wrong":"ran; our contract draft was wrong, not the code","ran_fixture":"ran; the contract check could not run (RAISES, DEP_MISSING or TIMEOUT)","ran":"ran on a synthesized input","unverified":"unverified (harvested, no recorded run)"},"row_route":"each syntology object on a benchmark's first row and on a paper carries sample_list (this site's paper page, #ran), syntology_url (syntology.ai's page for the paper, only when that route probed live) and mcp (the agent call that lists the paper's harvested samples; not a verdict on the paper)","developers":"https://syntology.ai/developers"},"not_shown":{"libraries":"the archive has no per-task library table","trend_sparklines":"the Trend column of the benchmarks table was a rendered image; it is not in the archive","social_and_latest_sorts":"stars and social signals are not in the archive"},"record_sha256":"7b4e2c3cc11351ec633ac0d98f259462ea4ec9f545128821e233192c7718ca16","record_changed_at":"2026-10-05","record_changed_at_basis":"changed"}