{"about":{"non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","site":"https://codewithpapers.app","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page","syntology":{"site":"https://syntology.ai","developers":"https://syntology.ai/developers","mcp":{"server":"https://syntology.ai/mcp","transport":"streamable-http","server_card":"https://syntology.ai/.well-known/mcp/server-card.json","auth":{"type":"trial token, no account","trial_token":"https://syntology.ai/api/oauth/trial/token","method":"POST","docs":"https://syntology.ai/developers"}},"have":"https://syntology.ai/api/graph/have?x=<method, arXiv id or title> (free, answers coverage only)","paper_base":"https://syntology.ai/paper/","atlas_base":"https://app.syntology.ai/?focus="},"machine_readable":[{"url":"https://codewithpapers.app/llms.txt","what":"the machine catalog: every machine-readable file, counted"},{"url":"https://codewithpapers.app/index/manifest.json","what":"paper-to-code index by arXiv id, with Syntology's counts"},{"url":"https://codewithpapers.app/search/manifest.json","what":"site search index (titles, authors) and its files"},{"url":"https://codewithpapers.app/download","what":"bulk files: Syntology's layer, described there"},{"url":"https://codewithpapers.app/build_manifest.json","what":"the build record: inputs, counts, exclusions, probes"}]},"url":"/task/math","name":"Math","slug":"math","description_markdown":null,"categories":[],"source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","slug_source":"archive_url"},"counts":{"papers_tagged":1596,"papers_with_code":765,"benchmarks":0,"benchmark_tables_in_archive":0,"benchmark_tables_shown":0,"benchmark_tables_withheld_as_spam":0,"benchmark_definition":"a leaderboard table with at least one row; benchmark_tables_shown also counts the zero-row tables; benchmark_tables_in_archive adds the tables withheld as spam","datasets":6,"subtasks":0,"parent_tasks":0},"benchmarks":[],"datasets":[{"url":"/dataset/gsm-plus","name":"GSM-Plus","full_name":"","num_papers_in_archive":17},{"url":"/dataset/mathbench","name":"MathBench","full_name":"MathBench: Evaluating the Theory and Application Proficiency of LLMs with a Hierarchical Mathematics Benchmark","num_papers_in_archive":16},{"url":"/dataset/dart-math-hard","name":"DART-Math-Hard","full_name":"","num_papers_in_archive":2},{"url":"/dataset/dart-math-uniform","name":"DART-Math-Uniform","full_name":"","num_papers_in_archive":1},{"url":"/dataset/mm-eval","name":"MM-Eval","full_name":"Modern Mongolian Evaluation","num_papers_in_archive":1},{"url":"/dataset/valimath","name":"ValiMath","full_name":"","num_papers_in_archive":1}],"subtasks":[],"parent_tasks":[],"papers":{"order":"repositories listed in the archive (most first), then date (newest first), then slug; the archive holds no stars","population":"every paper the archive tags with this task (distinct papers; spam titles not listed). 'of' counts the papers listed; until 2026-09-25 it counted only the papers with a code link, and that number is of_with_code","page":1,"pages_in_order":16,"rows_per_page":100,"shown":100,"of":1596,"of_with_code":765,"tagged_in_all":1596,"counts":{"archive_papers_tagged":1596,"with_a_code_link":765,"where_syntology_ran_a_sample":349,"not_listed_spam_title":0,"listed":1596,"listed_where_code_ran":349,"where_syntology_ran_a_sample_split":{"with_a_run_with_no_instrument_failure":277,"every_run_a_failure_of_syntologys_instrument":72,"listed_with_a_run_with_no_instrument_failure":277,"listed_every_run_a_failure_of_syntologys_instrument":72,"filter":{"states":["a run with no instrument failure","any run, instrument failures included"],"default":"a run with no instrument failure","note":"on the 'only where code ran' pages the default hides, in the browser, the rows where every run was a failure of Syntology's instrument; the second state shows them again. Rows are hidden, never re-ordered; these twins list every row"}},"definition":"distinct papers the archive tags; 'where Syntology ran a sample' counts papers with at least one harvested sample that ran, which is not a correctness claim"},"items":[{"url":"/paper/chain-of-thought-prompting-elicits-reasoning","slug":"chain-of-thought-prompting-elicits-reasoning","title":"Chain-of-Thought Prompting Elicits Reasoning in Large Language Models","date":"2022-01-28","arxiv_id":"2201.11903","repositories_listed":19,"syntology":{"n":7,"n_ran":5,"n_constructed":0,"n_ran_checked":4,"n_instrument":1,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":4,"n_pointer_only":1,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 4 with no instrument failure: 0 honoured, 0 violated, 4 with no contract checked; 1 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/chain-of-thought-prompting-elicits-reasoning#ran","syntology_url":"https://syntology.ai/paper/2201.11903","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2201.11903"}},"official":null}},{"url":"/paper/awq-activation-aware-weight-quantization-for","slug":"awq-activation-aware-weight-quantization-for","title":"AWQ: Activation-aware Weight Quantization for LLM Compression and Acceleration","date":"2023-06-01","arxiv_id":"2306.00978","repositories_listed":12,"syntology":{"n":18,"n_ran":16,"n_constructed":1,"n_ran_checked":6,"n_instrument":10,"n_unverified":2,"n_honours":1,"n_violates":0,"n_no_contract":5,"n_pointer_only":4,"phrase":"16 ran (of which 1 constructed an object rather than computing a result; 6 with no instrument failure: 1 honoured, 0 violated, 5 with no contract checked; 10 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/awq-activation-aware-weight-quantization-for#ran","syntology_url":"https://syntology.ai/paper/2306.00978","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2306.00978"}},"official":{"repos":["internlm/lmdeploy","mit-han-lab/llm-awq","nvidia/tensorrt-llm","vllm-project/vllm"],"state":"official (archive's flag): 9 ran","n_ran":9,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["found_in_text","listed","official"]}}},{"url":"/paper/gpt-4-technical-report-1","slug":"gpt-4-technical-report-1","title":"GPT-4 Technical Report","date":"2023-03-15","arxiv_id":"2303.08774","repositories_listed":11,"syntology":{"n":5,"n_ran":5,"n_constructed":0,"n_ran_checked":5,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":2,"n_no_contract":3,"n_pointer_only":1,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 0 honoured, 2 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/gpt-4-technical-report-1#ran","syntology_url":"https://syntology.ai/paper/2303.08774","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2303.08774"}},"official":{"repos":["openai/evals"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/translating-mathematical-formula-images-to","slug":"translating-mathematical-formula-images-to","title":"Translating Math Formula Images to LaTeX Sequences Using Deep Neural Networks with Sequence-level Training","date":"2019-08-29","arxiv_id":"1908.11415","repositories_listed":8,"syntology":null},{"url":"/paper/chatglm-a-family-of-large-language-models","slug":"chatglm-a-family-of-large-language-models","title":"ChatGLM: A Family of Large Language Models from GLM-130B to GLM-4 All Tools","date":"2024-06-18","arxiv_id":"2406.12793","repositories_listed":7,"syntology":{"n":29,"n_ran":21,"n_constructed":0,"n_ran_checked":15,"n_instrument":6,"n_unverified":8,"n_honours":1,"n_violates":0,"n_no_contract":14,"n_pointer_only":1,"phrase":"21 ran (of which 0 constructed an object rather than computing a result; 15 with no instrument failure: 1 honoured, 0 violated, 14 with no contract checked; 6 where Syntology's instrument failed) · 8 unverified","sample_list":"/paper/chatglm-a-family-of-large-language-models#ran","syntology_url":"https://syntology.ai/paper/2406.12793","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.12793"}},"official":{"repos":["thudm/chatglm-6b"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":1,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/palm-scaling-language-modeling-with-pathways-1","slug":"palm-scaling-language-modeling-with-pathways-1","title":"PaLM: Scaling Language Modeling with Pathways","date":"2022-04-05","arxiv_id":"2204.02311","repositories_listed":7,"syntology":{"n":37,"n_ran":32,"n_constructed":16,"n_ran_checked":24,"n_instrument":8,"n_unverified":5,"n_honours":2,"n_violates":1,"n_no_contract":21,"n_pointer_only":0,"phrase":"32 ran (of which 16 constructed an object rather than computing a result; 24 with no instrument failure: 2 honoured, 1 violated, 21 with no contract checked; 8 where Syntology's instrument failed) · 5 unverified","sample_list":"/paper/palm-scaling-language-modeling-with-pathways-1#ran","syntology_url":"https://syntology.ai/paper/2204.02311","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2204.02311"}},"official":null}},{"url":"/paper/qwen2-5-technical-report","slug":"qwen2-5-technical-report","title":"Qwen2.5 Technical Report","date":"2024-12-19","arxiv_id":"2412.15115","repositories_listed":6,"syntology":{"n":3,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/qwen2-5-technical-report#ran","syntology_url":"https://syntology.ai/paper/2412.15115","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2412.15115"}},"official":{"repos":["qwenlm/qwen2.5"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/mistral-7b","slug":"mistral-7b","title":"Mistral 7B","date":"2023-10-10","arxiv_id":"2310.06825","repositories_listed":6,"syntology":{"n":11,"n_ran":10,"n_constructed":0,"n_ran_checked":10,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":10,"n_pointer_only":2,"phrase":"10 ran (of which 0 constructed an object rather than computing a result; 10 with no instrument failure: 0 honoured, 0 violated, 10 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/mistral-7b#ran","syntology_url":"https://syntology.ai/paper/2310.06825","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2310.06825"}},"official":{"repos":["mistralai/mistral-src"],"state":"official (archive's flag): 9 ran","n_ran":9,"n_constructed":0,"n_ran_no_instrument_failure":9,"n_unverified":1,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/beyond-the-imitation-game-quantifying-and","slug":"beyond-the-imitation-game-quantifying-and","title":"Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models","date":"2022-06-09","arxiv_id":"2206.04615","repositories_listed":6,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":3,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/beyond-the-imitation-game-quantifying-and#ran","syntology_url":"https://syntology.ai/paper/2206.04615","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2206.04615"}},"official":{"repos":["google/BIG-bench"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/training-verifiers-to-solve-math-word","slug":"training-verifiers-to-solve-math-word","title":"Training Verifiers to Solve Math Word Problems","date":"2021-10-27","arxiv_id":"2110.14168","repositories_listed":6,"syntology":{"n":7,"n_ran":5,"n_constructed":0,"n_ran_checked":5,"n_instrument":0,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":5,"n_pointer_only":0,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 0 honoured, 0 violated, 5 with no contract checked; 0 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/training-verifiers-to-solve-math-word#ran","syntology_url":"https://syntology.ai/paper/2110.14168","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2110.14168"}},"official":{"repos":["openai/grade-school-math"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/process-reinforcement-through-implicit","slug":"process-reinforcement-through-implicit","title":"Process Reinforcement through Implicit Rewards","date":"2025-02-03","arxiv_id":"2502.01456","repositories_listed":5,"syntology":{"n":17,"n_ran":14,"n_constructed":0,"n_ran_checked":10,"n_instrument":4,"n_unverified":3,"n_honours":2,"n_violates":0,"n_no_contract":8,"n_pointer_only":2,"phrase":"14 ran (of which 0 constructed an object rather than computing a result; 10 with no instrument failure: 2 honoured, 0 violated, 8 with no contract checked; 4 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/process-reinforcement-through-implicit#ran","syntology_url":"https://syntology.ai/paper/2502.01456","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2502.01456"}},"official":{"repos":["prime-rl/prime"],"state":"official (archive's flag): 14 ran","n_ran":14,"n_constructed":0,"n_ran_no_instrument_failure":10,"n_unverified":3,"ran_from_kinds":["official"]}}},{"url":"/paper/deepseekmath-pushing-the-limits-of","slug":"deepseekmath-pushing-the-limits-of","title":"DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models","date":"2024-02-05","arxiv_id":"2402.03300","repositories_listed":5,"syntology":{"n":24,"n_ran":14,"n_constructed":0,"n_ran_checked":3,"n_instrument":11,"n_unverified":10,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":3,"phrase":"14 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 11 where Syntology's instrument failed) · 10 unverified","sample_list":"/paper/deepseekmath-pushing-the-limits-of#ran","syntology_url":"https://syntology.ai/paper/2402.03300","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2402.03300"}},"official":{"repos":["deepseek-ai/deepseek-math"],"state":"official (archive's flag): 8 ran","n_ran":8,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":1,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/full-page-handwriting-recognition-via-image","slug":"full-page-handwriting-recognition-via-image","title":"Full Page Handwriting Recognition via Image to Sequence Extraction","date":"2021-03-11","arxiv_id":"2103.06450","repositories_listed":5,"syntology":null},{"url":"/paper/measuring-mathematical-problem-solving-with","slug":"measuring-mathematical-problem-solving-with","title":"Measuring Mathematical Problem Solving With the MATH Dataset","date":"2021-03-05","arxiv_id":"2103.03874","repositories_listed":5,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":0,"n_instrument":3,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":1,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 3 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/measuring-mathematical-problem-solving-with#ran","syntology_url":"https://syntology.ai/paper/2103.03874","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2103.03874"}},"official":{"repos":["hendrycks/math"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official","unlocated"]}}},{"url":"/paper/the-matrix-calculus-you-need-for-deep","slug":"the-matrix-calculus-you-need-for-deep","title":"The Matrix Calculus You Need For Deep Learning","date":"2018-02-05","arxiv_id":"1802.01528","repositories_listed":5,"syntology":null},{"url":"/paper/mamut-a-novel-framework-for-modifying","slug":"mamut-a-novel-framework-for-modifying","title":"MAMUT: A Novel Framework for Modifying Mathematical Formulas for the Generation of Specialized Datasets for Language Model Training","date":"2025-02-28","arxiv_id":"2502.20855","repositories_listed":4,"syntology":null},{"url":"/paper/logic-rl-unleashing-llm-reasoning-with-rule","slug":"logic-rl-unleashing-llm-reasoning-with-rule","title":"Logic-RL: Unleashing LLM Reasoning with Rule-Based Reinforcement Learning","date":"2025-02-20","arxiv_id":"2502.14768","repositories_listed":4,"syntology":{"n":23,"n_ran":15,"n_constructed":0,"n_ran_checked":10,"n_instrument":5,"n_unverified":8,"n_honours":1,"n_violates":1,"n_no_contract":8,"n_pointer_only":0,"phrase":"15 ran (of which 0 constructed an object rather than computing a result; 10 with no instrument failure: 1 honoured, 1 violated, 8 with no contract checked; 5 where Syntology's instrument failed) · 8 unverified","sample_list":"/paper/logic-rl-unleashing-llm-reasoning-with-rule#ran","syntology_url":"https://syntology.ai/paper/2502.14768","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2502.14768"}},"official":{"repos":["Unakar/Logic-RL"],"state":"official (archive's flag): 10 ran","n_ran":10,"n_constructed":0,"n_ran_no_instrument_failure":7,"n_unverified":6,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/on-the-emergence-of-thinking-in-llms-i","slug":"on-the-emergence-of-thinking-in-llms-i","title":"On the Emergence of Thinking in LLMs I: Searching for the Right Intuition","date":"2025-02-10","arxiv_id":"2502.06773","repositories_listed":4,"syntology":{"n":13,"n_ran":11,"n_constructed":0,"n_ran_checked":6,"n_instrument":5,"n_unverified":2,"n_honours":1,"n_violates":2,"n_no_contract":3,"n_pointer_only":9,"phrase":"11 ran (of which 0 constructed an object rather than computing a result; 6 with no instrument failure: 1 honoured, 2 violated, 3 with no contract checked; 5 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/on-the-emergence-of-thinking-in-llms-i#ran","syntology_url":"https://syntology.ai/paper/2502.06773","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2502.06773"}},"official":{"repos":["GuanghaoYe/Emergence-of-Thinking"],"state":"official (archive's flag): 11 ran","n_ran":11,"n_constructed":0,"n_ran_no_instrument_failure":6,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/mario-eval-evaluate-your-math-llm-with-your","slug":"mario-eval-evaluate-your-math-llm-with-your","title":"MARIO Eval: Evaluate Your Math LLM with your Math LLM--A mathematical dataset evaluation toolkit","date":"2024-04-22","arxiv_id":"2404.13925","repositories_listed":4,"syntology":{"n":13,"n_ran":8,"n_constructed":0,"n_ran_checked":5,"n_instrument":3,"n_unverified":5,"n_honours":0,"n_violates":0,"n_no_contract":5,"n_pointer_only":13,"phrase":"8 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 0 honoured, 0 violated, 5 with no contract checked; 3 where Syntology's instrument failed) · 5 unverified","sample_list":"/paper/mario-eval-evaluate-your-math-llm-with-your#ran","syntology_url":"https://syntology.ai/paper/2404.13925","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2404.13925"}},"official":{"repos":["mario-math-reasoning/math_evaluation"],"state":"official (archive's flag): 8 ran","n_ran":8,"n_constructed":0,"n_ran_no_instrument_failure":5,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/starcoder-2-and-the-stack-v2-the-next","slug":"starcoder-2-and-the-stack-v2-the-next","title":"StarCoder 2 and The Stack v2: The Next Generation","date":"2024-02-29","arxiv_id":"2402.19173","repositories_listed":4,"syntology":null},{"url":"/paper/llemma-an-open-language-model-for-mathematics","slug":"llemma-an-open-language-model-for-mathematics","title":"Llemma: An Open Language Model For Mathematics","date":"2023-10-16","arxiv_id":"2310.10631","repositories_listed":4,"syntology":{"n":8,"n_ran":7,"n_constructed":0,"n_ran_checked":7,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":7,"n_pointer_only":0,"phrase":"7 ran (of which 0 constructed an object rather than computing a result; 7 with no instrument failure: 0 honoured, 0 violated, 7 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/llemma-an-open-language-model-for-mathematics#ran","syntology_url":"https://syntology.ai/paper/2310.10631","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2310.10631"}},"official":{"repos":["EleutherAI/math-lm","eleutherai/gpt-neox","wellecks/llmstep"],"state":"official (archive's flag): 6 ran","n_ran":6,"n_constructed":0,"n_ran_no_instrument_failure":6,"n_unverified":1,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/how-is-chatgpt-s-behavior-changing-over-time","slug":"how-is-chatgpt-s-behavior-changing-over-time","title":"How is ChatGPT's behavior changing over time?","date":"2023-07-18","arxiv_id":"2307.09009","repositories_listed":4,"syntology":{"n":6,"n_ran":6,"n_constructed":0,"n_ran_checked":4,"n_instrument":2,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":4,"n_pointer_only":5,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 4 with no instrument failure: 0 honoured, 0 violated, 4 with no contract checked; 2 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/how-is-chatgpt-s-behavior-changing-over-time#ran","syntology_url":"https://syntology.ai/paper/2307.09009","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2307.09009"}},"official":{"repos":["lchen001/llmdrift"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/language-models-are-multilingual-chain-of","slug":"language-models-are-multilingual-chain-of","title":"Language Models are Multilingual Chain-of-Thought Reasoners","date":"2022-10-06","arxiv_id":"2210.03057","repositories_listed":4,"syntology":null},{"url":"/paper/memorizing-transformers-1","slug":"memorizing-transformers-1","title":"Memorizing Transformers","date":"2022-03-16","arxiv_id":"2203.08913","repositories_listed":4,"syntology":{"n":34,"n_ran":23,"n_constructed":6,"n_ran_checked":18,"n_instrument":5,"n_unverified":11,"n_honours":3,"n_violates":1,"n_no_contract":14,"n_pointer_only":2,"phrase":"23 ran (of which 6 constructed an object rather than computing a result; 18 with no instrument failure: 3 honoured, 1 violated, 14 with no contract checked; 5 where Syntology's instrument failed) · 11 unverified","sample_list":"/paper/memorizing-transformers-1#ran","syntology_url":"https://syntology.ai/paper/2203.08913","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2203.08913"}},"official":{"repos":["lucidrains/memorizing-transformers-pytorch"],"state":"official (archive's flag): 12 ran","n_ran":12,"n_constructed":6,"n_ran_no_instrument_failure":8,"n_unverified":3,"ran_from_kinds":["listed","official","unlocated"]}}},{"url":"/paper/ttrl-test-time-reinforcement-learning","slug":"ttrl-test-time-reinforcement-learning","title":"TTRL: Test-Time Reinforcement Learning","date":"2025-04-22","arxiv_id":"2504.16084","repositories_listed":3,"syntology":{"n":16,"n_ran":14,"n_constructed":0,"n_ran_checked":9,"n_instrument":5,"n_unverified":2,"n_honours":2,"n_violates":0,"n_no_contract":7,"n_pointer_only":6,"phrase":"14 ran (of which 0 constructed an object rather than computing a result; 9 with no instrument failure: 2 honoured, 0 violated, 7 with no contract checked; 5 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/ttrl-test-time-reinforcement-learning#ran","syntology_url":"https://syntology.ai/paper/2504.16084","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2504.16084"}},"official":{"repos":["prime-rl/ttrl","tsinghuac3i/awesome-rl-reasoning-recipes"],"state":"official (archive's flag): 14 ran","n_ran":14,"n_constructed":0,"n_ran_no_instrument_failure":9,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/xlstm-7b-a-recurrent-llm-for-fast-and","slug":"xlstm-7b-a-recurrent-llm-for-fast-and","title":"xLSTM 7B: A Recurrent LLM for Fast and Efficient Inference","date":"2025-03-17","arxiv_id":"2503.13427","repositories_listed":3,"syntology":{"n":5,"n_ran":5,"n_constructed":0,"n_ran_checked":2,"n_instrument":3,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":0,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 3 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/xlstm-7b-a-recurrent-llm-for-fast-and#ran","syntology_url":"https://syntology.ai/paper/2503.13427","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2503.13427"}},"official":{"repos":["nx-ai/xlstm","nx-ai/xlstm-jax","nx-ai/mlstm_kernels"],"state":"official (archive's flag): 5 ran","n_ran":5,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/reasonflux-hierarchical-llm-reasoning-via","slug":"reasonflux-hierarchical-llm-reasoning-via","title":"ReasonFlux: Hierarchical LLM Reasoning via Scaling Thought Templates","date":"2025-02-10","arxiv_id":"2502.06772","repositories_listed":3,"syntology":null},{"url":"/paper/limo-less-is-more-for-reasoning","slug":"limo-less-is-more-for-reasoning","title":"LIMO: Less is More for Reasoning","date":"2025-02-05","arxiv_id":"2502.03387","repositories_listed":3,"syntology":null},{"url":"/paper/kimi-k1-5-scaling-reinforcement-learning-with","slug":"kimi-k1-5-scaling-reinforcement-learning-with","title":"Kimi k1.5: Scaling Reinforcement Learning with LLMs","date":"2025-01-22","arxiv_id":"2501.12599","repositories_listed":3,"syntology":{"n":10,"n_ran":7,"n_constructed":0,"n_ran_checked":5,"n_instrument":2,"n_unverified":3,"n_honours":0,"n_violates":1,"n_no_contract":4,"n_pointer_only":3,"phrase":"7 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 0 honoured, 1 violated, 4 with no contract checked; 2 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/kimi-k1-5-scaling-reinforcement-learning-with#ran","syntology_url":"https://syntology.ai/paper/2501.12599","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2501.12599"}},"official":null}},{"url":"/paper/rstar-math-small-llms-can-master-math","slug":"rstar-math-small-llms-can-master-math","title":"rStar-Math: Small LLMs Can Master Math Reasoning with Self-Evolved Deep Thinking","date":"2025-01-08","arxiv_id":"2501.04519","repositories_listed":3,"syntology":null},{"url":"/paper/bone-block-affine-transformation-as-parameter","slug":"bone-block-affine-transformation-as-parameter","title":"Balancing LoRA Performance and Efficiency with Simple Shard Sharing","date":"2024-09-19","arxiv_id":"2409.15371","repositories_listed":3,"syntology":null},{"url":"/paper/mutual-reasoning-makes-smaller-llms-stronger","slug":"mutual-reasoning-makes-smaller-llms-stronger","title":"Mutual Reasoning Makes Smaller LLMs Stronger Problem-Solvers","date":"2024-08-12","arxiv_id":"2408.06195","repositories_listed":3,"syntology":{"n":2,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/mutual-reasoning-makes-smaller-llms-stronger#ran","syntology_url":"https://syntology.ai/paper/2408.06195","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2408.06195"}},"official":{"repos":["zhentingqi/rstar"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/mavis-mathematical-visual-instruction-tuning","slug":"mavis-mathematical-visual-instruction-tuning","title":"MAVIS: Mathematical Visual Instruction Tuning with an Automatic Data Engine","date":"2024-07-11","arxiv_id":"2407.08739","repositories_listed":3,"syntology":null},{"url":"/paper/mathodyssey-benchmarking-mathematical-problem","slug":"mathodyssey-benchmarking-mathematical-problem","title":"MathOdyssey: Benchmarking Mathematical Problem-Solving Skills in Large Language Models Using Odyssey Math Data","date":"2024-06-26","arxiv_id":"2406.18321","repositories_listed":3,"syntology":{"n":5,"n_ran":5,"n_constructed":0,"n_ran_checked":4,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":4,"n_pointer_only":0,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 4 with no instrument failure: 0 honoured, 0 violated, 4 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/mathodyssey-benchmarking-mathematical-problem#ran","syntology_url":"https://syntology.ai/paper/2406.18321","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.18321"}},"official":null}},{"url":"/paper/rho-1-not-all-tokens-are-what-you-need","slug":"rho-1-not-all-tokens-are-what-you-need","title":"Rho-1: Not All Tokens Are What You Need","date":"2024-04-11","arxiv_id":"2404.07965","repositories_listed":3,"syntology":null},{"url":"/paper/chatglm-math-improving-math-problem-solving","slug":"chatglm-math-improving-math-problem-solving","title":"ChatGLM-Math: Improving Math Problem-Solving in Large Language Models with a Self-Critique Pipeline","date":"2024-04-03","arxiv_id":"2404.02893","repositories_listed":3,"syntology":{"n":4,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":3,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":1,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/chatglm-math-improving-math-problem-solving#ran","syntology_url":"https://syntology.ai/paper/2404.02893","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2404.02893"}},"official":{"repos":["thudm/chatglm-math"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":3,"ran_from_kinds":["official"]}}},{"url":"/paper/language-models-are-homer-simpson-safety-re","slug":"language-models-are-homer-simpson-safety-re","title":"Language Models are Homer Simpson! Safety Re-Alignment of Fine-tuned Language Models through Task Arithmetic","date":"2024-02-19","arxiv_id":"2402.11746","repositories_listed":3,"syntology":{"n":16,"n_ran":13,"n_constructed":0,"n_ran_checked":2,"n_instrument":11,"n_unverified":3,"n_honours":1,"n_violates":1,"n_no_contract":0,"n_pointer_only":9,"phrase":"13 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 1 honoured, 1 violated, 0 with no contract checked; 11 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/language-models-are-homer-simpson-safety-re#ran","syntology_url":"https://syntology.ai/paper/2402.11746","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2402.11746"}},"official":{"repos":["declare-lab/resta","hiyouga/llama-factory"],"state":"official (archive's flag): 6 ran","n_ran":6,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":1,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/noise-contrastive-alignment-of-language","slug":"noise-contrastive-alignment-of-language","title":"Noise Contrastive Alignment of Language Models with Explicit Rewards","date":"2024-02-08","arxiv_id":"2402.05369","repositories_listed":3,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":2,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/noise-contrastive-alignment-of-language#ran","syntology_url":"https://syntology.ai/paper/2402.05369","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2402.05369"}},"official":{"repos":["thu-ml/noise-contrastive-alignment"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/self-discover-large-language-models-self","slug":"self-discover-large-language-models-self","title":"Self-Discover: Large Language Models Self-Compose Reasoning Structures","date":"2024-02-06","arxiv_id":"2402.03620","repositories_listed":3,"syntology":{"n":3,"n_ran":0,"n_constructed":0,"n_ran_checked":0,"n_instrument":0,"n_unverified":3,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"0 ran · 3 unverified","sample_list":"/paper/self-discover-large-language-models-self#ran","syntology_url":"https://syntology.ai/paper/2402.03620","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2402.03620"}},"official":null}},{"url":"/paper/math-shepherd-a-label-free-step-by-step","slug":"math-shepherd-a-label-free-step-by-step","title":"Math-Shepherd: Verify and Reinforce LLMs Step-by-step without Human Annotations","date":"2023-12-14","arxiv_id":"2312.08935","repositories_listed":3,"syntology":null},{"url":"/paper/dspy-compiling-declarative-language-model","slug":"dspy-compiling-declarative-language-model","title":"DSPy: Compiling Declarative Language Model Calls into Self-Improving Pipelines","date":"2023-10-05","arxiv_id":"2310.03714","repositories_listed":3,"syntology":{"n":7,"n_ran":3,"n_constructed":0,"n_ran_checked":3,"n_instrument":0,"n_unverified":4,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 4 unverified","sample_list":"/paper/dspy-compiling-declarative-language-model#ran","syntology_url":"https://syntology.ai/paper/2310.03714","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2310.03714"}},"official":{"repos":["stanfordnlp/dspy"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/studying-large-language-model-generalization","slug":"studying-large-language-model-generalization","title":"Studying Large Language Model Generalization with Influence Functions","date":"2023-08-07","arxiv_id":"2308.03296","repositories_listed":3,"syntology":null},{"url":"/paper/leandojo-theorem-proving-with-retrieval-1","slug":"leandojo-theorem-proving-with-retrieval-1","title":"LeanDojo: Theorem Proving with Retrieval-Augmented Language Models","date":"2023-06-27","arxiv_id":"2306.15626","repositories_listed":3,"syntology":{"n":9,"n_ran":8,"n_constructed":0,"n_ran_checked":7,"n_instrument":1,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":7,"n_pointer_only":1,"phrase":"8 ran (of which 0 constructed an object rather than computing a result; 7 with no instrument failure: 0 honoured, 0 violated, 7 with no contract checked; 1 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/leandojo-theorem-proving-with-retrieval-1#ran","syntology_url":"https://syntology.ai/paper/2306.15626","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2306.15626"}},"official":{"repos":["lean-dojo/leandojo","lean-dojo/leandojochatgpt","lean-dojo/reprover"],"state":"official (archive's flag): 8 ran","n_ran":8,"n_constructed":0,"n_ran_no_instrument_failure":7,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/let-s-verify-step-by-step-1","slug":"let-s-verify-step-by-step-1","title":"Let's Verify Step by Step","date":"2023-05-31","arxiv_id":"2305.20050","repositories_listed":3,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":3,"n_instrument":0,"n_unverified":0,"n_honours":1,"n_violates":2,"n_no_contract":0,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 1 honoured, 2 violated, 0 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/let-s-verify-step-by-step-1#ran","syntology_url":"https://syntology.ai/paper/2305.20050","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2305.20050"}},"official":{"repos":["openai/prm800k"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/reasoning-with-language-model-is-planning","slug":"reasoning-with-language-model-is-planning","title":"Reasoning with Language Model is Planning with World Model","date":"2023-05-24","arxiv_id":"2305.14992","repositories_listed":3,"syntology":{"n":7,"n_ran":4,"n_constructed":2,"n_ran_checked":2,"n_instrument":2,"n_unverified":3,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":0,"phrase":"4 ran (of which 2 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 2 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/reasoning-with-language-model-is-planning#ran","syntology_url":"https://syntology.ai/paper/2305.14992","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2305.14992"}},"official":null}},{"url":"/paper/plan-and-solve-prompting-improving-zero-shot","slug":"plan-and-solve-prompting-improving-zero-shot","title":"Plan-and-Solve Prompting: Improving Zero-Shot Chain-of-Thought Reasoning by Large Language Models","date":"2023-05-06","arxiv_id":"2305.04091","repositories_listed":3,"syntology":{"n":2,"n_ran":0,"n_constructed":0,"n_ran_checked":0,"n_instrument":0,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":2,"phrase":"0 ran · 2 unverified","sample_list":"/paper/plan-and-solve-prompting-improving-zero-shot#ran","syntology_url":"https://syntology.ai/paper/2305.04091","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2305.04091"}},"official":{"repos":["agi-edgerunners/plan-and-solve-prompting"],"state":"official: harvested, nothing ran","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":2,"ran_from_kinds":[]}}},{"url":"/paper/agieval-a-human-centric-benchmark-for","slug":"agieval-a-human-centric-benchmark-for","title":"AGIEval: A Human-Centric Benchmark for Evaluating Foundation Models","date":"2023-04-13","arxiv_id":"2304.06364","repositories_listed":3,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/agieval-a-human-centric-benchmark-for#ran","syntology_url":"https://syntology.ai/paper/2304.06364","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2304.06364"}},"official":{"repos":["ruixiangcui/agieval"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/pal-program-aided-language-models","slug":"pal-program-aided-language-models","title":"PAL: Program-aided Language Models","date":"2022-11-18","arxiv_id":"2211.10435","repositories_listed":3,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":0,"n_instrument":3,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 3 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/pal-program-aided-language-models#ran","syntology_url":"https://syntology.ai/paper/2211.10435","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2211.10435"}},"official":null}},{"url":"/paper/self-consistency-improves-chain-of-thought","slug":"self-consistency-improves-chain-of-thought","title":"Self-Consistency Improves Chain of Thought Reasoning in Language Models","date":"2022-03-21","arxiv_id":"2203.11171","repositories_listed":3,"syntology":{"n":1,"n_ran":1,"n_constructed":1,"n_ran_checked":1,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"1 ran (of which 1 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified; the one sample that ran constructed an object rather than computing a result","sample_list":"/paper/self-consistency-improves-chain-of-thought#ran","syntology_url":"https://syntology.ai/paper/2203.11171","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2203.11171"}},"official":null}},{"url":"/paper/accelerating-sparse-deep-neural-networks","slug":"accelerating-sparse-deep-neural-networks","title":"Accelerating Sparse Deep Neural Networks","date":"2021-04-16","arxiv_id":"2104.08378","repositories_listed":3,"syntology":null},{"url":"/paper/are-nlp-models-really-able-to-solve-simple","slug":"are-nlp-models-really-able-to-solve-simple","title":"Are NLP Models really able to Solve Simple Math Word Problems?","date":"2021-03-12","arxiv_id":"2103.07191","repositories_listed":3,"syntology":null},{"url":"/paper/stereoset-measuring-stereotypical-bias-in","slug":"stereoset-measuring-stereotypical-bias-in","title":"StereoSet: Measuring stereotypical bias in pretrained language models","date":"2020-04-20","arxiv_id":"2004.09456","repositories_listed":3,"syntology":null},{"url":"/paper/enhancing-the-transformer-with-explicit-1","slug":"enhancing-the-transformer-with-explicit-1","title":"Enhancing the Transformer with Explicit Relational Encoding for Math Problem Solving","date":"2019-10-15","arxiv_id":"1910.06611","repositories_listed":3,"syntology":{"n":11,"n_ran":8,"n_constructed":0,"n_ran_checked":8,"n_instrument":0,"n_unverified":3,"n_honours":1,"n_violates":0,"n_no_contract":7,"n_pointer_only":0,"phrase":"8 ran (of which 0 constructed an object rather than computing a result; 8 with no instrument failure: 1 honoured, 0 violated, 7 with no contract checked; 0 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/enhancing-the-transformer-with-explicit-1#ran","syntology_url":"https://syntology.ai/paper/1910.06611","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1910.06611"}},"official":{"repos":["ischlag/TP-Transformer"],"state":"official (archive's flag): 8 ran","n_ran":8,"n_constructed":0,"n_ran_no_instrument_failure":8,"n_unverified":3,"ran_from_kinds":["official"]}}},{"url":"/paper/reasonflux-prm-trajectory-aware-prms-for-long","slug":"reasonflux-prm-trajectory-aware-prms-for-long","title":"ReasonFlux-PRM: Trajectory-Aware PRMs for Long Chain-of-Thought Reasoning in LLMs","date":"2025-06-23","arxiv_id":"2506.18896","repositories_listed":2,"syntology":{"n":5,"n_ran":4,"n_constructed":0,"n_ran_checked":2,"n_instrument":2,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":0,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 2 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/reasonflux-prm-trajectory-aware-prms-for-long#ran","syntology_url":"https://syntology.ai/paper/2506.18896","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2506.18896"}},"official":{"repos":["gen-verse/reasonflux"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/advancing-multimodal-reasoning-via","slug":"advancing-multimodal-reasoning-via","title":"Advancing Multimodal Reasoning via Reinforcement Learning with Cold Start","date":"2025-05-28","arxiv_id":"2505.22334","repositories_listed":2,"syntology":null},{"url":"/paper/unsupervised-post-training-for-multi-modal","slug":"unsupervised-post-training-for-multi-modal","title":"Unsupervised Post-Training for Multi-Modal LLM Reasoning via GRPO","date":"2025-05-28","arxiv_id":"2505.22453","repositories_listed":2,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/unsupervised-post-training-for-multi-modal#ran","syntology_url":"https://syntology.ai/paper/2505.22453","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2505.22453"}},"official":{"repos":["hiyouga/easyr1","waltonfuture/mm-upt"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/mm-prm-enhancing-multimodal-mathematical","slug":"mm-prm-enhancing-multimodal-mathematical","title":"MM-PRM: Enhancing Multimodal Mathematical Reasoning with Scalable Step-Level Supervision","date":"2025-05-19","arxiv_id":"2505.13427","repositories_listed":2,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":1,"n_instrument":1,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/mm-prm-enhancing-multimodal-mathematical#ran","syntology_url":"https://syntology.ai/paper/2505.13427","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2505.13427"}},"official":{"repos":["modalminds/mm-prm"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/agent-rl-scaling-law-agent-rl-with","slug":"agent-rl-scaling-law-agent-rl-with","title":"Agent RL Scaling Law: Agent RL with Spontaneous Code Execution for Mathematical Problem Solving","date":"2025-05-12","arxiv_id":"2505.07773","repositories_listed":2,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/agent-rl-scaling-law-agent-rl-with#ran","syntology_url":"https://syntology.ai/paper/2505.07773","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2505.07773"}},"official":{"repos":["anonymize-author/agentrl","yyht/openrlhf_async_pipline"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/vl-rethinker-incentivizing-self-reflection-of","slug":"vl-rethinker-incentivizing-self-reflection-of","title":"VL-Rethinker: Incentivizing Self-Reflection of Vision-Language Models with Reinforcement Learning","date":"2025-04-10","arxiv_id":"2504.08837","repositories_listed":2,"syntology":{"n":12,"n_ran":8,"n_constructed":0,"n_ran_checked":5,"n_instrument":3,"n_unverified":4,"n_honours":0,"n_violates":0,"n_no_contract":5,"n_pointer_only":1,"phrase":"8 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 0 honoured, 0 violated, 5 with no contract checked; 3 where Syntology's instrument failed) · 4 unverified","sample_list":"/paper/vl-rethinker-incentivizing-self-reflection-of#ran","syntology_url":"https://syntology.ai/paper/2504.08837","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2504.08837"}},"official":null}},{"url":"/paper/challenging-the-boundaries-of-reasoning-an","slug":"challenging-the-boundaries-of-reasoning-an","title":"Challenging the Boundaries of Reasoning: An Olympiad-Level Math Benchmark for Large Language Models","date":"2025-03-27","arxiv_id":"2503.21380","repositories_listed":2,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":3,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/challenging-the-boundaries-of-reasoning-an#ran","syntology_url":"https://syntology.ai/paper/2503.21380","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2503.21380"}},"official":{"repos":["RUCAIBox/Slow_Thinking_with_LLMs","rucaibox/olymmath"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/llms-can-easily-learn-to-reason-from","slug":"llms-can-easily-learn-to-reason-from","title":"LLMs Can Easily Learn to Reason from Demonstrations Structure, not content, is what matters!","date":"2025-02-11","arxiv_id":"2502.07374","repositories_listed":2,"syntology":null},{"url":"/paper/s1-simple-test-time-scaling","slug":"s1-simple-test-time-scaling","title":"s1: Simple test-time scaling","date":"2025-01-31","arxiv_id":"2501.19393","repositories_listed":2,"syntology":{"n":4,"n_ran":4,"n_constructed":0,"n_ran_checked":1,"n_instrument":3,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":1,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 3 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/s1-simple-test-time-scaling#ran","syntology_url":"https://syntology.ai/paper/2501.19393","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2501.19393"}},"official":{"repos":["simplescaling/s1"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/leveraging-online-olympiad-level-math","slug":"leveraging-online-olympiad-level-math","title":"Leveraging Online Olympiad-Level Math Problems for LLMs Training and Contamination-Resistant Evaluation","date":"2025-01-24","arxiv_id":"2501.14275","repositories_listed":2,"syntology":{"n":11,"n_ran":10,"n_constructed":0,"n_ran_checked":10,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":10,"n_pointer_only":0,"phrase":"10 ran (of which 0 constructed an object rather than computing a result; 10 with no instrument failure: 0 honoured, 0 violated, 10 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/leveraging-online-olympiad-level-math#ran","syntology_url":"https://syntology.ai/paper/2501.14275","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2501.14275"}},"official":{"repos":["dsl-lab/aops"],"state":"official (archive's flag): 10 ran","n_ran":10,"n_constructed":0,"n_ran_no_instrument_failure":10,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/offline-reinforcement-learning-for-llm-multi","slug":"offline-reinforcement-learning-for-llm-multi","title":"Offline Reinforcement Learning for LLM Multi-Step Reasoning","date":"2024-12-20","arxiv_id":"2412.16145","repositories_listed":2,"syntology":{"n":3,"n_ran":2,"n_constructed":0,"n_ran_checked":1,"n_instrument":1,"n_unverified":1,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/offline-reinforcement-learning-for-llm-multi#ran","syntology_url":"https://syntology.ai/paper/2412.16145","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2412.16145"}},"official":{"repos":["jwhj/oreo"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/free-process-rewards-without-process-labels","slug":"free-process-rewards-without-process-labels","title":"Free Process Rewards without Process Labels","date":"2024-12-02","arxiv_id":"2412.01981","repositories_listed":2,"syntology":{"n":22,"n_ran":14,"n_constructed":0,"n_ran_checked":8,"n_instrument":6,"n_unverified":8,"n_honours":5,"n_violates":0,"n_no_contract":3,"n_pointer_only":1,"phrase":"14 ran (of which 0 constructed an object rather than computing a result; 8 with no instrument failure: 5 honoured, 0 violated, 3 with no contract checked; 6 where Syntology's instrument failed) · 8 unverified","sample_list":"/paper/free-process-rewards-without-process-labels#ran","syntology_url":"https://syntology.ai/paper/2412.01981","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2412.01981"}},"official":{"repos":["lifan-yuan/implicitprm"],"state":"official (archive's flag): 14 ran","n_ran":14,"n_constructed":0,"n_ran_no_instrument_failure":8,"n_unverified":8,"ran_from_kinds":["official"]}}},{"url":"/paper/enhancing-reasoning-capabilities-of-llms-via","slug":"enhancing-reasoning-capabilities-of-llms-via","title":"Enhancing Reasoning Capabilities of LLMs via Principled Synthetic Logic Corpus","date":"2024-11-19","arxiv_id":"2411.12498","repositories_listed":2,"syntology":null},{"url":"/paper/unlocking-state-tracking-in-linear-rnns","slug":"unlocking-state-tracking-in-linear-rnns","title":"Unlocking State-Tracking in Linear RNNs Through Negative Eigenvalues","date":"2024-11-19","arxiv_id":"2411.12537","repositories_listed":2,"syntology":{"n":8,"n_ran":5,"n_constructed":0,"n_ran_checked":2,"n_instrument":3,"n_unverified":3,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":8,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 3 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/unlocking-state-tracking-in-linear-rnns#ran","syntology_url":"https://syntology.ai/paper/2411.12537","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2411.12537"}},"official":{"repos":["automl/unlocking_state_tracking"],"state":"official (archive's flag): 5 ran","n_ran":5,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":3,"ran_from_kinds":["official"]}}},{"url":"/paper/flaming-hot-initiation-with-regular-execution","slug":"flaming-hot-initiation-with-regular-execution","title":"Flaming-hot Initiation with Regular Execution Sampling for Large Language Models","date":"2024-10-28","arxiv_id":"2410.21236","repositories_listed":2,"syntology":{"n":11,"n_ran":7,"n_constructed":0,"n_ran_checked":2,"n_instrument":5,"n_unverified":4,"n_honours":1,"n_violates":1,"n_no_contract":0,"n_pointer_only":1,"phrase":"7 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 1 honoured, 1 violated, 0 with no contract checked; 5 where Syntology's instrument failed) · 4 unverified","sample_list":"/paper/flaming-hot-initiation-with-regular-execution#ran","syntology_url":"https://syntology.ai/paper/2410.21236","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2410.21236"}},"official":null}},{"url":"/paper/enhancing-multi-step-reasoning-abilities-of","slug":"enhancing-multi-step-reasoning-abilities-of","title":"Enhancing Multi-Step Reasoning Abilities of Language Models through Direct Q-Function Optimization","date":"2024-10-11","arxiv_id":"2410.09302","repositories_listed":2,"syntology":null},{"url":"/paper/supercorrect-supervising-and-correcting","slug":"supercorrect-supervising-and-correcting","title":"SuperCorrect: Supervising and Correcting Language Models with Error-Driven Insights","date":"2024-10-11","arxiv_id":"2410.09008","repositories_listed":2,"syntology":{"n":1,"n_ran":0,"n_constructed":0,"n_ran_checked":0,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":1,"phrase":"0 ran · 1 unverified","sample_list":"/paper/supercorrect-supervising-and-correcting#ran","syntology_url":"https://syntology.ai/paper/2410.09008","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2410.09008"}},"official":{"repos":["yangling0818/supercorrect-llm"],"state":"official: harvested, nothing ran","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":1,"ran_from_kinds":[]}}},{"url":"/paper/omni-math-a-universal-olympiad-level","slug":"omni-math-a-universal-olympiad-level","title":"Omni-MATH: A Universal Olympiad Level Mathematic Benchmark For Large Language Models","date":"2024-10-10","arxiv_id":"2410.07985","repositories_listed":2,"syntology":{"n":19,"n_ran":13,"n_constructed":0,"n_ran_checked":6,"n_instrument":7,"n_unverified":6,"n_honours":1,"n_violates":5,"n_no_contract":0,"n_pointer_only":19,"phrase":"13 ran (of which 0 constructed an object rather than computing a result; 6 with no instrument failure: 1 honoured, 5 violated, 0 with no contract checked; 7 where Syntology's instrument failed) · 6 unverified","sample_list":"/paper/omni-math-a-universal-olympiad-level#ran","syntology_url":"https://syntology.ai/paper/2410.07985","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2410.07985"}},"official":{"repos":["kbsdjames/omni-math","kbsdjames/omni-math-rule"],"state":"official (archive's flag): 13 ran","n_ran":13,"n_constructed":0,"n_ran_no_instrument_failure":6,"n_unverified":6,"ran_from_kinds":["official"]}}},{"url":"/paper/towards-the-pedagogical-steering-of-large","slug":"towards-the-pedagogical-steering-of-large","title":"Towards the Pedagogical Steering of Large Language Models for Tutoring: A Case Study with Modeling Productive Failure","date":"2024-10-03","arxiv_id":"2410.03781","repositories_listed":2,"syntology":null},{"url":"/paper/training-language-models-to-self-correct-via","slug":"training-language-models-to-self-correct-via","title":"Training Language Models to Self-Correct via Reinforcement Learning","date":"2024-09-19","arxiv_id":"2409.12917","repositories_listed":2,"syntology":null},{"url":"/paper/qwen2-5-coder-technical-report","slug":"qwen2-5-coder-technical-report","title":"Qwen2.5-Coder Technical Report","date":"2024-09-18","arxiv_id":"2409.12186","repositories_listed":2,"syntology":{"n":9,"n_ran":8,"n_constructed":0,"n_ran_checked":6,"n_instrument":2,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":6,"n_pointer_only":9,"phrase":"8 ran (of which 0 constructed an object rather than computing a result; 6 with no instrument failure: 0 honoured, 0 violated, 6 with no contract checked; 2 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/qwen2-5-coder-technical-report#ran","syntology_url":"https://syntology.ai/paper/2409.12186","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2409.12186"}},"official":{"repos":["qwenlm/qwen2.5-coder"],"state":"official (archive's flag): 8 ran","n_ran":8,"n_constructed":0,"n_ran_no_instrument_failure":6,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/vae-explainer-supplement-learning-variational","slug":"vae-explainer-supplement-learning-variational","title":"VAE Explainer: Supplement Learning Variational Autoencoders with Interactive Visualization","date":"2024-09-13","arxiv_id":"2409.09011","repositories_listed":2,"syntology":null},{"url":"/paper/2408-00724","slug":"2408-00724","title":"Inference Scaling Laws: An Empirical Analysis of Compute-Optimal Inference for Problem-Solving with Language Models","date":"2024-08-01","arxiv_id":"2408.00724","repositories_listed":2,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":1,"n_instrument":1,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":2,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/2408-00724#ran","syntology_url":"https://syntology.ai/paper/2408.00724","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2408.00724"}},"official":null}},{"url":"/paper/large-language-monkeys-scaling-inference","slug":"large-language-monkeys-scaling-inference","title":"Large Language Monkeys: Scaling Inference Compute with Repeated Sampling","date":"2024-07-31","arxiv_id":"2407.21787","repositories_listed":2,"syntology":{"n":5,"n_ran":4,"n_constructed":0,"n_ran_checked":2,"n_instrument":2,"n_unverified":1,"n_honours":0,"n_violates":2,"n_no_contract":0,"n_pointer_only":5,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 2 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/large-language-monkeys-scaling-inference#ran","syntology_url":"https://syntology.ai/paper/2407.21787","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2407.21787"}},"official":{"repos":["scalingintelligence/large_language_monkeys"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/unpacking-dpo-and-ppo-disentangling-best","slug":"unpacking-dpo-and-ppo-disentangling-best","title":"Unpacking DPO and PPO: Disentangling Best Practices for Learning from Preference Feedback","date":"2024-06-13","arxiv_id":"2406.09279","repositories_listed":2,"syntology":{"n":19,"n_ran":15,"n_constructed":0,"n_ran_checked":2,"n_instrument":13,"n_unverified":4,"n_honours":1,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"15 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 1 honoured, 0 violated, 1 with no contract checked; 13 where Syntology's instrument failed) · 4 unverified","sample_list":"/paper/unpacking-dpo-and-ppo-disentangling-best#ran","syntology_url":"https://syntology.ai/paper/2406.09279","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.09279"}},"official":{"repos":["allenai/open-instruct","hamishivi/easylm"],"state":"official (archive's flag): 15 ran","n_ran":15,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":4,"ran_from_kinds":["official"]}}},{"url":"/paper/monte-carlo-tree-search-boosts-reasoning-via","slug":"monte-carlo-tree-search-boosts-reasoning-via","title":"Monte Carlo Tree Search Boosts Reasoning via Iterative Preference Learning","date":"2024-05-01","arxiv_id":"2405.00451","repositories_listed":2,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":1,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":1,"n_no_contract":0,"n_pointer_only":1,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 1 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/monte-carlo-tree-search-boosts-reasoning-via#ran","syntology_url":"https://syntology.ai/paper/2405.00451","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2405.00451"}},"official":{"repos":["YuxiXie/MCTS-DPO"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/common-7b-language-models-already-possess","slug":"common-7b-language-models-already-possess","title":"Common 7B Language Models Already Possess Strong Math Capabilities","date":"2024-03-07","arxiv_id":"2403.04706","repositories_listed":2,"syntology":{"n":28,"n_ran":21,"n_constructed":0,"n_ran_checked":15,"n_instrument":6,"n_unverified":7,"n_honours":0,"n_violates":0,"n_no_contract":15,"n_pointer_only":12,"phrase":"21 ran (of which 0 constructed an object rather than computing a result; 15 with no instrument failure: 0 honoured, 0 violated, 15 with no contract checked; 6 where Syntology's instrument failed) · 7 unverified","sample_list":"/paper/common-7b-language-models-already-possess#ran","syntology_url":"https://syntology.ai/paper/2403.04706","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2403.04706"}},"official":{"repos":["xwin-lm/xwin-lm"],"state":"official (archive's flag): 9 ran","n_ran":9,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":3,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/mathwell-generating-educational-math-word","slug":"mathwell-generating-educational-math-word","title":"MATHWELL: Generating Educational Math Word Problems Using Teacher Annotations","date":"2024-02-24","arxiv_id":"2402.15861","repositories_listed":2,"syntology":null},{"url":"/paper/automathtext-autonomous-data-selection-with","slug":"automathtext-autonomous-data-selection-with","title":"Autonomous Data Selection with Zero-shot Generative Classifiers for Mathematical Texts","date":"2024-02-12","arxiv_id":"2402.07625","repositories_listed":2,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":3,"n_instrument":0,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":2,"n_pointer_only":3,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 1 honoured, 0 violated, 2 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/automathtext-autonomous-data-selection-with#ran","syntology_url":"https://syntology.ai/paper/2402.07625","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2402.07625"}},"official":{"repos":["hiyouga/llama-factory","yifanzhang-pro/automathtext"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/mario-math-reasoning-with-code-interpreter","slug":"mario-math-reasoning-with-code-interpreter","title":"MARIO: MAth Reasoning with code Interpreter Output -- A Reproducible Pipeline","date":"2024-01-16","arxiv_id":"2401.08190","repositories_listed":2,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":0,"n_instrument":3,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 3 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/mario-math-reasoning-with-code-interpreter#ran","syntology_url":"https://syntology.ai/paper/2401.08190","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2401.08190"}},"official":{"repos":["mario-math-reasoning/mario"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/tuning-language-models-by-proxy","slug":"tuning-language-models-by-proxy","title":"Tuning Language Models by Proxy","date":"2024-01-16","arxiv_id":"2401.08565","repositories_listed":2,"syntology":{"n":10,"n_ran":9,"n_constructed":0,"n_ran_checked":4,"n_instrument":5,"n_unverified":1,"n_honours":1,"n_violates":0,"n_no_contract":3,"n_pointer_only":10,"phrase":"9 ran (of which 0 constructed an object rather than computing a result; 4 with no instrument failure: 1 honoured, 0 violated, 3 with no contract checked; 5 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/tuning-language-models-by-proxy#ran","syntology_url":"https://syntology.ai/paper/2401.08565","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2401.08565"}},"official":{"repos":["alisawuffles/proxy-tuning"],"state":"official (archive's flag): 9 ran","n_ran":9,"n_constructed":0,"n_ran_no_instrument_failure":4,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/rosa-accurate-parameter-efficient-fine-tuning","slug":"rosa-accurate-parameter-efficient-fine-tuning","title":"RoSA: Accurate Parameter-Efficient Fine-Tuning via Robust Adaptation","date":"2024-01-09","arxiv_id":"2401.04679","repositories_listed":2,"syntology":{"n":9,"n_ran":6,"n_constructed":0,"n_ran_checked":5,"n_instrument":1,"n_unverified":3,"n_honours":1,"n_violates":1,"n_no_contract":3,"n_pointer_only":0,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 1 honoured, 1 violated, 3 with no contract checked; 1 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/rosa-accurate-parameter-efficient-fine-tuning#ran","syntology_url":"https://syntology.ai/paper/2401.04679","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2401.04679"}},"official":{"repos":["ist-daslab/rosa"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/challenge-llms-to-reason-about-reasoning-a","slug":"challenge-llms-to-reason-about-reasoning-a","title":"MR-GSM8K: A Meta-Reasoning Benchmark for Large Language Model Evaluation","date":"2023-12-28","arxiv_id":"2312.17080","repositories_listed":2,"syntology":null},{"url":"/paper/yuan-2-0-a-large-language-model-with","slug":"yuan-2-0-a-large-language-model-with","title":"YUAN 2.0: A Large Language Model with Localized Filtering-based Attention","date":"2023-11-27","arxiv_id":"2311.15786","repositories_listed":2,"syntology":null},{"url":"/paper/mathgloss-building-mathematical-glossaries","slug":"mathgloss-building-mathematical-glossaries","title":"MathGloss: Building mathematical glossaries from text","date":"2023-11-21","arxiv_id":"2311.12649","repositories_listed":2,"syntology":null},{"url":"/paper/lumos-learning-agents-with-unified-data","slug":"lumos-learning-agents-with-unified-data","title":"Agent Lumos: Unified and Modular Training for Open-Source Language Agents","date":"2023-11-09","arxiv_id":"2311.05657","repositories_listed":2,"syntology":{"n":3,"n_ran":2,"n_constructed":0,"n_ran_checked":0,"n_instrument":2,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/lumos-learning-agents-with-unified-data#ran","syntology_url":"https://syntology.ai/paper/2311.05657","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2311.05657"}},"official":{"repos":["allenai/lumos"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/breaking-language-barriers-in-multilingual","slug":"breaking-language-barriers-in-multilingual","title":"Breaking Language Barriers in Multilingual Mathematical Reasoning: Insights and Observations","date":"2023-10-31","arxiv_id":"2310.20246","repositories_listed":2,"syntology":{"n":11,"n_ran":8,"n_constructed":0,"n_ran_checked":7,"n_instrument":1,"n_unverified":3,"n_honours":0,"n_violates":0,"n_no_contract":7,"n_pointer_only":3,"phrase":"8 ran (of which 0 constructed an object rather than computing a result; 7 with no instrument failure: 0 honoured, 0 violated, 7 with no contract checked; 1 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/breaking-language-barriers-in-multilingual#ran","syntology_url":"https://syntology.ai/paper/2310.20246","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2310.20246"}},"official":{"repos":["microsoft/MathOctopus"],"state":"official (archive's flag): 8 ran","n_ran":8,"n_constructed":0,"n_ran_no_instrument_failure":7,"n_unverified":3,"ran_from_kinds":["official"]}}},{"url":"/paper/how-abilities-in-large-language-models-are","slug":"how-abilities-in-large-language-models-are","title":"How Abilities in Large Language Models are Affected by Supervised Fine-tuning Data Composition","date":"2023-10-09","arxiv_id":"2310.05492","repositories_listed":2,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":1,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":2,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/how-abilities-in-large-language-models-are#ran","syntology_url":"https://syntology.ai/paper/2310.05492","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2310.05492"}},"official":{"repos":["ofa-sys/gsm8k-screl"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/language-agent-tree-search-unifies-reasoning","slug":"language-agent-tree-search-unifies-reasoning","title":"Language Agent Tree Search Unifies Reasoning Acting and Planning in Language Models","date":"2023-10-06","arxiv_id":"2310.04406","repositories_listed":2,"syntology":{"n":9,"n_ran":9,"n_constructed":0,"n_ran_checked":5,"n_instrument":4,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":4,"n_pointer_only":2,"phrase":"9 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 1 honoured, 0 violated, 4 with no contract checked; 4 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/language-agent-tree-search-unifies-reasoning#ran","syntology_url":"https://syntology.ai/paper/2310.04406","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2310.04406"}},"official":{"repos":["lapisrocks/languageagenttreesearch","andyz245/LanguageAgentTreeSearch"],"state":"official (archive's flag): 9 ran","n_ran":9,"n_constructed":0,"n_ran_no_instrument_failure":5,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/retrieval-augmented-generation-to-improve","slug":"retrieval-augmented-generation-to-improve","title":"Retrieval-augmented Generation to Improve Math Question-Answering: Trade-offs Between Groundedness and Human Preference","date":"2023-10-04","arxiv_id":"2310.03184","repositories_listed":2,"syntology":{"n":20,"n_ran":12,"n_constructed":0,"n_ran_checked":12,"n_instrument":0,"n_unverified":8,"n_honours":0,"n_violates":0,"n_no_contract":12,"n_pointer_only":0,"phrase":"12 ran (of which 0 constructed an object rather than computing a result; 12 with no instrument failure: 0 honoured, 0 violated, 12 with no contract checked; 0 where Syntology's instrument failed) · 8 unverified","sample_list":"/paper/retrieval-augmented-generation-to-improve#ran","syntology_url":"https://syntology.ai/paper/2310.03184","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2310.03184"}},"official":{"repos":["digitalharborfoundation/rag-for-math-qa"],"state":"official (archive's flag): 7 ran","n_ran":7,"n_constructed":0,"n_ran_no_instrument_failure":7,"n_unverified":8,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/snip-bridging-mathematical-symbolic-and","slug":"snip-bridging-mathematical-symbolic-and","title":"SNIP: Bridging Mathematical Symbolic and Numeric Realms with Unified Pre-training","date":"2023-10-03","arxiv_id":"2310.02227","repositories_listed":2,"syntology":{"n":5,"n_ran":5,"n_constructed":0,"n_ran_checked":2,"n_instrument":3,"n_unverified":0,"n_honours":2,"n_violates":0,"n_no_contract":0,"n_pointer_only":1,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 2 honoured, 0 violated, 0 with no contract checked; 3 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/snip-bridging-mathematical-symbolic-and#ran","syntology_url":"https://syntology.ai/paper/2310.02227","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2310.02227"}},"official":{"repos":["deep-symbolic-mathematics/Multimodal-Math-Pretraining","deep-symbolic-mathematics/Multimodal-Symbolic-Regression"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["official","unlocated"]}}},{"url":"/paper/qwen-technical-report","slug":"qwen-technical-report","title":"Qwen Technical Report","date":"2023-09-28","arxiv_id":"2309.16609","repositories_listed":2,"syntology":{"n":4,"n_ran":2,"n_constructed":0,"n_ran_checked":0,"n_instrument":2,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/qwen-technical-report#ran","syntology_url":"https://syntology.ai/paper/2309.16609","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2309.16609"}},"official":{"repos":["QwenLM/Qwen-7B","qwenlm/qwen"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/reconcile-round-table-conference-improves","slug":"reconcile-round-table-conference-improves","title":"ReConcile: Round-Table Conference Improves Reasoning via Consensus among Diverse LLMs","date":"2023-09-22","arxiv_id":"2309.13007","repositories_listed":2,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":3,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/reconcile-round-table-conference-improves#ran","syntology_url":"https://syntology.ai/paper/2309.13007","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2309.13007"}},"official":{"repos":["dinobby/reconcile"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/an-empirical-study-on-challenging-math","slug":"an-empirical-study-on-challenging-math","title":"MathChat: Converse to Tackle Challenging Math Problems with LLM Agents","date":"2023-06-02","arxiv_id":"2306.01337","repositories_listed":2,"syntology":null},{"url":"/paper/creator-disentangling-abstract-and-concrete","slug":"creator-disentangling-abstract-and-concrete","title":"CREATOR: Tool Creation for Disentangling Abstract and Concrete Reasoning of Large Language Models","date":"2023-05-23","arxiv_id":"2305.14318","repositories_listed":2,"syntology":null},{"url":"/paper/reticl-sequential-retrieval-of-in-context","slug":"reticl-sequential-retrieval-of-in-context","title":"RetICL: Sequential Retrieval of In-Context Examples with Reinforcement Learning","date":"2023-05-23","arxiv_id":"2305.14502","repositories_listed":2,"syntology":null},{"url":"/paper/codet5-open-code-large-language-models-for","slug":"codet5-open-code-large-language-models-for","title":"CodeT5+: Open Code Large Language Models for Code Understanding and Generation","date":"2023-05-13","arxiv_id":"2305.07922","repositories_listed":2,"syntology":{"n":4,"n_ran":3,"n_constructed":0,"n_ran_checked":2,"n_instrument":1,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 1 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/codet5-open-code-large-language-models-for#ran","syntology_url":"https://syntology.ai/paper/2305.07922","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2305.07922"}},"official":{"repos":["salesforce/codet5"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":1,"ran_from_kinds":["official"]}}}],"syntology_records":69,"syntology_note":"a paper without a record is not a recorded non-run: it may lack an arXiv id or simply be absent from the graph layer"},"key_notes":{"n_ran_checked":"legacy name, kept unchanged so existing readers do not break: it counts the samples that ran with no instrument failure (honoured, violated, and ran with no contract checked); it does not mean a contract was checked, and the pages print it as 'K with no instrument failure', not 'K checked'","n_constructed":"a sub-count of the samples that ran, never subtracted from them and never a failure: an executed sample whose run returned an instance of its own class (fixture_out_type equals the entry name): the run built an object and did not compute a result (Syntology's RAN record, counts.constructed)"},"since_archive_judged":{"papers":0,"papers_placed":0,"judged_by":{"model":"a model","models_as_recorded":[],"from":"lb_rows_run_verified.jsonl: each record's `model` field (the extractor's call that wrote decisions[])"},"note":"a model's judgement from the paper's own tables that they report on this leaderboard, not a result: no value from a paper that was not placed is printed or carried here; papers counts each paper once over these leaderboards","backfill":"8,886 of 18,637 newer papers checked so far (733 more had nothing to place; 41 could not be read)","backfill_complete":false,"leaderboards":[]},"papers_where_code_ran":349,"papers_where_code_ran_first_page":"/task/math/papers/ran/1","papers_where_code_ran_caption":"We ran code from the paper's repository; we did not run it on this task or check it against the task's benchmarks.","pages":[{"order":"archive","page":2,"of":16,"url":"/task/math/papers/2","json":"/task/math/papers/2.json","rows":100},{"order":"archive","page":3,"of":16,"url":"/task/math/papers/3","json":"/task/math/papers/3.json","rows":100},{"order":"archive","page":4,"of":16,"url":"/task/math/papers/4","json":"/task/math/papers/4.json","rows":100},{"order":"archive","page":5,"of":16,"url":"/task/math/papers/5","json":"/task/math/papers/5.json","rows":100},{"order":"archive","page":6,"of":16,"url":"/task/math/papers/6","json":"/task/math/papers/6.json","rows":100},{"order":"archive","page":7,"of":16,"url":"/task/math/papers/7","json":"/task/math/papers/7.json","rows":100},{"order":"archive","page":8,"of":16,"url":"/task/math/papers/8","json":"/task/math/papers/8.json","rows":100},{"order":"archive","page":9,"of":16,"url":"/task/math/papers/9","json":"/task/math/papers/9.json","rows":100},{"order":"archive","page":10,"of":16,"url":"/task/math/papers/10","json":"/task/math/papers/10.json","rows":100},{"order":"archive","page":11,"of":16,"url":"/task/math/papers/11","json":"/task/math/papers/11.json","rows":100},{"order":"archive","page":12,"of":16,"url":"/task/math/papers/12","json":"/task/math/papers/12.json","rows":100},{"order":"archive","page":13,"of":16,"url":"/task/math/papers/13","json":"/task/math/papers/13.json","rows":100},{"order":"archive","page":14,"of":16,"url":"/task/math/papers/14","json":"/task/math/papers/14.json","rows":100},{"order":"archive","page":15,"of":16,"url":"/task/math/papers/15","json":"/task/math/papers/15.json","rows":100},{"order":"archive","page":16,"of":16,"url":"/task/math/papers/16","json":"/task/math/papers/16.json","rows":96},{"order":"ran","page":1,"of":4,"url":"/task/math/papers/ran/1","json":"/task/math/papers/ran/1.json","rows":100},{"order":"ran","page":2,"of":4,"url":"/task/math/papers/ran/2","json":"/task/math/papers/ran/2.json","rows":100},{"order":"ran","page":3,"of":4,"url":"/task/math/papers/ran/3","json":"/task/math/papers/ran/3.json","rows":100},{"order":"ran","page":4,"of":4,"url":"/task/math/papers/ran/4","json":"/task/math/papers/ran/4.json","rows":49}],"description_links":{"kept":0,"unwrapped_to_text":0,"bare_urls_linked":0,"relative_images_dropped":0,"rule":"internal links are kept only when the target slug exists in the catalog"},"syntology":{"read_at":"2026-10-05T00:36:11+00:00","claim":"Per-sample execution status on synthesized fixtures ('ran N of M samples'); not a correctness claim and not a ranking signal.","status_vocabulary":{"ran_honours":"ran, honoured the contract we drafted","ran_violates":"ran, violated the contract we drafted","ran_draft_wrong":"ran; our contract draft was wrong, not the code","ran_fixture":"ran; the contract check could not run (RAISES, DEP_MISSING or TIMEOUT)","ran":"ran on a synthesized input","unverified":"unverified (harvested, no recorded run)"},"row_route":"each syntology object on a benchmark's first row and on a paper carries sample_list (this site's paper page, #ran), syntology_url (syntology.ai's page for the paper, only when that route probed live) and mcp (the agent call that lists the paper's harvested samples; not a verdict on the paper)","developers":"https://syntology.ai/developers"},"not_shown":{"libraries":"the archive has no per-task library table","trend_sparklines":"the Trend column of the benchmarks table was a rendered image; it is not in the archive","social_and_latest_sorts":"stars and social signals are not in the archive"},"record_sha256":"3a77998e8b39fe9bfcf4cec9f49206ede88032d0851e3a871b39b42dccd33b18","record_changed_at":"2026-10-05","record_changed_at_basis":"changed"}