{"about":{"non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","site":"https://codewithpapers.app","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page","syntology":{"site":"https://syntology.ai","developers":"https://syntology.ai/developers","mcp":{"server":"https://syntology.ai/mcp","transport":"streamable-http","server_card":"https://syntology.ai/.well-known/mcp/server-card.json","auth":{"type":"trial token, no account","trial_token":"https://syntology.ai/api/oauth/trial/token","method":"POST","docs":"https://syntology.ai/developers"}},"have":"https://syntology.ai/api/graph/have?x=<method, arXiv id or title> (free, answers coverage only)","paper_base":"https://syntology.ai/paper/","atlas_base":"https://app.syntology.ai/?focus="},"machine_readable":[{"url":"https://codewithpapers.app/llms.txt","what":"the machine catalog: every machine-readable file, counted"},{"url":"https://codewithpapers.app/index/manifest.json","what":"paper-to-code index by arXiv id, with Syntology's counts"},{"url":"https://codewithpapers.app/search/manifest.json","what":"site search index (titles, authors) and its files"},{"url":"https://codewithpapers.app/download","what":"bulk files: Syntology's layer, described there"},{"url":"https://codewithpapers.app/build_manifest.json","what":"the build record: inputs, counts, exclusions, probes"}]},"url":"/task/humaneval","name":"HumanEval","slug":"humaneval","description_markdown":null,"categories":[],"source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","slug_source":"archive_url"},"counts":{"papers_tagged":264,"papers_with_code":135,"benchmarks":0,"benchmark_tables_in_archive":0,"benchmark_tables_shown":0,"benchmark_tables_withheld_as_spam":0,"benchmark_definition":"a leaderboard table with at least one row; benchmark_tables_shown also counts the zero-row tables; benchmark_tables_in_archive adds the tables withheld as spam","datasets":1,"subtasks":0,"parent_tasks":0},"benchmarks":[],"datasets":[{"url":"/dataset/humaneval","name":"HumanEval","full_name":"","num_papers_in_archive":1201}],"subtasks":[],"parent_tasks":[],"papers":{"order":"repositories listed in the archive (most first), then date (newest first), then slug; the archive holds no stars","population":"every paper the archive tags with this task (distinct papers; spam titles not listed). 'of' counts the papers listed; until 2026-09-25 it counted only the papers with a code link, and that number is of_with_code","page":1,"pages_in_order":3,"rows_per_page":100,"shown":100,"of":264,"of_with_code":135,"tagged_in_all":264,"counts":{"archive_papers_tagged":264,"with_a_code_link":135,"where_syntology_ran_a_sample":76,"not_listed_spam_title":0,"listed":264,"listed_where_code_ran":76,"where_syntology_ran_a_sample_split":{"with_a_run_with_no_instrument_failure":63,"every_run_a_failure_of_syntologys_instrument":13,"listed_with_a_run_with_no_instrument_failure":63,"listed_every_run_a_failure_of_syntologys_instrument":13,"filter":{"states":["a run with no instrument failure","any run, instrument failures included"],"default":"a run with no instrument failure","note":"on the 'only where code ran' pages the default hides, in the browser, the rows where every run was a failure of Syntology's instrument; the second state shows them again. Rows are hidden, never re-ordered; these twins list every row"}},"definition":"distinct papers the archive tags; 'where Syntology ran a sample' counts papers with at least one harvested sample that ran, which is not a correctness claim"},"items":[{"url":"/paper/evaluating-large-language-models-trained-on","slug":"evaluating-large-language-models-trained-on","title":"Evaluating Large Language Models Trained on Code","date":"2021-07-07","arxiv_id":"2107.03374","repositories_listed":13,"syntology":{"n":39,"n_ran":26,"n_constructed":0,"n_ran_checked":22,"n_instrument":4,"n_unverified":13,"n_honours":1,"n_violates":0,"n_no_contract":21,"n_pointer_only":4,"phrase":"26 ran (of which 0 constructed an object rather than computing a result; 22 with no instrument failure: 1 honoured, 0 violated, 21 with no contract checked; 4 where Syntology's instrument failed) · 13 unverified","sample_list":"/paper/evaluating-large-language-models-trained-on#ran","syntology_url":"https://syntology.ai/paper/2107.03374","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2107.03374"}},"official":{"repos":["openai/human-eval"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":3,"ran_from_kinds":["found_in_text","listed","official"]}}},{"url":"/paper/a-conversational-paradigm-for-program","slug":"a-conversational-paradigm-for-program","title":"CodeGen: An Open Large Language Model for Code with Multi-Turn Program Synthesis","date":"2022-03-25","arxiv_id":"2203.13474","repositories_listed":8,"syntology":{"n":10,"n_ran":10,"n_constructed":0,"n_ran_checked":3,"n_instrument":7,"n_unverified":0,"n_honours":1,"n_violates":1,"n_no_contract":1,"n_pointer_only":1,"phrase":"10 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 1 honoured, 1 violated, 1 with no contract checked; 7 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/a-conversational-paradigm-for-program#ran","syntology_url":"https://syntology.ai/paper/2203.13474","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2203.13474"}},"official":{"repos":["salesforce/CodeGen","salesforce/jaxformer"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/chatglm-a-family-of-large-language-models","slug":"chatglm-a-family-of-large-language-models","title":"ChatGLM: A Family of Large Language Models from GLM-130B to GLM-4 All Tools","date":"2024-06-18","arxiv_id":"2406.12793","repositories_listed":7,"syntology":{"n":29,"n_ran":21,"n_constructed":0,"n_ran_checked":15,"n_instrument":6,"n_unverified":8,"n_honours":1,"n_violates":0,"n_no_contract":14,"n_pointer_only":1,"phrase":"21 ran (of which 0 constructed an object rather than computing a result; 15 with no instrument failure: 1 honoured, 0 violated, 14 with no contract checked; 6 where Syntology's instrument failed) · 8 unverified","sample_list":"/paper/chatglm-a-family-of-large-language-models#ran","syntology_url":"https://syntology.ai/paper/2406.12793","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.12793"}},"official":{"repos":["thudm/chatglm-6b"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":1,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/qwen2-technical-report","slug":"qwen2-technical-report","title":"Qwen2 Technical Report","date":"2024-07-15","arxiv_id":"2407.10671","repositories_listed":6,"syntology":null},{"url":"/paper/reflexion-language-agents-with-verbal","slug":"reflexion-language-agents-with-verbal","title":"Reflexion: Language Agents with Verbal Reinforcement Learning","date":"2023-03-20","arxiv_id":"2303.11366","repositories_listed":5,"syntology":{"n":9,"n_ran":8,"n_constructed":0,"n_ran_checked":5,"n_instrument":3,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":5,"n_pointer_only":1,"phrase":"8 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 0 honoured, 0 violated, 5 with no contract checked; 3 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/reflexion-language-agents-with-verbal#ran","syntology_url":"https://syntology.ai/paper/2303.11366","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2303.11366"}},"official":{"repos":["noahshinn024/reflexion"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/wizardcoder-empowering-code-large-language","slug":"wizardcoder-empowering-code-large-language","title":"WizardCoder: Empowering Code Large Language Models with Evol-Instruct","date":"2023-06-14","arxiv_id":"2306.08568","repositories_listed":4,"syntology":{"n":7,"n_ran":6,"n_constructed":0,"n_ran_checked":4,"n_instrument":2,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":4,"n_pointer_only":2,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 4 with no instrument failure: 0 honoured, 0 violated, 4 with no contract checked; 2 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/wizardcoder-empowering-code-large-language#ran","syntology_url":"https://syntology.ai/paper/2306.08568","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2306.08568"}},"official":{"repos":["nlpxucan/wizardlm"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/starcoder-may-the-source-be-with-you","slug":"starcoder-may-the-source-be-with-you","title":"StarCoder: may the source be with you!","date":"2023-05-09","arxiv_id":"2305.06161","repositories_listed":4,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":1,"n_instrument":1,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/starcoder-may-the-source-be-with-you#ran","syntology_url":"https://syntology.ai/paper/2305.06161","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2305.06161"}},"official":null}},{"url":"/paper/soap-enhancing-efficiency-of-generated-code","slug":"soap-enhancing-efficiency-of-generated-code","title":"EffiLearner: Enhancing Efficiency of Generated Code via Self-Optimization","date":"2024-05-24","arxiv_id":"2405.15189","repositories_listed":3,"syntology":{"n":17,"n_ran":12,"n_constructed":0,"n_ran_checked":8,"n_instrument":4,"n_unverified":5,"n_honours":6,"n_violates":0,"n_no_contract":2,"n_pointer_only":0,"phrase":"12 ran (of which 0 constructed an object rather than computing a result; 8 with no instrument failure: 6 honoured, 0 violated, 2 with no contract checked; 4 where Syntology's instrument failed) · 5 unverified","sample_list":"/paper/soap-enhancing-efficiency-of-generated-code#ran","syntology_url":"https://syntology.ai/paper/2405.15189","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2405.15189"}},"official":{"repos":["huangd1999/effilearner","huangd1999/soap"],"state":"official (archive's flag): 12 ran","n_ran":12,"n_constructed":0,"n_ran_no_instrument_failure":8,"n_unverified":5,"ran_from_kinds":["official"]}}},{"url":"/paper/rlhf-workflow-from-reward-modeling-to-online","slug":"rlhf-workflow-from-reward-modeling-to-online","title":"RLHF Workflow: From Reward Modeling to Online RLHF","date":"2024-05-13","arxiv_id":"2405.07863","repositories_listed":3,"syntology":{"n":1,"n_ran":0,"n_constructed":0,"n_ran_checked":0,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":1,"phrase":"0 ran · 1 unverified","sample_list":"/paper/rlhf-workflow-from-reward-modeling-to-online#ran","syntology_url":"https://syntology.ai/paper/2405.07863","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2405.07863"}},"official":{"repos":["RLHFlow/RLHF-Reward-Modeling","rlhflow/online-rlhf"],"state":"official: harvested, nothing ran","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":1,"ran_from_kinds":[]}}},{"url":"/paper/magicoder-source-code-is-all-you-need","slug":"magicoder-source-code-is-all-you-need","title":"Magicoder: Empowering Code Generation with OSS-Instruct","date":"2023-12-04","arxiv_id":"2312.02120","repositories_listed":3,"syntology":{"n":2,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/magicoder-source-code-is-all-you-need#ran","syntology_url":"https://syntology.ai/paper/2312.02120","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2312.02120"}},"official":{"repos":["ise-uiuc/magicoder"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/octopack-instruction-tuning-code-large","slug":"octopack-instruction-tuning-code-large","title":"OctoPack: Instruction Tuning Code Large Language Models","date":"2023-08-14","arxiv_id":"2308.07124","repositories_listed":3,"syntology":{"n":13,"n_ran":9,"n_constructed":0,"n_ran_checked":3,"n_instrument":6,"n_unverified":4,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":0,"phrase":"9 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 6 where Syntology's instrument failed) · 4 unverified","sample_list":"/paper/octopack-instruction-tuning-code-large#ran","syntology_url":"https://syntology.ai/paper/2308.07124","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2308.07124"}},"official":{"repos":["bigcode-project/bigcode-evaluation-harness","bigcode-project/octopack"],"state":"official (archive's flag): 9 ran","n_ran":9,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":4,"ran_from_kinds":["official"]}}},{"url":"/paper/recode-robustness-evaluation-of-code","slug":"recode-robustness-evaluation-of-code","title":"ReCode: Robustness Evaluation of Code Generation Models","date":"2022-12-20","arxiv_id":"2212.10264","repositories_listed":3,"syntology":{"n":5,"n_ran":5,"n_constructed":0,"n_ran_checked":3,"n_instrument":2,"n_unverified":0,"n_honours":3,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 3 honoured, 0 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/recode-robustness-evaluation-of-code#ran","syntology_url":"https://syntology.ai/paper/2212.10264","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2212.10264"}},"official":{"repos":["amazon-science/recode"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":0,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/selfcodealign-self-alignment-for-code","slug":"selfcodealign-self-alignment-for-code","title":"SelfCodeAlign: Self-Alignment for Code Generation","date":"2024-10-31","arxiv_id":"2410.24198","repositories_listed":2,"syntology":{"n":37,"n_ran":30,"n_constructed":3,"n_ran_checked":21,"n_instrument":9,"n_unverified":7,"n_honours":1,"n_violates":0,"n_no_contract":20,"n_pointer_only":0,"phrase":"30 ran (of which 3 constructed an object rather than computing a result; 21 with no instrument failure: 1 honoured, 0 violated, 20 with no contract checked; 9 where Syntology's instrument failed) · 7 unverified","sample_list":"/paper/selfcodealign-self-alignment-for-code#ran","syntology_url":"https://syntology.ai/paper/2410.24198","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2410.24198"}},"official":{"repos":["bigcode-project/selfcodealign"],"state":"official (archive's flag): 9 ran","n_ran":9,"n_constructed":3,"n_ran_no_instrument_failure":3,"n_unverified":3,"ran_from_kinds":["community","listed","official"]}}},{"url":"/paper/training-language-models-to-self-correct-via","slug":"training-language-models-to-self-correct-via","title":"Training Language Models to Self-Correct via Reinforcement Learning","date":"2024-09-19","arxiv_id":"2409.12917","repositories_listed":2,"syntology":null},{"url":"/paper/planning-in-natural-language-improves-llm","slug":"planning-in-natural-language-improves-llm","title":"Planning In Natural Language Improves LLM Search For Code Generation","date":"2024-09-05","arxiv_id":"2409.03733","repositories_listed":2,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":1,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/planning-in-natural-language-improves-llm#ran","syntology_url":"https://syntology.ai/paper/2409.03733","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2409.03733"}},"official":{"repos":["scaleapi/plansearch"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/a-survey-on-large-language-models-for-code","slug":"a-survey-on-large-language-models-for-code","title":"A Survey on Large Language Models for Code Generation","date":"2024-06-01","arxiv_id":"2406.00515","repositories_listed":2,"syntology":{"n":1,"n_ran":0,"n_constructed":0,"n_ran_checked":0,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"0 ran · 1 unverified","sample_list":"/paper/a-survey-on-large-language-models-for-code#ran","syntology_url":"https://syntology.ai/paper/2406.00515","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.00515"}},"official":{"repos":["juyongjiang/codellmsurvey"],"state":"official: not harvested","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":[]}}},{"url":"/paper/mapcoder-multi-agent-code-generation-for","slug":"mapcoder-multi-agent-code-generation-for","title":"MapCoder: Multi-Agent Code Generation for Competitive Problem Solving","date":"2024-05-18","arxiv_id":"2405.11403","repositories_listed":2,"syntology":{"n":10,"n_ran":6,"n_constructed":0,"n_ran_checked":6,"n_instrument":0,"n_unverified":4,"n_honours":0,"n_violates":0,"n_no_contract":6,"n_pointer_only":0,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 6 with no instrument failure: 0 honoured, 0 violated, 6 with no contract checked; 0 where Syntology's instrument failed) · 4 unverified","sample_list":"/paper/mapcoder-multi-agent-code-generation-for#ran","syntology_url":"https://syntology.ai/paper/2405.11403","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2405.11403"}},"official":{"repos":["md-ashraful-pramanik/mapcoder"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/codeultrafeedback-an-llm-as-a-judge-dataset","slug":"codeultrafeedback-an-llm-as-a-judge-dataset","title":"CodeUltraFeedback: An LLM-as-a-Judge Dataset for Aligning Large Language Models to Coding Preferences","date":"2024-03-14","arxiv_id":"2403.09032","repositories_listed":2,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/codeultrafeedback-an-llm-as-a-judge-dataset#ran","syntology_url":"https://syntology.ai/paper/2403.09032","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2403.09032"}},"official":{"repos":["martin-wey/codeultrafeedback"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/inficoder-eval-systematically-evaluating-the","slug":"inficoder-eval-systematically-evaluating-the","title":"InfiBench: Evaluating the Question-Answering Capabilities of Code Large Language Models","date":"2024-03-11","arxiv_id":"2404.07940","repositories_listed":2,"syntology":{"n":13,"n_ran":11,"n_constructed":0,"n_ran_checked":8,"n_instrument":3,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":8,"n_pointer_only":0,"phrase":"11 ran (of which 0 constructed an object rather than computing a result; 8 with no instrument failure: 0 honoured, 0 violated, 8 with no contract checked; 3 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/inficoder-eval-systematically-evaluating-the#ran","syntology_url":"https://syntology.ai/paper/2404.07940","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2404.07940"}},"official":{"repos":["infi-coder/infibench-evaluation-harness","infi-coder/infibench-evaluator"],"state":"official (archive's flag): 11 ran","n_ran":11,"n_constructed":0,"n_ran_no_instrument_failure":8,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/generation-meets-verification-accelerating","slug":"generation-meets-verification-accelerating","title":"Generation Meets Verification: Accelerating Large Language Model Inference with Smart Parallel Auto-Correct Decoding","date":"2024-02-19","arxiv_id":"2402.11809","repositories_listed":2,"syntology":{"n":11,"n_ran":9,"n_constructed":0,"n_ran_checked":2,"n_instrument":7,"n_unverified":2,"n_honours":1,"n_violates":0,"n_no_contract":1,"n_pointer_only":4,"phrase":"9 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 1 honoured, 0 violated, 1 with no contract checked; 7 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/generation-meets-verification-accelerating#ran","syntology_url":"https://syntology.ai/paper/2402.11809","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2402.11809"}},"official":{"repos":["cteant/space","hiyouga/llama-factory"],"state":"official (archive's flag): 9 ran","n_ran":9,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/unsupervised-evaluation-of-code-llms-with","slug":"unsupervised-evaluation-of-code-llms-with","title":"Unsupervised Evaluation of Code LLMs with Round-Trip Correctness","date":"2024-02-13","arxiv_id":"2402.08699","repositories_listed":2,"syntology":{"n":4,"n_ran":3,"n_constructed":0,"n_ran_checked":1,"n_instrument":2,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 2 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/unsupervised-evaluation-of-code-llms-with#ran","syntology_url":"https://syntology.ai/paper/2402.08699","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2402.08699"}},"official":null}},{"url":"/paper/language-agent-tree-search-unifies-reasoning","slug":"language-agent-tree-search-unifies-reasoning","title":"Language Agent Tree Search Unifies Reasoning Acting and Planning in Language Models","date":"2023-10-06","arxiv_id":"2310.04406","repositories_listed":2,"syntology":{"n":9,"n_ran":9,"n_constructed":0,"n_ran_checked":5,"n_instrument":4,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":4,"n_pointer_only":2,"phrase":"9 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 1 honoured, 0 violated, 4 with no contract checked; 4 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/language-agent-tree-search-unifies-reasoning#ran","syntology_url":"https://syntology.ai/paper/2310.04406","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2310.04406"}},"official":{"repos":["lapisrocks/languageagenttreesearch","andyz245/LanguageAgentTreeSearch"],"state":"official (archive's flag): 9 ran","n_ran":9,"n_constructed":0,"n_ran_no_instrument_failure":5,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/baichuan-2-open-large-scale-language-models","slug":"baichuan-2-open-large-scale-language-models","title":"Baichuan 2: Open Large-scale Language Models","date":"2023-09-19","arxiv_id":"2309.10305","repositories_listed":2,"syntology":null},{"url":"/paper/code-llama-open-foundation-models-for-code","slug":"code-llama-open-foundation-models-for-code","title":"Code Llama: Open Foundation Models for Code","date":"2023-08-24","arxiv_id":"2308.12950","repositories_listed":2,"syntology":{"n":5,"n_ran":3,"n_constructed":0,"n_ran_checked":1,"n_instrument":2,"n_unverified":2,"n_honours":0,"n_violates":1,"n_no_contract":0,"n_pointer_only":4,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 1 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/code-llama-open-foundation-models-for-code#ran","syntology_url":"https://syntology.ai/paper/2308.12950","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2308.12950"}},"official":{"repos":["facebookresearch/codellama"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/classeval-a-manually-crafted-benchmark-for","slug":"classeval-a-manually-crafted-benchmark-for","title":"ClassEval: A Manually-Crafted Benchmark for Evaluating LLMs on Class-level Code Generation","date":"2023-08-03","arxiv_id":"2308.01861","repositories_listed":2,"syntology":null},{"url":"/paper/codet5-open-code-large-language-models-for","slug":"codet5-open-code-large-language-models-for","title":"CodeT5+: Open Code Large Language Models for Code Understanding and Generation","date":"2023-05-13","arxiv_id":"2305.07922","repositories_listed":2,"syntology":{"n":4,"n_ran":3,"n_constructed":0,"n_ran_checked":2,"n_instrument":1,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 1 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/codet5-open-code-large-language-models-for#ran","syntology_url":"https://syntology.ai/paper/2305.07922","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2305.07922"}},"official":{"repos":["salesforce/codet5"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/codegeex-a-pre-trained-model-for-code","slug":"codegeex-a-pre-trained-model-for-code","title":"CodeGeeX: A Pre-Trained Model for Code Generation with Multilingual Benchmarking on HumanEval-X","date":"2023-03-30","arxiv_id":"2303.17568","repositories_listed":2,"syntology":{"n":8,"n_ran":6,"n_constructed":0,"n_ran_checked":6,"n_instrument":0,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":6,"n_pointer_only":1,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 6 with no instrument failure: 0 honoured, 0 violated, 6 with no contract checked; 0 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/codegeex-a-pre-trained-model-for-code#ran","syntology_url":"https://syntology.ai/paper/2303.17568","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2303.17568"}},"official":{"repos":["THUDM/CodeGeeX"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/multi-lingual-evaluation-of-code-generation","slug":"multi-lingual-evaluation-of-code-generation","title":"Multi-lingual Evaluation of Code Generation Models","date":"2022-10-26","arxiv_id":"2210.14868","repositories_listed":2,"syntology":{"n":6,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":5,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 5 unverified","sample_list":"/paper/multi-lingual-evaluation-of-code-generation#ran","syntology_url":"https://syntology.ai/paper/2210.14868","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2210.14868"}},"official":{"repos":["amazon-research/mbxp-exec-eval"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":3,"ran_from_kinds":["official"]}}},{"url":"/paper/rethinking-verification-for-llm-code","slug":"rethinking-verification-for-llm-code","title":"Rethinking Verification for LLM Code Generation: From Generation to Testing","date":"2025-07-09","arxiv_id":"2507.06920","repositories_listed":1,"syntology":null},{"url":"/paper/any4-learned-4-bit-numeric-representation-for","slug":"any4-learned-4-bit-numeric-representation-for","title":"any4: Learned 4-bit Numeric Representation for LLMs","date":"2025-07-07","arxiv_id":"2507.04610","repositories_listed":1,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":1,"n_instrument":1,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":2,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/any4-learned-4-bit-numeric-representation-for#ran","syntology_url":"https://syntology.ai/paper/2507.04610","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2507.04610"}},"official":{"repos":["facebookresearch/any4"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/agentgroupchat-v2-divide-and-conquer-is-what","slug":"agentgroupchat-v2-divide-and-conquer-is-what","title":"AgentGroupChat-V2: Divide-and-Conquer Is What LLM-Based Multi-Agent System Need","date":"2025-06-18","arxiv_id":"2506.15451","repositories_listed":1,"syntology":null},{"url":"/paper/self-correcting-code-generation-using-small","slug":"self-correcting-code-generation-using-small","title":"Self-Correcting Code Generation Using Small Language Models","date":"2025-05-29","arxiv_id":"2505.23060","repositories_listed":1,"syntology":null},{"url":"/paper/invisible-entropy-towards-safe-and-efficient","slug":"invisible-entropy-towards-safe-and-efficient","title":"Invisible Entropy: Towards Safe and Efficient Low-Entropy LLM Watermarking","date":"2025-05-20","arxiv_id":"2505.14112","repositories_listed":1,"syntology":null},{"url":"/paper/warm-up-before-you-train-unlocking-general","slug":"warm-up-before-you-train-unlocking-general","title":"Warm Up Before You Train: Unlocking General Reasoning in Resource-Constrained Settings","date":"2025-05-19","arxiv_id":"2505.13718","repositories_listed":1,"syntology":null},{"url":"/paper/halo-hierarchical-autonomous-logic-oriented","slug":"halo-hierarchical-autonomous-logic-oriented","title":"HALO: Hierarchical Autonomous Logic-Oriented Orchestration for Multi-Agent LLM Systems","date":"2025-05-17","arxiv_id":"2505.13516","repositories_listed":1,"syntology":null},{"url":"/paper/rethinking-repetition-problems-of-llms-in","slug":"rethinking-repetition-problems-of-llms-in","title":"Rethinking Repetition Problems of LLMs in Code Generation","date":"2025-05-15","arxiv_id":"2505.10402","repositories_listed":1,"syntology":null},{"url":"/paper/enhancing-code-generation-via-bidirectional","slug":"enhancing-code-generation-via-bidirectional","title":"Enhancing Code Generation via Bidirectional Comment-Level Mutual Grounding","date":"2025-05-12","arxiv_id":"2505.07768","repositories_listed":1,"syntology":null},{"url":"/paper/web-bench-a-llm-code-benchmark-based-on-web","slug":"web-bench-a-llm-code-benchmark-based-on-web","title":"Web-Bench: A LLM Code Benchmark Based on Web Standards and Frameworks","date":"2025-05-12","arxiv_id":"2505.07473","repositories_listed":1,"syntology":null},{"url":"/paper/rewriting-pre-training-data-boosts-llm","slug":"rewriting-pre-training-data-boosts-llm","title":"Rewriting Pre-Training Data Boosts LLM Performance in Math and Code","date":"2025-05-05","arxiv_id":"2505.02881","repositories_listed":1,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":0,"n_instrument":2,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":2,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/rewriting-pre-training-data-boosts-llm#ran","syntology_url":"https://syntology.ai/paper/2505.02881","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2505.02881"}},"official":{"repos":["rioyokotalab/swallow-code-math"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/datadecide-how-to-predict-best-pretraining","slug":"datadecide-how-to-predict-best-pretraining","title":"DataDecide: How to Predict Best Pretraining Data with Small Experiments","date":"2025-04-15","arxiv_id":"2504.11393","repositories_listed":1,"syntology":{"n":14,"n_ran":11,"n_constructed":0,"n_ran_checked":10,"n_instrument":1,"n_unverified":3,"n_honours":1,"n_violates":1,"n_no_contract":8,"n_pointer_only":1,"phrase":"11 ran (of which 0 constructed an object rather than computing a result; 10 with no instrument failure: 1 honoured, 1 violated, 8 with no contract checked; 1 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/datadecide-how-to-predict-best-pretraining#ran","syntology_url":"https://syntology.ai/paper/2504.11393","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2504.11393"}},"official":null}},{"url":"/paper/repost-scalable-repository-level-coding","slug":"repost-scalable-repository-level-coding","title":"RepoST: Scalable Repository-Level Coding Environment Construction with Sandbox Testing","date":"2025-03-10","arxiv_id":"2503.07358","repositories_listed":1,"syntology":{"n":18,"n_ran":11,"n_constructed":0,"n_ran_checked":11,"n_instrument":0,"n_unverified":7,"n_honours":0,"n_violates":0,"n_no_contract":11,"n_pointer_only":0,"phrase":"11 ran (of which 0 constructed an object rather than computing a result; 11 with no instrument failure: 0 honoured, 0 violated, 11 with no contract checked; 0 where Syntology's instrument failed) · 7 unverified","sample_list":"/paper/repost-scalable-repository-level-coding#ran","syntology_url":"https://syntology.ai/paper/2503.07358","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2503.07358"}},"official":{"repos":["yiqingxyq/RepoST"],"state":"official (archive's flag): 11 ran","n_ran":11,"n_constructed":0,"n_ran_no_instrument_failure":11,"n_unverified":7,"ran_from_kinds":["official"]}}},{"url":"/paper/throwbench-benchmarking-llms-by-predicting","slug":"throwbench-benchmarking-llms-by-predicting","title":"ThrowBench: Benchmarking LLMs by Predicting Runtime Exceptions","date":"2025-03-06","arxiv_id":"2503.04241","repositories_listed":1,"syntology":null},{"url":"/paper/kodcode-a-diverse-challenging-and-verifiable","slug":"kodcode-a-diverse-challenging-and-verifiable","title":"KodCode: A Diverse, Challenging, and Verifiable Synthetic Dataset for Coding","date":"2025-03-04","arxiv_id":"2503.02951","repositories_listed":1,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/kodcode-a-diverse-challenging-and-verifiable#ran","syntology_url":"https://syntology.ai/paper/2503.02951","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2503.02951"}},"official":{"repos":["KodCode-AI/kodcode"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/nexus-a-lightweight-and-scalable-multi-agent","slug":"nexus-a-lightweight-and-scalable-multi-agent","title":"Nexus: A Lightweight and Scalable Multi-Agent Framework for Complex Tasks Automation","date":"2025-02-26","arxiv_id":"2502.19091","repositories_listed":1,"syntology":null},{"url":"/paper/codecriticbench-a-holistic-code-critique","slug":"codecriticbench-a-holistic-code-critique","title":"CodeCriticBench: A Holistic Code Critique Benchmark for Large Language Models","date":"2025-02-23","arxiv_id":"2502.16614","repositories_listed":1,"syntology":null},{"url":"/paper/code-vision-evaluating-multimodal-llms-logic","slug":"code-vision-evaluating-multimodal-llms-logic","title":"Code-Vision: Evaluating Multimodal LLMs Logic Understanding and Code Generation Capabilities","date":"2025-02-17","arxiv_id":"2502.11829","repositories_listed":1,"syntology":null},{"url":"/paper/masrouter-learning-to-route-llms-for-multi","slug":"masrouter-learning-to-route-llms-for-multi","title":"MasRouter: Learning to Route LLMs for Multi-Agent Systems","date":"2025-02-16","arxiv_id":"2502.11133","repositories_listed":1,"syntology":{"n":13,"n_ran":13,"n_constructed":0,"n_ran_checked":10,"n_instrument":3,"n_unverified":0,"n_honours":1,"n_violates":1,"n_no_contract":8,"n_pointer_only":4,"phrase":"13 ran (of which 0 constructed an object rather than computing a result; 10 with no instrument failure: 1 honoured, 1 violated, 8 with no contract checked; 3 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/masrouter-learning-to-route-llms-for-multi#ran","syntology_url":"https://syntology.ai/paper/2502.11133","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2502.11133"}},"official":{"repos":["yanweiyue/masrouter"],"state":"official (archive's flag): 13 ran","n_ran":13,"n_constructed":0,"n_ran_no_instrument_failure":10,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/copyspec-accelerating-llms-with-speculative","slug":"copyspec-accelerating-llms-with-speculative","title":"CopySpec: Accelerating LLMs with Speculative Copy-and-Paste Without Compromising Quality","date":"2025-02-13","arxiv_id":"2502.08923","repositories_listed":1,"syntology":null},{"url":"/paper/codesim-multi-agent-code-generation-and-1","slug":"codesim-multi-agent-code-generation-and-1","title":"CODESIM: Multi-Agent Code Generation and Problem Solving through Simulation-Driven Planning and Debugging","date":"2025-02-08","arxiv_id":"2502.05664","repositories_listed":1,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":3,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/codesim-multi-agent-code-generation-and-1#ran","syntology_url":"https://syntology.ai/paper/2502.05664","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2502.05664"}},"official":null}},{"url":"/paper/learning-to-generate-unit-tests-for-automated","slug":"learning-to-generate-unit-tests-for-automated","title":"Learning to Generate Unit Tests for Automated Debugging","date":"2025-02-03","arxiv_id":"2502.01619","repositories_listed":1,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":1,"n_instrument":2,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/learning-to-generate-unit-tests-for-automated#ran","syntology_url":"https://syntology.ai/paper/2502.01619","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2502.01619"}},"official":{"repos":["archiki/utgendebug"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/how-to-select-datapoints-for-efficient-human","slug":"how-to-select-datapoints-for-efficient-human","title":"How to Select Datapoints for Efficient Human Evaluation of NLG Models?","date":"2025-01-30","arxiv_id":"2501.18251","repositories_listed":1,"syntology":null},{"url":"/paper/coconut-structural-code-understanding-does","slug":"coconut-structural-code-understanding-does","title":"CoCoNUT: Structural Code Understanding does not fall out of a tree","date":"2025-01-27","arxiv_id":"2501.16456","repositories_listed":1,"syntology":null},{"url":"/paper/mygo-multiplex-cot-a-method-for-self","slug":"mygo-multiplex-cot-a-method-for-self","title":"MyGO Multiplex CoT: A Method for Self-Reflection in Large Language Models via Double Chain of Thought Thinking","date":"2025-01-20","arxiv_id":"2501.13117","repositories_listed":1,"syntology":null},{"url":"/paper/humaneval-pro-and-mbpp-pro-evaluating-large","slug":"humaneval-pro-and-mbpp-pro-evaluating-large","title":"HumanEval Pro and MBPP Pro: Evaluating Large Language Models on Self-invoking Code Generation","date":"2024-12-30","arxiv_id":"2412.21199","repositories_listed":1,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":0,"n_instrument":3,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":3,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 3 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/humaneval-pro-and-mbpp-pro-evaluating-large#ran","syntology_url":"https://syntology.ai/paper/2412.21199","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2412.21199"}},"official":{"repos":["CodeEval-Pro/CodeEval-Pro"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/inference-scaling-scriptsize-mathtt-f-laws","slug":"inference-scaling-scriptsize-mathtt-f-laws","title":"Inference Scaling fLaws: The Limits of LLM Resampling with Imperfect Verifiers","date":"2024-11-26","arxiv_id":"2411.17501","repositories_listed":1,"syntology":{"n":13,"n_ran":11,"n_constructed":0,"n_ran_checked":3,"n_instrument":8,"n_unverified":2,"n_honours":0,"n_violates":1,"n_no_contract":2,"n_pointer_only":0,"phrase":"11 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 1 violated, 2 with no contract checked; 8 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/inference-scaling-scriptsize-mathtt-f-laws#ran","syntology_url":"https://syntology.ai/paper/2411.17501","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2411.17501"}},"official":{"repos":["benediktstroebl/inference-scaling-limits"],"state":"official (archive's flag): 11 ran","n_ran":11,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/planning-driven-programming-a-large-language","slug":"planning-driven-programming-a-large-language","title":"Planning-Driven Programming: A Large Language Model Programming Workflow","date":"2024-11-21","arxiv_id":"2411.14503","repositories_listed":1,"syntology":{"n":9,"n_ran":9,"n_constructed":0,"n_ran_checked":4,"n_instrument":5,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":4,"n_pointer_only":0,"phrase":"9 ran (of which 0 constructed an object rather than computing a result; 4 with no instrument failure: 0 honoured, 0 violated, 4 with no contract checked; 5 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/planning-driven-programming-a-large-language#ran","syntology_url":"https://syntology.ai/paper/2411.14503","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2411.14503"}},"official":{"repos":["you68681/lpw"],"state":"official (archive's flag): 9 ran","n_ran":9,"n_constructed":0,"n_ran_no_instrument_failure":4,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/perfcodegen-improving-performance-of-llm","slug":"perfcodegen-improving-performance-of-llm","title":"PerfCodeGen: Improving Performance of LLM Generated Code with Execution Feedback","date":"2024-11-18","arxiv_id":"2412.03578","repositories_listed":1,"syntology":{"n":3,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":2,"n_honours":0,"n_violates":1,"n_no_contract":0,"n_pointer_only":1,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 1 violated, 0 with no contract checked; 0 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/perfcodegen-improving-performance-of-llm#ran","syntology_url":"https://syntology.ai/paper/2412.03578","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2412.03578"}},"official":{"repos":["SalesforceAIResearch/perfcodegen"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/intertrans-leveraging-transitive-intermediate","slug":"intertrans-leveraging-transitive-intermediate","title":"InterTrans: Leveraging Transitive Intermediate Translations to Enhance LLM-based Code Translation","date":"2024-11-01","arxiv_id":"2411.01063","repositories_listed":1,"syntology":null},{"url":"/paper/can-language-models-replace-programmers","slug":"can-language-models-replace-programmers","title":"Can Language Models Replace Programmers for Coding? REPOCOD Says 'Not Yet'","date":"2024-10-29","arxiv_id":"2410.21647","repositories_listed":1,"syntology":null},{"url":"/paper/falcon-feedback-driven-adaptive-long-short","slug":"falcon-feedback-driven-adaptive-long-short","title":"FALCON: Feedback-driven Adaptive Long/short-term memory reinforced Coding Optimization system","date":"2024-10-28","arxiv_id":"2410.21349","repositories_listed":1,"syntology":null},{"url":"/paper/mhumaneval-a-multilingual-benchmark-to","slug":"mhumaneval-a-multilingual-benchmark-to","title":"mHumanEval -- A Multilingual Benchmark to Evaluate Large Language Models for Code Generation","date":"2024-10-19","arxiv_id":"2410.15037","repositories_listed":1,"syntology":null},{"url":"/paper/humaneval-v-evaluating-visual-understanding","slug":"humaneval-v-evaluating-visual-understanding","title":"HumanEval-V: Evaluating Visual Understanding and Reasoning Abilities of Large Multimodal Models Through Coding Tasks","date":"2024-10-16","arxiv_id":"2410.12381","repositories_listed":1,"syntology":{"n":6,"n_ran":6,"n_constructed":0,"n_ran_checked":2,"n_instrument":4,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":6,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 4 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/humaneval-v-evaluating-visual-understanding#ran","syntology_url":"https://syntology.ai/paper/2410.12381","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2410.12381"}},"official":{"repos":["HumanEval-V/HumanEval-V-Benchmark"],"state":"official (archive's flag): 6 ran","n_ran":6,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/one-language-many-gaps-evaluating-dialect","slug":"one-language-many-gaps-evaluating-dialect","title":"One Language, Many Gaps: Evaluating Dialect Fairness and Robustness of Large Language Models in Reasoning Tasks","date":"2024-10-14","arxiv_id":"2410.11005","repositories_listed":1,"syntology":{"n":7,"n_ran":4,"n_constructed":0,"n_ran_checked":2,"n_instrument":2,"n_unverified":3,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":0,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 2 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/one-language-many-gaps-evaluating-dialect#ran","syntology_url":"https://syntology.ai/paper/2410.11005","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2410.11005"}},"official":{"repos":["fangru-lin/redial_dialect_robustness_fairness"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":3,"ran_from_kinds":["official"]}}},{"url":"/paper/kv-prediction-for-improved-time-to-first","slug":"kv-prediction-for-improved-time-to-first","title":"KV Prediction for Improved Time to First Token","date":"2024-10-10","arxiv_id":"2410.08391","repositories_listed":1,"syntology":null},{"url":"/paper/training-language-models-on-synthetic-edit","slug":"training-language-models-on-synthetic-edit","title":"Training Language Models on Synthetic Edit Sequences Improves Code Synthesis","date":"2024-10-03","arxiv_id":"2410.02749","repositories_listed":1,"syntology":{"n":1,"n_ran":0,"n_constructed":0,"n_ran_checked":0,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"0 ran · 1 unverified","sample_list":"/paper/training-language-models-on-synthetic-edit#ran","syntology_url":"https://syntology.ai/paper/2410.02749","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2410.02749"}},"official":{"repos":["upiterbarg/lintseq"],"state":"official: harvested, nothing ran","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":1,"ran_from_kinds":[]}}},{"url":"/paper/from-code-to-correctness-closing-the-last","slug":"from-code-to-correctness-closing-the-last","title":"From Code to Correctness: Closing the Last Mile of Code Generation with Hierarchical Debugging","date":"2024-10-02","arxiv_id":"2410.01215","repositories_listed":1,"syntology":{"n":9,"n_ran":8,"n_constructed":0,"n_ran_checked":5,"n_instrument":3,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":5,"n_pointer_only":0,"phrase":"8 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 0 honoured, 0 violated, 5 with no contract checked; 3 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/from-code-to-correctness-closing-the-last#ran","syntology_url":"https://syntology.ai/paper/2410.01215","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2410.01215"}},"official":{"repos":["YerbaPage/MGDebugger"],"state":"official (archive's flag): 8 ran","n_ran":8,"n_constructed":0,"n_ran_no_instrument_failure":5,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/rgd-multi-llm-based-agent-debugger-via","slug":"rgd-multi-llm-based-agent-debugger-via","title":"RGD: Multi-LLM Based Agent Debugger via Refinement and Generation Guidance","date":"2024-10-02","arxiv_id":"2410.01242","repositories_listed":1,"syntology":null},{"url":"/paper/amr-evol-adaptive-modular-response-evolution","slug":"amr-evol-adaptive-modular-response-evolution","title":"AMR-Evol: Adaptive Modular Response Evolution Elicits Better Knowledge Distillation for Large Language Models in Code Generation","date":"2024-10-01","arxiv_id":"2410.00558","repositories_listed":1,"syntology":null},{"url":"/paper/rethinking-the-influence-of-source-code-on","slug":"rethinking-the-influence-of-source-code-on","title":"Measuring the Influence of Incorrect Code on Test Generation","date":"2024-09-14","arxiv_id":"2409.09464","repositories_listed":1,"syntology":null},{"url":"/paper/policy-filtration-in-rlhf-to-fine-tune-llm","slug":"policy-filtration-in-rlhf-to-fine-tune-llm","title":"Policy Filtration in RLHF to Fine-Tune LLM for Code Generation","date":"2024-09-11","arxiv_id":"2409.06957","repositories_listed":1,"syntology":{"n":1,"n_ran":0,"n_constructed":0,"n_ran_checked":0,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"0 ran · 1 unverified","sample_list":"/paper/policy-filtration-in-rlhf-to-fine-tune-llm#ran","syntology_url":"https://syntology.ai/paper/2409.06957","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2409.06957"}},"official":{"repos":["swtheing/pf-ppo-rlhf"],"state":"official: harvested, nothing ran","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":1,"ran_from_kinds":[]}}},{"url":"/paper/multi-programming-language-ensemble-for-code","slug":"multi-programming-language-ensemble-for-code","title":"Multi-Programming Language Ensemble for Code Generation in Large Language Model","date":"2024-09-06","arxiv_id":"2409.04114","repositories_listed":1,"syntology":null},{"url":"/paper/how-do-your-code-llms-perform-empowering-code","slug":"how-do-your-code-llms-perform-empowering-code","title":"How Do Your Code LLMs Perform? Empowering Code Instruction Tuning with High-Quality Data","date":"2024-09-05","arxiv_id":"2409.03810","repositories_listed":1,"syntology":null},{"url":"/paper/codexgraph-bridging-large-language-models-and","slug":"codexgraph-bridging-large-language-models-and","title":"CodexGraph: Bridging Large Language Models and Code Repositories via Code Graph Databases","date":"2024-08-07","arxiv_id":"2408.03910","repositories_listed":1,"syntology":{"n":5,"n_ran":4,"n_constructed":0,"n_ran_checked":1,"n_instrument":3,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 3 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/codexgraph-bridging-large-language-models-and#ran","syntology_url":"https://syntology.ai/paper/2408.03910","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2408.03910"}},"official":{"repos":["modelscope/modelscope-agent"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/2408-00994","slug":"2408-00994","title":"ArchCode: Incorporating Software Requirements in Code Generation with Large Language Models","date":"2024-08-02","arxiv_id":"2408.00994","repositories_listed":1,"syntology":null},{"url":"/paper/scaling-granite-code-models-to-128k-context","slug":"scaling-granite-code-models-to-128k-context","title":"Scaling Granite Code Models to 128K Context","date":"2024-07-18","arxiv_id":"2407.13739","repositories_listed":1,"syntology":null},{"url":"/paper/inversecoder-unleashing-the-power-of","slug":"inversecoder-unleashing-the-power-of","title":"InverseCoder: Self-improving Instruction-Tuned Code LLMs with Inverse-Instruct","date":"2024-07-08","arxiv_id":"2407.05700","repositories_listed":1,"syntology":{"n":11,"n_ran":11,"n_constructed":0,"n_ran_checked":2,"n_instrument":9,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":11,"phrase":"11 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 9 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/inversecoder-unleashing-the-power-of#ran","syntology_url":"https://syntology.ai/paper/2407.05700","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2407.05700"}},"official":{"repos":["wyt2000/InverseCoder"],"state":"official (archive's flag): 11 ran","n_ran":11,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/res-q-evaluating-code-editing-large-language","slug":"res-q-evaluating-code-editing-large-language","title":"RES-Q: Evaluating Code-Editing Large Language Model Systems at the Repository Scale","date":"2024-06-24","arxiv_id":"2406.16801","repositories_listed":1,"syntology":{"n":6,"n_ran":6,"n_constructed":0,"n_ran_checked":1,"n_instrument":5,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 5 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/res-q-evaluating-code-editing-large-language#ran","syntology_url":"https://syntology.ai/paper/2406.16801","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.16801"}},"official":{"repos":["qurrent-ai/res-q"],"state":"official (archive's flag): 6 ran","n_ran":6,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/sharelora-parameter-efficient-and-robust","slug":"sharelora-parameter-efficient-and-robust","title":"ShareLoRA: Parameter Efficient and Robust Large Language Model Fine-tuning via Shared Low-Rank Adaptation","date":"2024-06-16","arxiv_id":"2406.10785","repositories_listed":1,"syntology":{"n":6,"n_ran":5,"n_constructed":0,"n_ran_checked":0,"n_instrument":5,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":6,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 5 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/sharelora-parameter-efficient-and-robust#ran","syntology_url":"https://syntology.ai/paper/2406.10785","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.10785"}},"official":{"repos":["Rain9876/ShareLoRA"],"state":"official (archive's flag): 5 ran","n_ran":5,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/can-ai-beat-undergraduates-in-entry-level","slug":"can-ai-beat-undergraduates-in-entry-level","title":"JavaBench: A Benchmark of Object-Oriented Code Generation for Evaluating Large Language Models","date":"2024-06-10","arxiv_id":"2406.12902","repositories_listed":1,"syntology":null},{"url":"/paper/how-efficient-is-llm-generated-code-a","slug":"how-efficient-is-llm-generated-code-a","title":"How Efficient is LLM-Generated Code? A Rigorous & High-Standard Benchmark","date":"2024-06-10","arxiv_id":"2406.06647","repositories_listed":1,"syntology":{"n":4,"n_ran":0,"n_constructed":0,"n_ran_checked":0,"n_instrument":0,"n_unverified":4,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"0 ran · 4 unverified","sample_list":"/paper/how-efficient-is-llm-generated-code-a#ran","syntology_url":"https://syntology.ai/paper/2406.06647","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.06647"}},"official":{"repos":["q-rz/enamel"],"state":"official: harvested, nothing ran","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":4,"ran_from_kinds":[]}}},{"url":"/paper/semcoder-training-code-language-models-with","slug":"semcoder-training-code-language-models-with","title":"SemCoder: Training Code Language Models with Comprehensive Semantics Reasoning","date":"2024-06-03","arxiv_id":"2406.01006","repositories_listed":1,"syntology":{"n":15,"n_ran":12,"n_constructed":0,"n_ran_checked":6,"n_instrument":6,"n_unverified":3,"n_honours":1,"n_violates":0,"n_no_contract":5,"n_pointer_only":0,"phrase":"12 ran (of which 0 constructed an object rather than computing a result; 6 with no instrument failure: 1 honoured, 0 violated, 5 with no contract checked; 6 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/semcoder-training-code-language-models-with#ran","syntology_url":"https://syntology.ai/paper/2406.01006","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.01006"}},"official":{"repos":["arise-lab/semcoder"],"state":"official (archive's flag): 12 ran","n_ran":12,"n_constructed":0,"n_ran_no_instrument_failure":6,"n_unverified":3,"ran_from_kinds":["official"]}}},{"url":"/paper/automatic-instruction-evolving-for-large","slug":"automatic-instruction-evolving-for-large","title":"Automatic Instruction Evolving for Large Language Models","date":"2024-06-02","arxiv_id":"2406.00770","repositories_listed":1,"syntology":{"n":2,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/automatic-instruction-evolving-for-large#ran","syntology_url":"https://syntology.ai/paper/2406.00770","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.00770"}},"official":null}},{"url":"/paper/reflectioncoder-learning-from-reflection","slug":"reflectioncoder-learning-from-reflection","title":"ReflectionCoder: Learning from Reflection Sequence for Enhanced One-off Code Generation","date":"2024-05-27","arxiv_id":"2405.17057","repositories_listed":1,"syntology":{"n":1,"n_ran":0,"n_constructed":0,"n_ran_checked":0,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"0 ran · 1 unverified","sample_list":"/paper/reflectioncoder-learning-from-reflection#ran","syntology_url":"https://syntology.ai/paper/2405.17057","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2405.17057"}},"official":{"repos":["sensellm/reflectioncoder"],"state":"official: harvested, nothing ran","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":1,"ran_from_kinds":[]}}},{"url":"/paper/instruction-tuning-with-loss-over","slug":"instruction-tuning-with-loss-over","title":"Instruction Tuning With Loss Over Instructions","date":"2024-05-23","arxiv_id":"2405.14394","repositories_listed":1,"syntology":{"n":20,"n_ran":16,"n_constructed":0,"n_ran_checked":11,"n_instrument":5,"n_unverified":4,"n_honours":0,"n_violates":1,"n_no_contract":10,"n_pointer_only":20,"phrase":"16 ran (of which 0 constructed an object rather than computing a result; 11 with no instrument failure: 0 honoured, 1 violated, 10 with no contract checked; 5 where Syntology's instrument failed) · 4 unverified","sample_list":"/paper/instruction-tuning-with-loss-over#ran","syntology_url":"https://syntology.ai/paper/2405.14394","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2405.14394"}},"official":{"repos":["zhengxiangshi/instructionmodelling"],"state":"official (archive's flag): 16 ran","n_ran":16,"n_constructed":0,"n_ran_no_instrument_failure":11,"n_unverified":4,"ran_from_kinds":["official"]}}},{"url":"/paper/unchosen-experts-can-contribute-too","slug":"unchosen-experts-can-contribute-too","title":"Unchosen Experts Can Contribute Too: Unleashing MoE Models' Power by Self-Contrast","date":"2024-05-23","arxiv_id":"2405.14507","repositories_listed":1,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":1,"n_instrument":1,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":2,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/unchosen-experts-can-contribute-too#ran","syntology_url":"https://syntology.ai/paper/2405.14507","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2405.14507"}},"official":{"repos":["davidfanzz/scmoe"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/can-github-issues-be-solved-with-tree-of","slug":"can-github-issues-be-solved-with-tree-of","title":"Can Github issues be solved with Tree Of Thoughts?","date":"2024-05-20","arxiv_id":"2405.13057","repositories_listed":1,"syntology":null},{"url":"/paper/multiple-choice-questions-are-efficient-and","slug":"multiple-choice-questions-are-efficient-and","title":"Multiple-Choice Questions are Efficient and Robust LLM Evaluators","date":"2024-05-20","arxiv_id":"2405.11966","repositories_listed":1,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":3,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/multiple-choice-questions-are-efficient-and#ran","syntology_url":"https://syntology.ai/paper/2405.11966","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2405.11966"}},"official":{"repos":["geralt-targaryen/mc-evaluation"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/mhpp-exploring-the-capabilities-and","slug":"mhpp-exploring-the-capabilities-and","title":"MHPP: Exploring the Capabilities and Limitations of Language Models Beyond Basic Code Generation","date":"2024-05-19","arxiv_id":"2405.11430","repositories_listed":1,"syntology":null},{"url":"/paper/naturalcodebench-examining-coding-performance","slug":"naturalcodebench-examining-coding-performance","title":"NaturalCodeBench: Examining Coding Performance Mismatch on HumanEval and Natural User Prompts","date":"2024-05-07","arxiv_id":"2405.04520","repositories_listed":1,"syntology":null},{"url":"/paper/better-faster-large-language-models-via-multi","slug":"better-faster-large-language-models-via-multi","title":"Better & Faster Large Language Models via Multi-token Prediction","date":"2024-04-30","arxiv_id":"2404.19737","repositories_listed":1,"syntology":null},{"url":"/paper/layer-skip-enabling-early-exit-inference-and","slug":"layer-skip-enabling-early-exit-inference-and","title":"LayerSkip: Enabling Early Exit Inference and Self-Speculative Decoding","date":"2024-04-25","arxiv_id":"2404.16710","repositories_listed":1,"syntology":{"n":5,"n_ran":4,"n_constructed":1,"n_ran_checked":1,"n_instrument":3,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":5,"phrase":"4 ran (of which 1 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 3 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/layer-skip-enabling-early-exit-inference-and#ran","syntology_url":"https://syntology.ai/paper/2404.16710","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2404.16710"}},"official":{"repos":["facebookresearch/layerskip"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":1,"n_ran_no_instrument_failure":1,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/xft-unlocking-the-power-of-code-instruction","slug":"xft-unlocking-the-power-of-code-instruction","title":"XFT: Unlocking the Power of Code Instruction Tuning by Simply Merging Upcycled Mixture-of-Experts","date":"2024-04-23","arxiv_id":"2404.15247","repositories_listed":1,"syntology":{"n":14,"n_ran":11,"n_constructed":0,"n_ran_checked":2,"n_instrument":9,"n_unverified":3,"n_honours":1,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"11 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 1 honoured, 0 violated, 1 with no contract checked; 9 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/xft-unlocking-the-power-of-code-instruction#ran","syntology_url":"https://syntology.ai/paper/2404.15247","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2404.15247"}},"official":{"repos":["ise-uiuc/xft"],"state":"official (archive's flag): 11 ran","n_ran":11,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":3,"ran_from_kinds":["official"]}}},{"url":"/paper/comments-as-natural-logic-pivots-improve-code","slug":"comments-as-natural-logic-pivots-improve-code","title":"Comments as Natural Logic Pivots: Improve Code Generation via Comment Perspective","date":"2024-04-11","arxiv_id":"2404.07549","repositories_listed":1,"syntology":null},{"url":"/paper/the-realhumaneval-evaluating-large-language","slug":"the-realhumaneval-evaluating-large-language","title":"The RealHumanEval: Evaluating Large Language Models' Abilities to Support Programmers","date":"2024-04-03","arxiv_id":"2404.02806","repositories_listed":1,"syntology":null},{"url":"/paper/self-organized-agents-a-llm-multi-agent","slug":"self-organized-agents-a-llm-multi-agent","title":"Self-Organized Agents: A LLM Multi-Agent Framework toward Ultra Large-Scale Code Generation and Optimization","date":"2024-04-02","arxiv_id":"2404.02183","repositories_listed":1,"syntology":{"n":11,"n_ran":8,"n_constructed":0,"n_ran_checked":1,"n_instrument":7,"n_unverified":3,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"8 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 7 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/self-organized-agents-a-llm-multi-agent#ran","syntology_url":"https://syntology.ai/paper/2404.02183","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2404.02183"}},"official":{"repos":["tsukushiai/self-organized-agent"],"state":"official (archive's flag): 8 ran","n_ran":8,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":3,"ran_from_kinds":["official"]}}},{"url":"/paper/top-leaderboard-ranking-top-coding","slug":"top-leaderboard-ranking-top-coding","title":"Top Leaderboard Ranking = Top Coding Proficiency, Always? EvoEval: Evolving Coding Benchmarks via LLM","date":"2024-03-28","arxiv_id":"2403.19114","repositories_listed":1,"syntology":{"n":6,"n_ran":5,"n_constructed":0,"n_ran_checked":3,"n_instrument":2,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":0,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 2 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/top-leaderboard-ranking-top-coding#ran","syntology_url":"https://syntology.ai/paper/2403.19114","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2403.19114"}},"official":{"repos":["evo-eval/evoeval"],"state":"official (archive's flag): 5 ran","n_ran":5,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/cycle-learning-to-self-refine-the-code","slug":"cycle-learning-to-self-refine-the-code","title":"CYCLE: Learning to Self-Refine the Code Generation","date":"2024-03-27","arxiv_id":"2403.18746","repositories_listed":1,"syntology":null},{"url":"/paper/investigating-the-performance-of-language","slug":"investigating-the-performance-of-language","title":"Investigating the Performance of Language Models for Completing Code in Functional Programming Languages: a Haskell Case Study","date":"2024-03-22","arxiv_id":"2403.15185","repositories_listed":1,"syntology":null},{"url":"/paper/autodev-automated-ai-driven-development","slug":"autodev-automated-ai-driven-development","title":"AutoDev: Automated AI-Driven Development","date":"2024-03-13","arxiv_id":"2403.08299","repositories_listed":1,"syntology":null},{"url":"/paper/llm4decompile-decompiling-binary-code-with","slug":"llm4decompile-decompiling-binary-code-with","title":"LLM4Decompile: Decompiling Binary Code with Large Language Models","date":"2024-03-08","arxiv_id":"2403.05286","repositories_listed":1,"syntology":{"n":13,"n_ran":9,"n_constructed":0,"n_ran_checked":4,"n_instrument":5,"n_unverified":4,"n_honours":1,"n_violates":0,"n_no_contract":3,"n_pointer_only":1,"phrase":"9 ran (of which 0 constructed an object rather than computing a result; 4 with no instrument failure: 1 honoured, 0 violated, 3 with no contract checked; 5 where Syntology's instrument failed) · 4 unverified","sample_list":"/paper/llm4decompile-decompiling-binary-code-with#ran","syntology_url":"https://syntology.ai/paper/2403.05286","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2403.05286"}},"official":{"repos":["albertan017/LLM4Decompile"],"state":"official (archive's flag): 9 ran","n_ran":9,"n_constructed":0,"n_ran_no_instrument_failure":4,"n_unverified":4,"ran_from_kinds":["official"]}}}],"syntology_records":57,"syntology_note":"a paper without a record is not a recorded non-run: it may lack an arXiv id or simply be absent from the graph layer"},"key_notes":{"n_ran_checked":"legacy name, kept unchanged so existing readers do not break: it counts the samples that ran with no instrument failure (honoured, violated, and ran with no contract checked); it does not mean a contract was checked, and the pages print it as 'K with no instrument failure', not 'K checked'","n_constructed":"a sub-count of the samples that ran, never subtracted from them and never a failure: an executed sample whose run returned an instance of its own class (fixture_out_type equals the entry name): the run built an object and did not compute a result (Syntology's RAN record, counts.constructed)"},"since_archive_judged":{"papers":0,"papers_placed":0,"judged_by":{"model":"a model","models_as_recorded":[],"from":"lb_rows_run_verified.jsonl: each record's `model` field (the extractor's call that wrote decisions[])"},"note":"a model's judgement from the paper's own tables that they report on this leaderboard, not a result: no value from a paper that was not placed is printed or carried here; papers counts each paper once over these leaderboards","backfill":"8,886 of 18,637 newer papers checked so far (733 more had nothing to place; 41 could not be read)","backfill_complete":false,"leaderboards":[]},"papers_where_code_ran":76,"papers_where_code_ran_first_page":"/task/humaneval/papers/ran/1","papers_where_code_ran_caption":"We ran code from the paper's repository; we did not run it on this task or check it against the task's benchmarks.","pages":[{"order":"archive","page":2,"of":3,"url":"/task/humaneval/papers/2","json":"/task/humaneval/papers/2.json","rows":100},{"order":"archive","page":3,"of":3,"url":"/task/humaneval/papers/3","json":"/task/humaneval/papers/3.json","rows":64},{"order":"ran","page":1,"of":1,"url":"/task/humaneval/papers/ran/1","json":"/task/humaneval/papers/ran/1.json","rows":76}],"description_links":{"kept":0,"unwrapped_to_text":0,"bare_urls_linked":0,"relative_images_dropped":0,"rule":"internal links are kept only when the target slug exists in the catalog"},"syntology":{"read_at":"2026-10-05T00:36:11+00:00","claim":"Per-sample execution status on synthesized fixtures ('ran N of M samples'); not a correctness claim and not a ranking signal.","status_vocabulary":{"ran_honours":"ran, honoured the contract we drafted","ran_violates":"ran, violated the contract we drafted","ran_draft_wrong":"ran; our contract draft was wrong, not the code","ran_fixture":"ran; the contract check could not run (RAISES, DEP_MISSING or TIMEOUT)","ran":"ran on a synthesized input","unverified":"unverified (harvested, no recorded run)"},"row_route":"each syntology object on a benchmark's first row and on a paper carries sample_list (this site's paper page, #ran), syntology_url (syntology.ai's page for the paper, only when that route probed live) and mcp (the agent call that lists the paper's harvested samples; not a verdict on the paper)","developers":"https://syntology.ai/developers"},"not_shown":{"libraries":"the archive has no per-task library table","trend_sparklines":"the Trend column of the benchmarks table was a rendered image; it is not in the archive","social_and_latest_sorts":"stars and social signals are not in the archive"},"record_sha256":"32c9ed37a6089bbf745c31d444dbfa44f55d0baa30a6b738041da6feb6d1a636","record_changed_at":"2026-10-05","record_changed_at_basis":"changed"}