{"about":{"non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","site":"https://codewithpapers.app","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page","syntology":{"site":"https://syntology.ai","developers":"https://syntology.ai/developers","mcp":{"server":"https://syntology.ai/mcp","transport":"streamable-http","server_card":"https://syntology.ai/.well-known/mcp/server-card.json","auth":{"type":"trial token, no account","trial_token":"https://syntology.ai/api/oauth/trial/token","method":"POST","docs":"https://syntology.ai/developers"}},"have":"https://syntology.ai/api/graph/have?x=<method, arXiv id or title> (free, answers coverage only)","paper_base":"https://syntology.ai/paper/","atlas_base":"https://app.syntology.ai/?focus="},"machine_readable":[{"url":"https://codewithpapers.app/llms.txt","what":"the machine catalog: every machine-readable file, counted"},{"url":"https://codewithpapers.app/index/manifest.json","what":"paper-to-code index by arXiv id, with Syntology's counts"},{"url":"https://codewithpapers.app/search/manifest.json","what":"site search index (titles, authors) and its files"},{"url":"https://codewithpapers.app/download","what":"bulk files: Syntology's layer, described there"},{"url":"https://codewithpapers.app/build_manifest.json","what":"the build record: inputs, counts, exclusions, probes"}]},"url":"/task/code-generation","name":"Code Generation","slug":"code-generation","description_markdown":"**Code Generation** is an important field to predict explicit code or program structure from multimodal data sources such as incomplete code, programs in another programming language, natural language descriptions or execution examples. Code Generation tools can assist the development of automatic programming tools to improve programming productivity.\r\n\r\n\r\n<span class=\"description-source\">Source: [Deep Learning for Source Code Modeling and Generation ](https://arxiv.org/abs/2002.05442)</span>\r\n\r\nImage source: [Measuring Coding Challenge Competence With APPS](https://paperswithcode.com/paper/measuring-coding-challenge-competence-with)","categories":[{"name":"Computer Code","url":"/area/computer-code"},{"name":"Natural Language Processing","url":"/area/natural-language-processing"},{"name":"Reasoning","url":"/area/reasoning"}],"source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","slug_source":"archive_url"},"counts":{"papers_tagged":1697,"papers_with_code":745,"benchmarks":27,"benchmark_tables_in_archive":27,"benchmark_tables_shown":27,"benchmark_tables_withheld_as_spam":0,"benchmark_definition":"a leaderboard table with at least one row; benchmark_tables_shown also counts the zero-row tables; benchmark_tables_in_archive adds the tables withheld as spam","datasets":70,"subtasks":5,"parent_tasks":0},"benchmarks":[{"leaderboard":"/sota/code-generation-on-mbpp","slug":"code-generation-on-mbpp","dataset":"MBPP","dataset_url":"/dataset/mbpp","rows_in_archive":99,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"EG-CFG (DeepSeek-V3-0324)","paper_title":"Execution Guided Line-by-Line Code Generation","paper_url":"/paper/execution-guided-line-by-line-code-generation","paper_date":"2025-06-12","arxiv_id":"2506.10948","code_links":[{"title":"boazlavon/eg_cfg","url":"https://github.com/boazlavon/eg_cfg"}],"syntology":null}},{"leaderboard":"/sota/code-generation-on-apps","slug":"code-generation-on-apps","dataset":"APPS","dataset_url":"/dataset/apps","rows_in_archive":18,"metrics":["Introductory Pass@1","Interview Pass@1","Competition Pass@1","Competition Pass@any","Interview Pass@any","Introductory Pass@any","Competition Pass@5","Interview Pass@5","Introductory Pass@5","Competition Pass@1000","Interview Pass@1000","Introductory Pass@1000","Pass@1"],"first_row_in_archive_order":{"model":"LPW (GPT-4o)","paper_title":"Planning-Driven Programming: A Large Language Model Programming Workflow","paper_url":"/paper/planning-driven-programming-a-large-language","paper_date":"2024-11-21","arxiv_id":"2411.14503","code_links":[{"title":"you68681/lpw","url":"https://github.com/you68681/lpw"}],"syntology":{"n":9,"n_ran":9,"n_constructed":0,"n_ran_checked":4,"n_instrument":5,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":4,"n_pointer_only":0,"phrase":"9 ran (of which 0 constructed an object rather than computing a result; 4 with no instrument failure: 0 honoured, 0 violated, 4 with no contract checked; 5 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/planning-driven-programming-a-large-language#ran","syntology_url":"https://syntology.ai/paper/2411.14503","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2411.14503"}},"official":{"repos":["you68681/lpw"],"state":"official (archive's flag): 9 ran","n_ran":9,"n_constructed":0,"n_ran_no_instrument_failure":4,"n_unverified":0,"ran_from_kinds":["official"]}}}},{"leaderboard":"/sota/code-generation-on-conala","slug":"code-generation-on-conala","dataset":"CoNaLa","dataset_url":"/dataset/conala","rows_in_archive":14,"metrics":["BLEU","Exact Match Accuracy"],"first_row_in_archive_order":{"model":"PanGu-Coder-FT-I","paper_title":"Fine-Tuning Large Language Models for Answering Programming Questions with Code Snippets","paper_url":"/paper/fine-tuning-large-language-models-for","paper_date":"2023-06-26","arxiv_id":null,"code_links":[],"syntology":null}},{"leaderboard":"/sota/code-generation-on-django","slug":"code-generation-on-django","dataset":"Django","dataset_url":"/dataset/django","rows_in_archive":11,"metrics":["Accuracy","BLEU Score"],"first_row_in_archive_order":{"model":"MarianCG","paper_title":"MarianCG: a code generation transformer model inspired by machine translation","paper_url":"/paper/mariancg-a-code-generation-transformer-model","paper_date":"2022-11-22","arxiv_id":null,"code_links":[{"title":"AhmedSSoliman/MarianCG-NL-to-Code","url":"https://github.com/AhmedSSoliman/MarianCG-NL-to-Code"}],"syntology":null}},{"leaderboard":"/sota/code-generation-on-wikisql","slug":"code-generation-on-wikisql","dataset":"WikiSQL","dataset_url":"/dataset/wikisql","rows_in_archive":10,"metrics":["Execution Accuracy","Exact Match Accuracy"],"first_row_in_archive_order":{"model":"NL2SQL-RULE","paper_title":"Content Enhanced BERT-based Text-to-SQL Generation","paper_url":"/paper/content-enhanced-bert-based-text-to-sql","paper_date":"2019-10-16","arxiv_id":"1910.07179","code_links":[{"title":"guotong1988/NL2SQL-RULE","url":"https://github.com/guotong1988/NL2SQL-RULE"},{"title":"guotong1988/NL2SQL-BERT","url":"https://github.com/guotong1988/NL2SQL-BERT"},{"title":"shivam017arora/Conversational-BI","url":"https://github.com/shivam017arora/Conversational-BI"},{"title":"yangyucheng000/University","url":"https://github.com/yangyucheng000/University/tree/main/model-2/bert_generation"},{"title":"realsonalkumar/Mish-Mash-Hackathon","url":"https://github.com/realsonalkumar/Mish-Mash-Hackathon"}],"syntology":null}},{"leaderboard":"/sota/code-generation-on-res-q","slug":"code-generation-on-res-q","dataset":"RES-Q","dataset_url":"/dataset/res-q","rows_in_archive":9,"metrics":["pass@1"],"first_row_in_archive_order":{"model":"QurrentOS-coder + Claude 3.5 Sonnet","paper_title":"RES-Q: Evaluating Code-Editing Large Language Model Systems at the Repository Scale","paper_url":"/paper/res-q-evaluating-code-editing-large-language","paper_date":"2024-06-24","arxiv_id":"2406.16801","code_links":[{"title":"qurrent-ai/res-q","url":"https://github.com/qurrent-ai/res-q"}],"syntology":{"n":6,"n_ran":6,"n_constructed":0,"n_ran_checked":1,"n_instrument":5,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 5 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/res-q-evaluating-code-editing-large-language#ran","syntology_url":"https://syntology.ai/paper/2406.16801","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.16801"}},"official":{"repos":["qurrent-ai/res-q"],"state":"official (archive's flag): 6 ran","n_ran":6,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}}},{"leaderboard":"/sota/code-generation-on-codecontests","slug":"code-generation-on-codecontests","dataset":"CodeContests","dataset_url":"/dataset/codecontests","rows_in_archive":8,"metrics":["Test Set pass@1","Test Set pass@5","Val Set pass@1","Val Set pass@5"],"first_row_in_archive_order":{"model":"EG-CFG (DeepSeek-V3-0324)","paper_title":"Execution Guided Line-by-Line Code Generation","paper_url":"/paper/execution-guided-line-by-line-code-generation","paper_date":"2025-06-12","arxiv_id":"2506.10948","code_links":[{"title":"boazlavon/eg_cfg","url":"https://github.com/boazlavon/eg_cfg"}],"syntology":null}},{"leaderboard":"/sota/code-generation-on-humaneval","slug":"code-generation-on-humaneval","dataset":"HumanEval","dataset_url":"/dataset/humaneval","rows_in_archive":8,"metrics":["Pass@1"],"first_row_in_archive_order":{"model":"DeepSeek-R1 (MGDebugger)","paper_title":"From Code to Correctness: Closing the Last Mile of Code Generation with Hierarchical Debugging","paper_url":"/paper/from-code-to-correctness-closing-the-last","paper_date":"2024-10-02","arxiv_id":"2410.01215","code_links":[{"title":"YerbaPage/MGDebugger","url":"https://github.com/YerbaPage/MGDebugger"}],"syntology":{"n":9,"n_ran":8,"n_constructed":0,"n_ran_checked":5,"n_instrument":3,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":5,"n_pointer_only":0,"phrase":"8 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 0 honoured, 0 violated, 5 with no contract checked; 3 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/from-code-to-correctness-closing-the-last#ran","syntology_url":"https://syntology.ai/paper/2410.01215","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2410.01215"}},"official":{"repos":["YerbaPage/MGDebugger"],"state":"official (archive's flag): 8 ran","n_ran":8,"n_constructed":0,"n_ran_no_instrument_failure":5,"n_unverified":1,"ran_from_kinds":["official"]}}}},{"leaderboard":"/sota/code-generation-on-pecc","slug":"code-generation-on-pecc","dataset":"PECC","dataset_url":"/dataset/pecc","rows_in_archive":8,"metrics":["Pass@3"],"first_row_in_archive_order":{"model":"Claude 3 Haiku","paper_title":"PECC: Problem Extraction and Coding Challenges","paper_url":"/paper/pecc-problem-extraction-and-coding-challenges","paper_date":"2024-04-29","arxiv_id":"2404.18766","code_links":[{"title":"hallerpatrick/pecc","url":"https://github.com/hallerpatrick/pecc"}],"syntology":null}},{"leaderboard":"/sota/code-generation-on-webapp1k-react","slug":"code-generation-on-webapp1k-react","dataset":"WebApp1K-React","dataset_url":"/dataset/webapp1k-react","rows_in_archive":8,"metrics":["pass@1"],"first_row_in_archive_order":{"model":"o1-preview","paper_title":"A Case Study of Web App Coding with OpenAI Reasoning Models","paper_url":"/paper/a-case-study-of-web-app-coding-with-openai","paper_date":"2024-09-19","arxiv_id":"2409.13773","code_links":[{"title":"onekq/webapp1k","url":"https://github.com/onekq/webapp1k"}],"syntology":null}},{"leaderboard":"/sota/code-generation-on-conala-ext","slug":"code-generation-on-conala-ext","dataset":"CoNaLa-Ext","dataset_url":"/dataset/conala-ext","rows_in_archive":6,"metrics":["BLEU"],"first_row_in_archive_order":{"model":"BART W/ Mined","paper_title":"Reading StackOverflow Encourages Cheating: Adding Question Text Improves Extractive Code Generation","paper_url":"/paper/reading-stackoverflow-encourages-cheating","paper_date":"2021-06-08","arxiv_id":"2106.04447","code_links":[{"title":"gabeorlanski/stackoverflow-encourages-cheating","url":"https://github.com/gabeorlanski/stackoverflow-encourages-cheating"}],"syntology":null}},{"leaderboard":"/sota/code-generation-on-webapp1k-duo-react","slug":"code-generation-on-webapp1k-duo-react","dataset":"WebApp1k-Duo-React","dataset_url":"/dataset/webapp1k-duo-react","rows_in_archive":6,"metrics":["pass@1"],"first_row_in_archive_order":{"model":"claude-3-5-sonnet","paper_title":"A Case Study of Web App Coding with OpenAI Reasoning Models","paper_url":"/paper/a-case-study-of-web-app-coding-with-openai","paper_date":"2024-09-19","arxiv_id":"2409.13773","code_links":[{"title":"onekq/webapp1k","url":"https://github.com/onekq/webapp1k"}],"syntology":null}},{"leaderboard":"/sota/code-generation-on-dseval-leetcode","slug":"code-generation-on-dseval-leetcode","dataset":"DSEval-LeetCode","dataset_url":"/dataset/dseval-leetcode","rows_in_archive":5,"metrics":["Pass Rate","w/o Intact","w/o PE"],"first_row_in_archive_order":{"model":"Jupyter-AI","paper_title":null,"paper_url":null,"paper_date":"","arxiv_id":null,"code_links":[],"syntology":null}},{"leaderboard":"/sota/code-generation-on-turbulence","slug":"code-generation-on-turbulence","dataset":"Turbulence","dataset_url":"/dataset/turbulence","rows_in_archive":5,"metrics":["CorrSc"],"first_row_in_archive_order":{"model":"GPT-4","paper_title":"Turbulence: Systematically and Automatically Testing Instruction-Tuned Large Language Models for Code","paper_url":"/paper/turbulence-systematically-and-automatically","paper_date":"2023-12-22","arxiv_id":"2312.14856","code_links":[{"title":"shahinhonarvar/turbulence-benchmark","url":"https://github.com/shahinhonarvar/turbulence-benchmark"}],"syntology":{"n":4,"n_ran":4,"n_constructed":0,"n_ran_checked":4,"n_instrument":0,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":3,"n_pointer_only":0,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 4 with no instrument failure: 1 honoured, 0 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/turbulence-systematically-and-automatically#ran","syntology_url":"https://syntology.ai/paper/2312.14856","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2312.14856"}},"official":{"repos":["shahinhonarvar/turbulence-benchmark"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":4,"n_unverified":0,"ran_from_kinds":["official"]}}}},{"leaderboard":"/sota/code-generation-on-livecodebench","slug":"code-generation-on-livecodebench","dataset":"Livecodebench","dataset_url":null,"rows_in_archive":3,"metrics":["Acc"],"first_row_in_archive_order":{"model":"Xolver","paper_title":"Xolver: Multi-Agent Reasoning with Holistic Experience Learning Just Like an Olympiad Team","paper_url":"/paper/xolver-multi-agent-reasoning-with-holistic","paper_date":"2025-06-17","arxiv_id":"2506.14234","code_links":[{"title":"kagnlp/Xolver","url":"https://github.com/kagnlp/Xolver"}],"syntology":null}},{"leaderboard":"/sota/code-generation-on-shellcode-ia32","slug":"code-generation-on-shellcode-ia32","dataset":"Shellcode_IA32","dataset_url":"/dataset/shellcode-ia32","rows_in_archive":3,"metrics":["BLEU-4","Exact Match Accuracy"],"first_row_in_archive_order":{"model":"CodeBERT","paper_title":"Can We Generate Shellcodes via Natural Language? An Empirical Study","paper_url":"/paper/can-we-generate-shellcodes-via-natural","paper_date":"2022-02-08","arxiv_id":"2202.03755","code_links":[{"title":"dessertlab/Shellcode_IA32","url":"https://github.com/dessertlab/Shellcode_IA32"}],"syntology":null}},{"leaderboard":"/sota/code-generation-on-taco-code","slug":"code-generation-on-taco-code","dataset":"TACO-BAAI","dataset_url":"/dataset/taco-topics-in-algorithmic-code-generation","rows_in_archive":3,"metrics":["easy pass@1"],"first_row_in_archive_order":{"model":"GPT-4","paper_title":"TACO: Topics in Algorithmic COde generation dataset","paper_url":"/paper/taco-topics-in-algorithmic-code-generation","paper_date":"2023-12-22","arxiv_id":"2312.14852","code_links":[{"title":"flagopen/taco","url":"https://github.com/flagopen/taco"}],"syntology":{"n":2,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/taco-topics-in-algorithmic-code-generation#ran","syntology_url":"https://syntology.ai/paper/2312.14852","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2312.14852"}},"official":{"repos":["flagopen/taco"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":1,"ran_from_kinds":["official"]}}}},{"leaderboard":"/sota/code-generation-on-bigcodebench-complete","slug":"code-generation-on-bigcodebench-complete","dataset":"BigCodeBench-Complete","dataset_url":"/dataset/bigcodebench","rows_in_archive":2,"metrics":["Pass@1"],"first_row_in_archive_order":{"model":"GPT-4o-2024-05-13","paper_title":"BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions","paper_url":"/paper/bigcodebench-benchmarking-code-generation","paper_date":"2024-06-22","arxiv_id":"2406.15877","code_links":[{"title":"mlfoundations/Evalchemy","url":"https://github.com/mlfoundations/Evalchemy"},{"title":"bigcode-project/bigcodebench","url":"https://github.com/bigcode-project/bigcodebench"},{"title":"bigcode-project/bigcodebench-annotation","url":"https://github.com/bigcode-project/bigcodebench-annotation"},{"title":"stovecat/convcodeworld","url":"https://github.com/stovecat/convcodeworld"}],"syntology":{"n":6,"n_ran":5,"n_constructed":0,"n_ran_checked":1,"n_instrument":4,"n_unverified":1,"n_honours":0,"n_violates":1,"n_no_contract":0,"n_pointer_only":1,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 1 violated, 0 with no contract checked; 4 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/bigcodebench-benchmarking-code-generation#ran","syntology_url":"https://syntology.ai/paper/2406.15877","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.15877"}},"official":{"repos":["bigcode-project/bigcodebench","bigcode-project/bigcodebench-annotation"],"state":"official (archive's flag): 5 ran","n_ran":5,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":1,"ran_from_kinds":["official"]}}}},{"leaderboard":"/sota/code-generation-on-bigcodebench-instruct","slug":"code-generation-on-bigcodebench-instruct","dataset":"BigCodeBench-Instruct","dataset_url":"/dataset/bigcodebench","rows_in_archive":2,"metrics":["Pass@1"],"first_row_in_archive_order":{"model":"GPT-4o-2024-05-13","paper_title":"BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions","paper_url":"/paper/bigcodebench-benchmarking-code-generation","paper_date":"2024-06-22","arxiv_id":"2406.15877","code_links":[{"title":"mlfoundations/Evalchemy","url":"https://github.com/mlfoundations/Evalchemy"},{"title":"bigcode-project/bigcodebench","url":"https://github.com/bigcode-project/bigcodebench"},{"title":"bigcode-project/bigcodebench-annotation","url":"https://github.com/bigcode-project/bigcodebench-annotation"},{"title":"stovecat/convcodeworld","url":"https://github.com/stovecat/convcodeworld"}],"syntology":{"n":6,"n_ran":5,"n_constructed":0,"n_ran_checked":1,"n_instrument":4,"n_unverified":1,"n_honours":0,"n_violates":1,"n_no_contract":0,"n_pointer_only":1,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 1 violated, 0 with no contract checked; 4 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/bigcodebench-benchmarking-code-generation#ran","syntology_url":"https://syntology.ai/paper/2406.15877","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.15877"}},"official":{"repos":["bigcode-project/bigcodebench","bigcode-project/bigcodebench-annotation"],"state":"official (archive's flag): 5 ran","n_ran":5,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":1,"ran_from_kinds":["official"]}}}},{"leaderboard":"/sota/code-generation-on-codexglue-codesearchnet","slug":"code-generation-on-codexglue-codesearchnet","dataset":"CodeXGLUE - CodeSearchNet","dataset_url":"/dataset/codexglue","rows_in_archive":2,"metrics":["Java/EM","Python/EM","Java/BLEU","Python/BLEU","Java/CodeBLEU","Python/CodeBLEU"],"first_row_in_archive_order":{"model":"Redcoder-ext","paper_title":"Retrieval Augmented Code Generation and Summarization","paper_url":"/paper/retrieval-augmented-code-generation-and","paper_date":"2021-08-26","arxiv_id":"2108.11601","code_links":[{"title":"kagnlp/CodeGenerator","url":"https://github.com/kagnlp/CodeGenerator"},{"title":"rizwan09/redcoder","url":"https://github.com/rizwan09/redcoder"}],"syntology":{"n":3,"n_ran":2,"n_constructed":0,"n_ran_checked":1,"n_instrument":1,"n_unverified":1,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":3,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/retrieval-augmented-code-generation-and#ran","syntology_url":"https://syntology.ai/paper/2108.11601","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2108.11601"}},"official":{"repos":["rizwan09/redcoder"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":1,"ran_from_kinds":["official"]}}}},{"leaderboard":"/sota/code-generation-on-concode","slug":"code-generation-on-concode","dataset":"CONCODE","dataset_url":"/dataset/concode","rows_in_archive":2,"metrics":["Exact Match","BLEU","CodeBLEU"],"first_row_in_archive_order":{"model":"Redcoder-ext","paper_title":"Retrieval Augmented Code Generation and Summarization","paper_url":"/paper/retrieval-augmented-code-generation-and","paper_date":"2021-08-26","arxiv_id":"2108.11601","code_links":[{"title":"kagnlp/CodeGenerator","url":"https://github.com/kagnlp/CodeGenerator"},{"title":"rizwan09/redcoder","url":"https://github.com/rizwan09/redcoder"}],"syntology":{"n":3,"n_ran":2,"n_constructed":0,"n_ran_checked":1,"n_instrument":1,"n_unverified":1,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":3,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/retrieval-augmented-code-generation-and#ran","syntology_url":"https://syntology.ai/paper/2108.11601","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2108.11601"}},"official":{"repos":["rizwan09/redcoder"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":1,"ran_from_kinds":["official"]}}}},{"leaderboard":"/sota/code-generation-on-humaneval-et","slug":"code-generation-on-humaneval-et","dataset":"HumanEval-ET","dataset_url":"/dataset/humaneval-et","rows_in_archive":2,"metrics":["Pass@1"],"first_row_in_archive_order":{"model":"EG-CFG (DeepSeek-V3-0324)","paper_title":"Execution Guided Line-by-Line Code Generation","paper_url":"/paper/execution-guided-line-by-line-code-generation","paper_date":"2025-06-12","arxiv_id":"2506.10948","code_links":[{"title":"boazlavon/eg_cfg","url":"https://github.com/boazlavon/eg_cfg"}],"syntology":null}},{"leaderboard":"/sota/code-generation-on-mbpp-et","slug":"code-generation-on-mbpp-et","dataset":"MBPP-ET","dataset_url":"/dataset/mbpp-et","rows_in_archive":2,"metrics":["Pass@1"],"first_row_in_archive_order":{"model":"EG-CFG (DeepSeek-V3-0324)","paper_title":"Execution Guided Line-by-Line Code Generation","paper_url":"/paper/execution-guided-line-by-line-code-generation","paper_date":"2025-06-12","arxiv_id":"2506.10948","code_links":[{"title":"boazlavon/eg_cfg","url":"https://github.com/boazlavon/eg_cfg"}],"syntology":null}},{"leaderboard":"/sota/code-generation-on-verified-smart-contract","slug":"code-generation-on-verified-smart-contract","dataset":"Verified Smart Contract Code Comments","dataset_url":"/dataset/verified-smart-contract-code-comments","rows_in_archive":2,"metrics":["BLEU score"],"first_row_in_archive_order":{"model":"GPT-J 6B Smart Contract","paper_title":"Efficient Avoidance of Vulnerabilities in Auto-completed Smart Contract Code Using Vulnerability-constrained Decoding","paper_url":"/paper/efficient-avoidance-of-vulnerabilities-in","paper_date":"2023-09-18","arxiv_id":"2309.09826","code_links":[],"syntology":null}},{"leaderboard":"/sota/code-generation-on-android-repos","slug":"code-generation-on-android-repos","dataset":"Android Repos","dataset_url":null,"rows_in_archive":1,"metrics":["Perplexity"],"first_row_in_archive_order":{"model":"Entity Type Model","paper_title":"Building Language Models for Text with Named Entities","paper_url":"/paper/building-language-models-for-text-with-named","paper_date":"2018-05-13","arxiv_id":"1805.04836","code_links":[{"title":"uclanlp/NamedEntityLanguageModel","url":"https://github.com/uclanlp/NamedEntityLanguageModel"},{"title":"devanshbatra04/Named-Entity-Language-Models","url":"https://github.com/devanshbatra04/Named-Entity-Language-Models"}],"syntology":{"n":1,"n_ran":0,"n_constructed":0,"n_ran_checked":0,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"0 ran · 1 unverified","sample_list":"/paper/building-language-models-for-text-with-named#ran","syntology_url":"https://syntology.ai/paper/1805.04836","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1805.04836"}},"official":{"repos":["uclanlp/NamedEntityLanguageModel"],"state":"official: harvested, nothing ran","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":1,"ran_from_kinds":[]}}}},{"leaderboard":"/sota/code-generation-on-floco","slug":"code-generation-on-floco","dataset":"FloCo","dataset_url":"/dataset/floco","rows_in_archive":1,"metrics":["BLEU","CodeBLEU"],"first_row_in_archive_order":{"model":"FloCo-T5","paper_title":"Towards Making Flowchart Images Machine Interpretable","paper_url":"/paper/towards-making-flowchart-images-machine-1","paper_date":"2025-01-29","arxiv_id":"2501.17441","code_links":[{"title":"vl2g/floco","url":"https://github.com/vl2g/floco"}],"syntology":null}},{"leaderboard":"/sota/code-generation-on-verilogeval","slug":"code-generation-on-verilogeval","dataset":"VerilogEval","dataset_url":"/dataset/verilogeval","rows_in_archive":1,"metrics":["Pass Rate"],"first_row_in_archive_order":{"model":"Nexus (Claude 3.5 Sonnet)","paper_title":"Nexus: A Lightweight and Scalable Multi-Agent Framework for Complex Tasks Automation","paper_url":"/paper/nexus-a-lightweight-and-scalable-multi-agent","paper_date":"2025-02-26","arxiv_id":"2502.19091","code_links":[{"title":"PrimisAI/nexus","url":"https://github.com/PrimisAI/nexus"}],"syntology":null}}],"datasets":[{"url":"/dataset/humaneval","name":"HumanEval","full_name":"","num_papers_in_archive":1201},{"url":"/dataset/mbpp","name":"MBPP","full_name":"Mostly Basic Python Programming","num_papers_in_archive":666},{"url":"/dataset/wikisql","name":"WikiSQL","full_name":"WikiSQL","num_papers_in_archive":267},{"url":"/dataset/codexglue","name":"CodeXGLUE","full_name":"","num_papers_in_archive":205},{"url":"/dataset/apps","name":"APPS","full_name":"Automated Programming Progress Standard","num_papers_in_archive":180},{"url":"/dataset/codecontests","name":"CodeContests","full_name":"","num_papers_in_archive":84},{"url":"/dataset/conala","name":"CoNaLa","full_name":"CMU CoNaLa, the Code/Natural Language Challenge","num_papers_in_archive":77},{"url":"/dataset/ds-1000","name":"DS-1000","full_name":"","num_papers_in_archive":72},{"url":"/dataset/concode","name":"CONCODE","full_name":"","num_papers_in_archive":46},{"url":"/dataset/bigcodebench","name":"BigCodeBench","full_name":"","num_papers_in_archive":38},{"url":"/dataset/classeval","name":"ClassEval","full_name":"","num_papers_in_archive":27},{"url":"/dataset/django","name":"Django","full_name":"Django","num_papers_in_archive":22},{"url":"/dataset/hearthstone","name":"Hearthstone","full_name":"","num_papers_in_archive":22},{"url":"/dataset/securityeval","name":"SecurityEval","full_name":"","num_papers_in_archive":17},{"url":"/dataset/humaneval-x","name":"HumanEval-X","full_name":"","num_papers_in_archive":16},{"url":"/dataset/juice","name":"JuICe","full_name":"JuICe Dataset","num_papers_in_archive":16},{"url":"/dataset/humaneval-et","name":"HumanEval-ET","full_name":"","num_papers_in_archive":15},{"url":"/dataset/mbpp-et","name":"MBPP-ET","full_name":"","num_papers_in_archive":15},{"url":"/dataset/llamea","name":"LLaMEA","full_name":"algorithms and experiments from the paper","num_papers_in_archive":12},{"url":"/dataset/criticbench","name":"CriticBench","full_name":"","num_papers_in_archive":10},{"url":"/dataset/mconala","name":"MCoNaLa","full_name":"Multilingual CoNaLa","num_papers_in_archive":10},{"url":"/dataset/data-science-problems","name":"Data Science Problems","full_name":"","num_papers_in_archive":7},{"url":"/dataset/humaneval-xl","name":"HumanEval-XL","full_name":"","num_papers_in_archive":6},{"url":"/dataset/lyra","name":"Lyra","full_name":"","num_papers_in_archive":6},{"url":"/dataset/shellcode-ia32","name":"Shellcode_IA32","full_name":"Shellcode_IA32","num_papers_in_archive":6},{"url":"/dataset/biocoder","name":"BioCoder","full_name":"","num_papers_in_archive":5},{"url":"/dataset/pytorrent","name":"PyTorrent","full_name":"PyTorrent","num_papers_in_archive":5},{"url":"/dataset/safim","name":"SAFIM","full_name":"Syntax-Aware Fill-In-the-Middle","num_papers_in_archive":5},{"url":"/dataset/conala-ext","name":"CoNaLa-Ext","full_name":"CoNaLa Extended With Question Text","num_papers_in_archive":4},{"url":"/dataset/dseval-leetcode","name":"DSEval-LeetCode","full_name":"","num_papers_in_archive":4},{"url":"/dataset/leetcode-hard","name":"LeetCode-Hard","full_name":"","num_papers_in_archive":4},{"url":"/dataset/mmcode","name":"MMCode","full_name":"","num_papers_in_archive":3},{"url":"/dataset/spectre-v1","name":"Spectre-v1","full_name":"","num_papers_in_archive":3},{"url":"/dataset/webapp1k-react","name":"WebApp1K-React","full_name":"","num_papers_in_archive":3},{"url":"/dataset/disl","name":"DISL","full_name":"Fueling Research with A Large Dataset of Solidity Smart Contracts","num_papers_in_archive":2},{"url":"/dataset/dseval-exercise","name":"DSEval-Exercise","full_name":"","num_papers_in_archive":2},{"url":"/dataset/dseval","name":"DSEval-Kaggle","full_name":"","num_papers_in_archive":2},{"url":"/dataset/dseval-so","name":"DSEval-SO","full_name":"","num_papers_in_archive":2},{"url":"/dataset/evil","name":"EVIL","full_name":"","num_papers_in_archive":2},{"url":"/dataset/planetarium","name":"Planetarium","full_name":"","num_papers_in_archive":2},{"url":"/dataset/rmcbench","name":"RMCBench","full_name":"","num_papers_in_archive":2},{"url":"/dataset/sltrans","name":"SLTrans","full_name":"","num_papers_in_archive":2},{"url":"/dataset/verilogeval","name":"VerilogEval","full_name":"","num_papers_in_archive":2},{"url":"/dataset/codegen4libs-dataset","name":"CodeGen4Libs Dataset","full_name":"","num_papers_in_archive":1},{"url":"/dataset/coffe","name":"COFFE","full_name":"COFFE: A Code Efficiency Benchmark for Code Generation","num_papers_in_archive":1},{"url":"/dataset/crust-bench","name":"CRUST-bench","full_name":"","num_papers_in_archive":1},{"url":"/dataset/synthetic-grade-school-math-sgsm","name":"Educational Grade School Math (EGSM)","full_name":"","num_papers_in_archive":1},{"url":"/dataset/evil-decoders","name":"EVIL-Decoders","full_name":"","num_papers_in_archive":1},{"url":"/dataset/evil-encoders","name":"EVIL-Encoders","full_name":"","num_papers_in_archive":1},{"url":"/dataset/flat-real-world-simulink-models","name":"Flat Real World Simulink Models","full_name":"","num_papers_in_archive":1},{"url":"/dataset/floco","name":"FloCo","full_name":"Flow chart Image to Code","num_papers_in_archive":1},{"url":"/dataset/genotex","name":"GenoTEX","full_name":"An LLM Agent Benchmark for Automated Gene Expression Data Analysis","num_papers_in_archive":1},{"url":"/dataset/migration-bench-java-full","name":"migration-bench-java-full","full_name":"","num_papers_in_archive":1},{"url":"/dataset/migration-bench-java-selected","name":"migration-bench-java-selected","full_name":"","num_papers_in_archive":1},{"url":"/dataset/migration-bench-java-utg","name":"migration-bench-java-utg","full_name":"","num_papers_in_archive":1},{"url":"/dataset/obscurax","name":"ObscuraX","full_name":"ObscuraX","num_papers_in_archive":1},{"url":"/dataset/openapi-code-completion","name":"OpenAPI completion refined","full_name":"","num_papers_in_archive":1},{"url":"/dataset/pecc","name":"PECC","full_name":"PECC: Problem Extraction and Coding Challenges","num_papers_in_archive":1},{"url":"/dataset/projecteval","name":"ProjectEval","full_name":"","num_papers_in_archive":1},{"url":"/dataset/res-q","name":"RES-Q","full_name":"RES-Q: Evaluating Code-Editing Large Language Model Systems at the Repository Scale","num_papers_in_archive":1},{"url":"/dataset/soeval","name":"SOEval","full_name":"","num_papers_in_archive":1},{"url":"/dataset/taco-topics-in-algorithmic-code-generation","name":"TACO-BAAI","full_name":"Topics in Algorithmic Code generation dataset","num_papers_in_archive":1},{"url":"/dataset/tfix-s-code-patch-data","name":"TFix's Code Patches Data","full_name":"","num_papers_in_archive":1},{"url":"/dataset/turbulence","name":"Turbulence","full_name":"Turbulence","num_papers_in_archive":1},{"url":"/dataset/verified-smart-contract-code-comments","name":"Verified Smart Contract Code Comments","full_name":"","num_papers_in_archive":1},{"url":"/dataset/verified-smart-contracts","name":"Verified Smart Contracts","full_name":"","num_papers_in_archive":1},{"url":"/dataset/verireason-rtl-coder-7b-reasoning-tb","name":"Verireason-RTL-Coder_7b_reasoning_tb","full_name":"VeriReason Verilog Dataset with Reasoning, Testbench, and Simulation Results","num_papers_in_archive":1},{"url":"/dataset/verireason-rtl-coder-7b-reasoning-tb-simple","name":"Verireason-RTL-Coder_7b_reasoning_tb_simple","full_name":"Simple Problems of VeriReason Verilog Dataset with Reasoning, Testbench, and Simulation Results","num_papers_in_archive":1},{"url":"/dataset/webapp1k-duo-react","name":"WebApp1k-Duo-React","full_name":"","num_papers_in_archive":1},{"url":"/dataset/webgen-bench","name":"WebGen-Bench","full_name":"","num_papers_in_archive":1}],"subtasks":[{"url":"/task/class-level-code-generation","name":"Class-level Code Generation"},{"url":"/task/code-documentation-generation","name":"Code Documentation Generation"},{"url":"/task/code-translation","name":"Code Translation"},{"url":"/task/github-issue-resolution","name":"GitHub issue resolution"},{"url":"/task/library-oriented-code-generation","name":"Library-Oriented Code Generation"}],"parent_tasks":[],"papers":{"order":"repositories listed in the archive (most first), then date (newest first), then slug; the archive holds no stars","population":"every paper the archive tags with this task (distinct papers; spam titles not listed). 'of' counts the papers listed; until 2026-09-25 it counted only the papers with a code link, and that number is of_with_code","page":1,"pages_in_order":17,"rows_per_page":100,"shown":100,"of":1697,"of_with_code":745,"tagged_in_all":1697,"counts":{"archive_papers_tagged":1697,"with_a_code_link":745,"where_syntology_ran_a_sample":280,"not_listed_spam_title":0,"listed":1697,"listed_where_code_ran":280,"where_syntology_ran_a_sample_split":{"with_a_run_with_no_instrument_failure":224,"every_run_a_failure_of_syntologys_instrument":56,"listed_with_a_run_with_no_instrument_failure":224,"listed_every_run_a_failure_of_syntologys_instrument":56,"filter":{"states":["a run with no instrument failure","any run, instrument failures included"],"default":"a run with no instrument failure","note":"on the 'only where code ran' pages the default hides, in the browser, the rows where every run was a failure of Syntology's instrument; the second state shows them again. Rows are hidden, never re-ordered; these twins list every row"}},"definition":"distinct papers the archive tags; 'where Syntology ran a sample' counts papers with at least one harvested sample that ran, which is not a correctness claim"},"items":[{"url":"/paper/llama-open-and-efficient-foundation-language-1","slug":"llama-open-and-efficient-foundation-language-1","title":"LLaMA: Open and Efficient Foundation Language Models","date":"2023-02-27","arxiv_id":"2302.13971","repositories_listed":57,"syntology":{"n":58,"n_ran":37,"n_constructed":9,"n_ran_checked":25,"n_instrument":12,"n_unverified":21,"n_honours":3,"n_violates":0,"n_no_contract":22,"n_pointer_only":4,"phrase":"37 ran (of which 9 constructed an object rather than computing a result; 25 with no instrument failure: 3 honoured, 0 violated, 22 with no contract checked; 12 where Syntology's instrument failed) · 21 unverified","sample_list":"/paper/llama-open-and-efficient-foundation-language-1#ran","syntology_url":"https://syntology.ai/paper/2302.13971","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2302.13971"}},"official":{"repos":["facebookresearch/llama"],"state":"official: no sample here; runs from other or unrecorded repositories","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed","unlocated"]}}},{"url":"/paper/joint-face-detection-and-alignment-using","slug":"joint-face-detection-and-alignment-using","title":"Joint Face Detection and Alignment using Multi-task Cascaded Convolutional Networks","date":"2016-04-11","arxiv_id":"1604.02878","repositories_listed":42,"syntology":{"n":9,"n_ran":4,"n_constructed":0,"n_ran_checked":3,"n_instrument":1,"n_unverified":5,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":3,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 1 where Syntology's instrument failed) · 5 unverified","sample_list":"/paper/joint-face-detection-and-alignment-using#ran","syntology_url":"https://syntology.ai/paper/1604.02878","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1604.02878"}},"official":null}},{"url":"/paper/llama-2-open-foundation-and-fine-tuned-chat","slug":"llama-2-open-foundation-and-fine-tuned-chat","title":"Llama 2: Open Foundation and Fine-Tuned Chat Models","date":"2023-07-18","arxiv_id":"2307.09288","repositories_listed":19,"syntology":{"n":52,"n_ran":33,"n_constructed":7,"n_ran_checked":20,"n_instrument":13,"n_unverified":19,"n_honours":1,"n_violates":1,"n_no_contract":18,"n_pointer_only":20,"phrase":"33 ran (of which 7 constructed an object rather than computing a result; 20 with no instrument failure: 1 honoured, 1 violated, 18 with no contract checked; 13 where Syntology's instrument failed) · 19 unverified","sample_list":"/paper/llama-2-open-foundation-and-fine-tuned-chat#ran","syntology_url":"https://syntology.ai/paper/2307.09288","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2307.09288"}},"official":{"repos":["facebookresearch/llama"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/evaluating-large-language-models-trained-on","slug":"evaluating-large-language-models-trained-on","title":"Evaluating Large Language Models Trained on Code","date":"2021-07-07","arxiv_id":"2107.03374","repositories_listed":13,"syntology":{"n":39,"n_ran":26,"n_constructed":0,"n_ran_checked":22,"n_instrument":4,"n_unverified":13,"n_honours":1,"n_violates":0,"n_no_contract":21,"n_pointer_only":4,"phrase":"26 ran (of which 0 constructed an object rather than computing a result; 22 with no instrument failure: 1 honoured, 0 violated, 21 with no contract checked; 4 where Syntology's instrument failed) · 13 unverified","sample_list":"/paper/evaluating-large-language-models-trained-on#ran","syntology_url":"https://syntology.ai/paper/2107.03374","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2107.03374"}},"official":{"repos":["openai/human-eval"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":3,"ran_from_kinds":["found_in_text","listed","official"]}}},{"url":"/paper/gpt-4-technical-report-1","slug":"gpt-4-technical-report-1","title":"GPT-4 Technical Report","date":"2023-03-15","arxiv_id":"2303.08774","repositories_listed":11,"syntology":{"n":5,"n_ran":5,"n_constructed":0,"n_ran_checked":5,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":2,"n_no_contract":3,"n_pointer_only":1,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 0 honoured, 2 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/gpt-4-technical-report-1#ran","syntology_url":"https://syntology.ai/paper/2303.08774","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2303.08774"}},"official":{"repos":["openai/evals"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/pix2code-generating-code-from-a-graphical","slug":"pix2code-generating-code-from-a-graphical","title":"pix2code: Generating Code from a Graphical User Interface Screenshot","date":"2017-05-22","arxiv_id":"1705.07962","repositories_listed":11,"syntology":null},{"url":"/paper/swe-bench-can-language-models-resolve-real","slug":"swe-bench-can-language-models-resolve-real","title":"SWE-bench: Can Language Models Resolve Real-World GitHub Issues?","date":"2023-10-10","arxiv_id":"2310.06770","repositories_listed":8,"syntology":{"n":5,"n_ran":3,"n_constructed":0,"n_ran_checked":3,"n_instrument":0,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/swe-bench-can-language-models-resolve-real#ran","syntology_url":"https://syntology.ai/paper/2310.06770","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2310.06770"}},"official":null}},{"url":"/paper/a-conversational-paradigm-for-program","slug":"a-conversational-paradigm-for-program","title":"CodeGen: An Open Large Language Model for Code with Multi-Turn Program Synthesis","date":"2022-03-25","arxiv_id":"2203.13474","repositories_listed":8,"syntology":{"n":10,"n_ran":10,"n_constructed":0,"n_ran_checked":3,"n_instrument":7,"n_unverified":0,"n_honours":1,"n_violates":1,"n_no_contract":1,"n_pointer_only":1,"phrase":"10 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 1 honoured, 1 violated, 1 with no contract checked; 7 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/a-conversational-paradigm-for-program#ran","syntology_url":"https://syntology.ai/paper/2203.13474","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2203.13474"}},"official":{"repos":["salesforce/CodeGen","salesforce/jaxformer"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/santacoder-don-t-reach-for-the-stars","slug":"santacoder-don-t-reach-for-the-stars","title":"SantaCoder: don't reach for the stars!","date":"2023-01-09","arxiv_id":"2301.03988","repositories_listed":7,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":1,"n_instrument":2,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":3,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/santacoder-don-t-reach-for-the-stars#ran","syntology_url":"https://syntology.ai/paper/2301.03988","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2301.03988"}},"official":{"repos":["bigcode-project/bigcode-evaluation-harness","bigcode-project/starcoder"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/palm-scaling-language-modeling-with-pathways-1","slug":"palm-scaling-language-modeling-with-pathways-1","title":"PaLM: Scaling Language Modeling with Pathways","date":"2022-04-05","arxiv_id":"2204.02311","repositories_listed":7,"syntology":{"n":37,"n_ran":32,"n_constructed":16,"n_ran_checked":24,"n_instrument":8,"n_unverified":5,"n_honours":2,"n_violates":1,"n_no_contract":21,"n_pointer_only":0,"phrase":"32 ran (of which 16 constructed an object rather than computing a result; 24 with no instrument failure: 2 honoured, 1 violated, 21 with no contract checked; 8 where Syntology's instrument failed) · 5 unverified","sample_list":"/paper/palm-scaling-language-modeling-with-pathways-1#ran","syntology_url":"https://syntology.ai/paper/2204.02311","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2204.02311"}},"official":null}},{"url":"/paper/codexglue-a-machine-learning-benchmark","slug":"codexglue-a-machine-learning-benchmark","title":"CodeXGLUE: A Machine Learning Benchmark Dataset for Code Understanding and Generation","date":"2021-02-09","arxiv_id":"2102.04664","repositories_listed":7,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/codexglue-a-machine-learning-benchmark#ran","syntology_url":"https://syntology.ai/paper/2102.04664","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2102.04664"}},"official":{"repos":["microsoft/CodeXGLUE"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/structvae-tree-structured-latent-variable","slug":"structvae-tree-structured-latent-variable","title":"StructVAE: Tree-structured Latent Variable Models for Semi-supervised Semantic Parsing","date":"2018-06-20","arxiv_id":"1806.07832","repositories_listed":7,"syntology":null},{"url":"/paper/mixtral-of-experts","slug":"mixtral-of-experts","title":"Mixtral of Experts","date":"2024-01-08","arxiv_id":"2401.04088","repositories_listed":6,"syntology":{"n":5,"n_ran":5,"n_constructed":5,"n_ran_checked":5,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":5,"n_pointer_only":0,"phrase":"5 ran (of which 5 constructed an object rather than computing a result; 5 with no instrument failure: 0 honoured, 0 violated, 5 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified; every one of the 5 samples that ran constructed an object rather than computing a result","sample_list":"/paper/mixtral-of-experts#ran","syntology_url":"https://syntology.ai/paper/2401.04088","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2401.04088"}},"official":null}},{"url":"/paper/mistral-7b","slug":"mistral-7b","title":"Mistral 7B","date":"2023-10-10","arxiv_id":"2310.06825","repositories_listed":6,"syntology":{"n":11,"n_ran":10,"n_constructed":0,"n_ran_checked":10,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":10,"n_pointer_only":2,"phrase":"10 ran (of which 0 constructed an object rather than computing a result; 10 with no instrument failure: 0 honoured, 0 violated, 10 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/mistral-7b#ran","syntology_url":"https://syntology.ai/paper/2310.06825","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2310.06825"}},"official":{"repos":["mistralai/mistral-src"],"state":"official (archive's flag): 9 ran","n_ran":9,"n_constructed":0,"n_ran_no_instrument_failure":9,"n_unverified":1,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/a-parallel-corpus-of-python-functions-and","slug":"a-parallel-corpus-of-python-functions-and","title":"A parallel corpus of Python functions and documentation strings for automated code documentation and code generation","date":"2017-07-07","arxiv_id":"1707.02275","repositories_listed":6,"syntology":{"n":3,"n_ran":2,"n_constructed":0,"n_ran_checked":0,"n_instrument":2,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":3,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/a-parallel-corpus-of-python-functions-and#ran","syntology_url":"https://syntology.ai/paper/1707.02275","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1707.02275"}},"official":{"repos":["Avmb/code-docstring-corpus"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/a-syntactic-neural-model-for-general-purpose","slug":"a-syntactic-neural-model-for-general-purpose","title":"A Syntactic Neural Model for General-Purpose Code Generation","date":"2017-04-06","arxiv_id":"1704.01696","repositories_listed":6,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":2,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":2,"n_no_contract":0,"n_pointer_only":3,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 2 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/a-syntactic-neural-model-for-general-purpose#ran","syntology_url":"https://syntology.ai/paper/1704.01696","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1704.01696"}},"official":null}},{"url":"/paper/codet5-identifier-aware-unified-pre-trained","slug":"codet5-identifier-aware-unified-pre-trained","title":"CodeT5: Identifier-aware Unified Pre-trained Encoder-Decoder Models for Code Understanding and Generation","date":"2021-09-02","arxiv_id":"2109.00859","repositories_listed":5,"syntology":{"n":11,"n_ran":6,"n_constructed":0,"n_ran_checked":5,"n_instrument":1,"n_unverified":5,"n_honours":0,"n_violates":0,"n_no_contract":5,"n_pointer_only":0,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 0 honoured, 0 violated, 5 with no contract checked; 1 where Syntology's instrument failed) · 5 unverified","sample_list":"/paper/codet5-identifier-aware-unified-pre-trained#ran","syntology_url":"https://syntology.ai/paper/2109.00859","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2109.00859"}},"official":{"repos":["salesforce/codet5"],"state":"official: harvested, nothing ran","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":1,"ran_from_kinds":["listed"]}}},{"url":"/paper/content-enhanced-bert-based-text-to-sql","slug":"content-enhanced-bert-based-text-to-sql","title":"Content Enhanced BERT-based Text-to-SQL Generation","date":"2019-10-16","arxiv_id":"1910.07179","repositories_listed":5,"syntology":null},{"url":"/paper/qwen3-technical-report","slug":"qwen3-technical-report","title":"Qwen3 Technical Report","date":"2025-05-14","arxiv_id":"2505.09388","repositories_listed":4,"syntology":null},{"url":"/paper/aflow-automating-agentic-workflow-generation","slug":"aflow-automating-agentic-workflow-generation","title":"AFlow: Automating Agentic Workflow Generation","date":"2024-10-14","arxiv_id":"2410.10762","repositories_listed":4,"syntology":null},{"url":"/paper/bigcodebench-benchmarking-code-generation","slug":"bigcodebench-benchmarking-code-generation","title":"BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions","date":"2024-06-22","arxiv_id":"2406.15877","repositories_listed":4,"syntology":{"n":6,"n_ran":5,"n_constructed":0,"n_ran_checked":1,"n_instrument":4,"n_unverified":1,"n_honours":0,"n_violates":1,"n_no_contract":0,"n_pointer_only":1,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 1 violated, 0 with no contract checked; 4 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/bigcodebench-benchmarking-code-generation#ran","syntology_url":"https://syntology.ai/paper/2406.15877","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.15877"}},"official":{"repos":["bigcode-project/bigcodebench","bigcode-project/bigcodebench-annotation"],"state":"official (archive's flag): 5 ran","n_ran":5,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/starcoder-2-and-the-stack-v2-the-next","slug":"starcoder-2-and-the-stack-v2-the-next","title":"StarCoder 2 and The Stack v2: The Next Generation","date":"2024-02-29","arxiv_id":"2402.19173","repositories_listed":4,"syntology":null},{"url":"/paper/how-is-chatgpt-s-behavior-changing-over-time","slug":"how-is-chatgpt-s-behavior-changing-over-time","title":"How is ChatGPT's behavior changing over time?","date":"2023-07-18","arxiv_id":"2307.09009","repositories_listed":4,"syntology":{"n":6,"n_ran":6,"n_constructed":0,"n_ran_checked":4,"n_instrument":2,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":4,"n_pointer_only":5,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 4 with no instrument failure: 0 honoured, 0 violated, 4 with no contract checked; 2 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/how-is-chatgpt-s-behavior-changing-over-time#ran","syntology_url":"https://syntology.ai/paper/2307.09009","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2307.09009"}},"official":{"repos":["lchen001/llmdrift"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/wizardcoder-empowering-code-large-language","slug":"wizardcoder-empowering-code-large-language","title":"WizardCoder: Empowering Code Large Language Models with Evol-Instruct","date":"2023-06-14","arxiv_id":"2306.08568","repositories_listed":4,"syntology":{"n":7,"n_ran":6,"n_constructed":0,"n_ran_checked":4,"n_instrument":2,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":4,"n_pointer_only":2,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 4 with no instrument failure: 0 honoured, 0 violated, 4 with no contract checked; 2 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/wizardcoder-empowering-code-large-language#ran","syntology_url":"https://syntology.ai/paper/2306.08568","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2306.08568"}},"official":{"repos":["nlpxucan/wizardlm"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/starcoder-may-the-source-be-with-you","slug":"starcoder-may-the-source-be-with-you","title":"StarCoder: may the source be with you!","date":"2023-05-09","arxiv_id":"2305.06161","repositories_listed":4,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":1,"n_instrument":1,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/starcoder-may-the-source-be-with-you#ran","syntology_url":"https://syntology.ai/paper/2305.06161","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2305.06161"}},"official":null}},{"url":"/paper/training-a-helpful-and-harmless-assistant","slug":"training-a-helpful-and-harmless-assistant","title":"Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback","date":"2022-04-12","arxiv_id":"2204.05862","repositories_listed":4,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":1,"n_instrument":1,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":2,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/training-a-helpful-and-harmless-assistant#ran","syntology_url":"https://syntology.ai/paper/2204.05862","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2204.05862"}},"official":{"repos":["anthropics/hh-rlhf"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/an-empirical-cybersecurity-evaluation-of","slug":"an-empirical-cybersecurity-evaluation-of","title":"Asleep at the Keyboard? Assessing the Security of GitHub Copilot's Code Contributions","date":"2021-08-20","arxiv_id":"2108.09293","repositories_listed":4,"syntology":null},{"url":"/paper/tranx-a-transition-based-neural-abstract","slug":"tranx-a-transition-based-neural-abstract","title":"TRANX: A Transition-based Neural Abstract Syntax Parser for Semantic Parsing and Code Generation","date":"2018-10-05","arxiv_id":"1810.02720","repositories_listed":4,"syntology":{"n":7,"n_ran":7,"n_constructed":0,"n_ran_checked":6,"n_instrument":1,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":5,"n_pointer_only":2,"phrase":"7 ran (of which 0 constructed an object rather than computing a result; 6 with no instrument failure: 1 honoured, 0 violated, 5 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/tranx-a-transition-based-neural-abstract#ran","syntology_url":"https://syntology.ai/paper/1810.02720","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1810.02720"}},"official":{"repos":["pcyin/tranX"],"state":"official (archive's flag): 5 ran","n_ran":5,"n_constructed":0,"n_ran_no_instrument_failure":5,"n_unverified":0,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/chartgalaxy-a-dataset-for-infographic-chart","slug":"chartgalaxy-a-dataset-for-infographic-chart","title":"ChartGalaxy: A Dataset for Infographic Chart Understanding and Generation","date":"2025-05-24","arxiv_id":"2505.18668","repositories_listed":3,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":0,"n_instrument":3,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 3 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/chartgalaxy-a-dataset-for-infographic-chart#ran","syntology_url":"https://syntology.ai/paper/2505.18668","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2505.18668"}},"official":{"repos":["chartgalaxy/chartgalaxy"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/soap-enhancing-efficiency-of-generated-code","slug":"soap-enhancing-efficiency-of-generated-code","title":"EffiLearner: Enhancing Efficiency of Generated Code via Self-Optimization","date":"2024-05-24","arxiv_id":"2405.15189","repositories_listed":3,"syntology":{"n":17,"n_ran":12,"n_constructed":0,"n_ran_checked":8,"n_instrument":4,"n_unverified":5,"n_honours":6,"n_violates":0,"n_no_contract":2,"n_pointer_only":0,"phrase":"12 ran (of which 0 constructed an object rather than computing a result; 8 with no instrument failure: 6 honoured, 0 violated, 2 with no contract checked; 4 where Syntology's instrument failed) · 5 unverified","sample_list":"/paper/soap-enhancing-efficiency-of-generated-code#ran","syntology_url":"https://syntology.ai/paper/2405.15189","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2405.15189"}},"official":{"repos":["huangd1999/effilearner","huangd1999/soap"],"state":"official (archive's flag): 12 ran","n_ran":12,"n_constructed":0,"n_ran_no_instrument_failure":8,"n_unverified":5,"ran_from_kinds":["official"]}}},{"url":"/paper/mmcode-evaluating-multi-modal-code-large","slug":"mmcode-evaluating-multi-modal-code-large","title":"MMCode: Benchmarking Multimodal Large Language Models for Code Generation with Visually Rich Programming Problems","date":"2024-04-15","arxiv_id":"2404.09486","repositories_listed":3,"syntology":{"n":16,"n_ran":13,"n_constructed":0,"n_ran_checked":3,"n_instrument":10,"n_unverified":3,"n_honours":1,"n_violates":0,"n_no_contract":2,"n_pointer_only":16,"phrase":"13 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 1 honoured, 0 violated, 2 with no contract checked; 10 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/mmcode-evaluating-multi-modal-code-large#ran","syntology_url":"https://syntology.ai/paper/2404.09486","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2404.09486"}},"official":{"repos":["happylkx/mmcode","likaixin2000/mmcode"],"state":"official (archive's flag): 13 ran","n_ran":13,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":3,"ran_from_kinds":["official"]}}},{"url":"/paper/knowledge-fusion-of-large-language-models","slug":"knowledge-fusion-of-large-language-models","title":"Knowledge Fusion of Large Language Models","date":"2024-01-19","arxiv_id":"2401.10491","repositories_listed":3,"syntology":{"n":4,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":3,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":4,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/knowledge-fusion-of-large-language-models#ran","syntology_url":"https://syntology.ai/paper/2401.10491","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2401.10491"}},"official":{"repos":["fanqiwan/fusellm"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/domain-specific-code-language-models","slug":"domain-specific-code-language-models","title":"MonoCoder: Domain-Specific Code Language Model for HPC Codes and Tasks","date":"2023-12-20","arxiv_id":"2312.13322","repositories_listed":3,"syntology":null},{"url":"/paper/magicoder-source-code-is-all-you-need","slug":"magicoder-source-code-is-all-you-need","title":"Magicoder: Empowering Code Generation with OSS-Instruct","date":"2023-12-04","arxiv_id":"2312.02120","repositories_listed":3,"syntology":{"n":2,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/magicoder-source-code-is-all-you-need#ran","syntology_url":"https://syntology.ai/paper/2312.02120","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2312.02120"}},"official":{"repos":["ise-uiuc/magicoder"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/askit-unified-programming-interface-for","slug":"askit-unified-programming-interface-for","title":"AskIt: Unified Programming Interface for Programming with Large Language Models","date":"2023-08-29","arxiv_id":"2308.15645","repositories_listed":3,"syntology":null},{"url":"/paper/octopack-instruction-tuning-code-large","slug":"octopack-instruction-tuning-code-large","title":"OctoPack: Instruction Tuning Code Large Language Models","date":"2023-08-14","arxiv_id":"2308.07124","repositories_listed":3,"syntology":{"n":13,"n_ran":9,"n_constructed":0,"n_ran_checked":3,"n_instrument":6,"n_unverified":4,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":0,"phrase":"9 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 6 where Syntology's instrument failed) · 4 unverified","sample_list":"/paper/octopack-instruction-tuning-code-large#ran","syntology_url":"https://syntology.ai/paper/2308.07124","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2308.07124"}},"official":{"repos":["bigcode-project/bigcode-evaluation-harness","bigcode-project/octopack"],"state":"official (archive's flag): 9 ran","n_ran":9,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":4,"ran_from_kinds":["official"]}}},{"url":"/paper/factool-factuality-detection-in-generative-ai","slug":"factool-factuality-detection-in-generative-ai","title":"FacTool: Factuality Detection in Generative AI -- A Tool Augmented Framework for Multi-Task and Multi-Domain Scenarios","date":"2023-07-25","arxiv_id":"2307.13528","repositories_listed":3,"syntology":{"n":14,"n_ran":11,"n_constructed":0,"n_ran_checked":9,"n_instrument":2,"n_unverified":3,"n_honours":0,"n_violates":0,"n_no_contract":9,"n_pointer_only":0,"phrase":"11 ran (of which 0 constructed an object rather than computing a result; 9 with no instrument failure: 0 honoured, 0 violated, 9 with no contract checked; 2 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/factool-factuality-detection-in-generative-ai#ran","syntology_url":"https://syntology.ai/paper/2307.13528","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2307.13528"}},"official":{"repos":["gair-nlp/factool"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/recode-robustness-evaluation-of-code","slug":"recode-robustness-evaluation-of-code","title":"ReCode: Robustness Evaluation of Code Generation Models","date":"2022-12-20","arxiv_id":"2212.10264","repositories_listed":3,"syntology":{"n":5,"n_ran":5,"n_constructed":0,"n_ran_checked":3,"n_instrument":2,"n_unverified":0,"n_honours":3,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 3 honoured, 0 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/recode-robustness-evaluation-of-code#ran","syntology_url":"https://syntology.ai/paper/2212.10264","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2212.10264"}},"official":{"repos":["amazon-science/recode"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":0,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/incoder-a-generative-model-for-code-infilling","slug":"incoder-a-generative-model-for-code-infilling","title":"InCoder: A Generative Model for Code Infilling and Synthesis","date":"2022-04-12","arxiv_id":"2204.05999","repositories_listed":3,"syntology":{"n":10,"n_ran":6,"n_constructed":0,"n_ran_checked":3,"n_instrument":3,"n_unverified":4,"n_honours":0,"n_violates":1,"n_no_contract":2,"n_pointer_only":2,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 1 violated, 2 with no contract checked; 3 where Syntology's instrument failed) · 4 unverified","sample_list":"/paper/incoder-a-generative-model-for-code-infilling#ran","syntology_url":"https://syntology.ai/paper/2204.05999","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2204.05999"}},"official":{"repos":["dpfried/incoder"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/programming-puzzles","slug":"programming-puzzles","title":"Programming Puzzles","date":"2021-06-10","arxiv_id":"2106.05784","repositories_listed":3,"syntology":null},{"url":"/paper/measuring-coding-challenge-competence-with","slug":"measuring-coding-challenge-competence-with","title":"Measuring Coding Challenge Competence With APPS","date":"2021-05-20","arxiv_id":"2105.09938","repositories_listed":3,"syntology":{"n":11,"n_ran":7,"n_constructed":0,"n_ran_checked":5,"n_instrument":2,"n_unverified":4,"n_honours":0,"n_violates":0,"n_no_contract":5,"n_pointer_only":2,"phrase":"7 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 0 honoured, 0 violated, 5 with no contract checked; 2 where Syntology's instrument failed) · 4 unverified","sample_list":"/paper/measuring-coding-challenge-competence-with#ran","syntology_url":"https://syntology.ai/paper/2105.09938","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2105.09938"}},"official":{"repos":["hendrycks/apps"],"state":"official (archive's flag): 7 ran","n_ran":7,"n_constructed":0,"n_ran_no_instrument_failure":5,"n_unverified":4,"ran_from_kinds":["official"]}}},{"url":"/paper/prophetnet-x-large-scale-pre-training-models","slug":"prophetnet-x-large-scale-pre-training-models","title":"ProphetNet-X: Large-Scale Pre-training Models for English, Chinese, Multi-lingual, Dialog, and Code Generation","date":"2021-04-16","arxiv_id":"2104.08006","repositories_listed":3,"syntology":null},{"url":"/paper/co-evolving-llm-coder-and-unit-tester-via","slug":"co-evolving-llm-coder-and-unit-tester-via","title":"Co-Evolving LLM Coder and Unit Tester via Reinforcement Learning","date":"2025-06-03","arxiv_id":"2506.03136","repositories_listed":2,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":0,"n_instrument":2,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":2,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/co-evolving-llm-coder-and-unit-tester-via#ran","syntology_url":"https://syntology.ai/paper/2506.03136","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2506.03136"}},"official":{"repos":["gen-verse/cure"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/2505-23881","slug":"2505-23881","title":"Using Reasoning Models to Generate Search Heuristics that Solve Open Instances of Combinatorial Design Problems","date":"2025-05-29","arxiv_id":"2505.23881","repositories_listed":2,"syntology":null},{"url":"/paper/dkv-cache-the-cache-for-diffusion-language","slug":"dkv-cache-the-cache-for-diffusion-language","title":"dKV-Cache: The Cache for Diffusion Language Models","date":"2025-05-21","arxiv_id":"2505.15781","repositories_listed":2,"syntology":null},{"url":"/paper/r-d-agent-quant-a-multi-agent-framework-for","slug":"r-d-agent-quant-a-multi-agent-framework-for","title":"R&D-Agent-Quant: A Multi-Agent Framework for Data-Centric Factors and Model Joint Optimization","date":"2025-05-21","arxiv_id":"2505.15155","repositories_listed":2,"syntology":{"n":10,"n_ran":7,"n_constructed":0,"n_ran_checked":7,"n_instrument":0,"n_unverified":3,"n_honours":0,"n_violates":0,"n_no_contract":7,"n_pointer_only":0,"phrase":"7 ran (of which 0 constructed an object rather than computing a result; 7 with no instrument failure: 0 honoured, 0 violated, 7 with no contract checked; 0 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/r-d-agent-quant-a-multi-agent-framework-for#ran","syntology_url":"https://syntology.ai/paper/2505.15155","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2505.15155"}},"official":{"repos":["microsoft/rd-agent","microsoft/qlib"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":2,"ran_from_kinds":["found_in_text","official"]}}},{"url":"/paper/clever-a-curated-benchmark-for-formally","slug":"clever-a-curated-benchmark-for-formally","title":"CLEVER: A Curated Benchmark for Formally Verified Code Generation","date":"2025-05-20","arxiv_id":"2505.13938","repositories_listed":2,"syntology":null},{"url":"/paper/data-efficient-llm-fine-tuning-for-code","slug":"data-efficient-llm-fine-tuning-for-code","title":"Data-efficient LLM Fine-tuning for Code Generation","date":"2025-04-17","arxiv_id":"2504.12687","repositories_listed":2,"syntology":null},{"url":"/paper/faircode-evaluating-social-bias-of-llms-in","slug":"faircode-evaluating-social-bias-of-llms-in","title":"FairCoder: Evaluating Social Bias of LLMs in Code Generation","date":"2025-01-09","arxiv_id":"2501.05396","repositories_listed":2,"syntology":null},{"url":"/paper/search-o1-agentic-search-enhanced-large","slug":"search-o1-agentic-search-enhanced-large","title":"Search-o1: Agentic Search-Enhanced Large Reasoning Models","date":"2025-01-09","arxiv_id":"2501.05366","repositories_listed":2,"syntology":null},{"url":"/paper/how-well-do-llms-generate-code-for-different","slug":"how-well-do-llms-generate-code-for-different","title":"Top General Performance = Top Domain Performance? DomainCodeBench: A Multi-domain Code Generation Benchmark","date":"2024-12-24","arxiv_id":"2412.18573","repositories_listed":2,"syntology":null},{"url":"/paper/selfcodealign-self-alignment-for-code","slug":"selfcodealign-self-alignment-for-code","title":"SelfCodeAlign: Self-Alignment for Code Generation","date":"2024-10-31","arxiv_id":"2410.24198","repositories_listed":2,"syntology":{"n":37,"n_ran":30,"n_constructed":3,"n_ran_checked":21,"n_instrument":9,"n_unverified":7,"n_honours":1,"n_violates":0,"n_no_contract":20,"n_pointer_only":0,"phrase":"30 ran (of which 3 constructed an object rather than computing a result; 21 with no instrument failure: 1 honoured, 0 violated, 20 with no contract checked; 9 where Syntology's instrument failed) · 7 unverified","sample_list":"/paper/selfcodealign-self-alignment-for-code#ran","syntology_url":"https://syntology.ai/paper/2410.24198","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2410.24198"}},"official":{"repos":["bigcode-project/selfcodealign"],"state":"official (archive's flag): 9 ran","n_ran":9,"n_constructed":3,"n_ran_no_instrument_failure":3,"n_unverified":3,"ran_from_kinds":["community","listed","official"]}}},{"url":"/paper/effi-code-unleashing-code-efficiency-in","slug":"effi-code-unleashing-code-efficiency-in","title":"EffiCoder: Enhancing Code Generation in Large Language Models through Efficiency-Aware Fine-tuning","date":"2024-10-14","arxiv_id":"2410.10209","repositories_listed":2,"syntology":{"n":15,"n_ran":15,"n_constructed":0,"n_ran_checked":13,"n_instrument":2,"n_unverified":0,"n_honours":9,"n_violates":0,"n_no_contract":4,"n_pointer_only":0,"phrase":"15 ran (of which 0 constructed an object rather than computing a result; 13 with no instrument failure: 9 honoured, 0 violated, 4 with no contract checked; 2 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/effi-code-unleashing-code-efficiency-in#ran","syntology_url":"https://syntology.ai/paper/2410.10209","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2410.10209"}},"official":{"repos":["huangd1999/effi-code","huangd1999/efficoder"],"state":"official (archive's flag): 15 ran","n_ran":15,"n_constructed":0,"n_ran_no_instrument_failure":13,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/swiftkv-fast-prefill-optimized-inference-with","slug":"swiftkv-fast-prefill-optimized-inference-with","title":"SwiftKV: Fast Prefill-Optimized Inference with Knowledge-Preserving Model Transformation","date":"2024-10-04","arxiv_id":"2410.03960","repositories_listed":2,"syntology":null},{"url":"/paper/qwen2-5-coder-technical-report","slug":"qwen2-5-coder-technical-report","title":"Qwen2.5-Coder Technical Report","date":"2024-09-18","arxiv_id":"2409.12186","repositories_listed":2,"syntology":{"n":9,"n_ran":8,"n_constructed":0,"n_ran_checked":6,"n_instrument":2,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":6,"n_pointer_only":9,"phrase":"8 ran (of which 0 constructed an object rather than computing a result; 6 with no instrument failure: 0 honoured, 0 violated, 6 with no contract checked; 2 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/qwen2-5-coder-technical-report#ran","syntology_url":"https://syntology.ai/paper/2409.12186","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2409.12186"}},"official":{"repos":["qwenlm/qwen2.5-coder"],"state":"official (archive's flag): 8 ran","n_ran":8,"n_constructed":0,"n_ran_no_instrument_failure":6,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/hyperagent-generalist-software-engineering","slug":"hyperagent-generalist-software-engineering","title":"HyperAgent: Generalist Software Engineering Agents to Solve Coding Tasks at Scale","date":"2024-09-09","arxiv_id":"2409.16299","repositories_listed":2,"syntology":null},{"url":"/paper/planning-in-natural-language-improves-llm","slug":"planning-in-natural-language-improves-llm","title":"Planning In Natural Language Improves LLM Search For Code Generation","date":"2024-09-05","arxiv_id":"2409.03733","repositories_listed":2,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":1,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/planning-in-natural-language-improves-llm#ran","syntology_url":"https://syntology.ai/paper/2409.03733","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2409.03733"}},"official":{"repos":["scaleapi/plansearch"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/examination-of-code-generated-by-large","slug":"examination-of-code-generated-by-large","title":"Examination of Code generated by Large Language Models","date":"2024-08-29","arxiv_id":"2408.16601","repositories_listed":2,"syntology":null},{"url":"/paper/2408-02487","slug":"2408-02487","title":"LiCoEval: Evaluating LLMs on License Compliance in Code Generation","date":"2024-08-05","arxiv_id":"2408.02487","repositories_listed":2,"syntology":null},{"url":"/paper/ecco-can-we-improve-model-generated-code","slug":"ecco-can-we-improve-model-generated-code","title":"ECCO: Can We Improve Model-Generated Code Efficiency Without Sacrificing Functional Correctness?","date":"2024-07-19","arxiv_id":"2407.14044","repositories_listed":2,"syntology":{"n":14,"n_ran":10,"n_constructed":0,"n_ran_checked":6,"n_instrument":4,"n_unverified":4,"n_honours":0,"n_violates":0,"n_no_contract":6,"n_pointer_only":5,"phrase":"10 ran (of which 0 constructed an object rather than computing a result; 6 with no instrument failure: 0 honoured, 0 violated, 6 with no contract checked; 4 where Syntology's instrument failed) · 4 unverified","sample_list":"/paper/ecco-can-we-improve-model-generated-code#ran","syntology_url":"https://syntology.ai/paper/2407.14044","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2407.14044"}},"official":{"repos":["codeeff/ecco"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":2,"ran_from_kinds":["found_in_text","official"]}}},{"url":"/paper/beyond-correctness-benchmarking-multi","slug":"beyond-correctness-benchmarking-multi","title":"Beyond Correctness: Benchmarking Multi-dimensional Code Generation for Large Language Models","date":"2024-07-16","arxiv_id":"2407.11470","repositories_listed":2,"syntology":{"n":16,"n_ran":12,"n_constructed":0,"n_ran_checked":5,"n_instrument":7,"n_unverified":4,"n_honours":1,"n_violates":1,"n_no_contract":3,"n_pointer_only":0,"phrase":"12 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 1 honoured, 1 violated, 3 with no contract checked; 7 where Syntology's instrument failed) · 4 unverified","sample_list":"/paper/beyond-correctness-benchmarking-multi#ran","syntology_url":"https://syntology.ai/paper/2407.11470","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2407.11470"}},"official":{"repos":["jszheng21/race"],"state":"official (archive's flag): 12 ran","n_ran":12,"n_constructed":0,"n_ran_no_instrument_failure":5,"n_unverified":4,"ran_from_kinds":["official"]}}},{"url":"/paper/buffer-of-thoughts-thought-augmented","slug":"buffer-of-thoughts-thought-augmented","title":"Buffer of Thoughts: Thought-Augmented Reasoning with Large Language Models","date":"2024-06-06","arxiv_id":"2406.04271","repositories_listed":2,"syntology":{"n":6,"n_ran":5,"n_constructed":1,"n_ran_checked":3,"n_instrument":2,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":1,"phrase":"5 ran (of which 1 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 2 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/buffer-of-thoughts-thought-augmented#ran","syntology_url":"https://syntology.ai/paper/2406.04271","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.04271"}},"official":{"repos":["yangling0818/buffer-of-thought-llm"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":1,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/a-survey-on-large-language-models-for-code","slug":"a-survey-on-large-language-models-for-code","title":"A Survey on Large Language Models for Code Generation","date":"2024-06-01","arxiv_id":"2406.00515","repositories_listed":2,"syntology":{"n":1,"n_ran":0,"n_constructed":0,"n_ran_checked":0,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"0 ran · 1 unverified","sample_list":"/paper/a-survey-on-large-language-models-for-code#ran","syntology_url":"https://syntology.ai/paper/2406.00515","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.00515"}},"official":{"repos":["juyongjiang/codellmsurvey"],"state":"official: not harvested","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":[]}}},{"url":"/paper/alchemistcoder-harmonizing-and-eliciting-code","slug":"alchemistcoder-harmonizing-and-eliciting-code","title":"AlchemistCoder: Harmonizing and Eliciting Code Capability by Hindsight Tuning on Multi-source Data","date":"2024-05-29","arxiv_id":"2405.19265","repositories_listed":2,"syntology":null},{"url":"/paper/optimizing-large-language-models-for-openapi","slug":"optimizing-large-language-models-for-openapi","title":"Optimizing Large Language Models for OpenAPI Code Completion","date":"2024-05-24","arxiv_id":"2405.15729","repositories_listed":2,"syntology":null},{"url":"/paper/mapcoder-multi-agent-code-generation-for","slug":"mapcoder-multi-agent-code-generation-for","title":"MapCoder: Multi-Agent Code Generation for Competitive Problem Solving","date":"2024-05-18","arxiv_id":"2405.11403","repositories_listed":2,"syntology":{"n":10,"n_ran":6,"n_constructed":0,"n_ran_checked":6,"n_instrument":0,"n_unverified":4,"n_honours":0,"n_violates":0,"n_no_contract":6,"n_pointer_only":0,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 6 with no instrument failure: 0 honoured, 0 violated, 6 with no contract checked; 0 where Syntology's instrument failed) · 4 unverified","sample_list":"/paper/mapcoder-multi-agent-code-generation-for#ran","syntology_url":"https://syntology.ai/paper/2405.11403","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2405.11403"}},"official":{"repos":["md-ashraful-pramanik/mapcoder"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/granite-code-models-a-family-of-open","slug":"granite-code-models-a-family-of-open","title":"Granite Code Models: A Family of Open Foundation Models for Code Intelligence","date":"2024-05-07","arxiv_id":"2405.04324","repositories_listed":2,"syntology":{"n":4,"n_ran":4,"n_constructed":0,"n_ran_checked":4,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":4,"n_pointer_only":4,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 4 with no instrument failure: 0 honoured, 0 violated, 4 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/granite-code-models-a-family-of-open#ran","syntology_url":"https://syntology.ai/paper/2405.04324","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2405.04324"}},"official":{"repos":["ibm-granite/granite-code-models"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/constrained-decoding-for-secure-code","slug":"constrained-decoding-for-secure-code","title":"Constrained Decoding for Secure Code Generation","date":"2024-04-30","arxiv_id":"2405.00218","repositories_listed":2,"syntology":{"n":13,"n_ran":9,"n_constructed":0,"n_ran_checked":5,"n_instrument":4,"n_unverified":4,"n_honours":0,"n_violates":1,"n_no_contract":4,"n_pointer_only":1,"phrase":"9 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 0 honoured, 1 violated, 4 with no contract checked; 4 where Syntology's instrument failed) · 4 unverified","sample_list":"/paper/constrained-decoding-for-secure-code#ran","syntology_url":"https://syntology.ai/paper/2405.00218","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2405.00218"}},"official":{"repos":["dynamite321/codeguardplus"],"state":"official (archive's flag): 9 ran","n_ran":9,"n_constructed":0,"n_ran_no_instrument_failure":5,"n_unverified":4,"ran_from_kinds":["official"]}}},{"url":"/paper/codebenchgen-creating-scalable-execution","slug":"codebenchgen-creating-scalable-execution","title":"CodeBenchGen: Creating Scalable Execution-based Code Generation Benchmarks","date":"2024-03-31","arxiv_id":"2404.00566","repositories_listed":2,"syntology":null},{"url":"/paper/devbench-a-comprehensive-benchmark-for","slug":"devbench-a-comprehensive-benchmark-for","title":"Prompting Large Language Models to Tackle the Full Software Development Lifecycle: A Case Study","date":"2024-03-13","arxiv_id":"2403.08604","repositories_listed":2,"syntology":{"n":4,"n_ran":4,"n_constructed":0,"n_ran_checked":3,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":2,"n_no_contract":1,"n_pointer_only":3,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 2 violated, 1 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/devbench-a-comprehensive-benchmark-for#ran","syntology_url":"https://syntology.ai/paper/2403.08604","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2403.08604"}},"official":{"repos":["open-compass/devbench","open-compass/deveval"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/inficoder-eval-systematically-evaluating-the","slug":"inficoder-eval-systematically-evaluating-the","title":"InfiBench: Evaluating the Question-Answering Capabilities of Code Large Language Models","date":"2024-03-11","arxiv_id":"2404.07940","repositories_listed":2,"syntology":{"n":13,"n_ran":11,"n_constructed":0,"n_ran_checked":8,"n_instrument":3,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":8,"n_pointer_only":0,"phrase":"11 ran (of which 0 constructed an object rather than computing a result; 8 with no instrument failure: 0 honoured, 0 violated, 8 with no contract checked; 3 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/inficoder-eval-systematically-evaluating-the#ran","syntology_url":"https://syntology.ai/paper/2404.07940","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2404.07940"}},"official":{"repos":["infi-coder/infibench-evaluation-harness","infi-coder/infibench-evaluator"],"state":"official (archive's flag): 11 ran","n_ran":11,"n_constructed":0,"n_ran_no_instrument_failure":8,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/grillbot-in-practice-lessons-and-tradeoffs","slug":"grillbot-in-practice-lessons-and-tradeoffs","title":"GRILLBot In Practice: Lessons and Tradeoffs Deploying Large Language Models for Adaptable Conversational Task Assistants","date":"2024-02-12","arxiv_id":"2402.07647","repositories_listed":2,"syntology":null},{"url":"/paper/code-generation-with-alphacodium-from-prompt","slug":"code-generation-with-alphacodium-from-prompt","title":"Code Generation with AlphaCodium: From Prompt Engineering to Flow Engineering","date":"2024-01-16","arxiv_id":"2401.08500","repositories_listed":2,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":1,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":1,"n_no_contract":0,"n_pointer_only":2,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 1 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/code-generation-with-alphacodium-from-prompt#ran","syntology_url":"https://syntology.ai/paper/2401.08500","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2401.08500"}},"official":{"repos":["codium-ai/alphacodium"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/parameter-efficient-sparsity-crafting-from","slug":"parameter-efficient-sparsity-crafting-from","title":"Parameter-Efficient Sparsity Crafting from Dense to Mixture-of-Experts for Instruction Tuning on General Tasks","date":"2024-01-05","arxiv_id":"2401.02731","repositories_listed":2,"syntology":null},{"url":"/paper/astraios-parameter-efficient-instruction","slug":"astraios-parameter-efficient-instruction","title":"Astraios: Parameter-Efficient Instruction Tuning Code Large Language Models","date":"2024-01-01","arxiv_id":"2401.00788","repositories_listed":2,"syntology":{"n":7,"n_ran":7,"n_constructed":0,"n_ran_checked":1,"n_instrument":6,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":4,"phrase":"7 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 6 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/astraios-parameter-efficient-instruction#ran","syntology_url":"https://syntology.ai/paper/2401.00788","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2401.00788"}},"official":{"repos":["bigcode-project/astraios","agi-edgerunners/llm-adapters"],"state":"official (archive's flag): 7 ran","n_ran":7,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/zeroquant-4-2-redefining-llms-quantization","slug":"zeroquant-4-2-redefining-llms-quantization","title":"ZeroQuant(4+2): Redefining LLMs Quantization with a New FP6-Centric Strategy for Diverse Generative Tasks","date":"2023-12-14","arxiv_id":"2312.08583","repositories_listed":2,"syntology":null},{"url":"/paper/yuan-2-0-a-large-language-model-with","slug":"yuan-2-0-a-large-language-model-with","title":"YUAN 2.0: A Large Language Model with Localized Filtering-based Attention","date":"2023-11-27","arxiv_id":"2311.15786","repositories_listed":2,"syntology":null},{"url":"/paper/clover-closed-loop-verifiable-code-generation","slug":"clover-closed-loop-verifiable-code-generation","title":"Clover: Closed-Loop Verifiable Code Generation","date":"2023-10-26","arxiv_id":"2310.17807","repositories_listed":2,"syntology":null},{"url":"/paper/automatic-unit-test-data-generation-and-actor","slug":"automatic-unit-test-data-generation-and-actor","title":"Automatic Unit Test Data Generation and Actor-Critic Reinforcement Learning for Code Synthesis","date":"2023-10-20","arxiv_id":"2310.13669","repositories_listed":2,"syntology":null},{"url":"/paper/neural-rankers-for-code-generation-via-inter","slug":"neural-rankers-for-code-generation-via-inter","title":"Functional Overlap Reranking for Neural Code Generation","date":"2023-10-16","arxiv_id":"2311.03366","repositories_listed":2,"syntology":null},{"url":"/paper/how-abilities-in-large-language-models-are","slug":"how-abilities-in-large-language-models-are","title":"How Abilities in Large Language Models are Affected by Supervised Fine-tuning Data Composition","date":"2023-10-09","arxiv_id":"2310.05492","repositories_listed":2,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":1,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":2,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/how-abilities-in-large-language-models-are#ran","syntology_url":"https://syntology.ai/paper/2310.05492","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2310.05492"}},"official":{"repos":["ofa-sys/gsm8k-screl"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/language-agent-tree-search-unifies-reasoning","slug":"language-agent-tree-search-unifies-reasoning","title":"Language Agent Tree Search Unifies Reasoning Acting and Planning in Language Models","date":"2023-10-06","arxiv_id":"2310.04406","repositories_listed":2,"syntology":{"n":9,"n_ran":9,"n_constructed":0,"n_ran_checked":5,"n_instrument":4,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":4,"n_pointer_only":2,"phrase":"9 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 1 honoured, 0 violated, 4 with no contract checked; 4 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/language-agent-tree-search-unifies-reasoning#ran","syntology_url":"https://syntology.ai/paper/2310.04406","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2310.04406"}},"official":{"repos":["lapisrocks/languageagenttreesearch","andyz245/LanguageAgentTreeSearch"],"state":"official (archive's flag): 9 ran","n_ran":9,"n_constructed":0,"n_ran_no_instrument_failure":5,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/cat-lm-training-language-models-on-aligned","slug":"cat-lm-training-language-models-on-aligned","title":"CAT-LM: Training Language Models on Aligned Code And Tests","date":"2023-10-02","arxiv_id":"2310.01602","repositories_listed":2,"syntology":null},{"url":"/paper/l2mac-large-language-model-automatic-computer","slug":"l2mac-large-language-model-automatic-computer","title":"L2MAC: Large Language Model Automatic Computer for Extensive Code Generation","date":"2023-10-02","arxiv_id":"2310.02003","repositories_listed":2,"syntology":{"n":14,"n_ran":10,"n_constructed":0,"n_ran_checked":10,"n_instrument":0,"n_unverified":4,"n_honours":0,"n_violates":0,"n_no_contract":10,"n_pointer_only":0,"phrase":"10 ran (of which 0 constructed an object rather than computing a result; 10 with no instrument failure: 0 honoured, 0 violated, 10 with no contract checked; 0 where Syntology's instrument failed) · 4 unverified","sample_list":"/paper/l2mac-large-language-model-automatic-computer#ran","syntology_url":"https://syntology.ai/paper/2310.02003","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2310.02003"}},"official":{"repos":["samholt/l2mac","vanderschaarlab/l2mac"],"state":"official (archive's flag): 10 ran","n_ran":10,"n_constructed":0,"n_ran_no_instrument_failure":10,"n_unverified":4,"ran_from_kinds":["official"]}}},{"url":"/paper/code-llama-open-foundation-models-for-code","slug":"code-llama-open-foundation-models-for-code","title":"Code Llama: Open Foundation Models for Code","date":"2023-08-24","arxiv_id":"2308.12950","repositories_listed":2,"syntology":{"n":5,"n_ran":3,"n_constructed":0,"n_ran_checked":1,"n_instrument":2,"n_unverified":2,"n_honours":0,"n_violates":1,"n_no_contract":0,"n_pointer_only":4,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 1 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/code-llama-open-foundation-models-for-code#ran","syntology_url":"https://syntology.ai/paper/2308.12950","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2308.12950"}},"official":{"repos":["facebookresearch/codellama"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/classeval-a-manually-crafted-benchmark-for","slug":"classeval-a-manually-crafted-benchmark-for","title":"ClassEval: A Manually-Crafted Benchmark for Evaluating LLMs on Class-level Code Generation","date":"2023-08-03","arxiv_id":"2308.01861","repositories_listed":2,"syntology":null},{"url":"/paper/metagpt-meta-programming-for-multi-agent","slug":"metagpt-meta-programming-for-multi-agent","title":"MetaGPT: Meta Programming for A Multi-Agent Collaborative Framework","date":"2023-08-01","arxiv_id":"2308.00352","repositories_listed":2,"syntology":null},{"url":"/paper/intercode-standardizing-and-benchmarking","slug":"intercode-standardizing-and-benchmarking","title":"InterCode: Standardizing and Benchmarking Interactive Coding with Execution Feedback","date":"2023-06-26","arxiv_id":"2306.14898","repositories_listed":2,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":0,"n_instrument":2,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/intercode-standardizing-and-benchmarking#ran","syntology_url":"https://syntology.ai/paper/2306.14898","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2306.14898"}},"official":{"repos":["princeton-nlp/intercode"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/knowledge-driven-robot-program-synthesis-from","slug":"knowledge-driven-robot-program-synthesis-from","title":"Knowledge-Driven Robot Program Synthesis from Human VR Demonstrations","date":"2023-06-05","arxiv_id":"2306.02739","repositories_listed":2,"syntology":null},{"url":"/paper/flexible-grammar-based-constrained-decoding","slug":"flexible-grammar-based-constrained-decoding","title":"Grammar-Constrained Decoding for Structured NLP Tasks without Finetuning","date":"2023-05-23","arxiv_id":"2305.13971","repositories_listed":2,"syntology":{"n":14,"n_ran":7,"n_constructed":0,"n_ran_checked":5,"n_instrument":2,"n_unverified":7,"n_honours":0,"n_violates":1,"n_no_contract":4,"n_pointer_only":0,"phrase":"7 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 0 honoured, 1 violated, 4 with no contract checked; 2 where Syntology's instrument failed) · 7 unverified","sample_list":"/paper/flexible-grammar-based-constrained-decoding#ran","syntology_url":"https://syntology.ai/paper/2305.13971","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2305.13971"}},"official":{"repos":["epfl-dlab/gcd"],"state":"official (archive's flag): 7 ran","n_ran":7,"n_constructed":0,"n_ran_no_instrument_failure":5,"n_unverified":7,"ran_from_kinds":["official"]}}},{"url":"/paper/mpi-rical-data-driven-mpi-distributed","slug":"mpi-rical-data-driven-mpi-distributed","title":"MPI-rical: Data-Driven MPI Distributed Parallelism Assistance with Transformers","date":"2023-05-16","arxiv_id":"2305.09438","repositories_listed":2,"syntology":null},{"url":"/paper/codet5-open-code-large-language-models-for","slug":"codet5-open-code-large-language-models-for","title":"CodeT5+: Open Code Large Language Models for Code Understanding and Generation","date":"2023-05-13","arxiv_id":"2305.07922","repositories_listed":2,"syntology":{"n":4,"n_ran":3,"n_constructed":0,"n_ran_checked":2,"n_instrument":1,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 1 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/codet5-open-code-large-language-models-for#ran","syntology_url":"https://syntology.ai/paper/2305.07922","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2305.07922"}},"official":{"repos":["salesforce/codet5"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/large-language-models-are-state-of-the-art-1","slug":"large-language-models-are-state-of-the-art-1","title":"ICE-Score: Instructing Large Language Models to Evaluate Code","date":"2023-04-27","arxiv_id":"2304.14317","repositories_listed":2,"syntology":{"n":4,"n_ran":3,"n_constructed":0,"n_ran_checked":3,"n_instrument":0,"n_unverified":1,"n_honours":2,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 2 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/large-language-models-are-state-of-the-art-1#ran","syntology_url":"https://syntology.ai/paper/2304.14317","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2304.14317"}},"official":{"repos":["terryyz/llm-code-eval","terryyz/ice-score"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/teaching-large-language-models-to-self-debug","slug":"teaching-large-language-models-to-self-debug","title":"Teaching Large Language Models to Self-Debug","date":"2023-04-11","arxiv_id":"2304.05128","repositories_listed":2,"syntology":{"n":1,"n_ran":0,"n_constructed":0,"n_ran_checked":0,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"0 ran · 1 unverified","sample_list":"/paper/teaching-large-language-models-to-self-debug#ran","syntology_url":"https://syntology.ai/paper/2304.05128","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2304.05128"}},"official":null}},{"url":"/paper/codegeex-a-pre-trained-model-for-code","slug":"codegeex-a-pre-trained-model-for-code","title":"CodeGeeX: A Pre-Trained Model for Code Generation with Multilingual Benchmarking on HumanEval-X","date":"2023-03-30","arxiv_id":"2303.17568","repositories_listed":2,"syntology":{"n":8,"n_ran":6,"n_constructed":0,"n_ran_checked":6,"n_instrument":0,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":6,"n_pointer_only":1,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 6 with no instrument failure: 0 honoured, 0 violated, 6 with no contract checked; 0 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/codegeex-a-pre-trained-model-for-code#ran","syntology_url":"https://syntology.ai/paper/2303.17568","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2303.17568"}},"official":{"repos":["THUDM/CodeGeeX"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/decomposed-diffusion-models-for-high-quality","slug":"decomposed-diffusion-models-for-high-quality","title":"VideoFusion: Decomposed Diffusion Models for High-Quality Video Generation","date":"2023-03-15","arxiv_id":"2303.08320","repositories_listed":2,"syntology":null},{"url":"/paper/learning-performance-improving-code-edits","slug":"learning-performance-improving-code-edits","title":"Learning Performance-Improving Code Edits","date":"2023-02-15","arxiv_id":"2302.07867","repositories_listed":2,"syntology":{"n":19,"n_ran":8,"n_constructed":0,"n_ran_checked":3,"n_instrument":5,"n_unverified":11,"n_honours":2,"n_violates":0,"n_no_contract":1,"n_pointer_only":19,"phrase":"8 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 2 honoured, 0 violated, 1 with no contract checked; 5 where Syntology's instrument failed) · 11 unverified","sample_list":"/paper/learning-performance-improving-code-edits#ran","syntology_url":"https://syntology.ai/paper/2302.07867","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2302.07867"}},"official":{"repos":["madaan/pie-perf"],"state":"official (archive's flag): 8 ran","n_ran":8,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":11,"ran_from_kinds":["official"]}}},{"url":"/paper/ds-1000-a-natural-and-reliable-benchmark-for","slug":"ds-1000-a-natural-and-reliable-benchmark-for","title":"DS-1000: A Natural and Reliable Benchmark for Data Science Code Generation","date":"2022-11-18","arxiv_id":"2211.11501","repositories_listed":2,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":0,"n_instrument":2,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":2,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/ds-1000-a-natural-and-reliable-benchmark-for#ran","syntology_url":"https://syntology.ai/paper/2211.11501","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2211.11501"}},"official":{"repos":["HKUNLP/DS-1000"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/multi-lingual-evaluation-of-code-generation","slug":"multi-lingual-evaluation-of-code-generation","title":"Multi-lingual Evaluation of Code Generation Models","date":"2022-10-26","arxiv_id":"2210.14868","repositories_listed":2,"syntology":{"n":6,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":5,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 5 unverified","sample_list":"/paper/multi-lingual-evaluation-of-code-generation#ran","syntology_url":"https://syntology.ai/paper/2210.14868","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2210.14868"}},"official":{"repos":["amazon-research/mbxp-exec-eval"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":3,"ran_from_kinds":["official"]}}},{"url":"/paper/language-models-of-code-are-few-shot","slug":"language-models-of-code-are-few-shot","title":"Language Models of Code are Few-Shot Commonsense Learners","date":"2022-10-13","arxiv_id":"2210.07128","repositories_listed":2,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":1,"n_instrument":1,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":2,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/language-models-of-code-are-few-shot#ran","syntology_url":"https://syntology.ai/paper/2210.07128","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2210.07128"}},"official":{"repos":["madaan/cocogen"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}}],"syntology_records":62,"syntology_note":"a paper without a record is not a recorded non-run: it may lack an arXiv id or simply be absent from the graph layer"},"key_notes":{"n_ran_checked":"legacy name, kept unchanged so existing readers do not break: it counts the samples that ran with no instrument failure (honoured, violated, and ran with no contract checked); it does not mean a contract was checked, and the pages print it as 'K with no instrument failure', not 'K checked'","n_constructed":"a sub-count of the samples that ran, never subtracted from them and never a failure: an executed sample whose run returned an instance of its own class (fixture_out_type equals the entry name): the run built an object and did not compute a result (Syntology's RAN record, counts.constructed)"},"since_archive_judged":{"papers":109,"papers_placed":6,"judged_by":{"model":"Claude Sonnet 4.5","models_as_recorded":["global.anthropic.claude-sonnet-4-5-20250929-v1:0"],"from":"lb_rows_run_verified.jsonl: each record's `model` field (the extractor's call that wrote decisions[])"},"note":"a model's judgement from the paper's own tables that they report on this leaderboard, not a result: no value from a paper that was not placed is printed or carried here; papers counts each paper once over these leaderboards","backfill":"8,886 of 18,637 newer papers checked so far (733 more had nothing to place; 41 could not be read)","backfill_complete":false,"leaderboards":[{"leaderboard":"/sota/code-generation-on-mbpp","since_archive":"/sota/code-generation-on-mbpp#since-judged","papers_judged":57,"placed":4},{"leaderboard":"/sota/code-generation-on-apps","since_archive":"/sota/code-generation-on-apps#since-judged","papers_judged":1,"placed":0},{"leaderboard":"/sota/code-generation-on-humaneval","since_archive":"/sota/code-generation-on-humaneval#since-judged","papers_judged":54,"placed":0},{"leaderboard":"/sota/code-generation-on-livecodebench","since_archive":"/sota/code-generation-on-livecodebench#since-judged","papers_judged":16,"placed":2},{"leaderboard":"/sota/code-generation-on-bigcodebench-instruct","since_archive":"/sota/code-generation-on-bigcodebench-instruct#since-judged","papers_judged":2,"placed":0}]},"papers_where_code_ran":280,"papers_where_code_ran_first_page":"/task/code-generation/papers/ran/1","papers_where_code_ran_caption":"We ran code from the paper's repository; we did not run it on this task or check it against the task's benchmarks.","pages":[{"order":"archive","page":2,"of":17,"url":"/task/code-generation/papers/2","json":"/task/code-generation/papers/2.json","rows":100},{"order":"archive","page":3,"of":17,"url":"/task/code-generation/papers/3","json":"/task/code-generation/papers/3.json","rows":100},{"order":"archive","page":4,"of":17,"url":"/task/code-generation/papers/4","json":"/task/code-generation/papers/4.json","rows":100},{"order":"archive","page":5,"of":17,"url":"/task/code-generation/papers/5","json":"/task/code-generation/papers/5.json","rows":100},{"order":"archive","page":6,"of":17,"url":"/task/code-generation/papers/6","json":"/task/code-generation/papers/6.json","rows":100},{"order":"archive","page":7,"of":17,"url":"/task/code-generation/papers/7","json":"/task/code-generation/papers/7.json","rows":100},{"order":"archive","page":8,"of":17,"url":"/task/code-generation/papers/8","json":"/task/code-generation/papers/8.json","rows":100},{"order":"archive","page":9,"of":17,"url":"/task/code-generation/papers/9","json":"/task/code-generation/papers/9.json","rows":100},{"order":"archive","page":10,"of":17,"url":"/task/code-generation/papers/10","json":"/task/code-generation/papers/10.json","rows":100},{"order":"archive","page":11,"of":17,"url":"/task/code-generation/papers/11","json":"/task/code-generation/papers/11.json","rows":100},{"order":"archive","page":12,"of":17,"url":"/task/code-generation/papers/12","json":"/task/code-generation/papers/12.json","rows":100},{"order":"archive","page":13,"of":17,"url":"/task/code-generation/papers/13","json":"/task/code-generation/papers/13.json","rows":100},{"order":"archive","page":14,"of":17,"url":"/task/code-generation/papers/14","json":"/task/code-generation/papers/14.json","rows":100},{"order":"archive","page":15,"of":17,"url":"/task/code-generation/papers/15","json":"/task/code-generation/papers/15.json","rows":100},{"order":"archive","page":16,"of":17,"url":"/task/code-generation/papers/16","json":"/task/code-generation/papers/16.json","rows":100},{"order":"archive","page":17,"of":17,"url":"/task/code-generation/papers/17","json":"/task/code-generation/papers/17.json","rows":97},{"order":"ran","page":1,"of":3,"url":"/task/code-generation/papers/ran/1","json":"/task/code-generation/papers/ran/1.json","rows":100},{"order":"ran","page":2,"of":3,"url":"/task/code-generation/papers/ran/2","json":"/task/code-generation/papers/ran/2.json","rows":100},{"order":"ran","page":3,"of":3,"url":"/task/code-generation/papers/ran/3","json":"/task/code-generation/papers/ran/3.json","rows":80}],"description_links":{"kept":1,"unwrapped_to_text":0,"bare_urls_linked":0,"relative_images_dropped":0,"rule":"internal links are kept only when the target slug exists in the catalog"},"syntology":{"read_at":"2026-10-05T00:36:11+00:00","claim":"Per-sample execution status on synthesized fixtures ('ran N of M samples'); not a correctness claim and not a ranking signal.","status_vocabulary":{"ran_honours":"ran, honoured the contract we drafted","ran_violates":"ran, violated the contract we drafted","ran_draft_wrong":"ran; our contract draft was wrong, not the code","ran_fixture":"ran; the contract check could not run (RAISES, DEP_MISSING or TIMEOUT)","ran":"ran on a synthesized input","unverified":"unverified (harvested, no recorded run)"},"row_route":"each syntology object on a benchmark's first row and on a paper carries sample_list (this site's paper page, #ran), syntology_url (syntology.ai's page for the paper, only when that route probed live) and mcp (the agent call that lists the paper's harvested samples; not a verdict on the paper)","developers":"https://syntology.ai/developers"},"not_shown":{"libraries":"the archive has no per-task library table","trend_sparklines":"the Trend column of the benchmarks table was a rendered image; it is not in the archive","social_and_latest_sorts":"stars and social signals are not in the archive"},"record_sha256":"3398f5bade78cd19a14d3d987af9f6e4f52333c335255784ce84e616f6be2561","record_changed_at":"2026-10-05","record_changed_at_basis":"changed"}