{"about":{"non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","site":"https://codewithpapers.app","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page","syntology":{"site":"https://syntology.ai","developers":"https://syntology.ai/developers","mcp":{"server":"https://syntology.ai/mcp","transport":"streamable-http","server_card":"https://syntology.ai/.well-known/mcp/server-card.json","auth":{"type":"trial token, no account","trial_token":"https://syntology.ai/api/oauth/trial/token","method":"POST","docs":"https://syntology.ai/developers"}},"have":"https://syntology.ai/api/graph/have?x=<method, arXiv id or title> (free, answers coverage only)","paper_base":"https://syntology.ai/paper/","atlas_base":"https://app.syntology.ai/?focus="},"machine_readable":[{"url":"https://codewithpapers.app/llms.txt","what":"the machine catalog: every machine-readable file, counted"},{"url":"https://codewithpapers.app/index/manifest.json","what":"paper-to-code index by arXiv id, with Syntology's counts"},{"url":"https://codewithpapers.app/search/manifest.json","what":"site search index (titles, authors) and its files"},{"url":"https://codewithpapers.app/download","what":"bulk files: Syntology's layer, described there"},{"url":"https://codewithpapers.app/build_manifest.json","what":"the build record: inputs, counts, exclusions, probes"}]},"url":"/task/program-synthesis","name":"Program Synthesis","slug":"program-synthesis","description_markdown":"Program synthesis is the process of automatically generating a program or code snippet that satisfies a given specification or set of requirements. This can include generating code from a formal specification, a natural language description, or example inputs and outputs. The primary goal of program synthesis is to minimize human intervention in the coding process, reduce errors, and improve productivity.\r\n\r\nProgram synthesis often involves the use of advanced algorithms, artificial intelligence, and machine learning techniques to search the space of possible programs that meet the given constraints. This process can be guided by a variety of techniques, such as constraint solving, symbolic execution, and genetic algorithms.","categories":[{"name":"Computer Code","url":"/area/computer-code"},{"name":"Reasoning","url":"/area/reasoning"}],"source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","slug_source":"archive_url"},"counts":{"papers_tagged":423,"papers_with_code":179,"benchmarks":3,"benchmark_tables_in_archive":3,"benchmark_tables_shown":3,"benchmark_tables_withheld_as_spam":0,"benchmark_definition":"a leaderboard table with at least one row; benchmark_tables_shown also counts the zero-row tables; benchmark_tables_in_archive adds the tables withheld as spam","datasets":5,"subtasks":5,"parent_tasks":0},"benchmarks":[{"leaderboard":"/sota/program-synthesis-on-spoc-testp","slug":"program-synthesis-on-spoc-testp","dataset":"SPoC TestP","dataset_url":"/dataset/spoc","rows_in_archive":2,"metrics":["Success rate @budget 100"],"first_row_in_archive_order":{"model":"DrRepair","paper_title":"Graph-based, Self-Supervised Program Repair from Diagnostic Feedback","paper_url":"/paper/graph-based-self-supervised-program-repair","paper_date":"2020-05-20","arxiv_id":"2005.10636","code_links":[{"title":"michiyasunaga/DrRepair","url":"https://github.com/michiyasunaga/DrRepair"},{"title":"worksheets/0x01838644","url":"https://worksheets.codalab.org/worksheets/0x01838644724a433c932bef4cb5c42fbd"}],"syntology":{"n":6,"n_ran":4,"n_constructed":2,"n_ran_checked":3,"n_instrument":1,"n_unverified":2,"n_honours":1,"n_violates":0,"n_no_contract":2,"n_pointer_only":0,"phrase":"4 ran (of which 2 constructed an object rather than computing a result; 3 with no instrument failure: 1 honoured, 0 violated, 2 with no contract checked; 1 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/graph-based-self-supervised-program-repair#ran","syntology_url":"https://syntology.ai/paper/2005.10636","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2005.10636"}},"official":{"repos":["michiyasunaga/DrRepair","worksheets.codalab.org/worksheets/0x01838644724a433c932bef4cb5c42fbd"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":2,"n_ran_no_instrument_failure":3,"n_unverified":2,"ran_from_kinds":["official"]}}}},{"leaderboard":"/sota/program-synthesis-on-spoc-testw","slug":"program-synthesis-on-spoc-testw","dataset":"SPoC TestW","dataset_url":"/dataset/spoc","rows_in_archive":2,"metrics":["Success rate @budget 100"],"first_row_in_archive_order":{"model":"DrRepair","paper_title":"Graph-based, Self-Supervised Program Repair from Diagnostic Feedback","paper_url":"/paper/graph-based-self-supervised-program-repair","paper_date":"2020-05-20","arxiv_id":"2005.10636","code_links":[{"title":"michiyasunaga/DrRepair","url":"https://github.com/michiyasunaga/DrRepair"},{"title":"worksheets/0x01838644","url":"https://worksheets.codalab.org/worksheets/0x01838644724a433c932bef4cb5c42fbd"}],"syntology":{"n":6,"n_ran":4,"n_constructed":2,"n_ran_checked":3,"n_instrument":1,"n_unverified":2,"n_honours":1,"n_violates":0,"n_no_contract":2,"n_pointer_only":0,"phrase":"4 ran (of which 2 constructed an object rather than computing a result; 3 with no instrument failure: 1 honoured, 0 violated, 2 with no contract checked; 1 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/graph-based-self-supervised-program-repair#ran","syntology_url":"https://syntology.ai/paper/2005.10636","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2005.10636"}},"official":{"repos":["michiyasunaga/DrRepair","worksheets.codalab.org/worksheets/0x01838644724a433c932bef4cb5c42fbd"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":2,"n_ran_no_instrument_failure":3,"n_unverified":2,"ran_from_kinds":["official"]}}}},{"leaderboard":"/sota/program-synthesis-on-algolisp","slug":"program-synthesis-on-algolisp","dataset":"AlgoLisp","dataset_url":null,"rows_in_archive":1,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"CodeTrans-MT-TF-Small","paper_title":"CodeTrans: Towards Cracking the Language of Silicon's Code Through Self-Supervised Deep Learning and High Performance Computing","paper_url":"/paper/codetrans-towards-cracking-the-language-of","paper_date":"2021-04-06","arxiv_id":"2104.02443","code_links":[{"title":"agemagician/CodeTrans","url":"https://github.com/agemagician/CodeTrans"}],"syntology":null}}],"datasets":[{"url":"/dataset/concode","name":"CONCODE","full_name":"","num_papers_in_archive":46},{"url":"/dataset/sketchgraphs","name":"SketchGraphs","full_name":null,"num_papers_in_archive":16},{"url":"/dataset/spoc","name":"SPoC","full_name":"Pseudocode-to-Code","num_papers_in_archive":16},{"url":"/dataset/xcodeeval","name":"xCodeEval","full_name":"xCodeEval","num_papers_in_archive":15},{"url":"/dataset/psb2","name":"PSB2","full_name":"The Second Program Synthesis Benchmark Suite","num_papers_in_archive":1}],"subtasks":[{"url":"/task/enumerative-search","name":"Enumerative Search"},{"url":"/task/program-repair","name":"Program Repair"},{"url":"/task/sql-synthesis","name":"SQL Synthesis"},{"url":"/task/type-prediction","name":"Type prediction"},{"url":"/task/value-prediction","name":"Value prediction"}],"parent_tasks":[],"papers":{"order":"repositories listed in the archive (most first), then date (newest first), then slug; the archive holds no stars","population":"every paper the archive tags with this task (distinct papers; spam titles not listed). 'of' counts the papers listed; until 2026-09-25 it counted only the papers with a code link, and that number is of_with_code","page":1,"pages_in_order":5,"rows_per_page":100,"shown":100,"of":423,"of_with_code":179,"tagged_in_all":423,"counts":{"archive_papers_tagged":423,"with_a_code_link":179,"where_syntology_ran_a_sample":69,"not_listed_spam_title":0,"listed":423,"listed_where_code_ran":69,"where_syntology_ran_a_sample_split":{"with_a_run_with_no_instrument_failure":48,"every_run_a_failure_of_syntologys_instrument":21,"listed_with_a_run_with_no_instrument_failure":48,"listed_every_run_a_failure_of_syntologys_instrument":21,"filter":{"states":["a run with no instrument failure","any run, instrument failures included"],"default":"a run with no instrument failure","note":"on the 'only where code ran' pages the default hides, in the browser, the rows where every run was a failure of Syntology's instrument; the second state shows them again. Rows are hidden, never re-ordered; these twins list every row"}},"definition":"distinct papers the archive tags; 'where Syntology ran a sample' counts papers with at least one harvested sample that ran, which is not a correctness claim"},"items":[{"url":"/paper/a-conversational-paradigm-for-program","slug":"a-conversational-paradigm-for-program","title":"CodeGen: An Open Large Language Model for Code with Multi-Turn Program Synthesis","date":"2022-03-25","arxiv_id":"2203.13474","repositories_listed":8,"syntology":{"n":10,"n_ran":10,"n_constructed":0,"n_ran_checked":3,"n_instrument":7,"n_unverified":0,"n_honours":1,"n_violates":1,"n_no_contract":1,"n_pointer_only":1,"phrase":"10 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 1 honoured, 1 violated, 1 with no contract checked; 7 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/a-conversational-paradigm-for-program#ran","syntology_url":"https://syntology.ai/paper/2203.13474","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2203.13474"}},"official":{"repos":["salesforce/CodeGen","salesforce/jaxformer"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/large-language-models-are-human-level-prompt","slug":"large-language-models-are-human-level-prompt","title":"Large Language Models Are Human-Level Prompt Engineers","date":"2022-11-03","arxiv_id":"2211.01910","repositories_listed":5,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/large-language-models-are-human-level-prompt#ran","syntology_url":"https://syntology.ai/paper/2211.01910","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2211.01910"}},"official":{"repos":["keirp/automatic_prompt_engineer"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/xcodeeval-a-large-scale-multilingual","slug":"xcodeeval-a-large-scale-multilingual","title":"xCodeEval: A Large Scale Multilingual Multitask Benchmark for Code Understanding, Generation, Translation and Retrieval","date":"2023-03-06","arxiv_id":"2303.03004","repositories_listed":4,"syntology":{"n":6,"n_ran":5,"n_constructed":0,"n_ran_checked":4,"n_instrument":1,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":4,"n_pointer_only":0,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 4 with no instrument failure: 0 honoured, 0 violated, 4 with no contract checked; 1 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/xcodeeval-a-large-scale-multilingual#ran","syntology_url":"https://syntology.ai/paper/2303.03004","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2303.03004"}},"official":{"repos":["ntunlp/xCodeEval"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":4,"n_unverified":1,"ran_from_kinds":["named_in_paper","official"]}}},{"url":"/paper/memory-augmented-policy-optimization-for","slug":"memory-augmented-policy-optimization-for","title":"Memory Augmented Policy Optimization for Program Synthesis and Semantic Parsing","date":"2018-07-06","arxiv_id":"1807.02322","repositories_listed":4,"syntology":{"n":15,"n_ran":7,"n_constructed":0,"n_ran_checked":7,"n_instrument":0,"n_unverified":8,"n_honours":0,"n_violates":0,"n_no_contract":7,"n_pointer_only":0,"phrase":"7 ran (of which 0 constructed an object rather than computing a result; 7 with no instrument failure: 0 honoured, 0 violated, 7 with no contract checked; 0 where Syntology's instrument failed) · 8 unverified","sample_list":"/paper/memory-augmented-policy-optimization-for#ran","syntology_url":"https://syntology.ai/paper/1807.02322","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1807.02322"}},"official":{"repos":["crazydonkey200/neural-symbolic-machines"],"state":"official (archive's flag): 7 ran","n_ran":7,"n_constructed":0,"n_ran_no_instrument_failure":7,"n_unverified":6,"ran_from_kinds":["official"]}}},{"url":"/paper/neural-program-synthesis-with-priority-queue","slug":"neural-program-synthesis-with-priority-queue","title":"Neural Program Synthesis with Priority Queue Training","date":"2018-01-10","arxiv_id":"1801.03526","repositories_listed":4,"syntology":null},{"url":"/paper/incoder-a-generative-model-for-code-infilling","slug":"incoder-a-generative-model-for-code-infilling","title":"InCoder: A Generative Model for Code Infilling and Synthesis","date":"2022-04-12","arxiv_id":"2204.05999","repositories_listed":3,"syntology":{"n":10,"n_ran":6,"n_constructed":0,"n_ran_checked":3,"n_instrument":3,"n_unverified":4,"n_honours":0,"n_violates":1,"n_no_contract":2,"n_pointer_only":2,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 1 violated, 2 with no contract checked; 3 where Syntology's instrument failed) · 4 unverified","sample_list":"/paper/incoder-a-generative-model-for-code-infilling#ran","syntology_url":"https://syntology.ai/paper/2204.05999","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2204.05999"}},"official":{"repos":["dpfried/incoder"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/program-synthesis-for-the-oeis","slug":"program-synthesis-for-the-oeis","title":"Learning Program Synthesis for Integer Sequences from Scratch","date":"2022-02-24","arxiv_id":"2202.11908","repositories_listed":3,"syntology":null},{"url":"/paper/programming-puzzles","slug":"programming-puzzles","title":"Programming Puzzles","date":"2021-06-10","arxiv_id":"2106.05784","repositories_listed":3,"syntology":null},{"url":"/paper/dreamcoder-growing-generalizable","slug":"dreamcoder-growing-generalizable","title":"DreamCoder: Growing generalizable, interpretable knowledge with wake-sleep Bayesian program learning","date":"2020-06-15","arxiv_id":"2006.08381","repositories_listed":3,"syntology":{"n":11,"n_ran":6,"n_constructed":0,"n_ran_checked":5,"n_instrument":1,"n_unverified":5,"n_honours":0,"n_violates":0,"n_no_contract":5,"n_pointer_only":11,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 0 honoured, 0 violated, 5 with no contract checked; 1 where Syntology's instrument failed) · 5 unverified","sample_list":"/paper/dreamcoder-growing-generalizable#ran","syntology_url":"https://syntology.ai/paper/2006.08381","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2006.08381"}},"official":{"repos":["CatherineWong/dreamcoder"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/robustfill-neural-program-learning-under","slug":"robustfill-neural-program-learning-under","title":"RobustFill: Neural Program Learning under Noisy I/O","date":"2017-03-21","arxiv_id":"1703.07469","repositories_listed":3,"syntology":{"n":8,"n_ran":5,"n_constructed":0,"n_ran_checked":3,"n_instrument":2,"n_unverified":3,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":1,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 2 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/robustfill-neural-program-learning-under#ran","syntology_url":"https://syntology.ai/paper/1703.07469","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1703.07469"}},"official":null}},{"url":"/paper/deepcoder-learning-to-write-programs","slug":"deepcoder-learning-to-write-programs","title":"DeepCoder: Learning to Write Programs","date":"2016-11-07","arxiv_id":"1611.01989","repositories_listed":3,"syntology":{"n":8,"n_ran":8,"n_constructed":0,"n_ran_checked":5,"n_instrument":3,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":5,"n_pointer_only":3,"phrase":"8 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 0 honoured, 0 violated, 5 with no contract checked; 3 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/deepcoder-learning-to-write-programs#ran","syntology_url":"https://syntology.ai/paper/1611.01989","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1611.01989"}},"official":null}},{"url":"/paper/clever-a-curated-benchmark-for-formally","slug":"clever-a-curated-benchmark-for-formally","title":"CLEVER: A Curated Benchmark for Formally Verified Code Generation","date":"2025-05-20","arxiv_id":"2505.13938","repositories_listed":2,"syntology":null},{"url":"/paper/tikzero-zero-shot-text-guided-graphics","slug":"tikzero-zero-shot-text-guided-graphics","title":"TikZero: Zero-Shot Text-Guided Graphics Program Synthesis","date":"2025-03-14","arxiv_id":"2503.11509","repositories_listed":2,"syntology":null},{"url":"/paper/mapcoder-multi-agent-code-generation-for","slug":"mapcoder-multi-agent-code-generation-for","title":"MapCoder: Multi-Agent Code Generation for Competitive Problem Solving","date":"2024-05-18","arxiv_id":"2405.11403","repositories_listed":2,"syntology":{"n":10,"n_ran":6,"n_constructed":0,"n_ran_checked":6,"n_instrument":0,"n_unverified":4,"n_honours":0,"n_violates":0,"n_no_contract":6,"n_pointer_only":0,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 6 with no instrument failure: 0 honoured, 0 violated, 6 with no contract checked; 0 where Syntology's instrument failed) · 4 unverified","sample_list":"/paper/mapcoder-multi-agent-code-generation-for#ran","syntology_url":"https://syntology.ai/paper/2405.11403","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2405.11403"}},"official":{"repos":["md-ashraful-pramanik/mapcoder"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/bring-your-own-kg-self-supervised-program","slug":"bring-your-own-kg-self-supervised-program","title":"Bring Your Own KG: Self-Supervised Program Synthesis for Zero-Shot KGQA","date":"2023-11-14","arxiv_id":"2311.07850","repositories_listed":2,"syntology":null},{"url":"/paper/knowledge-driven-robot-program-synthesis-from","slug":"knowledge-driven-robot-program-synthesis-from","title":"Knowledge-Driven Robot Program Synthesis from Human VR Demonstrations","date":"2023-06-05","arxiv_id":"2306.02739","repositories_listed":2,"syntology":null},{"url":"/paper/codegen2-lessons-for-training-llms-on","slug":"codegen2-lessons-for-training-llms-on","title":"CodeGen2: Lessons for Training LLMs on Programming and Natural Languages","date":"2023-05-03","arxiv_id":"2305.02309","repositories_listed":2,"syntology":null},{"url":"/paper/learning-programs-with-magic-values","slug":"learning-programs-with-magic-values","title":"Learning programs with magic values","date":"2022-08-05","arxiv_id":"2208.03238","repositories_listed":2,"syntology":null},{"url":"/paper/coderl-mastering-code-generation-through","slug":"coderl-mastering-code-generation-through","title":"CodeRL: Mastering Code Generation through Pretrained Models and Deep Reinforcement Learning","date":"2022-07-05","arxiv_id":"2207.01780","repositories_listed":2,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/coderl-mastering-code-generation-through#ran","syntology_url":"https://syntology.ai/paper/2207.01780","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2207.01780"}},"official":{"repos":["salesforce/coderl"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/communicating-natural-programs-to-humans-and","slug":"communicating-natural-programs-to-humans-and","title":"Communicating Natural Programs to Humans and Machines","date":"2021-06-15","arxiv_id":"2106.07824","repositories_listed":2,"syntology":null},{"url":"/paper/graph-based-self-supervised-program-repair","slug":"graph-based-self-supervised-program-repair","title":"Graph-based, Self-Supervised Program Repair from Diagnostic Feedback","date":"2020-05-20","arxiv_id":"2005.10636","repositories_listed":2,"syntology":{"n":6,"n_ran":4,"n_constructed":2,"n_ran_checked":3,"n_instrument":1,"n_unverified":2,"n_honours":1,"n_violates":0,"n_no_contract":2,"n_pointer_only":0,"phrase":"4 ran (of which 2 constructed an object rather than computing a result; 3 with no instrument failure: 1 honoured, 0 violated, 2 with no contract checked; 1 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/graph-based-self-supervised-program-repair#ran","syntology_url":"https://syntology.ai/paper/2005.10636","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2005.10636"}},"official":{"repos":["michiyasunaga/DrRepair","worksheets.codalab.org/worksheets/0x01838644724a433c932bef4cb5c42fbd"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":2,"n_ran_no_instrument_failure":3,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/tf-coder-program-synthesis-for-tensor","slug":"tf-coder-program-synthesis-for-tensor","title":"TF-Coder: Program Synthesis for Tensor Manipulations","date":"2020-03-19","arxiv_id":"2003.09040","repositories_listed":2,"syntology":{"n":14,"n_ran":8,"n_constructed":0,"n_ran_checked":8,"n_instrument":0,"n_unverified":6,"n_honours":0,"n_violates":0,"n_no_contract":8,"n_pointer_only":0,"phrase":"8 ran (of which 0 constructed an object rather than computing a result; 8 with no instrument failure: 0 honoured, 0 violated, 8 with no contract checked; 0 where Syntology's instrument failed) · 6 unverified","sample_list":"/paper/tf-coder-program-synthesis-for-tensor#ran","syntology_url":"https://syntology.ai/paper/2003.09040","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2003.09040"}},"official":{"repos":["google-research/tensorflow-coder"],"state":"official (archive's flag): 7 ran","n_ran":7,"n_constructed":0,"n_ran_no_instrument_failure":7,"n_unverified":1,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/improving-molecular-design-by-stochastic","slug":"improving-molecular-design-by-stochastic","title":"Improving Molecular Design by Stochastic Iterative Target Augmentation","date":"2020-02-11","arxiv_id":"2002.04720","repositories_listed":2,"syntology":{"n":8,"n_ran":6,"n_constructed":0,"n_ran_checked":6,"n_instrument":0,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":6,"n_pointer_only":1,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 6 with no instrument failure: 0 honoured, 0 violated, 6 with no contract checked; 0 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/improving-molecular-design-by-stochastic#ran","syntology_url":"https://syntology.ai/paper/2002.04720","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2002.04720"}},"official":{"repos":["yangkevin2/icml2020-stochastic-iterative-target-augmentation"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/natural-to-formal-language-generation-using-1","slug":"natural-to-formal-language-generation-using-1","title":"Mapping Natural-language Problems to Formal-language Solutions Using Structured Neural Representations","date":"2019-10-05","arxiv_id":"1910.02339","repositories_listed":2,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":0,"n_instrument":3,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":3,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 3 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/natural-to-formal-language-generation-using-1#ran","syntology_url":"https://syntology.ai/paper/1910.02339","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1910.02339"}},"official":null}},{"url":"/paper/houdini-lifelong-learning-as-program","slug":"houdini-lifelong-learning-as-program","title":"HOUDINI: Lifelong Learning as Program Synthesis","date":"2018-03-31","arxiv_id":"1804.00218","repositories_listed":2,"syntology":{"n":3,"n_ran":2,"n_constructed":0,"n_ran_checked":0,"n_instrument":2,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":3,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/houdini-lifelong-learning-as-program#ran","syntology_url":"https://syntology.ai/paper/1804.00218","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1804.00218"}},"official":{"repos":["capergroup/houdini"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/transductively-informed-inductive-program","slug":"transductively-informed-inductive-program","title":"Transductively Informed Inductive Program Synthesis","date":"2025-05-20","arxiv_id":"2505.14744","repositories_listed":1,"syntology":null},{"url":"/paper/2505-10819","slug":"2505-10819","title":"PoE-World: Compositional World Modeling with Products of Programmatic Experts","date":"2025-05-16","arxiv_id":"2505.10819","repositories_listed":1,"syntology":null},{"url":"/paper/enhancing-code-generation-via-bidirectional","slug":"enhancing-code-generation-via-bidirectional","title":"Enhancing Code Generation via Bidirectional Comment-Level Mutual Grounding","date":"2025-05-12","arxiv_id":"2505.07768","repositories_listed":1,"syntology":null},{"url":"/paper/rewriting-pre-training-data-boosts-llm","slug":"rewriting-pre-training-data-boosts-llm","title":"Rewriting Pre-Training Data Boosts LLM Performance in Math and Code","date":"2025-05-05","arxiv_id":"2505.02881","repositories_listed":1,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":0,"n_instrument":2,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":2,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/rewriting-pre-training-data-boosts-llm#ran","syntology_url":"https://syntology.ai/paper/2505.02881","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2505.02881"}},"official":{"repos":["rioyokotalab/swallow-code-math"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/osvbench-benchmarking-llms-on-specification","slug":"osvbench-benchmarking-llms-on-specification","title":"OSVBench: Benchmarking LLMs on Specification Generation Tasks for Operating System Verification","date":"2025-04-29","arxiv_id":"2504.20964","repositories_listed":1,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":1,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/osvbench-benchmarking-llms-on-specification#ran","syntology_url":"https://syntology.ai/paper/2504.20964","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2504.20964"}},"official":{"repos":["lishangyu-hkust/osvbench"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/tinyversegp-towards-a-modular-cross-domain","slug":"tinyversegp-towards-a-modular-cross-domain","title":"TinyverseGP: Towards a Modular Cross-domain Benchmarking Framework for Genetic Programming","date":"2025-04-14","arxiv_id":"2504.10253","repositories_listed":1,"syntology":null},{"url":"/paper/autoiot-llm-driven-automated-natural-language","slug":"autoiot-llm-driven-automated-natural-language","title":"AutoIOT: LLM-Driven Automated Natural Language Programming for AIoT Applications","date":"2025-03-07","arxiv_id":"2503.05346","repositories_listed":1,"syntology":null},{"url":"/paper/factorio-learning-environment","slug":"factorio-learning-environment","title":"Factorio Learning Environment","date":"2025-03-06","arxiv_id":"2503.09617","repositories_listed":1,"syntology":null},{"url":"/paper/2503-00686","slug":"2503-00686","title":"GPIoT: Tailoring Small Language Models for IoT Program Synthesis and Development","date":"2025-03-02","arxiv_id":"2503.00686","repositories_listed":1,"syntology":null},{"url":"/paper/program-synthesis-dialog-agents-for","slug":"program-synthesis-dialog-agents-for","title":"Program Synthesis Dialog Agents for Interactive Decision-Making","date":"2025-02-26","arxiv_id":"2502.19610","repositories_listed":1,"syntology":null},{"url":"/paper/codesim-multi-agent-code-generation-and-1","slug":"codesim-multi-agent-code-generation-and-1","title":"CODESIM: Multi-Agent Code Generation and Problem Solving through Simulation-Driven Planning and Debugging","date":"2025-02-08","arxiv_id":"2502.05664","repositories_listed":1,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":3,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/codesim-multi-agent-code-generation-and-1#ran","syntology_url":"https://syntology.ai/paper/2502.05664","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2502.05664"}},"official":null}},{"url":"/paper/conceptsearch-towards-efficient-program","slug":"conceptsearch-towards-efficient-program","title":"ConceptSearch: Towards Efficient Program Search Using LLMs for Abstraction and Reasoning Corpus (ARC)","date":"2024-12-10","arxiv_id":"2412.07322","repositories_listed":1,"syntology":null},{"url":"/paper/arc-prize-2024-technical-report","slug":"arc-prize-2024-technical-report","title":"ARC Prize 2024: Technical Report","date":"2024-12-05","arxiv_id":"2412.04604","repositories_listed":1,"syntology":{"n":4,"n_ran":4,"n_constructed":0,"n_ran_checked":4,"n_instrument":0,"n_unverified":0,"n_honours":3,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 4 with no instrument failure: 3 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/arc-prize-2024-technical-report#ran","syntology_url":"https://syntology.ai/paper/2412.04604","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2412.04604"}},"official":{"repos":["michaelhodel/re-arc"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":4,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/searching-latent-program-spaces","slug":"searching-latent-program-spaces","title":"Searching Latent Program Spaces","date":"2024-11-13","arxiv_id":"2411.08706","repositories_listed":1,"syntology":{"n":9,"n_ran":4,"n_constructed":3,"n_ran_checked":3,"n_instrument":1,"n_unverified":5,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":0,"phrase":"4 ran (of which 3 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 1 where Syntology's instrument failed) · 5 unverified","sample_list":"/paper/searching-latent-program-spaces#ran","syntology_url":"https://syntology.ai/paper/2411.08706","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2411.08706"}},"official":{"repos":["clement-bonnet/lpn"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":3,"n_ran_no_instrument_failure":3,"n_unverified":5,"ran_from_kinds":["official"]}}},{"url":"/paper/the-surprising-effectiveness-of-test-time","slug":"the-surprising-effectiveness-of-test-time","title":"The Surprising Effectiveness of Test-Time Training for Few-Shot Learning","date":"2024-11-11","arxiv_id":"2411.07279","repositories_listed":1,"syntology":{"n":10,"n_ran":5,"n_constructed":0,"n_ran_checked":3,"n_instrument":2,"n_unverified":5,"n_honours":3,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 3 honoured, 0 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 5 unverified","sample_list":"/paper/the-surprising-effectiveness-of-test-time#ran","syntology_url":"https://syntology.ai/paper/2411.07279","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2411.07279"}},"official":{"repos":["ekinakyurek/marc"],"state":"official (archive's flag): 5 ran","n_ran":5,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":5,"ran_from_kinds":["official"]}}},{"url":"/paper/combining-induction-and-transduction-for","slug":"combining-induction-and-transduction-for","title":"Combining Induction and Transduction for Abstract Reasoning","date":"2024-11-04","arxiv_id":"2411.02272","repositories_listed":1,"syntology":{"n":19,"n_ran":17,"n_constructed":0,"n_ran_checked":12,"n_instrument":5,"n_unverified":2,"n_honours":3,"n_violates":2,"n_no_contract":7,"n_pointer_only":19,"phrase":"17 ran (of which 0 constructed an object rather than computing a result; 12 with no instrument failure: 3 honoured, 2 violated, 7 with no contract checked; 5 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/combining-induction-and-transduction-for#ran","syntology_url":"https://syntology.ai/paper/2411.02272","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2411.02272"}},"official":{"repos":["xu3kev/barc"],"state":"official (archive's flag): 17 ran","n_ran":17,"n_constructed":0,"n_ran_no_instrument_failure":12,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/from-easy-to-hard-tackling-quantum-problems","slug":"from-easy-to-hard-tackling-quantum-problems","title":"Reinforcement learning with learned gadgets to tackle hard quantum problems on real hardware","date":"2024-10-31","arxiv_id":"2411.00230","repositories_listed":1,"syntology":{"n":3,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/from-easy-to-hard-tackling-quantum-problems#ran","syntology_url":"https://syntology.ai/paper/2411.00230","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2411.00230"}},"official":{"repos":["aqasch/gadget_rl"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/tackling-the-abstraction-and-reasoning-corpus-1","slug":"tackling-the-abstraction-and-reasoning-corpus-1","title":"Tackling the Abstraction and Reasoning Corpus with Vision Transformers: the Importance of 2D Representation, Positions, and Objects","date":"2024-10-08","arxiv_id":"2410.06405","repositories_listed":1,"syntology":{"n":5,"n_ran":4,"n_constructed":0,"n_ran_checked":2,"n_instrument":2,"n_unverified":1,"n_honours":1,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 1 honoured, 0 violated, 1 with no contract checked; 2 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/tackling-the-abstraction-and-reasoning-corpus-1#ran","syntology_url":"https://syntology.ai/paper/2410.06405","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2410.06405"}},"official":{"repos":["khalil-research/ViTARC"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/ma-rlhf-reinforcement-learning-from-human","slug":"ma-rlhf-reinforcement-learning-from-human","title":"MA-RLHF: Reinforcement Learning from Human Feedback with Macro Actions","date":"2024-10-03","arxiv_id":"2410.02743","repositories_listed":1,"syntology":null},{"url":"/paper/mathdsl-a-domain-specific-language-for","slug":"mathdsl-a-domain-specific-language-for","title":"MathDSL: A Domain-Specific Language for Concise Mathematical Solutions Via Program Synthesis","date":"2024-09-26","arxiv_id":"2409.17490","repositories_listed":1,"syntology":null},{"url":"/paper/autosafecoder-a-multi-agent-framework-for","slug":"autosafecoder-a-multi-agent-framework-for","title":"AutoSafeCoder: A Multi-Agent Framework for Securing LLM Code Generation through Static Analysis and Fuzz Testing","date":"2024-09-16","arxiv_id":"2409.10737","repositories_listed":1,"syntology":{"n":5,"n_ran":2,"n_constructed":0,"n_ran_checked":1,"n_instrument":1,"n_unverified":3,"n_honours":0,"n_violates":1,"n_no_contract":0,"n_pointer_only":5,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 1 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/autosafecoder-a-multi-agent-framework-for#ran","syntology_url":"https://syntology.ai/paper/2409.10737","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2409.10737"}},"official":{"repos":["secureaiautonomylab/autosafecoder"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":3,"ran_from_kinds":["official"]}}},{"url":"/paper/h-arc-a-robust-estimate-of-human-performance","slug":"h-arc-a-robust-estimate-of-human-performance","title":"H-ARC: A Robust Estimate of Human Performance on the Abstraction and Reasoning Corpus Benchmark","date":"2024-09-02","arxiv_id":"2409.01374","repositories_listed":1,"syntology":null},{"url":"/paper/syntax-guided-procedural-synthesis-of","slug":"syntax-guided-procedural-synthesis-of","title":"Procedural Synthesis of Synthesizable Molecules","date":"2024-08-24","arxiv_id":"2409.05873","repositories_listed":1,"syntology":{"n":9,"n_ran":6,"n_constructed":1,"n_ran_checked":4,"n_instrument":2,"n_unverified":3,"n_honours":0,"n_violates":2,"n_no_contract":2,"n_pointer_only":0,"phrase":"6 ran (of which 1 constructed an object rather than computing a result; 4 with no instrument failure: 0 honoured, 2 violated, 2 with no contract checked; 2 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/syntax-guided-procedural-synthesis-of#ran","syntology_url":"https://syntology.ai/paper/2409.05873","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2409.05873"}},"official":{"repos":["shiningsunnyday/synthesisnet"],"state":"official (archive's flag): 6 ran","n_ran":6,"n_constructed":1,"n_ran_no_instrument_failure":4,"n_unverified":3,"ran_from_kinds":["official"]}}},{"url":"/paper/relational-decomposition-for-program","slug":"relational-decomposition-for-program","title":"Relational decomposition for program synthesis","date":"2024-08-22","arxiv_id":"2408.12212","repositories_listed":1,"syntology":{"n":14,"n_ran":11,"n_constructed":0,"n_ran_checked":6,"n_instrument":5,"n_unverified":3,"n_honours":0,"n_violates":0,"n_no_contract":6,"n_pointer_only":14,"phrase":"11 ran (of which 0 constructed an object rather than computing a result; 6 with no instrument failure: 0 honoured, 0 violated, 6 with no contract checked; 5 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/relational-decomposition-for-program#ran","syntology_url":"https://syntology.ai/paper/2408.12212","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2408.12212"}},"official":{"repos":["celinehocquette/ijcai25-relational-decomposition"],"state":"official (archive's flag): 11 ran","n_ran":11,"n_constructed":0,"n_ran_no_instrument_failure":6,"n_unverified":3,"ran_from_kinds":["official"]}}},{"url":"/paper/codeupdatearena-benchmarking-knowledge","slug":"codeupdatearena-benchmarking-knowledge","title":"CodeUpdateArena: Benchmarking Knowledge Editing on API Updates","date":"2024-07-08","arxiv_id":"2407.06249","repositories_listed":1,"syntology":null},{"url":"/paper/bug-in-the-code-stack-can-llms-find-bugs-in","slug":"bug-in-the-code-stack-can-llms-find-bugs-in","title":"Bug In the Code Stack: Can LLMs Find Bugs in Large Python Code Stacks","date":"2024-06-21","arxiv_id":"2406.15325","repositories_listed":1,"syntology":null},{"url":"/paper/fvel-interactive-formal-verification","slug":"fvel-interactive-formal-verification","title":"FVEL: Interactive Formal Verification Environment with Large Language Models via Theorem Proving","date":"2024-06-20","arxiv_id":"2406.14408","repositories_listed":1,"syntology":{"n":5,"n_ran":4,"n_constructed":0,"n_ran_checked":2,"n_instrument":2,"n_unverified":1,"n_honours":0,"n_violates":1,"n_no_contract":1,"n_pointer_only":5,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 1 violated, 1 with no contract checked; 2 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/fvel-interactive-formal-verification#ran","syntology_url":"https://syntology.ai/paper/2406.14408","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.14408"}},"official":{"repos":["fveler/fvel"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/how-efficient-is-llm-generated-code-a","slug":"how-efficient-is-llm-generated-code-a","title":"How Efficient is LLM-Generated Code? A Rigorous & High-Standard Benchmark","date":"2024-06-10","arxiv_id":"2406.06647","repositories_listed":1,"syntology":{"n":4,"n_ran":0,"n_constructed":0,"n_ran_checked":0,"n_instrument":0,"n_unverified":4,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"0 ran · 4 unverified","sample_list":"/paper/how-efficient-is-llm-generated-code-a#ran","syntology_url":"https://syntology.ai/paper/2406.06647","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.06647"}},"official":{"repos":["q-rz/enamel"],"state":"official: harvested, nothing ran","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":4,"ran_from_kinds":[]}}},{"url":"/paper/learning-to-reason-via-program-generation","slug":"learning-to-reason-via-program-generation","title":"Learning to Reason via Program Generation, Emulation, and Search","date":"2024-05-25","arxiv_id":"2405.16337","repositories_listed":1,"syntology":null},{"url":"/paper/generating-code-world-models-with-large","slug":"generating-code-world-models-with-large","title":"Generating Code World Models with Large Language Models Guided by Monte Carlo Tree Search","date":"2024-05-24","arxiv_id":"2405.15383","repositories_listed":1,"syntology":{"n":4,"n_ran":4,"n_constructed":0,"n_ran_checked":3,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":1,"n_no_contract":2,"n_pointer_only":0,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 1 violated, 2 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/generating-code-world-models-with-large#ran","syntology_url":"https://syntology.ai/paper/2405.15383","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2405.15383"}},"official":{"repos":["nicoladainese96/code-world-models"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/goals-as-reward-producing-programs","slug":"goals-as-reward-producing-programs","title":"Goals as Reward-Producing Programs","date":"2024-05-21","arxiv_id":"2405.13242","repositories_listed":1,"syntology":{"n":3,"n_ran":2,"n_constructed":0,"n_ran_checked":0,"n_instrument":2,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/goals-as-reward-producing-programs#ran","syntology_url":"https://syntology.ai/paper/2405.13242","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2405.13242"}},"official":{"repos":["guydav/goals-as-reward-producing-programs"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/large-language-models-synergize-with","slug":"large-language-models-synergize-with","title":"Large Language Models Synergize with Automated Machine Learning","date":"2024-05-06","arxiv_id":"2405.03727","repositories_listed":1,"syntology":null},{"url":"/paper/top-leaderboard-ranking-top-coding","slug":"top-leaderboard-ranking-top-coding","title":"Top Leaderboard Ranking = Top Coding Proficiency, Always? EvoEval: Evolving Coding Benchmarks via LLM","date":"2024-03-28","arxiv_id":"2403.19114","repositories_listed":1,"syntology":{"n":6,"n_ran":5,"n_constructed":0,"n_ran_checked":3,"n_instrument":2,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":0,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 2 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/top-leaderboard-ranking-top-coding#ran","syntology_url":"https://syntology.ai/paper/2403.19114","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2403.19114"}},"official":{"repos":["evo-eval/evoeval"],"state":"official (archive's flag): 5 ran","n_ran":5,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/watchat-explaining-perplexing-programs-by","slug":"watchat-explaining-perplexing-programs-by","title":"WatChat: Explaining perplexing programs by debugging mental models","date":"2024-03-08","arxiv_id":"2403.05334","repositories_listed":1,"syntology":null},{"url":"/paper/constrained-decoding-for-code-language-models","slug":"constrained-decoding-for-code-language-models","title":"Constrained Decoding for Fill-in-the-Middle Code Language Models via Efficient Left and Right Quotienting of Context-Sensitive Grammars","date":"2024-02-28","arxiv_id":"2402.17988","repositories_listed":1,"syntology":{"n":6,"n_ran":4,"n_constructed":0,"n_ran_checked":2,"n_instrument":2,"n_unverified":2,"n_honours":1,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 1 honoured, 0 violated, 1 with no contract checked; 2 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/constrained-decoding-for-code-language-models#ran","syntology_url":"https://syntology.ai/paper/2402.17988","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2402.17988"}},"official":{"repos":["amazon-science/incremental-parsing"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/humaneval-on-latest-gpt-models-2024","slug":"humaneval-on-latest-gpt-models-2024","title":"HumanEval on Latest GPT Models -- 2024","date":"2024-02-20","arxiv_id":"2402.14852","repositories_listed":1,"syntology":{"n":3,"n_ran":2,"n_constructed":0,"n_ran_checked":0,"n_instrument":2,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":1,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/humaneval-on-latest-gpt-models-2024#ran","syntology_url":"https://syntology.ai/paper/2402.14852","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2402.14852"}},"official":{"repos":["daniel442li/gpt-human-eval"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/ltl-learning-on-gpus","slug":"ltl-learning-on-gpus","title":"LTL learning on GPUs","date":"2024-02-19","arxiv_id":"2402.12373","repositories_listed":1,"syntology":null},{"url":"/paper/swissnyf-tool-grounded-llm-agents-for-black","slug":"swissnyf-tool-grounded-llm-agents-for-black","title":"SwissNYF: Tool Grounded LLM Agents for Black Box Setting","date":"2024-02-15","arxiv_id":"2402.10051","repositories_listed":1,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":0,"n_instrument":2,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":2,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/swissnyf-tool-grounded-llm-agents-for-black#ran","syntology_url":"https://syntology.ai/paper/2402.10051","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2402.10051"}},"official":{"repos":["iclr-dummy-user/swissnyf"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/pix2code-learning-to-compose-neural-visual","slug":"pix2code-learning-to-compose-neural-visual","title":"Pix2Code: Learning to Compose Neural Visual Concepts as Programs","date":"2024-02-13","arxiv_id":"2402.08280","repositories_listed":1,"syntology":{"n":2,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":1,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":1,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/pix2code-learning-to-compose-neural-visual#ran","syntology_url":"https://syntology.ai/paper/2402.08280","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2402.08280"}},"official":{"repos":["ml-research/pix2code"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/codeit-self-improving-language-models-with","slug":"codeit-self-improving-language-models-with","title":"CodeIt: Self-Improving Language Models with Prioritized Hindsight Replay","date":"2024-02-07","arxiv_id":"2402.04858","repositories_listed":1,"syntology":{"n":41,"n_ran":32,"n_constructed":4,"n_ran_checked":10,"n_instrument":22,"n_unverified":9,"n_honours":4,"n_violates":2,"n_no_contract":4,"n_pointer_only":41,"phrase":"32 ran (of which 4 constructed an object rather than computing a result; 10 with no instrument failure: 4 honoured, 2 violated, 4 with no contract checked; 22 where Syntology's instrument failed) · 9 unverified","sample_list":"/paper/codeit-self-improving-language-models-with#ran","syntology_url":"https://syntology.ai/paper/2402.04858","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2402.04858"}},"official":{"repos":["Qualcomm-AI-research/codeit"],"state":"official (archive's flag): 32 ran","n_ran":32,"n_constructed":4,"n_ran_no_instrument_failure":10,"n_unverified":9,"ran_from_kinds":["official"]}}},{"url":"/paper/opening-the-ai-black-box-program-synthesis","slug":"opening-the-ai-black-box-program-synthesis","title":"Opening the AI black box: program synthesis via mechanistic interpretability","date":"2024-02-07","arxiv_id":"2402.05110","repositories_listed":1,"syntology":{"n":4,"n_ran":3,"n_constructed":0,"n_ran_checked":0,"n_instrument":3,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":4,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 3 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/opening-the-ai-black-box-program-synthesis#ran","syntology_url":"https://syntology.ai/paper/2402.05110","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2402.05110"}},"official":{"repos":["ejmichaud/neural-verification"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/regal-refactoring-programs-to-discover","slug":"regal-refactoring-programs-to-discover","title":"ReGAL: Refactoring Programs to Discover Generalizable Abstractions","date":"2024-01-29","arxiv_id":"2401.16467","repositories_listed":1,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":2,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":2,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/regal-refactoring-programs-to-discover#ran","syntology_url":"https://syntology.ai/paper/2401.16467","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2401.16467"}},"official":{"repos":["esteng/regal_program_learning"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/dalex-lexicase-like-selection-via-diverse","slug":"dalex-lexicase-like-selection-via-diverse","title":"DALex: Lexicase-like Selection via Diverse Aggregation","date":"2024-01-23","arxiv_id":"2401.12424","repositories_listed":1,"syntology":null},{"url":"/paper/analyzing-the-effectiveness-of-large-language","slug":"analyzing-the-effectiveness-of-large-language","title":"Analyzing the Effectiveness of Large Language Models on Text-to-SQL Synthesis","date":"2024-01-22","arxiv_id":"2401.12379","repositories_listed":1,"syntology":null},{"url":"/paper/codescholar-growing-idiomatic-code-examples","slug":"codescholar-growing-idiomatic-code-examples","title":"CodeScholar: Growing Idiomatic Code Examples","date":"2023-12-23","arxiv_id":"2312.15157","repositories_listed":1,"syntology":null},{"url":"/paper/automating-the-design-of-multigrid-methods","slug":"automating-the-design-of-multigrid-methods","title":"Automating the Design of Multigrid Methods with Evolutionary Program Synthesis","date":"2023-12-22","arxiv_id":"2312.14875","repositories_listed":1,"syntology":null},{"url":"/paper/ken-kernel-extensions-using-natural-language","slug":"ken-kernel-extensions-using-natural-language","title":"KEN: Kernel Extensions using Natural Language","date":"2023-12-09","arxiv_id":"2312.05531","repositories_listed":1,"syntology":null},{"url":"/paper/generating-pragmatic-examples-to-train-neural","slug":"generating-pragmatic-examples-to-train-neural","title":"Generating Pragmatic Examples to Train Neural Program Synthesizers","date":"2023-11-09","arxiv_id":"2311.05740","repositories_listed":1,"syntology":{"n":3,"n_ran":3,"n_constructed":2,"n_ran_checked":3,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":1,"n_no_contract":2,"n_pointer_only":3,"phrase":"3 ran (of which 2 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 1 violated, 2 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/generating-pragmatic-examples-to-train-neural#ran","syntology_url":"https://syntology.ai/paper/2311.05740","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2311.05740"}},"official":{"repos":["saujasv/generating-pragmatic-examples"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":2,"n_ran_no_instrument_failure":3,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/lilo-learning-interpretable-libraries-by","slug":"lilo-learning-interpretable-libraries-by","title":"LILO: Learning Interpretable Libraries by Compressing and Documenting Code","date":"2023-10-30","arxiv_id":"2310.19791","repositories_listed":1,"syntology":{"n":9,"n_ran":8,"n_constructed":0,"n_ran_checked":8,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":8,"n_pointer_only":9,"phrase":"8 ran (of which 0 constructed an object rather than computing a result; 8 with no instrument failure: 0 honoured, 0 violated, 8 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/lilo-learning-interpretable-libraries-by#ran","syntology_url":"https://syntology.ai/paper/2310.19791","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2310.19791"}},"official":{"repos":["gabegrand/lilo"],"state":"official (archive's flag): 8 ran","n_ran":8,"n_constructed":0,"n_ran_no_instrument_failure":8,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/learning-of-generalizable-and-interpretable","slug":"learning-of-generalizable-and-interpretable","title":"Learning of Generalizable and Interpretable Knowledge in Grid-Based Reinforcement Learning Environments","date":"2023-09-07","arxiv_id":"2309.03651","repositories_listed":1,"syntology":null},{"url":"/paper/amortizing-pragmatic-program-synthesis-with","slug":"amortizing-pragmatic-program-synthesis-with","title":"Amortizing Pragmatic Program Synthesis with Rankings","date":"2023-09-01","arxiv_id":"2309.03225","repositories_listed":1,"syntology":null},{"url":"/paper/learning-mdl-logic-programs-from-noisy-data","slug":"learning-mdl-logic-programs-from-noisy-data","title":"Learning MDL logic programs from noisy data","date":"2023-08-18","arxiv_id":"2308.09393","repositories_listed":1,"syntology":{"n":16,"n_ran":13,"n_constructed":0,"n_ran_checked":9,"n_instrument":4,"n_unverified":3,"n_honours":2,"n_violates":0,"n_no_contract":7,"n_pointer_only":16,"phrase":"13 ran (of which 0 constructed an object rather than computing a result; 9 with no instrument failure: 2 honoured, 0 violated, 7 with no contract checked; 4 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/learning-mdl-logic-programs-from-noisy-data#ran","syntology_url":"https://syntology.ai/paper/2308.09393","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2308.09393"}},"official":{"repos":["celinehocquette/aaai24-maxsynth"],"state":"official (archive's flag): 13 ran","n_ran":13,"n_constructed":0,"n_ran_no_instrument_failure":9,"n_unverified":3,"ran_from_kinds":["official"]}}},{"url":"/paper/learning-logic-programs-by-discovering-higher","slug":"learning-logic-programs-by-discovering-higher","title":"Learning logic programs by discovering higher-order abstractions","date":"2023-08-16","arxiv_id":"2308.08334","repositories_listed":1,"syntology":null},{"url":"/paper/enhancing-network-management-using-code","slug":"enhancing-network-management-using-code","title":"Enhancing Network Management Using Code Generated by Large Language Models","date":"2023-08-11","arxiv_id":"2308.06261","repositories_listed":1,"syntology":null},{"url":"/paper/chatgpt-for-gtfs-from-words-to-information","slug":"chatgpt-for-gtfs-from-words-to-information","title":"ChatGPT for GTFS: Benchmarking LLMs on GTFS Understanding and Retrieval","date":"2023-08-04","arxiv_id":"2308.02618","repositories_listed":1,"syntology":null},{"url":"/paper/gran-is-superior-to-graphrnn-node-orderings","slug":"gran-is-superior-to-graphrnn-node-orderings","title":"GRAN is superior to GraphRNN: node orderings, kernel- and graph embeddings-based metrics for graph generators","date":"2023-07-13","arxiv_id":"2307.06709","repositories_listed":1,"syntology":null},{"url":"/paper/rltf-reinforcement-learning-from-unit-test","slug":"rltf-reinforcement-learning-from-unit-test","title":"RLTF: Reinforcement Learning from Unit Test Feedback","date":"2023-07-10","arxiv_id":"2307.04349","repositories_listed":1,"syntology":{"n":9,"n_ran":6,"n_constructed":0,"n_ran_checked":3,"n_instrument":3,"n_unverified":3,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":2,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 3 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/rltf-reinforcement-learning-from-unit-test#ran","syntology_url":"https://syntology.ai/paper/2307.04349","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2307.04349"}},"official":{"repos":["zyq-scut/rltf"],"state":"official (archive's flag): 6 ran","n_ran":6,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":3,"ran_from_kinds":["official"]}}},{"url":"/paper/lambdabeam-neural-program-search-with-higher","slug":"lambdabeam-neural-program-search-with-higher","title":"LambdaBeam: Neural Program Search with Higher-Order Functions and Lambdas","date":"2023-06-03","arxiv_id":"2306.02049","repositories_listed":1,"syntology":{"n":7,"n_ran":6,"n_constructed":0,"n_ran_checked":6,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":6,"n_pointer_only":0,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 6 with no instrument failure: 0 honoured, 0 violated, 6 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/lambdabeam-neural-program-search-with-higher#ran","syntology_url":"https://syntology.ai/paper/2306.02049","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2306.02049"}},"official":null}},{"url":"/paper/anpl-compiling-natural-programs-with","slug":"anpl-compiling-natural-programs-with","title":"ANPL: Towards Natural Programming with Interactive Decomposition","date":"2023-05-29","arxiv_id":"2305.18498","repositories_listed":1,"syntology":null},{"url":"/paper/search-based-regular-expression-inference-on","slug":"search-based-regular-expression-inference-on","title":"Search-Based Regular Expression Inference on a GPU","date":"2023-05-29","arxiv_id":"2305.18575","repositories_listed":1,"syntology":null},{"url":"/paper/gorilla-large-language-model-connected-with","slug":"gorilla-large-language-model-connected-with","title":"Gorilla: Large Language Model Connected with Massive APIs","date":"2023-05-24","arxiv_id":"2305.15334","repositories_listed":1,"syntology":null},{"url":"/paper/critic-large-language-models-can-self-correct","slug":"critic-large-language-models-can-self-correct","title":"CRITIC: Large Language Models Can Self-Correct with Tool-Interactive Critiquing","date":"2023-05-19","arxiv_id":"2305.11738","repositories_listed":1,"syntology":{"n":5,"n_ran":4,"n_constructed":0,"n_ran_checked":0,"n_instrument":4,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 4 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/critic-large-language-models-can-self-correct#ran","syntology_url":"https://syntology.ai/paper/2305.11738","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2305.11738"}},"official":{"repos":["microsoft/ProphetNet"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/probabilistic-lexicase-selection","slug":"probabilistic-lexicase-selection","title":"Probabilistic Lexicase Selection","date":"2023-05-19","arxiv_id":"2305.11681","repositories_listed":1,"syntology":null},{"url":"/paper/evidence-of-meaning-in-language-models","slug":"evidence-of-meaning-in-language-models","title":"Emergent Representations of Program Semantics in Language Models Trained on Programs","date":"2023-05-18","arxiv_id":"2305.11169","repositories_listed":1,"syntology":{"n":13,"n_ran":12,"n_constructed":0,"n_ran_checked":0,"n_instrument":12,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":13,"phrase":"12 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 12 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/evidence-of-meaning-in-language-models#ran","syntology_url":"https://syntology.ai/paper/2305.11169","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2305.11169"}},"official":{"repos":["charlesjin/emergent-semantics"],"state":"official (archive's flag): 12 ran","n_ran":12,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/learning-to-parallelize-with-openmp-by","slug":"learning-to-parallelize-with-openmp-by","title":"Learning to Parallelize with OpenMP by Augmented Heterogeneous AST Representation","date":"2023-05-09","arxiv_id":"2305.05779","repositories_listed":1,"syntology":null},{"url":"/paper/is-your-code-generated-by-chatgpt-really-1","slug":"is-your-code-generated-by-chatgpt-really-1","title":"Is Your Code Generated by ChatGPT Really Correct? Rigorous Evaluation of Large Language Models for Code Generation","date":"2023-05-02","arxiv_id":"2305.01210","repositories_listed":1,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/is-your-code-generated-by-chatgpt-really-1#ran","syntology_url":"https://syntology.ai/paper/2305.01210","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2305.01210"}},"official":{"repos":["evalplus/evalplus"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/outline-then-details-syntactically-guided","slug":"outline-then-details-syntactically-guided","title":"Outline, Then Details: Syntactically Guided Coarse-To-Fine Code Generation","date":"2023-04-28","arxiv_id":"2305.00909","repositories_listed":1,"syntology":{"n":9,"n_ran":7,"n_constructed":1,"n_ran_checked":5,"n_instrument":2,"n_unverified":2,"n_honours":1,"n_violates":0,"n_no_contract":4,"n_pointer_only":0,"phrase":"7 ran (of which 1 constructed an object rather than computing a result; 5 with no instrument failure: 1 honoured, 0 violated, 4 with no contract checked; 2 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/outline-then-details-syntactically-guided#ran","syntology_url":"https://syntology.ai/paper/2305.00909","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2305.00909"}},"official":{"repos":["vita-group/chaincoder"],"state":"official (archive's flag): 7 ran","n_ran":7,"n_constructed":1,"n_ran_no_instrument_failure":5,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/improving-code-generation-by-training-with","slug":"improving-code-generation-by-training-with","title":"Improving Code Generation by Training with Natural Language Feedback","date":"2023-03-28","arxiv_id":"2303.16749","repositories_listed":1,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/improving-code-generation-by-training-with#ran","syntology_url":"https://syntology.ai/paper/2303.16749","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2303.16749"}},"official":{"repos":["nyu-mll/ILF-for-code-generation"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/wikicoder-learning-to-write-knowledge-powered","slug":"wikicoder-learning-to-write-knowledge-powered","title":"WikiCoder: Learning to Write Knowledge-Powered Code","date":"2023-03-15","arxiv_id":"2303.08574","repositories_listed":1,"syntology":null},{"url":"/paper/controlling-large-language-models-to-generate","slug":"controlling-large-language-models-to-generate","title":"Large Language Models for Code: Security Hardening and Adversarial Testing","date":"2023-02-10","arxiv_id":"2302.05319","repositories_listed":1,"syntology":null},{"url":"/paper/exploring-data-augmentation-for-code","slug":"exploring-data-augmentation-for-code","title":"Exploring Data Augmentation for Code Generation Tasks","date":"2023-02-05","arxiv_id":"2302.03499","repositories_listed":1,"syntology":null},{"url":"/paper/neuri-diversifying-dnn-generation-via","slug":"neuri-diversifying-dnn-generation-via","title":"NeuRI: Diversifying DNN Generation via Inductive Rule Inference","date":"2023-02-04","arxiv_id":"2302.02261","repositories_listed":1,"syntology":null},{"url":"/paper/execution-based-code-generation-using-deep","slug":"execution-based-code-generation-using-deep","title":"Execution-based Code Generation using Deep Reinforcement Learning","date":"2023-01-31","arxiv_id":"2301.13816","repositories_listed":1,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/execution-based-code-generation-using-deep#ran","syntology_url":"https://syntology.ai/paper/2301.13816","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2301.13816"}},"official":{"repos":["reddy-lab-code-research/PPOCoder"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/padl-language-directed-physics-based","slug":"padl-language-directed-physics-based","title":"PADL: Language-Directed Physics-Based Character Control","date":"2023-01-31","arxiv_id":"2301.13868","repositories_listed":1,"syntology":null},{"url":"/paper/parsel-a-unified-natural-language-framework","slug":"parsel-a-unified-natural-language-framework","title":"Parsel: Algorithmic Reasoning with Language Models by Composing Decompositions","date":"2022-12-20","arxiv_id":"2212.10561","repositories_listed":1,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":1,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/parsel-a-unified-natural-language-framework#ran","syntology_url":"https://syntology.ai/paper/2212.10561","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2212.10561"}},"official":{"repos":["ezelikman/parsel"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}}],"syntology_records":51,"syntology_note":"a paper without a record is not a recorded non-run: it may lack an arXiv id or simply be absent from the graph layer"},"key_notes":{"n_ran_checked":"legacy name, kept unchanged so existing readers do not break: it counts the samples that ran with no instrument failure (honoured, violated, and ran with no contract checked); it does not mean a contract was checked, and the pages print it as 'K with no instrument failure', not 'K checked'","n_constructed":"a sub-count of the samples that ran, never subtracted from them and never a failure: an executed sample whose run returned an instance of its own class (fixture_out_type equals the entry name): the run built an object and did not compute a result (Syntology's RAN record, counts.constructed)"},"since_archive_judged":{"papers":0,"papers_placed":0,"judged_by":{"model":"a model","models_as_recorded":[],"from":"lb_rows_run_verified.jsonl: each record's `model` field (the extractor's call that wrote decisions[])"},"note":"a model's judgement from the paper's own tables that they report on this leaderboard, not a result: no value from a paper that was not placed is printed or carried here; papers counts each paper once over these leaderboards","backfill":"8,886 of 18,637 newer papers checked so far (733 more had nothing to place; 41 could not be read)","backfill_complete":false,"leaderboards":[]},"papers_where_code_ran":69,"papers_where_code_ran_first_page":"/task/program-synthesis/papers/ran/1","papers_where_code_ran_caption":"We ran code from the paper's repository; we did not run it on this task or check it against the task's benchmarks.","pages":[{"order":"archive","page":2,"of":5,"url":"/task/program-synthesis/papers/2","json":"/task/program-synthesis/papers/2.json","rows":100},{"order":"archive","page":3,"of":5,"url":"/task/program-synthesis/papers/3","json":"/task/program-synthesis/papers/3.json","rows":100},{"order":"archive","page":4,"of":5,"url":"/task/program-synthesis/papers/4","json":"/task/program-synthesis/papers/4.json","rows":100},{"order":"archive","page":5,"of":5,"url":"/task/program-synthesis/papers/5","json":"/task/program-synthesis/papers/5.json","rows":23},{"order":"ran","page":1,"of":1,"url":"/task/program-synthesis/papers/ran/1","json":"/task/program-synthesis/papers/ran/1.json","rows":69}],"description_links":{"kept":0,"unwrapped_to_text":0,"bare_urls_linked":0,"relative_images_dropped":0,"rule":"internal links are kept only when the target slug exists in the catalog"},"syntology":{"read_at":"2026-10-05T00:36:11+00:00","claim":"Per-sample execution status on synthesized fixtures ('ran N of M samples'); not a correctness claim and not a ranking signal.","status_vocabulary":{"ran_honours":"ran, honoured the contract we drafted","ran_violates":"ran, violated the contract we drafted","ran_draft_wrong":"ran; our contract draft was wrong, not the code","ran_fixture":"ran; the contract check could not run (RAISES, DEP_MISSING or TIMEOUT)","ran":"ran on a synthesized input","unverified":"unverified (harvested, no recorded run)"},"row_route":"each syntology object on a benchmark's first row and on a paper carries sample_list (this site's paper page, #ran), syntology_url (syntology.ai's page for the paper, only when that route probed live) and mcp (the agent call that lists the paper's harvested samples; not a verdict on the paper)","developers":"https://syntology.ai/developers"},"not_shown":{"libraries":"the archive has no per-task library table","trend_sparklines":"the Trend column of the benchmarks table was a rendered image; it is not in the archive","social_and_latest_sorts":"stars and social signals are not in the archive"},"record_sha256":"3af92f4ffceb7d4e6b9d9b3e1f1b8752d33e2400f97155e37211da8b9c486a50","record_changed_at":"2026-10-05","record_changed_at_basis":"changed"}