{"about":{"non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","site":"https://codewithpapers.app","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page","syntology":{"site":"https://syntology.ai","developers":"https://syntology.ai/developers","mcp":{"server":"https://syntology.ai/mcp","transport":"streamable-http","server_card":"https://syntology.ai/.well-known/mcp/server-card.json","auth":{"type":"trial token, no account","trial_token":"https://syntology.ai/api/oauth/trial/token","method":"POST","docs":"https://syntology.ai/developers"}},"have":"https://syntology.ai/api/graph/have?x=<method, arXiv id or title> (free, answers coverage only)","paper_base":"https://syntology.ai/paper/","atlas_base":"https://app.syntology.ai/?focus="},"machine_readable":[{"url":"https://codewithpapers.app/llms.txt","what":"the machine catalog: every machine-readable file, counted"},{"url":"https://codewithpapers.app/index/manifest.json","what":"paper-to-code index by arXiv id, with Syntology's counts"},{"url":"https://codewithpapers.app/search/manifest.json","what":"site search index (titles, authors) and its files"},{"url":"https://codewithpapers.app/download","what":"bulk files: Syntology's layer, described there"},{"url":"https://codewithpapers.app/build_manifest.json","what":"the build record: inputs, counts, exclusions, probes"}]},"url":"/task/code-completion","name":"Code Completion","slug":"code-completion","description_markdown":null,"categories":[{"name":"Computer Code","url":"/area/computer-code"}],"source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","slug_source":"archive_url"},"counts":{"papers_tagged":212,"papers_with_code":108,"benchmarks":6,"benchmark_tables_in_archive":6,"benchmark_tables_shown":6,"benchmark_tables_withheld_as_spam":0,"benchmark_definition":"a leaderboard table with at least one row; benchmark_tables_shown also counts the zero-row tables; benchmark_tables_in_archive adds the tables withheld as spam","datasets":12,"subtasks":1,"parent_tasks":0},"benchmarks":[{"leaderboard":"/sota/code-completion-on-safim","slug":"code-completion-on-safim","dataset":"SAFIM","dataset_url":"/dataset/safim","rows_in_archive":15,"metrics":["Average","Algorithmic","Control","API"],"first_row_in_archive_order":{"model":"deepseek-coder-33b-base","paper_title":"Evaluation of LLMs on Syntax-Aware Code Fill-in-the-Middle Tasks","paper_url":"/paper/evaluation-of-llms-on-syntax-aware-code-fill","paper_date":"2024-03-07","arxiv_id":"2403.04814","code_links":[{"title":"gonglinyuan/safim","url":"https://github.com/gonglinyuan/safim"}],"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":1,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":1,"n_no_contract":0,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 1 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/evaluation-of-llms-on-syntax-aware-code-fill#ran","syntology_url":"https://syntology.ai/paper/2403.04814","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2403.04814"}},"official":{"repos":["gonglinyuan/safim"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}}},{"leaderboard":"/sota/code-completion-on-codexglue-github-java","slug":"code-completion-on-codexglue-github-java","dataset":"CodeXGLUE - Github Java Corpus","dataset_url":"/dataset/codexglue","rows_in_archive":3,"metrics":["Accuracy (token-level)","EM (line-level)","Edit Sim (line-level)"],"first_row_in_archive_order":{"model":"CodeGPT-adapted","paper_title":"CodeXGLUE: A Machine Learning Benchmark Dataset for Code Understanding and Generation","paper_url":"/paper/codexglue-a-machine-learning-benchmark","paper_date":"2021-02-09","arxiv_id":"2102.04664","code_links":[{"title":"microsoft/CodeXGLUE","url":"https://github.com/microsoft/CodeXGLUE"},{"title":"facebookresearch/CodeGen","url":"https://github.com/facebookresearch/CodeGen"},{"title":"sberbank-ai/fusion_brain_aij2021","url":"https://github.com/sberbank-ai/fusion_brain_aij2021"},{"title":"kilimanj4r0/code-summarization-beyond-function-level","url":"https://github.com/kilimanj4r0/code-summarization-beyond-function-level"},{"title":"yueyuel/programgen-lms-reliability","url":"https://github.com/yueyuel/programgen-lms-reliability"},{"title":"Avmb/semantic_neq_game","url":"https://github.com/Avmb/semantic_neq_game"},{"title":"deeplearnxmu/unigencoder","url":"https://github.com/deeplearnxmu/unigencoder"}],"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/codexglue-a-machine-learning-benchmark#ran","syntology_url":"https://syntology.ai/paper/2102.04664","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2102.04664"}},"official":{"repos":["microsoft/CodeXGLUE"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}}},{"leaderboard":"/sota/code-completion-on-codexglue-py150","slug":"code-completion-on-codexglue-py150","dataset":"CodeXGLUE - PY150","dataset_url":"/dataset/codexglue","rows_in_archive":3,"metrics":["Accuracy (token-level)","EM (line-level)","Edit Sim (line-level)"],"first_row_in_archive_order":{"model":"CodeGPT-adapted","paper_title":"CodeXGLUE: A Machine Learning Benchmark Dataset for Code Understanding and Generation","paper_url":"/paper/codexglue-a-machine-learning-benchmark","paper_date":"2021-02-09","arxiv_id":"2102.04664","code_links":[{"title":"microsoft/CodeXGLUE","url":"https://github.com/microsoft/CodeXGLUE"},{"title":"facebookresearch/CodeGen","url":"https://github.com/facebookresearch/CodeGen"},{"title":"sberbank-ai/fusion_brain_aij2021","url":"https://github.com/sberbank-ai/fusion_brain_aij2021"},{"title":"kilimanj4r0/code-summarization-beyond-function-level","url":"https://github.com/kilimanj4r0/code-summarization-beyond-function-level"},{"title":"yueyuel/programgen-lms-reliability","url":"https://github.com/yueyuel/programgen-lms-reliability"},{"title":"Avmb/semantic_neq_game","url":"https://github.com/Avmb/semantic_neq_game"},{"title":"deeplearnxmu/unigencoder","url":"https://github.com/deeplearnxmu/unigencoder"}],"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/codexglue-a-machine-learning-benchmark#ran","syntology_url":"https://syntology.ai/paper/2102.04664","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2102.04664"}},"official":{"repos":["microsoft/CodeXGLUE"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}}},{"leaderboard":"/sota/code-completion-on-dotprompts","slug":"code-completion-on-dotprompts","dataset":"DotPrompts","dataset_url":"/dataset/dotprompts","rows_in_archive":3,"metrics":["Compilation Rate"],"first_row_in_archive_order":{"model":"SantaCoder-MGD","paper_title":"Guiding Language Models of Code with Global Context using Monitors","paper_url":"/paper/guiding-language-models-of-code-with-global","paper_date":"2023-06-19","arxiv_id":"2306.10763","code_links":[{"title":"microsoft/monitors4codegen","url":"https://github.com/microsoft/monitors4codegen"}],"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":1,"n_instrument":1,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/guiding-language-models-of-code-with-global#ran","syntology_url":"https://syntology.ai/paper/2306.10763","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2306.10763"}},"official":{"repos":["microsoft/monitors4codegen"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}}},{"leaderboard":"/sota/code-completion-on-defects4j","slug":"code-completion-on-defects4j","dataset":"Defects4J","dataset_url":"/dataset/defects4j","rows_in_archive":2,"metrics":["Compilation Rate","Pass@1","BLEU"],"first_row_in_archive_order":{"model":"Rambo","paper_title":"RAMBO: Enhancing RAG-based Repository-Level Method Body Completion","paper_url":"/paper/rambo-enhancing-rag-based-repository-level","paper_date":"2024-09-23","arxiv_id":"2409.15204","code_links":[{"title":"ise-uet-vnu/rambo","url":"https://github.com/ise-uet-vnu/rambo"}],"syntology":null}},{"leaderboard":"/sota/code-completion-on-rambo-benchmark","slug":"code-completion-on-rambo-benchmark","dataset":"Rambo Benchmark","dataset_url":null,"rows_in_archive":2,"metrics":["Compilation Rate","BLEU"],"first_row_in_archive_order":{"model":"Rambo","paper_title":"RAMBO: Enhancing RAG-based Repository-Level Method Body Completion","paper_url":"/paper/rambo-enhancing-rag-based-repository-level","paper_date":"2024-09-23","arxiv_id":"2409.15204","code_links":[{"title":"ise-uet-vnu/rambo","url":"https://github.com/ise-uet-vnu/rambo"}],"syntology":null}}],"datasets":[{"url":"/dataset/codexglue","name":"CodeXGLUE","full_name":"","num_papers_in_archive":205},{"url":"/dataset/defects4j","name":"Defects4J","full_name":"Defects4J","num_papers_in_archive":9},{"url":"/dataset/pytorrent","name":"PyTorrent","full_name":"PyTorrent","num_papers_in_archive":5},{"url":"/dataset/rtl-repo","name":"RTL-Repo","full_name":"","num_papers_in_archive":5},{"url":"/dataset/safim","name":"SAFIM","full_name":"Syntax-Aware Fill-In-the-Middle","num_papers_in_archive":5},{"url":"/dataset/mmcode","name":"MMCode","full_name":"","num_papers_in_archive":3},{"url":"/dataset/slnet","name":"SLNET","full_name":"SLNET: A Redistributable Corpus of 3rd-party Simulink Models","num_papers_in_archive":3},{"url":"/dataset/spectre-v1","name":"Spectre-v1","full_name":"","num_papers_in_archive":3},{"url":"/dataset/dotprompts","name":"DotPrompts","full_name":"","num_papers_in_archive":1},{"url":"/dataset/openapi-code-completion","name":"OpenAPI completion refined","full_name":"","num_papers_in_archive":1},{"url":"/dataset/verified-smart-contract-code-comments","name":"Verified Smart Contract Code Comments","full_name":"","num_papers_in_archive":1},{"url":"/dataset/verified-smart-contracts","name":"Verified Smart Contracts","full_name":"","num_papers_in_archive":1}],"subtasks":[{"url":"/task/openapi-code-completion","name":"OpenAPI code completion"}],"parent_tasks":[],"papers":{"order":"repositories listed in the archive (most first), then date (newest first), then slug; the archive holds no stars","population":"every paper the archive tags with this task (distinct papers; spam titles not listed). 'of' counts the papers listed; until 2026-09-25 it counted only the papers with a code link, and that number is of_with_code","page":1,"pages_in_order":3,"rows_per_page":100,"shown":100,"of":212,"of_with_code":108,"tagged_in_all":212,"counts":{"archive_papers_tagged":212,"with_a_code_link":108,"where_syntology_ran_a_sample":42,"not_listed_spam_title":0,"listed":212,"listed_where_code_ran":42,"where_syntology_ran_a_sample_split":{"with_a_run_with_no_instrument_failure":33,"every_run_a_failure_of_syntologys_instrument":9,"listed_with_a_run_with_no_instrument_failure":33,"listed_every_run_a_failure_of_syntologys_instrument":9,"filter":{"states":["a run with no instrument failure","any run, instrument failures included"],"default":"a run with no instrument failure","note":"on the 'only where code ran' pages the default hides, in the browser, the rows where every run was a failure of Syntology's instrument; the second state shows them again. Rows are hidden, never re-ordered; these twins list every row"}},"definition":"distinct papers the archive tags; 'where Syntology ran a sample' counts papers with at least one harvested sample that ran, which is not a correctness claim"},"items":[{"url":"/paper/codexglue-a-machine-learning-benchmark","slug":"codexglue-a-machine-learning-benchmark","title":"CodeXGLUE: A Machine Learning Benchmark Dataset for Code Understanding and Generation","date":"2021-02-09","arxiv_id":"2102.04664","repositories_listed":7,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/codexglue-a-machine-learning-benchmark#ran","syntology_url":"https://syntology.ai/paper/2102.04664","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2102.04664"}},"official":{"repos":["microsoft/CodeXGLUE"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/starcoder-2-and-the-stack-v2-the-next","slug":"starcoder-2-and-the-stack-v2-the-next","title":"StarCoder 2 and The Stack v2: The Next Generation","date":"2024-02-29","arxiv_id":"2402.19173","repositories_listed":4,"syntology":null},{"url":"/paper/datasculpt-crafting-data-landscapes-for-llm","slug":"datasculpt-crafting-data-landscapes-for-llm","title":"DataSculpt: Crafting Data Landscapes for Long-Context LLMs through Multi-Objective Partitioning","date":"2024-09-02","arxiv_id":"2409.00997","repositories_listed":3,"syntology":null},{"url":"/paper/longllmlingua-accelerating-and-enhancing-llms","slug":"longllmlingua-accelerating-and-enhancing-llms","title":"LongLLMLingua: Accelerating and Enhancing LLMs in Long Context Scenarios via Prompt Compression","date":"2023-10-10","arxiv_id":"2310.06839","repositories_listed":3,"syntology":null},{"url":"/paper/longbench-a-bilingual-multitask-benchmark-for","slug":"longbench-a-bilingual-multitask-benchmark-for","title":"LongBench: A Bilingual, Multitask Benchmark for Long Context Understanding","date":"2023-08-28","arxiv_id":"2308.14508","repositories_listed":3,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/longbench-a-bilingual-multitask-benchmark-for#ran","syntology_url":"https://syntology.ai/paper/2308.14508","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2308.14508"}},"official":{"repos":["thudm/longbench"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/open-vocabulary-learning-on-source-code-with","slug":"open-vocabulary-learning-on-source-code-with","title":"Open Vocabulary Learning on Source Code with a Graph-Structured Cache","date":"2018-10-18","arxiv_id":"1810.08305","repositories_listed":3,"syntology":null},{"url":"/paper/on-the-workflows-and-smells-of-leaderboard","slug":"on-the-workflows-and-smells-of-leaderboard","title":"On the Workflows and Smells of Leaderboard Operations (LBOps): An Exploratory Study of Foundation Model Leaderboards","date":"2024-07-04","arxiv_id":"2407.04065","repositories_listed":2,"syntology":null},{"url":"/paper/optimizing-large-language-models-for-openapi","slug":"optimizing-large-language-models-for-openapi","title":"Optimizing Large Language Models for OpenAPI Code Completion","date":"2024-05-24","arxiv_id":"2405.15729","repositories_listed":2,"syntology":null},{"url":"/paper/exploring-safety-generalization-challenges-of","slug":"exploring-safety-generalization-challenges-of","title":"CodeAttack: Revealing Safety Generalization Challenges of Large Language Models via Code Completion","date":"2024-03-12","arxiv_id":"2403.07865","repositories_listed":2,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":1,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/exploring-safety-generalization-challenges-of#ran","syntology_url":"https://syntology.ai/paper/2403.07865","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2403.07865"}},"official":{"repos":["renqibing/CodeAttack"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/scope-is-all-you-need-transforming-llms-for","slug":"scope-is-all-you-need-transforming-llms-for","title":"Scope is all you need: Transforming LLMs for HPC Code","date":"2023-08-18","arxiv_id":"2308.09440","repositories_listed":2,"syntology":null},{"url":"/paper/mpi-rical-data-driven-mpi-distributed","slug":"mpi-rical-data-driven-mpi-distributed","title":"MPI-rical: Data-Driven MPI Distributed Parallelism Assistance with Transformers","date":"2023-05-16","arxiv_id":"2305.09438","repositories_listed":2,"syntology":null},{"url":"/paper/codet5-open-code-large-language-models-for","slug":"codet5-open-code-large-language-models-for","title":"CodeT5+: Open Code Large Language Models for Code Understanding and Generation","date":"2023-05-13","arxiv_id":"2305.07922","repositories_listed":2,"syntology":{"n":4,"n_ran":3,"n_constructed":0,"n_ran_checked":2,"n_instrument":1,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 1 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/codet5-open-code-large-language-models-for#ran","syntology_url":"https://syntology.ai/paper/2305.07922","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2305.07922"}},"official":{"repos":["salesforce/codet5"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/codekgc-code-language-model-for-generative","slug":"codekgc-code-language-model-for-generative","title":"CodeKGC: Code Language Model for Generative Knowledge Graph Construction","date":"2023-04-18","arxiv_id":"2304.09048","repositories_listed":2,"syntology":{"n":16,"n_ran":14,"n_constructed":0,"n_ran_checked":13,"n_instrument":1,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":13,"n_pointer_only":1,"phrase":"14 ran (of which 0 constructed an object rather than computing a result; 13 with no instrument failure: 0 honoured, 0 violated, 13 with no contract checked; 1 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/codekgc-code-language-model-for-generative#ran","syntology_url":"https://syntology.ai/paper/2304.09048","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2304.09048"}},"official":{"repos":["zjunlp/deepke"],"state":"official (archive's flag): 14 ran","n_ran":14,"n_constructed":0,"n_ran_no_instrument_failure":13,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/more-than-you-ve-asked-for-a-comprehensive","slug":"more-than-you-ve-asked-for-a-comprehensive","title":"Not what you've signed up for: Compromising Real-World LLM-Integrated Applications with Indirect Prompt Injection","date":"2023-02-23","arxiv_id":"2302.12173","repositories_listed":2,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":2,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/more-than-you-ve-asked-for-a-comprehensive#ran","syntology_url":"https://syntology.ai/paper/2302.12173","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2302.12173"}},"official":{"repos":["greshake/llm-security","greshake/lm-safety"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/multi-lingual-evaluation-of-code-generation","slug":"multi-lingual-evaluation-of-code-generation","title":"Multi-lingual Evaluation of Code Generation Models","date":"2022-10-26","arxiv_id":"2210.14868","repositories_listed":2,"syntology":{"n":6,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":5,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 5 unverified","sample_list":"/paper/multi-lingual-evaluation-of-code-generation#ran","syntology_url":"https://syntology.ai/paper/2210.14868","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2210.14868"}},"official":{"repos":["amazon-research/mbxp-exec-eval"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":3,"ran_from_kinds":["official"]}}},{"url":"/paper/unixcoder-unified-cross-modal-pre-training","slug":"unixcoder-unified-cross-modal-pre-training","title":"UniXcoder: Unified Cross-Modal Pre-training for Code Representation","date":"2022-03-08","arxiv_id":"2203.03850","repositories_listed":2,"syntology":{"n":4,"n_ran":2,"n_constructed":2,"n_ran_checked":2,"n_instrument":0,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":0,"phrase":"2 ran (of which 2 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 0 where Syntology's instrument failed) · 2 unverified; every one of the 2 samples that ran constructed an object rather than computing a result","sample_list":"/paper/unixcoder-unified-cross-modal-pre-training#ran","syntology_url":"https://syntology.ai/paper/2203.03850","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2203.03850"}},"official":{"repos":["microsoft/CodeBERT"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":1,"n_ran_no_instrument_failure":1,"n_unverified":1,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/energy-based-models-for-code-generation-under","slug":"energy-based-models-for-code-generation-under","title":"Energy-Based Models for Code Generation under Compilability Constraints","date":"2021-06-09","arxiv_id":"2106.04985","repositories_listed":2,"syntology":null},{"url":"/paper/neural-software-analysis","slug":"neural-software-analysis","title":"Neural Software Analysis","date":"2020-11-16","arxiv_id":"2011.07986","repositories_listed":2,"syntology":null},{"url":"/paper/structural-language-models-for-any-code","slug":"structural-language-models-for-any-code","title":"Structural Language Models of Code","date":"2019-09-30","arxiv_id":"1910.00577","repositories_listed":2,"syntology":{"n":8,"n_ran":5,"n_constructed":0,"n_ran_checked":5,"n_instrument":0,"n_unverified":3,"n_honours":0,"n_violates":0,"n_no_contract":5,"n_pointer_only":0,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 0 honoured, 0 violated, 5 with no contract checked; 0 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/structural-language-models-for-any-code#ran","syntology_url":"https://syntology.ai/paper/1910.00577","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1910.00577"}},"official":{"repos":["tech-srl/slm-code-generation"],"state":"official (archive's flag): 5 ran","n_ran":5,"n_constructed":0,"n_ran_no_instrument_failure":5,"n_unverified":3,"ran_from_kinds":["official"]}}},{"url":"/paper/seed-coder-let-the-code-model-curate-data-for","slug":"seed-coder-let-the-code-model-curate-data-for","title":"Seed-Coder: Let the Code Model Curate Data for Itself","date":"2025-06-04","arxiv_id":"2506.03524","repositories_listed":1,"syntology":null},{"url":"/paper/swe-dev-evaluating-and-training-autonomous","slug":"swe-dev-evaluating-and-training-autonomous","title":"SWE-Dev: Evaluating and Training Autonomous Feature-Driven Software Development","date":"2025-05-22","arxiv_id":"2505.16975","repositories_listed":1,"syntology":{"n":5,"n_ran":5,"n_constructed":0,"n_ran_checked":1,"n_instrument":4,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":5,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 4 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/swe-dev-evaluating-and-training-autonomous#ran","syntology_url":"https://syntology.ai/paper/2505.16975","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2505.16975"}},"official":{"repos":["dorothyduuu/swe-dev"],"state":"official (archive's flag): 5 ran","n_ran":5,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/can-you-really-trust-code-copilots-evaluating","slug":"can-you-really-trust-code-copilots-evaluating","title":"Can You Really Trust Code Copilots? Evaluating Large Language Models from a Code Security Perspective","date":"2025-05-15","arxiv_id":"2505.10494","repositories_listed":1,"syntology":null},{"url":"/paper/obscuracoder-powering-efficient-code-lm-pre","slug":"obscuracoder-powering-efficient-code-lm-pre","title":"ObscuraCoder: Powering Efficient Code LM Pre-Training Via Obfuscation Grounding","date":"2025-03-27","arxiv_id":"2504.00019","repositories_listed":1,"syntology":null},{"url":"/paper/logquant-log-distributed-2-bit-quantization","slug":"logquant-log-distributed-2-bit-quantization","title":"LogQuant: Log-Distributed 2-Bit Quantization of KV Cache with Superior Accuracy Preservation","date":"2025-03-25","arxiv_id":"2503.19950","repositories_listed":1,"syntology":null},{"url":"/paper/castle-benchmarking-dataset-for-static-code","slug":"castle-benchmarking-dataset-for-static-code","title":"CASTLE: Benchmarking Dataset for Static Code Analyzers and LLMs towards CWE Detection","date":"2025-03-12","arxiv_id":"2503.09433","repositories_listed":1,"syntology":null},{"url":"/paper/enhancing-high-quality-code-generation-in","slug":"enhancing-high-quality-code-generation-in","title":"Enhancing High-Quality Code Generation in Large Language Models with Comparative Prefix-Tuning","date":"2025-03-12","arxiv_id":"2503.09020","repositories_listed":1,"syntology":null},{"url":"/paper/longspec-long-context-speculative-decoding","slug":"longspec-long-context-speculative-decoding","title":"LongSpec: Long-Context Speculative Decoding with Efficient Drafting and Verification","date":"2025-02-24","arxiv_id":"2502.17421","repositories_listed":1,"syntology":{"n":10,"n_ran":8,"n_constructed":0,"n_ran_checked":6,"n_instrument":2,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":6,"n_pointer_only":0,"phrase":"8 ran (of which 0 constructed an object rather than computing a result; 6 with no instrument failure: 0 honoured, 0 violated, 6 with no contract checked; 2 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/longspec-long-context-speculative-decoding#ran","syntology_url":"https://syntology.ai/paper/2502.17421","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2502.17421"}},"official":{"repos":["sail-sg/longspec"],"state":"official (archive's flag): 8 ran","n_ran":8,"n_constructed":0,"n_ran_no_instrument_failure":6,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/how-to-get-your-llm-to-generate-challenging","slug":"how-to-get-your-llm-to-generate-challenging","title":"How to Get Your LLM to Generate Challenging Problems for Evaluation","date":"2025-02-20","arxiv_id":"2502.14678","repositories_listed":1,"syntology":null},{"url":"/paper/green-code-optimizing-energy-efficiency-in","slug":"green-code-optimizing-energy-efficiency-in","title":"GREEN-CODE: Learning to Optimize Energy Efficiency in LLM-based Code Generation","date":"2025-01-19","arxiv_id":"2501.11006","repositories_listed":1,"syntology":null},{"url":"/paper/gitchameleon-unmasking-the-version-switching","slug":"gitchameleon-unmasking-the-version-switching","title":"GitChameleon: Unmasking the Version-Switching Capabilities of Code Generation Models","date":"2024-11-05","arxiv_id":"2411.05830","repositories_listed":1,"syntology":null},{"url":"/paper/can-language-models-replace-programmers","slug":"can-language-models-replace-programmers","title":"Can Language Models Replace Programmers for Coding? REPOCOD Says 'Not Yet'","date":"2024-10-29","arxiv_id":"2410.21647","repositories_listed":1,"syntology":null},{"url":"/paper/building-a-coding-assistant-via-the-retrieval","slug":"building-a-coding-assistant-via-the-retrieval","title":"Building A Coding Assistant via the Retrieval-Augmented Language Model","date":"2024-10-21","arxiv_id":"2410.16229","repositories_listed":1,"syntology":null},{"url":"/paper/contextual-augmented-multi-model-programming","slug":"contextual-augmented-multi-model-programming","title":"Contextual Augmented Multi-Model Programming (CAMP): A Hybrid Local-Cloud Copilot Framework","date":"2024-10-20","arxiv_id":"2410.15285","repositories_listed":1,"syntology":null},{"url":"/paper/aixcoder-7b-a-lightweight-and-effective-large","slug":"aixcoder-7b-a-lightweight-and-effective-large","title":"aiXcoder-7B: A Lightweight and Effective Large Language Model for Code Processing","date":"2024-10-17","arxiv_id":"2410.13187","repositories_listed":1,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/aixcoder-7b-a-lightweight-and-effective-large#ran","syntology_url":"https://syntology.ai/paper/2410.13187","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2410.13187"}},"official":{"repos":["aixcoder-plugin/aixcoder-7b"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/kv-prediction-for-improved-time-to-first","slug":"kv-prediction-for-improved-time-to-first","title":"KV Prediction for Improved Time to First Token","date":"2024-10-10","arxiv_id":"2410.08391","repositories_listed":1,"syntology":null},{"url":"/paper/cursorcore-assist-programming-through","slug":"cursorcore-assist-programming-through","title":"CursorCore: Assist Programming through Aligning Anything","date":"2024-10-09","arxiv_id":"2410.07002","repositories_listed":1,"syntology":{"n":13,"n_ran":13,"n_constructed":0,"n_ran_checked":11,"n_instrument":2,"n_unverified":0,"n_honours":3,"n_violates":0,"n_no_contract":8,"n_pointer_only":0,"phrase":"13 ran (of which 0 constructed an object rather than computing a result; 11 with no instrument failure: 3 honoured, 0 violated, 8 with no contract checked; 2 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/cursorcore-assist-programming-through#ran","syntology_url":"https://syntology.ai/paper/2410.07002","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2410.07002"}},"official":{"repos":["TechxGenus/CursorCore"],"state":"official (archive's flag): 13 ran","n_ran":13,"n_constructed":0,"n_ran_no_instrument_failure":11,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/codev-bench-how-do-llms-understand-developer","slug":"codev-bench-how-do-llms-understand-developer","title":"Codev-Bench: How Do LLMs Understand Developer-Centric Code Completion?","date":"2024-10-02","arxiv_id":"2410.01353","repositories_listed":1,"syntology":{"n":6,"n_ran":6,"n_constructed":0,"n_ran_checked":2,"n_instrument":4,"n_unverified":0,"n_honours":0,"n_violates":2,"n_no_contract":0,"n_pointer_only":6,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 2 violated, 0 with no contract checked; 4 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/codev-bench-how-do-llms-understand-developer#ran","syntology_url":"https://syntology.ai/paper/2410.01353","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2410.01353"}},"official":{"repos":["LingmaTongyi/Codev-Bench"],"state":"official (archive's flag): 6 ran","n_ran":6,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/rambo-enhancing-rag-based-repository-level","slug":"rambo-enhancing-rag-based-repository-level","title":"RAMBO: Enhancing RAG-based Repository-Level Method Body Completion","date":"2024-09-23","arxiv_id":"2409.15204","repositories_listed":1,"syntology":null},{"url":"/paper/revisiting-verilogeval-newer-llms-in-context","slug":"revisiting-verilogeval-newer-llms-in-context","title":"Revisiting VerilogEval: A Year of Improvements in Large-Language Models for Hardware Code Generation","date":"2024-08-20","arxiv_id":"2408.11053","repositories_listed":1,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":1,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/revisiting-verilogeval-newer-llms-in-context#ran","syntology_url":"https://syntology.ai/paper/2408.11053","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2408.11053"}},"official":{"repos":["nvlabs/verilog-eval"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/while-github-copilot-excels-at-coding-does-it","slug":"while-github-copilot-excels-at-coding-does-it","title":"Security Attacks on LLM-based Code Completion Tools","date":"2024-08-20","arxiv_id":"2408.11006","repositories_listed":1,"syntology":null},{"url":"/paper/scaling-granite-code-models-to-128k-context","slug":"scaling-granite-code-models-to-128k-context","title":"Scaling Granite Code Models to 128K Context","date":"2024-07-18","arxiv_id":"2407.13739","repositories_listed":1,"syntology":null},{"url":"/paper/hierarchical-context-pruning-optimizing-real","slug":"hierarchical-context-pruning-optimizing-real","title":"Hierarchical Context Pruning: Optimizing Real-World Code Completion with Repository-Level Pretrained Code LLMs","date":"2024-06-26","arxiv_id":"2406.18294","repositories_listed":1,"syntology":{"n":1,"n_ran":0,"n_constructed":0,"n_ran_checked":0,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"0 ran · 1 unverified","sample_list":"/paper/hierarchical-context-pruning-optimizing-real#ran","syntology_url":"https://syntology.ai/paper/2406.18294","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.18294"}},"official":{"repos":["hambaobao/hcp-coder"],"state":"official: harvested, nothing ran","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":1,"ran_from_kinds":[]}}},{"url":"/paper/long-code-arena-a-set-of-benchmarks-for-long","slug":"long-code-arena-a-set-of-benchmarks-for-long","title":"Long Code Arena: a Set of Benchmarks for Long-Context Code Models","date":"2024-06-17","arxiv_id":"2406.11612","repositories_listed":1,"syntology":{"n":7,"n_ran":6,"n_constructed":0,"n_ran_checked":4,"n_instrument":2,"n_unverified":1,"n_honours":3,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 4 with no instrument failure: 3 honoured, 0 violated, 1 with no contract checked; 2 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/long-code-arena-a-set-of-benchmarks-for-long#ran","syntology_url":"https://syntology.ai/paper/2406.11612","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.11612"}},"official":{"repos":["jetbrains-research/lca-baselines"],"state":"official (archive's flag): 6 ran","n_ran":6,"n_constructed":0,"n_ran_no_instrument_failure":4,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/versicode-towards-version-controllable-code","slug":"versicode-towards-version-controllable-code","title":"VersiCode: Towards Version-controllable Code Generation","date":"2024-06-11","arxiv_id":"2406.07411","repositories_listed":1,"syntology":null},{"url":"/paper/an-llm-assisted-easy-to-trigger-backdoor","slug":"an-llm-assisted-easy-to-trigger-backdoor","title":"An LLM-Assisted Easy-to-Trigger Backdoor Attack on Code Completion Models: Injecting Disguised Vulnerabilities against Strong Detection","date":"2024-06-10","arxiv_id":"2406.06822","repositories_listed":1,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":0,"n_instrument":3,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":3,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 3 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/an-llm-assisted-easy-to-trigger-backdoor#ran","syntology_url":"https://syntology.ai/paper/2406.06822","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.06822"}},"official":{"repos":["datasec-lab/codebreaker"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/semcoder-training-code-language-models-with","slug":"semcoder-training-code-language-models-with","title":"SemCoder: Training Code Language Models with Comprehensive Semantics Reasoning","date":"2024-06-03","arxiv_id":"2406.01006","repositories_listed":1,"syntology":{"n":15,"n_ran":12,"n_constructed":0,"n_ran_checked":6,"n_instrument":6,"n_unverified":3,"n_honours":1,"n_violates":0,"n_no_contract":5,"n_pointer_only":0,"phrase":"12 ran (of which 0 constructed an object rather than computing a result; 6 with no instrument failure: 1 honoured, 0 violated, 5 with no contract checked; 6 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/semcoder-training-code-language-models-with#ran","syntology_url":"https://syntology.ai/paper/2406.01006","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.01006"}},"official":{"repos":["arise-lab/semcoder"],"state":"official (archive's flag): 12 ran","n_ran":12,"n_constructed":0,"n_ran_no_instrument_failure":6,"n_unverified":3,"ran_from_kinds":["official"]}}},{"url":"/paper/dataflow-guided-retrieval-augmentation-for","slug":"dataflow-guided-retrieval-augmentation-for","title":"Dataflow-Guided Retrieval Augmentation for Repository-Level Code Completion","date":"2024-05-30","arxiv_id":"2405.19782","repositories_listed":1,"syntology":{"n":4,"n_ran":4,"n_constructed":4,"n_ran_checked":4,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":4,"n_pointer_only":4,"phrase":"4 ran (of which 4 constructed an object rather than computing a result; 4 with no instrument failure: 0 honoured, 0 violated, 4 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified; every one of the 4 samples that ran constructed an object rather than computing a result","sample_list":"/paper/dataflow-guided-retrieval-augmentation-for#ran","syntology_url":"https://syntology.ai/paper/2405.19782","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2405.19782"}},"official":{"repos":["nju-websoft/DraCo"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":4,"n_ran_no_instrument_failure":4,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/superposed-decoding-multiple-generations-from","slug":"superposed-decoding-multiple-generations-from","title":"Superposed Decoding: Multiple Generations from a Single Autoregressive Inference Pass","date":"2024-05-28","arxiv_id":"2405.18400","repositories_listed":1,"syntology":{"n":11,"n_ran":11,"n_constructed":1,"n_ran_checked":8,"n_instrument":3,"n_unverified":0,"n_honours":1,"n_violates":1,"n_no_contract":6,"n_pointer_only":11,"phrase":"11 ran (of which 1 constructed an object rather than computing a result; 8 with no instrument failure: 1 honoured, 1 violated, 6 with no contract checked; 3 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/superposed-decoding-multiple-generations-from#ran","syntology_url":"https://syntology.ai/paper/2405.18400","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2405.18400"}},"official":{"repos":["raivnlab/superposeddecoding"],"state":"official (archive's flag): 11 ran","n_ran":11,"n_constructed":1,"n_ran_no_instrument_failure":8,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/a-transformer-based-approach-for-smart","slug":"a-transformer-based-approach-for-smart","title":"A Transformer-Based Approach for Smart Invocation of Automatic Code Completion","date":"2024-05-23","arxiv_id":"2405.14753","repositories_listed":1,"syntology":null},{"url":"/paper/autocoder-enhancing-code-large-language-model","slug":"autocoder-enhancing-code-large-language-model","title":"AutoCoder: Enhancing Code Large Language Model with \\textsc{AIEV-Instruct}","date":"2024-05-23","arxiv_id":"2405.14906","repositories_listed":1,"syntology":null},{"url":"/paper/does-your-neural-code-completion-model-use-my","slug":"does-your-neural-code-completion-model-use-my","title":"Does Your Neural Code Completion Model Use My Code? A Membership Inference Approach","date":"2024-04-22","arxiv_id":"2404.14296","repositories_listed":1,"syntology":null},{"url":"/paper/pctoolkit-a-unified-plug-and-play-prompt","slug":"pctoolkit-a-unified-plug-and-play-prompt","title":"PCToolkit: A Unified Plug-and-Play Prompt Compression Toolkit of Large Language Models","date":"2024-03-26","arxiv_id":"2403.17411","repositories_listed":1,"syntology":null},{"url":"/paper/investigating-the-performance-of-language","slug":"investigating-the-performance-of-language","title":"Investigating the Performance of Language Models for Completing Code in Functional Programming Languages: a Haskell Case Study","date":"2024-03-22","arxiv_id":"2403.15185","repositories_listed":1,"syntology":null},{"url":"/paper/repohyper-better-context-retrieval-is-all-you","slug":"repohyper-better-context-retrieval-is-all-you","title":"RepoHyper: Search-Expand-Refine on Semantic Graphs for Repository-Level Code Completion","date":"2024-03-10","arxiv_id":"2403.06095","repositories_listed":1,"syntology":{"n":2,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/repohyper-better-context-retrieval-is-all-you#ran","syntology_url":"https://syntology.ai/paper/2403.06095","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2403.06095"}},"official":{"repos":["fsoft-ai4code/repohyper"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/evaluation-of-llms-on-syntax-aware-code-fill","slug":"evaluation-of-llms-on-syntax-aware-code-fill","title":"Evaluation of LLMs on Syntax-Aware Code Fill-in-the-Middle Tasks","date":"2024-03-07","arxiv_id":"2403.04814","repositories_listed":1,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":1,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":1,"n_no_contract":0,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 1 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/evaluation-of-llms-on-syntax-aware-code-fill#ran","syntology_url":"https://syntology.ai/paper/2403.04814","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2403.04814"}},"official":{"repos":["gonglinyuan/safim"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/ircoder-intermediate-representations-make","slug":"ircoder-intermediate-representations-make","title":"IRCoder: Intermediate Representations Make Language Models Robust Multilingual Code Generators","date":"2024-03-06","arxiv_id":"2403.03894","repositories_listed":1,"syntology":null},{"url":"/paper/codechameleon-personalized-encryption","slug":"codechameleon-personalized-encryption","title":"CodeChameleon: Personalized Encryption Framework for Jailbreaking Large Language Models","date":"2024-02-26","arxiv_id":"2402.16717","repositories_listed":1,"syntology":{"n":8,"n_ran":5,"n_constructed":0,"n_ran_checked":2,"n_instrument":3,"n_unverified":3,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":0,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 3 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/codechameleon-personalized-encryption#ran","syntology_url":"https://syntology.ai/paper/2402.16717","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2402.16717"}},"official":{"repos":["huizhang-l/codechameleon"],"state":"official (archive's flag): 5 ran","n_ran":5,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":3,"ran_from_kinds":["official"]}}},{"url":"/paper/language-models-for-code-completion-a","slug":"language-models-for-code-completion-a","title":"Language Models for Code Completion: A Practical Evaluation","date":"2024-02-25","arxiv_id":"2402.16197","repositories_listed":1,"syntology":null},{"url":"/paper/break-the-sequential-dependency-of-llm","slug":"break-the-sequential-dependency-of-llm","title":"Break the Sequential Dependency of LLM Inference Using Lookahead Decoding","date":"2024-02-03","arxiv_id":"2402.02057","repositories_listed":1,"syntology":null},{"url":"/paper/effibench-benchmarking-the-efficiency-of","slug":"effibench-benchmarking-the-efficiency-of","title":"EffiBench: Benchmarking the Efficiency of Automatically Generated Code","date":"2024-02-03","arxiv_id":"2402.02037","repositories_listed":1,"syntology":{"n":5,"n_ran":3,"n_constructed":0,"n_ran_checked":1,"n_instrument":2,"n_unverified":2,"n_honours":0,"n_violates":1,"n_no_contract":0,"n_pointer_only":5,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 1 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/effibench-benchmarking-the-efficiency-of#ran","syntology_url":"https://syntology.ai/paper/2402.02037","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2402.02037"}},"official":{"repos":["huangd1999/EffiBench"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/stepcoder-improve-code-generation-with","slug":"stepcoder-improve-code-generation-with","title":"StepCoder: Improve Code Generation with Reinforcement Learning from Compiler Feedback","date":"2024-02-02","arxiv_id":"2402.01391","repositories_listed":1,"syntology":null},{"url":"/paper/can-large-language-models-write-parallel-code","slug":"can-large-language-models-write-parallel-code","title":"Can Large Language Models Write Parallel Code?","date":"2024-01-23","arxiv_id":"2401.12554","repositories_listed":1,"syntology":{"n":19,"n_ran":18,"n_constructed":0,"n_ran_checked":13,"n_instrument":5,"n_unverified":1,"n_honours":3,"n_violates":5,"n_no_contract":5,"n_pointer_only":0,"phrase":"18 ran (of which 0 constructed an object rather than computing a result; 13 with no instrument failure: 3 honoured, 5 violated, 5 with no contract checked; 5 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/can-large-language-models-write-parallel-code#ran","syntology_url":"https://syntology.ai/paper/2401.12554","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2401.12554"}},"official":{"repos":["parallelcodefoundry/ParEval"],"state":"official (archive's flag): 18 ran","n_ran":18,"n_constructed":0,"n_ran_no_instrument_failure":13,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/langbridge-multilingual-reasoning-without","slug":"langbridge-multilingual-reasoning-without","title":"LangBridge: Multilingual Reasoning Without Multilingual Supervision","date":"2024-01-19","arxiv_id":"2401.10695","repositories_listed":1,"syntology":{"n":12,"n_ran":5,"n_constructed":0,"n_ran_checked":3,"n_instrument":2,"n_unverified":7,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":12,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 2 where Syntology's instrument failed) · 7 unverified","sample_list":"/paper/langbridge-multilingual-reasoning-without#ran","syntology_url":"https://syntology.ai/paper/2401.10695","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2401.10695"}},"official":{"repos":["kaistAI/LangBridge"],"state":"official (archive's flag): 5 ran","n_ran":5,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":7,"ran_from_kinds":["official"]}}},{"url":"/paper/when-neural-code-completion-models-size-up","slug":"when-neural-code-completion-models-size-up","title":"When Neural Code Completion Models Size up the Situation: Attaining Cheaper and Faster Completion through Dynamic Model Inference","date":"2024-01-18","arxiv_id":"2401.09964","repositories_listed":1,"syntology":null},{"url":"/paper/traces-of-memorisation-in-large-language","slug":"traces-of-memorisation-in-large-language","title":"Traces of Memorisation in Large Language Models for Code","date":"2023-12-18","arxiv_id":"2312.11658","repositories_listed":1,"syntology":null},{"url":"/paper/breaking-the-silence-the-threats-of-using","slug":"breaking-the-silence-the-threats-of-using","title":"Breaking the Silence: the Threats of Using LLMs in Software Engineering","date":"2023-12-13","arxiv_id":"2312.08055","repositories_listed":1,"syntology":null},{"url":"/paper/inspect-intrinsic-and-systematic-probing","slug":"inspect-intrinsic-and-systematic-probing","title":"INSPECT: Intrinsic and Systematic Probing Evaluation for Code Transformers","date":"2023-12-08","arxiv_id":"2312.05092","repositories_listed":1,"syntology":null},{"url":"/paper/gencodesearchnet-a-benchmark-test-suite-for","slug":"gencodesearchnet-a-benchmark-test-suite-for","title":"GenCodeSearchNet: A Benchmark Test Suite for Evaluating Generalization in Programming Language Understanding","date":"2023-11-16","arxiv_id":"2311.09707","repositories_listed":1,"syntology":null},{"url":"/paper/crosscodeeval-a-diverse-and-multilingual","slug":"crosscodeeval-a-diverse-and-multilingual","title":"CrossCodeEval: A Diverse and Multilingual Benchmark for Cross-File Code Completion","date":"2023-10-17","arxiv_id":"2310.11248","repositories_listed":1,"syntology":{"n":6,"n_ran":4,"n_constructed":0,"n_ran_checked":4,"n_instrument":0,"n_unverified":2,"n_honours":0,"n_violates":1,"n_no_contract":3,"n_pointer_only":0,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 4 with no instrument failure: 0 honoured, 1 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/crosscodeeval-a-diverse-and-multilingual#ran","syntology_url":"https://syntology.ai/paper/2310.11248","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2310.11248"}},"official":null}},{"url":"/paper/ada-instruct-adapting-instruction-generators","slug":"ada-instruct-adapting-instruction-generators","title":"Ada-Instruct: Adapting Instruction Generators for Complex Reasoning","date":"2023-10-06","arxiv_id":"2310.04484","repositories_listed":1,"syntology":null},{"url":"/paper/bamboo-a-comprehensive-benchmark-for","slug":"bamboo-a-comprehensive-benchmark-for","title":"BAMBOO: A Comprehensive Benchmark for Evaluating Long Text Modeling Capacities of Large Language Models","date":"2023-09-23","arxiv_id":"2309.13345","repositories_listed":1,"syntology":{"n":4,"n_ran":4,"n_constructed":0,"n_ran_checked":1,"n_instrument":3,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":4,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 3 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/bamboo-a-comprehensive-benchmark-for#ran","syntology_url":"https://syntology.ai/paper/2309.13345","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2309.13345"}},"official":{"repos":["rucaibox/bamboo"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/layoutnuwa-revealing-the-hidden-layout","slug":"layoutnuwa-revealing-the-hidden-layout","title":"LayoutNUWA: Revealing the Hidden Layout Expertise of Large Language Models","date":"2023-09-18","arxiv_id":"2309.09506","repositories_listed":1,"syntology":null},{"url":"/paper/codemark-imperceptible-watermarking-for-code","slug":"codemark-imperceptible-watermarking-for-code","title":"CodeMark: Imperceptible Watermarking for Code Datasets against Neural Code Completion Models","date":"2023-08-28","arxiv_id":"2308.14401","repositories_listed":1,"syntology":null},{"url":"/paper/longcoder-a-long-range-pre-trained-language","slug":"longcoder-a-long-range-pre-trained-language","title":"LongCoder: A Long-Range Pre-trained Language Model for Code Completion","date":"2023-06-26","arxiv_id":"2306.14893","repositories_listed":1,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/longcoder-a-long-range-pre-trained-language#ran","syntology_url":"https://syntology.ai/paper/2306.14893","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2306.14893"}},"official":{"repos":["microsoft/CodeBERT"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/guiding-language-models-of-code-with-global","slug":"guiding-language-models-of-code-with-global","title":"Guiding Language Models of Code with Global Context using Monitors","date":"2023-06-19","arxiv_id":"2306.10763","repositories_listed":1,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":1,"n_instrument":1,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/guiding-language-models-of-code-with-global#ran","syntology_url":"https://syntology.ai/paper/2306.10763","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2306.10763"}},"official":{"repos":["microsoft/monitors4codegen"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/large-language-models-of-code-fail-at","slug":"large-language-models-of-code-fail-at","title":"Large Language Models of Code Fail at Completing Code with Potential Bugs","date":"2023-06-06","arxiv_id":"2306.03438","repositories_listed":1,"syntology":{"n":10,"n_ran":10,"n_constructed":0,"n_ran_checked":9,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":9,"n_pointer_only":1,"phrase":"10 ran (of which 0 constructed an object rather than computing a result; 9 with no instrument failure: 0 honoured, 0 violated, 9 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/large-language-models-of-code-fail-at#ran","syntology_url":"https://syntology.ai/paper/2306.03438","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2306.03438"}},"official":{"repos":["amazon-science/buggy-code-completion"],"state":"official (archive's flag): 10 ran","n_ran":10,"n_constructed":0,"n_ran_no_instrument_failure":9,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/repobench-benchmarking-repository-level-code","slug":"repobench-benchmarking-repository-level-code","title":"RepoBench: Benchmarking Repository-Level Code Auto-Completion Systems","date":"2023-06-05","arxiv_id":"2306.03091","repositories_listed":1,"syntology":null},{"url":"/paper/how-effective-are-neural-networks-for-fixing","slug":"how-effective-are-neural-networks-for-fixing","title":"How Effective Are Neural Networks for Fixing Security Vulnerabilities","date":"2023-05-29","arxiv_id":"2305.18607","repositories_listed":1,"syntology":{"n":4,"n_ran":4,"n_constructed":0,"n_ran_checked":0,"n_instrument":4,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":4,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 4 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/how-effective-are-neural-networks-for-fixing#ran","syntology_url":"https://syntology.ai/paper/2305.18607","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2305.18607"}},"official":{"repos":["lin-tan/llm-vul"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/merge-fast-private-text-generation","slug":"merge-fast-private-text-generation","title":"MERGE: Fast Private Text Generation","date":"2023-05-25","arxiv_id":"2305.15769","repositories_listed":1,"syntology":null},{"url":"/paper/enriching-source-code-with-contextual-data","slug":"enriching-source-code-with-contextual-data","title":"Enriching Source Code with Contextual Data for Code Completion Models: An Empirical Study","date":"2023-04-24","arxiv_id":"2304.12269","repositories_listed":1,"syntology":null},{"url":"/paper/repocoder-repository-level-code-completion","slug":"repocoder-repository-level-code-completion","title":"RepoCoder: Repository-Level Code Completion Through Iterative Retrieval and Generation","date":"2023-03-22","arxiv_id":"2303.12570","repositories_listed":1,"syntology":{"n":7,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":6,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 6 unverified","sample_list":"/paper/repocoder-repository-level-code-completion#ran","syntology_url":"https://syntology.ai/paper/2303.12570","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2303.12570"}},"official":{"repos":["microsoft/codet"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":6,"ran_from_kinds":["official"]}}},{"url":"/paper/llmseceval-a-dataset-of-natural-language","slug":"llmseceval-a-dataset-of-natural-language","title":"LLMSecEval: A Dataset of Natural Language Prompts for Security Evaluations","date":"2023-03-16","arxiv_id":"2303.09384","repositories_listed":1,"syntology":null},{"url":"/paper/exploring-chatgpt-s-ability-to-rank-content-a","slug":"exploring-chatgpt-s-ability-to-rank-content-a","title":"Exploring ChatGPT's Ability to Rank Content: A Preliminary Study on Consistency with Human Preferences","date":"2023-03-14","arxiv_id":"2303.07610","repositories_listed":1,"syntology":null},{"url":"/paper/learning-deep-semantics-for-test-completion","slug":"learning-deep-semantics-for-test-completion","title":"Learning Deep Semantics for Test Completion","date":"2023-02-20","arxiv_id":"2302.10166","repositories_listed":1,"syntology":null},{"url":"/paper/automating-code-related-tasks-through","slug":"automating-code-related-tasks-through","title":"Automating Code-Related Tasks Through Transformers: The Impact of Pre-training","date":"2023-02-08","arxiv_id":"2302.04048","repositories_listed":1,"syntology":null},{"url":"/paper/execution-based-code-generation-using-deep","slug":"execution-based-code-generation-using-deep","title":"Execution-based Code Generation using Deep Reinforcement Learning","date":"2023-01-31","arxiv_id":"2301.13816","repositories_listed":1,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/execution-based-code-generation-using-deep#ran","syntology_url":"https://syntology.ai/paper/2301.13816","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2301.13816"}},"official":{"repos":["reddy-lab-code-research/PPOCoder"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/cocomic-code-completion-by-jointly-modeling","slug":"cocomic-code-completion-by-jointly-modeling","title":"CoCoMIC: Code Completion By Jointly Modeling In-file and Cross-file Context","date":"2022-12-20","arxiv_id":"2212.10007","repositories_listed":1,"syntology":{"n":2,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/cocomic-code-completion-by-jointly-modeling#ran","syntology_url":"https://syntology.ai/paper/2212.10007","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2212.10007"}},"official":{"repos":["amazon-science/cocomic"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/syntax-aware-on-the-fly-code-completion","slug":"syntax-aware-on-the-fly-code-completion","title":"Syntax-Aware On-the-Fly Code Completion","date":"2022-11-09","arxiv_id":"2211.04673","repositories_listed":1,"syntology":null},{"url":"/paper/reading-between-the-lines-modeling-user","slug":"reading-between-the-lines-modeling-user","title":"Reading Between the Lines: Modeling User Behavior and Costs in AI-Assisted Programming","date":"2022-10-25","arxiv_id":"2210.14306","repositories_listed":1,"syntology":null},{"url":"/paper/learning-to-prevent-profitless-neural-code","slug":"learning-to-prevent-profitless-neural-code","title":"Don't Complete It! Preventing Unhelpful Code Completion for Productive and Sustainable Neural Code Completion Systems","date":"2022-09-13","arxiv_id":"2209.05948","repositories_listed":1,"syntology":null},{"url":"/paper/metatptrans-a-meta-learning-approach-for","slug":"metatptrans-a-meta-learning-approach-for","title":"MetaTPTrans: A Meta Learning Approach for Multilingual Code Representation Learning","date":"2022-06-13","arxiv_id":"2206.06460","repositories_listed":1,"syntology":null},{"url":"/paper/productivity-assessment-of-neural-code","slug":"productivity-assessment-of-neural-code","title":"Productivity Assessment of Neural Code Completion","date":"2022-05-13","arxiv_id":"2205.06537","repositories_listed":1,"syntology":null},{"url":"/paper/reacc-a-retrieval-augmented-code-completion","slug":"reacc-a-retrieval-augmented-code-completion","title":"ReACC: A Retrieval-Augmented Code Completion Framework","date":"2022-03-15","arxiv_id":"2203.07722","repositories_listed":1,"syntology":null},{"url":"/paper/codefill-multi-token-code-completion-by","slug":"codefill-multi-token-code-completion-by","title":"CodeFill: Multi-token Code Completion by Jointly Learning from Structure and Naming Sequences","date":"2022-02-14","arxiv_id":"2202.06689","repositories_listed":1,"syntology":null},{"url":"/paper/what-do-they-capture-a-structural-analysis-of","slug":"what-do-they-capture-a-structural-analysis-of","title":"What Do They Capture? -- A Structural Analysis of Pre-Trained Language Models for Source Code","date":"2022-02-14","arxiv_id":"2202.06840","repositories_listed":1,"syntology":null},{"url":"/paper/time-efficient-code-completion-model-for-the","slug":"time-efficient-code-completion-model-for-the","title":"Time-Efficient Code Completion Model for the R Programming Language","date":"2021-08-01","arxiv_id":null,"repositories_listed":1,"syntology":null},{"url":"/paper/a-syntax-guided-edit-decoder-for-neural","slug":"a-syntax-guided-edit-decoder-for-neural","title":"A Syntax-Guided Edit Decoder for Neural Program Repair","date":"2021-06-15","arxiv_id":"2106.08253","repositories_listed":1,"syntology":{"n":14,"n_ran":12,"n_constructed":1,"n_ran_checked":9,"n_instrument":3,"n_unverified":2,"n_honours":0,"n_violates":1,"n_no_contract":8,"n_pointer_only":1,"phrase":"12 ran (of which 1 constructed an object rather than computing a result; 9 with no instrument failure: 0 honoured, 1 violated, 8 with no contract checked; 3 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/a-syntax-guided-edit-decoder-for-neural#ran","syntology_url":"https://syntology.ai/paper/2106.08253","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2106.08253"}},"official":null}},{"url":"/paper/a-simple-approach-for-handling-out-of","slug":"a-simple-approach-for-handling-out-of","title":"A Simple Approach for Handling Out-of-Vocabulary Identifiers in Deep Learning for Source Code","date":"2020-10-23","arxiv_id":"2010.12663","repositories_listed":1,"syntology":null},{"url":"/paper/learning-to-execute-programs-with-instruction","slug":"learning-to-execute-programs-with-instruction","title":"Learning to Execute Programs with Instruction Pointer Attention Graph Neural Networks","date":"2020-10-23","arxiv_id":"2010.12621","repositories_listed":1,"syntology":null},{"url":"/paper/neural-code-completion-with-anonymized","slug":"neural-code-completion-with-anonymized","title":"On the Embeddings of Variables in Recurrent Neural Networks for Source Code","date":"2020-10-23","arxiv_id":"2010.12693","repositories_listed":1,"syntology":null}],"syntology_records":37,"syntology_note":"a paper without a record is not a recorded non-run: it may lack an arXiv id or simply be absent from the graph layer"},"key_notes":{"n_ran_checked":"legacy name, kept unchanged so existing readers do not break: it counts the samples that ran with no instrument failure (honoured, violated, and ran with no contract checked); it does not mean a contract was checked, and the pages print it as 'K with no instrument failure', not 'K checked'","n_constructed":"a sub-count of the samples that ran, never subtracted from them and never a failure: an executed sample whose run returned an instance of its own class (fixture_out_type equals the entry name): the run built an object and did not compute a result (Syntology's RAN record, counts.constructed)"},"since_archive_judged":{"papers":0,"papers_placed":0,"judged_by":{"model":"a model","models_as_recorded":[],"from":"lb_rows_run_verified.jsonl: each record's `model` field (the extractor's call that wrote decisions[])"},"note":"a model's judgement from the paper's own tables that they report on this leaderboard, not a result: no value from a paper that was not placed is printed or carried here; papers counts each paper once over these leaderboards","backfill":"8,886 of 18,637 newer papers checked so far (733 more had nothing to place; 41 could not be read)","backfill_complete":false,"leaderboards":[]},"papers_where_code_ran":42,"papers_where_code_ran_first_page":"/task/code-completion/papers/ran/1","papers_where_code_ran_caption":"We ran code from the paper's repository; we did not run it on this task or check it against the task's benchmarks.","pages":[{"order":"archive","page":2,"of":3,"url":"/task/code-completion/papers/2","json":"/task/code-completion/papers/2.json","rows":100},{"order":"archive","page":3,"of":3,"url":"/task/code-completion/papers/3","json":"/task/code-completion/papers/3.json","rows":12},{"order":"ran","page":1,"of":1,"url":"/task/code-completion/papers/ran/1","json":"/task/code-completion/papers/ran/1.json","rows":42}],"description_links":{"kept":0,"unwrapped_to_text":0,"bare_urls_linked":0,"relative_images_dropped":0,"rule":"internal links are kept only when the target slug exists in the catalog"},"syntology":{"read_at":"2026-10-05T00:36:11+00:00","claim":"Per-sample execution status on synthesized fixtures ('ran N of M samples'); not a correctness claim and not a ranking signal.","status_vocabulary":{"ran_honours":"ran, honoured the contract we drafted","ran_violates":"ran, violated the contract we drafted","ran_draft_wrong":"ran; our contract draft was wrong, not the code","ran_fixture":"ran; the contract check could not run (RAISES, DEP_MISSING or TIMEOUT)","ran":"ran on a synthesized input","unverified":"unverified (harvested, no recorded run)"},"row_route":"each syntology object on a benchmark's first row and on a paper carries sample_list (this site's paper page, #ran), syntology_url (syntology.ai's page for the paper, only when that route probed live) and mcp (the agent call that lists the paper's harvested samples; not a verdict on the paper)","developers":"https://syntology.ai/developers"},"not_shown":{"libraries":"the archive has no per-task library table","trend_sparklines":"the Trend column of the benchmarks table was a rendered image; it is not in the archive","social_and_latest_sorts":"stars and social signals are not in the archive"},"record_sha256":"5fdb0c024a5a887ee58fad42357c9c752fac3a5383ea5d2080c904945ddeb880","record_changed_at":"2026-10-05","record_changed_at_basis":"changed"}