{"about":{"non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","site":"https://codewithpapers.app","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page","syntology":{"site":"https://syntology.ai","developers":"https://syntology.ai/developers","mcp":{"server":"https://syntology.ai/mcp","transport":"streamable-http","server_card":"https://syntology.ai/.well-known/mcp/server-card.json","auth":{"type":"trial token, no account","trial_token":"https://syntology.ai/api/oauth/trial/token","method":"POST","docs":"https://syntology.ai/developers"}},"have":"https://syntology.ai/api/graph/have?x=<method, arXiv id or title> (free, answers coverage only)","paper_base":"https://syntology.ai/paper/","atlas_base":"https://app.syntology.ai/?focus="},"machine_readable":[{"url":"https://codewithpapers.app/llms.txt","what":"the machine catalog: every machine-readable file, counted"},{"url":"https://codewithpapers.app/index/manifest.json","what":"paper-to-code index by arXiv id, with Syntology's counts"},{"url":"https://codewithpapers.app/search/manifest.json","what":"site search index (titles, authors) and its files"},{"url":"https://codewithpapers.app/download","what":"bulk files: Syntology's layer, described there"},{"url":"https://codewithpapers.app/build_manifest.json","what":"the build record: inputs, counts, exclusions, probes"}]},"url":"/task/code-summarization-1","name":"Code Summarization","slug":"code-summarization-1","description_markdown":null,"categories":[],"source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","slug_source":"archive_url"},"counts":{"papers_tagged":163,"papers_with_code":82,"benchmarks":0,"benchmark_tables_in_archive":0,"benchmark_tables_shown":0,"benchmark_tables_withheld_as_spam":0,"benchmark_definition":"a leaderboard table with at least one row; benchmark_tables_shown also counts the zero-row tables; benchmark_tables_in_archive adds the tables withheld as spam","datasets":8,"subtasks":0,"parent_tasks":0},"benchmarks":[],"datasets":[{"url":"/dataset/codexglue","name":"CodeXGLUE","full_name":"","num_papers_in_archive":205},{"url":"/dataset/mconala","name":"MCoNaLa","full_name":"Multilingual CoNaLa","num_papers_in_archive":10},{"url":"/dataset/pytorrent","name":"PyTorrent","full_name":"PyTorrent","num_papers_in_archive":5},{"url":"/dataset/deepcom-java","name":"DeepCom-Java","full_name":"","num_papers_in_archive":2},{"url":"/dataset/cosqa-1","name":"CoSQA+","full_name":"CoSQA_Plus","num_papers_in_archive":1},{"url":"/dataset/hybrid-deepcom-java","name":"Java scripts","full_name":"","num_papers_in_archive":1},{"url":"/dataset/notebookcdg","name":"notebookcdg","full_name":"","num_papers_in_archive":1},{"url":"/dataset/parallelcorpus-python","name":"ParallelCorpus-Python","full_name":"","num_papers_in_archive":1}],"subtasks":[],"parent_tasks":[],"papers":{"order":"repositories listed in the archive (most first), then date (newest first), then slug; the archive holds no stars","population":"every paper the archive tags with this task (distinct papers; spam titles not listed). 'of' counts the papers listed; until 2026-09-25 it counted only the papers with a code link, and that number is of_with_code","page":1,"pages_in_order":2,"rows_per_page":100,"shown":100,"of":163,"of_with_code":82,"tagged_in_all":163,"counts":{"archive_papers_tagged":163,"with_a_code_link":82,"where_syntology_ran_a_sample":15,"not_listed_spam_title":0,"listed":163,"listed_where_code_ran":15,"where_syntology_ran_a_sample_split":{"with_a_run_with_no_instrument_failure":14,"every_run_a_failure_of_syntologys_instrument":1,"listed_with_a_run_with_no_instrument_failure":14,"listed_every_run_a_failure_of_syntologys_instrument":1,"filter":{"states":["a run with no instrument failure","any run, instrument failures included"],"default":"a run with no instrument failure","note":"on the 'only where code ran' pages the default hides, in the browser, the rows where every run was a failure of Syntology's instrument; the second state shows them again. Rows are hidden, never re-ordered; these twins list every row"}},"definition":"distinct papers the archive tags; 'where Syntology ran a sample' counts papers with at least one harvested sample that ran, which is not a correctness claim"},"items":[{"url":"/paper/a-transformer-based-approach-for-source-code","slug":"a-transformer-based-approach-for-source-code","title":"A Transformer-based Approach for Source Code Summarization","date":"2020-05-01","arxiv_id":"2005.00653","repositories_listed":9,"syntology":null},{"url":"/paper/codexglue-a-machine-learning-benchmark","slug":"codexglue-a-machine-learning-benchmark","title":"CodeXGLUE: A Machine Learning Benchmark Dataset for Code Understanding and Generation","date":"2021-02-09","arxiv_id":"2102.04664","repositories_listed":7,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/codexglue-a-machine-learning-benchmark#ran","syntology_url":"https://syntology.ai/paper/2102.04664","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2102.04664"}},"official":{"repos":["microsoft/CodeXGLUE"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/recommendations-for-datasets-for-source-code","slug":"recommendations-for-datasets-for-source-code","title":"Recommendations for Datasets for Source Code Summarization","date":"2019-04-04","arxiv_id":"1904.02660","repositories_listed":7,"syntology":null},{"url":"/paper/code2seq-generating-sequences-from-structured","slug":"code2seq-generating-sequences-from-structured","title":"code2seq: Generating Sequences from Structured Representations of Code","date":"2018-08-04","arxiv_id":"1808.01400","repositories_listed":6,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":3,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/code2seq-generating-sequences-from-structured#ran","syntology_url":"https://syntology.ai/paper/1808.01400","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1808.01400"}},"official":{"repos":["tech-srl/code2seq"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/codet5-identifier-aware-unified-pre-trained","slug":"codet5-identifier-aware-unified-pre-trained","title":"CodeT5: Identifier-aware Unified Pre-trained Encoder-Decoder Models for Code Understanding and Generation","date":"2021-09-02","arxiv_id":"2109.00859","repositories_listed":5,"syntology":{"n":11,"n_ran":6,"n_constructed":0,"n_ran_checked":5,"n_instrument":1,"n_unverified":5,"n_honours":0,"n_violates":0,"n_no_contract":5,"n_pointer_only":0,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 0 honoured, 0 violated, 5 with no contract checked; 1 where Syntology's instrument failed) · 5 unverified","sample_list":"/paper/codet5-identifier-aware-unified-pre-trained#ran","syntology_url":"https://syntology.ai/paper/2109.00859","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2109.00859"}},"official":{"repos":["salesforce/codet5"],"state":"official: harvested, nothing ran","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":1,"ran_from_kinds":["listed"]}}},{"url":"/paper/codet5-open-code-large-language-models-for","slug":"codet5-open-code-large-language-models-for","title":"CodeT5+: Open Code Large Language Models for Code Understanding and Generation","date":"2023-05-13","arxiv_id":"2305.07922","repositories_listed":2,"syntology":{"n":4,"n_ran":3,"n_constructed":0,"n_ran_checked":2,"n_instrument":1,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 1 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/codet5-open-code-large-language-models-for#ran","syntology_url":"https://syntology.ai/paper/2305.07922","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2305.07922"}},"official":{"repos":["salesforce/codet5"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/graph-conditioned-sparse-attention-for","slug":"graph-conditioned-sparse-attention-for","title":"Graph Conditioned Sparse-Attention for Improved Source Code Understanding","date":"2021-12-01","arxiv_id":"2112.00663","repositories_listed":2,"syntology":null},{"url":"/paper/retrieval-augmented-code-generation-and","slug":"retrieval-augmented-code-generation-and","title":"Retrieval Augmented Code Generation and Summarization","date":"2021-08-26","arxiv_id":"2108.11601","repositories_listed":2,"syntology":{"n":3,"n_ran":2,"n_constructed":0,"n_ran_checked":1,"n_instrument":1,"n_unverified":1,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":3,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/retrieval-augmented-code-generation-and#ran","syntology_url":"https://syntology.ai/paper/2108.11601","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2108.11601"}},"official":{"repos":["rizwan09/redcoder"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/haconvgnn-hierarchical-attention-based","slug":"haconvgnn-hierarchical-attention-based","title":"HAConvGNN: Hierarchical Attention Based Convolutional Graph Neural Network for Code Documentation Generation in Jupyter Notebooks","date":"2021-03-31","arxiv_id":"2104.01002","repositories_listed":2,"syntology":null},{"url":"/paper/unified-pre-training-for-program","slug":"unified-pre-training-for-program","title":"Unified Pre-training for Program Understanding and Generation","date":"2021-03-10","arxiv_id":"2103.06333","repositories_listed":2,"syntology":null},{"url":"/paper/improved-code-summarization-via-a-graph","slug":"improved-code-summarization-via-a-graph","title":"Improved Code Summarization via a Graph Neural Network","date":"2020-04-06","arxiv_id":"2004.02843","repositories_listed":2,"syntology":null},{"url":"/paper/code-generation-as-a-dual-task-of-code","slug":"code-generation-as-a-dual-task-of-code","title":"Code Generation as a Dual Task of Code Summarization","date":"2019-10-14","arxiv_id":"1910.05923","repositories_listed":2,"syntology":{"n":7,"n_ran":7,"n_constructed":0,"n_ran_checked":5,"n_instrument":2,"n_unverified":0,"n_honours":2,"n_violates":0,"n_no_contract":3,"n_pointer_only":4,"phrase":"7 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 2 honoured, 0 violated, 3 with no contract checked; 2 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/code-generation-as-a-dual-task-of-code#ran","syntology_url":"https://syntology.ai/paper/1910.05923","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1910.05923"}},"official":{"repos":["Bolin0215/CSCGDual"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/improving-automatic-source-code-summarization","slug":"improving-automatic-source-code-summarization","title":"Improving Automatic Source Code Summarization via Deep Reinforcement Learning","date":"2018-11-17","arxiv_id":"1811.07234","repositories_listed":2,"syntology":{"n":3,"n_ran":2,"n_constructed":0,"n_ran_checked":2,"n_instrument":0,"n_unverified":1,"n_honours":1,"n_violates":1,"n_no_contract":0,"n_pointer_only":3,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 1 honoured, 1 violated, 0 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/improving-automatic-source-code-summarization#ran","syntology_url":"https://syntology.ai/paper/1811.07234","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1811.07234"}},"official":null}},{"url":"/paper/variational-prefix-tuning-for-diverse-and","slug":"variational-prefix-tuning-for-diverse-and","title":"Variational Prefix Tuning for Diverse and Accurate Code Summarization Using Pre-trained Language Models","date":"2025-05-14","arxiv_id":"2505.09062","repositories_listed":1,"syntology":null},{"url":"/paper/code-summarization-beyond-function-level","slug":"code-summarization-beyond-function-level","title":"Code Summarization Beyond Function Level","date":"2025-02-23","arxiv_id":"2502.16704","repositories_listed":1,"syntology":null},{"url":"/paper/how-to-select-pre-trained-code-models-for","slug":"how-to-select-pre-trained-code-models-for","title":"How to Select Pre-Trained Code Models for Reuse? A Learning Perspective","date":"2025-01-07","arxiv_id":"2501.03783","repositories_listed":1,"syntology":null},{"url":"/paper/transducer-tuning-efficient-model-adaptation","slug":"transducer-tuning-efficient-model-adaptation","title":"Transducer Tuning: Efficient Model Adaptation for Software Tasks Using Code Property Graphs","date":"2024-12-18","arxiv_id":"2412.13467","repositories_listed":1,"syntology":null},{"url":"/paper/model-editing-for-llms4code-how-far-are-we","slug":"model-editing-for-llms4code-how-far-are-we","title":"Model Editing for LLMs4Code: How Far are We?","date":"2024-11-11","arxiv_id":"2411.06638","repositories_listed":1,"syntology":null},{"url":"/paper/building-a-coding-assistant-via-the-retrieval","slug":"building-a-coding-assistant-via-the-retrieval","title":"Building A Coding Assistant via the Retrieval-Augmented Language Model","date":"2024-10-21","arxiv_id":"2410.16229","repositories_listed":1,"syntology":null},{"url":"/paper/xmainframe-a-large-language-model-for","slug":"xmainframe-a-large-language-model-for","title":"XMainframe: A Large Language Model for Mainframe Modernization","date":"2024-08-05","arxiv_id":"2408.04660","repositories_listed":1,"syntology":null},{"url":"/paper/source-code-summarization-in-the-era-of-large","slug":"source-code-summarization-in-the-era-of-large","title":"Source Code Summarization in the Era of Large Language Models","date":"2024-07-09","arxiv_id":"2407.07959","repositories_listed":1,"syntology":null},{"url":"/paper/inversecoder-unleashing-the-power-of","slug":"inversecoder-unleashing-the-power-of","title":"InverseCoder: Self-improving Instruction-Tuned Code LLMs with Inverse-Instruct","date":"2024-07-08","arxiv_id":"2407.05700","repositories_listed":1,"syntology":{"n":11,"n_ran":11,"n_constructed":0,"n_ran_checked":2,"n_instrument":9,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":11,"phrase":"11 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 9 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/inversecoder-unleashing-the-power-of#ran","syntology_url":"https://syntology.ai/paper/2407.05700","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2407.05700"}},"official":{"repos":["wyt2000/InverseCoder"],"state":"official (archive's flag): 11 ran","n_ran":11,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/autocoder-enhancing-code-large-language-model","slug":"autocoder-enhancing-code-large-language-model","title":"AutoCoder: Enhancing Code Large Language Model with \\textsc{AIEV-Instruct}","date":"2024-05-23","arxiv_id":"2405.14906","repositories_listed":1,"syntology":null},{"url":"/paper/analyzing-the-performance-of-large-language","slug":"analyzing-the-performance-of-large-language","title":"Analyzing the Performance of Large Language Models on Code Summarization","date":"2024-04-10","arxiv_id":"2404.08018","repositories_listed":1,"syntology":null},{"url":"/paper/csa-trans-code-structure-aware-transformer","slug":"csa-trans-code-structure-aware-transformer","title":"CSA-Trans: Code Structure Aware Transformer for AST","date":"2024-04-07","arxiv_id":"2404.05767","repositories_listed":1,"syntology":{"n":5,"n_ran":5,"n_constructed":0,"n_ran_checked":2,"n_instrument":3,"n_unverified":0,"n_honours":0,"n_violates":1,"n_no_contract":1,"n_pointer_only":5,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 1 violated, 1 with no contract checked; 3 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/csa-trans-code-structure-aware-transformer#ran","syntology_url":"https://syntology.ai/paper/2404.05767","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2404.05767"}},"official":{"repos":["saeyoon17/code-structure-aware-transformer"],"state":"official (archive's flag): 5 ran","n_ran":5,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/empirical-studies-of-parameter-efficient","slug":"empirical-studies-of-parameter-efficient","title":"Empirical Studies of Parameter Efficient Methods for Large Language Models of Code and Knowledge Transfer to R","date":"2024-03-16","arxiv_id":"2405.01553","repositories_listed":1,"syntology":null},{"url":"/paper/eyetrans-merging-human-and-machine-attention","slug":"eyetrans-merging-human-and-machine-attention","title":"EyeTrans: Merging Human and Machine Attention for Neural Code Summarization","date":"2024-02-21","arxiv_id":"2402.14096","repositories_listed":1,"syntology":null},{"url":"/paper/studying-vulnerable-code-entities-in-r","slug":"studying-vulnerable-code-entities-in-r","title":"Studying Vulnerable Code Entities in R","date":"2024-02-06","arxiv_id":"2402.04421","repositories_listed":1,"syntology":null},{"url":"/paper/a-prompt-learning-framework-for-source-code","slug":"a-prompt-learning-framework-for-source-code","title":"A Prompt Learning Framework for Source Code Summarization","date":"2023-12-26","arxiv_id":"2312.16066","repositories_listed":1,"syntology":null},{"url":"/paper/binary-code-summarization-benchmarking","slug":"binary-code-summarization-benchmarking","title":"Binary Code Summarization: Benchmarking ChatGPT/GPT-4 and Other Large Language Models","date":"2023-12-15","arxiv_id":"2312.09601","repositories_listed":1,"syntology":null},{"url":"/paper/breaking-the-silence-the-threats-of-using","slug":"breaking-the-silence-the-threats-of-using","title":"Breaking the Silence: the Threats of Using LLMs in Software Engineering","date":"2023-12-13","arxiv_id":"2312.08055","repositories_listed":1,"syntology":null},{"url":"/paper/understanding-code-semantics-an-evaluation-of","slug":"understanding-code-semantics-an-evaluation-of","title":"Understanding Code Semantics: An Evaluation of Transformer Models in Summarization","date":"2023-10-25","arxiv_id":"2310.16314","repositories_listed":1,"syntology":null},{"url":"/paper/cp-bcs-binary-code-summarization-guided-by","slug":"cp-bcs-binary-code-summarization-guided-by","title":"CP-BCS: Binary Code Summarization Guided by Control Flow Graph and Pseudo Code","date":"2023-10-24","arxiv_id":"2310.16853","repositories_listed":1,"syntology":null},{"url":"/paper/beyond-accuracy-evaluating-self-consistency","slug":"beyond-accuracy-evaluating-self-consistency","title":"Beyond Accuracy: Evaluating Self-Consistency of Code Large Language Models with IdentityChain","date":"2023-10-21","arxiv_id":"2310.14053","repositories_listed":1,"syntology":{"n":14,"n_ran":10,"n_constructed":2,"n_ran_checked":4,"n_instrument":6,"n_unverified":4,"n_honours":0,"n_violates":1,"n_no_contract":3,"n_pointer_only":0,"phrase":"10 ran (of which 2 constructed an object rather than computing a result; 4 with no instrument failure: 0 honoured, 1 violated, 3 with no contract checked; 6 where Syntology's instrument failed) · 4 unverified","sample_list":"/paper/beyond-accuracy-evaluating-self-consistency#ran","syntology_url":"https://syntology.ai/paper/2310.14053","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2310.14053"}},"official":{"repos":["marcusm117/IdentityChain"],"state":"official (archive's flag): 10 ran","n_ran":10,"n_constructed":2,"n_ran_no_instrument_failure":4,"n_unverified":4,"ran_from_kinds":["official"]}}},{"url":"/paper/revisiting-file-context-for-source-code","slug":"revisiting-file-context-for-source-code","title":"Revisiting File Context for Source Code Summarization","date":"2023-09-05","arxiv_id":"2309.02326","repositories_listed":1,"syntology":null},{"url":"/paper/distilled-gpt-for-source-code-summarization","slug":"distilled-gpt-for-source-code-summarization","title":"Distilled GPT for Source Code Summarization","date":"2023-08-28","arxiv_id":"2308.14731","repositories_listed":1,"syntology":null},{"url":"/paper/sotana-the-open-source-software-development","slug":"sotana-the-open-source-software-development","title":"SoTaNa: The Open-Source Software Development Assistant","date":"2023-08-25","arxiv_id":"2308.13416","repositories_listed":1,"syntology":null},{"url":"/paper/semantic-similarity-loss-for-neural-source","slug":"semantic-similarity-loss-for-neural-source","title":"Semantic Similarity Loss for Neural Source Code Summarization","date":"2023-08-14","arxiv_id":"2308.07429","repositories_listed":1,"syntology":null},{"url":"/paper/statement-based-memory-for-neural-source-code","slug":"statement-based-memory-for-neural-source-code","title":"Statement-based Memory for Neural Source Code Summarization","date":"2023-07-21","arxiv_id":"2307.11709","repositories_listed":1,"syntology":null},{"url":"/paper/transcoder-towards-unified-transferable-code","slug":"transcoder-towards-unified-transferable-code","title":"TransCoder: Towards Unified Transferable Code Representation Learning Inspired by Human Skills","date":"2023-05-23","arxiv_id":"2306.07285","repositories_listed":1,"syntology":{"n":10,"n_ran":5,"n_constructed":0,"n_ran_checked":4,"n_instrument":1,"n_unverified":5,"n_honours":0,"n_violates":0,"n_no_contract":4,"n_pointer_only":0,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 4 with no instrument failure: 0 honoured, 0 violated, 4 with no contract checked; 1 where Syntology's instrument failed) · 5 unverified","sample_list":"/paper/transcoder-towards-unified-transferable-code#ran","syntology_url":"https://syntology.ai/paper/2306.07285","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2306.07285"}},"official":{"repos":["qiushisun/transcoder"],"state":"official (archive's flag): 5 ran","n_ran":5,"n_constructed":0,"n_ran_no_instrument_failure":4,"n_unverified":5,"ran_from_kinds":["official"]}}},{"url":"/paper/tram-a-token-level-retrieval-augmented","slug":"tram-a-token-level-retrieval-augmented","title":"Tram: A Token-level Retrieval-augmented Mechanism for Source Code Summarization","date":"2023-05-18","arxiv_id":"2305.11074","repositories_listed":1,"syntology":null},{"url":"/paper/the-vault-a-comprehensive-multilingual","slug":"the-vault-a-comprehensive-multilingual","title":"The Vault: A Comprehensive Multilingual Dataset for Advancing Code Understanding and Generation","date":"2023-05-09","arxiv_id":"2305.06156","repositories_listed":1,"syntology":null},{"url":"/paper/better-language-models-of-code-through-self","slug":"better-language-models-of-code-through-self","title":"Better Language Models of Code through Self-Improvement","date":"2023-04-02","arxiv_id":"2304.01228","repositories_listed":1,"syntology":null},{"url":"/paper/one-adapter-for-all-programming-languages","slug":"one-adapter-for-all-programming-languages","title":"One Adapter for All Programming Languages? Adapter Tuning for Code Search and Summarization","date":"2023-03-28","arxiv_id":"2303.15822","repositories_listed":1,"syntology":null},{"url":"/paper/automating-code-related-tasks-through","slug":"automating-code-related-tasks-through","title":"Automating Code-Related Tasks Through Transformers: The Impact of Pre-training","date":"2023-02-08","arxiv_id":"2302.04048","repositories_listed":1,"syntology":null},{"url":"/paper/exploring-data-augmentation-for-code","slug":"exploring-data-augmentation-for-code","title":"Exploring Data Augmentation for Code Generation Tasks","date":"2023-02-05","arxiv_id":"2302.03499","repositories_listed":1,"syntology":null},{"url":"/paper/ernie-code-beyond-english-centric-cross","slug":"ernie-code-beyond-english-centric-cross","title":"ERNIE-Code: Beyond English-Centric Cross-lingual Pretraining for Programming Languages","date":"2022-12-13","arxiv_id":"2212.06742","repositories_listed":1,"syntology":null},{"url":"/paper/clawsat-towards-both-robust-and-accurate-code","slug":"clawsat-towards-both-robust-and-accurate-code","title":"CLAWSAT: Towards Both Robust and Accurate Code Models","date":"2022-11-21","arxiv_id":"2211.11711","repositories_listed":1,"syntology":null},{"url":"/paper/boosting-code-summarization-by-embedding-code","slug":"boosting-code-summarization-by-embedding-code","title":"Boosting Code Summarization by Embedding Code Structures","date":"2022-10-01","arxiv_id":null,"repositories_listed":1,"syntology":null},{"url":"/paper/extreme-multi-domain-multi-task-learning-with","slug":"extreme-multi-domain-multi-task-learning-with","title":"Extreme Multi-Domain, Multi-Task Learning With Unified Text-to-Text Transfer Transformers","date":"2022-09-21","arxiv_id":"2209.10106","repositories_listed":1,"syntology":null},{"url":"/paper/no-more-fine-tuning-an-experimental","slug":"no-more-fine-tuning-an-experimental","title":"No More Fine-Tuning? An Experimental Evaluation of Prompt Tuning in Code Intelligence","date":"2022-07-24","arxiv_id":"2207.11680","repositories_listed":1,"syntology":{"n":14,"n_ran":7,"n_constructed":0,"n_ran_checked":3,"n_instrument":4,"n_unverified":7,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":2,"phrase":"7 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 4 where Syntology's instrument failed) · 7 unverified","sample_list":"/paper/no-more-fine-tuning-an-experimental#ran","syntology_url":"https://syntology.ai/paper/2207.11680","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2207.11680"}},"official":{"repos":["adf1178/pt4code"],"state":"official (archive's flag): 7 ran","n_ran":7,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":7,"ran_from_kinds":["official"]}}},{"url":"/paper/an-extractive-and-abstractive-framework-for","slug":"an-extractive-and-abstractive-framework-for","title":"An Extractive-and-Abstractive Framework for Source Code Summarization","date":"2022-06-15","arxiv_id":"2206.07245","repositories_listed":1,"syntology":null},{"url":"/paper/metatptrans-a-meta-learning-approach-for","slug":"metatptrans-a-meta-learning-approach-for","title":"MetaTPTrans: A Meta Learning Approach for Multilingual Code Representation Learning","date":"2022-06-13","arxiv_id":"2206.06460","repositories_listed":1,"syntology":null},{"url":"/paper/summarize-and-generate-to-back-translate-1","slug":"summarize-and-generate-to-back-translate-1","title":"Summarize and Generate to Back-translate: Unsupervised Translation of Programming Languages","date":"2022-05-23","arxiv_id":"2205.11116","repositories_listed":1,"syntology":null},{"url":"/paper/a-neural-network-architecture-for-program-1","slug":"a-neural-network-architecture-for-program-1","title":"A Neural Network Architecture for Program Understanding Inspired by Human Behaviors","date":"2022-05-10","arxiv_id":"2206.04730","repositories_listed":1,"syntology":{"n":9,"n_ran":5,"n_constructed":0,"n_ran_checked":5,"n_instrument":0,"n_unverified":4,"n_honours":0,"n_violates":0,"n_no_contract":5,"n_pointer_only":0,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 0 honoured, 0 violated, 5 with no contract checked; 0 where Syntology's instrument failed) · 4 unverified","sample_list":"/paper/a-neural-network-architecture-for-program-1#ran","syntology_url":"https://syntology.ai/paper/2206.04730","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2206.04730"}},"official":{"repos":["recklessronan/pgnn-ek"],"state":"official: no sample here; runs from other or unrecorded repositories","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["found_in_text"]}}},{"url":"/paper/gypsum-learning-hybrid-representations-for","slug":"gypsum-learning-hybrid-representations-for","title":"GypSum: Learning Hybrid Representations for Code Summarization","date":"2022-04-26","arxiv_id":"2204.12916","repositories_listed":1,"syntology":null},{"url":"/paper/m2ts-multi-scale-multi-modal-approach-based","slug":"m2ts-multi-scale-multi-modal-approach-based","title":"M2TS: Multi-Scale Multi-Modal Approach Based on Transformer for Source Code Summarization","date":"2022-03-18","arxiv_id":"2203.09707","repositories_listed":1,"syntology":null},{"url":"/paper/mconala-a-benchmark-for-code-generation-from","slug":"mconala-a-benchmark-for-code-generation-from","title":"MCoNaLa: A Benchmark for Code Generation from Multiple Natural Languages","date":"2022-03-16","arxiv_id":"2203.08388","repositories_listed":1,"syntology":null},{"url":"/paper/source-code-summarization-with-structural","slug":"source-code-summarization-with-structural","title":"Source Code Summarization with Structural Relative Position Guided Transformer","date":"2022-02-14","arxiv_id":"2202.06521","repositories_listed":1,"syntology":null},{"url":"/paper/what-do-they-capture-a-structural-analysis-of","slug":"what-do-they-capture-a-structural-analysis-of","title":"What Do They Capture? -- A Structural Analysis of Pre-Trained Language Models for Source Code","date":"2022-02-14","arxiv_id":"2202.06840","repositories_listed":1,"syntology":null},{"url":"/paper/compositionality-aware-graph2seq-learning","slug":"compositionality-aware-graph2seq-learning","title":"Compositionality-Aware Graph2Seq Learning","date":"2022-01-28","arxiv_id":"2201.12178","repositories_listed":1,"syntology":null},{"url":"/paper/assemble-foundation-models-for-automatic-code","slug":"assemble-foundation-models-for-automatic-code","title":"Assemble Foundation Models for Automatic Code Summarization","date":"2022-01-13","arxiv_id":"2201.05222","repositories_listed":1,"syntology":null},{"url":"/paper/integrating-tree-path-in-transformer-for-code","slug":"integrating-tree-path-in-transformer-for-code","title":"Integrating Tree Path in Transformer for Code Representation","date":"2021-12-01","arxiv_id":null,"repositories_listed":1,"syntology":null},{"url":"/paper/leveraging-unsupervised-learning-to-summarize","slug":"leveraging-unsupervised-learning-to-summarize","title":"Leveraging Unsupervised Learning to Summarize APIs Discussed in Stack Overflow","date":"2021-11-27","arxiv_id":"2111.13962","repositories_listed":1,"syntology":null},{"url":"/paper/gn-transformer-fusing-sequence-and-graph","slug":"gn-transformer-fusing-sequence-and-graph","title":"GN-Transformer: Fusing Sequence and Graph Representation for Improved Code Summarization","date":"2021-11-17","arxiv_id":"2111.08874","repositories_listed":1,"syntology":null},{"url":"/paper/modeling-hierarchical-syntax-structure-with","slug":"modeling-hierarchical-syntax-structure-with","title":"Modeling Hierarchical Syntax Structure with Triplet Position for Source Code Summarization","date":"2021-11-16","arxiv_id":null,"repositories_listed":1,"syntology":null},{"url":"/paper/graphsearchnet-enhancing-gnns-via-capturing","slug":"graphsearchnet-enhancing-gnns-via-capturing","title":"GraphSearchNet: Enhancing GNNs via Capturing Global Dependencies for Semantic Code Search","date":"2021-11-04","arxiv_id":"2111.02671","repositories_listed":1,"syntology":null},{"url":"/paper/evaluation-methodologies-for-code-learning","slug":"evaluation-methodologies-for-code-learning","title":"Impact of Evaluation Methodologies on Code Summarization","date":"2021-08-22","arxiv_id":"2108.09619","repositories_listed":1,"syntology":null},{"url":"/paper/autoencoders-as-tools-for-program-synthesis","slug":"autoencoders-as-tools-for-program-synthesis","title":"Autoencoders as Tools for Program Synthesis","date":"2021-08-16","arxiv_id":"2108.07129","repositories_listed":1,"syntology":null},{"url":"/paper/neural-code-summarization-how-far-are-we","slug":"neural-code-summarization-how-far-are-we","title":"On the Evaluation of Neural Code Summarization","date":"2021-07-15","arxiv_id":"2107.07112","repositories_listed":1,"syntology":null},{"url":"/paper/code-to-comment-translation-a-comparative","slug":"code-to-comment-translation-a-comparative","title":"Code to Comment Translation: A Comparative Study on Model Effectiveness & Errors","date":"2021-06-15","arxiv_id":"2106.08415","repositories_listed":1,"syntology":null},{"url":"/paper/codesc-a-large-code-description-parallel","slug":"codesc-a-large-code-description-parallel","title":"CoDesc: A Large Code-Description Parallel Dataset","date":"2021-05-29","arxiv_id":"2105.14220","repositories_listed":1,"syntology":{"n":8,"n_ran":7,"n_constructed":0,"n_ran_checked":7,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":1,"n_no_contract":6,"n_pointer_only":0,"phrase":"7 ran (of which 0 constructed an object rather than computing a result; 7 with no instrument failure: 0 honoured, 1 violated, 6 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/codesc-a-large-code-description-parallel#ran","syntology_url":"https://syntology.ai/paper/2105.14220","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2105.14220"}},"official":{"repos":["csebuetnlp/CoDesc"],"state":"official (archive's flag): 7 ran","n_ran":7,"n_constructed":0,"n_ran_no_instrument_failure":7,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/treebert-a-tree-based-pre-trained-model-for","slug":"treebert-a-tree-based-pre-trained-model-for","title":"TreeBERT: A Tree-Based Pre-Trained Model for Programming Language","date":"2021-05-26","arxiv_id":"2105.12485","repositories_listed":1,"syntology":null},{"url":"/paper/cotext-multi-task-learning-with-code-text","slug":"cotext-multi-task-learning-with-code-text","title":"CoTexT: Multi-task Learning with Code-Text Transformer","date":"2021-05-18","arxiv_id":"2105.08645","repositories_listed":1,"syntology":null},{"url":"/paper/project-level-encoding-for-neural-source-code","slug":"project-level-encoding-for-neural-source-code","title":"Project-Level Encoding for Neural Source Code Summarization of Subroutines","date":"2021-03-22","arxiv_id":"2103.11599","repositories_listed":1,"syntology":null},{"url":"/paper/language-agnostic-representation-learning-of-1","slug":"language-agnostic-representation-learning-of-1","title":"Language-Agnostic Representation Learning of Source Code from Structure and Context","date":"2021-03-21","arxiv_id":"2103.11318","repositories_listed":1,"syntology":{"n":8,"n_ran":6,"n_constructed":6,"n_ran_checked":6,"n_instrument":0,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":6,"n_pointer_only":0,"phrase":"6 ran (of which 6 constructed an object rather than computing a result; 6 with no instrument failure: 0 honoured, 0 violated, 6 with no contract checked; 0 where Syntology's instrument failed) · 2 unverified; every one of the 6 samples that ran constructed an object rather than computing a result","sample_list":"/paper/language-agnostic-representation-learning-of-1#ran","syntology_url":"https://syntology.ai/paper/2103.11318","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2103.11318"}},"official":{"repos":["danielzuegner/code-transformer"],"state":"official (archive's flag): 6 ran","n_ran":6,"n_constructed":6,"n_ran_no_instrument_failure":6,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/improving-code-summarization-with-block-wise","slug":"improving-code-summarization-with-block-wise","title":"Improving Code Summarization with Block-wise Abstract Syntax Tree Splitting","date":"2021-03-14","arxiv_id":"2103.07845","repositories_listed":1,"syntology":null},{"url":"/paper/neural-code-summarization","slug":"neural-code-summarization","title":"Neural Code Summarization","date":"2021-02-26","arxiv_id":"2103.01025","repositories_listed":1,"syntology":null},{"url":"/paper/sit3-code-summarization-with-structure","slug":"sit3-code-summarization-with-structure","title":"Code Summarization with Structure-induced Transformer","date":"2020-12-29","arxiv_id":"2012.14710","repositories_listed":1,"syntology":null},{"url":"/paper/graphcodebert-pre-training-code","slug":"graphcodebert-pre-training-code","title":"GraphCodeBERT: Pre-training Code Representations with Data Flow","date":"2020-09-17","arxiv_id":"2009.08366","repositories_listed":1,"syntology":null},{"url":"/paper/automatic-code-summarization-via-multi","slug":"automatic-code-summarization-via-multi","title":"Retrieval-Augmented Generation for Code Summarization via Hybrid GNN","date":"2020-06-09","arxiv_id":"2006.05405","repositories_listed":1,"syntology":null},{"url":"/paper/automatic-source-code-summarization-with","slug":"automatic-source-code-summarization-with","title":"Automatic Source Code Summarization with Extended Tree-LSTM","date":"2019-06-19","arxiv_id":"1906.08094","repositories_listed":1,"syntology":null},{"url":null,"slug":"rethinking-the-effects-of-data-contamination","title":"Rethinking the effects of data contamination in Code Intelligence","date":"2025-06-03","arxiv_id":"2506.02791","repositories_listed":0,"syntology":null},{"url":null,"slug":"an-llm-as-judge-metric-for-bridging-the-gap","title":"An LLM-as-Judge Metric for Bridging the Gap with Human Evaluation in SE Tasks","date":"2025-05-27","arxiv_id":"2505.20854","repositories_listed":0,"syntology":null},{"url":null,"slug":"leancode-understanding-models-better-for-code","title":"LEANCODE: Understanding Models Better for Code Simplification of Pre-trained Large Language Models","date":"2025-05-20","arxiv_id":"2505.14759","repositories_listed":0,"syntology":null},{"url":null,"slug":"evaloop-assessing-llm-robustness-in","title":"EVALOOP: Assessing LLM Robustness in Programming from a Self-consistency Perspective","date":"2025-05-18","arxiv_id":"2505.12185","repositories_listed":0,"syntology":null},{"url":null,"slug":"large-language-models-are-qualified-benchmark","title":"Large Language Models are Qualified Benchmark Builders: Rebuilding Pre-Training Datasets for Advancing Code Intelligence Tasks","date":"2025-04-28","arxiv_id":"2504.19444","repositories_listed":0,"syntology":null},{"url":null,"slug":"code-craft-hierarchical-graph-based-code","title":"Code-Craft: Hierarchical Graph-Based Code Summarization for Enhanced Context Retrieval","date":"2025-04-11","arxiv_id":"2504.08975","repositories_listed":0,"syntology":null},{"url":null,"slug":"enhancing-code-llm-training-with-programmer","title":"Enhancing Code LLM Training with Programmer Attention","date":"2025-03-19","arxiv_id":"2503.14936","repositories_listed":0,"syntology":null},{"url":null,"slug":"commenting-higher-level-code-unit-full-code","title":"Commenting Higher-level Code Unit: Full Code, Reduced Code, or Hierarchical Code Summarization","date":"2025-03-13","arxiv_id":"2503.10737","repositories_listed":0,"syntology":null},{"url":null,"slug":"energy-conscious-llm-decoding-impact-of-text","title":"Energy-Conscious LLM Decoding: Impact of Text Generation Strategies on GPU Energy Consumption","date":"2025-02-17","arxiv_id":"2502.11723","repositories_listed":0,"syntology":null},{"url":null,"slug":"can-llms-replace-human-evaluators-an","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","date":"2025-02-10","arxiv_id":"2502.06193","repositories_listed":0,"syntology":null},{"url":null,"slug":"can-large-language-models-understand-1","title":"Can Large Language Models Understand Intermediate Representations?","date":"2025-02-07","arxiv_id":"2502.06854","repositories_listed":0,"syntology":null},{"url":null,"slug":"should-code-models-learn-pedagogically-a","title":"Should Code Models Learn Pedagogically? A Preliminary Evaluation of Curriculum Learning for Real-World Software Engineering Tasks","date":"2025-02-06","arxiv_id":"2502.03806","repositories_listed":0,"syntology":null},{"url":null,"slug":"analysis-of-student-llm-interaction-in-a","title":"Analysis of Student-LLM Interaction in a Software Engineering Project","date":"2025-02-03","arxiv_id":"2502.01273","repositories_listed":0,"syntology":null},{"url":null,"slug":"hierarchical-repository-level-code","title":"Hierarchical Repository-Level Code Summarization for Business Applications Using Local LLMs","date":"2025-01-14","arxiv_id":"2501.07857","repositories_listed":0,"syntology":null},{"url":null,"slug":"analysis-on-llms-performance-for-code","title":"Analysis on LLMs Performance for Code Summarization","date":"2024-12-22","arxiv_id":"2412.17094","repositories_listed":0,"syntology":null},{"url":null,"slug":"on-the-compression-of-language-models-for","title":"On the Compression of Language Models for Code: An Empirical Study on CodeBERT","date":"2024-12-18","arxiv_id":"2412.13737","repositories_listed":0,"syntology":null},{"url":null,"slug":"geocode-gpt-a-large-language-model-for","title":"GeoCode-GPT: A Large Language Model for Geospatial Code Generation Tasks","date":"2024-10-22","arxiv_id":"2410.17031","repositories_listed":0,"syntology":null},{"url":null,"slug":"etf-an-entity-tracing-framework-for","title":"ETF: An Entity Tracing Framework for Hallucination Detection in Code Summaries","date":"2024-10-17","arxiv_id":"2410.14748","repositories_listed":0,"syntology":null}],"syntology_records":15,"syntology_note":"a paper without a record is not a recorded non-run: it may lack an arXiv id or simply be absent from the graph layer"},"key_notes":{"n_ran_checked":"legacy name, kept unchanged so existing readers do not break: it counts the samples that ran with no instrument failure (honoured, violated, and ran with no contract checked); it does not mean a contract was checked, and the pages print it as 'K with no instrument failure', not 'K checked'","n_constructed":"a sub-count of the samples that ran, never subtracted from them and never a failure: an executed sample whose run returned an instance of its own class (fixture_out_type equals the entry name): the run built an object and did not compute a result (Syntology's RAN record, counts.constructed)"},"since_archive_judged":{"papers":0,"papers_placed":0,"judged_by":{"model":"a model","models_as_recorded":[],"from":"lb_rows_run_verified.jsonl: each record's `model` field (the extractor's call that wrote decisions[])"},"note":"a model's judgement from the paper's own tables that they report on this leaderboard, not a result: no value from a paper that was not placed is printed or carried here; papers counts each paper once over these leaderboards","backfill":"8,886 of 18,637 newer papers checked so far (733 more had nothing to place; 41 could not be read)","backfill_complete":false,"leaderboards":[]},"papers_where_code_ran":15,"papers_where_code_ran_first_page":"/task/code-summarization-1/papers/ran/1","papers_where_code_ran_caption":"We ran code from the paper's repository; we did not run it on this task or check it against the task's benchmarks.","pages":[{"order":"archive","page":2,"of":2,"url":"/task/code-summarization-1/papers/2","json":"/task/code-summarization-1/papers/2.json","rows":63},{"order":"ran","page":1,"of":1,"url":"/task/code-summarization-1/papers/ran/1","json":"/task/code-summarization-1/papers/ran/1.json","rows":15}],"description_links":{"kept":0,"unwrapped_to_text":0,"bare_urls_linked":0,"relative_images_dropped":0,"rule":"internal links are kept only when the target slug exists in the catalog"},"syntology":{"read_at":"2026-10-05T00:36:11+00:00","claim":"Per-sample execution status on synthesized fixtures ('ran N of M samples'); not a correctness claim and not a ranking signal.","status_vocabulary":{"ran_honours":"ran, honoured the contract we drafted","ran_violates":"ran, violated the contract we drafted","ran_draft_wrong":"ran; our contract draft was wrong, not the code","ran_fixture":"ran; the contract check could not run (RAISES, DEP_MISSING or TIMEOUT)","ran":"ran on a synthesized input","unverified":"unverified (harvested, no recorded run)"},"row_route":"each syntology object on a benchmark's first row and on a paper carries sample_list (this site's paper page, #ran), syntology_url (syntology.ai's page for the paper, only when that route probed live) and mcp (the agent call that lists the paper's harvested samples; not a verdict on the paper)","developers":"https://syntology.ai/developers"},"not_shown":{"libraries":"the archive has no per-task library table","trend_sparklines":"the Trend column of the benchmarks table was a rendered image; it is not in the archive","social_and_latest_sorts":"stars and social signals are not in the archive"},"record_sha256":"a3e76bb0f22921c90e2de1f4262d542038c4f3dc877e9ae896ab6c5e8747b917","record_changed_at":"2026-10-05","record_changed_at_basis":"changed"}