{"about":{"non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","site":"https://codewithpapers.app","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page","syntology":{"site":"https://syntology.ai","developers":"https://syntology.ai/developers","mcp":{"server":"https://syntology.ai/mcp","transport":"streamable-http","server_card":"https://syntology.ai/.well-known/mcp/server-card.json","auth":{"type":"trial token, no account","trial_token":"https://syntology.ai/api/oauth/trial/token","method":"POST","docs":"https://syntology.ai/developers"}},"have":"https://syntology.ai/api/graph/have?x=<method, arXiv id or title> (free, answers coverage only)","paper_base":"https://syntology.ai/paper/","atlas_base":"https://app.syntology.ai/?focus="},"machine_readable":[{"url":"https://codewithpapers.app/llms.txt","what":"the machine catalog: every machine-readable file, counted"},{"url":"https://codewithpapers.app/index/manifest.json","what":"paper-to-code index by arXiv id, with Syntology's counts"},{"url":"https://codewithpapers.app/search/manifest.json","what":"site search index (titles, authors) and its files"},{"url":"https://codewithpapers.app/download","what":"bulk files: Syntology's layer, described there"},{"url":"https://codewithpapers.app/build_manifest.json","what":"the build record: inputs, counts, exclusions, probes"}]},"url":"/task/software-testing","name":"software testing","slug":"software-testing","description_markdown":null,"categories":[],"source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","slug_source":"archive_url"},"counts":{"papers_tagged":135,"papers_with_code":39,"benchmarks":0,"benchmark_tables_in_archive":0,"benchmark_tables_shown":0,"benchmark_tables_withheld_as_spam":0,"benchmark_definition":"a leaderboard table with at least one row; benchmark_tables_shown also counts the zero-row tables; benchmark_tables_in_archive adds the tables withheld as spam","datasets":1,"subtasks":0,"parent_tasks":0},"benchmarks":[],"datasets":[{"url":"/dataset/solidiffy-differencing-contract-pairs-and","name":"SoliDiffy Differencing Contract Pairs and Edit Scripts","full_name":"","num_papers_in_archive":1}],"subtasks":[],"parent_tasks":[],"papers":{"order":"repositories listed in the archive (most first), then date (newest first), then slug; the archive holds no stars","population":"every paper the archive tags with this task (distinct papers; spam titles not listed). 'of' counts the papers listed; until 2026-09-25 it counted only the papers with a code link, and that number is of_with_code","page":1,"pages_in_order":2,"rows_per_page":100,"shown":100,"of":135,"of_with_code":39,"tagged_in_all":135,"counts":{"archive_papers_tagged":135,"with_a_code_link":39,"where_syntology_ran_a_sample":5,"not_listed_spam_title":0,"listed":135,"listed_where_code_ran":5,"where_syntology_ran_a_sample_split":{"with_a_run_with_no_instrument_failure":5,"every_run_a_failure_of_syntologys_instrument":0,"listed_with_a_run_with_no_instrument_failure":5,"listed_every_run_a_failure_of_syntologys_instrument":0,"filter":{"states":["a run with no instrument failure","any run, instrument failures included"],"default":"a run with no instrument failure","note":"on the 'only where code ran' pages the default hides, in the browser, the rows where every run was a failure of Syntology's instrument; the second state shows them again. Rows are hidden, never re-ordered; these twins list every row"}},"definition":"distinct papers the archive tags; 'where Syntology ran a sample' counts papers with at least one harvested sample that ran, which is not a correctness claim"},"items":[{"url":"/paper/coverup-coverage-guided-llm-based-test","slug":"coverup-coverage-guided-llm-based-test","title":"CoverUp: Effective High Coverage Test Generation for Python","date":"2024-03-24","arxiv_id":"2403.16218","repositories_listed":3,"syntology":{"n":17,"n_ran":12,"n_constructed":0,"n_ran_checked":8,"n_instrument":4,"n_unverified":5,"n_honours":0,"n_violates":0,"n_no_contract":8,"n_pointer_only":0,"phrase":"12 ran (of which 0 constructed an object rather than computing a result; 8 with no instrument failure: 0 honoured, 0 violated, 8 with no contract checked; 4 where Syntology's instrument failed) · 5 unverified","sample_list":"/paper/coverup-coverage-guided-llm-based-test#ran","syntology_url":"https://syntology.ai/paper/2403.16218","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2403.16218"}},"official":{"repos":["plasma-umass/coverup","plasma-umass/coverup-eval"],"state":"official (archive's flag): 12 ran","n_ran":12,"n_constructed":0,"n_ran_no_instrument_failure":8,"n_unverified":5,"ran_from_kinds":["official"]}}},{"url":"/paper/tensorfuzz-debugging-neural-networks-with","slug":"tensorfuzz-debugging-neural-networks-with","title":"TensorFuzz: Debugging Neural Networks with Coverage-Guided Fuzzing","date":"2018-07-28","arxiv_id":"1807.10875","repositories_listed":3,"syntology":{"n":12,"n_ran":8,"n_constructed":0,"n_ran_checked":7,"n_instrument":1,"n_unverified":4,"n_honours":0,"n_violates":0,"n_no_contract":7,"n_pointer_only":0,"phrase":"8 ran (of which 0 constructed an object rather than computing a result; 7 with no instrument failure: 0 honoured, 0 violated, 7 with no contract checked; 1 where Syntology's instrument failed) · 4 unverified","sample_list":"/paper/tensorfuzz-debugging-neural-networks-with#ran","syntology_url":"https://syntology.ai/paper/1807.10875","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1807.10875"}},"official":null}},{"url":"/paper/navigating-the-growing-field-of-research-on","slug":"navigating-the-growing-field-of-research-on","title":"Navigating the growing field of research on AI for software testing -- the taxonomy for AI-augmented software testing and an ontology-driven literature survey","date":"2025-06-17","arxiv_id":"2506.14640","repositories_listed":2,"syntology":null},{"url":"/paper/fairness-aware-configuration-of-machine","slug":"fairness-aware-configuration-of-machine","title":"Fairness-aware Configuration of Machine Learning Libraries","date":"2022-02-13","arxiv_id":"2202.06196","repositories_listed":2,"syntology":null},{"url":"/paper/black-box-explanation-of-object-detectors-via","slug":"black-box-explanation-of-object-detectors-via","title":"Black-box Explanation of Object Detectors via Saliency Maps","date":"2020-06-05","arxiv_id":"2006.03204","repositories_listed":2,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":2,"n_instrument":1,"n_unverified":0,"n_honours":2,"n_violates":0,"n_no_contract":0,"n_pointer_only":3,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 2 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/black-box-explanation-of-object-detectors-via#ran","syntology_url":"https://syntology.ai/paper/2006.03204","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2006.03204"}},"official":null}},{"url":"/paper/test-it-before-you-trust-it-applying-software","slug":"test-it-before-you-trust-it-applying-software","title":"Test It Before You Trust It: Applying Software Testing for Trustworthy In-context Learning","date":"2025-04-26","arxiv_id":"2504.18827","repositories_listed":1,"syntology":null},{"url":"/paper/towards-trustworthy-gui-agents-a-survey","slug":"towards-trustworthy-gui-agents-a-survey","title":"Towards Trustworthy GUI Agents: A Survey","date":"2025-03-30","arxiv_id":"2503.23434","repositories_listed":1,"syntology":null},{"url":"/paper/assessing-data-augmentation-induced-bias-in","slug":"assessing-data-augmentation-induced-bias-in","title":"Assessing Data Augmentation-Induced Bias in Training and Testing of Machine Learning Models","date":"2025-02-03","arxiv_id":"2502.01825","repositories_listed":1,"syntology":null},{"url":"/paper/can-search-based-testing-with-pareto","slug":"can-search-based-testing-with-pareto","title":"Can Search-Based Testing with Pareto Optimization Effectively Cover Failure-Revealing Test Inputs?","date":"2024-10-15","arxiv_id":"2410.11769","repositories_listed":1,"syntology":null},{"url":"/paper/leveraging-large-language-models-for-15","slug":"leveraging-large-language-models-for-15","title":"Leveraging Large Language Models for Enhancing the Understandability of Generated Unit Tests","date":"2024-08-21","arxiv_id":"2408.11710","repositories_listed":1,"syntology":null},{"url":"/paper/code-agents-are-state-of-the-art-software","slug":"code-agents-are-state-of-the-art-software","title":"SWT-Bench: Testing and Validating Real-World Bug-Fixes with Code Agents","date":"2024-06-18","arxiv_id":"2406.12952","repositories_listed":1,"syntology":{"n":6,"n_ran":5,"n_constructed":0,"n_ran_checked":1,"n_instrument":4,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 4 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/code-agents-are-state-of-the-art-software#ran","syntology_url":"https://syntology.ai/paper/2406.12952","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.12952"}},"official":{"repos":["logic-star-ai/swt-bench"],"state":"official (archive's flag): 5 ran","n_ran":5,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/llm-powered-test-case-generation-for","slug":"llm-powered-test-case-generation-for","title":"LLM-Powered Test Case Generation for Detecting Bugs in Plausible Programs","date":"2024-04-16","arxiv_id":"2404.10304","repositories_listed":1,"syntology":null},{"url":"/paper/comprehensive-evaluation-and-insights-into-1","slug":"comprehensive-evaluation-and-insights-into-1","title":"Comprehensive Evaluation and Insights into the Use of Large Language Models in the Automation of Behavior-Driven Development Acceptance Test Formulation","date":"2024-03-22","arxiv_id":"2403.14965","repositories_listed":1,"syntology":null},{"url":"/paper/towards-principled-representation-learning-1","slug":"towards-principled-representation-learning-1","title":"Towards Principled Representation Learning from Videos for Reinforcement Learning","date":"2024-03-20","arxiv_id":"2403.13765","repositories_listed":1,"syntology":null},{"url":"/paper/quantest-entanglement-guided-testing-of","slug":"quantest-entanglement-guided-testing-of","title":"QuanTest: Entanglement-Guided Testing of Quantum Neural Network Systems","date":"2024-02-20","arxiv_id":"2402.12950","repositories_listed":1,"syntology":null},{"url":"/paper/large-language-models-based-fuzzing","slug":"large-language-models-based-fuzzing","title":"On the Challenges of Fuzzing Techniques via Large Language Models","date":"2024-02-01","arxiv_id":"2402.00350","repositories_listed":1,"syntology":null},{"url":"/paper/generative-ai-to-generate-test-data","slug":"generative-ai-to-generate-test-data","title":"Generative AI to Generate Test Data Generators","date":"2024-01-31","arxiv_id":"2401.17626","repositories_listed":1,"syntology":null},{"url":"/paper/interevo-tr-interactive-evolutionary-test","slug":"interevo-tr-interactive-evolutionary-test","title":"InterEvo-TR: Interactive Evolutionary Test Generation With Readability Assessment","date":"2024-01-13","arxiv_id":"2401.07072","repositories_listed":1,"syntology":null},{"url":"/paper/irg-generating-synthetic-relational-databases","slug":"irg-generating-synthetic-relational-databases","title":"IRG: Generating Synthetic Relational Databases using Deep Learning with Insightful Relational Understanding","date":"2023-12-23","arxiv_id":"2312.15187","repositories_listed":1,"syntology":null},{"url":"/paper/test-case-recommendations-with-distributed","slug":"test-case-recommendations-with-distributed","title":"Test Case Recommendations with Distributed Representation of Code Syntactic Features","date":"2023-10-04","arxiv_id":"2310.03174","repositories_listed":1,"syntology":null},{"url":"/paper/towards-efficient-fine-tuning-of-pre-trained","slug":"towards-efficient-fine-tuning-of-pre-trained","title":"Towards Efficient Fine-tuning of Pre-trained Code Models: An Experimental Study and Beyond","date":"2023-04-11","arxiv_id":"2304.05216","repositories_listed":1,"syntology":null},{"url":"/paper/testing-the-channels-of-convolutional-neural","slug":"testing-the-channels-of-convolutional-neural","title":"Testing the Channels of Convolutional Neural Networks","date":"2023-03-06","arxiv_id":"2303.03400","repositories_listed":1,"syntology":null},{"url":"/paper/reasoning-based-software-testing","slug":"reasoning-based-software-testing","title":"Reasoning-Based Software Testing","date":"2023-03-02","arxiv_id":"2303.01302","repositories_listed":1,"syntology":null},{"url":"/paper/2302-07646","slug":"2302-07646","title":"Genetic Micro-Programs for Automated Software Testing with Large Path Coverage","date":"2023-02-14","arxiv_id":"2302.07646","repositories_listed":1,"syntology":null},{"url":"/paper/perfect-is-the-enemy-of-test-oracle","slug":"perfect-is-the-enemy-of-test-oracle","title":"Perfect is the enemy of test oracle","date":"2023-02-03","arxiv_id":"2302.01488","repositories_listed":1,"syntology":null},{"url":"/paper/boosting-synthetic-data-generation-with","slug":"boosting-synthetic-data-generation-with","title":"Boosting Synthetic Data Generation with Effective Nonlinear Causal Discovery","date":"2023-01-18","arxiv_id":"2301.07427","repositories_listed":1,"syntology":null},{"url":"/paper/a-comparison-of-reinforcement-learning-1","slug":"a-comparison-of-reinforcement-learning-1","title":"A Comparison of Reinforcement Learning Frameworks for Software Testing Tasks","date":"2022-08-25","arxiv_id":"2208.12136","repositories_listed":1,"syntology":null},{"url":"/paper/differential-testing-for-machine-learning-an","slug":"differential-testing-for-machine-learning-an","title":"Differential testing for machine learning: an analysis for classification algorithms beyond deep learning","date":"2022-07-25","arxiv_id":"2207.11976","repositories_listed":1,"syntology":null},{"url":"/paper/an-efficiency-study-for-splade-models","slug":"an-efficiency-study-for-splade-models","title":"An Efficiency Study for SPLADE Models","date":"2022-07-08","arxiv_id":"2207.03834","repositories_listed":1,"syntology":null},{"url":"/paper/pipelines-for-social-bias-testing-of-large","slug":"pipelines-for-social-bias-testing-of-large","title":"Pipelines for Social Bias Testing of Large Language Models","date":"2022-05-01","arxiv_id":null,"repositories_listed":1,"syntology":null},{"url":"/paper/efficient-and-effective-generation-of-test","slug":"efficient-and-effective-generation-of-test","title":"Efficient and Effective Generation of Test Cases for Pedestrian Detection -- Search-based Software Testing of Baidu Apollo in SVL","date":"2021-09-16","arxiv_id":"2109.07960","repositories_listed":1,"syntology":null},{"url":"/paper/software-engineering-for-ai-based-systems-a","slug":"software-engineering-for-ai-based-systems-a","title":"Software Engineering for AI-Based Systems: A Survey","date":"2021-05-05","arxiv_id":"2105.01984","repositories_listed":1,"syntology":null},{"url":"/paper/smoke-testing-for-machine-learning-simple","slug":"smoke-testing-for-machine-learning-simple","title":"Smoke Testing for Machine Learning: Simple Tests to Discover Severe Defects","date":"2020-09-03","arxiv_id":"2009.01521","repositories_listed":1,"syntology":null},{"url":"/paper/business-negotiation-definition-language","slug":"business-negotiation-definition-language","title":"Business Negotiation Definition Language","date":"2020-01-04","arxiv_id":"2001.10799","repositories_listed":1,"syntology":null},{"url":"/paper/an-autonomous-performance-testing-framework","slug":"an-autonomous-performance-testing-framework","title":"An Autonomous Performance Testing Framework using Self-Adaptive Fuzzy Reinforcement Learning","date":"2019-08-19","arxiv_id":"1908.06900","repositories_listed":1,"syntology":null},{"url":"/paper/boosting-operational-dnn-testing-efficiency","slug":"boosting-operational-dnn-testing-efficiency","title":"Boosting Operational DNN Testing Efficiency through Conditioning","date":"2019-06-06","arxiv_id":"1906.02533","repositories_listed":1,"syntology":null},{"url":"/paper/recurrent-neural-networks-for-fuzz-testing","slug":"recurrent-neural-networks-for-fuzz-testing","title":"Recurrent Neural Networks for Fuzz Testing Web Browsers","date":"2018-12-12","arxiv_id":"1812.04852","repositories_listed":1,"syntology":null},{"url":"/paper/secbench-a-database-of-real-security","slug":"secbench-a-database-of-real-security","title":"SECBENCH: A Database of Real Security Vulnerabilities","date":"2017-10-31","arxiv_id":null,"repositories_listed":1,"syntology":null},{"url":"/paper/gpflow-a-gaussian-process-library-using","slug":"gpflow-a-gaussian-process-library-using","title":"GPflow: A Gaussian process library using TensorFlow","date":"2016-10-27","arxiv_id":"1610.08733","repositories_listed":1,"syntology":null},{"url":null,"slug":"guaranteed-guess-a-language-modeling-approach","title":"Guaranteed Guess: A Language Modeling Approach for CISC-to-RISC Transpilation with Testing Guarantees","date":"2025-06-17","arxiv_id":"2506.14606","repositories_listed":0,"syntology":null},{"url":"/paper/intentest-stress-testing-for-intent-integrity","slug":"intentest-stress-testing-for-intent-integrity","title":"IntenTest: Stress Testing for Intent Integrity in API-Calling LLM Agents","date":"2025-06-09","arxiv_id":"2506.07524","repositories_listed":0,"syntology":{"n":9,"n_ran":8,"n_constructed":0,"n_ran_checked":8,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":8,"n_pointer_only":0,"phrase":"8 ran (of which 0 constructed an object rather than computing a result; 8 with no instrument failure: 0 honoured, 0 violated, 8 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/intentest-stress-testing-for-intent-integrity#ran","syntology_url":"https://syntology.ai/paper/2506.07524","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2506.07524"}},"official":null}},{"url":null,"slug":"the-impact-of-software-testing-with-quantum","title":"The Impact of Software Testing with Quantum Optimization Meets Machine Learning","date":"2025-06-02","arxiv_id":"2506.02090","repositories_listed":0,"syntology":null},{"url":null,"slug":"evogpt-enhancing-test-suite-robustness-via","title":"EvoGPT: Enhancing Test Suite Robustness via LLM-Based Generation and Genetic Optimization","date":"2025-05-18","arxiv_id":"2505.12424","repositories_listed":0,"syntology":null},{"url":null,"slug":"on-the-need-for-a-statistical-foundation-in","title":"On the Need for a Statistical Foundation in Scenario-Based Testing of Autonomous Vehicles","date":"2025-05-04","arxiv_id":"2505.02274","repositories_listed":0,"syntology":null},{"url":null,"slug":"automated-unit-test-case-generation-a","title":"Automated Unit Test Case Generation: A Systematic Literature Review","date":"2025-04-29","arxiv_id":"2504.20357","repositories_listed":0,"syntology":null},{"url":null,"slug":"harden-and-catch-for-just-in-time-assured-llm","title":"Harden and Catch for Just-in-Time Assured LLM-Based Software Testing: Open Research Challenges","date":"2025-04-23","arxiv_id":"2504.16472","repositories_listed":0,"syntology":null},{"url":null,"slug":"expectations-vs-reality-a-secondary-study-on","title":"Expectations vs Reality -- A Secondary Study on AI Adoption in Software Testing","date":"2025-04-07","arxiv_id":"2504.04921","repositories_listed":0,"syntology":null},{"url":null,"slug":"from-code-generation-to-software-testing-ai","title":"From Code Generation to Software Testing: AI Copilot with Context-Based RAG","date":"2025-04-02","arxiv_id":"2504.01866","repositories_listed":0,"syntology":null},{"url":null,"slug":"integrating-artificial-intelligence-with","title":"Integrating Artificial Intelligence with Human Expertise: An In-depth Analysis of ChatGPT's Capabilities in Generating Metamorphic Relations","date":"2025-03-28","arxiv_id":"2503.22141","repositories_listed":0,"syntology":null},{"url":null,"slug":"vulnerability-detection-from-formal","title":"Vulnerability Detection: From Formal Verification to Large Language Models and Hybrid Approaches: A Comprehensive Overview","date":"2025-03-13","arxiv_id":"2503.10784","repositories_listed":0,"syntology":null},{"url":null,"slug":"rule-guided-reinforcement-learning-policy","title":"Rule-Guided Reinforcement Learning Policy Evaluation and Improvement","date":"2025-03-12","arxiv_id":"2503.09270","repositories_listed":0,"syntology":null},{"url":null,"slug":"toolfuzz-automated-agent-tool-testing","title":"ToolFuzz -- Automated Agent Tool Testing","date":"2025-03-06","arxiv_id":"2503.04479","repositories_listed":0,"syntology":null},{"url":null,"slug":"wip-assessing-the-effectiveness-of-chatgpt-in","title":"WIP: Assessing the Effectiveness of ChatGPT in Preparatory Testing Activities","date":"2025-03-05","arxiv_id":"2503.03951","repositories_listed":0,"syntology":null},{"url":null,"slug":"2503-00795","title":"Towards Reliable LLM-Driven Fuzz Testing: Vision and Road Ahead","date":"2025-03-02","arxiv_id":"2503.00795","repositories_listed":0,"syntology":null},{"url":null,"slug":"clover-a-test-case-generation-benchmark-with","title":"CLOVER: A Test Case Generation Benchmark with Coverage, Long-Context, and Verification","date":"2025-02-12","arxiv_id":"2502.08806","repositories_listed":0,"syntology":null},{"url":null,"slug":"identifying-flaky-tests-in-quantum-code-a","title":"Identifying Flaky Tests in Quantum Code: A Machine Learning Approach","date":"2025-02-06","arxiv_id":"2502.04471","repositories_listed":0,"syntology":null},{"url":null,"slug":"a-systematic-approach-for-assessing-large","title":"A Systematic Approach for Assessing Large Language Models' Test Case Generation Capability","date":"2025-02-05","arxiv_id":"2502.02866","repositories_listed":0,"syntology":null},{"url":null,"slug":"toward-neurosymbolic-program-comprehension","title":"Toward Neurosymbolic Program Comprehension","date":"2025-02-03","arxiv_id":"2502.01806","repositories_listed":0,"syntology":null},{"url":null,"slug":"many-objective-neuroevolution-for-testing","title":"Many-Objective Neuroevolution for Testing Games","date":"2025-01-14","arxiv_id":"2501.07954","repositories_listed":0,"syntology":null},{"url":null,"slug":"an-efficient-approach-to-represent-enterprise","title":"An efficient approach to represent enterprise web application structure using Large Language Model in the service of Intelligent Quality Engineering","date":"2025-01-12","arxiv_id":"2501.06837","repositories_listed":0,"syntology":null},{"url":null,"slug":"the-potential-of-llms-in-automating-software","title":"The Potential of LLMs in Automating Software Testing: From Generation to Reporting","date":"2024-12-31","arxiv_id":"2501.00217","repositories_listed":0,"syntology":null},{"url":null,"slug":"design-choices-made-by-llm-based-test","title":"Design choices made by LLM-based test generators prevent them from finding bugs","date":"2024-12-18","arxiv_id":"2412.14137","repositories_listed":0,"syntology":null},{"url":null,"slug":"reinforcement-learning-from-automatic-1","title":"Reinforcement Learning from Automatic Feedback for High-Quality Unit Test Generation","date":"2024-12-18","arxiv_id":"2412.14308","repositories_listed":0,"syntology":null},{"url":null,"slug":"cpp-ut-bench-can-llms-write-complex-unit","title":"CPP-UT-Bench: Can LLMs Write Complex Unit Tests in C++?","date":"2024-12-03","arxiv_id":"2412.02735","repositories_listed":0,"syntology":null},{"url":null,"slug":"software-testing-for-project-report","title":"Software testing for project report.","date":"2024-11-20","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"valtest-automated-validation-of-language","title":"VALTEST: Automated Validation of Language Model Generated Test Cases","date":"2024-11-13","arxiv_id":"2411.08254","repositories_listed":0,"syntology":null},{"url":null,"slug":"taegan-generating-synthetic-tabular-data-for","title":"TAEGAN: Generating Synthetic Tabular Data For Data Augmentation","date":"2024-10-02","arxiv_id":"2410.01933","repositories_listed":0,"syntology":null},{"url":null,"slug":"on-the-effectiveness-of-llms-for-manual-test","title":"On the Effectiveness of LLMs for Manual Test Verifications","date":"2024-09-19","arxiv_id":"2409.12405","repositories_listed":0,"syntology":null},{"url":null,"slug":"computer-vision-intelligence-test-modeling","title":"Computer Vision Intelligence Test Modeling and Generation: A Case Study on Smart OCR","date":"2024-09-14","arxiv_id":"2410.03536","repositories_listed":0,"syntology":null},{"url":null,"slug":"exploring-the-integration-of-large-language-1","title":"Exploring the Integration of Large Language Models in Industrial Test Maintenance Processes","date":"2024-09-10","arxiv_id":"2409.06416","repositories_listed":0,"syntology":null},{"url":null,"slug":"the-future-of-software-testing-ai-powered","title":"The Future of Software Testing: AI-Powered Test Case Generation and Validation","date":"2024-09-09","arxiv_id":"2409.05808","repositories_listed":0,"syntology":null},{"url":null,"slug":"the-role-of-artificial-intelligence-and","title":"The Role of Artificial Intelligence and Machine Learning in Software Testing","date":"2024-09-04","arxiv_id":"2409.02693","repositories_listed":0,"syntology":null},{"url":null,"slug":"testing-and-evaluation-of-large-language","title":"Testing and Evaluation of Large Language Models: Correctness, Non-Toxicity, and Fairness","date":"2024-08-31","arxiv_id":"2409.00551","repositories_listed":0,"syntology":null},{"url":null,"slug":"a-system-for-automated-unit-test-generation","title":"A System for Automated Unit Test Generation Using Large Language Models and Assessment of Generated Test Suites","date":"2024-08-14","arxiv_id":"2408.07846","repositories_listed":0,"syntology":null},{"url":null,"slug":"2408-01916","title":"MAO: A Framework for Process Model Generation with Multi-Agent Orchestration","date":"2024-08-04","arxiv_id":"2408.01916","repositories_listed":0,"syntology":null},{"url":null,"slug":"fuzztherest-an-intelligent-automated-black","title":"FuzzTheREST: An Intelligent Automated Black-box RESTful API Fuzzer","date":"2024-07-19","arxiv_id":"2407.14361","repositories_listed":0,"syntology":null},{"url":null,"slug":"exploring-fuzzing-as-data-augmentation-for","title":"Data Augmentation by Fuzzing for Neural Test Generation","date":"2024-06-12","arxiv_id":"2406.08665","repositories_listed":0,"syntology":null},{"url":null,"slug":"bugblitz-ai-an-intelligent-qa-assistant","title":"BugBlitz-AI: An Intelligent QA Assistant","date":"2024-05-17","arxiv_id":"2406.04356","repositories_listed":0,"syntology":null},{"url":null,"slug":"artificial-intelligence-for-context-aware","title":"Artificial intelligence for context-aware visual change detection in software test automation","date":"2024-05-01","arxiv_id":"2405.00874","repositories_listed":0,"syntology":null},{"url":null,"slug":"fuzzy-inference-system-for-test-case","title":"Fuzzy Inference System for Test Case Prioritization in Software Testing","date":"2024-04-25","arxiv_id":"2404.16395","repositories_listed":0,"syntology":null},{"url":null,"slug":"tasks-people-prompt-a-taxonomy-of-llm","title":"Tasks People Prompt: A Taxonomy of LLM Downstream Tasks in Software Verification and Falsification Approaches","date":"2024-04-14","arxiv_id":"2404.09384","repositories_listed":0,"syntology":null},{"url":null,"slug":"machine-learning-robustness-a-primer","title":"Machine Learning Robustness: A Primer","date":"2024-04-01","arxiv_id":"2404.00897","repositories_listed":0,"syntology":null},{"url":null,"slug":"ritfis-robust-input-testing-framework-for","title":"RITFIS: Robust input testing framework for LLMs-based intelligent software","date":"2024-02-21","arxiv_id":"2402.13518","repositories_listed":0,"syntology":null},{"url":null,"slug":"unitsyn-a-large-scale-dataset-capable-of","title":"UniTSyn: A Large-Scale Dataset Capable of Enhancing the Prowess of Large Language Models for Program Testing","date":"2024-02-04","arxiv_id":"2402.03396","repositories_listed":0,"syntology":null},{"url":null,"slug":"code-aware-prompting-a-study-of-coverage","title":"Code-Aware Prompting: A study of Coverage Guided Test Generation in Regression Setting using LLM","date":"2024-01-31","arxiv_id":"2402.00097","repositories_listed":0,"syntology":null},{"url":null,"slug":"mutation-based-consistency-testing-for","title":"Mutation-based Consistency Testing for Evaluating the Code Understanding Capability of LLMs","date":"2024-01-11","arxiv_id":"2401.05940","repositories_listed":0,"syntology":null},{"url":null,"slug":"a-case-study-on-test-case-construction-with","title":"A Case Study on Test Case Construction with Large Language Models: Unveiling Practical Insights and Challenges","date":"2023-12-19","arxiv_id":"2312.12598","repositories_listed":0,"syntology":null},{"url":null,"slug":"studying-the-practices-of-testing-machine","title":"An empirical study of testing machine learning in the wild","date":"2023-12-19","arxiv_id":"2312.12604","repositories_listed":0,"syntology":null},{"url":null,"slug":"can-chatgpt-advance-software-testing","title":"Can ChatGPT advance software testing intelligence? An experience report on metamorphic testing","date":"2023-10-30","arxiv_id":"2310.19204","repositories_listed":0,"syntology":null},{"url":null,"slug":"reinforcement-learning-from-automatic","title":"Reinforcement Learning from Automatic Feedback for High-Quality Unit Test Generation","date":"2023-10-03","arxiv_id":"2310.02368","repositories_listed":0,"syntology":null},{"url":null,"slug":"testlab-an-intelligent-automated-software","title":"TestLab: An Intelligent Automated Software Testing Framework","date":"2023-06-06","arxiv_id":"2306.03602","repositories_listed":0,"syntology":null},{"url":null,"slug":"rethink-diversity-in-deep-learning-testing","title":"Rethinking Diversity in Deep Neural Network Testing","date":"2023-05-25","arxiv_id":"2305.15698","repositories_listed":0,"syntology":null},{"url":null,"slug":"validating-multimedia-content-moderation","title":"Validating Multimedia Content Moderation Software via Semantic Fusion","date":"2023-05-23","arxiv_id":"2305.13623","repositories_listed":0,"syntology":null},{"url":null,"slug":"comparing-software-developers-with-chatgpt-an","title":"Comparing Software Developers with ChatGPT: An Empirical Investigation","date":"2023-05-19","arxiv_id":"2305.11837","repositories_listed":0,"syntology":null},{"url":null,"slug":"information-theoretic-testing-and-debugging","title":"Information-Theoretic Testing and Debugging of Fairness Defects in Deep Neural Networks","date":"2023-04-09","arxiv_id":"2304.04199","repositories_listed":0,"syntology":null},{"url":null,"slug":"constrained-adversarial-learning-and-its","title":"Constrained Adversarial Learning for Automated Software Testing: a literature review","date":"2023-03-14","arxiv_id":"2303.07546","repositories_listed":0,"syntology":null},{"url":null,"slug":"object-oriented-test-case-generation-using","title":"Object-Oriented Test Case Generation Using Teaching Learning-Based Optimization (TLBO) Algorithm","date":"2022-10-14","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"diverget-a-search-based-software-testing","title":"DiverGet: A Search-Based Software Testing Approach for Deep Neural Network Quantization Assessment","date":"2022-07-13","arxiv_id":"2207.06282","repositories_listed":0,"syntology":null},{"url":null,"slug":"guiding-the-retraining-of-convolutional","title":"Guiding the retraining of convolutional neural networks against adversarial inputs","date":"2022-07-08","arxiv_id":"2207.03689","repositories_listed":0,"syntology":null},{"url":null,"slug":"test2vec-an-execution-trace-embedding-for","title":"Test2Vec: An Execution Trace Embedding for Test Case Prioritization","date":"2022-06-28","arxiv_id":"2206.15428","repositories_listed":0,"syntology":null}],"syntology_records":5,"syntology_note":"a paper without a record is not a recorded non-run: it may lack an arXiv id or simply be absent from the graph layer"},"key_notes":{"n_ran_checked":"legacy name, kept unchanged so existing readers do not break: it counts the samples that ran with no instrument failure (honoured, violated, and ran with no contract checked); it does not mean a contract was checked, and the pages print it as 'K with no instrument failure', not 'K checked'","n_constructed":"a sub-count of the samples that ran, never subtracted from them and never a failure: an executed sample whose run returned an instance of its own class (fixture_out_type equals the entry name): the run built an object and did not compute a result (Syntology's RAN record, counts.constructed)"},"since_archive_judged":{"papers":0,"papers_placed":0,"judged_by":{"model":"a model","models_as_recorded":[],"from":"lb_rows_run_verified.jsonl: each record's `model` field (the extractor's call that wrote decisions[])"},"note":"a model's judgement from the paper's own tables that they report on this leaderboard, not a result: no value from a paper that was not placed is printed or carried here; papers counts each paper once over these leaderboards","backfill":"8,886 of 18,637 newer papers checked so far (733 more had nothing to place; 41 could not be read)","backfill_complete":false,"leaderboards":[]},"papers_where_code_ran":5,"papers_where_code_ran_first_page":"/task/software-testing/papers/ran/1","papers_where_code_ran_caption":"We ran code from the paper's repository; we did not run it on this task or check it against the task's benchmarks.","pages":[{"order":"archive","page":2,"of":2,"url":"/task/software-testing/papers/2","json":"/task/software-testing/papers/2.json","rows":35},{"order":"ran","page":1,"of":1,"url":"/task/software-testing/papers/ran/1","json":"/task/software-testing/papers/ran/1.json","rows":5}],"description_links":{"kept":0,"unwrapped_to_text":0,"bare_urls_linked":0,"relative_images_dropped":0,"rule":"internal links are kept only when the target slug exists in the catalog"},"syntology":{"read_at":"2026-10-05T00:36:11+00:00","claim":"Per-sample execution status on synthesized fixtures ('ran N of M samples'); not a correctness claim and not a ranking signal.","status_vocabulary":{"ran_honours":"ran, honoured the contract we drafted","ran_violates":"ran, violated the contract we drafted","ran_draft_wrong":"ran; our contract draft was wrong, not the code","ran_fixture":"ran; the contract check could not run (RAISES, DEP_MISSING or TIMEOUT)","ran":"ran on a synthesized input","unverified":"unverified (harvested, no recorded run)"},"row_route":"each syntology object on a benchmark's first row and on a paper carries sample_list (this site's paper page, #ran), syntology_url (syntology.ai's page for the paper, only when that route probed live) and mcp (the agent call that lists the paper's harvested samples; not a verdict on the paper)","developers":"https://syntology.ai/developers"},"not_shown":{"libraries":"the archive has no per-task library table","trend_sparklines":"the Trend column of the benchmarks table was a rendered image; it is not in the archive","social_and_latest_sorts":"stars and social signals are not in the archive"},"record_sha256":"451e681e276fea9e4e3c1169cb454678ac16fe4adf38cd6f4437db3e7e47a30b","record_changed_at":"2026-10-05","record_changed_at_basis":"changed"}