{"about":{"non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","site":"https://codewithpapers.app","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page","syntology":{"site":"https://syntology.ai","developers":"https://syntology.ai/developers","mcp":{"server":"https://syntology.ai/mcp","transport":"streamable-http","server_card":"https://syntology.ai/.well-known/mcp/server-card.json","auth":{"type":"trial token, no account","trial_token":"https://syntology.ai/api/oauth/trial/token","method":"POST","docs":"https://syntology.ai/developers"}},"have":"https://syntology.ai/api/graph/have?x=<method, arXiv id or title> (free, answers coverage only)","paper_base":"https://syntology.ai/paper/","atlas_base":"https://app.syntology.ai/?focus="},"machine_readable":[{"url":"https://codewithpapers.app/llms.txt","what":"the machine catalog: every machine-readable file, counted"},{"url":"https://codewithpapers.app/index/manifest.json","what":"paper-to-code index by arXiv id, with Syntology's counts"},{"url":"https://codewithpapers.app/search/manifest.json","what":"site search index (titles, authors) and its files"},{"url":"https://codewithpapers.app/download","what":"bulk files: Syntology's layer, described there"},{"url":"https://codewithpapers.app/build_manifest.json","what":"the build record: inputs, counts, exclusions, probes"}]},"url":"/task/legal-reasoning","name":"Legal Reasoning","slug":"legal-reasoning","description_markdown":null,"categories":[{"name":"Natural Language Processing","url":"/area/natural-language-processing"}],"source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","slug_source":"derived"},"counts":{"papers_tagged":92,"papers_with_code":30,"benchmarks":2,"benchmark_tables_in_archive":2,"benchmark_tables_shown":2,"benchmark_tables_withheld_as_spam":0,"benchmark_definition":"a leaderboard table with at least one row; benchmark_tables_shown also counts the zero-row tables; benchmark_tables_in_archive adds the tables withheld as spam","datasets":0,"subtasks":0,"parent_tasks":0},"benchmarks":[{"leaderboard":"/sota/legal-reasoning-on-legalbench-issue-spotting","slug":"legal-reasoning-on-legalbench-issue-spotting","dataset":"LegalBench (Issue-spotting)","dataset_url":null,"rows_in_archive":3,"metrics":["Balanced Accuracy"],"first_row_in_archive_order":{"model":"GPT-4","paper_title":null,"paper_url":null,"paper_date":"","arxiv_id":null,"code_links":[],"syntology":null}},{"leaderboard":"/sota/legal-reasoning-on-legalbench-rule-recall","slug":"legal-reasoning-on-legalbench-rule-recall","dataset":"LegalBench (Rule-recall)","dataset_url":null,"rows_in_archive":1,"metrics":["Balanced Accuracy"],"first_row_in_archive_order":{"model":"GPT-4","paper_title":"GPT-4 Technical Report","paper_url":"/paper/gpt-4-technical-report-1","paper_date":"2023-03-15","arxiv_id":"2303.08774","code_links":[{"title":"openai/evals","url":"https://github.com/openai/evals"},{"title":"shmsw25/factscore","url":"https://github.com/shmsw25/factscore"},{"title":"unispac/visual-adversarial-examples-jailbreak-large-language-models","url":"https://github.com/unispac/visual-adversarial-examples-jailbreak-large-language-models"},{"title":"gpt4life/alpagasus","url":"https://github.com/gpt4life/alpagasus"},{"title":"emrgnt-cmplxty/zero-shot-replication","url":"https://github.com/emrgnt-cmplxty/zero-shot-replication"},{"title":"ethz-privsec/superhuman-ai-consistency","url":"https://github.com/ethz-privsec/superhuman-ai-consistency"},{"title":"ethz-spylab/superhuman-ai-consistency","url":"https://github.com/ethz-spylab/superhuman-ai-consistency"},{"title":"eternityyw/tram-benchmark","url":"https://github.com/eternityyw/tram-benchmark"},{"title":"AUCOHL/RTL-Repo","url":"https://github.com/AUCOHL/RTL-Repo"},{"title":"zach-zhiling-zheng/reticular_chemist","url":"https://github.com/zach-zhiling-zheng/reticular_chemist"},{"title":"lflage/openfactscore","url":"https://github.com/lflage/openfactscore"}],"syntology":{"n":5,"n_ran":5,"n_constructed":0,"n_ran_checked":5,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":2,"n_no_contract":3,"n_pointer_only":1,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 0 honoured, 2 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/gpt-4-technical-report-1#ran","syntology_url":"https://syntology.ai/paper/2303.08774","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2303.08774"}},"official":{"repos":["openai/evals"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}}}],"datasets":[],"subtasks":[],"parent_tasks":[],"papers":{"order":"repositories listed in the archive (most first), then date (newest first), then slug; the archive holds no stars","population":"every paper the archive tags with this task (distinct papers; spam titles not listed). 'of' counts the papers listed; until 2026-09-25 it counted only the papers with a code link, and that number is of_with_code","page":1,"pages_in_order":1,"rows_per_page":100,"shown":92,"of":92,"of_with_code":30,"tagged_in_all":92,"counts":{"archive_papers_tagged":92,"with_a_code_link":30,"where_syntology_ran_a_sample":7,"not_listed_spam_title":0,"listed":92,"listed_where_code_ran":7,"where_syntology_ran_a_sample_split":{"with_a_run_with_no_instrument_failure":6,"every_run_a_failure_of_syntologys_instrument":1,"listed_with_a_run_with_no_instrument_failure":6,"listed_every_run_a_failure_of_syntologys_instrument":1,"filter":{"states":["a run with no instrument failure","any run, instrument failures included"],"default":"a run with no instrument failure","note":"on the 'only where code ran' pages the default hides, in the browser, the rows where every run was a failure of Syntology's instrument; the second state shows them again. Rows are hidden, never re-ordered; these twins list every row"}},"definition":"distinct papers the archive tags; 'where Syntology ran a sample' counts papers with at least one harvested sample that ran, which is not a correctness claim"},"items":[{"url":"/paper/gpt-4-technical-report-1","slug":"gpt-4-technical-report-1","title":"GPT-4 Technical Report","date":"2023-03-15","arxiv_id":"2303.08774","repositories_listed":11,"syntology":{"n":5,"n_ran":5,"n_constructed":0,"n_ran_checked":5,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":2,"n_no_contract":3,"n_pointer_only":1,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 0 honoured, 2 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/gpt-4-technical-report-1#ran","syntology_url":"https://syntology.ai/paper/2303.08774","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2303.08774"}},"official":{"repos":["openai/evals"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/disc-lawllm-fine-tuning-large-language-models","slug":"disc-lawllm-fine-tuning-large-language-models","title":"DISC-LawLLM: Fine-tuning Large Language Models for Intelligent Legal Services","date":"2023-09-20","arxiv_id":"2309.11325","repositories_listed":2,"syntology":{"n":6,"n_ran":5,"n_constructed":0,"n_ran_checked":5,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":5,"n_pointer_only":0,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 0 honoured, 0 violated, 5 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/disc-lawllm-fine-tuning-large-language-models#ran","syntology_url":"https://syntology.ai/paper/2309.11325","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2309.11325"}},"official":{"repos":["fudandisc/disc-lawllm"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":1,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/scale-scaling-up-the-complexity-for-advanced","slug":"scale-scaling-up-the-complexity-for-advanced","title":"One Law, Many Languages: Benchmarking Multilingual Legal Reasoning for Judicial Support","date":"2023-06-15","arxiv_id":"2306.09237","repositories_listed":2,"syntology":null},{"url":"/paper/parameter-efficient-fine-tuning-llama-3-1-for","slug":"parameter-efficient-fine-tuning-llama-3-1-for","title":"Parameter Efficient Fine Tuning Llama 3.1 for Answering Arabic Legal Questions: A Case Study on Jordanian Laws","date":"2025-06-02","arxiv_id":null,"repositories_listed":1,"syntology":null},{"url":"/paper/llm-based-hse-compliance-assessment-benchmark","slug":"llm-based-hse-compliance-assessment-benchmark","title":"LLM-based HSE Compliance Assessment: Benchmark, Performance, and Advancements","date":"2025-05-29","arxiv_id":"2505.22959","repositories_listed":1,"syntology":null},{"url":"/paper/incorporating-legal-structure-in-retrieval","slug":"incorporating-legal-structure-in-retrieval","title":"Incorporating Legal Structure in Retrieval-Augmented Generation: A Case Study on Copyright Fair Use","date":"2025-05-04","arxiv_id":"2505.02164","repositories_listed":1,"syntology":null},{"url":"/paper/casegen-a-benchmark-for-multi-stage-legal","slug":"casegen-a-benchmark-for-multi-stage-legal","title":"CaseGen: A Benchmark for Multi-Stage Legal Case Documents Generation","date":"2025-02-25","arxiv_id":"2502.17943","repositories_listed":1,"syntology":null},{"url":"/paper/jurex-4e-juridical-expert-annotated-four","slug":"jurex-4e-juridical-expert-annotated-four","title":"JUREX-4E: Juridical Expert-Annotated Four-Element Knowledge Base for Legal Reasoning","date":"2025-02-24","arxiv_id":"2502.17166","repositories_listed":1,"syntology":{"n":15,"n_ran":14,"n_constructed":0,"n_ran_checked":13,"n_instrument":1,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":13,"n_pointer_only":0,"phrase":"14 ran (of which 0 constructed an object rather than computing a result; 13 with no instrument failure: 0 honoured, 0 violated, 13 with no contract checked; 1 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/jurex-4e-juridical-expert-annotated-four#ran","syntology_url":"https://syntology.ai/paper/2502.17166","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2502.17166"}},"official":{"repos":["thulawtech/jurex"],"state":"official (archive's flag): 14 ran","n_ran":14,"n_constructed":0,"n_ran_no_instrument_failure":13,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/nitibench-a-comprehensive-studies-of-llm","slug":"nitibench-a-comprehensive-studies-of-llm","title":"NitiBench: A Comprehensive Studies of LLM Frameworks Capabilities for Thai Legal Question Answering","date":"2025-02-15","arxiv_id":"2502.10868","repositories_listed":1,"syntology":null},{"url":"/paper/elevating-legal-llm-responses-harnessing","slug":"elevating-legal-llm-responses-harnessing","title":"Elevating Legal LLM Responses: Harnessing Trainable Logical Structures and Semantic Knowledge with Legal Reasoning","date":"2025-02-11","arxiv_id":"2502.07912","repositories_listed":1,"syntology":null},{"url":"/paper/lawgpt-knowledge-guided-data-generation-and","slug":"lawgpt-knowledge-guided-data-generation-and","title":"LawGPT: Knowledge-Guided Data Generation and Its Application to Legal LLM","date":"2025-02-10","arxiv_id":"2502.06572","repositories_listed":1,"syntology":null},{"url":"/paper/investigating-the-shortcomings-of-llms-in","slug":"investigating-the-shortcomings-of-llms-in","title":"Investigating the Shortcomings of LLMs in Step-by-Step Legal Reasoning","date":"2025-02-08","arxiv_id":"2502.05675","repositories_listed":1,"syntology":null},{"url":"/paper/weak-to-strong-generalization-beyond-accuracy","slug":"weak-to-strong-generalization-beyond-accuracy","title":"Weak-to-Strong Generalization beyond Accuracy: a Pilot Study in Safety, Toxicity, and Legal Reasoning","date":"2024-10-16","arxiv_id":"2410.12621","repositories_listed":1,"syntology":null},{"url":"/paper/developing-a-pragmatic-benchmark-for","slug":"developing-a-pragmatic-benchmark-for","title":"Developing a Pragmatic Benchmark for Assessing Korean Legal Language Understanding in Large Language Models","date":"2024-10-11","arxiv_id":"2410.08731","repositories_listed":1,"syntology":null},{"url":"/paper/can-large-language-models-grasp-legal","slug":"can-large-language-models-grasp-legal","title":"Can Large Language Models Grasp Legal Theories? Enhance Legal Reasoning with Insights from Multi-Agent Collaboration","date":"2024-10-03","arxiv_id":"2410.02507","repositories_listed":1,"syntology":{"n":3,"n_ran":2,"n_constructed":0,"n_ran_checked":2,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":3,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/can-large-language-models-grasp-legal#ran","syntology_url":"https://syntology.ai/paper/2410.02507","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2410.02507"}},"official":{"repos":["yuanwk99/malr"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/lekube-a-legal-knowledge-update-benchmark","slug":"lekube-a-legal-knowledge-update-benchmark","title":"LeKUBE: A Legal Knowledge Update BEnchmark","date":"2024-07-19","arxiv_id":"2407.14192","repositories_listed":1,"syntology":null},{"url":"/paper/ai-driven-statutory-reasoning-via-software","slug":"ai-driven-statutory-reasoning-via-software","title":"Software Engineering Methods For AI-Driven Deductive Legal Reasoning","date":"2024-04-15","arxiv_id":"2404.09868","repositories_listed":1,"syntology":null},{"url":"/paper/flawn-t5-an-empirical-examination-of","slug":"flawn-t5-an-empirical-examination-of","title":"LawInstruct: A Resource for Studying Language Model Adaptation to the Legal Domain","date":"2024-04-02","arxiv_id":"2404.02127","repositories_listed":1,"syntology":null},{"url":"/paper/ecthr-pcr-a-dataset-for-precedent","slug":"ecthr-pcr-a-dataset-for-precedent","title":"ECtHR-PCR: A Dataset for Precedent Understanding and Prior Case Retrieval in the European Court of Human Rights","date":"2024-03-31","arxiv_id":"2404.00596","repositories_listed":1,"syntology":null},{"url":"/paper/chain-of-logic-rule-based-reasoning-with","slug":"chain-of-logic-rule-based-reasoning-with","title":"Chain of Logic: Rule-Based Reasoning with Large Language Models","date":"2024-02-16","arxiv_id":"2402.10400","repositories_listed":1,"syntology":null},{"url":"/paper/tmid-a-comprehensive-real-world-dataset-for","slug":"tmid-a-comprehensive-real-world-dataset-for","title":"TMID: A Comprehensive Real-world Dataset for Trademark Infringement Detection in E-Commerce","date":"2023-12-08","arxiv_id":"2312.05103","repositories_listed":1,"syntology":null},{"url":"/paper/modeling-legal-reasoning-lm-annotation-at-the","slug":"modeling-legal-reasoning-lm-annotation-at-the","title":"Modeling Legal Reasoning: LM Annotation at the Edge of Human Agreement","date":"2023-10-27","arxiv_id":"2310.18440","repositories_listed":1,"syntology":{"n":4,"n_ran":4,"n_constructed":0,"n_ran_checked":4,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":4,"n_pointer_only":4,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 4 with no instrument failure: 0 honoured, 0 violated, 4 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/modeling-legal-reasoning-lm-annotation-at-the#ran","syntology_url":"https://syntology.ai/paper/2310.18440","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2310.18440"}},"official":{"repos":["rosthalken/legal-interpretation"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":4,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/can-chatgpt-perform-reasoning-using-the-irac","slug":"can-chatgpt-perform-reasoning-using-the-irac","title":"Can ChatGPT Perform Reasoning Using the IRAC Method in Analyzing Legal Scenarios Like a Lawyer?","date":"2023-10-23","arxiv_id":"2310.14880","repositories_listed":1,"syntology":null},{"url":"/paper/a-comprehensive-evaluation-of-large-language-1","slug":"a-comprehensive-evaluation-of-large-language-1","title":"A Comprehensive Evaluation of Large Language Models on Legal Judgment Prediction","date":"2023-10-18","arxiv_id":"2310.11761","repositories_listed":1,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":0,"n_instrument":2,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":2,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/a-comprehensive-evaluation-of-large-language-1#ran","syntology_url":"https://syntology.ai/paper/2310.11761","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2310.11761"}},"official":{"repos":["srhthu/lm-compeval-legal"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/legalbench-a-collaboratively-built-benchmark-1","slug":"legalbench-a-collaboratively-built-benchmark-1","title":"LegalBench: A Collaboratively Built Benchmark for Measuring Legal Reasoning in Large Language Models","date":"2023-08-20","arxiv_id":"2308.11462","repositories_listed":1,"syntology":{"n":3,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":2,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":3,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 0 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/legalbench-a-collaboratively-built-benchmark-1#ran","syntology_url":"https://syntology.ai/paper/2308.11462","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2308.11462"}},"official":{"repos":["hazyresearch/legalbench"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/legalbench-prototyping-a-collaborative","slug":"legalbench-prototyping-a-collaborative","title":"LegalBench: Prototyping a Collaborative Benchmark for Legal Reasoning","date":"2022-09-13","arxiv_id":"2209.06120","repositories_listed":1,"syntology":null},{"url":"/paper/claim-extraction-and-law-matching-for-covid","slug":"claim-extraction-and-law-matching-for-covid","title":"Claim Extraction and Law Matching for COVID-19-related Legislation","date":"2022-06-01","arxiv_id":null,"repositories_listed":1,"syntology":null},{"url":"/paper/designing-normative-theories-of-ethical","slug":"designing-normative-theories-of-ethical","title":"Designing Normative Theories for Ethical and Legal Reasoning: LogiKEy Framework, Methodology, and Tool Support","date":"2019-03-25","arxiv_id":"1903.10187","repositories_listed":1,"syntology":null},{"url":"/paper/passing-the-brazilian-oab-exam-data","slug":"passing-the-brazilian-oab-exam-data","title":"Passing the Brazilian OAB Exam: data preparation and some experiments","date":"2017-12-14","arxiv_id":"1712.05128","repositories_listed":1,"syntology":null},{"url":"/paper/causality-and-responsibility-for-formal","slug":"causality-and-responsibility-for-formal","title":"Causality and Responsibility for Formal Verification and Beyond","date":"2016-08-29","arxiv_id":"1608.07879","repositories_listed":1,"syntology":null},{"url":null,"slug":"indianbailjudgments-1200-a-multi-attribute","title":"IndianBailJudgments-1200: A Multi-Attribute Dataset for Legal NLP on Indian Bail Orders","date":"2025-07-03","arxiv_id":"2507.02506","repositories_listed":0,"syntology":null},{"url":null,"slug":"large-language-models-acing-chartered","title":"Large Language Models Acing Chartered Accountancy","date":"2025-06-26","arxiv_id":"2506.21031","repositories_listed":0,"syntology":null},{"url":null,"slug":"chancery-evaluating-corporate-governance","title":"CHANCERY: Evaluating Corporate Governance Reasoning Capabilities in Language Models","date":"2025-06-05","arxiv_id":"2506.04636","repositories_listed":0,"syntology":null},{"url":null,"slug":"when-fairness-isn-t-statistical-the-limits-of","title":"When Fairness Isn't Statistical: The Limits of Machine Learning in Evaluating Legal Reasoning","date":"2025-06-04","arxiv_id":"2506.03913","repositories_listed":0,"syntology":null},{"url":null,"slug":"lexam-benchmarking-legal-reasoning-on-340-law","title":"LEXam: Benchmarking Legal Reasoning on 340 Law Exams","date":"2025-05-19","arxiv_id":"2505.12864","repositories_listed":0,"syntology":null},{"url":null,"slug":"synlexlm-scaling-legal-llms-with-synthetic","title":"SynLexLM: Scaling Legal LLMs with Synthetic Data and Curriculum Learning","date":"2025-04-26","arxiv_id":"2504.18762","repositories_listed":0,"syntology":null},{"url":null,"slug":"engineering-the-law-machine-learning","title":"Engineering the Law-Machine Learning Translation Problem: Developing Legally Aligned Models","date":"2025-04-23","arxiv_id":"2504.16969","repositories_listed":0,"syntology":null},{"url":null,"slug":"continual-pre-training-is-not-what-you-need","title":"Continual Pre-Training is (not) What You Need in Domain Adaption","date":"2025-04-18","arxiv_id":"2504.13603","repositories_listed":0,"syntology":null},{"url":null,"slug":"kfineval-pilot-a-comprehensive-benchmark","title":"KFinEval-Pilot: A Comprehensive Benchmark Suite for Korean Financial Language Understanding","date":"2025-04-17","arxiv_id":"2504.13216","repositories_listed":0,"syntology":null},{"url":null,"slug":"an-explicit-syllogistic-legal-reasoning","title":"An Explicit Syllogistic Legal Reasoning Framework for Large Language Models","date":"2025-04-05","arxiv_id":"2504.04042","repositories_listed":0,"syntology":null},{"url":null,"slug":"evaluating-test-time-scaling-llms-for-legal","title":"Evaluating Test-Time Scaling LLMs for Legal Reasoning: OpenAI o1, DeepSeek-R1, and Beyond","date":"2025-03-20","arxiv_id":"2503.16040","repositories_listed":0,"syntology":null},{"url":null,"slug":"adaptively-evaluating-models-with-task","title":"Adaptively profiling models with task elicitation","date":"2025-03-03","arxiv_id":"2503.01986","repositories_listed":0,"syntology":null},{"url":null,"slug":"towards-robust-legal-reasoning-harnessing","title":"Towards Robust Legal Reasoning: Harnessing Logical LLMs in Law","date":"2025-02-24","arxiv_id":"2502.17638","repositories_listed":0,"syntology":null},{"url":null,"slug":"logical-lease-litigation-prolog-and-llms-for","title":"Logical Lease Litigation: Prolog and LLMs for Rental Law Compliance in New York","date":"2025-02-13","arxiv_id":"2502.09204","repositories_listed":0,"syntology":null},{"url":null,"slug":"artificial-intelligence-and-legal-analysis","title":"Artificial Intelligence and Legal Analysis: Implications for Legal Education and the Profession","date":"2025-02-04","arxiv_id":"2502.03487","repositories_listed":0,"syntology":null},{"url":null,"slug":"domaino1s-guiding-llm-reasoning-for","title":"Domaino1s: Guiding LLM Reasoning for Explainable Answers in High-Stakes Domains","date":"2025-01-24","arxiv_id":"2501.14431","repositories_listed":0,"syntology":null},{"url":null,"slug":"legal-evalutions-and-challenges-of-large","title":"Legal Evalutions and Challenges of Large Language Models","date":"2024-11-15","arxiv_id":"2411.10137","repositories_listed":0,"syntology":null},{"url":null,"slug":"lar-echr-a-new-legal-argument-reasoning-task","title":"LAR-ECHR: A New Legal Argument Reasoning Task and Dataset for Cases of the European Court of Human Rights","date":"2024-10-17","arxiv_id":"2410.13352","repositories_listed":0,"syntology":null},{"url":null,"slug":"krag-framework-for-enhancing-llms-in-the","title":"KRAG Framework for Enhancing LLMs in the Legal Domain","date":"2024-10-10","arxiv_id":"2410.07551","repositories_listed":0,"syntology":null},{"url":null,"slug":"using-llms-to-discover-legal-factors","title":"Using LLMs to Discover Legal Factors","date":"2024-10-10","arxiv_id":"2410.07504","repositories_listed":0,"syntology":null},{"url":null,"slug":"legilm-a-fine-tuned-legal-language-model-for","title":"LegiLM: A Fine-Tuned Legal Language Model for Data Compliance","date":"2024-09-09","arxiv_id":"2409.13721","repositories_listed":0,"syntology":null},{"url":null,"slug":"lapis-language-model-augmented-police","title":"LAPIS: Language Model-Augmented Police Investigation System","date":"2024-07-19","arxiv_id":"2407.20248","repositories_listed":0,"syntology":null},{"url":null,"slug":"formalising-anti-discrimination-law-in","title":"Formalising Anti-Discrimination Law in Automated Decision Systems","date":"2024-06-29","arxiv_id":"2407.00400","repositories_listed":0,"syntology":null},{"url":null,"slug":"bridging-law-and-data-augmenting-reasoning","title":"Bridging Law and Data: Augmenting Reasoning via a Semi-Structured Dataset with IRAC methodology","date":"2024-06-19","arxiv_id":"2406.13217","repositories_listed":0,"syntology":null},{"url":null,"slug":"towards-supporting-legal-argumentation-with","title":"Towards Supporting Legal Argumentation with NLP: Is More Data Really All You Need?","date":"2024-06-16","arxiv_id":"2406.10974","repositories_listed":0,"syntology":null},{"url":null,"slug":"explainable-machine-learning-multi-label","title":"Explainable machine learning multi-label classification of Spanish legal judgements","date":"2024-05-27","arxiv_id":"2405.17610","repositories_listed":0,"syntology":null},{"url":null,"slug":"paramanu-ayn-an-efficient-novel-generative","title":"PARAMANU-AYN: Pretrain from scratch or Continual Pretraining of LLMs for Legal Domain Adaptation?","date":"2024-03-20","arxiv_id":"2403.13681","repositories_listed":0,"syntology":null},{"url":null,"slug":"advancing-legal-reasoning-the-integration-of","title":"Advancing Legal Reasoning: The Integration of AI to Navigate Complexities and Biases in Global Jurisprudence with Semi-Automated Arbitration Processes (SAAPs)","date":"2024-02-06","arxiv_id":"2402.04140","repositories_listed":0,"syntology":null},{"url":null,"slug":"aalap-ai-assistant-for-legal-paralegal","title":"Aalap: AI Assistant for Legal & Paralegal Functions in India","date":"2024-01-30","arxiv_id":"2402.01758","repositories_listed":0,"syntology":null},{"url":null,"slug":"automated-legal-reasoning-with-discretion-to","title":"Automated legal reasoning with discretion to act using s(LAW)","date":"2024-01-25","arxiv_id":"2401.14511","repositories_listed":0,"syntology":null},{"url":null,"slug":"enhancing-logical-reasoning-in-large-language","title":"Enhancing Logical Reasoning in Large Language Models to Facilitate Legal Applications","date":"2023-11-22","arxiv_id":"2311.13095","repositories_listed":0,"syntology":null},{"url":null,"slug":"large-language-models-in-cryptocurrency","title":"Large Language Models in Cryptocurrency Securities Cases: Can a GPT Model Meaningfully Assist Lawyers?","date":"2023-08-11","arxiv_id":"2308.06032","repositories_listed":0,"syntology":null},{"url":null,"slug":"exploring-the-psychology-of-gpt-4-s-moral-and","title":"Exploring the psychology of LLMs' Moral and Legal Reasoning","date":"2023-08-02","arxiv_id":"2308.01264","repositories_listed":0,"syntology":null},{"url":null,"slug":"argumentation-schemes-for-blockchain","title":"Argumentation Schemes for Blockchain Deanonymization","date":"2023-05-26","arxiv_id":"2305.16883","repositories_listed":0,"syntology":null},{"url":null,"slug":"algorithmic-unfairness-through-the-lens-of-eu","title":"Algorithmic Unfairness through the Lens of EU Non-Discrimination Law: Or Why the Law is not a Decision Tree","date":"2023-05-05","arxiv_id":"2305.13938","repositories_listed":0,"syntology":null},{"url":null,"slug":"zero-shot-transfer-of-legal-judgement","title":"Zero-shot Transfer of Article-aware Legal Outcome Classification for European Court of Human Rights Cases","date":"2023-02-01","arxiv_id":"2302.00609","repositories_listed":0,"syntology":null},{"url":null,"slug":"law-to-binary-tree-an-formal-interpretation","title":"Law to Binary Tree -- An Formal Interpretation of Legal Natural Language","date":"2022-12-16","arxiv_id":"2212.08335","repositories_listed":0,"syntology":null},{"url":null,"slug":"legal-prompting-teaching-a-language-model-to","title":"Legal Prompting: Teaching a Language Model to Think Like a Lawyer","date":"2022-12-02","arxiv_id":"2212.01326","repositories_listed":0,"syntology":null},{"url":null,"slug":"using-argumentation-schemes-to-model-legal","title":"Using Argumentation Schemes to Model Legal Reasoning","date":"2022-10-01","arxiv_id":"2210.00315","repositories_listed":0,"syntology":null},{"url":null,"slug":"law-informs-code-a-legal-informatics-approach","title":"Law Informs Code: A Legal Informatics Approach to Aligning Artificial Intelligence with Humans","date":"2022-09-14","arxiv_id":"2209.13020","repositories_listed":0,"syntology":null},{"url":null,"slug":"legal-detection-of-ai-products-based-on","title":"An Argumentation-Based Legal Reasoning Approach for DL-Ontology","date":"2022-09-07","arxiv_id":"2209.03070","repositories_listed":0,"syntology":null},{"url":null,"slug":"non-determinism-and-the-lawlessness-of-ml","title":"Non-Determinism and the Lawlessness of Machine Learning Code","date":"2022-06-23","arxiv_id":"2206.11834","repositories_listed":0,"syntology":null},{"url":null,"slug":"searching-for-legal-documents-at-paragraph","title":"Searching for Legal Documents at Paragraph Level: Automating Label Generation and Use of an Extended Attention Mask for Boosting Neural Models of Semantic Similarity","date":"2021-11-01","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"exploiting-domain-specific-knowledge-for","title":"Exploiting Domain-Specific Knowledge for Judgment Prediction Is No Panacea","date":"2021-09-01","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"new-algebraic-normative-theories-for-ethical","title":"New Algebraic Normative Theories for Ethical and Legal Reasoning in the LogiKEy Framework","date":"2021-07-25","arxiv_id":"2107.11838","repositories_listed":0,"syntology":null},{"url":null,"slug":"autolaw-augmented-legal-reasoning-through","title":"AutoLAW: Augmented Legal Reasoning through Legal Precedent Prediction","date":"2021-06-30","arxiv_id":"2106.16034","repositories_listed":0,"syntology":null},{"url":null,"slug":"antitrust-and-artificial-intelligence-aai","title":"Antitrust and Artificial Intelligence (AAI): Antitrust Vigilance Lifecycle and AI Legal Reasoning Autonomy","date":"2020-12-23","arxiv_id":"2012.13016","repositories_listed":0,"syntology":null},{"url":null,"slug":"legal-sentiment-analysis-and-opinion-mining","title":"Legal Sentiment Analysis and Opinion Mining (LSAOM): Assimilating Advances in Autonomous AI Legal Reasoning","date":"2020-10-02","arxiv_id":"2010.02726","repositories_listed":0,"syntology":null},{"url":null,"slug":"legal-judgment-prediction-ljp-amid-the-advent","title":"Legal Judgment Prediction (LJP) Amid the Advent of Autonomous AI Legal Reasoning","date":"2020-09-29","arxiv_id":"2009.14620","repositories_listed":0,"syntology":null},{"url":null,"slug":"the-next-era-of-american-law-amid-the-advent","title":"The Next Era of American Law Amid the Advent of Autonomous AI Legal Reasoning","date":"2020-09-21","arxiv_id":"2009.11647","repositories_listed":0,"syntology":null},{"url":null,"slug":"deconstructing-legal-text-object-oriented","title":"Deconstructing Legal Text_Object Oriented Design in Legal Adjudication","date":"2020-09-13","arxiv_id":"2009.06054","repositories_listed":0,"syntology":null},{"url":null,"slug":"ai-and-legal-argumentation-aligning-the","title":"AI and Legal Argumentation: Aligning the Autonomous Levels of AI Legal Reasoning","date":"2020-09-11","arxiv_id":"2009.11180","repositories_listed":0,"syntology":null},{"url":null,"slug":"robustness-and-overcoming-brittleness-of-ai","title":"Robustness and Overcoming Brittleness of AI-Enabled Legal Micro-Directives: The Role of Autonomous Levels of AI Legal Reasoning","date":"2020-08-31","arxiv_id":"2009.02243","repositories_listed":0,"syntology":null},{"url":null,"slug":"an-impact-model-of-ai-on-the-principles-of","title":"An Impact Model of AI on the Principles of Justice: Encompassing the Autonomous Levels of AI Legal Reasoning","date":"2020-08-26","arxiv_id":"2008.12615","repositories_listed":0,"syntology":null},{"url":null,"slug":"multidimensionality-of-legal-singularity","title":"Multidimensionality of Legal Singularity: Parametric Analysis and the Autonomous Levels of AI Legal Reasoning","date":"2020-08-24","arxiv_id":"2008.10575","repositories_listed":0,"syntology":null},{"url":null,"slug":"authorized-and-unauthorized-practices-of-law","title":"Authorized and Unauthorized Practices of Law: The Role of Autonomous Levels of AI Legal Reasoning","date":"2020-08-19","arxiv_id":"2008.09507","repositories_listed":0,"syntology":null},{"url":null,"slug":"turing-test-and-the-practice-of-law-the-role","title":"Turing Test and the Practice of Law: The Role of Autonomous Levels of AI Legal Reasoning","date":"2020-08-18","arxiv_id":"2008.07743","repositories_listed":0,"syntology":null},{"url":null,"slug":"an-ontological-ai-and-law-framework-for-the","title":"An Ontological AI-and-Law Framework for the Autonomous Levels of AI Legal Reasoning","date":"2020-08-04","arxiv_id":"2008.07328","repositories_listed":0,"syntology":null},{"url":null,"slug":"encoding-legal-balancing-automating-an","title":"Modelling Value-oriented Legal Reasoning in LogiKEy","date":"2020-06-23","arxiv_id":"2006.12789","repositories_listed":0,"syntology":null},{"url":null,"slug":"proceedings-first-workshop-on-causal","title":"Proceedings First Workshop on Causal Reasoning for Embedded and safety-critical Systems Technologies","date":"2016-08-26","arxiv_id":"1608.07398","repositories_listed":0,"syntology":null},{"url":null,"slug":"automated-reasoning-for-robot-ethics","title":"Automated Reasoning for Robot Ethics","date":"2015-02-20","arxiv_id":"1502.05838","repositories_listed":0,"syntology":null},{"url":null,"slug":"automated-reasoning-in-deontic-logic","title":"Automated Reasoning in Deontic Logic","date":"2014-11-18","arxiv_id":"1411.4823","repositories_listed":0,"syntology":null}],"syntology_records":7,"syntology_note":"a paper without a record is not a recorded non-run: it may lack an arXiv id or simply be absent from the graph layer"},"key_notes":{"n_ran_checked":"legacy name, kept unchanged so existing readers do not break: it counts the samples that ran with no instrument failure (honoured, violated, and ran with no contract checked); it does not mean a contract was checked, and the pages print it as 'K with no instrument failure', not 'K checked'","n_constructed":"a sub-count of the samples that ran, never subtracted from them and never a failure: an executed sample whose run returned an instance of its own class (fixture_out_type equals the entry name): the run built an object and did not compute a result (Syntology's RAN record, counts.constructed)"},"since_archive_judged":{"papers":0,"papers_placed":0,"judged_by":{"model":"a model","models_as_recorded":[],"from":"lb_rows_run_verified.jsonl: each record's `model` field (the extractor's call that wrote decisions[])"},"note":"a model's judgement from the paper's own tables that they report on this leaderboard, not a result: no value from a paper that was not placed is printed or carried here; papers counts each paper once over these leaderboards","backfill":"8,886 of 18,637 newer papers checked so far (733 more had nothing to place; 41 could not be read)","backfill_complete":false,"leaderboards":[]},"papers_where_code_ran":7,"papers_where_code_ran_first_page":"/task/legal-reasoning/papers/ran/1","papers_where_code_ran_caption":"We ran code from the paper's repository; we did not run it on this task or check it against the task's benchmarks.","pages":[{"order":"ran","page":1,"of":1,"url":"/task/legal-reasoning/papers/ran/1","json":"/task/legal-reasoning/papers/ran/1.json","rows":7}],"description_links":{"kept":0,"unwrapped_to_text":0,"bare_urls_linked":0,"relative_images_dropped":0,"rule":"internal links are kept only when the target slug exists in the catalog"},"syntology":{"read_at":"2026-10-05T00:36:11+00:00","claim":"Per-sample execution status on synthesized fixtures ('ran N of M samples'); not a correctness claim and not a ranking signal.","status_vocabulary":{"ran_honours":"ran, honoured the contract we drafted","ran_violates":"ran, violated the contract we drafted","ran_draft_wrong":"ran; our contract draft was wrong, not the code","ran_fixture":"ran; the contract check could not run (RAISES, DEP_MISSING or TIMEOUT)","ran":"ran on a synthesized input","unverified":"unverified (harvested, no recorded run)"},"row_route":"each syntology object on a benchmark's first row and on a paper carries sample_list (this site's paper page, #ran), syntology_url (syntology.ai's page for the paper, only when that route probed live) and mcp (the agent call that lists the paper's harvested samples; not a verdict on the paper)","developers":"https://syntology.ai/developers"},"not_shown":{"libraries":"the archive has no per-task library table","trend_sparklines":"the Trend column of the benchmarks table was a rendered image; it is not in the archive","social_and_latest_sorts":"stars and social signals are not in the archive"},"record_sha256":"d0513954b8892c0ab66c1f5ae8bc135785fc04795b870741c004d68a9bb7c63a","record_changed_at":"2026-10-05","record_changed_at_basis":"changed"}