{"about":{"non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","site":"https://codewithpapers.app","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page","syntology":{"site":"https://syntology.ai","developers":"https://syntology.ai/developers","mcp":{"server":"https://syntology.ai/mcp","transport":"streamable-http","server_card":"https://syntology.ai/.well-known/mcp/server-card.json","auth":{"type":"trial token, no account","trial_token":"https://syntology.ai/api/oauth/trial/token","method":"POST","docs":"https://syntology.ai/developers"}},"have":"https://syntology.ai/api/graph/have?x=<method, arXiv id or title> (free, answers coverage only)","paper_base":"https://syntology.ai/paper/","atlas_base":"https://app.syntology.ai/?focus="},"machine_readable":[{"url":"https://codewithpapers.app/llms.txt","what":"the machine catalog: every machine-readable file, counted"},{"url":"https://codewithpapers.app/index/manifest.json","what":"paper-to-code index by arXiv id, with Syntology's counts"},{"url":"https://codewithpapers.app/search/manifest.json","what":"site search index (titles, authors) and its files"},{"url":"https://codewithpapers.app/download","what":"bulk files: Syntology's layer, described there"},{"url":"https://codewithpapers.app/build_manifest.json","what":"the build record: inputs, counts, exclusions, probes"}]},"url":"/task/logical-reasoning","name":"Logical Reasoning","slug":"logical-reasoning","description_markdown":null,"categories":[{"name":"Methodology","url":"/area/methodology"},{"name":"Miscellaneous","url":"/area/miscellaneous"},{"name":"Reasoning","url":"/area/reasoning"}],"source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","slug_source":"archive_url"},"counts":{"papers_tagged":747,"papers_with_code":330,"benchmarks":10,"benchmark_tables_in_archive":10,"benchmark_tables_shown":10,"benchmark_tables_withheld_as_spam":0,"benchmark_definition":"a leaderboard table with at least one row; benchmark_tables_shown also counts the zero-row tables; benchmark_tables_in_archive adds the tables withheld as spam","datasets":18,"subtasks":22,"parent_tasks":0},"benchmarks":[{"leaderboard":"/sota/logical-reasoning-on-lingoly","slug":"logical-reasoning-on-lingoly","dataset":"LingOly","dataset_url":"/dataset/lingoly","rows_in_archive":11,"metrics":["Delta_NoContext","Exact Match Accuracy"],"first_row_in_archive_order":{"model":"Claude Opus","paper_title":"LINGOLY: A Benchmark of Olympiad-Level Linguistic Reasoning Puzzles in Low-Resource and Extinct Languages","paper_url":"/paper/lingoly-a-benchmark-of-olympiad-level","paper_date":"2024-06-10","arxiv_id":"2406.06196","code_links":[{"title":"am-bean/lingOly","url":"https://github.com/am-bean/lingOly"}],"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":1,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":1,"n_no_contract":0,"n_pointer_only":2,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 1 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/lingoly-a-benchmark-of-olympiad-level#ran","syntology_url":"https://syntology.ai/paper/2406.06196","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.06196"}},"official":{"repos":["am-bean/lingOly"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}}},{"leaderboard":"/sota/logical-reasoning-on-big-bench-formal","slug":"logical-reasoning-on-big-bench-formal","dataset":"BIG-bench (Formal Fallacies Syllogisms Negation)","dataset_url":"/dataset/big-bench","rows_in_archive":9,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"PaLM 2 (few-shot, k=3, Direct)","paper_title":"PaLM 2 Technical Report","paper_url":"/paper/palm-2-technical-report-1","paper_date":"2023-05-17","arxiv_id":"2305.10403","code_links":[{"title":"eternityyw/tram-benchmark","url":"https://github.com/eternityyw/tram-benchmark"}],"syntology":null}},{"leaderboard":"/sota/logical-reasoning-on-big-bench-penguins-in-a","slug":"logical-reasoning-on-big-bench-penguins-in-a","dataset":"BIG-bench (Penguins In A Table)","dataset_url":"/dataset/big-bench","rows_in_archive":9,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"PaLM 2 (few-shot, k=3, CoT)","paper_title":"PaLM 2 Technical Report","paper_url":"/paper/palm-2-technical-report-1","paper_date":"2023-05-17","arxiv_id":"2305.10403","code_links":[{"title":"eternityyw/tram-benchmark","url":"https://github.com/eternityyw/tram-benchmark"}],"syntology":null}},{"leaderboard":"/sota/logical-reasoning-on-big-bench-reasoning","slug":"logical-reasoning-on-big-bench-reasoning","dataset":"BIG-bench (Reasoning About Colored Objects)","dataset_url":"/dataset/big-bench","rows_in_archive":9,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"PaLM 2 (few-shot, k=3, CoT)","paper_title":"PaLM 2 Technical Report","paper_url":"/paper/palm-2-technical-report-1","paper_date":"2023-05-17","arxiv_id":"2305.10403","code_links":[{"title":"eternityyw/tram-benchmark","url":"https://github.com/eternityyw/tram-benchmark"}],"syntology":null}},{"leaderboard":"/sota/logical-reasoning-on-big-bench-temporal","slug":"logical-reasoning-on-big-bench-temporal","dataset":"BIG-bench (Temporal Sequences)","dataset_url":"/dataset/big-bench","rows_in_archive":9,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"PaLM 2 (few-shot, k=3, CoT)","paper_title":"PaLM 2 Technical Report","paper_url":"/paper/palm-2-technical-report-1","paper_date":"2023-05-17","arxiv_id":"2305.10403","code_links":[{"title":"eternityyw/tram-benchmark","url":"https://github.com/eternityyw/tram-benchmark"}],"syntology":null}},{"leaderboard":"/sota/logical-reasoning-on-big-bench-logic-grid","slug":"logical-reasoning-on-big-bench-logic-grid","dataset":"BIG-bench (Logic Grid Puzzle)","dataset_url":"/dataset/big-bench","rows_in_archive":4,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"Chinchilla-70B (few-shot, k=5)","paper_title":"Training Compute-Optimal Large Language Models","paper_url":"/paper/training-compute-optimal-large-language","paper_date":"2022-03-29","arxiv_id":"2203.15556","code_links":[{"title":"karpathy/llama2.c","url":"https://github.com/karpathy/llama2.c"},{"title":"nkluge-correa/teenytinyllama","url":"https://github.com/nkluge-correa/teenytinyllama"}],"syntology":{"n":11,"n_ran":8,"n_constructed":3,"n_ran_checked":5,"n_instrument":3,"n_unverified":3,"n_honours":2,"n_violates":0,"n_no_contract":3,"n_pointer_only":4,"phrase":"8 ran (of which 3 constructed an object rather than computing a result; 5 with no instrument failure: 2 honoured, 0 violated, 3 with no contract checked; 3 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/training-compute-optimal-large-language#ran","syntology_url":"https://syntology.ai/paper/2203.15556","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2203.15556"}},"official":null}}},{"leaderboard":"/sota/logical-reasoning-on-big-bench-strategyqa","slug":"logical-reasoning-on-big-bench-strategyqa","dataset":"BIG-bench (StrategyQA)","dataset_url":"/dataset/big-bench","rows_in_archive":4,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"PaLM-540B (few-shot, k=5)","paper_title":"PaLM: Scaling Language Modeling with Pathways","paper_url":"/paper/palm-scaling-language-modeling-with-pathways-1","paper_date":"2022-04-05","arxiv_id":"2204.02311","code_links":[{"title":"lucidrains/CoCa-pytorch","url":"https://github.com/lucidrains/CoCa-pytorch"},{"title":"lucidrains/PaLM-pytorch","url":"https://github.com/lucidrains/PaLM-pytorch"},{"title":"google/paxml","url":"https://github.com/google/paxml"},{"title":"foundation-model-stack/fms-fsdp","url":"https://github.com/foundation-model-stack/fms-fsdp"},{"title":"lucidrains/PaLM-jax","url":"https://github.com/lucidrains/PaLM-jax"},{"title":"chrisociepa/allamo","url":"https://github.com/chrisociepa/allamo"},{"title":"conceptofmind/PaLM-flax","url":"https://github.com/conceptofmind/PaLM-flax"}],"syntology":{"n":37,"n_ran":32,"n_constructed":16,"n_ran_checked":24,"n_instrument":8,"n_unverified":5,"n_honours":2,"n_violates":1,"n_no_contract":21,"n_pointer_only":0,"phrase":"32 ran (of which 16 constructed an object rather than computing a result; 24 with no instrument failure: 2 honoured, 1 violated, 21 with no contract checked; 8 where Syntology's instrument failed) · 5 unverified","sample_list":"/paper/palm-scaling-language-modeling-with-pathways-1#ran","syntology_url":"https://syntology.ai/paper/2204.02311","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2204.02311"}},"official":null}}},{"leaderboard":"/sota/logical-reasoning-on-ruworldtree","slug":"logical-reasoning-on-ruworldtree","dataset":"RuWorldTree","dataset_url":"/dataset/ruworldtree","rows_in_archive":4,"metrics":["Accuracy "],"first_row_in_archive_order":{"model":"Human benchmark","paper_title":"TAPE: Assessing Few-shot Russian Language Understanding","paper_url":"/paper/tape-assessing-few-shot-russian-language","paper_date":"2022-10-23","arxiv_id":"2210.12813","code_links":[{"title":"RussianNLP/TAPE","url":"https://github.com/RussianNLP/TAPE"}],"syntology":null}},{"leaderboard":"/sota/logical-reasoning-on-winograd-automatic","slug":"logical-reasoning-on-winograd-automatic","dataset":"Winograd Automatic","dataset_url":"/dataset/winograd-automatic","rows_in_archive":4,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"Human benchmark","paper_title":"TAPE: Assessing Few-shot Russian Language Understanding","paper_url":"/paper/tape-assessing-few-shot-russian-language","paper_date":"2022-10-23","arxiv_id":"2210.12813","code_links":[{"title":"RussianNLP/TAPE","url":"https://github.com/RussianNLP/TAPE"}],"syntology":null}},{"leaderboard":"/sota/logical-reasoning-on-big-bench-logical","slug":"logical-reasoning-on-big-bench-logical","dataset":"BIG-bench (Logical Fallacy Detection)","dataset_url":"/dataset/big-bench","rows_in_archive":2,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"Chinchilla-70B (few-shot, k=5)","paper_title":"Training Compute-Optimal Large Language Models","paper_url":"/paper/training-compute-optimal-large-language","paper_date":"2022-03-29","arxiv_id":"2203.15556","code_links":[{"title":"karpathy/llama2.c","url":"https://github.com/karpathy/llama2.c"},{"title":"nkluge-correa/teenytinyllama","url":"https://github.com/nkluge-correa/teenytinyllama"}],"syntology":{"n":11,"n_ran":8,"n_constructed":3,"n_ran_checked":5,"n_instrument":3,"n_unverified":3,"n_honours":2,"n_violates":0,"n_no_contract":3,"n_pointer_only":4,"phrase":"8 ran (of which 3 constructed an object rather than computing a result; 5 with no instrument failure: 2 honoured, 0 violated, 3 with no contract checked; 3 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/training-compute-optimal-large-language#ran","syntology_url":"https://syntology.ai/paper/2203.15556","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2203.15556"}},"official":null}}}],"datasets":[{"url":"/dataset/big-bench","name":"BIG-bench","full_name":"Beyond the Imitation Game Benchmark","num_papers_in_archive":349},{"url":"/dataset/gsm-plus","name":"GSM-Plus","full_name":"","num_papers_in_archive":17},{"url":"/dataset/smart-101","name":"SMART-101","full_name":"Simple Multimodal Algorithmic Reasoning Task Dataset","num_papers_in_archive":6},{"url":"/dataset/fld","name":"FLD","full_name":"Formal Logic Deduction","num_papers_in_archive":5},{"url":"/dataset/lingoly","name":"LingOly","full_name":"","num_papers_in_archive":4},{"url":"/dataset/multiq","name":"MultiQ","full_name":"MultiQ","num_papers_in_archive":3},{"url":"/dataset/chegeka","name":"CheGeKa","full_name":"CheGeKa","num_papers_in_archive":2},{"url":"/dataset/ethics-2","name":"Ethics (per ethics)","full_name":"","num_papers_in_archive":2},{"url":"/dataset/qure","name":"QuRe","full_name":"","num_papers_in_archive":2},{"url":"/dataset/ruopenbookqa","name":"RuOpenBookQA","full_name":"RuOpenBookQA","num_papers_in_archive":2},{"url":"/dataset/ruworldtree","name":"RuWorldTree","full_name":"RuWorldTree","num_papers_in_archive":2},{"url":"/dataset/truthquest","name":"TruthQuest","full_name":"","num_papers_in_archive":2},{"url":"/dataset/inferential-strategies","name":"Inferential-Strategies","full_name":"","num_papers_in_archive":1},{"url":"/dataset/justlogic","name":"JustLogic","full_name":"","num_papers_in_archive":1},{"url":"/dataset/oxford-ontology-library","name":"Oxford Ontology Library","full_name":"","num_papers_in_archive":1},{"url":"/dataset/polymath","name":"PolyMATH","full_name":"","num_papers_in_archive":1},{"url":"/dataset/rebus","name":"REBUS","full_name":"A Robust Evaluation Benchmark of Understanding Symbols","num_papers_in_archive":1},{"url":"/dataset/winograd-automatic","name":"Winograd Automatic","full_name":"Winograd","num_papers_in_archive":1}],"subtasks":[{"url":"/task/analytic-entailment","name":"Analytic Entailment"},{"url":"/task/checkmate-in-one","name":"Checkmate In One"},{"url":"/task/code-line-descriptions","name":"Code Line Descriptions"},{"url":"/task/college-mathematics","name":"College Mathematics"},{"url":"/task/date-understanding","name":"Date Understanding"},{"url":"/task/elementary-mathematics","name":"Elementary Mathematics"},{"url":"/task/entailed-polarity","name":"Entailed Polarity"},{"url":"/task/epistemic-reasoning","name":"Epistemic Reasoning"},{"url":"/task/evaluating-information-essentiality","name":"Evaluating Information Essentiality"},{"url":"/task/logic-grid-puzzle","name":"Logic Grid Puzzle"},{"url":"/task/logical-args","name":"Logical Args"},{"url":"/task/logical-fallacy-detection","name":"Logical Fallacy Detection"},{"url":"/task/logical-sequence","name":"Logical Sequence"},{"url":"/task/metaphor-boolean","name":"Metaphor Boolean"},{"url":"/task/navigate","name":"Navigate"},{"url":"/task/novel-concepts","name":"Novel Concepts"},{"url":"/task/penguins-in-a-table","name":"Penguins In A Table"},{"url":"/task/physical-intuition","name":"Physical Intuition"},{"url":"/task/presuppositions-as-nli","name":"Presuppositions As NLI"},{"url":"/task/reasoning-about-colored-objects","name":"Reasoning About Colored Objects"},{"url":"/task/strategyqa","name":"StrategyQA"},{"url":"/task/temporal-sequences","name":"Temporal Sequences"}],"parent_tasks":[],"papers":{"order":"repositories listed in the archive (most first), then date (newest first), then slug; the archive holds no stars","population":"every paper the archive tags with this task (distinct papers; spam titles not listed). 'of' counts the papers listed; until 2026-09-25 it counted only the papers with a code link, and that number is of_with_code","page":1,"pages_in_order":8,"rows_per_page":100,"shown":100,"of":747,"of_with_code":330,"tagged_in_all":747,"counts":{"archive_papers_tagged":747,"with_a_code_link":330,"where_syntology_ran_a_sample":113,"not_listed_spam_title":0,"listed":747,"listed_where_code_ran":113,"where_syntology_ran_a_sample_split":{"with_a_run_with_no_instrument_failure":91,"every_run_a_failure_of_syntologys_instrument":22,"listed_with_a_run_with_no_instrument_failure":91,"listed_every_run_a_failure_of_syntologys_instrument":22,"filter":{"states":["a run with no instrument failure","any run, instrument failures included"],"default":"a run with no instrument failure","note":"on the 'only where code ran' pages the default hides, in the browser, the rows where every run was a failure of Syntology's instrument; the second state shows them again. Rows are hidden, never re-ordered; these twins list every row"}},"definition":"distinct papers the archive tags; 'where Syntology ran a sample' counts papers with at least one harvested sample that ran, which is not a correctness claim"},"items":[{"url":"/paper/beta-embeddings-for-multi-hop-logical","slug":"beta-embeddings-for-multi-hop-logical","title":"Beta Embeddings for Multi-Hop Logical Reasoning in Knowledge Graphs","date":"2020-10-22","arxiv_id":"2010.11465","repositories_listed":9,"syntology":{"n":14,"n_ran":9,"n_constructed":0,"n_ran_checked":6,"n_instrument":3,"n_unverified":5,"n_honours":2,"n_violates":0,"n_no_contract":4,"n_pointer_only":2,"phrase":"9 ran (of which 0 constructed an object rather than computing a result; 6 with no instrument failure: 2 honoured, 0 violated, 4 with no contract checked; 3 where Syntology's instrument failed) · 5 unverified","sample_list":"/paper/beta-embeddings-for-multi-hop-logical#ran","syntology_url":"https://syntology.ai/paper/2010.11465","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2010.11465"}},"official":null}},{"url":"/paper/palm-scaling-language-modeling-with-pathways-1","slug":"palm-scaling-language-modeling-with-pathways-1","title":"PaLM: Scaling Language Modeling with Pathways","date":"2022-04-05","arxiv_id":"2204.02311","repositories_listed":7,"syntology":{"n":37,"n_ran":32,"n_constructed":16,"n_ran_checked":24,"n_instrument":8,"n_unverified":5,"n_honours":2,"n_violates":1,"n_no_contract":21,"n_pointer_only":0,"phrase":"32 ran (of which 16 constructed an object rather than computing a result; 24 with no instrument failure: 2 honoured, 1 violated, 21 with no contract checked; 8 where Syntology's instrument failed) · 5 unverified","sample_list":"/paper/palm-scaling-language-modeling-with-pathways-1#ran","syntology_url":"https://syntology.ai/paper/2204.02311","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2204.02311"}},"official":null}},{"url":"/paper/mmmu-a-massive-multi-discipline-multimodal","slug":"mmmu-a-massive-multi-discipline-multimodal","title":"MMMU: A Massive Multi-discipline Multimodal Understanding and Reasoning Benchmark for Expert AGI","date":"2023-11-27","arxiv_id":"2311.16502","repositories_listed":5,"syntology":{"n":14,"n_ran":13,"n_constructed":0,"n_ran_checked":8,"n_instrument":5,"n_unverified":1,"n_honours":1,"n_violates":0,"n_no_contract":7,"n_pointer_only":2,"phrase":"13 ran (of which 0 constructed an object rather than computing a result; 8 with no instrument failure: 1 honoured, 0 violated, 7 with no contract checked; 5 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/mmmu-a-massive-multi-discipline-multimodal#ran","syntology_url":"https://syntology.ai/paper/2311.16502","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2311.16502"}},"official":{"repos":["MMMU-Benchmark/MMMU"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["listed","official","unlocated"]}}},{"url":"/paper/scaling-synthetic-data-creation-with","slug":"scaling-synthetic-data-creation-with","title":"Scaling Synthetic Data Creation with 1,000,000,000 Personas","date":"2024-06-28","arxiv_id":"2406.20094","repositories_listed":4,"syntology":{"n":4,"n_ran":4,"n_constructed":0,"n_ran_checked":3,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":1,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/scaling-synthetic-data-creation-with#ran","syntology_url":"https://syntology.ai/paper/2406.20094","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.20094"}},"official":{"repos":["tencent-ailab/persona-hub"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/large-language-models-are-zero-shot-reasoners","slug":"large-language-models-are-zero-shot-reasoners","title":"Large Language Models are Zero-Shot Reasoners","date":"2022-05-24","arxiv_id":"2205.11916","repositories_listed":4,"syntology":{"n":4,"n_ran":3,"n_constructed":0,"n_ran_checked":3,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":1,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/large-language-models-are-zero-shot-reasoners#ran","syntology_url":"https://syntology.ai/paper/2205.11916","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2205.11916"}},"official":{"repos":["kojima-takeshi188/zero_shot_cot"],"state":"official: harvested, nothing ran","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":1,"ran_from_kinds":["listed"]}}},{"url":"/paper/hunyuan-large-an-open-source-moe-model-with","slug":"hunyuan-large-an-open-source-moe-model-with","title":"Hunyuan-Large: An Open-Source MoE Model with 52 Billion Activated Parameters by Tencent","date":"2024-11-04","arxiv_id":"2411.02265","repositories_listed":3,"syntology":null},{"url":"/paper/knowledge-fusion-of-large-language-models","slug":"knowledge-fusion-of-large-language-models","title":"Knowledge Fusion of Large Language Models","date":"2024-01-19","arxiv_id":"2401.10491","repositories_listed":3,"syntology":{"n":4,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":3,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":4,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/knowledge-fusion-of-large-language-models#ran","syntology_url":"https://syntology.ai/paper/2401.10491","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2401.10491"}},"official":{"repos":["fanqiwan/fusellm"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/learning-deductive-reasoning-from-synthetic","slug":"learning-deductive-reasoning-from-synthetic","title":"Learning Deductive Reasoning from Synthetic Corpus based on Formal Logic","date":"2023-08-11","arxiv_id":"2308.07336","repositories_listed":3,"syntology":{"n":2,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/learning-deductive-reasoning-from-synthetic#ran","syntology_url":"https://syntology.ai/paper/2308.07336","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2308.07336"}},"official":{"repos":["hitachi-nlp/fld"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/metalogic-logical-reasoning-explanations-with","slug":"metalogic-logical-reasoning-explanations-with","title":"MetaLogic: Logical Reasoning Explanations with Fine-Grained Structure","date":"2022-10-22","arxiv_id":"2210.12487","repositories_listed":3,"syntology":null},{"url":"/paper/scaling-language-models-methods-analysis-1","slug":"scaling-language-models-methods-analysis-1","title":"Scaling Language Models: Methods, Analysis & Insights from Training Gopher","date":"2021-12-08","arxiv_id":"2112.11446","repositories_listed":3,"syntology":null},{"url":"/paper/neural-logic-reasoning","slug":"neural-logic-reasoning","title":"Neural Logic Reasoning","date":"2020-08-20","arxiv_id":"2008.09514","repositories_listed":3,"syntology":{"n":3,"n_ran":2,"n_constructed":0,"n_ran_checked":2,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/neural-logic-reasoning#ran","syntology_url":"https://syntology.ai/paper/2008.09514","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2008.09514"}},"official":{"repos":["rutgerswiselab/NLR"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/neural-collaborative-reasoning","slug":"neural-collaborative-reasoning","title":"Neural Collaborative Reasoning","date":"2020-05-16","arxiv_id":"2005.08129","repositories_listed":3,"syntology":null},{"url":"/paper/satnet-bridging-deep-learning-and-logical","slug":"satnet-bridging-deep-learning-and-logical","title":"SATNet: Bridging deep learning and logical reasoning using a differentiable satisfiability solver","date":"2019-05-29","arxiv_id":"1905.12149","repositories_listed":3,"syntology":{"n":5,"n_ran":4,"n_constructed":0,"n_ran_checked":2,"n_instrument":2,"n_unverified":1,"n_honours":1,"n_violates":1,"n_no_contract":0,"n_pointer_only":0,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 1 honoured, 1 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/satnet-bridging-deep-learning-and-logical#ran","syntology_url":"https://syntology.ai/paper/1905.12149","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1905.12149"}},"official":{"repos":["Kyubyong/sudoku","locuslab/SATNet"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/a-dataset-and-architecture-for-visual","slug":"a-dataset-and-architecture-for-visual","title":"A Dataset and Architecture for Visual Reasoning with a Working Memory","date":"2018-03-16","arxiv_id":"1803.06092","repositories_listed":3,"syntology":{"n":5,"n_ran":5,"n_constructed":0,"n_ran_checked":0,"n_instrument":5,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 5 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/a-dataset-and-architecture-for-visual#ran","syntology_url":"https://syntology.ai/paper/1803.06092","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1803.06092"}},"official":{"repos":["google/cog"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/infigui-r1-advancing-multimodal-gui-agents","slug":"infigui-r1-advancing-multimodal-gui-agents","title":"InfiGUI-R1: Advancing Multimodal GUI Agents from Reactive Actors to Deliberative Reasoners","date":"2025-04-19","arxiv_id":"2504.14239","repositories_listed":2,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":3,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/infigui-r1-advancing-multimodal-gui-agents#ran","syntology_url":"https://syntology.ai/paper/2504.14239","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2504.14239"}},"official":{"repos":["reallm-labs/infigui-r1"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/textgames-learning-to-self-play-text-based","slug":"textgames-learning-to-self-play-text-based","title":"TextGames: Learning to Self-Play Text-Based Puzzle Games via Language Model Reasoning","date":"2025-02-25","arxiv_id":"2502.18431","repositories_listed":2,"syntology":{"n":3,"n_ran":2,"n_constructed":0,"n_ran_checked":0,"n_instrument":2,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":3,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/textgames-learning-to-self-play-text-based#ran","syntology_url":"https://syntology.ai/paper/2502.18431","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2502.18431"}},"official":{"repos":["fhudi/textgames"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/enhancing-reasoning-capabilities-of-llms-via","slug":"enhancing-reasoning-capabilities-of-llms-via","title":"Enhancing Reasoning Capabilities of LLMs via Principled Synthetic Logic Corpus","date":"2024-11-19","arxiv_id":"2411.12498","repositories_listed":2,"syntology":null},{"url":"/paper/llava-o1-let-vision-language-models-reason","slug":"llava-o1-let-vision-language-models-reason","title":"LLaVA-CoT: Let Vision Language Models Reason Step-by-Step","date":"2024-11-15","arxiv_id":"2411.10440","repositories_listed":2,"syntology":{"n":6,"n_ran":6,"n_constructed":0,"n_ran_checked":4,"n_instrument":2,"n_unverified":0,"n_honours":0,"n_violates":1,"n_no_contract":3,"n_pointer_only":0,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 4 with no instrument failure: 0 honoured, 1 violated, 3 with no contract checked; 2 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/llava-o1-let-vision-language-models-reason#ran","syntology_url":"https://syntology.ai/paper/2411.10440","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2411.10440"}},"official":{"repos":["PKU-YuanGroup/LLaVA-CoT"],"state":"official (archive's flag): 6 ran","n_ran":6,"n_constructed":0,"n_ran_no_instrument_failure":4,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/gsm-symbolic-understanding-the-limitations-of","slug":"gsm-symbolic-understanding-the-limitations-of","title":"GSM-Symbolic: Understanding the Limitations of Mathematical Reasoning in Large Language Models","date":"2024-10-07","arxiv_id":"2410.05229","repositories_listed":2,"syntology":null},{"url":"/paper/needlebench-can-llms-do-retrieval-and","slug":"needlebench-can-llms-do-retrieval-and","title":"NeedleBench: Can LLMs Do Retrieval and Reasoning in Information-Dense Context?","date":"2024-07-16","arxiv_id":"2407.11963","repositories_listed":2,"syntology":null},{"url":"/paper/gtbench-uncovering-the-strategic-reasoning","slug":"gtbench-uncovering-the-strategic-reasoning","title":"GTBench: Uncovering the Strategic Reasoning Limitations of LLMs via Game-Theoretic Evaluations","date":"2024-02-19","arxiv_id":"2402.12348","repositories_listed":2,"syntology":null},{"url":"/paper/efficiently-programming-large-language-models","slug":"efficiently-programming-large-language-models","title":"SGLang: Efficient Execution of Structured Language Model Programs","date":"2023-12-12","arxiv_id":"2312.07104","repositories_listed":2,"syntology":{"n":17,"n_ran":16,"n_constructed":0,"n_ran_checked":9,"n_instrument":7,"n_unverified":1,"n_honours":3,"n_violates":1,"n_no_contract":5,"n_pointer_only":0,"phrase":"16 ran (of which 0 constructed an object rather than computing a result; 9 with no instrument failure: 3 honoured, 1 violated, 5 with no contract checked; 7 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/efficiently-programming-large-language-models#ran","syntology_url":"https://syntology.ai/paper/2312.07104","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2312.07104"}},"official":{"repos":["sgl-project/sglang"],"state":"official (archive's flag): 7 ran","n_ran":7,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/generating-by-understanding-neural-visual","slug":"generating-by-understanding-neural-visual","title":"Generating by Understanding: Neural Visual Generation with Logical Symbol Groundings","date":"2023-10-26","arxiv_id":"2310.17451","repositories_listed":2,"syntology":null},{"url":"/paper/logicllm-exploring-self-supervised-logic","slug":"logicllm-exploring-self-supervised-logic","title":"Exploring Self-supervised Logic-enhanced Training for Large Language Models","date":"2023-05-23","arxiv_id":"2305.13718","repositories_listed":2,"syntology":{"n":8,"n_ran":6,"n_constructed":0,"n_ran_checked":5,"n_instrument":1,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":5,"n_pointer_only":0,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 0 honoured, 0 violated, 5 with no contract checked; 1 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/logicllm-exploring-self-supervised-logic#ran","syntology_url":"https://syntology.ai/paper/2305.13718","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2305.13718"}},"official":{"repos":["sparkjiao/logicllm","sparkjiao/merit-v2"],"state":"official (archive's flag): 6 ran","n_ran":6,"n_constructed":0,"n_ran_no_instrument_failure":5,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/polytuplet-loss-a-reverse-approach-to","slug":"polytuplet-loss-a-reverse-approach-to","title":"Deep Manifold Learning for Reading Comprehension and Logical Reasoning Tasks with Polytuplet Loss","date":"2023-04-03","arxiv_id":"2304.01046","repositories_listed":2,"syntology":null},{"url":"/paper/bloomberggpt-a-large-language-model-for","slug":"bloomberggpt-a-large-language-model-for","title":"BloombergGPT: A Large Language Model for Finance","date":"2023-03-30","arxiv_id":"2303.17564","repositories_listed":2,"syntology":null},{"url":"/paper/explicit-planning-helps-language-models-in","slug":"explicit-planning-helps-language-models-in","title":"Explicit Planning Helps Language Models in Logical Reasoning","date":"2023-03-28","arxiv_id":"2303.15714","repositories_listed":2,"syntology":{"n":1,"n_ran":1,"n_constructed":1,"n_ran_checked":1,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"1 ran (of which 1 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified; the one sample that ran constructed an object rather than computing a result","sample_list":"/paper/explicit-planning-helps-language-models-in#ran","syntology_url":"https://syntology.ai/paper/2303.15714","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2303.15714"}},"official":{"repos":["cindermond/explicit-planning-for-reasoning","cindermond/leap"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":1,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/reasoning-with-language-model-prompting-a","slug":"reasoning-with-language-model-prompting-a","title":"Reasoning with Language Model Prompting: A Survey","date":"2022-12-19","arxiv_id":"2212.09597","repositories_listed":2,"syntology":null},{"url":"/paper/unigeo-unifying-geometry-logical-reasoning","slug":"unigeo-unifying-geometry-logical-reasoning","title":"UniGeo: Unifying Geometry Logical Reasoning via Reformulating Mathematical Expression","date":"2022-12-06","arxiv_id":"2212.02746","repositories_listed":2,"syntology":{"n":4,"n_ran":2,"n_constructed":2,"n_ran_checked":2,"n_instrument":0,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":4,"phrase":"2 ran (of which 2 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 0 where Syntology's instrument failed) · 2 unverified; every one of the 2 samples that ran constructed an object rather than computing a result","sample_list":"/paper/unigeo-unifying-geometry-logical-reasoning#ran","syntology_url":"https://syntology.ai/paper/2212.02746","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2212.02746"}},"official":{"repos":["chen-judge/unigeo"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":2,"n_ran_no_instrument_failure":2,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/investigating-the-robustness-of-natural","slug":"investigating-the-robustness-of-natural","title":"Investigating the Robustness of Natural Language Generation from Logical Forms via Counterfactual Samples","date":"2022-10-16","arxiv_id":"2210.08548","repositories_listed":2,"syntology":null},{"url":"/paper/dynamic-prompt-learning-via-policy-gradient","slug":"dynamic-prompt-learning-via-policy-gradient","title":"Dynamic Prompt Learning via Policy Gradient for Semi-structured Mathematical Reasoning","date":"2022-09-29","arxiv_id":"2209.14610","repositories_listed":2,"syntology":{"n":7,"n_ran":3,"n_constructed":0,"n_ran_checked":1,"n_instrument":2,"n_unverified":4,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 4 unverified","sample_list":"/paper/dynamic-prompt-learning-via-policy-gradient#ran","syntology_url":"https://syntology.ai/paper/2209.14610","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2209.14610"}},"official":null}},{"url":"/paper/training-compute-optimal-large-language","slug":"training-compute-optimal-large-language","title":"Training Compute-Optimal Large Language Models","date":"2022-03-29","arxiv_id":"2203.15556","repositories_listed":2,"syntology":{"n":11,"n_ran":8,"n_constructed":3,"n_ran_checked":5,"n_instrument":3,"n_unverified":3,"n_honours":2,"n_violates":0,"n_no_contract":3,"n_pointer_only":4,"phrase":"8 ran (of which 3 constructed an object rather than computing a result; 5 with no instrument failure: 2 honoured, 0 violated, 3 with no contract checked; 3 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/training-compute-optimal-large-language#ran","syntology_url":"https://syntology.ai/paper/2203.15556","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2203.15556"}},"official":null}},{"url":"/paper/a-neural-symbolic-approach-to-natural","slug":"a-neural-symbolic-approach-to-natural","title":"A Neural-Symbolic Approach to Natural Language Understanding","date":"2022-03-20","arxiv_id":"2203.10557","repositories_listed":2,"syntology":null},{"url":"/paper/a-neuro-vector-symbolic-architecture-for","slug":"a-neuro-vector-symbolic-architecture-for","title":"A Neuro-vector-symbolic Architecture for Solving Raven's Progressive Matrices","date":"2022-03-09","arxiv_id":"2203.04571","repositories_listed":2,"syntology":{"n":9,"n_ran":9,"n_constructed":0,"n_ran_checked":7,"n_instrument":2,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":7,"n_pointer_only":0,"phrase":"9 ran (of which 0 constructed an object rather than computing a result; 7 with no instrument failure: 0 honoured, 0 violated, 7 with no contract checked; 2 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/a-neuro-vector-symbolic-architecture-for#ran","syntology_url":"https://syntology.ai/paper/2203.04571","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2203.04571"}},"official":{"repos":["ibm/neuro-vector-symbolic-architectures"],"state":"official (archive's flag): 9 ran","n_ran":9,"n_constructed":0,"n_ran_no_instrument_failure":7,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/quantifying-adaptability-in-pre-trained","slug":"quantifying-adaptability-in-pre-trained","title":"Quantifying Adaptability in Pre-trained Language Models with 500 Tasks","date":"2021-12-06","arxiv_id":"2112.03204","repositories_listed":2,"syntology":{"n":8,"n_ran":7,"n_constructed":0,"n_ran_checked":6,"n_instrument":1,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":6,"n_pointer_only":1,"phrase":"7 ran (of which 0 constructed an object rather than computing a result; 6 with no instrument failure: 0 honoured, 0 violated, 6 with no contract checked; 1 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/quantifying-adaptability-in-pre-trained#ran","syntology_url":"https://syntology.ai/paper/2112.03204","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2112.03204"}},"official":{"repos":["belindal/taskbench500","facebookresearch/task_bench"],"state":"official (archive's flag): 7 ran","n_ran":7,"n_constructed":0,"n_ran_no_instrument_failure":6,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/conditionalqa-a-complex-reading-comprehension","slug":"conditionalqa-a-complex-reading-comprehension","title":"ConditionalQA: A Complex Reading Comprehension Dataset with Conditional Answers","date":"2021-10-13","arxiv_id":"2110.06884","repositories_listed":2,"syntology":{"n":3,"n_ran":2,"n_constructed":0,"n_ran_checked":1,"n_instrument":1,"n_unverified":1,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":3,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/conditionalqa-a-complex-reading-comprehension#ran","syntology_url":"https://syntology.ai/paper/2110.06884","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2110.06884"}},"official":{"repos":["haitian-sun/conditionalqa"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/discriminative-reasoning-for-document-level","slug":"discriminative-reasoning-for-document-level","title":"Discriminative Reasoning for Document-level Relation Extraction","date":"2021-06-03","arxiv_id":"2106.01562","repositories_listed":2,"syntology":{"n":10,"n_ran":8,"n_constructed":6,"n_ran_checked":8,"n_instrument":0,"n_unverified":2,"n_honours":1,"n_violates":0,"n_no_contract":7,"n_pointer_only":0,"phrase":"8 ran (of which 6 constructed an object rather than computing a result; 8 with no instrument failure: 1 honoured, 0 violated, 7 with no contract checked; 0 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/discriminative-reasoning-for-document-level#ran","syntology_url":"https://syntology.ai/paper/2106.01562","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2106.01562"}},"official":{"repos":["xwjim/DRN"],"state":"official (archive's flag): 8 ran","n_ran":8,"n_constructed":6,"n_ran_no_instrument_failure":8,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/fact-driven-logical-reasoning","slug":"fact-driven-logical-reasoning","title":"Fact-driven Logical Reasoning for Machine Reading Comprehension","date":"2021-05-21","arxiv_id":"2105.10334","repositories_listed":2,"syntology":null},{"url":"/paper/logic-driven-context-extension-and-data","slug":"logic-driven-context-extension-and-data","title":"Logic-Driven Context Extension and Data Augmentation for Logical Reasoning of Text","date":"2021-05-08","arxiv_id":"2105.03659","repositories_listed":2,"syntology":null},{"url":"/paper/dagn-discourse-aware-graph-network-for","slug":"dagn-discourse-aware-graph-network-for","title":"DAGN: Discourse-Aware Graph Network for Logical Reasoning","date":"2021-03-26","arxiv_id":"2103.14349","repositories_listed":2,"syntology":{"n":5,"n_ran":2,"n_constructed":0,"n_ran_checked":2,"n_instrument":0,"n_unverified":3,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":5,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 0 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/dagn-discourse-aware-graph-network-for#ran","syntology_url":"https://syntology.ai/paper/2103.14349","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2103.14349"}},"official":{"repos":["Eleanor-H/DAGN"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":3,"ran_from_kinds":["official"]}}},{"url":"/paper/neural-software-analysis","slug":"neural-software-analysis","title":"Neural Software Analysis","date":"2020-11-16","arxiv_id":"2011.07986","repositories_listed":2,"syntology":null},{"url":"/paper/measuring-systematic-generalization-in-neural","slug":"measuring-systematic-generalization-in-neural","title":"Measuring Systematic Generalization in Neural Proof Generation with Transformers","date":"2020-09-30","arxiv_id":"2009.14786","repositories_listed":2,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/measuring-systematic-generalization-in-neural#ran","syntology_url":"https://syntology.ai/paper/2009.14786","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2009.14786"}},"official":{"repos":["NicolasAG/SGinPG"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/logiqa-a-challenge-dataset-for-machine","slug":"logiqa-a-challenge-dataset-for-machine","title":"LogiQA: A Challenge Dataset for Machine Reading Comprehension with Logical Reasoning","date":"2020-07-16","arxiv_id":"2007.08124","repositories_listed":2,"syntology":null},{"url":"/paper/switchblade-a-neural-network-for-hard-2d","slug":"switchblade-a-neural-network-for-hard-2d","title":"Matrix Shuffle-Exchange Networks for Hard 2D Tasks","date":"2020-06-29","arxiv_id":"2006.15892","repositories_listed":2,"syntology":null},{"url":"/paper/logical-neural-networks","slug":"logical-neural-networks","title":"Logical Neural Networks","date":"2020-06-23","arxiv_id":"2006.13155","repositories_listed":2,"syntology":{"n":15,"n_ran":2,"n_constructed":0,"n_ran_checked":2,"n_instrument":0,"n_unverified":13,"n_honours":0,"n_violates":2,"n_no_contract":0,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 2 violated, 0 with no contract checked; 0 where Syntology's instrument failed) · 13 unverified","sample_list":"/paper/logical-neural-networks#ran","syntology_url":"https://syntology.ai/paper/2006.13155","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2006.13155"}},"official":{"repos":["IBM/LNN"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":12,"ran_from_kinds":["official"]}}},{"url":"/paper/mmm-multi-stage-multi-task-learning-for-multi","slug":"mmm-multi-stage-multi-task-learning-for-multi","title":"MMM: Multi-stage Multi-task Learning for Multi-choice Reading Comprehension","date":"2019-10-01","arxiv_id":"1910.00458","repositories_listed":2,"syntology":null},{"url":"/paper/ontology-reasoning-with-deep-neural-networks","slug":"ontology-reasoning-with-deep-neural-networks","title":"Ontology Reasoning with Deep Neural Networks","date":"2018-08-24","arxiv_id":"1808.07980","repositories_listed":2,"syntology":null},{"url":"/paper/chains-of-reasoning-over-entities-relations","slug":"chains-of-reasoning-over-entities-relations","title":"Chains of Reasoning over Entities, Relations, and Text using Recurrent Neural Networks","date":"2016-07-05","arxiv_id":"1607.01426","repositories_listed":2,"syntology":null},{"url":"/paper/logic-tensor-networks-deep-learning-and","slug":"logic-tensor-networks-deep-learning-and","title":"Logic Tensor Networks: Deep Learning and Logical Reasoning from Data and Knowledge","date":"2016-06-14","arxiv_id":"1606.04422","repositories_listed":2,"syntology":{"n":1,"n_ran":0,"n_constructed":0,"n_ran_checked":0,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"0 ran · 1 unverified","sample_list":"/paper/logic-tensor-networks-deep-learning-and#ran","syntology_url":"https://syntology.ai/paper/1606.04422","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1606.04422"}},"official":null}},{"url":"/paper/soundmind-rl-incentivized-logic-reasoning-for","slug":"soundmind-rl-incentivized-logic-reasoning-for","title":"SoundMind: RL-Incentivized Logic Reasoning for Audio-Language Models","date":"2025-06-15","arxiv_id":"2506.12935","repositories_listed":1,"syntology":{"n":16,"n_ran":10,"n_constructed":0,"n_ran_checked":5,"n_instrument":5,"n_unverified":6,"n_honours":0,"n_violates":1,"n_no_contract":4,"n_pointer_only":4,"phrase":"10 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 0 honoured, 1 violated, 4 with no contract checked; 5 where Syntology's instrument failed) · 6 unverified","sample_list":"/paper/soundmind-rl-incentivized-logic-reasoning-for#ran","syntology_url":"https://syntology.ai/paper/2506.12935","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2506.12935"}},"official":{"repos":["xid32/soundmind"],"state":"official (archive's flag): 10 ran","n_ran":10,"n_constructed":0,"n_ran_no_instrument_failure":5,"n_unverified":6,"ran_from_kinds":["official"]}}},{"url":"/paper/evinet-evidential-reasoning-network-for","slug":"evinet-evidential-reasoning-network-for","title":"EviNet: Evidential Reasoning Network for Resilient Graph Learning in the Open and Noisy Environments","date":"2025-06-08","arxiv_id":"2506.07288","repositories_listed":1,"syntology":null},{"url":"/paper/are-llms-reliable-translators-of-logical","slug":"are-llms-reliable-translators-of-logical","title":"Are LLMs Reliable Translators of Logical Reasoning Across Lexically Diversified Contexts?","date":"2025-06-05","arxiv_id":"2506.04575","repositories_listed":1,"syntology":null},{"url":"/paper/dissecting-logical-reasoning-in-llms-a-fine","slug":"dissecting-logical-reasoning-in-llms-a-fine","title":"Dissecting Logical Reasoning in LLMs: A Fine-Grained Evaluation and Supervision Study","date":"2025-06-05","arxiv_id":"2506.04810","repositories_listed":1,"syntology":null},{"url":"/paper/climate-finance-bench","slug":"climate-finance-bench","title":"Climate Finance Bench","date":"2025-05-28","arxiv_id":"2505.22752","repositories_listed":1,"syntology":null},{"url":"/paper/a-structured-unplugged-approach-for","slug":"a-structured-unplugged-approach-for","title":"A Structured Unplugged Approach for Foundational AI Literacy in Primary Education","date":"2025-05-27","arxiv_id":"2505.21398","repositories_listed":1,"syntology":null},{"url":"/paper/cross-from-left-to-right-brain-adaptive-text","slug":"cross-from-left-to-right-brain-adaptive-text","title":"Cross from Left to Right Brain: Adaptive Text Dreamer for Vision-and-Language Navigation","date":"2025-05-27","arxiv_id":"2505.20897","repositories_listed":1,"syntology":null},{"url":"/paper/sv-trusteval-c-evaluating-structure-and","slug":"sv-trusteval-c-evaluating-structure-and","title":"SV-TrustEval-C: Evaluating Structure and Semantic Reasoning in Large Language Models for Source Code Vulnerability Analysis","date":"2025-05-27","arxiv_id":"2505.20630","repositories_listed":1,"syntology":null},{"url":"/paper/chain-of-thought-for-autonomous-driving-a","slug":"chain-of-thought-for-autonomous-driving-a","title":"Chain-of-Thought for Autonomous Driving: A Comprehensive Survey and Future Prospects","date":"2025-05-26","arxiv_id":"2505.20223","repositories_listed":1,"syntology":null},{"url":"/paper/large-language-models-for-planning-a","slug":"large-language-models-for-planning-a","title":"Large Language Models for Planning: A Comprehensive and Systematic Survey","date":"2025-05-26","arxiv_id":"2505.19683","repositories_listed":1,"syntology":null},{"url":"/paper/surrogate-signals-from-format-and-length","slug":"surrogate-signals-from-format-and-length","title":"Surrogate Signals from Format and Length: Reinforcement Learning for Solving Mathematical Problems without Ground Truth Answers","date":"2025-05-26","arxiv_id":"2505.19439","repositories_listed":1,"syntology":null},{"url":"/paper/synlogic-synthesizing-verifiable-reasoning","slug":"synlogic-synthesizing-verifiable-reasoning","title":"SynLogic: Synthesizing Verifiable Reasoning Data at Scale for Learning Logical Reasoning and Beyond","date":"2025-05-26","arxiv_id":"2505.19641","repositories_listed":1,"syntology":null},{"url":"/paper/chartsketcher-reasoning-with-multimodal","slug":"chartsketcher-reasoning-with-multimodal","title":"ChartSketcher: Reasoning with Multimodal Feedback and Reflection for Chart Understanding","date":"2025-05-25","arxiv_id":"2505.19076","repositories_listed":1,"syntology":null},{"url":"/paper/do-large-language-models-excel-in-complex","slug":"do-large-language-models-excel-in-complex","title":"Do Large Language Models Excel in Complex Logical Reasoning with Formal Language?","date":"2025-05-22","arxiv_id":"2505.16998","repositories_listed":1,"syntology":null},{"url":"/paper/sudoku-bench-evaluating-creative-reasoning","slug":"sudoku-bench-evaluating-creative-reasoning","title":"Sudoku-Bench: Evaluating creative reasoning with Sudoku variants","date":"2025-05-22","arxiv_id":"2505.16135","repositories_listed":1,"syntology":null},{"url":"/paper/learning-to-reason-via-mixture-of-thought-for","slug":"learning-to-reason-via-mixture-of-thought-for","title":"Learning to Reason via Mixture-of-Thought for Logical Reasoning","date":"2025-05-21","arxiv_id":"2505.15817","repositories_listed":1,"syntology":null},{"url":"/paper/nover-incentive-training-for-language-models","slug":"nover-incentive-training-for-language-models","title":"NOVER: Incentive Training for Language Models via Verifier-Free Reinforcement Learning","date":"2025-05-21","arxiv_id":"2505.16022","repositories_listed":1,"syntology":null},{"url":"/paper/reasoning-ocr-can-large-multimodal-models","slug":"reasoning-ocr-can-large-multimodal-models","title":"Reasoning-OCR: Can Large Multimodal Models Solve Complex Logical Reasoning Problems from OCR Cues?","date":"2025-05-19","arxiv_id":"2505.12766","repositories_listed":1,"syntology":null},{"url":"/paper/barrel-boundary-aware-reasoning-for-factual","slug":"barrel-boundary-aware-reasoning-for-factual","title":"BARREL: Boundary-Aware Reasoning for Factual and Reliable LRMs","date":"2025-05-18","arxiv_id":"2505.13529","repositories_listed":1,"syntology":null},{"url":"/paper/logicocr-do-your-large-multimodal-models","slug":"logicocr-do-your-large-multimodal-models","title":"LogicOCR: Do Your Large Multimodal Models Excel at Logical Reasoning on Text-Rich Images?","date":"2025-05-18","arxiv_id":"2505.12307","repositories_listed":1,"syntology":null},{"url":"/paper/reasoning-capabilities-and-invariability-of","slug":"reasoning-capabilities-and-invariability-of","title":"Reasoning Capabilities and Invariability of Large Language Models","date":"2025-05-01","arxiv_id":"2505.00776","repositories_listed":1,"syntology":null},{"url":"/paper/crave-a-conflicting-reasoning-approach-for","slug":"crave-a-conflicting-reasoning-approach-for","title":"CRAVE: A Conflicting Reasoning Approach for Explainable Claim Verification Using LLMs","date":"2025-04-21","arxiv_id":"2504.14905","repositories_listed":1,"syntology":null},{"url":"/paper/enhancing-the-geometric-problem-solving","slug":"enhancing-the-geometric-problem-solving","title":"Enhancing the Geometric Problem-Solving Ability of Multimodal LLMs via Symbolic-Neural Integration","date":"2025-04-17","arxiv_id":"2504.12773","repositories_listed":1,"syntology":null},{"url":"/paper/alice-proactive-learning-with-teacher-s","slug":"alice-proactive-learning-with-teacher-s","title":"Alice: Proactive Learning with Teacher's Demonstrations for Weak-to-Strong Generalization","date":"2025-04-09","arxiv_id":"2504.07316","repositories_listed":1,"syntology":null},{"url":"/paper/socrates-or-smartypants-testing-logic","slug":"socrates-or-smartypants-testing-logic","title":"Socrates or Smartypants: Testing Logic Reasoning Capabilities of Large Language Models with Logic Programming-based Test Oracles","date":"2025-04-09","arxiv_id":"2504.12312","repositories_listed":1,"syntology":null},{"url":"/paper/envisioning-beyond-the-pixels-benchmarking","slug":"envisioning-beyond-the-pixels-benchmarking","title":"Envisioning Beyond the Pixels: Benchmarking Reasoning-Informed Visual Editing","date":"2025-04-03","arxiv_id":"2504.02826","repositories_listed":1,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":0,"n_instrument":2,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":2,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/envisioning-beyond-the-pixels-benchmarking#ran","syntology_url":"https://syntology.ai/paper/2504.02826","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2504.02826"}},"official":{"repos":["phoenixz810/risebench"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/adaptive-rectification-sampling-for-test-time","slug":"adaptive-rectification-sampling-for-test-time","title":"Adaptive Rectification Sampling for Test-Time Compute Scaling","date":"2025-04-02","arxiv_id":"2504.01317","repositories_listed":1,"syntology":null},{"url":"/paper/exploring-the-effect-of-reinforcement","slug":"exploring-the-effect-of-reinforcement","title":"Exploring the Effect of Reinforcement Learning on Video Understanding: Insights from SEED-Bench-R1","date":"2025-03-31","arxiv_id":"2503.24376","repositories_listed":1,"syntology":{"n":9,"n_ran":7,"n_constructed":0,"n_ran_checked":7,"n_instrument":0,"n_unverified":2,"n_honours":3,"n_violates":0,"n_no_contract":4,"n_pointer_only":0,"phrase":"7 ran (of which 0 constructed an object rather than computing a result; 7 with no instrument failure: 3 honoured, 0 violated, 4 with no contract checked; 0 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/exploring-the-effect-of-reinforcement#ran","syntology_url":"https://syntology.ai/paper/2503.24376","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2503.24376"}},"official":{"repos":["tencentarc/seed-bench-r1"],"state":"official (archive's flag): 7 ran","n_ran":7,"n_constructed":0,"n_ran_no_instrument_failure":7,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/questbench-can-llms-ask-the-right-question-to","slug":"questbench-can-llms-ask-the-right-question-to","title":"QuestBench: Can LLMs ask the right question to acquire information in reasoning tasks?","date":"2025-03-28","arxiv_id":"2503.22674","repositories_listed":1,"syntology":{"n":12,"n_ran":11,"n_constructed":0,"n_ran_checked":11,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":11,"n_pointer_only":0,"phrase":"11 ran (of which 0 constructed an object rather than computing a result; 11 with no instrument failure: 0 honoured, 0 violated, 11 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/questbench-can-llms-ask-the-right-question-to#ran","syntology_url":"https://syntology.ai/paper/2503.22674","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2503.22674"}},"official":{"repos":["google-deepmind/questbench"],"state":"official (archive's flag): 11 ran","n_ran":11,"n_constructed":0,"n_ran_no_instrument_failure":11,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/medagent-pro-towards-evidence-based-multi","slug":"medagent-pro-towards-evidence-based-multi","title":"MedAgent-Pro: Towards Evidence-based Multi-modal Medical Diagnosis via Reasoning Agentic Workflow","date":"2025-03-21","arxiv_id":"2503.18968","repositories_listed":1,"syntology":null},{"url":"/paper/measuring-ai-ability-to-complete-long-tasks","slug":"measuring-ai-ability-to-complete-long-tasks","title":"Measuring AI Ability to Complete Long Tasks","date":"2025-03-18","arxiv_id":"2503.14499","repositories_listed":1,"syntology":null},{"url":"/paper/lmm-r1-empowering-3b-lmms-with-strong","slug":"lmm-r1-empowering-3b-lmms-with-strong","title":"LMM-R1: Empowering 3B LMMs with Strong Reasoning Abilities Through Two-Stage Rule-Based RL","date":"2025-03-10","arxiv_id":"2503.07536","repositories_listed":1,"syntology":{"n":3,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/lmm-r1-empowering-3b-lmms-with-strong#ran","syntology_url":"https://syntology.ai/paper/2503.07536","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2503.07536"}},"official":null}},{"url":"/paper/knowlogic-a-benchmark-for-commonsense","slug":"knowlogic-a-benchmark-for-commonsense","title":"SCoRE: Benchmarking Long-Chain Reasoning in Commonsense Scenarios","date":"2025-03-08","arxiv_id":"2503.06218","repositories_listed":1,"syntology":null},{"url":"/paper/three-tiers-of-computation-in-transformers","slug":"three-tiers-of-computation-in-transformers","title":"Three tiers of computation in transformers and in brain architectures","date":"2025-03-05","arxiv_id":"2503.04848","repositories_listed":1,"syntology":null},{"url":"/paper/delta-a-decoding-strategy-based-on-logit","slug":"delta-a-decoding-strategy-based-on-logit","title":"DeLTa: A Decoding Strategy based on Logit Trajectory Prediction Improves Factuality and Reasoning Ability","date":"2025-03-04","arxiv_id":"2503.02343","repositories_listed":1,"syntology":null},{"url":"/paper/kgcompiler-deep-learning-compilation","slug":"kgcompiler-deep-learning-compilation","title":"KGCompiler: Deep Learning Compilation Optimization for Knowledge Graph Complex Logical Query Answering","date":"2025-03-04","arxiv_id":"2503.02172","repositories_listed":1,"syntology":null},{"url":"/paper/nexus-a-lightweight-and-scalable-multi-agent","slug":"nexus-a-lightweight-and-scalable-multi-agent","title":"Nexus: A Lightweight and Scalable Multi-Agent Framework for Complex Tasks Automation","date":"2025-02-26","arxiv_id":"2502.19091","repositories_listed":1,"syntology":null},{"url":"/paper/autologi-automated-generation-of-logic","slug":"autologi-automated-generation-of-logic","title":"AutoLogi: Automated Generation of Logic Puzzles for Evaluating Reasoning Abilities of Large Language Models","date":"2025-02-24","arxiv_id":"2502.16906","repositories_listed":1,"syntology":null},{"url":"/paper/from-system-1-to-system-2-a-survey-of","slug":"from-system-1-to-system-2-a-survey-of","title":"From System 1 to System 2: A Survey of Reasoning Large Language Models","date":"2025-02-24","arxiv_id":"2502.17419","repositories_listed":1,"syntology":null},{"url":"/paper/r1-onevision-an-open-source-multimodal-large","slug":"r1-onevision-an-open-source-multimodal-large","title":"R1-Onevision：An Open-Source Multimodal Large Language Model Capable of Deep Reasoning","date":"2025-02-24","arxiv_id":null,"repositories_listed":1,"syntology":null},{"url":"/paper/on-the-logical-skills-of-large-language","slug":"on-the-logical-skills-of-large-language","title":"On the logical skills of large language models: evaluations using arbitrarily complex first-order logic problems","date":"2025-02-20","arxiv_id":"2502.14180","repositories_listed":1,"syntology":null},{"url":"/paper/integrating-expert-knowledge-into-logical","slug":"integrating-expert-knowledge-into-logical","title":"Integrating Expert Knowledge into Logical Programs via LLMs","date":"2025-02-17","arxiv_id":"2502.12275","repositories_listed":1,"syntology":null},{"url":"/paper/unveiling-the-magic-of-code-reasoning-through","slug":"unveiling-the-magic-of-code-reasoning-through","title":"Unveiling the Magic of Code Reasoning through Hypothesis Decomposition and Amendment","date":"2025-02-17","arxiv_id":"2502.13170","repositories_listed":1,"syntology":{"n":2,"n_ran":0,"n_constructed":0,"n_ran_checked":0,"n_instrument":0,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":2,"phrase":"0 ran · 2 unverified","sample_list":"/paper/unveiling-the-magic-of-code-reasoning-through#ran","syntology_url":"https://syntology.ai/paper/2502.13170","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2502.13170"}},"official":{"repos":["tntwow/code_reasoning"],"state":"official: harvested, nothing ran","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":2,"ran_from_kinds":[]}}},{"url":"/paper/exposing-numeracy-gaps-a-benchmark-to","slug":"exposing-numeracy-gaps-a-benchmark-to","title":"Exposing Numeracy Gaps: A Benchmark to Evaluate Fundamental Numerical Abilities in Large Language Models","date":"2025-02-16","arxiv_id":"2502.11075","repositories_listed":1,"syntology":null},{"url":"/paper/large-language-models-meet-symbolic-provers","slug":"large-language-models-meet-symbolic-provers","title":"Large Language Models Meet Symbolic Provers for Logical Reasoning Evaluation","date":"2025-02-10","arxiv_id":"2502.06563","repositories_listed":1,"syntology":{"n":6,"n_ran":3,"n_constructed":1,"n_ran_checked":2,"n_instrument":1,"n_unverified":3,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":6,"phrase":"3 ran (of which 1 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 1 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/large-language-models-meet-symbolic-provers#ran","syntology_url":"https://syntology.ai/paper/2502.06563","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2502.06563"}},"official":{"repos":["opendatalab/provergen"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":1,"n_ran_no_instrument_failure":2,"n_unverified":3,"ran_from_kinds":["official"]}}},{"url":"/paper/sedareval-automated-evaluation-using-self","slug":"sedareval-automated-evaluation-using-self","title":"SedarEval: Automated Evaluation using Self-Adaptive Rubrics","date":"2025-01-26","arxiv_id":"2501.15595","repositories_listed":1,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":0,"n_instrument":2,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":2,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/sedareval-automated-evaluation-using-self#ran","syntology_url":"https://syntology.ai/paper/2501.15595","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2501.15595"}},"official":{"repos":["wwn1233/sedareval"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/justlogic-a-comprehensive-benchmark-for","slug":"justlogic-a-comprehensive-benchmark-for","title":"JustLogic: A Comprehensive Benchmark for Evaluating Deductive Reasoning in Large Language Models","date":"2025-01-24","arxiv_id":"2501.14851","repositories_listed":1,"syntology":null},{"url":"/paper/pike-rag-specialized-knowledge-and-rationale","slug":"pike-rag-specialized-knowledge-and-rationale","title":"PIKE-RAG: sPecIalized KnowledgE and Rationale Augmented Generation","date":"2025-01-20","arxiv_id":"2501.11551","repositories_listed":1,"syntology":{"n":2,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/pike-rag-specialized-knowledge-and-rationale#ran","syntology_url":"https://syntology.ai/paper/2501.11551","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2501.11551"}},"official":{"repos":["microsoft/pike-rag"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/assessing-the-alignment-of-fol-closeness","slug":"assessing-the-alignment-of-fol-closeness","title":"Assessing the Alignment of FOL Closeness Metrics with Human Judgement","date":"2025-01-15","arxiv_id":"2501.08613","repositories_listed":1,"syntology":null},{"url":"/paper/leapvad-a-leap-in-autonomous-driving-via","slug":"leapvad-a-leap-in-autonomous-driving-via","title":"LeapVAD: A Leap in Autonomous Driving via Cognitive Perception and Dual-Process Thinking","date":"2025-01-14","arxiv_id":"2501.08168","repositories_listed":1,"syntology":{"n":5,"n_ran":4,"n_constructed":0,"n_ran_checked":3,"n_instrument":1,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":1,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 1 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/leapvad-a-leap-in-autonomous-driving-via#ran","syntology_url":"https://syntology.ai/paper/2501.08168","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2501.08168"}},"official":null}},{"url":"/paper/ocrbench-v2-an-improved-benchmark-for","slug":"ocrbench-v2-an-improved-benchmark-for","title":"OCRBench v2: An Improved Benchmark for Evaluating Large Multimodal Models on Visual Text Localization and Reasoning","date":"2024-12-31","arxiv_id":"2501.00321","repositories_listed":1,"syntology":{"n":10,"n_ran":9,"n_constructed":0,"n_ran_checked":5,"n_instrument":4,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":5,"n_pointer_only":0,"phrase":"9 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 0 honoured, 0 violated, 5 with no contract checked; 4 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/ocrbench-v2-an-improved-benchmark-for#ran","syntology_url":"https://syntology.ai/paper/2501.00321","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2501.00321"}},"official":{"repos":["yuliang-liu/multimodalocr"],"state":"official (archive's flag): 9 ran","n_ran":9,"n_constructed":0,"n_ran_no_instrument_failure":5,"n_unverified":1,"ran_from_kinds":["official"]}}}],"syntology_records":38,"syntology_note":"a paper without a record is not a recorded non-run: it may lack an arXiv id or simply be absent from the graph layer"},"key_notes":{"n_ran_checked":"legacy name, kept unchanged so existing readers do not break: it counts the samples that ran with no instrument failure (honoured, violated, and ran with no contract checked); it does not mean a contract was checked, and the pages print it as 'K with no instrument failure', not 'K checked'","n_constructed":"a sub-count of the samples that ran, never subtracted from them and never a failure: an executed sample whose run returned an instance of its own class (fixture_out_type equals the entry name): the run built an object and did not compute a result (Syntology's RAN record, counts.constructed)"},"since_archive_judged":{"papers":0,"papers_placed":0,"judged_by":{"model":"a model","models_as_recorded":[],"from":"lb_rows_run_verified.jsonl: each record's `model` field (the extractor's call that wrote decisions[])"},"note":"a model's judgement from the paper's own tables that they report on this leaderboard, not a result: no value from a paper that was not placed is printed or carried here; papers counts each paper once over these leaderboards","backfill":"8,886 of 18,637 newer papers checked so far (733 more had nothing to place; 41 could not be read)","backfill_complete":false,"leaderboards":[]},"papers_where_code_ran":113,"papers_where_code_ran_first_page":"/task/logical-reasoning/papers/ran/1","papers_where_code_ran_caption":"We ran code from the paper's repository; we did not run it on this task or check it against the task's benchmarks.","pages":[{"order":"archive","page":2,"of":8,"url":"/task/logical-reasoning/papers/2","json":"/task/logical-reasoning/papers/2.json","rows":100},{"order":"archive","page":3,"of":8,"url":"/task/logical-reasoning/papers/3","json":"/task/logical-reasoning/papers/3.json","rows":100},{"order":"archive","page":4,"of":8,"url":"/task/logical-reasoning/papers/4","json":"/task/logical-reasoning/papers/4.json","rows":100},{"order":"archive","page":5,"of":8,"url":"/task/logical-reasoning/papers/5","json":"/task/logical-reasoning/papers/5.json","rows":100},{"order":"archive","page":6,"of":8,"url":"/task/logical-reasoning/papers/6","json":"/task/logical-reasoning/papers/6.json","rows":100},{"order":"archive","page":7,"of":8,"url":"/task/logical-reasoning/papers/7","json":"/task/logical-reasoning/papers/7.json","rows":100},{"order":"archive","page":8,"of":8,"url":"/task/logical-reasoning/papers/8","json":"/task/logical-reasoning/papers/8.json","rows":47},{"order":"ran","page":1,"of":2,"url":"/task/logical-reasoning/papers/ran/1","json":"/task/logical-reasoning/papers/ran/1.json","rows":100},{"order":"ran","page":2,"of":2,"url":"/task/logical-reasoning/papers/ran/2","json":"/task/logical-reasoning/papers/ran/2.json","rows":13}],"description_links":{"kept":0,"unwrapped_to_text":0,"bare_urls_linked":0,"relative_images_dropped":0,"rule":"internal links are kept only when the target slug exists in the catalog"},"syntology":{"read_at":"2026-10-05T00:36:11+00:00","claim":"Per-sample execution status on synthesized fixtures ('ran N of M samples'); not a correctness claim and not a ranking signal.","status_vocabulary":{"ran_honours":"ran, honoured the contract we drafted","ran_violates":"ran, violated the contract we drafted","ran_draft_wrong":"ran; our contract draft was wrong, not the code","ran_fixture":"ran; the contract check could not run (RAISES, DEP_MISSING or TIMEOUT)","ran":"ran on a synthesized input","unverified":"unverified (harvested, no recorded run)"},"row_route":"each syntology object on a benchmark's first row and on a paper carries sample_list (this site's paper page, #ran), syntology_url (syntology.ai's page for the paper, only when that route probed live) and mcp (the agent call that lists the paper's harvested samples; not a verdict on the paper)","developers":"https://syntology.ai/developers"},"not_shown":{"libraries":"the archive has no per-task library table","trend_sparklines":"the Trend column of the benchmarks table was a rendered image; it is not in the archive","social_and_latest_sorts":"stars and social signals are not in the archive"},"record_sha256":"a7b2c368aa20d0e4d0840ee66f7e2e7067dcec8a9860d744fdb4ad6e27c70e7e","record_changed_at":"2026-10-05","record_changed_at_basis":"changed"}