{"about":{"non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","site":"https://codewithpapers.app","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page","syntology":{"site":"https://syntology.ai","developers":"https://syntology.ai/developers","mcp":{"server":"https://syntology.ai/mcp","transport":"streamable-http","server_card":"https://syntology.ai/.well-known/mcp/server-card.json","auth":{"type":"trial token, no account","trial_token":"https://syntology.ai/api/oauth/trial/token","method":"POST","docs":"https://syntology.ai/developers"}},"have":"https://syntology.ai/api/graph/have?x=<method, arXiv id or title> (free, answers coverage only)","paper_base":"https://syntology.ai/paper/","atlas_base":"https://app.syntology.ai/?focus="},"machine_readable":[{"url":"https://codewithpapers.app/llms.txt","what":"the machine catalog: every machine-readable file, counted"},{"url":"https://codewithpapers.app/index/manifest.json","what":"paper-to-code index by arXiv id, with Syntology's counts"},{"url":"https://codewithpapers.app/search/manifest.json","what":"site search index (titles, authors) and its files"},{"url":"https://codewithpapers.app/download","what":"bulk files: Syntology's layer, described there"},{"url":"https://codewithpapers.app/build_manifest.json","what":"the build record: inputs, counts, exclusions, probes"}]},"url":"/task/strategyqa","name":"StrategyQA","slug":"strategyqa","description_markdown":"StrategyQA aims to measure the ability of models to answer questions that require multi-step implicit reasoning.\r\n\r\nSource: [BIG-bench](https://github.com/google/BIG-bench/tree/main/bigbench/benchmark_tasks/strategyqa)","categories":[{"name":"Methodology","url":"/area/methodology"},{"name":"Miscellaneous","url":"/area/miscellaneous"},{"name":"Reasoning","url":"/area/reasoning"}],"source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","slug_source":"derived"},"counts":{"papers_tagged":40,"papers_with_code":20,"benchmarks":0,"benchmark_tables_in_archive":0,"benchmark_tables_shown":0,"benchmark_tables_withheld_as_spam":0,"benchmark_definition":"a leaderboard table with at least one row; benchmark_tables_shown also counts the zero-row tables; benchmark_tables_in_archive adds the tables withheld as spam","datasets":0,"subtasks":0,"parent_tasks":1},"benchmarks":[],"datasets":[],"subtasks":[],"parent_tasks":[{"url":"/task/logical-reasoning","name":"Logical Reasoning"}],"papers":{"order":"repositories listed in the archive (most first), then date (newest first), then slug; the archive holds no stars","population":"every paper the archive tags with this task (distinct papers; spam titles not listed). 'of' counts the papers listed; until 2026-09-25 it counted only the papers with a code link, and that number is of_with_code","page":1,"pages_in_order":1,"rows_per_page":100,"shown":40,"of":40,"of_with_code":20,"tagged_in_all":40,"counts":{"archive_papers_tagged":40,"with_a_code_link":20,"where_syntology_ran_a_sample":11,"not_listed_spam_title":0,"listed":40,"listed_where_code_ran":11,"where_syntology_ran_a_sample_split":{"with_a_run_with_no_instrument_failure":10,"every_run_a_failure_of_syntologys_instrument":1,"listed_with_a_run_with_no_instrument_failure":10,"listed_every_run_a_failure_of_syntologys_instrument":1,"filter":{"states":["a run with no instrument failure","any run, instrument failures included"],"default":"a run with no instrument failure","note":"on the 'only where code ran' pages the default hides, in the browser, the rows where every run was a failure of Syntology's instrument; the second state shows them again. Rows are hidden, never re-ordered; these twins list every row"}},"definition":"distinct papers the archive tags; 'where Syntology ran a sample' counts papers with at least one harvested sample that ran, which is not a correctness claim"},"items":[{"url":"/paper/palm-scaling-language-modeling-with-pathways-1","slug":"palm-scaling-language-modeling-with-pathways-1","title":"PaLM: Scaling Language Modeling with Pathways","date":"2022-04-05","arxiv_id":"2204.02311","repositories_listed":7,"syntology":{"n":37,"n_ran":32,"n_constructed":16,"n_ran_checked":24,"n_instrument":8,"n_unverified":5,"n_honours":2,"n_violates":1,"n_no_contract":21,"n_pointer_only":0,"phrase":"32 ran (of which 16 constructed an object rather than computing a result; 24 with no instrument failure: 2 honoured, 1 violated, 21 with no contract checked; 8 where Syntology's instrument failed) · 5 unverified","sample_list":"/paper/palm-scaling-language-modeling-with-pathways-1#ran","syntology_url":"https://syntology.ai/paper/2204.02311","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2204.02311"}},"official":null}},{"url":"/paper/mutual-reasoning-makes-smaller-llms-stronger","slug":"mutual-reasoning-makes-smaller-llms-stronger","title":"Mutual Reasoning Makes Smaller LLMs Stronger Problem-Solvers","date":"2024-08-12","arxiv_id":"2408.06195","repositories_listed":3,"syntology":{"n":2,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/mutual-reasoning-makes-smaller-llms-stronger#ran","syntology_url":"https://syntology.ai/paper/2408.06195","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2408.06195"}},"official":{"repos":["zhentingqi/rstar"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/a-prefrontal-cortex-inspired-architecture-for","slug":"a-prefrontal-cortex-inspired-architecture-for","title":"Improving Planning with Large Language Models: A Modular Agentic Architecture","date":"2023-09-30","arxiv_id":"2310.00194","repositories_listed":3,"syntology":null},{"url":"/paper/self-consistency-improves-chain-of-thought","slug":"self-consistency-improves-chain-of-thought","title":"Self-Consistency Improves Chain of Thought Reasoning in Language Models","date":"2022-03-21","arxiv_id":"2203.11171","repositories_listed":3,"syntology":{"n":1,"n_ran":1,"n_constructed":1,"n_ran_checked":1,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"1 ran (of which 1 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified; the one sample that ran constructed an object rather than computing a result","sample_list":"/paper/self-consistency-improves-chain-of-thought#ran","syntology_url":"https://syntology.ai/paper/2203.11171","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2203.11171"}},"official":null}},{"url":"/paper/scaling-language-models-methods-analysis-1","slug":"scaling-language-models-methods-analysis-1","title":"Scaling Language Models: Methods, Analysis & Insights from Training Gopher","date":"2021-12-08","arxiv_id":"2112.11446","repositories_listed":3,"syntology":null},{"url":"/paper/training-compute-optimal-large-language","slug":"training-compute-optimal-large-language","title":"Training Compute-Optimal Large Language Models","date":"2022-03-29","arxiv_id":"2203.15556","repositories_listed":2,"syntology":{"n":11,"n_ran":8,"n_constructed":3,"n_ran_checked":5,"n_instrument":3,"n_unverified":3,"n_honours":2,"n_violates":0,"n_no_contract":3,"n_pointer_only":4,"phrase":"8 ran (of which 3 constructed an object rather than computing a result; 5 with no instrument failure: 2 honoured, 0 violated, 3 with no contract checked; 3 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/training-compute-optimal-large-language#ran","syntology_url":"https://syntology.ai/paper/2203.15556","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2203.15556"}},"official":null}},{"url":"/paper/delta-a-decoding-strategy-based-on-logit","slug":"delta-a-decoding-strategy-based-on-logit","title":"DeLTa: A Decoding Strategy based on Logit Trajectory Prediction Improves Factuality and Reasoning Ability","date":"2025-03-04","arxiv_id":"2503.02343","repositories_listed":1,"syntology":null},{"url":"/paper/voting-or-consensus-decision-making-in-multi","slug":"voting-or-consensus-decision-making-in-multi","title":"Voting or Consensus? Decision-Making in Multi-Agent Debate","date":"2025-02-26","arxiv_id":"2502.19130","repositories_listed":1,"syntology":null},{"url":"/paper/autoreason-automatic-few-shot-reasoning","slug":"autoreason-automatic-few-shot-reasoning","title":"AutoReason: Automatic Few-Shot Reasoning Decomposition","date":"2024-12-09","arxiv_id":"2412.06975","repositories_listed":1,"syntology":null},{"url":"/paper/rationale-aware-answer-verification-by","slug":"rationale-aware-answer-verification-by","title":"Rationale-Aware Answer Verification by Pairwise Self-Evaluation","date":"2024-10-07","arxiv_id":"2410.04838","repositories_listed":1,"syntology":{"n":7,"n_ran":6,"n_constructed":0,"n_ran_checked":1,"n_instrument":5,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 5 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/rationale-aware-answer-verification-by#ran","syntology_url":"https://syntology.ai/paper/2410.04838","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2410.04838"}},"official":{"repos":["akirakawabata/reps"],"state":"official (archive's flag): 6 ran","n_ran":6,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/unchosen-experts-can-contribute-too","slug":"unchosen-experts-can-contribute-too","title":"Unchosen Experts Can Contribute Too: Unleashing MoE Models' Power by Self-Contrast","date":"2024-05-23","arxiv_id":"2405.14507","repositories_listed":1,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":1,"n_instrument":1,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":2,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/unchosen-experts-can-contribute-too#ran","syntology_url":"https://syntology.ai/paper/2405.14507","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2405.14507"}},"official":{"repos":["davidfanzz/scmoe"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/cr-lt-kgqa-a-knowledge-graph-question","slug":"cr-lt-kgqa-a-knowledge-graph-question","title":"CR-LT-KGQA: A Knowledge Graph Question Answering Dataset Requiring Commonsense Reasoning and Long-Tail Knowledge","date":"2024-03-03","arxiv_id":"2403.01395","repositories_listed":1,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":1,"n_no_contract":0,"n_pointer_only":1,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 1 violated, 0 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/cr-lt-kgqa-a-knowledge-graph-question#ran","syntology_url":"https://syntology.ai/paper/2403.01395","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2403.01395"}},"official":{"repos":["d3mlab/cr-lt-kgqa"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/distillation-contrastive-decoding-improving","slug":"distillation-contrastive-decoding-improving","title":"Distillation Contrastive Decoding: Improving LLMs Reasoning with Contrastive Decoding and Distillation","date":"2024-02-21","arxiv_id":"2402.14874","repositories_listed":1,"syntology":{"n":10,"n_ran":6,"n_constructed":0,"n_ran_checked":3,"n_instrument":3,"n_unverified":4,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":10,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 3 where Syntology's instrument failed) · 4 unverified","sample_list":"/paper/distillation-contrastive-decoding-improving#ran","syntology_url":"https://syntology.ai/paper/2402.14874","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2402.14874"}},"official":{"repos":["pphuc25/distil-cd"],"state":"official (archive's flag): 6 ran","n_ran":6,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":4,"ran_from_kinds":["official"]}}},{"url":"/paper/escape-sky-high-cost-early-stopping-self","slug":"escape-sky-high-cost-early-stopping-self","title":"Escape Sky-high Cost: Early-stopping Self-Consistency for Multi-step Reasoning","date":"2024-01-19","arxiv_id":"2401.10480","repositories_listed":1,"syntology":{"n":8,"n_ran":7,"n_constructed":0,"n_ran_checked":4,"n_instrument":3,"n_unverified":1,"n_honours":3,"n_violates":0,"n_no_contract":1,"n_pointer_only":8,"phrase":"7 ran (of which 0 constructed an object rather than computing a result; 4 with no instrument failure: 3 honoured, 0 violated, 1 with no contract checked; 3 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/escape-sky-high-cost-early-stopping-self#ran","syntology_url":"https://syntology.ai/paper/2401.10480","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2401.10480"}},"official":{"repos":["yiwei98/esc"],"state":"official (archive's flag): 7 ran","n_ran":7,"n_constructed":0,"n_ran_no_instrument_failure":4,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/tailoring-self-rationalizers-with-multi","slug":"tailoring-self-rationalizers-with-multi","title":"Tailoring Self-Rationalizers with Multi-Reward Distillation","date":"2023-11-06","arxiv_id":"2311.02805","repositories_listed":1,"syntology":{"n":10,"n_ran":6,"n_constructed":0,"n_ran_checked":6,"n_instrument":0,"n_unverified":4,"n_honours":0,"n_violates":0,"n_no_contract":6,"n_pointer_only":10,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 6 with no instrument failure: 0 honoured, 0 violated, 6 with no contract checked; 0 where Syntology's instrument failed) · 4 unverified","sample_list":"/paper/tailoring-self-rationalizers-with-multi#ran","syntology_url":"https://syntology.ai/paper/2311.02805","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2311.02805"}},"official":{"repos":["ink-usc/rationalemultirewarddistillation"],"state":"official (archive's flag): 6 ran","n_ran":6,"n_constructed":0,"n_ran_no_instrument_failure":6,"n_unverified":4,"ran_from_kinds":["official"]}}},{"url":"/paper/teaching-smaller-language-models-to","slug":"teaching-smaller-language-models-to","title":"Teaching Smaller Language Models To Generalise To Unseen Compositional Questions","date":"2023-08-02","arxiv_id":"2308.00946","repositories_listed":1,"syntology":null},{"url":"/paper/knowledge-augmented-reasoning-distillation-1","slug":"knowledge-augmented-reasoning-distillation-1","title":"Knowledge-Augmented Reasoning Distillation for Small Language Models in Knowledge-Intensive Tasks","date":"2023-05-28","arxiv_id":"2305.18395","repositories_listed":1,"syntology":{"n":6,"n_ran":5,"n_constructed":0,"n_ran_checked":3,"n_instrument":2,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":1,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 2 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/knowledge-augmented-reasoning-distillation-1#ran","syntology_url":"https://syntology.ai/paper/2305.18395","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2305.18395"}},"official":{"repos":["nardien/kard"],"state":"official (archive's flag): 5 ran","n_ran":5,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/visconde-multi-document-qa-with-gpt-3-and","slug":"visconde-multi-document-qa-with-gpt-3-and","title":"Visconde: Multi-document QA with GPT-3 and Neural Reranking","date":"2022-12-19","arxiv_id":"2212.09656","repositories_listed":1,"syntology":null},{"url":"/paper/distilling-multi-step-reasoning-capabilities","slug":"distilling-multi-step-reasoning-capabilities","title":"Distilling Reasoning Capabilities into Smaller Language Models","date":"2022-12-01","arxiv_id":"2212.00193","repositories_listed":1,"syntology":null},{"url":"/paper/did-aristotle-use-a-laptop-a-question","slug":"did-aristotle-use-a-laptop-a-question","title":"Did Aristotle Use a Laptop? A Question Answering Benchmark with Implicit Reasoning Strategies","date":"2021-01-06","arxiv_id":"2101.02235","repositories_listed":1,"syntology":null},{"url":null,"slug":"fusing-bidirectional-chains-of-thought-and","title":"Fusing Bidirectional Chains of Thought and Reward Mechanisms A Method for Enhancing Question-Answering Capabilities of Large Language Models for Chinese Intangible Cultural Heritage","date":"2025-05-13","arxiv_id":"2505.08167","repositories_listed":0,"syntology":null},{"url":null,"slug":"rule-guided-feedback-enhancing-reasoning-by","title":"Rule-Guided Feedback: Enhancing Reasoning by Enforcing Rule Adherence in Large Language Models","date":"2025-03-14","arxiv_id":"2503.11336","repositories_listed":0,"syntology":null},{"url":null,"slug":"unraveling-indirect-in-context-learning-using","title":"Unraveling Indirect In-Context Learning Using Influence Functions","date":"2025-01-01","arxiv_id":"2501.01473","repositories_listed":0,"syntology":null},{"url":null,"slug":"dialectical-behavior-therapy-approach-to-llm","title":"Dialectical Behavior Therapy Approach to LLM Prompting","date":"2024-10-10","arxiv_id":"2410.07768","repositories_listed":0,"syntology":null},{"url":null,"slug":"a-looming-replication-crisis-in-evaluating","title":"A Looming Replication Crisis in Evaluating Behavior in Language Models? Evidence and Solutions","date":"2024-09-30","arxiv_id":"2409.20303","repositories_listed":0,"syntology":null},{"url":null,"slug":"proof-of-thought-neurosymbolic-program","title":"Proof of Thought : Neurosymbolic Program Synthesis allows Robust and Interpretable Reasoning","date":"2024-09-25","arxiv_id":"2409.17270","repositories_listed":0,"syntology":null},{"url":null,"slug":"meta-prompting-optimized-retrieval-augmented","title":"Meta-prompting Optimized Retrieval-augmented Generation","date":"2024-07-04","arxiv_id":"2407.03955","repositories_listed":0,"syntology":null},{"url":null,"slug":"question-analysis-prompting-improves-llm","title":"Question-Analysis Prompting Improves LLM Performance in Reasoning Tasks","date":"2024-07-04","arxiv_id":"2407.03624","repositories_listed":0,"syntology":null},{"url":null,"slug":"advancing-process-verification-for-large","title":"Advancing Process Verification for Large Language Models via Tree-Based Preference Learning","date":"2024-06-29","arxiv_id":"2407.00390","repositories_listed":0,"syntology":null},{"url":null,"slug":"improving-attributed-text-generation-of-large","title":"Improving Attributed Text Generation of Large Language Models via Preference Learning","date":"2024-03-27","arxiv_id":"2403.18381","repositories_listed":0,"syntology":null},{"url":null,"slug":"towards-uncertainty-aware-language-agent","title":"Towards Uncertainty-Aware Language Agent","date":"2024-01-25","arxiv_id":"2401.14016","repositories_listed":0,"syntology":null},{"url":null,"slug":"iag-induction-augmented-generation-framework","title":"IAG: Induction-Augmented Generation Framework for Answering Reasoning Questions","date":"2023-11-30","arxiv_id":"2311.18397","repositories_listed":0,"syntology":null},{"url":"/paper/the-art-of-llm-refinement-ask-refine-and","slug":"the-art-of-llm-refinement-ask-refine-and","title":"The ART of LLM Refinement: Ask, Refine, and Trust","date":"2023-11-14","arxiv_id":"2311.07961","repositories_listed":0,"syntology":null},{"url":null,"slug":"large-language-models-are-also-good","title":"Large Language Models Are Also Good Prototypical Commonsense Reasoners","date":"2023-09-22","arxiv_id":"2309.13165","repositories_listed":0,"syntology":null},{"url":null,"slug":"answering-unseen-questions-with-smaller","title":"Answering Unseen Questions With Smaller Language Models Using Rationale Generation and Dense Retrieval","date":"2023-08-09","arxiv_id":"2308.04711","repositories_listed":0,"syntology":null},{"url":null,"slug":"deduction-under-perturbed-evidence-probing","title":"Deduction under Perturbed Evidence: Probing Student Simulation Capabilities of Large Language Models","date":"2023-05-23","arxiv_id":"2305.14507","repositories_listed":0,"syntology":null},{"url":null,"slug":"selfzcot-a-self-prompt-zero-shot-cot-from","title":"Hint of Thought prompting: an explainable and zero-shot approach to reasoning tasks with LLMs","date":"2023-05-19","arxiv_id":"2305.11461","repositories_listed":0,"syntology":null},{"url":null,"slug":"self-evaluation-guided-beam-search-for-1","title":"Self-Evaluation Guided Beam Search for Reasoning","date":"2023-05-01","arxiv_id":"2305.00633","repositories_listed":0,"syntology":null},{"url":null,"slug":"learning-to-decompose-hypothetical-question","title":"Learning to Decompose: Hypothetical Question Decomposition Based on Comparable Texts","date":"2022-10-30","arxiv_id":"2210.16865","repositories_listed":0,"syntology":null},{"url":null,"slug":"better-retrieval-may-not-lead-to-better","title":"Better Retrieval May Not Lead to Better Question Answering","date":"2022-05-07","arxiv_id":"2205.03685","repositories_listed":0,"syntology":null}],"syntology_records":11,"syntology_note":"a paper without a record is not a recorded non-run: it may lack an arXiv id or simply be absent from the graph layer"},"key_notes":{"n_ran_checked":"legacy name, kept unchanged so existing readers do not break: it counts the samples that ran with no instrument failure (honoured, violated, and ran with no contract checked); it does not mean a contract was checked, and the pages print it as 'K with no instrument failure', not 'K checked'","n_constructed":"a sub-count of the samples that ran, never subtracted from them and never a failure: an executed sample whose run returned an instance of its own class (fixture_out_type equals the entry name): the run built an object and did not compute a result (Syntology's RAN record, counts.constructed)"},"since_archive_judged":{"papers":0,"papers_placed":0,"judged_by":{"model":"a model","models_as_recorded":[],"from":"lb_rows_run_verified.jsonl: each record's `model` field (the extractor's call that wrote decisions[])"},"note":"a model's judgement from the paper's own tables that they report on this leaderboard, not a result: no value from a paper that was not placed is printed or carried here; papers counts each paper once over these leaderboards","backfill":"8,886 of 18,637 newer papers checked so far (733 more had nothing to place; 41 could not be read)","backfill_complete":false,"leaderboards":[]},"papers_where_code_ran":11,"papers_where_code_ran_first_page":"/task/strategyqa/papers/ran/1","papers_where_code_ran_caption":"We ran code from the paper's repository; we did not run it on this task or check it against the task's benchmarks.","pages":[{"order":"ran","page":1,"of":1,"url":"/task/strategyqa/papers/ran/1","json":"/task/strategyqa/papers/ran/1.json","rows":11}],"description_links":{"kept":0,"unwrapped_to_text":0,"bare_urls_linked":0,"relative_images_dropped":0,"rule":"internal links are kept only when the target slug exists in the catalog"},"syntology":{"read_at":"2026-10-05T00:36:11+00:00","claim":"Per-sample execution status on synthesized fixtures ('ran N of M samples'); not a correctness claim and not a ranking signal.","status_vocabulary":{"ran_honours":"ran, honoured the contract we drafted","ran_violates":"ran, violated the contract we drafted","ran_draft_wrong":"ran; our contract draft was wrong, not the code","ran_fixture":"ran; the contract check could not run (RAISES, DEP_MISSING or TIMEOUT)","ran":"ran on a synthesized input","unverified":"unverified (harvested, no recorded run)"},"row_route":"each syntology object on a benchmark's first row and on a paper carries sample_list (this site's paper page, #ran), syntology_url (syntology.ai's page for the paper, only when that route probed live) and mcp (the agent call that lists the paper's harvested samples; not a verdict on the paper)","developers":"https://syntology.ai/developers"},"not_shown":{"libraries":"the archive has no per-task library table","trend_sparklines":"the Trend column of the benchmarks table was a rendered image; it is not in the archive","social_and_latest_sorts":"stars and social signals are not in the archive"},"record_sha256":"7fa421104b082165283e9a2080a8afae9fa58707e165579cb023c98c94c11be4","record_changed_at":"2026-10-05","record_changed_at_basis":"changed"}