{"about":{"non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","site":"https://codewithpapers.app","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page","syntology":{"site":"https://syntology.ai","developers":"https://syntology.ai/developers","mcp":{"server":"https://syntology.ai/mcp","transport":"streamable-http","server_card":"https://syntology.ai/.well-known/mcp/server-card.json","auth":{"type":"trial token, no account","trial_token":"https://syntology.ai/api/oauth/trial/token","method":"POST","docs":"https://syntology.ai/developers"}},"have":"https://syntology.ai/api/graph/have?x=<method, arXiv id or title> (free, answers coverage only)","paper_base":"https://syntology.ai/paper/","atlas_base":"https://app.syntology.ai/?focus="},"machine_readable":[{"url":"https://codewithpapers.app/llms.txt","what":"the machine catalog: every machine-readable file, counted"},{"url":"https://codewithpapers.app/index/manifest.json","what":"paper-to-code index by arXiv id, with Syntology's counts"},{"url":"https://codewithpapers.app/search/manifest.json","what":"site search index (titles, authors) and its files"},{"url":"https://codewithpapers.app/download","what":"bulk files: Syntology's layer, described there"},{"url":"https://codewithpapers.app/build_manifest.json","what":"the build record: inputs, counts, exclusions, probes"}]},"url":"/task/lexical-entailment","name":"Lexical Entailment","slug":"lexical-entailment","description_markdown":null,"categories":[],"source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","slug_source":"archive_url"},"counts":{"papers_tagged":45,"papers_with_code":18,"benchmarks":0,"benchmark_tables_in_archive":0,"benchmark_tables_shown":0,"benchmark_tables_withheld_as_spam":0,"benchmark_definition":"a leaderboard table with at least one row; benchmark_tables_shown also counts the zero-row tables; benchmark_tables_in_archive adds the tables withheld as spam","datasets":6,"subtasks":0,"parent_tasks":0},"benchmarks":[],"datasets":[{"url":"/dataset/hyperlex","name":"HyperLex","full_name":"","num_papers_in_archive":29},{"url":"/dataset/evalution","name":"EVALution","full_name":"EVALution","num_papers_in_archive":28},{"url":"/dataset/terra","name":"TERRa","full_name":"Textual Entailment Recognition for Russian","num_papers_in_archive":7},{"url":"/dataset/imppres","name":"IMPPRES","full_name":"","num_papers_in_archive":6},{"url":"/dataset/sherliic","name":"SherLIiC","full_name":"SherLIiC","num_papers_in_archive":6},{"url":"/dataset/persian-nlp-hub","name":"Persian NLP Hub","full_name":"","num_papers_in_archive":1}],"subtasks":[],"parent_tasks":[],"papers":{"order":"repositories listed in the archive (most first), then date (newest first), then slug; the archive holds no stars","population":"every paper the archive tags with this task (distinct papers; spam titles not listed). 'of' counts the papers listed; until 2026-09-25 it counted only the papers with a code link, and that number is of_with_code","page":1,"pages_in_order":1,"rows_per_page":100,"shown":45,"of":45,"of_with_code":18,"tagged_in_all":45,"counts":{"archive_papers_tagged":45,"with_a_code_link":18,"where_syntology_ran_a_sample":3,"not_listed_spam_title":0,"listed":45,"listed_where_code_ran":3,"where_syntology_ran_a_sample_split":{"with_a_run_with_no_instrument_failure":3,"every_run_a_failure_of_syntologys_instrument":0,"listed_with_a_run_with_no_instrument_failure":3,"listed_every_run_a_failure_of_syntologys_instrument":0,"filter":{"states":["a run with no instrument failure","any run, instrument failures included"],"default":"a run with no instrument failure","note":"on the 'only where code ran' pages the default hides, in the browser, the rows where every run was a failure of Syntology's instrument; the second state shows them again. Rows are hidden, never re-ordered; these twins list every row"}},"definition":"distinct papers the archive tags; 'where Syntology ran a sample' counts papers with at least one harvested sample that ran, which is not a correctness claim"},"items":[{"url":"/paper/russiansuperglue-a-russian-language","slug":"russiansuperglue-a-russian-language","title":"RussianSuperGLUE: A Russian Language Understanding Evaluation Benchmark","date":"2020-10-29","arxiv_id":"2010.15925","repositories_listed":2,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/russiansuperglue-a-russian-language#ran","syntology_url":"https://syntology.ai/paper/2010.15925","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2010.15925"}},"official":{"repos":["RussianNLP/RussianSuperGLUE"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/textattack-a-framework-for-adversarial","slug":"textattack-a-framework-for-adversarial","title":"TextAttack: A Framework for Adversarial Attacks, Data Augmentation, and Adversarial Training in NLP","date":"2020-04-29","arxiv_id":"2005.05909","repositories_listed":2,"syntology":null},{"url":"/paper/hierarchical-density-order-embeddings","slug":"hierarchical-density-order-embeddings","title":"Hierarchical Density Order Embeddings","date":"2018-04-26","arxiv_id":"1804.09843","repositories_listed":2,"syntology":null},{"url":"/paper/emergent-visual-semantic-hierarchies-in-image","slug":"emergent-visual-semantic-hierarchies-in-image","title":"Emergent Visual-Semantic Hierarchies in Image-Text Representations","date":"2024-07-11","arxiv_id":"2407.08521","repositories_listed":1,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":3,"n_instrument":0,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":2,"n_pointer_only":3,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 1 honoured, 0 violated, 2 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/emergent-visual-semantic-hierarchies-in-image#ran","syntology_url":"https://syntology.ai/paper/2407.08521","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2407.08521"}},"official":{"repos":["TAU-VAILab/hierarcaps"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/taxollama-wordnet-based-model-for-solving","slug":"taxollama-wordnet-based-model-for-solving","title":"TaxoLLaMA: WordNet-based Model for Solving Multiple Lexical Semantic Tasks","date":"2024-03-14","arxiv_id":"2403.09207","repositories_listed":1,"syntology":null},{"url":"/paper/testing-large-language-models-on","slug":"testing-large-language-models-on","title":"Testing Large Language Models on Compositionality and Inference with Phrase-Level Adjective-Noun Entailment","date":"2022-10-01","arxiv_id":null,"repositories_listed":1,"syntology":null},{"url":"/paper/continuous-entailment-patterns-for-lexical","slug":"continuous-entailment-patterns-for-lexical","title":"Continuous Entailment Patterns for Lexical Inference in Context","date":"2021-09-08","arxiv_id":"2109.03695","repositories_listed":1,"syntology":null},{"url":"/paper/mining-knowledge-for-natural-language","slug":"mining-knowledge-for-natural-language","title":"Mining Knowledge for Natural Language Inference from Wikipedia Categories","date":"2020-10-03","arxiv_id":"2010.01239","repositories_listed":1,"syntology":{"n":2,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/mining-knowledge-for-natural-language#ran","syntology_url":"https://syntology.ai/paper/2010.01239","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2010.01239"}},"official":{"repos":["ZeweiChu/WikiNLI"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/modular-representation-underlies-systematic","slug":"modular-representation-underlies-systematic","title":"Neural Natural Language Inference Models Partially Embed Theories of Lexical Entailment and Negation","date":"2020-04-30","arxiv_id":"2004.14623","repositories_listed":1,"syntology":null},{"url":"/paper/cate-category-name-guidedword-embedding","slug":"cate-category-name-guidedword-embedding","title":"Discriminative Topic Mining via Category-Name Guided Text Embedding","date":"2019-08-20","arxiv_id":"1908.07162","repositories_listed":1,"syntology":null},{"url":"/paper/generalized-tuning-of-distributional-word","slug":"generalized-tuning-of-distributional-word","title":"Generalized Tuning of Distributional Word Vectors for Monolingual and Cross-Lingual Lexical Entailment","date":"2019-07-01","arxiv_id":null,"repositories_listed":1,"syntology":null},{"url":"/paper/sherliic-a-typed-event-focused-lexical","slug":"sherliic-a-typed-event-focused-lexical","title":"SherLIiC: A Typed Event-Focused Lexical Inference Benchmark for Evaluating Natural Language Inference","date":"2019-06-04","arxiv_id":"1906.01393","repositories_listed":1,"syntology":null},{"url":"/paper/specialising-word-vectors-for-lexical","slug":"specialising-word-vectors-for-lexical","title":"Specialising Word Vectors for Lexical Entailment","date":"2018-06-01","arxiv_id":null,"repositories_listed":1,"syntology":null},{"url":"/paper/scoring-lexical-entailment-with-a-supervised","slug":"scoring-lexical-entailment-with-a-supervised","title":"Scoring Lexical Entailment with a Supervised Directional Similarity Network","date":"2018-05-23","arxiv_id":"1805.09355","repositories_listed":1,"syntology":null},{"url":"/paper/specialising-word-vectors-for-lexical-1","slug":"specialising-word-vectors-for-lexical-1","title":"Specialising Word Vectors for Lexical Entailment","date":"2017-10-17","arxiv_id":"1710.06371","repositories_listed":1,"syntology":null},{"url":"/paper/a-consolidated-open-knowledge-representation","slug":"a-consolidated-open-knowledge-representation","title":"A Consolidated Open Knowledge Representation for Multiple Texts","date":"2017-04-01","arxiv_id":null,"repositories_listed":1,"syntology":null},{"url":"/paper/representing-meaning-with-a-combination-of","slug":"representing-meaning-with-a-combination-of","title":"Representing Meaning with a Combination of Logical and Distributional Models","date":"2015-05-26","arxiv_id":"1505.06816","repositories_listed":1,"syntology":null},{"url":"/paper/experiments-with-three-approaches-to","slug":"experiments-with-three-approaches-to","title":"Experiments with Three Approaches to Recognizing Lexical Entailment","date":"2014-01-31","arxiv_id":"1401.8269","repositories_listed":1,"syntology":null},{"url":null,"slug":"talking-the-talk-does-not-entail-walking-the","title":"Talking the Talk Does Not Entail Walking the Walk: On the Limits of Large Language Models in Lexical Entailment Recognition","date":"2024-06-21","arxiv_id":"2406.14894","repositories_listed":0,"syntology":null},{"url":null,"slug":"a-retrofitting-model-for-incorporating","title":"A Retrofitting Model for Incorporating Semantic Relations into Word Embeddings","date":"2020-12-01","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"bmeaut-at-semeval-2020-task-2-lexical","title":"BMEAUT at SemEval-2020 Task 2: Lexical Entailment with Semantic Graphs","date":"2020-12-01","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"semeval-2020-task-2-predicting-multilingual","title":"SemEval-2020 Task 2: Predicting Multilingual and Cross-Lingual (Graded) Lexical Entailment","date":"2020-12-01","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"shikeblcu-at-semeval-2020-task-2-an-external","title":"SHIKEBLCU at SemEval-2020 Task 2: An External Knowledge-enhanced Matrix for Multilingual and Cross-Lingual Lexical Entailment","date":"2020-12-01","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"ualberta-at-semeval-2020-task-2-using","title":"UAlberta at SemEval-2020 Task 2: Using Translations to Predict Cross-Lingual Entailment","date":"2020-12-01","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"visual-objects-as-context-exploiting-visual","title":"Visual Objects As Context: Exploiting Visual Objects for Lexical Entailment","date":"2020-11-01","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"hypernymy-detection-for-low-resource","title":"Hypernymy Detection for Low-Resource Languages via Meta Learning","date":"2020-07-01","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"word-attribute-prediction-enhanced-by-lexical","title":"Word Attribute Prediction Enhanced by Lexical Entailment Tasks","date":"2020-05-01","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"specializing-distributional-vectors-of-all","title":"Specializing Distributional Vectors of All Words for Lexical Entailment","date":"2019-08-01","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"multilingual-and-cross-lingual-graded-lexical","title":"Multilingual and Cross-Lingual Graded Lexical Entailment","date":"2019-07-01","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"embedding-wordnet-knowledge-for-textual","title":"Embedding WordNet Knowledge for Textual Entailment","date":"2018-08-01","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"integrating-multiplicative-features-into","title":"Integrating Multiplicative Features into Supervised Distributional Methods for Lexical Entailment","date":"2018-04-24","arxiv_id":"1804.08845","repositories_listed":0,"syntology":null},{"url":null,"slug":"learning-word-embeddings-for-hyponymy-with","title":"Learning Word Embeddings for Hyponymy with Entailment-Based Distributional Semantics","date":"2017-10-06","arxiv_id":"1710.02437","repositories_listed":0,"syntology":null},{"url":null,"slug":"hierarchical-embeddings-for-hypernymy","title":"Hierarchical Embeddings for Hypernymy Detection and Directionality","date":"2017-07-23","arxiv_id":"1707.07273","repositories_listed":0,"syntology":null},{"url":null,"slug":"word-vector-space-specialisation","title":"Word Vector Space Specialisation","date":"2017-04-01","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":"/paper/hyperlex-a-large-scale-evaluation-of-graded","slug":"hyperlex-a-large-scale-evaluation-of-graded","title":"HyperLex: A Large-Scale Evaluation of Graded Lexical Entailment","date":"2016-08-06","arxiv_id":"1608.02117","repositories_listed":0,"syntology":null},{"url":null,"slug":"dual-density-operators-and-natural-language","title":"Dual Density Operators and Natural Language Meaning","date":"2016-08-04","arxiv_id":"1608.01401","repositories_listed":0,"syntology":null},{"url":null,"slug":"a-vector-space-for-distributional-semantics","title":"A Vector Space for Distributional Semantics for Entailment","date":"2016-07-13","arxiv_id":"1607.03780","repositories_listed":0,"syntology":null},{"url":null,"slug":"sparse-bilingual-word-representations-for","title":"Sparse Bilingual Word Representations for Cross-lingual Lexical Entailment","date":"2016-06-01","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"relations-such-as-hypernymy-identifying-and","title":"Relations such as Hypernymy: Identifying and Exploiting Hearst Patterns in Distributional Vectors for Lexical Entailment","date":"2016-05-18","arxiv_id":"1605.05433","repositories_listed":0,"syntology":null},{"url":null,"slug":"graded-entailment-for-compositional","title":"Graded Entailment for Compositional Distributional Semantics","date":"2016-01-19","arxiv_id":"1601.04908","repositories_listed":0,"syntology":null},{"url":null,"slug":"exploiting-image-generality-for-lexical","title":"Exploiting Image Generality for Lexical Entailment Detection","date":"2015-07-01","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"distributional-sentence-entailment-using","title":"Distributional Sentence Entailment Using Density Matrices","date":"2015-06-22","arxiv_id":"1506.06534","repositories_listed":0,"syntology":null},{"url":null,"slug":"do-supervised-distributional-methods-really","title":"Do Supervised Distributional Methods Really Learn Lexical Inference Relations?","date":"2015-05-01","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"tiered-clustering-to-improve-lexical","title":"Tiered Clustering to Improve Lexical Entailment","date":"2014-12-02","arxiv_id":"1412.0751","repositories_listed":0,"syntology":null},{"url":null,"slug":"distributional-lexical-entailment-by-topic","title":"Distributional Lexical Entailment by Topic Coherence","date":"2014-04-01","arxiv_id":null,"repositories_listed":0,"syntology":null}],"syntology_records":3,"syntology_note":"a paper without a record is not a recorded non-run: it may lack an arXiv id or simply be absent from the graph layer"},"key_notes":{"n_ran_checked":"legacy name, kept unchanged so existing readers do not break: it counts the samples that ran with no instrument failure (honoured, violated, and ran with no contract checked); it does not mean a contract was checked, and the pages print it as 'K with no instrument failure', not 'K checked'","n_constructed":"a sub-count of the samples that ran, never subtracted from them and never a failure: an executed sample whose run returned an instance of its own class (fixture_out_type equals the entry name): the run built an object and did not compute a result (Syntology's RAN record, counts.constructed)"},"since_archive_judged":{"papers":0,"papers_placed":0,"judged_by":{"model":"a model","models_as_recorded":[],"from":"lb_rows_run_verified.jsonl: each record's `model` field (the extractor's call that wrote decisions[])"},"note":"a model's judgement from the paper's own tables that they report on this leaderboard, not a result: no value from a paper that was not placed is printed or carried here; papers counts each paper once over these leaderboards","backfill":"8,886 of 18,637 newer papers checked so far (733 more had nothing to place; 41 could not be read)","backfill_complete":false,"leaderboards":[]},"papers_where_code_ran":3,"papers_where_code_ran_first_page":"/task/lexical-entailment/papers/ran/1","papers_where_code_ran_caption":"We ran code from the paper's repository; we did not run it on this task or check it against the task's benchmarks.","pages":[{"order":"ran","page":1,"of":1,"url":"/task/lexical-entailment/papers/ran/1","json":"/task/lexical-entailment/papers/ran/1.json","rows":3}],"description_links":{"kept":0,"unwrapped_to_text":0,"bare_urls_linked":0,"relative_images_dropped":0,"rule":"internal links are kept only when the target slug exists in the catalog"},"syntology":{"read_at":"2026-10-05T00:36:11+00:00","claim":"Per-sample execution status on synthesized fixtures ('ran N of M samples'); not a correctness claim and not a ranking signal.","status_vocabulary":{"ran_honours":"ran, honoured the contract we drafted","ran_violates":"ran, violated the contract we drafted","ran_draft_wrong":"ran; our contract draft was wrong, not the code","ran_fixture":"ran; the contract check could not run (RAISES, DEP_MISSING or TIMEOUT)","ran":"ran on a synthesized input","unverified":"unverified (harvested, no recorded run)"},"row_route":"each syntology object on a benchmark's first row and on a paper carries sample_list (this site's paper page, #ran), syntology_url (syntology.ai's page for the paper, only when that route probed live) and mcp (the agent call that lists the paper's harvested samples; not a verdict on the paper)","developers":"https://syntology.ai/developers"},"not_shown":{"libraries":"the archive has no per-task library table","trend_sparklines":"the Trend column of the benchmarks table was a rendered image; it is not in the archive","social_and_latest_sorts":"stars and social signals are not in the archive"},"record_sha256":"e69e16e4cd67a42bcda0985b3f5e13db56f583b0a7e3f646dedae999a51cd5cf","record_changed_at":"2026-10-05","record_changed_at_basis":"changed"}