{"about":{"non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","site":"https://codewithpapers.app","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page","syntology":{"site":"https://syntology.ai","developers":"https://syntology.ai/developers","mcp":{"server":"https://syntology.ai/mcp","transport":"streamable-http","server_card":"https://syntology.ai/.well-known/mcp/server-card.json","auth":{"type":"trial token, no account","trial_token":"https://syntology.ai/api/oauth/trial/token","method":"POST","docs":"https://syntology.ai/developers"}},"have":"https://syntology.ai/api/graph/have?x=<method, arXiv id or title> (free, answers coverage only)","paper_base":"https://syntology.ai/paper/","atlas_base":"https://app.syntology.ai/?focus="},"machine_readable":[{"url":"https://codewithpapers.app/llms.txt","what":"the machine catalog: every machine-readable file, counted"},{"url":"https://codewithpapers.app/index/manifest.json","what":"paper-to-code index by arXiv id, with Syntology's counts"},{"url":"https://codewithpapers.app/search/manifest.json","what":"site search index (titles, authors) and its files"},{"url":"https://codewithpapers.app/download","what":"bulk files: Syntology's layer, described there"},{"url":"https://codewithpapers.app/build_manifest.json","what":"the build record: inputs, counts, exclusions, probes"}]},"url":"/paper/wic-tsv-an-evaluation-benchmark-for-target","title":"WiC-TSV: An Evaluation Benchmark for Target Sense Verification of Words in Context","arxiv_id":"2004.15016","date":"2020-04-30","proceeding":"EACL 2021 2","authors":["Anna Breit","Artem Revenko","Kiamehr Rezaee","Mohammad Taher Pilehvar","Jose Camacho-Collados"],"abstract":"We present WiC-TSV, a new multi-domain evaluation benchmark for Word Sense Disambiguation. More specifically, we introduce a framework for Target Sense Verification of Words in Context which grounds its uniqueness in the formulation as a binary classification task thus being independent of external sense inventories, and the coverage of various domains. This makes the dataset highly flexible for the evaluation of a diverse set of models and systems in and across domains. WiC-TSV provides three different evaluation settings, depending on the input signals provided to the model. We set baseline performance on the dataset using state-of-the-art language models. Experimental results show that even though these models can perform decently on the task, there remains a gap between machine and human performance, especially in out-of-domain settings. WiC-TSV data is available at https://competitions.codalab.org/competitions/23683","url_abs":"https://arxiv.org/abs/2004.15016v3","url_pdf":"https://arxiv.org/pdf/2004.15016v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"wic-tsv-an-evaluation-benchmark-for-target","repo_url":"https://github.com/semantic-web-company/wic-tsv","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"binary-classification","task_name":"Binary Classification"},{"task_slug":"entity-linking","task_name":"Entity Linking"},{"task_slug":"word-sense-disambiguation","task_name":"Word Sense Disambiguation"}],"methods":[{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"attention-dropout","method_name":"Attention Dropout"},{"method_slug":"bert","method_name":"BERT"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"linear-warmup-with-linear-decay","method_name":"Linear Warmup With Linear Decay"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"weight-decay","method_name":"Weight Decay"},{"method_slug":"wordpiece","method_name":"WordPiece"}],"datasets_introduced":[{"slug":"wic-tsv","name":"WiC-TSV","full_name":"Words-in-Context: Target Sense Verification"}],"methods_introduced":[],"results":[{"leaderboard":"/sota/entity-linking-on-wic-tsv","task":"Entity Linking","dataset":"WiC-TSV","model":"Bert-base","rank_in_archive_order":4,"of":8,"metrics":{"Task 1 Accuracy: all":"75.3","Task 1 Accuracy: domain specific":"77.9","Task 1 Accuracy: general purpose":"73.3","Task 2 Accuracy: all":"71.7","Task 2 Accuracy: domain specific":"74.7","Task 2 Accuracy: general purpose":"68.6","Task 3 Accuracy: all":"76.6","Task 3 Accuracy: domain specific":"80.4","Task 3 Accuracy: general purpose":"73.5"},"uses_additional_data":false},{"leaderboard":"/sota/entity-linking-on-wic-tsv","task":"Entity Linking","dataset":"WiC-TSV","model":"Unsupervised Bert","rank_in_archive_order":5,"of":8,"metrics":{"Task 1 Accuracy: all":"54.4","Task 1 Accuracy: domain specific":"60.6","Task 1 Accuracy: general purpose":"49.2","Task 2 Accuracy: all":"62.8","Task 2 Accuracy: domain specific":"69.1","Task 2 Accuracy: general purpose":"57.6","Task 3 Accuracy: all":"60.5","Task 3 Accuracy: domain specific":"67.9","Task 3 Accuracy: general purpose":"54.4"},"uses_additional_data":false},{"leaderboard":"/sota/entity-linking-on-wic-tsv","task":"Entity Linking","dataset":"WiC-TSV","model":"FastText","rank_in_archive_order":6,"of":8,"metrics":{"Task 1 Accuracy: all":"53.7","Task 1 Accuracy: domain specific":"50.6","Task 1 Accuracy: general purpose":"56.2","Task 2 Accuracy: all":"52.7","Task 2 Accuracy: domain specific":"47.7","Task 2 Accuracy: general purpose":"56.8","Task 3 Accuracy: all":"53.4","Task 3 Accuracy: domain specific":"49.0","Task 3 Accuracy: general purpose":"57.1"},"uses_additional_data":false},{"leaderboard":"/sota/entity-linking-on-wic-tsv","task":"Entity Linking","dataset":"WiC-TSV","model":"All true","rank_in_archive_order":7,"of":8,"metrics":{"Task 1 Accuracy: all":"50.8","Task 1 Accuracy: domain specific":"47.0","Task 1 Accuracy: general purpose":"53.8","Task 2 Accuracy: all":"50.8","Task 2 Accuracy: domain specific":"47.0","Task 2 Accuracy: general purpose":"53.8","Task 3 Accuracy: all":"50.8","Task 3 Accuracy: domain specific":"47.0","Task 3 Accuracy: general purpose":"53.8"},"uses_additional_data":false},{"leaderboard":"/sota/entity-linking-on-wic-tsv","task":"Entity Linking","dataset":"WiC-TSV","model":"Human","rank_in_archive_order":8,"of":8,"metrics":{"Task 3 Accuracy: all":"85.3","Task 3 Accuracy: domain specific":"89.2","Task 3 Accuracy: general purpose":"82.1"},"uses_additional_data":false},{"leaderboard":"/sota/word-sense-disambiguation-on-wic-tsv","task":"Word Sense Disambiguation","dataset":"WiC-TSV","model":"Bert-base","rank_in_archive_order":4,"of":8,"metrics":{"Task 1 Accuracy: all":"75.3","Task 1 Accuracy: domain specific":"77.9","Task 1 Accuracy: general purpose":"73.3","Task 2 Accuracy: all":"71.7","Task 2 Accuracy: domain specific":"74.7","Task 2 Accuracy: general purpose":"68.6","Task 3 Accuracy: all":"76.6","Task 3 Accuracy: domain specific":"80.4","Task 3 Accuracy: general purpose":"73.5"},"uses_additional_data":false},{"leaderboard":"/sota/word-sense-disambiguation-on-wic-tsv","task":"Word Sense Disambiguation","dataset":"WiC-TSV","model":"Unsupervised Bert","rank_in_archive_order":5,"of":8,"metrics":{"Task 1 Accuracy: all":"54.4","Task 1 Accuracy: domain specific":"60.6","Task 1 Accuracy: general purpose":"49.2","Task 2 Accuracy: all":"62.8","Task 2 Accuracy: domain specific":"69.1","Task 2 Accuracy: general purpose":"57.6","Task 3 Accuracy: all":"60.5","Task 3 Accuracy: domain specific":"67.9","Task 3 Accuracy: general purpose":"54.4"},"uses_additional_data":false},{"leaderboard":"/sota/word-sense-disambiguation-on-wic-tsv","task":"Word Sense Disambiguation","dataset":"WiC-TSV","model":"FastText","rank_in_archive_order":6,"of":8,"metrics":{"Task 1 Accuracy: all":"53.7","Task 1 Accuracy: domain specific":"50.6","Task 1 Accuracy: general purpose":"56.2","Task 2 Accuracy: all":"52.7","Task 2 Accuracy: domain specific":"47.7","Task 2 Accuracy: general purpose":"56.8","Task 3 Accuracy: all":"53.4","Task 3 Accuracy: domain specific":"49.0","Task 3 Accuracy: general purpose":"57.1"},"uses_additional_data":false},{"leaderboard":"/sota/word-sense-disambiguation-on-wic-tsv","task":"Word Sense Disambiguation","dataset":"WiC-TSV","model":"All true","rank_in_archive_order":7,"of":8,"metrics":{"Task 1 Accuracy: all":"50.8","Task 1 Accuracy: domain specific":"47.0","Task 1 Accuracy: general purpose":"53.8","Task 2 Accuracy: all":"50.8","Task 2 Accuracy: domain specific":"47.0","Task 2 Accuracy: general purpose":"53.8","Task 3 Accuracy: all":"50.8","Task 3 Accuracy: domain specific":"47.0","Task 3 Accuracy: general purpose":"53.8"},"uses_additional_data":false},{"leaderboard":"/sota/word-sense-disambiguation-on-wic-tsv","task":"Word Sense Disambiguation","dataset":"WiC-TSV","model":"Human","rank_in_archive_order":8,"of":8,"metrics":{"Task 3 Accuracy: all":"85.3","Task 3 Accuracy: domain specific":"89.2","Task 3 Accuracy: general purpose":"82.1"},"uses_additional_data":false}],"syntology":{"syntology_url":null,"atlas_url":null,"mcp":null,"developers":"https://syntology.ai/developers","held":null,"reason":"arxiv_id_not_in_atlas","checked_against":{"atlas_ids":549449,"atlas_ids_sha256":"afd72399dafd9906e9062927f70655952385a3d75488558e1f0484e950ab09b7","read_at":"2026-10-05T01:46:21Z"},"by_repo":[{"repo":"semantic-web-company/wic-tsv","kind":"official","samples":0,"ran":0,"instrument":0,"unverified":0}],"by_repo_is":"one row per repository in the archive's code links, Syntology's graph links or the samples; samples 0 means no sample is linked to this paper, a statement about Syntology's coverage, not about the repository: with harvested_for_other_papers present, Syntology harvested the repository for that many other papers, and without it, Syntology harvested nothing from it; placed_by_identical_code counts samples whose link records no repository, placed here because identical code was harvested from this repository for another paper; ran counts executions on a synthesized input and instrument counts failures of Syntology's instrument, not of the code"},"code_links_note":null,"arxiv_metadata":null,"syntology_extracted_results":null,"record_sha256":"42b9e4ebcb0566bf24311fada079cef75e5841bfd9ad8bb9a16bb6f14eaab769","record_changed_at":"2026-09-28","record_changed_at_basis":"first_hashed"}