@inproceedings{chau-etal-2020-understanding,
title = "Understanding the Tradeoff between Cost and Quality of Expert Annotations for Keyphrase Extraction",
author = "Chau, Hung and
Balaneshin, Saeid and
Liu, Kai and
Linda, Ondrej",
editor = "Dipper, Stefanie and
Zeldes, Amir",
booktitle = "Proceedings of the 14th Linguistic Annotation Workshop",
month = dec,
year = "2020",
address = "Barcelona, Spain",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2020.law-1.7",
pages = "74--86",
abstract = "Generating expert ground truth annotations of documents can be a very expensive process. However, such annotations are essential for training domain-specific keyphrase extraction models, especially when utilizing data-intensive deep learning models in unique domains such as real-estate. Therefore, it is critical to optimize the manual annotation process to maximize the quality of the annotations while minimizing the cost of manual labor. To address this need, we explore multiple annotation strategies including self-review and peer-review as well as various methods of resolving annotator disagreements. We evaluate these annotation strategies with respect to their cost and on the task of learning keyphrase extraction models applied with an experimental dataset in the real-estate domain. The results demonstrate that different annotation strategies should be considered depending on specific metrics such as precision and recall.",
}
<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="chau-etal-2020-understanding">
<titleInfo>
<title>Understanding the Tradeoff between Cost and Quality of Expert Annotations for Keyphrase Extraction</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung</namePart>
<namePart type="family">Chau</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Saeid</namePart>
<namePart type="family">Balaneshin</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kai</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ondrej</namePart>
<namePart type="family">Linda</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2020-12</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 14th Linguistic Annotation Workshop</title>
</titleInfo>
<name type="personal">
<namePart type="given">Stefanie</namePart>
<namePart type="family">Dipper</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Amir</namePart>
<namePart type="family">Zeldes</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Barcelona, Spain</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>Generating expert ground truth annotations of documents can be a very expensive process. However, such annotations are essential for training domain-specific keyphrase extraction models, especially when utilizing data-intensive deep learning models in unique domains such as real-estate. Therefore, it is critical to optimize the manual annotation process to maximize the quality of the annotations while minimizing the cost of manual labor. To address this need, we explore multiple annotation strategies including self-review and peer-review as well as various methods of resolving annotator disagreements. We evaluate these annotation strategies with respect to their cost and on the task of learning keyphrase extraction models applied with an experimental dataset in the real-estate domain. The results demonstrate that different annotation strategies should be considered depending on specific metrics such as precision and recall.</abstract>
<identifier type="citekey">chau-etal-2020-understanding</identifier>
<location>
<url>https://aclanthology.org/2020.law-1.7</url>
</location>
<part>
<date>2020-12</date>
<extent unit="page">
<start>74</start>
<end>86</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Understanding the Tradeoff between Cost and Quality of Expert Annotations for Keyphrase Extraction
%A Chau, Hung
%A Balaneshin, Saeid
%A Liu, Kai
%A Linda, Ondrej
%Y Dipper, Stefanie
%Y Zeldes, Amir
%S Proceedings of the 14th Linguistic Annotation Workshop
%D 2020
%8 December
%I Association for Computational Linguistics
%C Barcelona, Spain
%F chau-etal-2020-understanding
%X Generating expert ground truth annotations of documents can be a very expensive process. However, such annotations are essential for training domain-specific keyphrase extraction models, especially when utilizing data-intensive deep learning models in unique domains such as real-estate. Therefore, it is critical to optimize the manual annotation process to maximize the quality of the annotations while minimizing the cost of manual labor. To address this need, we explore multiple annotation strategies including self-review and peer-review as well as various methods of resolving annotator disagreements. We evaluate these annotation strategies with respect to their cost and on the task of learning keyphrase extraction models applied with an experimental dataset in the real-estate domain. The results demonstrate that different annotation strategies should be considered depending on specific metrics such as precision and recall.
%U https://aclanthology.org/2020.law-1.7
%P 74-86
Markdown (Informal)
[Understanding the Tradeoff between Cost and Quality of Expert Annotations for Keyphrase Extraction](https://aclanthology.org/2020.law-1.7) (Chau et al., LAW 2020)
ACL