@inproceedings{mordido-meinel-2020-mark,
title = "Mark-Evaluate: Assessing Language Generation using Population Estimation Methods",
author = "Mordido, Gon{\c{c}}alo and
Meinel, Christoph",
editor = "Scott, Donia and
Bel, Nuria and
Zong, Chengqing",
booktitle = "Proceedings of the 28th International Conference on Computational Linguistics",
month = dec,
year = "2020",
address = "Barcelona, Spain (Online)",
publisher = "International Committee on Computational Linguistics",
url = "https://aclanthology.org/2020.coling-main.178",
doi = "10.18653/v1/2020.coling-main.178",
pages = "1963--1977",
abstract = "We propose a family of metrics to assess language generation derived from population estimation methods widely used in ecology. More specifically, we use mark-recapture and maximum-likelihood methods that have been applied over the past several decades to estimate the size of closed populations in the wild. We propose three novel metrics: $\textrm{ME}_{\textrm{Petersen}}$ and $\textrm{ME}_{\textrm{CAPTURE}}$, which retrieve a single-valued assessment, and $\textrm{ME}_{\textrm{Schnabel}}$ which returns a double-valued metric to assess the evaluation set in terms of quality and diversity, separately. In synthetic experiments, our family of methods is sensitive to drops in quality and diversity. Moreover, our methods show a higher correlation to human evaluation than existing metrics on several challenging tasks, namely unconditional language generation, machine translation, and text summarization.",
}
<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="mordido-meinel-2020-mark">
<titleInfo>
<title>Mark-Evaluate: Assessing Language Generation using Population Estimation Methods</title>
</titleInfo>
<name type="personal">
<namePart type="given">Gonçalo</namePart>
<namePart type="family">Mordido</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Christoph</namePart>
<namePart type="family">Meinel</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2020-12</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 28th International Conference on Computational Linguistics</title>
</titleInfo>
<name type="personal">
<namePart type="given">Donia</namePart>
<namePart type="family">Scott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nuria</namePart>
<namePart type="family">Bel</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Chengqing</namePart>
<namePart type="family">Zong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>International Committee on Computational Linguistics</publisher>
<place>
<placeTerm type="text">Barcelona, Spain (Online)</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>We propose a family of metrics to assess language generation derived from population estimation methods widely used in ecology. More specifically, we use mark-recapture and maximum-likelihood methods that have been applied over the past several decades to estimate the size of closed populations in the wild. We propose three novel metrics: ME_Petersen and ME_CAPTURE, which retrieve a single-valued assessment, and ME_Schnabel which returns a double-valued metric to assess the evaluation set in terms of quality and diversity, separately. In synthetic experiments, our family of methods is sensitive to drops in quality and diversity. Moreover, our methods show a higher correlation to human evaluation than existing metrics on several challenging tasks, namely unconditional language generation, machine translation, and text summarization.</abstract>
<identifier type="citekey">mordido-meinel-2020-mark</identifier>
<identifier type="doi">10.18653/v1/2020.coling-main.178</identifier>
<location>
<url>https://aclanthology.org/2020.coling-main.178</url>
</location>
<part>
<date>2020-12</date>
<extent unit="page">
<start>1963</start>
<end>1977</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Mark-Evaluate: Assessing Language Generation using Population Estimation Methods
%A Mordido, Gonçalo
%A Meinel, Christoph
%Y Scott, Donia
%Y Bel, Nuria
%Y Zong, Chengqing
%S Proceedings of the 28th International Conference on Computational Linguistics
%D 2020
%8 December
%I International Committee on Computational Linguistics
%C Barcelona, Spain (Online)
%F mordido-meinel-2020-mark
%X We propose a family of metrics to assess language generation derived from population estimation methods widely used in ecology. More specifically, we use mark-recapture and maximum-likelihood methods that have been applied over the past several decades to estimate the size of closed populations in the wild. We propose three novel metrics: ME_Petersen and ME_CAPTURE, which retrieve a single-valued assessment, and ME_Schnabel which returns a double-valued metric to assess the evaluation set in terms of quality and diversity, separately. In synthetic experiments, our family of methods is sensitive to drops in quality and diversity. Moreover, our methods show a higher correlation to human evaluation than existing metrics on several challenging tasks, namely unconditional language generation, machine translation, and text summarization.
%R 10.18653/v1/2020.coling-main.178
%U https://aclanthology.org/2020.coling-main.178
%U https://doi.org/10.18653/v1/2020.coling-main.178
%P 1963-1977
Markdown (Informal)
[Mark-Evaluate: Assessing Language Generation using Population Estimation Methods](https://aclanthology.org/2020.coling-main.178) (Mordido & Meinel, COLING 2020)
ACL