@inproceedings{mazuecos-etal-2020-role,
title = "On the role of effective and referring questions in {G}uess{W}hat?!",
author = "Mazuecos, Mauricio and
Testoni, Alberto and
Bernardi, Raffaella and
Benotti, Luciana",
editor = "Wang, Xin and
Thomason, Jesse and
Hu, Ronghang and
Chen, Xinlei and
Anderson, Peter and
Wu, Qi and
Celikyilmaz, Asli and
Baldridge, Jason and
Wang, William Yang",
booktitle = "Proceedings of the First Workshop on Advances in Language and Vision Research",
month = jul,
year = "2020",
address = "Online",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2020.alvr-1.4",
doi = "10.18653/v1/2020.alvr-1.4",
pages = "19--25",
abstract = "Task success is the standard metric used to evaluate referential visual dialogue systems. In this paper we propose two new metrics that evaluate how each question contributes to the goal. First, we measure how effective each question is by evaluating whether the question discards objects that are not the referent. Second, we define referring questions as those that univocally identify one object in the image. We report the new metrics for human dialogues and for state of the art publicly available models on GuessWhat?!. Regarding our first metric, we find that successful dialogues do not have a higher percentage of effective questions for most models. With respect to the second metric, humans make questions at the end of the dialogue that are referring, confirming their guess before guessing. Human dialogues that use this strategy have a higher task success but models do not seem to learn it.",
}
<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="mazuecos-etal-2020-role">
<titleInfo>
<title>On the role of effective and referring questions in GuessWhat?!</title>
</titleInfo>
<name type="personal">
<namePart type="given">Mauricio</namePart>
<namePart type="family">Mazuecos</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alberto</namePart>
<namePart type="family">Testoni</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Raffaella</namePart>
<namePart type="family">Bernardi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Luciana</namePart>
<namePart type="family">Benotti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2020-07</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the First Workshop on Advances in Language and Vision Research</title>
</titleInfo>
<name type="personal">
<namePart type="given">Xin</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jesse</namePart>
<namePart type="family">Thomason</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ronghang</namePart>
<namePart type="family">Hu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xinlei</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Peter</namePart>
<namePart type="family">Anderson</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Qi</namePart>
<namePart type="family">Wu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Asli</namePart>
<namePart type="family">Celikyilmaz</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jason</namePart>
<namePart type="family">Baldridge</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">William</namePart>
<namePart type="given">Yang</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Online</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>Task success is the standard metric used to evaluate referential visual dialogue systems. In this paper we propose two new metrics that evaluate how each question contributes to the goal. First, we measure how effective each question is by evaluating whether the question discards objects that are not the referent. Second, we define referring questions as those that univocally identify one object in the image. We report the new metrics for human dialogues and for state of the art publicly available models on GuessWhat?!. Regarding our first metric, we find that successful dialogues do not have a higher percentage of effective questions for most models. With respect to the second metric, humans make questions at the end of the dialogue that are referring, confirming their guess before guessing. Human dialogues that use this strategy have a higher task success but models do not seem to learn it.</abstract>
<identifier type="citekey">mazuecos-etal-2020-role</identifier>
<identifier type="doi">10.18653/v1/2020.alvr-1.4</identifier>
<location>
<url>https://aclanthology.org/2020.alvr-1.4</url>
</location>
<part>
<date>2020-07</date>
<extent unit="page">
<start>19</start>
<end>25</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T On the role of effective and referring questions in GuessWhat?!
%A Mazuecos, Mauricio
%A Testoni, Alberto
%A Bernardi, Raffaella
%A Benotti, Luciana
%Y Wang, Xin
%Y Thomason, Jesse
%Y Hu, Ronghang
%Y Chen, Xinlei
%Y Anderson, Peter
%Y Wu, Qi
%Y Celikyilmaz, Asli
%Y Baldridge, Jason
%Y Wang, William Yang
%S Proceedings of the First Workshop on Advances in Language and Vision Research
%D 2020
%8 July
%I Association for Computational Linguistics
%C Online
%F mazuecos-etal-2020-role
%X Task success is the standard metric used to evaluate referential visual dialogue systems. In this paper we propose two new metrics that evaluate how each question contributes to the goal. First, we measure how effective each question is by evaluating whether the question discards objects that are not the referent. Second, we define referring questions as those that univocally identify one object in the image. We report the new metrics for human dialogues and for state of the art publicly available models on GuessWhat?!. Regarding our first metric, we find that successful dialogues do not have a higher percentage of effective questions for most models. With respect to the second metric, humans make questions at the end of the dialogue that are referring, confirming their guess before guessing. Human dialogues that use this strategy have a higher task success but models do not seem to learn it.
%R 10.18653/v1/2020.alvr-1.4
%U https://aclanthology.org/2020.alvr-1.4
%U https://doi.org/10.18653/v1/2020.alvr-1.4
%P 19-25
Markdown (Informal)
[On the role of effective and referring questions in GuessWhat?!](https://aclanthology.org/2020.alvr-1.4) (Mazuecos et al., ALVR 2020)
ACL