@inproceedings{ghasemi-hiemstra-2021-bert,
title = "{BERT} meets Cranfield: Uncovering the Properties of Full Ranking on Fully Labeled Data",
author = "Ghasemi, Negin and
Hiemstra, Djoerd",
editor = "Sorodoc, Ionut-Teodor and
Sushil, Madhumita and
Takmaz, Ece and
Agirre, Eneko",
booktitle = "Proceedings of the 16th Conference of the European Chapter of the Association for Computational Linguistics: Student Research Workshop",
month = apr,
year = "2021",
address = "Online",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2021.eacl-srw.9",
doi = "10.18653/v1/2021.eacl-srw.9",
pages = "58--64",
abstract = "Recently, various information retrieval models have been proposed based on pre-trained BERT models, achieving outstanding performance. The majority of such models have been tested on data collections with partial relevance labels, where various potentially relevant documents have not been exposed to the annotators. Therefore, evaluating BERT-based rankers may lead to biased and unfair evaluation results, simply because a relevant document has not been exposed to the annotators while creating the collection. In our work, we aim to better understand a BERT-based ranker{'}s strengths compared to a BERT-based re-ranker and the initial ranker. To this aim, we investigate BERT-based rankers performance on the Cranfield collection, which comes with full relevance judgment on all documents in the collection. Our results demonstrate the BERT-based full ranker{'}s effectiveness, as opposed to the BERT-based re-ranker and BM25. Also, analysis shows that there are documents that the BERT-based full-ranker finds that were not found by the initial ranker.",
}
<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="ghasemi-hiemstra-2021-bert">
<titleInfo>
<title>BERT meets Cranfield: Uncovering the Properties of Full Ranking on Fully Labeled Data</title>
</titleInfo>
<name type="personal">
<namePart type="given">Negin</namePart>
<namePart type="family">Ghasemi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Djoerd</namePart>
<namePart type="family">Hiemstra</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2021-04</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 16th Conference of the European Chapter of the Association for Computational Linguistics: Student Research Workshop</title>
</titleInfo>
<name type="personal">
<namePart type="given">Ionut-Teodor</namePart>
<namePart type="family">Sorodoc</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Madhumita</namePart>
<namePart type="family">Sushil</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ece</namePart>
<namePart type="family">Takmaz</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Eneko</namePart>
<namePart type="family">Agirre</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Online</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>Recently, various information retrieval models have been proposed based on pre-trained BERT models, achieving outstanding performance. The majority of such models have been tested on data collections with partial relevance labels, where various potentially relevant documents have not been exposed to the annotators. Therefore, evaluating BERT-based rankers may lead to biased and unfair evaluation results, simply because a relevant document has not been exposed to the annotators while creating the collection. In our work, we aim to better understand a BERT-based ranker’s strengths compared to a BERT-based re-ranker and the initial ranker. To this aim, we investigate BERT-based rankers performance on the Cranfield collection, which comes with full relevance judgment on all documents in the collection. Our results demonstrate the BERT-based full ranker’s effectiveness, as opposed to the BERT-based re-ranker and BM25. Also, analysis shows that there are documents that the BERT-based full-ranker finds that were not found by the initial ranker.</abstract>
<identifier type="citekey">ghasemi-hiemstra-2021-bert</identifier>
<identifier type="doi">10.18653/v1/2021.eacl-srw.9</identifier>
<location>
<url>https://aclanthology.org/2021.eacl-srw.9</url>
</location>
<part>
<date>2021-04</date>
<extent unit="page">
<start>58</start>
<end>64</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T BERT meets Cranfield: Uncovering the Properties of Full Ranking on Fully Labeled Data
%A Ghasemi, Negin
%A Hiemstra, Djoerd
%Y Sorodoc, Ionut-Teodor
%Y Sushil, Madhumita
%Y Takmaz, Ece
%Y Agirre, Eneko
%S Proceedings of the 16th Conference of the European Chapter of the Association for Computational Linguistics: Student Research Workshop
%D 2021
%8 April
%I Association for Computational Linguistics
%C Online
%F ghasemi-hiemstra-2021-bert
%X Recently, various information retrieval models have been proposed based on pre-trained BERT models, achieving outstanding performance. The majority of such models have been tested on data collections with partial relevance labels, where various potentially relevant documents have not been exposed to the annotators. Therefore, evaluating BERT-based rankers may lead to biased and unfair evaluation results, simply because a relevant document has not been exposed to the annotators while creating the collection. In our work, we aim to better understand a BERT-based ranker’s strengths compared to a BERT-based re-ranker and the initial ranker. To this aim, we investigate BERT-based rankers performance on the Cranfield collection, which comes with full relevance judgment on all documents in the collection. Our results demonstrate the BERT-based full ranker’s effectiveness, as opposed to the BERT-based re-ranker and BM25. Also, analysis shows that there are documents that the BERT-based full-ranker finds that were not found by the initial ranker.
%R 10.18653/v1/2021.eacl-srw.9
%U https://aclanthology.org/2021.eacl-srw.9
%U https://doi.org/10.18653/v1/2021.eacl-srw.9
%P 58-64
Markdown (Informal)
[BERT meets Cranfield: Uncovering the Properties of Full Ranking on Fully Labeled Data](https://aclanthology.org/2021.eacl-srw.9) (Ghasemi & Hiemstra, EACL 2021)
ACL