@inproceedings{chung-etal-2020-extremely,
title = "Extremely Low Bit Transformer Quantization for On-Device Neural Machine Translation",
author = "Chung, Insoo and
Kim, Byeongwook and
Choi, Yoonjung and
Kwon, Se Jung and
Jeon, Yongkweon and
Park, Baeseong and
Kim, Sangha and
Lee, Dongsoo",
editor = "Cohn, Trevor and
He, Yulan and
Liu, Yang",
booktitle = "Findings of the Association for Computational Linguistics: EMNLP 2020",
month = nov,
year = "2020",
address = "Online",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2020.findings-emnlp.433",
doi = "10.18653/v1/2020.findings-emnlp.433",
pages = "4812--4826",
abstract = "The deployment of widely used Transformer architecture is challenging because of heavy computation load and memory overhead during inference, especially when the target device is limited in computational resources such as mobile or edge devices. Quantization is an effective technique to address such challenges. Our analysis shows that for a given number of quantization bits, each block of Transformer contributes to translation quality and inference computations in different manners. Moreover, even inside an embedding block, each word presents vastly different contributions. Correspondingly, we propose a mixed precision quantization strategy to represent Transformer weights by an extremely low number of bits (e.g., under 3 bits). For example, for each word in an embedding block, we assign different quantization bits based on statistical property. Our quantized Transformer model achieves 11.8{\mbox{$\times$}} smaller model size than the baseline model, with less than -0.5 BLEU. We achieve 8.3{\mbox{$\times$}} reduction in run-time memory footprints and 3.5{\mbox{$\times$}} speed up (Galaxy N10+) such that our proposed compression strategy enables efficient implementation for on-device NMT.",
}
<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="chung-etal-2020-extremely">
<titleInfo>
<title>Extremely Low Bit Transformer Quantization for On-Device Neural Machine Translation</title>
</titleInfo>
<name type="personal">
<namePart type="given">Insoo</namePart>
<namePart type="family">Chung</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Byeongwook</namePart>
<namePart type="family">Kim</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yoonjung</namePart>
<namePart type="family">Choi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Se</namePart>
<namePart type="given">Jung</namePart>
<namePart type="family">Kwon</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yongkweon</namePart>
<namePart type="family">Jeon</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Baeseong</namePart>
<namePart type="family">Park</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sangha</namePart>
<namePart type="family">Kim</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Dongsoo</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2020-11</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2020</title>
</titleInfo>
<name type="personal">
<namePart type="given">Trevor</namePart>
<namePart type="family">Cohn</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yulan</namePart>
<namePart type="family">He</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yang</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Online</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>The deployment of widely used Transformer architecture is challenging because of heavy computation load and memory overhead during inference, especially when the target device is limited in computational resources such as mobile or edge devices. Quantization is an effective technique to address such challenges. Our analysis shows that for a given number of quantization bits, each block of Transformer contributes to translation quality and inference computations in different manners. Moreover, even inside an embedding block, each word presents vastly different contributions. Correspondingly, we propose a mixed precision quantization strategy to represent Transformer weights by an extremely low number of bits (e.g., under 3 bits). For example, for each word in an embedding block, we assign different quantization bits based on statistical property. Our quantized Transformer model achieves 11.8\times smaller model size than the baseline model, with less than -0.5 BLEU. We achieve 8.3\times reduction in run-time memory footprints and 3.5\times speed up (Galaxy N10+) such that our proposed compression strategy enables efficient implementation for on-device NMT.</abstract>
<identifier type="citekey">chung-etal-2020-extremely</identifier>
<identifier type="doi">10.18653/v1/2020.findings-emnlp.433</identifier>
<location>
<url>https://aclanthology.org/2020.findings-emnlp.433</url>
</location>
<part>
<date>2020-11</date>
<extent unit="page">
<start>4812</start>
<end>4826</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Extremely Low Bit Transformer Quantization for On-Device Neural Machine Translation
%A Chung, Insoo
%A Kim, Byeongwook
%A Choi, Yoonjung
%A Kwon, Se Jung
%A Jeon, Yongkweon
%A Park, Baeseong
%A Kim, Sangha
%A Lee, Dongsoo
%Y Cohn, Trevor
%Y He, Yulan
%Y Liu, Yang
%S Findings of the Association for Computational Linguistics: EMNLP 2020
%D 2020
%8 November
%I Association for Computational Linguistics
%C Online
%F chung-etal-2020-extremely
%X The deployment of widely used Transformer architecture is challenging because of heavy computation load and memory overhead during inference, especially when the target device is limited in computational resources such as mobile or edge devices. Quantization is an effective technique to address such challenges. Our analysis shows that for a given number of quantization bits, each block of Transformer contributes to translation quality and inference computations in different manners. Moreover, even inside an embedding block, each word presents vastly different contributions. Correspondingly, we propose a mixed precision quantization strategy to represent Transformer weights by an extremely low number of bits (e.g., under 3 bits). For example, for each word in an embedding block, we assign different quantization bits based on statistical property. Our quantized Transformer model achieves 11.8\times smaller model size than the baseline model, with less than -0.5 BLEU. We achieve 8.3\times reduction in run-time memory footprints and 3.5\times speed up (Galaxy N10+) such that our proposed compression strategy enables efficient implementation for on-device NMT.
%R 10.18653/v1/2020.findings-emnlp.433
%U https://aclanthology.org/2020.findings-emnlp.433
%U https://doi.org/10.18653/v1/2020.findings-emnlp.433
%P 4812-4826
Markdown (Informal)
[Extremely Low Bit Transformer Quantization for On-Device Neural Machine Translation](https://aclanthology.org/2020.findings-emnlp.433) (Chung et al., Findings 2020)
ACL