%PDF-1.1
%
2 0 obj
<<
/Length 8628
>>
stream
BT
/T17 1 Tf
0.12 0 0 -0.12 80.4 670.78 Tm
/CS1 cs 0 sc
BX /GS1 gs EX
0 Tc
(DESCRIPTION)Tj
1000 0 TD
(OF)Tj
231 0 TD
(THE)Tj
331 0 TD
(UPENN)Tj
539 0 TD
(CAMP)Tj
463 0 TD
(SYSTEM)Tj
605 0 TD
(AS)Tj
221 0 TD
(USED)Tj
-2193 149 TD
(F)Tj
81 0 TD
(OR)Tj
246 0 TD
(COREFERENCE)Tj
/T16 1 Tf
-1419 253 TD
(Brec)Tj
190 0 TD
(k)Tj
85 0 TD
[(Baldwin,)-31000(T)]TJ
472 0 TD
[(om)-32000(Morton,)]TJ
540 0 TD
[(Amit)-31000(Bagga,)-31000(Jason)]TJ
848 0 TD
[(Baldridge,)-31000(Raman)-32000(Chandrasek)]TJ
1305 0 TD
(er,)Tj
-2830 116 TD
[(Alexis)-32000(Dimitriadis,)-28000(Kieran)-32000(Sn)]TJ
1259 0 TD
(yder,)Tj
247 0 TD
[(Magdalena)-32000(W)]TJ
588 0 TD
(olsk)Tj
161 0 TD
(a,)Tj
/T15 1 Tf
-1865 116 TD
[(Institute)-28000(for)-28000(Researc)]TJ
745 0 TD
[(h)-26000(i)0(n)-27000(Cognitiv)]TJ
478 0 TD
[(e)-27000(Science)]TJ
-853 117 TD
[(3401)-26000(W)]TJ
272 0 TD
(aln)Tj
109 0 TD
[(ut)-28000(St.)]TJ
244 0 TD
(400C)Tj
-761 116 TD
[(Philadelphia,)-27000(P)]TJ
554 0 TD
[(A)-28000(19104.)]TJ
357 0 TD
(USA)Tj
-781 116 TD
(Phone:)Tj
287 0 TD
[(\(215\))-27000(898-0329)]TJ
-240 116 TD
(F)Tj
47 0 TD
(ax:)Tj
146 0 TD
[(\(215\))-27000(573-9247)]TJ
-634 117 TD
(Email:)Tj
/T14 1 Tf
273 0 TD
(f)Tj
/T15 1 Tf
42 0 TD
(brec)Tj
151 0 TD
(k,tsmorton)Tj
/T14 1 Tf
396 0 TD
(g)Tj
/T15 1 Tf
42 0 TD
[(@linc.cis.up)-2000(enn.edu)]TJ
/T13 1 Tf
-2049 332 TD
(In)Tj
101 0 TD
[(tro)-3000(duction)]TJ
/T15 1 Tf
24 236 TD
[(In)-28000(this)-29000(pap)-2000(er)-28000(w)]TJ
559 0 TD
[(e)-28000(presen)]TJ
295 0 TD
[(t)-28000(some)-27000(adv)]TJ
395 0 TD
[(ances)-28000(made)-28000(to)-28000(the)-29000(CAMP)-29000(system)-28000(since)-28000(it's)-29000(inception)-28000(for)-28000(MUC-6.)]TJ
2413 0 TD
(Al-)Tj
-3787 100 TD
[(though)-23000(the)-23000(infrastructure)-22000(has)-22000(b)-2000(een)-23000(completely)-23000(re-implemen)]TJ
2128 0 TD
[(ted,)-24000(the)-23000(arc)]TJ
410 0 TD
[(hitecture)-22000(has)-22000(remained)-22000(fundamen)]TJ
1198.001 0 TD
(tally)Tj
-3736 99 TD
[(the)-29000(same{consequen)]TJ
733 0 TD
[(tly)-27000(w)]TJ
184 0 TD
[(e)-28000(will)-29000(fo)-2000(cus)-28000(some)-28000(adv)]TJ
772 0 TD
[(ances)-28000(w)]TJ
281 0 TD
28 Tc
[(eh)28000(a)]TJ
151 0 TD
(v)Tj
42 0 TD
0 Tc
[(e)-28000(made)-28000(in)-29000(our)-28000(understanding)-28000(of)-28000(coreference)-27000(and)]TJ
-2162.999 100 TD
[(then)-28000(discuss)-27000(the)-28000(p)-2000(erformance)-27000(of)-27000(the)-28000(system.)]TJ
/T13 1 Tf
0 236 TD
(Scoring)Tj
401 0 TD
(Coreference)Tj
616 0 TD
(Output)Tj
/T15 1 Tf
-1017 236 TD
[(Scoring)-23000(the)-24000(p)-2000(erformance)-22000(of)-24000(a)-23000(system)-24000(is)-23000(an)-24000(extremely)-23000(imp)-2000(ortan)]TJ
2233 0 TD
[(t)-23000(asp)-2000(ect)-24000(of)-23000(coreference)-23000(algorithm)-22000(p)-2000(erformance.)]TJ
-2233 100 TD
(The)Tj
179 0 TD
(score)Tj
217 0 TD
(for)Tj
136 0 TD
(a)Tj
78 0 TD
(particular)Tj
392 0 TD
(run)Tj
161 0 TD
(is)Tj
92 0 TD
(the)Tj
151 0 TD
(single)Tj
240 0 TD
(strongest)Tj
365 0 TD
(measure)Tj
332 0 TD
(of)Tj
103 0 TD
(ho)Tj
86 0 TD
(w)Tj
96 0 TD
(w)Tj
58 0 TD
(ell)Tj
119 0 TD
(the)Tj
152 0 TD
(system)Tj
284 0 TD
(is)Tj
92 0 TD
[(p)-2000(erforming)]TJ
433 0 TD
(and)Tj
-3766 100 TD
(it)Tj
89 0 TD
(can)Tj
159 0 TD
[(strongly)-32000(determine)]TJ
721 0 TD
(directions)Tj
385 0 TD
(for)Tj
133 0 TD
(further)Tj
286 0 TD
(impro)Tj
211 0 TD
(v)Tj
42 0 TD
(emen)Tj
187 0 TD
(ts.)Tj
142 0 TD
(In)Tj
110 0 TD
(this)Tj
168 0 TD
[(pap)-2000(er,)]TJ
264 0 TD
(w)Tj
58 0 TD
(e)Tj
71 0 TD
(presen)Tj
230 0 TD
(t)Tj
65 0 TD
(sev)Tj
112 0 TD
[(eral)-32000(di\013eren)]TJ
435 0 TD
(t)Tj
-3868 99 TD
[(scoring)-25000(algorithms)-26000(and)-27000(detail)-27000(their)-27000(resp)-2000(ectiv)]TJ
1603 0 TD
[(e)-26000(strengths)-27000(and)-26000(w)]TJ
642 0 TD
[(eaknesses)-26000(for)-26000(v)]TJ
533.001 0 TD
[(arying)-26000(classes)-26000(of)-27000(pro)-2000(cessing.)]TJ
1046 0 TD
(In)Tj
-3824 100 TD
[(particular,)-20000(w)]TJ
458 0 TD
[(e)-20000(describ)-2000(e)-19000(and)-20000(analyze)-19000(the)-20000(coreference)-19000(scoring)-18000(algorithm)-19000(used)-20000(to)-19000(ev)]TJ
2365 0 TD
[(aluate)-20000(the)-20000(coreference)-19000(systems)]TJ
-2823 99 TD
(in)Tj
102 0 TD
(the)Tj
149 0 TD
(sixth)Tj
211 0 TD
[(Message)-31000(Understanding)-32000(Conference)]TJ
1330 0 TD
[(\(MUC-6\)[MUC-6,)-27000(95)]TJ
756 0 TD
(].)Tj
99 0 TD
(W)Tj
78 0 TD
(e)Tj
70 0 TD
[(also)-32000(presen)]TJ
402 0 TD
(t)Tj
65 0 TD
(t)Tj
30 0 TD
(w)Tj
58 0 TD
[(o)-32000(shortcomings)]TJ
-3350 100 TD
[(of)-32000(this)]TJ
266 0 TD
(algorithm.)Tj
425 0 TD
(In)Tj
109 0 TD
(addition,)Tj
357 0 TD
(w)Tj
58 0 TD
[(e)-32000(presen)]TJ
299 0 TD
[(t)-32000(a)-32000(new)-32000(coreference)-31000(scoring)-31000(algorithm,)]TJ
1440 0 TD
[(our)-31000(B-CUBED)-32000(algorithm,)]TJ
-2954 100 TD
(whic)Tj
164 0 TD
28 Tc
(hw)Tj
132 0 TD
0 Tc
[(as)-26000(designed)-27000(to)-28000(o)]TJ
580 0 TD
(v)Tj
42 0 TD
[(ercome)-25000(the)-28000(shortcomings)-26000(of)-27000(the)-28000(MUC-6)-28000(algorithm.)]TJ
/T13 1 Tf
-918 236 TD
(Scoring)Tj
401 0 TD
(in)Tj
130 0 TD
(MUC-6/7:)Tj
560 0 TD
(Vilain)Tj
332 0 TD
(et)Tj
133 0 TD
(al.)Tj
/T15 1 Tf
-1556 236 TD
(Prior)Tj
227 0 TD
(to)Tj
114 0 TD
(Vilain)Tj
260 0 TD
(et)Tj
110 0 TD
(al.'s)Tj
184 0 TD
(coreference)Tj
440 0 TD
(scoring)Tj
296 0 TD
(algorithm)Tj
391 0 TD
[([Vilain,)-28000(95)]TJ
376 0 TD
(])Tj
64 0 TD
(there)Tj
225 0 TD
(had)Tj
175 0 TD
[(b)-2000(een)]TJ
209 0 TD
(a)Tj
82 0 TD
(graph)Tj
248 0 TD
(based)Tj
245 0 TD
(scoring)Tj
-3646 100 TD
(algorithm)Tj
388 0 TD
(\(Sundheim)Tj
430 0 TD
(et)Tj
106 0 TD
(al.\))Tj
186 0 TD
(whic)Tj
164 0 TD
(h)Tj
83 0 TD
[(pro)-2000(duced)]TJ
372 0 TD
(unin)Tj
159 0 TD
(tuitiv)Tj
198 0 TD
(e)Tj
75 0 TD
(results)Tj
274 0 TD
(for)Tj
137 0 TD
(ev)Tj
79 0 TD
(en)Tj
120 0 TD
(v)Tj
42 0 TD
(ery)Tj
150 0 TD
(simple)Tj
268 0 TD
(cases.)Tj
270 0 TD
[([Vilain,)-28000(95)]TJ
375.999 0 TD
(])Tj
-3877 99 TD
[(substituted)-26000(a)-24000(m)0(o)-2000(del-theoretic)-24000(scoring)-24000(algorithm)-23000(whic)]TJ
1906.999 0 TD
[(h)-25000(pro)-2000(duced)-24000(v)]TJ
472 0 TD
[(ery)-24000(in)]TJ
205 0 TD
(tuitiv)Tj
198 0 TD
[(e)-25000(results)-24000(for)-25000(the)-25000(t)]TJ
618 0 TD
[(yp)-2000(e)-25000(o)0(f)-25000(scoring)]TJ
-3400 100 TD
[(desired)-31000(in)-32000(MUC-6.)]TJ
728 0 TD
[(This)-31000(algorithm)-31000(computes)-32000(computes)-31000(the)]TJ
1471 0 TD
[(recall)-30000(error)-30000(b)]TJ
477 0 TD
[(y)-32000(taking)-31000(eac)]TJ
450 0 TD
[(h)-31000(equiv)]TJ
266 0 TD
[(alence)-31000(class)]TJ
/T12 1 Tf
452 0 TD
(S)Tj
/T15 1 Tf
-3844 100 TD
[(\(de\014ned)-26000(b)]TJ
360 0 TD
[(y)-25000(the)-25000(links)-25000(in)-26000(the)-26000(answ)]TJ
818 0 TD
[(er)-23000(k)]TJ
135 0 TD
[(ey\))-25000(and)-25000(determining)-25000(the)-26000(n)]TJ
941 0 TD
(um)Tj
113 0 TD
[(b)-2000(e)0(r)-25000(o)0(f)-25000(coreference)-24000(links)]TJ
/T12 1 Tf
854 0 TD
(m)Tj
/T15 1 Tf
98 0 TD
[(that)-25000(w)]TJ
235 0 TD
[(ould)-25000(ha)]TJ
268 0 TD
(v)Tj
42 0 TD
(e)Tj
-3864 99 TD
[(to)-32000(b)-2000(e)]TJ
224 0 TD
(added)Tj
250 0 TD
[(to)-32000(the)]TJ
254 0 TD
[(resp)-2000(onse)-31000(to)-32000(place)-32000(all)]TJ
784 0 TD
[(the)-32000(en)]TJ
228 0 TD
(tities)Tj
213 0 TD
(in)Tj
/T12 1 Tf
102 0 TD
(S)Tj
/T15 1 Tf
88 0 TD
(in)Tj
67 0 TD
[(to)-32000(the)]TJ
254 0 TD
[(same)-32000(equiv)]TJ
402 0 TD
[(alence)-32000(class)-31000(in)]TJ
555 0 TD
(the)Tj
148 0 TD
[(resp)-2000(onse.)]TJ
-3569 100 TD
[(Recall)-29000(error)-28000(then)-30000(is)-30000(the)-30000(sum)-29000(of)]TJ
/T12 1 Tf
1154 0 TD
(m)Tj
/T15 1 Tf
73 0 TD
[('s)-29000(divided)-30000(b)]TJ
424 0 TD
[(y)-30000(the)-30000(n)]TJ
263 0 TD
(um)Tj
113 0 TD
[(b)-2000(e)0(r)-29000(o)0(f)-30000(links)-29000(in)-30000(the)-30000(k)]TJ
728 0 TD
(ey)Tj
74 0 TD
(.)Tj
66 0 TD
[(Precision)-28000(error)-27000(is)-30000(computed)]TJ
-2895 100 TD
(b)Tj
44 0 TD
[(y)-27000(rev)]TJ
183 0 TD
[(ersing)-26000(the)-28000(roles)-26000(of)-28000(the)-28000(answ)]TJ
994 0 TD
[(er)-26000(k)]TJ
138 0 TD
[(ey)-27000(and)-27000(the)-28000(resp)-2000(onse.)]TJ
-1234 153 TD
[(The)-27000(full)-28000(details)-28000(of)-27000(the)-28000(algorithm)-27000(are)-26000(discussed)-28000(next.)]TJ
/T13 1 Tf
-125 236 TD
(The)Tj
229 0 TD
[(Mo)-3000(del)]TJ
347 0 TD
(Theoretic)Tj
506 0 TD
(Approac)Tj
412 0 TD
(h)Tj
100 0 TD
(T)Tj
69 0 TD
(o)Tj
93 0 TD
(The)Tj
229 0 TD
(Vilain)Tj
332 0 TD
(et.)Tj
176 0 TD
(al)Tj
123 0 TD
(Algorithm)Tj
/T11 1 Tf
507 -36 TD
(1)Tj
ET
1 i
72 84.58 187.2 0.48 re
f
BT
/T10 1 Tf
0.12 0 0 -0.12 83.04 78.22 Tm
(1)Tj
/T11 1 Tf
35 24 TD
[(The)-24000(exp)-2000(osition)-25000(of)-23000(this)-24000(scorer)-23000(has)-24000(b)-2000(een)-24000(tak)]TJ
1293.999 0 TD
[(en)-26000(nearly)-23000(en)]TJ
377 0 TD
[(tirely)-24000(from)-22000([Vilain,)-22000(95])]TJ
ET
endstream
endobj
3 0 obj
<<
/ProcSet [/PDF /Text /ImageB ]
/Font <<
/T10 4 0 R
/T11 5 0 R
/T12 6 0 R
/T13 7 0 R
/T14 8 0 R
/T15 9 0 R
/T16 10 0 R
/T17 11 0 R
>>
/ExtGState <<
/GS1 12 0 R
>>
/ColorSpace <<
/CS1 13 0 R
>>
>>
endobj
13 0 obj
[/CalGray <<
/Gamma 1.8
/WhitePoint [0.951 1 1.089]
>>
]
endobj
15 0 obj
<<
/Type /XObject
/Subtype /Image
/Name /Im1
/Width 107
/Height 100
/BitsPerComponent 8
/ColorSpace 13 0 R
/Length 540
/Filter /LZWDecode
>>
stream
?P8$
BaPd6DbQ8V-FcQv=HdR9$M'JeR K3|NkMIe8N VCt(dƉLi5
mR/B5)ZNT
_Wӫ=e`-\g6x0z(\ SaF6e-yjf4YڙdۭZ}6vv}ogo{}pw7=cl-ٶYv{]wx|^?'z}^g|~_?Ṷr>*K/:@L+(A)#?pRp@L<@I
)8ϵM>˲lFOODLqeƑmuǑ}!Ȓ*9RC3Rc:RRI2ɲ2K)2ܤn|3 1s3/l*M|2OsBO
NDPFt@@kD$T'8-AD%=IEKR5[MeY֕[DSMNE?95aؖ*.
endstream
endobj
16 0 obj
<<
/Type /XObject
/Subtype /Image
/Name /Im2
/Width 109
/Height 102
/BitsPerComponent 8
/ColorSpace 13 0 R
/Length 575
/Filter /LZWDecode
>>
stream
?P8$
BaPd6DbQ8V-FcQv=HdR9$M'JeRd]/H@ gM"sty8P Syi1Z\2MO5(]BkTѫ"iVWi(UA?id֭0;]=]mMy[_8UO`0u̝VheY\DS#qhbM-uL{Udji9]Ipx\>'n[.$~EG8Z1x|^?'z}^g|~_?~_~?rUPSk/0P{? lDP˪ë^ۺlS0:6E2)$#5B%Irc3'BRs)I+2-Jy/41A,1
TlMqs;N53Lt;ϴ̕9s2|+@Mt5EOTwD04ȹ/ D4RJqVdYRu[KGEWu{ZR6#KcEU FmgU1jTյ%o
<
endstream
endobj
18 0 obj
<<
/Length 6480
>>
stream
BX /GS1 gs EX
q
106.992 0 0 100.008 254.16 618.66 cm
/Im1 Do
Q
BT
/T9 1 Tf
0.12 0 0 -0.12 268.56 596.74 Tm
/CS1 cs 0 sc
0 Tc
[(Figure)-32000(1:)]TJ
/T15 1 Tf
414 0 TD
(T)Tj
53 0 TD
(ruth)Tj
ET
q
109.008 0 0 102.024 253.2 480.82 cm
/Im2 Do
Q
BT
/T9 1 Tf
0.12 0 0 -0.12 234.24 458.86 Tm
[(Figure)-31000(2:)]TJ
/T15 1 Tf
413 0 TD
[(Resp)-2000(onse:)]TJ
397 0 TD
[(Example)-27000(1)]TJ
-2162 265 TD
(In)Tj
113 0 TD
(the)Tj
153 0 TD
(description)Tj
434 0 TD
(of)Tj
104 0 TD
(the)Tj
153 0 TD
[(mo)-2000(del)]TJ
256 0 TD
(theoretic)Tj
355 0 TD
(algorithm,)Tj
413.999 0 TD
(the)Tj
152 0 TD
(terms)Tj
241 0 TD
(\\k)Tj
84 0 TD
(ey)Tj
74 0 TD
(,")Tj
103 0 TD
(and)Tj
171 0 TD
[(\\resp)-2000(onse")]TJ
428 0 TD
(are)Tj
148 0 TD
(de\014ned)Tj
295 0 TD
(in)Tj
107 0 TD
(the)Tj
-3785 99 TD
(follo)Tj
153 0 TD
[(wing)-27000(w)]TJ
256 0 TD
(a)Tj
40 0 TD
(y:)Tj
/T9 1 Tf
-448.999 220 TD
(k)Tj
47 0 TD
(ey)Tj
/T15 1 Tf
136 0 TD
[(refers)-27000(to)-27000(the)-28000(man)]TJ
624 0 TD
[(ually)-27000(annotated)-27000(coreference)-26000(c)]TJ
1059 0 TD
[(hains)-27000(\(the)-28000(truth\).)]TJ
/T9 1 Tf
-1865.999 166 TD
[(resp)-3000(onse)]TJ
/T15 1 Tf
400 0 TD
[(refers)-27000(to)-27000(the)-28000(coreference)-26000(c)]TJ
931 0 TD
[(hains)-27000(output)-28000(b)]TJ
533 0 TD
[(y)-28000(a)-27000(system.)]TJ
-1864 220 TD
(An)Tj
143 0 TD
(equiv)Tj
189 0 TD
(alence)Tj
256 0 TD
(set)Tj
137 0 TD
(is)Tj
91 0 TD
(the)Tj
150 0 TD
(transitiv)Tj
306 0 TD
(e)Tj
71 0 TD
(closure)Tj
284 0 TD
(of)Tj
102 0 TD
(a)Tj
76 0 TD
(coreference)Tj
434 0 TD
(c)Tj
35 0 TD
(hain.)Tj
238 0 TD
(The)Tj
178 0 TD
(algorithm)Tj
385 0 TD
(computes)Tj
377 0 TD
(recall)Tj
229 0 TD
(in)Tj
104 0 TD
(the)Tj
-3785 99 TD
(follo)Tj
153 0 TD
[(wing)-27000(w)]TJ
256 0 TD
(a)Tj
40 0 TD
(y)Tj
37 0 TD
(.)Tj
-360.999 154 TD
[(First,)-24000(let)-24000(S)-23000(b)-2000(e)-24000(a)0(n)-23000(equiv)]TJ
816 0 TD
[(alence)-23000(set)-24000(generated)-22000(b)]TJ
789 0 TD
[(y)-23000(the)-24000(k)]TJ
248 0 TD
(ey)Tj
74 0 TD
[(,)-24000(and)-23000(let)]TJ
/T12 1 Tf
320 0 TD
(R)Tj
/T8 1 Tf
63 12 TD
(1)Tj
/T12 1 Tf
51 -12 TD
14 Tc
(:::R)Tj
/T7 1 Tf
174 12 TD
(m)Tj
/T15 1 Tf
86 -12 TD
0 Tc
[(b)-2000(e)-24000(equiv)]TJ
298 0 TD
[(alence)-23000(classes)-22000(generated)]TJ
-3044 99 TD
(b)Tj
44 0 TD
[(y)-27000(the)-28000(resp)-2000(onse.)]TJ
582 0 TD
[(Then)-28000(w)]TJ
275 0 TD
[(e)-27000(de\014ne)-28000(the)-28000(follo)]TJ
600 0 TD
[(wing)-26000(functions)-28000(o)]TJ
595 0 TD
(v)Tj
42 0 TD
[(er)-26000(S:)]TJ
/T14 1 Tf
-2013 220 TD
(\017)Tj
/T15 1 Tf
83 0 TD
(p\(S\))Tj
197 0 TD
(is)Tj
96 0 TD
(a)Tj
82 0 TD
(partition)Tj
359 0 TD
(of)Tj
107 0 TD
(S)Tj
87 0 TD
(relativ)Tj
232 0 TD
(e)Tj
76 0 TD
(to)Tj
114 0 TD
(the)Tj
156 0 TD
[(resp)-2000(onse.)]TJ
406 0 TD
(Eac)Tj
134 0 TD
(h)Tj
85 0 TD
(subset)Tj
267 0 TD
(of)Tj
107 0 TD
(S)Tj
87 0 TD
(in)Tj
110 0 TD
(the)Tj
155 0 TD
(partition)Tj
359 0 TD
(is)Tj
96 0 TD
(formed)Tj
292 0 TD
(b)Tj
44 0 TD
(y)Tj
-3648 100 TD
(in)Tj
67 0 TD
[(tersecting)-27000(S)-29000(and)-28000(those)-28000(resp)-2000(onse)-27000(sets)]TJ
/T12 1 Tf
1334 0 TD
(R)Tj
/T7 1 Tf
63 12 TD
(i)Tj
/T15 1 Tf
56 -12 TD
[(that)-29000(o)]TJ
221 0 TD
(v)Tj
42 0 TD
[(erlap)-26000(S.)-29000(Note)-28000(that)-29000(the)-28000(equiv)]TJ
1019 0 TD
[(alence)-28000(classes)-28000(de\014ned)-28000(b)]TJ
846 0 TD
(y)Tj
-3648 99 TD
[(the)-28000(resp)-2000(onse)-27000(ma)]TJ
588 0 TD
[(y)-27000(include)-29000(implicit)-29000(singleton)-27000(sets)-28000(-)-28000(these)-28000(corresp)-2000(ond)-26000(to)-28000(elemen)]TJ
2218 0 TD
[(ts)-28000(that)-29000(are)-27000(men)]TJ
563 0 TD
[(tioned)-28000(in)]TJ
-3369 100 TD
[(the)-26000(k)]TJ
183 0 TD
[(ey)-26000(but)-27000(not)-26000(in)-26000(the)-27000(resp)-2000(onse.)]TJ
1008 0 TD
(F)Tj
47 0 TD
[(or)-26000(example,)-26000(sa)]TJ
521 0 TD
[(y)-25000(the)-27000(k)]TJ
253 0 TD
[(ey)-25000(generates)-25000(the)-26000(equiv)]TJ
800 0 TD
[(alence)-26000(class)-26000(S)-26000(=)]TJ
/T14 1 Tf
605 0 TD
(f)Tj
/T15 1 Tf
42 0 TD
26 Tc
(ABC)Tj
-3459 99 TD
(D)Tj
/T14 1 Tf
63 0 TD
(g)Tj
/T15 1 Tf
42 0 TD
0 Tc
[(,)-27000(and)-28000(the)-28000(resp)-2000(onse)-26000(is)-28000(simply)]TJ
/T12 1 Tf
1039 0 TD
(<)Tj
/T15 1 Tf
65 0 TD
(A-B)Tj
/T12 1 Tf
149 0 TD
(>)Tj
/T15 1 Tf
65 0 TD
(.)Tj
59 0 TD
[(The)-28000(relativ)]TJ
403 0 TD
[(e)-26000(partition)-27000(p\(S\))-29000(is)-27000(then)]TJ
/T14 1 Tf
867 0 TD
(f)Tj
/T15 1 Tf
42 0 TD
27 Tc
(AB)Tj
/T14 1 Tf
148 0 TD
(gf)Tj
/T15 1 Tf
111 0 TD
(C)Tj
/T14 1 Tf
60 0 TD
(g)Tj
/T15 1 Tf
69 0 TD
0 Tc
(and)Tj
/T14 1 Tf
161 0 TD
(f)Tj
/T15 1 Tf
42 0 TD
(D)Tj
/T14 1 Tf
63 0 TD
(g)Tj
/T15 1 Tf
42 0 TD
(.)Tj
/T14 1 Tf
-3573 166 TD
(\017)Tj
/T15 1 Tf
83 0 TD
[(c\(S\))-28000(is)-27000(the)-28000(minimal)-27000(n)]TJ
767 0 TD
(um)Tj
113 0 TD
[(b)-2000(e)0(r)-27000(o)0(f)-28000(\\correct")-24000(links)-28000(necessary)-25000(to)-28000(generate)-26000(the)-27000(equiv)]TJ
1928 0 TD
[(alence)-27000(class)-27000(S.)-27000(It)-28000(is)-27000(clear)]TJ
-2808 100 TD
[(that)-27000(c\(S\))-29000(is)-27000(one)-27000(less)-28000(than)-27000(the)-28000(cardinalit)]TJ
1424.999 0 TD
[(y)-27000(o)0(f)-27000(S)0(,)-28000(i.e.,)]TJ
116.001 183 TD
[(c\(S\))-24000(=)-23000(\()]TJ
/T14 1 Tf
291 0 TD
(j)Tj
/T15 1 Tf
23 0 TD
(S)Tj
/T14 1 Tf
46 0 TD
19 Tc
(j\000)Tj
/T15 1 Tf
125 0 TD
0 Tc
(1\))Tj
/T12 1 Tf
101 0 TD
(:)Tj
/T14 1 Tf
-2210 216 TD
(\017)Tj
/T15 1 Tf
83 0 TD
[(m\(S\))-31000(is)-30000(the)-30000(n)]TJ
485 0 TD
(um)Tj
113 0 TD
[(b)-2000(e)0(r)-30000(o)0(f)-30000(\\missing")-29000(links)-30000(in)-30000(the)-31000(resp)-2000(onse)-29000(relativ)]TJ
1641 0 TD
[(e)-29000(t)0(o)-30000(the)-30000(k)]TJ
357 0 TD
[(ey)-30000(set)-30000(S.)-30000(As)-30000(noted)-30000(ab)-2000(o)]TJ
830.001 0 TD
(v)Tj
42 0 TD
[(e,)-30000(this)]TJ
-3468.001 99 TD
[(is)-29000(the)-30000(n)]TJ
274 0 TD
(um)Tj
113 0 TD
[(b)-2000(e)0(r)-29000(o)0(f)-30000(links)-30000(necessary)-27000(to)-30000(fully)-30000(reunite)-30000(an)]TJ
1477 0 TD
[(y)-29000(comp)-2000(onen)]TJ
438 0 TD
[(ts)-29000(of)-29000(the)-30000(p\(S\))-31000(partition.)]TJ
906 0 TD
(W)Tj
78 0 TD
[(e)-30000(note)-30000(that)]TJ
-3286 100 TD
[(this)-27000(is)-28000(simply)-28000(one)-27000(few)]TJ
783 0 TD
[(er)-27000(than)-28000(the)-28000(n)]TJ
478 0 TD
(um)Tj
113 0 TD
[(b)-2000(e)0(r)-27000(o)0(f)-27000(elemen)]TJ
486 0 TD
[(ts)-28000(in)-28000(the)-28000(partition,)-27000(that)-28000(is,)]TJ
-390 183 TD
[(m\(S\))-24000(=)-22000(\()]TJ
/T14 1 Tf
322 0 TD
(j)Tj
/T15 1 Tf
23 0 TD
(p\(S\))Tj
/T14 1 Tf
158 0 TD
18 Tc
(j\000)Tj
/T15 1 Tf
124 0 TD
0 Tc
(1\))Tj
/T12 1 Tf
102 0 TD
(:)Tj
ET
endstream
endobj
19 0 obj
<<
/ProcSet [/PDF /Text /ImageB ]
/Font <<
/T7 20 0 R
/T8 21 0 R
/T9 22 0 R
/T12 6 0 R
/T14 8 0 R
/T15 9 0 R
>>
/XObject <<
/Im1 15 0 R
/Im2 16 0 R
>>
/ExtGState <<
/GS1 12 0 R
>>
/ColorSpace <<
/CS1 13 0 R
>>
>>
endobj
24 0 obj
<<
/Length 11265
>>
stream
BT
/T15 1 Tf
0.12 0 0 -0.12 87 708.7 Tm
/CS1 cs 0 sc
BX /GS1 gs EX
0 Tc
[(Lo)-2000(oking)]TJ
326 0 TD
(in)Tj
104 0 TD
(isolation)Tj
340 0 TD
(at)Tj
109 0 TD
(a)Tj
76 0 TD
(single)Tj
238 0 TD
(equiv)Tj
189 0 TD
(alence)Tj
257 0 TD
(class)Tj
202 0 TD
(in)Tj
104 0 TD
(the)Tj
150 0 TD
(k)Tj
42 0 TD
(ey)Tj
74 0 TD
(,)Tj
59 0 TD
(the)Tj
150 0 TD
(recall)Tj
/T6 1 Tf
229 0 TD
(err)Tj
104 0 TD
(or)Tj
/T15 1 Tf
121 0 TD
(for)Tj
135 0 TD
(that)Tj
187 0 TD
(class)Tj
202 0 TD
(is)Tj
90 0 TD
(just)Tj
172 0 TD
(the)Tj
-3785 100 TD
(n)Tj
44 0 TD
(um)Tj
113 0 TD
[(b)-2000(e)0(r)-27000(o)0(f)-28000(missing)-27000(links)-27000(divided)-28000(b)]TJ
1069 0 TD
[(y)-28000(the)-28000(n)]TJ
259 0 TD
(um)Tj
113 0 TD
[(b)-2000(e)0(r)-27000(o)0(f)-28000(correct)-26000(links,)-27000(i.e.,)]TJ
237 167 TD
(m\(S\))Tj
ET
1 i
292.2 672.22 21.6 0.48 re
f
BT
0.12 0 0 -0.12 294.12 663.1 Tm
(c\(S\))Tj
/T12 1 Tf
201 -57 TD
(:)Tj
/T15 1 Tf
-1927 278 TD
[(Recall)-27000(in)-28000(turn)-27000(is)]TJ
1585 71 TD
(c\(S\))Tj
/T14 1 Tf
166 0 TD
(\000)Tj
/T15 1 Tf
83 0 TD
(m\(S\))Tj
ET
277.2 623.62 51.48 0.48 re
f
BT
0.12 0 0 -0.12 294.12 614.5 Tm
(c\(S\))Tj
/T12 1 Tf
326 -57 TD
(;)Tj
/T15 1 Tf
-2177 192 TD
(whic)Tj
164 0 TD
[(h)-28000(equals)]TJ
1355 86 TD
(\()Tj
/T14 1 Tf
32 0 TD
(j)Tj
/T15 1 Tf
23 0 TD
(S)Tj
/T14 1 Tf
46 0 TD
19 Tc
(j\000)Tj
/T15 1 Tf
125 0 TD
0 Tc
(1\))Tj
/T14 1 Tf
92 0 TD
(\000)Tj
/T15 1 Tf
83 0 TD
(\()Tj
/T14 1 Tf
32 0 TD
(j)Tj
/T15 1 Tf
23 0 TD
(p\(S\))Tj
/T14 1 Tf
157 0 TD
19 Tc
(j\000)Tj
/T15 1 Tf
125 0 TD
0 Tc
(1\))Tj
ET
254.28 583.42 97.44 0.48 re
f
BT
/T14 1 Tf
0.12 0 0 -0.12 288.84 574.3 Tm
(j)Tj
/T15 1 Tf
23 0 TD
(S)Tj
/T14 1 Tf
46 0 TD
19 Tc
(j\000)Tj
/T15 1 Tf
125 0 TD
(1)Tj
/T12 1 Tf
367 -57 TD
(:)Tj
/T15 1 Tf
-2368 192 TD
0 Tc
[(The)-28000(whole)-27000(expression)-26000(can)-27000(no)]TJ
1044 0 TD
[(w)-27000(b)-2000(e)-28000(simpli\014ed)-28000(to)]TJ
/T14 1 Tf
696 167 TD
(j)Tj
/T15 1 Tf
23 0 TD
(S)Tj
/T14 1 Tf
46 0 TD
19 Tc
[(j\000)1000(j)]TJ
/T15 1 Tf
148 0 TD
0 Tc
(p\(S\))Tj
/T14 1 Tf
157 0 TD
(j)Tj
ET
280.8 533.62 47.64 0.48 re
f
BT
0.12 0 0 -0.12 290.52 524.5 Tm
(j)Tj
/T15 1 Tf
23 0 TD
(S)Tj
/T14 1 Tf
46 0 TD
19 Tc
(j\000)Tj
/T15 1 Tf
125 0 TD
(1)Tj
/T12 1 Tf
132 -57 TD
(:)Tj
/T15 1 Tf
1647 0 TD
0 Tc
(\(1\))Tj
-3669 278 TD
(Finally)Tj
248 0 TD
(,)Tj
59 0 TD
(extending)Tj
388 0 TD
(this)Tj
168 0 TD
(measure)Tj
331 0 TD
(from)Tj
203 0 TD
(a)Tj
77 0 TD
(single)Tj
238 0 TD
(k)Tj
42 0 TD
(ey)Tj
115 0 TD
(equiv)Tj
189 0 TD
(alence)Tj
256 0 TD
(class)Tj
201 0 TD
(to)Tj
109 0 TD
(an)Tj
122 0 TD
(en)Tj
81 0 TD
(tire)Tj
159 0 TD
(set)Tj
137 0 TD
(T)Tj
95 0 TD
(simply)Tj
272 0 TD
(requires)Tj
-3615 99 TD
[(summing)-28000(o)]TJ
396 0 TD
(v)Tj
42 0 TD
[(er)-25000(the)-28000(k)]TJ
280 0 TD
[(ey)-27000(equiv)]TJ
297 0 TD
[(alence)-27000(classes.)]TJ
546 0 TD
[(That)-28000(is,)]TJ
-54.999 224 TD
(R)Tj
/T8 1 Tf
61 12 TD
(T)Tj
/T15 1 Tf
74 -12 TD
(=)Tj
/T5 1 Tf
98 -119 TD
(P)Tj
/T15 1 Tf
88 63 TD
(\()Tj
/T14 1 Tf
32 0 TD
(j)Tj
/T15 1 Tf
23 0 TD
(S)Tj
/T8 1 Tf
46 12 TD
(i)Tj
/T14 1 Tf
23 -12 TD
18 Tc
[(j\000)-1000(j)]TJ
/T15 1 Tf
148 0 TD
0 Tc
(p\(S)Tj
/T8 1 Tf
124 12 TD
(i)Tj
/T15 1 Tf
23 -12 TD
(\))Tj
/T14 1 Tf
32 0 TD
(j)Tj
/T15 1 Tf
23 0 TD
(\))Tj
ET
280.68 461.5 71.4 0.48 re
f
BT
/T5 1 Tf
0.12 0 0 -0.12 291.72 459.94 Tm
(P)Tj
/T15 1 Tf
88 63 TD
(\()Tj
/T14 1 Tf
32 0 TD
(j)Tj
/T15 1 Tf
23 0 TD
(S)Tj
/T8 1 Tf
46 12 TD
(i)Tj
/T14 1 Tf
23 -12 TD
19 Tc
(j\000)Tj
/T15 1 Tf
125 0 TD
0 Tc
(1\))Tj
/T12 1 Tf
203 -57 TD
(:)Tj
/T15 1 Tf
1423 0 TD
(\(2\))Tj
-3669 278 TD
[(Precision)-26000(is)-27000(computed)-28000(b)]TJ
867 0 TD
[(y)-27000(switc)]TJ
254 0 TD
[(hing)-27000(the)-28000(roles)-27000(of)-27000(the)-28000(k)]TJ
801 0 TD
[(ey)-27000(and)-28000(resp)-2000(onse)-26000(in)-28000(the)-28000(ab)-2000(o)]TJ
975 0 TD
(v)Tj
42 0 TD
[(e)-26000(form)]TJ
230 0 TD
(ulation.)Tj
/T13 1 Tf
-3294 233 TD
(Example)Tj
/T15 1 Tf
0 234 TD
(F)Tj
47 0 TD
(or)Tj
111 0 TD
(example,)Tj
359 0 TD
(let)Tj
128 0 TD
(the)Tj
152 0 TD
(k)Tj
42 0 TD
(ey)Tj
116 0 TD
(con)Tj
123 0 TD
(tain)Tj
179 0 TD
(3)Tj
78 0 TD
(equiv)Tj
189 0 TD
(alence)Tj
257 0 TD
(classes)Tj
273 0 TD
(as)Tj
111 0 TD
(sho)Tj
119 0 TD
(wn)Tj
141 0 TD
(in)Tj
106 0 TD
(Figure)Tj
270 0 TD
(1.)Tj
128 0 TD
[(Supp)-2000(ose)]TJ
334 0 TD
(Figure)Tj
271 0 TD
(2)Tj
77 0 TD
(sho)Tj
119 0 TD
(ws)Tj
128 0 TD
(a)Tj
-3857.999 100 TD
[(resp)-2000(onse.)]TJ
381 0 TD
(F)Tj
47 0 TD
[(rom)-32000(Figure)-32000(3\(I\),)-32000(the)-32000(three)-32000(equiv)]TJ
1187 0 TD
[(alence)-32000(classes)-31000(in)-32000(the)]TJ
772 0 TD
(truth,)Tj
245 0 TD
(S)Tj
/T8 1 Tf
46 12 TD
(1)Tj
/T15 1 Tf
38 -12 TD
(,)Tj
56 0 TD
(S)Tj
/T8 1 Tf
46 12 TD
(2)Tj
/T15 1 Tf
38 -12 TD
(,)Tj
56 0 TD
[(and)-32000(S)]TJ
/T8 1 Tf
212 12 TD
(3)Tj
/T15 1 Tf
37 -12 TD
(,)Tj
57 0 TD
(are)Tj
/T14 1 Tf
143 0 TD
(f)Tj
/T15 1 Tf
42 0 TD
[(1,)-32000(2,)]TJ
195 0 TD
(3,)Tj
98 0 TD
(4,)Tj
98 0 TD
(5)Tj
/T14 1 Tf
42 0 TD
(g)Tj
/T15 1 Tf
42 0 TD
(,)Tj
/T14 1 Tf
-3878 99 TD
(f)Tj
/T15 1 Tf
42 0 TD
(6,)Tj
99 0 TD
(7)Tj
/T14 1 Tf
42 0 TD
(g)Tj
/T15 1 Tf
42 0 TD
(,)Tj
56 0 TD
(and)Tj
/T14 1 Tf
167 0 TD
(f)Tj
/T15 1 Tf
42 0 TD
(8,)Tj
99 0 TD
(9,)Tj
99 0 TD
(A,)Tj
118 0 TD
(B,)Tj
116 0 TD
(C)Tj
/T14 1 Tf
60 0 TD
(g)Tj
/T15 1 Tf
74 0 TD
[(resp)-2000(ectiv)]TJ
322 0 TD
(ely)Tj
97 0 TD
(.)Tj
76 0 TD
(And)Tj
187 0 TD
(the)Tj
149 0 TD
[(partitions)-32000(p\(S)]TJ
/T8 1 Tf
508.001 12 TD
(1)Tj
/T15 1 Tf
38 -12 TD
(\),)Tj
90 0 TD
(p\(S)Tj
/T8 1 Tf
124 12 TD
(2)Tj
/T15 1 Tf
38 -12 TD
(\),)Tj
90 0 TD
(and)Tj
167 0 TD
(p\(S)Tj
/T8 1 Tf
124 12 TD
(3)Tj
/T15 1 Tf
37 -12 TD
(\),)Tj
90 0 TD
(with)Tj
195 0 TD
[(resp)-2000(ect)]TJ
290 0 TD
(to)Tj
107 0 TD
(the)Tj
-3785 100 TD
[(resp)-2000(onse,)-23000(sho)]TJ
474 0 TD
[(wn)-22000(in)-23000(Figure)-22000(3\(I)-2000(I\),)-23000(are)]TJ
/T14 1 Tf
825 0 TD
(f)Tj
/T15 1 Tf
42 0 TD
[(1,)-23000(2,)-23000(3,)-24000(4,)-23000(5)]TJ
/T14 1 Tf
395 0 TD
(g)Tj
/T15 1 Tf
42 0 TD
(,)Tj
/T14 1 Tf
46 0 TD
(f)Tj
/T15 1 Tf
42 0 TD
[(6,)-23000(7)]TJ
/T14 1 Tf
130 0 TD
(g)Tj
/T15 1 Tf
42 0 TD
[(,)-23000(and)]TJ
/T14 1 Tf
203 0 TD
(f)Tj
/T15 1 Tf
42 0 TD
[(8,)-23000(9,)-23000(A,)-23000(B,)-23000(C)]TJ
/T14 1 Tf
449 0 TD
(g)Tj
/T15 1 Tf
64 0 TD
[(resp)-2000(ectiv)]TJ
322 0 TD
(ely)Tj
97 0 TD
(.)Tj
58 0 TD
[(Using)-22000(equation)-22000(2,)]TJ
-3272.998 100 TD
[(the)-28000(recall)-27000(can)-27000(no)]TJ
603 0 TD
[(w)-27000(b)-2000(e)-28000(calculated)-27000(in)-28000(the)-28000(follo)]TJ
984.999 0 TD
[(wing)-26000(w)]TJ
255 0 TD
(a)Tj
40 0 TD
(y:)Tj
-879.999 223 TD
[(Recall)-23000(=)]TJ
343 -56 TD
(\(5)Tj
/T14 1 Tf
93 0 TD
(\000)Tj
/T15 1 Tf
83 0 TD
18 Tc
[(1)18000(\)+\()18000(2)]TJ
/T14 1 Tf
267 0 TD
(\000)Tj
/T15 1 Tf
83 0 TD
[(1)18000(\)+\()18000(5)]TJ
/T14 1 Tf
268 0 TD
(\000)Tj
/T15 1 Tf
83 0 TD
0 Tc
(1\))Tj
ET
233.52 297.46 114.12 0.48 re
f
BT
0.12 0 0 -0.12 233.52 288.34 Tm
(\(5)Tj
/T14 1 Tf
93 0 TD
(\000)Tj
/T15 1 Tf
83 0 TD
18 Tc
[(1)18000(\)+\()18000(2)]TJ
/T14 1 Tf
267 0 TD
(\000)Tj
/T15 1 Tf
83 0 TD
[(1)18000(\)+\()18000(5)]TJ
/T14 1 Tf
268 0 TD
(\000)Tj
/T15 1 Tf
83 0 TD
0 Tc
(1\))Tj
107 -57 TD
22 Tc
(=9)Tj
/T12 1 Tf
129 0 TD
(=)Tj
/T15 1 Tf
42 0 TD
0 Tc
[(9)-22000(=)-22000(100%)]TJ
/T12 1 Tf
373 0 TD
(:)Tj
/T15 1 Tf
-2874 224 TD
(Similarly)Tj
319 0 TD
[(,)-27000(i)0(f)-27000(the)-27000(roles)-25000(of)-27000(the)-27000(k)]TJ
738 0 TD
[(ey)-26000(and)-27000(the)-27000(resp)-2000(onse)-25000(are)-26000(rev)]TJ
994 0 TD
[(ersed,)-25000(then)-27000(the)-27000(equiv)]TJ
753 0 TD
[(alence)-27000(classes)-25000(in)-27000(the)-27000(truth,)-28000(S)]TJ
/T8 1 Tf
1035.999 12 TD
(1)Tj
/T15 1 Tf
37 -12 TD
(,)Tj
-3877 100 TD
[(and)-27000(S)]TJ
/T8 1 Tf
207 12 TD
(2)Tj
/T15 1 Tf
37 -12 TD
[(,)-27000(are)]TJ
/T14 1 Tf
188 0 TD
(f)Tj
/T15 1 Tf
42 0 TD
[(1,)-27000(2,)-26000(3,)-27000(4,)-27000(5)]TJ
/T14 1 Tf
408.999 0 TD
(g)Tj
/T15 1 Tf
68 0 TD
(and)Tj
/T14 1 Tf
161 0 TD
(f)Tj
/T15 1 Tf
42 0 TD
[(6,)-26000(7,)-27000(8,)-27000(9,)-26000(A,)-28000(B,)-27000(C)]TJ
/T14 1 Tf
647.999 0 TD
(g)Tj
/T15 1 Tf
42 0 TD
[(],)-26000(and)-27000(the)-28000(partitions,)-26000(p\(S)]TJ
/T8 1 Tf
901.002 12 TD
(1)Tj
/T15 1 Tf
38 -12 TD
[(\),)-27000(and)-27000(p\(S)]TJ
/T8 1 Tf
367 12 TD
(2)Tj
/T15 1 Tf
38 -12 TD
[(\),)-27000(are)]TJ
/T14 1 Tf
220 0 TD
(f)Tj
/T15 1 Tf
42 0 TD
[(1,)-27000(2,)-26000(3,)-27000(4,)-27000(5)]TJ
/T14 1 Tf
409 0 TD
(g)Tj
/T15 1 Tf
-3859 99 TD
[(and)-27000([)]TJ
/T14 1 Tf
184 0 TD
(f)Tj
/T15 1 Tf
42 0 TD
[(6,)-27000(7)]TJ
/T14 1 Tf
134 0 TD
27 Tc
(gf)Tj
/T15 1 Tf
111 0 TD
0 Tc
[(8,)-27000(9,)-27000(A,)-28000(B,)-27000(C)]TJ
/T14 1 Tf
465.999 0 TD
(g)Tj
/T15 1 Tf
42 0 TD
[(])-28000(resp)-2000(ectiv)]TJ
373 0 TD
[(ely)-26000(\(Figure)-28000(3\(I)-2000(I)-2000(I\)\).)-28000(The)-28000(precision)-27000(can)-27000(no)]TJ
1464 0 TD
[(w)-27000(b)-2000(e)-28000(calculated)-27000(as:)]TJ
-1690.999 228 TD
[(Precision)-21000(=)]TJ
450 -56 TD
(\(5)Tj
/T14 1 Tf
92 0 TD
(\000)Tj
/T15 1 Tf
83 0 TD
[(1\))-19000(+)-18000(\()0(7)]TJ
/T14 1 Tf
268 0 TD
(\000)Tj
/T15 1 Tf
83 0 TD
(2\))Tj
ET
261 219.34 72 0.48 re
f
BT
0.12 0 0 -0.12 261 210.22 Tm
(\(5)Tj
/T14 1 Tf
92 0 TD
(\000)Tj
/T15 1 Tf
83 0 TD
[(1\))-19000(+)-18000(\()0(7)]TJ
/T14 1 Tf
268 0 TD
(\000)Tj
/T15 1 Tf
83 0 TD
(1\))Tj
107 -57 TD
22 Tc
(=9)Tj
/T12 1 Tf
129 0 TD
(=)Tj
/T15 1 Tf
42 0 TD
0 Tc
[(10)-21000(=)-23000(90%)]TJ
/T12 1 Tf
373 0 TD
(:)Tj
/T13 1 Tf
-2752 370 TD
(Shortcomings)Tj
700 0 TD
(of)Tj
128 0 TD
(the)Tj
194 0 TD
(Vilain)Tj
332 0 TD
(et.)Tj
176 0 TD
(al)Tj
123 0 TD
(Algorithm)Tj
/T15 1 Tf
-1653 234 TD
(Despite)Tj
304 0 TD
(the)Tj
148 0 TD
(adv)Tj
127 0 TD
[(ances)-31000(of)]TJ
326 0 TD
[(the)-32000(mo)-2000(del-theoretic)-32000(scorer,)-32000(it)]TJ
1103 0 TD
[(yields)-32000(unin)]TJ
397 0 TD
(tuitiv)Tj
198 0 TD
(e)Tj
70 0 TD
[(results)-32000(for)-32000(some)-32000(tasks.)]TJ
871 0 TD
[(There)-32000(are)]TJ
-3544 100 TD
(t)Tj
30 0 TD
(w)Tj
58 0 TD
[(o)-27000(main)-27000(reasons.)]TJ
13 217 TD
(1.)Tj
107 0 TD
(The)Tj
180 0 TD
(algorithm)Tj
387 0 TD
[(do)-2000(es)]TJ
197 0 TD
(not)Tj
157 0 TD
(giv)Tj
107 0 TD
(e)Tj
73 0 TD
(an)Tj
86 0 TD
(y)Tj
80 0 TD
(credit)Tj
244 0 TD
(for)Tj
137 0 TD
(separating)Tj
411 0 TD
(out)Tj
157 0 TD
(singletons)Tj
393 0 TD
(\(en)Tj
113 0 TD
(tities)Tj
217 0 TD
(that)Tj
189 0 TD
[(o)-2000(ccur)]TJ
234 0 TD
(in)Tj
106 0 TD
(c)Tj
35.001 0 TD
(hains)Tj
-3503 100 TD
[(consisting)-31000(only)-32000(of)]TJ
675 0 TD
[(one)-32000(elemen)]TJ
404 0 TD
(t,)Tj
88 0 TD
(the)Tj
148 0 TD
(en)Tj
81 0 TD
(tit)Tj
85 0 TD
(y)Tj
77 0 TD
[(itself)-6000(\))]TJ
244 0 TD
[(from)-32000(other)-32000(c)]TJ
458 0 TD
[(hains)-32000(whic)]TJ
386 0 TD
32 Tc
[(hh)32000(a)]TJ
164 0 TD
(v)Tj
42 0 TD
0 Tc
[(e)-31000(b)-2000(een)]TJ
269 0 TD
(iden)Tj
150 0 TD
(ti\014ed.)Tj
259 0 TD
(This)Tj
-3530 100 TD
(follo)Tj
153 0 TD
[(ws)-20000(from)-21000(the)-22000(con)]TJ
563 0 TD
(v)Tj
42 0 TD
(en)Tj
81 0 TD
[(tion)-19000(in)-22000(coreference)-20000(annotation)-20000(of)-22000(not)-21000(iden)]TJ
1467 0 TD
[(tifying)-22000(those)-21000(en)]TJ
549 0 TD
[(tities)-21000(that)-22000(are)-20000(mark)]TJ
690 0 TD
(able)Tj
-3544.999 99 TD
[(as)-17000(p)-2000(ossibly)-18000(coreferen)]TJ
727 0 TD
[(t)-17000(with)-19000(other)-18000(en)]TJ
518 0 TD
[(tities)-19000(in)-18000(the)-19000(text.)]TJ
622 0 TD
[(Rather,)-20000(en)]TJ
375 0 TD
[(tities)-19000(are)-17000(only)-18000(mark)]TJ
687 0 TD
[(ed)-17000(as)-18000(b)-2000(eing)-18000(coreferen)]TJ
732 0 TD
(t)Tj
ET
endstream
endobj
25 0 obj
<<
/ProcSet [/PDF /Text /ImageB ]
/Font <<
/T5 26 0 R
/T6 27 0 R
/T8 21 0 R
/T12 6 0 R
/T13 7 0 R
/T14 8 0 R
/T15 9 0 R
>>
/ExtGState <<
/GS1 12 0 R
>>
/ColorSpace <<
/CS1 13 0 R
>>
>>
endobj
29 0 obj
<<
/Length 12201
>>
stream
/CS2 CS 0 0 0 SC
0 J 0 j 1.35 w 10 M []0 d
BX /GS1 gs EX
1 i
184.25 683.5 m
184.25 666.733 169.13 653.125 150.5 653.125 c
131.87 653.125 116.75 666.733 116.75 683.5 c
116.75 700.267 131.87 713.875 150.5 713.875 c
169.13 713.875 184.25 700.267 184.25 683.5 c
h
177.625 626.5 m
177.625 619.048 167.041 613 154 613 c
140.959 613 130.375 619.048 130.375 626.5 c
130.375 633.952 140.959 640 154 640 c
167.041 640 177.625 633.952 177.625 626.5 c
h
187.75 568.5 m
187.75 551.733 172.63 538.125 154 538.125 c
135.37 538.125 120.25 551.733 120.25 568.5 c
120.25 585.267 135.37 598.875 154 598.875 c
172.63 598.875 187.75 585.267 187.75 568.5 c
s
0.337 w
177.625 592.5 m
177.625 564.555 167.041 541.875 154 541.875 c
140.959 541.875 130.375 564.555 130.375 592.5 c
130.375 620.445 140.959 643.125 154 643.125 c
167.041 643.125 177.625 620.445 177.625 592.5 c
s
1.35 w
295.75 686.5 m
295.75 669.733 280.63 656.125 262 656.125 c
243.37 656.125 228.25 669.733 228.25 686.5 c
228.25 703.267 243.37 716.875 262 716.875 c
280.63 716.875 295.75 703.267 295.75 686.5 c
h
285.625 626.5 m
285.625 619.048 275.041 613 262 613 c
248.959 613 238.375 619.048 238.375 626.5 c
238.375 633.952 248.959 640 262 640 c
275.041 640 285.625 633.952 285.625 626.5 c
h
295.75 568.5 m
295.75 551.733 280.63 538.125 262 538.125 c
243.37 538.125 228.25 551.733 228.25 568.5 c
228.25 585.267 243.37 598.875 262 598.875 c
280.63 598.875 295.75 585.267 295.75 568.5 c
s
0.337 w
282.625 687 m
282.625 675.822 272.041 666.75 259 666.75 c
245.959 666.75 235.375 675.822 235.375 687 c
235.375 698.178 245.959 707.25 259 707.25 c
272.041 707.25 282.625 698.178 282.625 687 c
h
275.5 626 m
275.5 622.274 269.452 619.25 262 619.25 c
254.548 619.25 248.5 622.274 248.5 626 c
248.5 629.726 254.548 632.75 262 632.75 c
269.452 632.75 275.5 629.726 275.5 626 c
h
278.915 567 m
278.915 554.903 269.097 545.085 257 545.085 c
244.903 545.085 235.085 554.903 235.085 567 c
235.085 579.097 244.903 588.915 257 588.915 c
269.097 588.915 278.915 579.097 278.915 567 c
h
447.75 686.5 m
447.75 669.733 432.63 656.125 414 656.125 c
395.37 656.125 380.25 669.733 380.25 686.5 c
380.25 703.267 395.37 716.875 414 716.875 c
432.63 716.875 447.75 703.267 447.75 686.5 c
s
1.35 w
434.235 624.5 m
434.235 619.929 426.514 616.22 417 616.22 c
407.486 616.22 399.765 619.929 399.765 624.5 c
399.765 629.071 407.486 632.78 417 632.78 c
426.514 632.78 434.235 629.071 434.235 624.5 c
h
433.415 686 m
433.415 673.903 423.597 664.085 411.5 664.085 c
399.403 664.085 389.585 673.903 389.585 686 c
389.585 698.097 399.403 707.915 411.5 707.915 c
423.597 707.915 433.415 698.097 433.415 686 c
h
438.415 567.5 m
438.415 555.999 428.597 546.665 416.5 546.665 c
404.403 546.665 394.585 555.999 394.585 567.5 c
394.585 579.001 404.403 588.335 416.5 588.335 c
428.597 588.335 438.415 579.001 438.415 567.5 c
s
0.337 w
439.625 589.5 m
439.625 561.555 429.041 538.875 416 538.875 c
402.959 538.875 392.375 561.555 392.375 589.5 c
392.375 617.445 402.959 640.125 416 640.125 c
429.041 640.125 439.625 617.445 439.625 589.5 c
h
170.625 684.5 m
170.625 673.322 160.041 664.25 147 664.25 c
133.959 664.25 123.375 673.322 123.375 684.5 c
123.375 695.678 133.959 704.75 147 704.75 c
160.041 704.75 170.625 695.678 170.625 684.5 c
s
BT
/F1 1 Tf
9.45 0 0 9.45 130.45 686.785 Tm
/CS2 cs 0 0 0 sc
(1)Tj
2.857 -1.429 TD
(4)Tj
-1.429 -5.357 TD
0.929 Tc
(67)Tj
-0.714 -5 TD
(8)Tj
1.429 0.357 TD
(9)Tj
1.071 -0.714 TD
(A)Tj
-0.714 -1.786 TD
(B)Tj
-1.429 0 TD
(C)Tj
10.714 14.286 TD
(1)Tj
0.714 -1.429 TD
(5)Tj
0.357 -5.714 TD
0.571 Tc
(67)Tj
1.429 -5.714 TD
(A)Tj
-0.714 -1.429 TD
(B)Tj
-1.429 0 TD
(C)Tj
18.214 7.143 TD
(7)Tj
-1.071 0 TD
(6)Tj
-17.5 -5.714 TD
(8)Tj
1.429 1.071 TD
(9)Tj
0.714 10.357 TD
(4)Tj
0.714 1.429 TD
(3)Tj
-1.429 0.714 TD
(2)Tj
15 -0.714 TD
(1)Tj
1.071 0.714 TD
(2)Tj
1.429 -0.714 TD
(3)Tj
-0.357 -1.786 TD
(4)Tj
-1.786 0.357 TD
(5)Tj
0 -11.429 TD
(8)Tj
1.429 0.714 TD
(9)Tj
1.071 -0.714 TD
(A)Tj
-0.357 -1.429 TD
(B)Tj
-1.786 0 TD
(C)Tj
10.8 0 0 10.8 127.075 491.035 Tm
0 Tc
0 Tw
(Thin = response)Tj
0 1.25 TD
(Thick = key)Tj
T*
(Figure I)Tj
9.688 -2.5 TD
(Thin = partition wrt/response)Tj
0 1.25 TD
(Thick = key)Tj
T*
(Figure II)Tj
14.688 -2.188 TD
(Thin = response)Tj
0 1.25 TD
(Thick = partition wrt/key)Tj
T*
(Figure III)Tj
9.45 0 0 9.45 140.575 693.535 Tm
(2)Tj
1.786 -0.357 TD
(3)Tj
-2.143 -1.786 TD
(5)Tj
/T9 1 Tf
0.12 0 0 -0.12 157.92 466.78 Tm
/CS1 cs 0 sc
[(Figure)-31000(3:)]TJ
/T15 1 Tf
413 0 TD
(Equiv)Tj
209 0 TD
[(alence)-27000(Classes)-26000(and)-28000(Their)-27000(P)]TJ
979 0 TD
[(artitions)-26000(F)]TJ
379 0 TD
[(or)-27000(Example)-27000(1)]TJ
-2488 260 TD
[(if)-21000(they)-22000(actually)-21000(are)-20000(coreferen)]TJ
1017 0 TD
[(t)-20000(with)-22000(other)-21000(en)]TJ
527 0 TD
[(tities)-22000(in)-21000(the)-22000(text.)]TJ
632 0 TD
[(This)-22000(p)-2000(oten)]TJ
387 0 TD
[(tial)-21000(shortcoming)-20000(could)-21000(b)-2000(e)-22000(easily)]TJ
-2563 100 TD
[(enough)-26000(o)]TJ
325 0 TD
(v)Tj
42 0 TD
[(ercome)-24000(with)-28000(di\013eren)]TJ
736 0 TD
[(t)-27000(annotation)-26000(con)]TJ
601 0 TD
(v)Tj
42 0 TD
(en)Tj
81 0 TD
[(tions)-25000(and)-27000(with)-27000(minor)-27000(c)]TJ
825 0 TD
[(hanges)-25000(to)-27000(the)-27000(algorithm,)-26000(but)]TJ
-2651.999 99 TD
(the)Tj
149 0 TD
(decision)Tj
320 0 TD
(to)Tj
107 0 TD
(annotate)Tj
352 0 TD
(singletons)Tj
390 0 TD
(is)Tj
90 0 TD
(a)Tj
75 0 TD
(bit)Tj
135 0 TD
(of)Tj
101 0 TD
(a)Tj
75 0 TD
(philosophical)Tj
505 0 TD
(issue.)Tj
250 0 TD
(On)Tj
144 0 TD
(the)Tj
150 0 TD
(one)Tj
158 0 TD
(hand)Tj
214 0 TD
[(singletons)-32000(do)]TJ
-3215 100 TD
[(form)-26000(equiv)]TJ
384 0 TD
[(alence)-26000(classes,)-25000(and)-26000(those)-26000(equiv)]TJ
1099 0 TD
[(alence)-26000(classes)-26000(are)-25000(signi\014can)]TJ
985 0 TD
[(t)-26000(i)0(n)-26000(that)-27000(they)-27000(are)-25000(NOT)-26000(coreferen)]TJ
1193 0 TD
(t)Tj
-3661 99 TD
[(with)-28000(another)-27000(phrase)-27000(in)-29000(the)-28000(text)-29000(and)-28000(they)-28000(ma)]TJ
1631 0 TD
[(y)-27000(pla)]TJ
180 0 TD
[(y)-28000(a)0(n)-28000(imp)-2000(ortan)]TJ
521 0 TD
[(t)-27000(role)-28000(in)-28000(other)-28000(equiv)]TJ
726 0 TD
[(alence)-27000(classes)-27000(out)]TJ
-3058 100 TD
[(side)-23000(the)-23000(immediate)-24000(text)-24000(\(as)-22000(in)-24000(cross)-22000(do)-2000(cumen)]TJ
1616 0 TD
[(t)-23000(coreference\).)]TJ
545 0 TD
[(On)-23000(the)-24000(other)-23000(hand,)-24000(if)-24000(coreference)-21000(is)-24000(view)]TJ
1449.001 0 TD
(ed)Tj
-3610 100 TD
[(as)-25000(b)-2000(eing)-27000(ab)-2000(out)-26000(the)-27000(relations)-25000(b)-2000(e)0(t)]TJ
1152 0 TD
(w)Tj
58 0 TD
[(een)-27000(en)]TJ
228 0 TD
[(tities,)-27000(then)-26000(p)-2000(erhaps)-26000(is)-27000(mak)]TJ
964 0 TD
[(es)-25000(little)-27000(sense)-26000(to)-26000(annotate)-26000(and)-26000(score)]TJ
-2402 99 TD
(singletons.)Tj
-107 163 TD
(2.)Tj
107 0 TD
(All)Tj
142 0 TD
[(errors)-32000(are)]TJ
388 0 TD
(considered)Tj
413 0 TD
(to)Tj
107 0 TD
2 Tc
(be)Tj
120 0 TD
0 Tc
(equal.)Tj
270 0 TD
(The)Tj
177 0 TD
(MUC)Tj
232 0 TD
(scoring)Tj
289 0 TD
(algorithm)Tj
385 0 TD
[(p)-2000(enalizes)]TJ
359 0 TD
(the)Tj
150 0 TD
(precision)Tj
353 0 TD
(n)Tj
44 0 TD
(um)Tj
113 0 TD
[(b)-2000(ers)]TJ
-3542 100 TD
[(equally)-23000(for)-23000(all)-24000(t)]TJ
546.999 0 TD
[(yp)-2000(es)-23000(of)-24000(errors.)]TJ
543 0 TD
[(It)-25000(is)-23000(our)-23000(p)-2000(osition)-24000(that,)-25000(for)-23000(certain)-23000(tasks,)-24000(some)-23000(coreference)-23000(errors)-21000(do)-24000(more)]TJ
-1089.999 100 TD
[(damage)-26000(than)-28000(others.)]TJ
0 131 TD
[(Consider)-20000(the)-23000(follo)]TJ
631 0 TD
[(wing)-20000(examples:)]TJ
579 0 TD
[(supp)-2000(ose)-21000(the)-22000(truth)-23000(con)]TJ
778 0 TD
[(tains)-20000(t)]TJ
226 0 TD
(w)Tj
58 0 TD
[(o)-22000(large)-20000(coreference)-20000(c)]TJ
717 0 TD
[(hains)-21000(and)-22000(one)-21000(small)]TJ
-2989 100 TD
[(one)-22000(\(Figure)-22000(1\),)-23000(and)-23000(supp)-2000(ose)-22000(Figures)-21000(2)-22000(and)-23000(4)-22000(sho)]TJ
1712.999 0 TD
21 Tc
(wt)Tj
111 0 TD
(w)Tj
58 0 TD
0 Tc
[(o)-22000(di\013eren)]TJ
332 0 TD
[(t)-22000(resp)-2000(onses.)]TJ
453 0 TD
(W)Tj
78 0 TD
[(e)-23000(will)-23000(explore)-21000(t)]TJ
525 0 TD
(w)Tj
58 0 TD
[(o)-22000(di\013eren)]TJ
332 0 TD
(t)Tj
-3659.999 99 TD
(precision)Tj
353 0 TD
(errors.)Tj
290 0 TD
(The)Tj
177 0 TD
(\014rst)Tj
178 0 TD
(error)Tj
211 0 TD
(will)Tj
164 0 TD
(connect)Tj
311 0 TD
(one)Tj
159 0 TD
(of)Tj
102 0 TD
(the)Tj
150 0 TD
(large)Tj
210 0 TD
(coreference)Tj
434 0 TD
(c)Tj
35 0 TD
(hains)Tj
223 0 TD
(with)Tj
196 0 TD
(the)Tj
150 0 TD
(small)Tj
224 0 TD
(one)Tj
-3567.001 100 TD
[(\(Figure)-31000(2\).)]TJ
444 0 TD
[(The)-32000(second)-31000(error)-29000(o)-2000(ccurs)-31000(when)-32000(the)-32000(t)]TJ
1313 0 TD
(w)Tj
58 0 TD
[(o)-31000(large)-30000(coreference)-30000(c)]TJ
746 0 TD
[(hains)-31000(are)-31000(related)-31000(b)]TJ
689 0 TD
[(y)-31000(the)-32000(erran)]TJ
411 0 TD
(t)Tj
-3661 100 TD
(coreferen)Tj
325 0 TD
[(t)-22000(link)-23000(\(Figure)-23000(4\).)]TJ
634.999 0 TD
[(It)-24000(is)-23000(our)-23000(p)-2000(osition)-23000(that)-23000(the)-24000(second)-23000(error)-21000(is)-23000(more)-23000(damaging)-21000(b)-2000(ecause,)-24000(compared)]TJ
-959.999 99 TD
[(to)-27000(the)-28000(\014rst)-27000(error,)-25000(the)-28000(second)-27000(error)-25000(mak)]TJ
1408 0 TD
[(es)-27000(more)-26000(en)]TJ
385 0 TD
[(tities)-28000(coreferen)]TJ
533 0 TD
[(t)-26000(that)-28000(should)-27000(not)-27000(b)-2000(e)0(.)]TJ
793 0 TD
[(This)-28000(distinction)]TJ
-3119.001 100 TD
[(is)-23000(not)-24000(re\015ected)-24000(in)-24000(the)-25000([Vilain,)-28000(95)]TJ
1161 0 TD
[(])-24000(scorer)-22000(whic)]TJ
448 0 TD
[(h)-24000(scores)-22000(b)-2000(oth)-25000(resp)-2000(onses)-22000(as)-24000(ha)]TJ
1049.001 0 TD
[(ving)-23000(a)-23000(precision)-23000(score)-23000(of)-24000(90%)]TJ
-2658 100 TD
[(\(Figure)-27000(6\).)]TJ
/T13 1 Tf
-208 298 TD
(Revisions)Tj
500 0 TD
(to)Tj
137 0 TD
(the)Tj
195 0 TD
(Algorithm:)Tj
585 0 TD
(Our)Tj
229 0 TD
(B-CUBED)Tj
561 0 TD
(Algorithm)Tj
/T11 1 Tf
507 -37 TD
(2)Tj
/T15 1 Tf
-2714 266 TD
[(Our)-32000(B-CUBED)]TJ
597 0 TD
[(algorithm)-32000(w)]TJ
442 0 TD
[(as)-32000(designed)-32000(to)]TJ
556 0 TD
(o)Tj
40 0 TD
(v)Tj
42 0 TD
[(ercome)-31000(the)]TJ
434 0 TD
(t)Tj
30 0 TD
(w)Tj
58 0 TD
[(o)-32000(shortcomings)-32000(of)]TJ
684 0 TD
(the)Tj
148 0 TD
(Vilain)Tj
252 0 TD
(et.)Tj
146 0 TD
[(al)-32000(algorithm.)]TJ
-3428.999 99 TD
[(Instead)-24000(of)-24000(lo)-2000(oking)-24000(at)-24000(the)-25000(links)-24000(pro)-2000(duced)-24000(b)]TJ
1503 0 TD
[(y)-24000(a)-24000(system,)-24000(our)-24000(algorithm)-23000(lo)-2000(oks)-24000(at)-24000(the)-24000(presence/absence)-23000(of)-24000(en)]TJ
2217 0 TD
(tities)Tj
-3720 100 TD
(relativ)Tj
232 0 TD
[(e)-21000(t)0(o)-22000(eac)]TJ
268 0 TD
[(h)-21000(o)0(f)-22000(the)-22000(other)-22000(en)]TJ
586 0 TD
[(tities)-22000(in)-22000(the)-23000(equiv)]TJ
620 0 TD
[(alence)-21000(classes)-21000(pro)-2000(duced.)]TJ
895 0 TD
[(Therefore,)-22000(w)]TJ
453 0 TD
[(e)-21000(compute)-23000(the)-22000(precision)]TJ
-3054 100 TD
[(and)-25000(recall)-25000(n)]TJ
423 0 TD
(um)Tj
113 0 TD
[(b)-2000(ers)-25000(for)-25000(eac)]TJ
415 0 TD
25 Tc
[(he)25000(n)]TJ
152 0 TD
0 Tc
(tit)Tj
85 0 TD
[(y)-25000(i)0(n)-26000(the)-26000(do)-2000(cumen)]TJ
628 0 TD
[(t,)-26000(whic)]TJ
245 0 TD
[(h)-26000(are)-24000(then)-26000(com)]TJ
541 0 TD
[(bined)-26000(to)-25000(pro)-2000(duce)-25000(\014nal)-26000(precision)-24000(and)]TJ
-2602 99 TD
[(recall)-23000(n)]TJ
262 0 TD
(um)Tj
113 0 TD
[(b)-2000(ers)-24000(for)-24000(the)-25000(en)]TJ
520 0 TD
[(tire)-24000(output.)]TJ
452 0 TD
[(The)-25000(formal)-24000(mo)-2000(del-theoretic)-23000(v)]TJ
1057 0 TD
[(ersion)-23000(of)-25000(our)-23000(algorithm)-24000(is)-24000(discussed)-24000(in)-25000(the)]TJ
-2404 100 TD
[(next)-28000(section.)]TJ
ET
72 79.9 187.2 0.48 re
f
BT
/T10 1 Tf
0.12 0 0 -0.12 83.04 73.42 Tm
(2)Tj
/T11 1 Tf
35 23 TD
[(The)-24000(main)-23000(idea)-23000(of)-24000(this)-23000(algorithm)-23000(w)]TJ
1058.999 0 TD
[(as)-24000(initially)-22000(put)-25000(forth)-23000(b)]TJ
687.999 0 TD
[(y)-24000(Alan)-24000(W.)-23000(Biermann)-23000(of)-23000(Duk)]TJ
880 0 TD
[(e)-24000(Univ)]TJ
202 0 TD
(ersit)Tj
132 0 TD
(y)Tj
31 0 TD
(.)Tj
ET
endstream
endobj
30 0 obj
<<
/ProcSet [/PDF /Text /ImageB ]
/Font <<
/F1 31 0 R
/T9 22 0 R
/T10 4 0 R
/T11 5 0 R
/T13 7 0 R
/T15 9 0 R
>>
/ExtGState <<
/GS1 12 0 R
>>
/ColorSpace <<
/CS1 13 0 R
/CS2 32 0 R
>>
>>
endobj
32 0 obj
[/CalRGB <<
/Gamma [1.8 1.8 1.8]
/Matrix [0.45 0.245 0.025 0.316 0.672 0.141 0.185 0.083 0.923]
/WhitePoint [0.951 1 1.089]
>>
]
endobj
33 0 obj
<<
/Type /XObject
/Subtype /Image
/Name /Im3
/Width 107
/Height 99
/BitsPerComponent 8
/ColorSpace 13 0 R
/Length 613
/Filter /LZWDecode
>>
stream
?P8$
BaPd6DbQ8V-FcQv=HdR9$M'JeR K3|NkMIe8N VCt(dƉLi5
mR/B5)ZNT
_Wӫ=e`-\g6x0zO0恎_qMn`&L{my&nS,{uUvJ66Vœqq8\>'FvxM?3ɹ?v,UjY(w/%7z=X?_ߗ7F:ߟ?ͬ@$,ApLNZ?n+
PACD,P;jBL[A;L1aǭY!ǭ011V8QĊư$SJҜ7-?A,1Jp'L2("kG9>;9S73?&ҊQ2ӻ\#DEѽ'KRl
GQ#E23PPl7$Re(RTGMYY5=Su5LU_2U̍rf-6jY+T2VQ[zR3μyONwEʣImwm^m>^l/}
endstream
endobj
35 0 obj
<<
/Length 11494
>>
stream
BX /GS1 gs EX
q
106.992 0 0 99 254.16 619.66 cm
/Im3 Do
Q
BT
/T9 1 Tf
0.12 0 0 -0.12 234.24 597.7 Tm
/CS1 cs 0 sc
0 Tc
[(Figure)-31000(4:)]TJ
/T15 1 Tf
413 0 TD
[(Resp)-2000(onse:)]TJ
397 0 TD
[(Example)-27000(2)]TJ
-1668 312 TD
(Precision)Tj
/T8 1 Tf
331 12 TD
(i)Tj
/T15 1 Tf
44 -12 TD
(=)Tj
/T6 1 Tf
98 -56 TD
(numb)Tj
194 0 TD
(er)Tj
109 0 TD
(of)Tj
113 0 TD
(c)Tj
34 0 TD
(orr)Tj
108 0 TD
(e)Tj
34 0 TD
(ct)Tj
101 0 TD
(elements)Tj
347 0 TD
(in)Tj
106 0 TD
(the)Tj
142 0 TD
(output)Tj
265 0 TD
(chain)Tj
229 0 TD
(c)Tj
34 0 TD
(ontaining)Tj
377 0 TD
(entity)Tj
/T4 1 Tf
206.001 12 TD
(i)Tj
ET
1 i
188.04 562.54 291.6 0.48 re
f
BT
/T6 1 Tf
0.12 0 0 -0.12 204.72 553.42 Tm
(numb)Tj
194 0 TD
(er)Tj
109 0 TD
(of)Tj
113 0 TD
(elements)Tj
346 0 TD
(in)Tj
106 0 TD
(the)Tj
142 0 TD
(output)Tj
265 0 TD
(chain)Tj
229 0 TD
(c)Tj
34 0 TD
(ontaining)Tj
377 0 TD
(entity)Tj
/T4 1 Tf
206.001 12 TD
(i)Tj
/T15 1 Tf
-2680 209 TD
(Recall)Tj
/T8 1 Tf
223 12 TD
(i)Tj
/T15 1 Tf
46 -12 TD
(=)Tj
/T6 1 Tf
98 -56 TD
(numb)Tj
194 0 TD
(er)Tj
108 0 TD
(of)Tj
114 0 TD
(c)Tj
34 0 TD
(orr)Tj
108 0 TD
(e)Tj
34 0 TD
(ct)Tj
101 0 TD
(elements)Tj
346 0 TD
(in)Tj
107 0 TD
(the)Tj
142 0 TD
(output)Tj
264 0 TD
(chain)Tj
230 0 TD
(c)Tj
34 0 TD
(ontaining)Tj
376 0 TD
(entity)Tj
/T4 1 Tf
206.001 12 TD
(i)Tj
ET
181.68 529.18 291.6 0.48 re
f
BT
/T6 1 Tf
0.12 0 0 -0.12 201.48 520.06 Tm
(numb)Tj
194 0 TD
(er)Tj
109 0 TD
(of)Tj
113 0 TD
(elements)Tj
346 0 TD
(in)Tj
107 0 TD
(the)Tj
142 0 TD
(truth)Tj
211 0 TD
(chain)Tj
229 0 TD
(c)Tj
34 0 TD
(ontaining)Tj
377 0 TD
(entity)Tj
/T4 1 Tf
206 12 TD
(i)Tj
/T9 1 Tf
-2325 203 TD
[(Figure)-31000(5:)]TJ
/T15 1 Tf
413 0 TD
[(De\014nitions)-28000(for)-27000(Precision)-26000(and)-28000(Recall)-27000(for)-27000(an)-27000(En)]TJ
1658 0 TD
(tit)Tj
85 0 TD
(y)Tj
/T12 1 Tf
72 0 TD
(i)Tj
/T15 1 Tf
-2925 248 TD
(F)Tj
47 0 TD
[(or)-27000(an)-27000(en)]TJ
298 0 TD
(tit)Tj
85 0 TD
(y)Tj
37 0 TD
(,)Tj
/T12 1 Tf
50 0 TD
(i)Tj
/T15 1 Tf
29 0 TD
28 Tc
(,w)Tj
109 0 TD
0 Tc
[(e)-27000(de\014ne)-28000(the)-28000(precision)-27000(and)-27000(recall)-27000(with)-28000(resp)-2000(ect)-27000(to)-27000(that)-28000(en)]TJ
2012 0 TD
(tit)Tj
85 0 TD
[(y)-28000(i)0(n)-28000(Figure)-27000(5.)]TJ
-2752 154 TD
[(The)-27000(\014nal)-28000(precision)-26000(and)-28000(recall)-26000(n)]TJ
1128 0 TD
(um)Tj
113 0 TD
[(b)-2000(ers)-27000(are)-27000(computed)-28000(b)]TJ
744 0 TD
[(y)-27000(the)-28000(follo)]TJ
367 0 TD
[(wing)-27000(t)]TJ
228 0 TD
(w)Tj
58 0 TD
[(o)-27000(form)]TJ
236 0 TD
(ulae:)Tj
-1706 237 TD
[(Final)-27000(Precision)-22000(=)]TJ
/T8 1 Tf
691 -104 TD
(N)Tj
/T5 1 Tf
-36 25 TD
(X)Tj
/T8 1 Tf
9 178 TD
(i=1)Tj
/T15 1 Tf
125 -99 TD
(w)Tj
/T8 1 Tf
60 12 TD
(i)Tj
/T14 1 Tf
41 -12 TD
(\003)Tj
/T6 1 Tf
60 0 TD
(Pr)Tj
87 0 TD
(e)Tj
34 0 TD
(cision)Tj
/T4 1 Tf
211 12 TD
(i)Tj
/T15 1 Tf
-1169 317 TD
[(Final)-28000(Recall)-22000(=)]TJ
/T8 1 Tf
584 -104 TD
(N)Tj
/T5 1 Tf
-35 25 TD
(X)Tj
/T8 1 Tf
8 177 TD
(i=1)Tj
/T15 1 Tf
126 -98 TD
(w)Tj
/T8 1 Tf
60 12 TD
(i)Tj
/T14 1 Tf
41 -12 TD
(\003)Tj
/T6 1 Tf
60 0 TD
(R)Tj
57 0 TD
(e)Tj
34 0 TD
(c)Tj
34 0 TD
[(al)-4000(l)]TJ
/T4 1 Tf
88 12 TD
(i)Tj
/T15 1 Tf
-2463 198 TD
(where)Tj
/T12 1 Tf
235 0 TD
(N)Tj
/T15 1 Tf
98 0 TD
[(is)-22000(the)-23000(n)]TJ
260 0 TD
(um)Tj
113 0 TD
2 Tc
[(be)2000(r)-20000(o)2000(f)-21000(e)2000(n)]TJ
311 0 TD
0 Tc
[(tities)-22000(in)-23000(the)-22000(do)-2000(cumen)]TJ
754 0 TD
[(t,)-24000(and)]TJ
/T12 1 Tf
235 0 TD
(w)Tj
/T7 1 Tf
59 12 TD
(i)Tj
/T15 1 Tf
51 -12 TD
[(is)-22000(the)-23000(w)]TJ
274 0 TD
(eigh)Tj
146 0 TD
[(t)-22000(assigned)-21000(to)-22000(en)]TJ
554 0 TD
(tit)Tj
85 0 TD
(y)Tj
/T12 1 Tf
66 0 TD
(i)Tj
/T15 1 Tf
51 0 TD
[(in)-23000(the)-23000(do)-2000(cumen)]TJ
553 0 TD
(t.)Tj
-3845 99 TD
[(It)-25000(should)-25000(b)-2000(e)-25000(noted)-25000(that)-26000(the)-25000(B-CUBED)-25000(algorithm)-23000(implicitly)-26000(o)]TJ
2200.999 0 TD
(v)Tj
42 0 TD
[(ercomes)-23000(the)-25000(\014rst)-25000(shortcoming)-23000(of)-25000(the)-26000(Vilain)-25000(et.)]TJ
-2242.999 100 TD
[(al)-25000(algorithm)-25000(b)]TJ
511 0 TD
[(y)-26000(calculating)-25000(the)-26000(precision)-25000(and)-25000(recall)-25000(n)]TJ
1397 0 TD
(um)Tj
113 0 TD
[(b)-2000(ers)-26000(for)-25000(eac)]TJ
416 0 TD
25 Tc
[(he)25000(n)]TJ
152 0 TD
0 Tc
(tit)Tj
85 0 TD
[(y)-26000(i)0(n)-26000(the)-26000(do)-2000(cumen)]TJ
629 0 TD
[(t)-26000(\(irresp)-2000(ectiv)]TJ
468 0 TD
25 Tc
[(eo)25000(f)]TJ
-3771 99 TD
0 Tc
[(whether)-27000(an)-28000(en)]TJ
515 0 TD
(tit)Tj
85 0 TD
[(y)-27000(i)0(s)-28000(part)-27000(of)-27000(a)-28000(coreference)-26000(c)]TJ
961 0 TD
(hain\).)Tj
-1436 154 TD
(Di\013eren)Tj
285 0 TD
(t)Tj
68 0 TD
(w)Tj
58 0 TD
(eigh)Tj
146 0 TD
(ting)Tj
179 0 TD
(sc)Tj
68 0 TD
(hemes)Tj
257 0 TD
[(pro)-2000(duce)]TJ
325 0 TD
(di\013eren)Tj
268 0 TD
(t)Tj
69 0 TD
(v)Tj
42 0 TD
(ersions)Tj
282 0 TD
(of)Tj
103 0 TD
(the)Tj
152 0 TD
(algorithm.)Tj
437 0 TD
(The)Tj
179 0 TD
(c)Tj
35 0 TD
(hoice)Tj
221 0 TD
(of)Tj
103 0 TD
(the)Tj
152 0 TD
(w)Tj
58 0 TD
(eigh)Tj
146 0 TD
(ting)Tj
-3758 99 TD
(sc)Tj
68 0 TD
[(heme)-27000(is)-28000(determined)-27000(b)]TJ
777 0 TD
[(y)-28000(the)-28000(task)-27000(for)-27000(whic)]TJ
684 0 TD
[(h)-27000(the)-28000(algorithm)-27000(is)-27000(going)-27000(to)-27000(b)-2000(e)-28000(used.)]TJ
-1404 154 TD
(When)Tj
254 0 TD
(coreference)Tj
440 0 TD
(\(or)Tj
146 0 TD
[(cross-do)-2000(cumen)]TJ
529 0 TD
(t)Tj
70 0 TD
(coreference\))Tj
472 0 TD
(is)Tj
96 0 TD
(used)Tj
202 0 TD
(for)Tj
140 0 TD
(an)Tj
128 0 TD
(information)Tj
462 0 TD
(extraction)Tj
407 0 TD
(task,)Tj
217 0 TD
(where)Tj
-3688 99 TD
(information)Tj
460 0 TD
[(ab)-2000(out)]TJ
247 0 TD
(ev)Tj
79 0 TD
(ery)Tj
151 0 TD
(en)Tj
81 0 TD
(tit)Tj
85 0 TD
(y)Tj
81 0 TD
(in)Tj
107 0 TD
(an)Tj
125 0 TD
(equiv)Tj
189 0 TD
(alence)Tj
259 0 TD
(class)Tj
205 0 TD
(is)Tj
93 0 TD
[(imp)-2000(ortan)]TJ
333 0 TD
(t,)Tj
95 0 TD
(the)Tj
153 0 TD
(w)Tj
58 0 TD
(eigh)Tj
146 0 TD
(ting)Tj
179 0 TD
(sc)Tj
68 0 TD
(heme)Tj
227 0 TD
(assigns)Tj
288 0 TD
(equal)Tj
-3709 100 TD
(w)Tj
58 0 TD
(eigh)Tj
146 0 TD
[(ts)-26000(for)-27000(ev)]TJ
297 0 TD
[(ery)-26000(en)]TJ
221 0 TD
(tit)Tj
85 0 TD
(y)Tj
/T12 1 Tf
71 0 TD
(i)Tj
/T15 1 Tf
29 0 TD
(.)Tj
60 0 TD
(F)Tj
47 0 TD
[(or)-26000(example,)-27000(the)-28000(w)]TJ
650 0 TD
(eigh)Tj
146 0 TD
[(t)-26000(assigned)-26000(to)-27000(eac)]TJ
601 0 TD
27 Tc
[(he)27000(n)]TJ
154 0 TD
0 Tc
(tit)Tj
85 0 TD
[(y)-27000(i)0(n)-28000(Figure)-26000(1)-27000(i)0(s)-27000(1/12.)]TJ
807 0 TD
[(As)-27000(sho)]TJ
241 0 TD
[(wn)-27000(in)]TJ
-3697.999 99 TD
[(Figure)-25000(6,)-26000(the)-26000(precision)-25000(scores)-25000(for)-25000(resp)-2000(onses)-24000(in)-27000(Figures)-25000(2)-25000(and)-26000(4)-25000(are)-25000(16/21)-24000(\(76%\))-25000(and)-26000(7/12)-24000(\(58%\))-26000(resp)-2000(ectiv)]TJ
3780.999 0 TD
(ely)Tj
97 0 TD
(,)Tj
-3877.999 100 TD
[(using)-25000(equal)-24000(w)]TJ
489 0 TD
(eigh)Tj
146 0 TD
[(ts)-24000(for)-24000(all)-25000(en)]TJ
407 0 TD
(tities.)Tj
239 0 TD
[(Recall)-25000(for)-24000(b)-2000(oth)-25000(resp)-2000(onses)-24000(is)-25000(100%.)]TJ
1265 0 TD
[(It)-25000(should)-24000(b)-2000(e)-26000(noted)-25000(that)-25000(the)-25000(algorithm)]TJ
-2545.999 100 TD
[(p)-2000(enalizes)]TJ
359 0 TD
(the)Tj
149 0 TD
(precision)Tj
353 0 TD
(n)Tj
44 0 TD
(um)Tj
113 0 TD
[(b)-2000(ers)]TJ
184 0 TD
(more)Tj
214 0 TD
(for)Tj
133 0 TD
(the)Tj
149 0 TD
[(error)-31000(made)]TJ
437 0 TD
(in)Tj
102 0 TD
(Figure)Tj
268 0 TD
(4)Tj
76 0 TD
(than)Tj
199 0 TD
(the)Tj
149 0 TD
(one)Tj
158 0 TD
(made)Tj
228 0 TD
(in)Tj
103 0 TD
(Figure)Tj
268 0 TD
(2.)Tj
119 0 TD
(As)Tj
-3805 99 TD
(eviden)Tj
231 0 TD
[(t)-24000(from)-23000(the)-24000(t)]TJ
417 0 TD
(w)Tj
58 0 TD
[(o)-23000(examples,)-23000(this)-24000(v)]TJ
642 0 TD
[(ersion)-22000(of)-24000(the)-24000(B-CUBED)-23000(algorithm)-23000(\(using)-23000(equal)-23000(w)]TJ
1770 0 TD
(eigh)Tj
146 0 TD
[(ts)-23000(for)-23000(eac)]TJ
325 0 TD
23 Tc
[(he)23000(n)]TJ
150 0 TD
0 Tc
(tit)Tj
85 0 TD
(y\))Tj
-3823.999 100 TD
[(is)-27000(a)-28000(precision)-26000(orien)]TJ
678 0 TD
[(ted)-27000(algorithm)-27000(i.e.)]TJ
663 0 TD
[(it)-29000(is)-27000(sensitiv)]TJ
436 0 TD
[(e)-27000(t)0(o)-27000(precision)-27000(errors.)]TJ
-1651.999 153 TD
[(But,)-23000(for)-20000(an)-21000(information)-21000(retriev)]TJ
1089.999 0 TD
[(al)-21000(\(IR\))-22000(task,)-22000(or)-20000(a)-21000(w)]TJ
675 0 TD
[(eb)-21000(searc)]TJ
284 0 TD
[(h)-20000(task,)-22000(where)-21000(an)-21000(user)-21000(is)-21000(presen)]TJ
1082 0 TD
[(ted)-21000(with)-22000(classes)-20000(of)]TJ
-3256 100 TD
[(do)-2000(cumen)]TJ
323 0 TD
[(ts)-29000(that)-29000(p)-2000(ertain)-28000(to)-29000(the)-29000(same)-28000(en)]TJ
1101 0 TD
(tit)Tj
85 0 TD
(y)Tj
37 0 TD
[(,)-29000(the)-30000(w)]TJ
255 0 TD
(eigh)Tj
146 0 TD
[(ting)-27000(sc)]TJ
238 0 TD
[(heme)-29000(assigns)-27000(equal)-28000(w)]TJ
775 0 TD
(eigh)Tj
146 0 TD
[(ts)-28000(to)-29000(eac)]TJ
310 0 TD
[(h)-28000(equiv)]TJ
263 0 TD
(alence)Tj
-3678.999 99 TD
(class.)Tj
253 0 TD
(The)Tj
179 0 TD
(w)Tj
58 0 TD
(eigh)Tj
146 0 TD
(t)Tj
68 0 TD
(for)Tj
135 0 TD
(eac)Tj
114 0 TD
(h)Tj
82 0 TD
(en)Tj
81 0 TD
(tit)Tj
85 0 TD
(y)Tj
80 0 TD
(within)Tj
267 0 TD
(an)Tj
124 0 TD
(equiv)Tj
189 0 TD
(alence)Tj
257 0 TD
(class)Tj
204 0 TD
(is)Tj
92 0 TD
(computed)Tj
391 0 TD
(b)Tj
44 0 TD
(y)Tj
80 0 TD
(dividing)Tj
329 0 TD
(the)Tj
152 0 TD
(w)Tj
58 0 TD
(eigh)Tj
146 0 TD
(t)Tj
68 0 TD
(of)Tj
103 0 TD
(the)Tj
-3785 100 TD
(equiv)Tj
189 0 TD
[(alence)-29000(class)-28000(b)]TJ
491 0 TD
[(y)-29000(the)-30000(n)]TJ
262 0 TD
(um)Tj
113 0 TD
2 Tc
[(be)2000(r)-27000(o)2000(f)-28000(e)2000(n)]TJ
325 0 TD
0 Tc
[(tities)-29000(in)-30000(that)-30000(class.)]TJ
722 0 TD
[(Recall)-29000(is)-29000(calculated)-29000(b)]TJ
775 0 TD
[(y)-29000(assigning)-28000(equal)-29000(w)]TJ
710 0 TD
(eigh)Tj
146 0 TD
[(ts)-28000(to)]TJ
-3732.999 99.999 TD
(eac)Tj
114 0 TD
[(h)-21000(equiv)]TJ
256 0 TD
[(alence)-20000(class)-21000(in)-21000(the)-22000(truth)-22000(while)-21000(precision)-21000(is)-21000(calculated)-21000(b)]TJ
1927 0 TD
[(y)-21000(assigning)-20000(equal)-20000(w)]TJ
685 0 TD
(eigh)Tj
146 0 TD
[(ts)-21000(to)-21000(eac)]TJ
295 0 TD
[(h)-21000(equiv)]TJ
256 0 TD
(alence)Tj
-3679 99 TD
[(class)-24000(in)-25000(the)-25000(resp)-2000(onse.)]TJ
793 0 TD
(F)Tj
47 0 TD
[(or)-25000(example,)-25000(in)-25000(Figure)-24000(2,)-25000(the)-26000(w)]TJ
1088 0 TD
(eigh)Tj
146 0 TD
[(ting)-24000(sc)]TJ
235 0 TD
[(heme)-24000(assigns)-24000(a)-24000(w)]TJ
613 0 TD
(eigh)Tj
146 0 TD
[(t)-24000(o)0(f)-25000(1/10)-23000(to)-25000(eac)]TJ
552 0 TD
24 Tc
[(he)24000(n)]TJ
151 0 TD
0 Tc
(tit)Tj
85 0 TD
(y)Tj
-3856 100 TD
(in)Tj
106 0 TD
(the)Tj
152 0 TD
(\014rst)Tj
181 0 TD
(equiv)Tj
189 0 TD
(alence)Tj
259 0 TD
(class,)Tj
229 0 TD
(and)Tj
171 0 TD
(a)Tj
78 0 TD
(w)Tj
58 0 TD
(eigh)Tj
146 0 TD
(t)Tj
69 0 TD
(of)Tj
103 0 TD
(1/14)Tj
203 0 TD
(to)Tj
111 0 TD
(eac)Tj
114 0 TD
(h)Tj
82 0 TD
(en)Tj
81 0 TD
(tit)Tj
85 0 TD
(y)Tj
81 0 TD
(in)Tj
106 0 TD
(the)Tj
153 0 TD
(second)Tj
277 0 TD
(equiv)Tj
189 0 TD
(alence)Tj
259 0 TD
(class,)Tj
229 0 TD
(when)Tj
-3711 100 TD
[(calculating)-27000(precision.)]TJ
798 0 TD
[(Using)-28000(this)-27000(w)]TJ
453 0 TD
(eigh)Tj
146 0 TD
[(ting)-27000(sc)]TJ
238 0 TD
[(heme,)-27000(the)-27000(precision)-27000(scores)-26000(for)-27000(resp)-2000(onses)-26000(in)-27000(Figures)-27000(2)-27000(and)-27000(4)-27000(are)]TJ
-1634.999 99 TD
[(39/49)-25000(\(79.6%\))-27000(and)-28000(3/4)-26000(\(75%\))-27000(resp)-2000(ectiv)]TJ
1423.999 0 TD
(ely)Tj
97 0 TD
(.)Tj
59 0 TD
[(Recall)-28000(for)-27000(b)-2000(oth)-28000(resp)-2000(onses)-26000(is)-28000(100%.)]TJ
ET
endstream
endobj
36 0 obj
<<
/ProcSet [/PDF /Text /ImageB ]
/Font <<
/T4 37 0 R
/T5 26 0 R
/T6 27 0 R
/T7 20 0 R
/T8 21 0 R
/T9 22 0 R
/T12 6 0 R
/T14 8 0 R
/T15 9 0 R
>>
/XObject <<
/Im3 33 0 R
>>
/ExtGState <<
/GS1 12 0 R
>>
/ColorSpace <<
/CS1 13 0 R
>>
>>
endobj
39 0 obj
<<
/Length 19125
>>
stream
/CS1 cs 0 sc
BX /GS1 gs EX
1 i
56.52 718.18 502.44 0.48 re
f
56.28 706.3 0.48 12 re
f
58.32 706.3 0.48 12 re
f
BT
/T15 1 Tf
0.12 0 0 -0.12 64.44 709.9 Tm
0 Tc
(Output)Tj
ET
116.64 706.3 0.48 12 re
f
BT
0.12 0 0 -0.12 122.76 709.9 Tm
[(MUC)-28000(Algorithm)]TJ
ET
200.28 706.3 0.48 12 re
f
BT
0.12 0 0 -0.12 206.4 709.9 Tm
[(B-CUBED)-27000(Algorithm)-28000(\(equal)-27000(w)]TJ
1124 0 TD
(eigh)Tj
146 0 TD
[(ts)-27000(for)-27000(ev)]TJ
298 0 TD
[(ery)-26000(en)]TJ
221 0 TD
(tit)Tj
85 0 TD
(y\))Tj
ET
556.68 706.3 0.48 12 re
f
558.72 706.3 0.48 12 re
f
56.52 705.94 502.44 0.48 re
f
56.28 693.94 0.48 12 re
f
58.32 693.94 0.48 12 re
f
116.64 693.94 0.48 12 re
f
200.28 693.94 0.48 12 re
f
556.68 693.94 0.48 12 re
f
558.72 693.94 0.48 12 re
f
56.28 681.94 0.48 12.12 re
f
58.32 681.94 0.48 12.12 re
f
116.64 681.94 0.48 12.12 re
f
BT
0.12 0 0 -0.12 122.76 685.54 Tm
(P:)Tj
/T8 1 Tf
134 -32 TD
(9)Tj
ET
136.8 687.7 8.04 0.48 re
f
BT
0.12 0 0 -0.12 136.8 682.06 Tm
(10)Tj
/T15 1 Tf
104 -29 TD
(\(90%\))Tj
ET
200.28 681.94 0.48 12.12 re
f
BT
0.12 0 0 -0.12 206.4 685.54 Tm
(P:)Tj
/T8 1 Tf
134 -32 TD
(1)Tj
ET
220.44 687.7 8.04 0.48 re
f
BT
0.12 0 0 -0.12 220.44 682.06 Tm
(12)Tj
/T15 1 Tf
104 -29 TD
27 Tc
(*[)Tj
/T8 1 Tf
102 -32 TD
(5)Tj
ET
245.28 687.7 4.08 0.48 re
f
BT
0.12 0 0 -0.12 245.28 682.06 Tm
(5)Tj
/T15 1 Tf
61 -29 TD
(+)Tj
/T8 1 Tf
93 -32 TD
(5)Tj
ET
263.76 687.7 4.08 0.48 re
f
BT
0.12 0 0 -0.12 263.76 682.06 Tm
(5)Tj
/T15 1 Tf
62 -29 TD
(+)Tj
/T8 1 Tf
93 -32 TD
(5)Tj
ET
282.36 687.7 4.08 0.48 re
f
BT
0.12 0 0 -0.12 282.36 682.06 Tm
(5)Tj
/T15 1 Tf
61 -29 TD
(+)Tj
/T8 1 Tf
93 -32 TD
(5)Tj
ET
300.84 687.7 4.08 0.48 re
f
BT
0.12 0 0 -0.12 300.84 682.06 Tm
(5)Tj
/T15 1 Tf
61 -29 TD
(+)Tj
/T8 1 Tf
93 -32 TD
(5)Tj
ET
319.32 687.7 4.08 0.48 re
f
BT
0.12 0 0 -0.12 319.32 682.06 Tm
(5)Tj
/T15 1 Tf
62 -29 TD
(+)Tj
/T8 1 Tf
93 -32 TD
(2)Tj
ET
337.92 687.7 4.08 0.48 re
f
BT
0.12 0 0 -0.12 337.92 682.06 Tm
(7)Tj
/T15 1 Tf
61 -29 TD
(+)Tj
/T8 1 Tf
93 -32 TD
(2)Tj
ET
356.4 687.7 4.08 0.48 re
f
BT
0.12 0 0 -0.12 356.4 682.06 Tm
(7)Tj
/T15 1 Tf
62 -29 TD
(+)Tj
/T8 1 Tf
93 -32 TD
(5)Tj
ET
375 687.7 4.08 0.48 re
f
BT
0.12 0 0 -0.12 375 682.06 Tm
(7)Tj
/T15 1 Tf
61 -29 TD
(+)Tj
/T8 1 Tf
93 -32 TD
(5)Tj
ET
393.48 687.7 4.08 0.48 re
f
BT
0.12 0 0 -0.12 393.48 682.06 Tm
(7)Tj
/T15 1 Tf
62 -29 TD
(+)Tj
/T8 1 Tf
93 -32 TD
(5)Tj
ET
412.08 687.7 4.08 0.48 re
f
BT
0.12 0 0 -0.12 412.08 682.06 Tm
(7)Tj
/T15 1 Tf
61 -29 TD
(+)Tj
/T8 1 Tf
93 -32 TD
(5)Tj
ET
430.56 687.7 4.08 0.48 re
f
BT
0.12 0 0 -0.12 430.56 682.06 Tm
(7)Tj
/T15 1 Tf
62 -29 TD
(+)Tj
/T8 1 Tf
93 -32 TD
(5)Tj
ET
449.16 687.7 4.08 0.48 re
f
BT
0.12 0 0 -0.12 449.16 682.06 Tm
(7)Tj
/T15 1 Tf
43 -29 TD
28 Tc
(]=)Tj
/T8 1 Tf
153 -32 TD
0 Tc
(16)Tj
ET
472.68 687.7 8.04 0.48 re
f
BT
0.12 0 0 -0.12 472.68 682.06 Tm
(21)Tj
/T15 1 Tf
104 -29 TD
(\(76%\))Tj
ET
556.68 681.94 0.48 12.12 re
f
558.72 681.94 0.48 12.12 re
f
56.28 669.94 0.48 12 re
f
58.32 669.94 0.48 12 re
f
BT
0.12 0 0 -0.12 64.44 673.54 Tm
[(Example)-27000(1)]TJ
ET
116.64 669.94 0.48 12 re
f
200.28 669.94 0.48 12 re
f
556.68 669.94 0.48 12 re
f
558.72 669.94 0.48 12 re
f
116.76 669.58 442.08 0.48 re
f
56.28 658.06 0.48 12 re
f
58.32 658.06 0.48 12 re
f
116.64 658.06 0.48 12 re
f
200.28 658.06 0.48 12 re
f
556.68 658.06 0.48 12 re
f
558.72 658.06 0.48 12 re
f
56.28 646.06 0.48 12.12 re
f
58.32 646.06 0.48 12.12 re
f
116.64 646.06 0.48 12.12 re
f
BT
0.12 0 0 -0.12 122.76 649.66 Tm
(R:)Tj
/T8 1 Tf
122 -32 TD
(9)Tj
ET
137.4 651.82 4.08 0.48 re
f
BT
0.12 0 0 -0.12 137.4 646.18 Tm
(9)Tj
/T15 1 Tf
71 -29 TD
(\(100%\))Tj
ET
200.28 646.06 0.48 12.12 re
f
BT
0.12 0 0 -0.12 206.4 649.66 Tm
(R:)Tj
/T8 1 Tf
139 -32 TD
(1)Tj
ET
221.04 651.82 8.04 0.48 re
f
BT
0.12 0 0 -0.12 221.04 646.18 Tm
(12)Tj
/T15 1 Tf
104 -29 TD
27 Tc
(*[)Tj
/T8 1 Tf
102 -32 TD
(5)Tj
ET
245.76 651.82 4.08 0.48 re
f
BT
0.12 0 0 -0.12 245.76 646.18 Tm
(5)Tj
/T15 1 Tf
62 -29 TD
(+)Tj
/T8 1 Tf
93 -32 TD
(5)Tj
ET
264.36 651.82 4.08 0.48 re
f
BT
0.12 0 0 -0.12 264.36 646.18 Tm
(5)Tj
/T15 1 Tf
61 -29 TD
(+)Tj
/T8 1 Tf
93 -32 TD
(5)Tj
ET
282.84 651.82 4.08 0.48 re
f
BT
0.12 0 0 -0.12 282.84 646.18 Tm
(5)Tj
/T15 1 Tf
62 -29 TD
(+)Tj
/T8 1 Tf
93 -32 TD
(5)Tj
ET
301.44 651.82 4.08 0.48 re
f
BT
0.12 0 0 -0.12 301.44 646.18 Tm
(5)Tj
/T15 1 Tf
61 -29 TD
(+)Tj
/T8 1 Tf
93 -32 TD
(5)Tj
ET
319.92 651.82 4.08 0.48 re
f
BT
0.12 0 0 -0.12 319.92 646.18 Tm
(5)Tj
/T15 1 Tf
62 -29 TD
(+)Tj
/T8 1 Tf
93 -32 TD
(2)Tj
ET
338.52 651.82 4.08 0.48 re
f
BT
0.12 0 0 -0.12 338.52 646.18 Tm
(2)Tj
/T15 1 Tf
61 -29 TD
(+)Tj
/T8 1 Tf
93 -32 TD
(2)Tj
ET
357 651.82 4.08 0.48 re
f
BT
0.12 0 0 -0.12 357 646.18 Tm
(2)Tj
/T15 1 Tf
62 -29 TD
(+)Tj
/T8 1 Tf
93 -32 TD
(5)Tj
ET
375.6 651.82 4.08 0.48 re
f
BT
0.12 0 0 -0.12 375.6 646.18 Tm
(5)Tj
/T15 1 Tf
61 -29 TD
(+)Tj
/T8 1 Tf
93 -32 TD
(5)Tj
ET
394.08 651.82 4.08 0.48 re
f
BT
0.12 0 0 -0.12 394.08 646.18 Tm
(5)Tj
/T15 1 Tf
62 -29 TD
(+)Tj
/T8 1 Tf
93 -32 TD
(5)Tj
ET
412.68 651.82 4.08 0.48 re
f
BT
0.12 0 0 -0.12 412.68 646.18 Tm
(5)Tj
/T15 1 Tf
61 -29 TD
(+)Tj
/T8 1 Tf
93 -32 TD
(5)Tj
ET
431.16 651.82 4.08 0.48 re
f
BT
0.12 0 0 -0.12 431.16 646.18 Tm
(5)Tj
/T15 1 Tf
62 -29 TD
(+)Tj
/T8 1 Tf
93 -32 TD
(5)Tj
ET
449.76 651.82 4.08 0.48 re
f
BT
0.12 0 0 -0.12 449.76 646.18 Tm
(5)Tj
/T15 1 Tf
43 -29 TD
0 Tc
[(])-27000(=)-28000(100%)]TJ
ET
556.68 646.06 0.48 12.12 re
f
558.72 646.06 0.48 12.12 re
f
56.28 634.06 0.48 12 re
f
58.32 634.06 0.48 12 re
f
116.64 634.06 0.48 12 re
f
200.28 634.06 0.48 12 re
f
556.68 634.06 0.48 12 re
f
558.72 634.06 0.48 12 re
f
56.52 633.7 502.44 0.48 re
f
56.28 621.7 0.48 12 re
f
58.32 621.7 0.48 12 re
f
116.64 621.7 0.48 12 re
f
200.28 621.7 0.48 12 re
f
556.68 621.7 0.48 12 re
f
558.72 621.7 0.48 12 re
f
56.28 609.7 0.48 12.12 re
f
58.32 609.7 0.48 12.12 re
f
116.64 609.7 0.48 12.12 re
f
BT
0.12 0 0 -0.12 122.76 613.3 Tm
(P:)Tj
/T8 1 Tf
134 -33 TD
(9)Tj
ET
136.8 615.58 8.04 0.48 re
f
BT
0.12 0 0 -0.12 136.8 609.82 Tm
(10)Tj
/T15 1 Tf
104 -29 TD
(\(90%\))Tj
ET
200.28 609.7 0.48 12.12 re
f
BT
0.12 0 0 -0.12 206.4 613.3 Tm
(P:)Tj
/T8 1 Tf
134 -33 TD
(1)Tj
ET
220.44 615.58 8.04 0.48 re
f
BT
0.12 0 0 -0.12 220.44 609.82 Tm
(12)Tj
/T15 1 Tf
104 -29 TD
27 Tc
(*[)Tj
/T8 1 Tf
119 -33 TD
(5)Tj
ET
245.28 615.58 8.04 0.48 re
f
BT
0.12 0 0 -0.12 245.28 609.82 Tm
0 Tc
(10)Tj
/T15 1 Tf
94 -29 TD
(+)Tj
/T8 1 Tf
110 -33 TD
(5)Tj
ET
267.72 615.58 8.04 0.48 re
f
BT
0.12 0 0 -0.12 267.72 609.82 Tm
(10)Tj
/T15 1 Tf
95 -29 TD
(+)Tj
/T8 1 Tf
109 -33 TD
(5)Tj
ET
290.28 615.58 8.04 0.48 re
f
BT
0.12 0 0 -0.12 290.28 609.82 Tm
(10)Tj
/T15 1 Tf
94 -29 TD
(+)Tj
/T8 1 Tf
110 -33 TD
(5)Tj
ET
312.72 615.58 8.04 0.48 re
f
BT
0.12 0 0 -0.12 312.72 609.82 Tm
(10)Tj
/T15 1 Tf
95 -29 TD
(+)Tj
/T8 1 Tf
109 -33 TD
(5)Tj
ET
335.28 615.58 8.04 0.48 re
f
BT
0.12 0 0 -0.12 335.28 609.82 Tm
(10)Tj
/T15 1 Tf
94 -29 TD
(+)Tj
/T8 1 Tf
93 -33 TD
(2)Tj
ET
357.72 615.58 4.08 0.48 re
f
BT
0.12 0 0 -0.12 357.72 609.82 Tm
(2)Tj
/T15 1 Tf
62 -29 TD
(+)Tj
/T8 1 Tf
93 -33 TD
(2)Tj
ET
376.32 615.58 4.08 0.48 re
f
BT
0.12 0 0 -0.12 376.32 609.82 Tm
(2)Tj
/T15 1 Tf
61 -29 TD
(+)Tj
/T8 1 Tf
110 -33 TD
(5)Tj
ET
394.8 615.58 8.04 0.48 re
f
BT
0.12 0 0 -0.12 394.8 609.82 Tm
(10)Tj
/T15 1 Tf
95 -29 TD
(+)Tj
/T8 1 Tf
110 -33 TD
(5)Tj
ET
417.36 615.58 8.04 0.48 re
f
BT
0.12 0 0 -0.12 417.36 609.82 Tm
(10)Tj
/T15 1 Tf
95 -29 TD
(+)Tj
/T8 1 Tf
109 -33 TD
(5)Tj
ET
439.92 615.58 8.04 0.48 re
f
BT
0.12 0 0 -0.12 439.92 609.82 Tm
(10)Tj
/T15 1 Tf
94 -29 TD
(+)Tj
/T8 1 Tf
110 -33 TD
(5)Tj
ET
462.36 615.58 8.04 0.48 re
f
BT
0.12 0 0 -0.12 462.36 609.82 Tm
(10)Tj
/T15 1 Tf
95 -29 TD
(+)Tj
/T8 1 Tf
109 -33 TD
(5)Tj
ET
484.92 615.58 8.04 0.48 re
f
BT
0.12 0 0 -0.12 484.92 609.82 Tm
(10)Tj
/T15 1 Tf
76 -29 TD
28 Tc
(]=)Tj
/T8 1 Tf
169 -33 TD
(7)Tj
ET
512.4 615.58 8.04 0.48 re
f
BT
0.12 0 0 -0.12 512.4 609.82 Tm
0 Tc
(12)Tj
/T15 1 Tf
104 -29 TD
(\(58%\))Tj
ET
556.68 609.7 0.48 12.12 re
f
558.72 609.7 0.48 12.12 re
f
56.28 597.7 0.48 12 re
f
58.32 597.7 0.48 12 re
f
BT
0.12 0 0 -0.12 64.44 601.3 Tm
[(Example)-27000(2)]TJ
ET
116.64 597.7 0.48 12 re
f
200.28 597.7 0.48 12 re
f
556.68 597.7 0.48 12 re
f
558.72 597.7 0.48 12 re
f
116.76 597.34 442.08 0.48 re
f
56.28 585.82 0.48 12 re
f
58.32 585.82 0.48 12 re
f
116.64 585.82 0.48 12 re
f
200.28 585.82 0.48 12 re
f
556.68 585.82 0.48 12 re
f
558.72 585.82 0.48 12 re
f
56.28 573.82 0.48 12.12 re
f
58.32 573.82 0.48 12.12 re
f
116.64 573.82 0.48 12.12 re
f
BT
0.12 0 0 -0.12 122.76 577.42 Tm
(R:)Tj
/T8 1 Tf
122 -32 TD
(9)Tj
ET
137.4 579.58 4.08 0.48 re
f
BT
0.12 0 0 -0.12 137.4 573.94 Tm
(9)Tj
/T15 1 Tf
71 -29 TD
(\(100%\))Tj
ET
200.28 573.82 0.48 12.12 re
f
BT
0.12 0 0 -0.12 206.4 577.42 Tm
(R:)Tj
/T8 1 Tf
139 -32 TD
(1)Tj
ET
221.04 579.58 8.04 0.48 re
f
BT
0.12 0 0 -0.12 221.04 573.94 Tm
(12)Tj
/T15 1 Tf
104 -29 TD
27 Tc
(*[)Tj
/T8 1 Tf
102 -32 TD
(5)Tj
ET
245.76 579.58 4.08 0.48 re
f
BT
0.12 0 0 -0.12 245.76 573.94 Tm
(5)Tj
/T15 1 Tf
62 -29 TD
(+)Tj
/T8 1 Tf
93 -32 TD
(5)Tj
ET
264.36 579.58 4.08 0.48 re
f
BT
0.12 0 0 -0.12 264.36 573.94 Tm
(5)Tj
/T15 1 Tf
61 -29 TD
(+)Tj
/T8 1 Tf
93 -32 TD
(5)Tj
ET
282.84 579.58 4.08 0.48 re
f
BT
0.12 0 0 -0.12 282.84 573.94 Tm
(5)Tj
/T15 1 Tf
62 -29 TD
(+)Tj
/T8 1 Tf
93 -32 TD
(5)Tj
ET
301.44 579.58 4.08 0.48 re
f
BT
0.12 0 0 -0.12 301.44 573.94 Tm
(5)Tj
/T15 1 Tf
61 -29 TD
(+)Tj
/T8 1 Tf
93 -32 TD
(5)Tj
ET
319.92 579.58 4.08 0.48 re
f
BT
0.12 0 0 -0.12 319.92 573.94 Tm
(5)Tj
/T15 1 Tf
62 -29 TD
(+)Tj
/T8 1 Tf
93 -32 TD
(2)Tj
ET
338.52 579.58 4.08 0.48 re
f
BT
0.12 0 0 -0.12 338.52 573.94 Tm
(2)Tj
/T15 1 Tf
61 -29 TD
(+)Tj
/T8 1 Tf
93 -32 TD
(2)Tj
ET
357 579.58 4.08 0.48 re
f
BT
0.12 0 0 -0.12 357 573.94 Tm
(2)Tj
/T15 1 Tf
62 -29 TD
(+)Tj
/T8 1 Tf
93 -32 TD
(5)Tj
ET
375.6 579.58 4.08 0.48 re
f
BT
0.12 0 0 -0.12 375.6 573.94 Tm
(5)Tj
/T15 1 Tf
61 -29 TD
(+)Tj
/T8 1 Tf
93 -32 TD
(5)Tj
ET
394.08 579.58 4.08 0.48 re
f
BT
0.12 0 0 -0.12 394.08 573.94 Tm
(5)Tj
/T15 1 Tf
62 -29 TD
(+)Tj
/T8 1 Tf
93 -32 TD
(5)Tj
ET
412.68 579.58 4.08 0.48 re
f
BT
0.12 0 0 -0.12 412.68 573.94 Tm
(5)Tj
/T15 1 Tf
61 -29 TD
(+)Tj
/T8 1 Tf
93 -32 TD
(5)Tj
ET
431.16 579.58 4.08 0.48 re
f
BT
0.12 0 0 -0.12 431.16 573.94 Tm
(5)Tj
/T15 1 Tf
62 -29 TD
(+)Tj
/T8 1 Tf
93 -32 TD
(5)Tj
ET
449.76 579.58 4.08 0.48 re
f
BT
0.12 0 0 -0.12 449.76 573.94 Tm
(5)Tj
/T15 1 Tf
43 -29 TD
0 Tc
[(])-27000(=)-28000(100%)]TJ
ET
556.68 573.82 0.48 12.12 re
f
558.72 573.82 0.48 12.12 re
f
56.28 561.82 0.48 12 re
f
58.32 561.82 0.48 12 re
f
116.64 561.82 0.48 12 re
f
200.28 561.82 0.48 12 re
f
556.68 561.82 0.48 12 re
f
558.72 561.82 0.48 12 re
f
56.52 561.46 502.44 0.48 re
f
BT
/T9 1 Tf
0.12 0 0 -0.12 185.52 543.58 Tm
[(Figure)-31000(6:)]TJ
/T15 1 Tf
413 0 TD
[(Scores)-26000(of)-28000(Both)-27000(Algorithms)-27000(on)-28000(the)-28000(Examples)]TJ
-1234 251 TD
[(Comparing)-22000(these)-25000(n)]TJ
679 0 TD
(um)Tj
113 0 TD
[(b)-2000(ers)-24000(to)-24000(the)-24000(ones)-24000(obtained)-24000(b)]TJ
976 0 TD
[(y)-24000(using)-24000(the)-25000(v)]TJ
464 0 TD
[(ersion)-22000(of)-25000(the)-24000(algorithm)-24000(whic)]TJ
1007 0 TD
[(h)-24000(assigns)-23000(equal)]TJ
-3364 99 TD
(w)Tj
58 0 TD
(eigh)Tj
146 0 TD
[(ts)-20000(to)-21000(eac)]TJ
294 0 TD
19 Tc
[(he)19000(n)]TJ
146 0 TD
0 Tc
(tit)Tj
85 0 TD
(y)Tj
37 0 TD
[(,)-23000(one)-20000(can)-21000(see)-20000(that)-21000(the)-21000(curren)]TJ
1003 0 TD
20 Tc
(tv)Tj
94 0 TD
0 Tc
[(ersion)-19000(is)-21000(m)]TJ
377 0 TD
(uc)Tj
81 0 TD
[(h)-21000(less)-20000(sensitiv)]TJ
482 0 TD
[(e)-20000(t)0(o)-21000(precision)-20000(errors.)-32000(Although)]TJ
-2802.999 100 TD
[(the)-26000(curren)]TJ
371 0 TD
25 Tc
(tv)Tj
99 0 TD
0 Tc
[(ersion)-23000(of)-26000(the)-26000(algorithm)-24000(do)-2000(es)-25000(p)-2000(enalize)-26000(the)-26000(precision)-24000(n)]TJ
1879.999 0 TD
(um)Tj
113 0 TD
[(b)-2000(ers)-25000(for)-25000(the)-26000(error)-24000(in)-25000(Figure)-25000(4)-25000(more)-25000(than)]TJ
-2462.999 100 TD
[(the)-28000(error)-26000(made)-27000(in)-28000(Figure)-27000(2,)-27000(it)-28000(is)-28000(less)-27000(sev)]TJ
1451 0 TD
[(ere)-26000(than)-28000(the)-28000(earlier)-26000(v)]TJ
766 0 TD
(ersion.)Tj
/T13 1 Tf
-2217 218 TD
(The)Tj
229 0 TD
[(Mo)-3000(del)]TJ
347 0 TD
(Theoretic)Tj
506 0 TD
(Approac)Tj
412 0 TD
(h)Tj
100 0 TD
(T)Tj
69 0 TD
(o)Tj
93 0 TD
(The)Tj
229 0 TD
(B-CUBED)Tj
560 0 TD
(Algorithm)Tj
/T15 1 Tf
-2545 217 TD
[(Let)-27000(S)-28000(b)-2000(e)-28000(a)0(n)-27000(equiv)]TJ
639 0 TD
[(alence)-26000(set)-28000(generated)-26000(b)]TJ
800 0 TD
[(y)-27000(the)-27000(k)]TJ
255 0 TD
(ey)Tj
74 0 TD
[(,)-27000(and)-27000(let)]TJ
/T12 1 Tf
331 0 TD
(R)Tj
/T8 1 Tf
63 12 TD
(1)Tj
/T12 1 Tf
51 -12 TD
14 Tc
(:::R)Tj
/T7 1 Tf
174 12 TD
(m)Tj
/T15 1 Tf
90 -12 TD
0 Tc
[(b)-2000(e)-28000(equiv)]TJ
302 0 TD
[(alence)-27000(classes)-26000(generated)-26000(b)]TJ
935 0 TD
[(y)-27000(the)]TJ
-3714 100 TD
[(resp)-2000(onse.)]TJ
368 0 TD
[(Then)-28000(w)]TJ
275 0 TD
[(e)-27000(de\014ne)-28000(the)-28000(follo)]TJ
600 0 TD
[(wing)-26000(functions)-28000(o)]TJ
595 0 TD
(v)Tj
42 0 TD
[(er)-26000(S:)]TJ
/T14 1 Tf
-1755 205 TD
(\017)Tj
/T15 1 Tf
83 0 TD
(p\(S\))Tj
194 0 TD
(is)Tj
94 0 TD
(a)Tj
79 0 TD
(partition)Tj
357 0 TD
(of)Tj
104 0 TD
(S)Tj
84 0 TD
(with)Tj
200 0 TD
[(resp)-2000(ect)]TJ
294 0 TD
(to)Tj
112 0 TD
(the)Tj
153 0 TD
[(resp)-2000(onse,)]TJ
372 0 TD
(i.e.)Tj
173 0 TD
(p\(S\))Tj
195 0 TD
(is)Tj
94 0 TD
(a)Tj
79 0 TD
(set)Tj
140 0 TD
(of)Tj
105 0 TD
(subsets)Tj
297 0 TD
(of)Tj
105 0 TD
(S)Tj
84 0 TD
(formed)Tj
289 0 TD
(b)Tj
44 0 TD
(y)Tj
-3648 100 TD
(in)Tj
67 0 TD
[(tersecting)-27000(S)-29000(with)-29000(those)-29000(resp)-2000(onse)-27000(sets)]TJ
/T12 1 Tf
1363 0 TD
(R)Tj
/T7 1 Tf
63 12 TD
(i)Tj
/T15 1 Tf
56 -12 TD
[(that)-29000(o)]TJ
221 0 TD
(v)Tj
42 0 TD
[(erlap)-26000(S.)-29000(Let)-29000(p\(S\))-30000(=)]TJ
/T14 1 Tf
734 0 TD
(f)Tj
/T15 1 Tf
42 0 TD
(P)Tj
/T8 1 Tf
57 12 TD
(1)Tj
/T12 1 Tf
36 -12 TD
(;)Tj
/T15 1 Tf
37 0 TD
(P)Tj
/T8 1 Tf
57 12 TD
(2)Tj
/T12 1 Tf
37 -12 TD
14 Tc
[(;:::)1000(;)]TJ
/T15 1 Tf
184 0 TD
(P)Tj
/T8 1 Tf
57 12 TD
(m)Tj
/T14 1 Tf
59 -12 TD
(g)Tj
/T15 1 Tf
70 0 TD
0 Tc
[(where)-28000(eac)]TJ
355 0 TD
28 Tc
(hP)Tj
/T8 1 Tf
131 12 TD
(j)Tj
/T15 1 Tf
-3668 87 TD
0 Tc
[(is)-27000(a)-27000(subset)-28000(of)-27000(S.)]TJ
/T14 1 Tf
-83 159 TD
(\017)Tj
/T15 1 Tf
83 0 TD
(m)Tj
/T8 1 Tf
69 12 TD
(j)Tj
/T15 1 Tf
24 -12 TD
[(\(S\))-29000(is)-27000(the)-28000(n)]TJ
409 0 TD
(um)Tj
113 0 TD
[(b)-2000(e)0(r)-28000(o)0(f)-27000(elemen)]TJ
487 0 TD
[(ts)-28000(that)-28000(are)-26000(missing)-27000(from)-28000(eac)]TJ
1018 0 TD
26 Tc
(hP)Tj
/T8 1 Tf
129 12 TD
(j)Tj
/T15 1 Tf
52 -12 TD
0 Tc
(relativ)Tj
232 0 TD
[(e)-27000(t)0(o)-27000(the)-28000(k)]TJ
350 0 TD
[(ey)-27000(set)-28000(S.)-27000(Therefore,)]TJ
-1413 164 TD
(m)Tj
/T8 1 Tf
69 12 TD
(j)Tj
/T15 1 Tf
25 -12 TD
[(\(S\))-24000(=)-22000(\()]TJ
/T14 1 Tf
253 0 TD
(j)Tj
/T15 1 Tf
23 0 TD
(S)Tj
/T14 1 Tf
46 0 TD
19 Tc
[(j\000)1000(j)]TJ
/T15 1 Tf
148 0 TD
(P)Tj
/T8 1 Tf
57 12 TD
(j)Tj
/T14 1 Tf
25 -12 TD
(j)Tj
/T15 1 Tf
23 0 TD
(\))Tj
/T12 1 Tf
60 0 TD
(:)Tj
/T15 1 Tf
-2282 205 TD
0 Tc
(Since)Tj
224 0 TD
(the)Tj
151 0 TD
(B-CUBED)Tj
423 0 TD
(algorithm)Tj
387 0 TD
[(lo)-2000(oks)]TJ
220 0 TD
(at)Tj
110 0 TD
(the)Tj
151 0 TD
(presence/absence)Tj
659 0 TD
(of)Tj
103 0 TD
(en)Tj
81 0 TD
(tities)Tj
216 0 TD
(relativ)Tj
232 0 TD
(e)Tj
71 0 TD
(to)Tj
109 0 TD
(eac)Tj
114 0 TD
(h)Tj
81 0 TD
(of)Tj
103 0 TD
(the)Tj
151 0 TD
(other)Tj
-3711 100 TD
(en)Tj
81 0 TD
(tities,)Tj
236 0 TD
(the)Tj
148 0 TD
(n)Tj
44 0 TD
(um)Tj
113 0 TD
[(b)-2000(e)0(r)-31000(o)0(f)-32000(missing)-32000(en)]TJ
630 0 TD
[(tities)-32000(in)-32000(an)-32000(en)]TJ
514 0 TD
[(tire)-31000(equiv)]TJ
345 0 TD
[(alence)-32000(set)-32000(is)-32000(calculated)-31000(b)]TJ
916 0 TD
[(y)-32000(adding)-31000(the)-32000(n)]TJ
543 0 TD
(um)Tj
113 0 TD
2 Tc
[(be)2000(r)-30000(o)2000(f)]TJ
-3683 100 TD
0 Tc
[(missing)-23000(en)]TJ
373 0 TD
[(tities)-24000(with)-25000(resp)-2000(ect)-23000(to)-24000(eac)]TJ
881.999 0 TD
23 Tc
[(he)23000(n)]TJ
150 0 TD
0 Tc
(tit)Tj
85 0 TD
[(y)-24000(i)0(n)-24000(that)-24000(equiv)]TJ
526 0 TD
[(alence)-23000(set.)]TJ
406 0 TD
[(Therefore,)-23000(the)-24000(n)]TJ
579 0 TD
(um)Tj
113 0 TD
[(b)-2000(e)0(r)-24000(o)0(f)-24000(missing)-23000(en)]TJ
606.001 0 TD
(tities)Tj
-3720 99 TD
[(for)-27000(the)-28000(en)]TJ
351 0 TD
[(tire)-27000(set)-28000(S)-28000(i)0(s)]TJ
/T7 1 Tf
1185 42 TD
(m)Tj
/T5 1 Tf
-31 25 TD
(X)Tj
/T7 1 Tf
3 177 TD
(j)Tj
/T8 1 Tf
30 0 TD
(=1)Tj
/T5 1 Tf
298 -177 TD
(X)Tj
/T8 1 Tf
-169 178 TD
[(for)-27000(eac)]TJ
198 0 TD
27 Tc
(he)Tj
/T3 1 Tf
121 0 TD
(2)Tj
/T8 1 Tf
72 0 TD
(P)Tj
/T2 1 Tf
45 8 TD
(j)Tj
/T15 1 Tf
36 -107 TD
(m)Tj
/T8 1 Tf
69 12 TD
(j)Tj
/T15 1 Tf
25 -12 TD
0 Tc
(\(S\))Tj
/T12 1 Tf
139 0 TD
(:)Tj
/T15 1 Tf
-2247 282 TD
[(The)-27000(recall)]TJ
/T6 1 Tf
392 0 TD
(err)Tj
104 0 TD
(or)Tj
/T15 1 Tf
105 0 TD
[(is)-28000(simply)-28000(the)-28000(n)]TJ
537 0 TD
(um)Tj
113 0 TD
[(b)-2000(e)0(r)-27000(o)0(f)-28000(missing)-27000(en)]TJ
617 0 TD
[(tities)-28000(divided)-28000(b)]TJ
545 0 TD
[(y)-27000(the)-29000(n)]TJ
259 0 TD
(um)Tj
113 0 TD
2 Tc
[(be)2000(r)-25000(o)2000(f)-26000(e)2000(n)]TJ
321 0 TD
0 Tc
[(tities)-28000(in)-28000(the)-28000(equiv-)]TJ
-3231 100 TD
[(alence)-27000(set,)-28000(i.e.,)]TJ
1822 87 TD
(m)Tj
/T8 1 Tf
69 12 TD
(j)Tj
/T15 1 Tf
25 -12 TD
(\(S\))Tj
ET
290.64 187.42 24.6 0.48 re
f
BT
/T14 1 Tf
0.12 0 0 -0.12 297.48 178.3 Tm
(j)Tj
/T15 1 Tf
23 0 TD
(S)Tj
/T14 1 Tf
46 0 TD
(j)Tj
/T12 1 Tf
117 -57 TD
(:)Tj
/T15 1 Tf
-2065 182 TD
[(Since)-28000(the)-28000(algorithm)-26000(lo)-2000(oks)-27000(at)-27000(eac)]TJ
1165.999 0 TD
27 Tc
[(he)27000(n)]TJ
154 0 TD
0 Tc
(tit)Tj
85 0 TD
[(y)-28000(i)0(n)-28000(a)0(n)-27000(equiv)]TJ
473 0 TD
[(alence)-27000(set,)-28000(the)-28000(recall)]TJ
/T6 1 Tf
766 0 TD
(err)Tj
104 0 TD
(or)Tj
/T15 1 Tf
105 0 TD
[(for)-27000(that)-28000(en)]TJ
388 0 TD
[(tire)-27000(set)-28000(is)]TJ
-1786.999 171 TD
(1)Tj
ET
243.48 138.34 11.16 0.48 re
f
BT
/T14 1 Tf
0.12 0 0 -0.12 243.48 129.22 Tm
(j)Tj
/T15 1 Tf
23 0 TD
(S)Tj
/T14 1 Tf
46 0 TD
(j)Tj
/T7 1 Tf
105 -161 TD
(m)Tj
/T5 1 Tf
-31 25 TD
(X)Tj
/T7 1 Tf
3 177 TD
(j)Tj
/T8 1 Tf
30 0 TD
(=1)Tj
/T5 1 Tf
298 -177 TD
(X)Tj
/T8 1 Tf
-169 179 TD
[(for)-27000(eac)]TJ
198 0 TD
27 Tc
(he)Tj
/T3 1 Tf
121 0 TD
(2)Tj
/T8 1 Tf
72 0 TD
(P)Tj
/T2 1 Tf
44.999 8 TD
(j)Tj
/T15 1 Tf
46 -164 TD
(m)Tj
/T8 1 Tf
69 12 TD
(j)Tj
/T15 1 Tf
25 -12 TD
0 Tc
(\(S\))Tj
ET
337.92 138.34 24.6 0.48 re
f
BT
/T14 1 Tf
0.12 0 0 -0.12 344.64 129.22 Tm
(j)Tj
/T15 1 Tf
23 0 TD
(S)Tj
/T14 1 Tf
46 0 TD
(j)Tj
/T12 1 Tf
117 -57 TD
(:)Tj
/T15 1 Tf
-2333 308 TD
[(Recall)-27000(in)-28000(turn)-27000(is)]TJ
1222 146 TD
(1)Tj
/T14 1 Tf
60 0 TD
(\000)Tj
/T15 1 Tf
118 -56 TD
(1)Tj
ET
252 83.86 11.16 0.48 re
f
BT
/T14 1 Tf
0.12 0 0 -0.12 252 74.74 Tm
(j)Tj
/T15 1 Tf
23 0 TD
(S)Tj
/T14 1 Tf
46 0 TD
(j)Tj
/T7 1 Tf
105 -161 TD
(m)Tj
/T5 1 Tf
-30 25 TD
(X)Tj
/T7 1 Tf
2 177 TD
(j)Tj
/T8 1 Tf
30 0 TD
(=1)Tj
/T5 1 Tf
299 -177 TD
(X)Tj
/T8 1 Tf
-170 179 TD
[(for)-28000(eac)]TJ
199 0 TD
26 Tc
(he)Tj
/T3 1 Tf
120 0 TD
(2)Tj
/T8 1 Tf
73 0 TD
(P)Tj
/T2 1 Tf
45 8 TD
(j)Tj
/T15 1 Tf
46 -164 TD
(m)Tj
/T8 1 Tf
69 12 TD
(j)Tj
/T15 1 Tf
24 -12 TD
0 Tc
(\(S\))Tj
ET
346.56 83.86 24.6 0.48 re
f
BT
/T14 1 Tf
0.12 0 0 -0.12 353.28 74.74 Tm
(j)Tj
/T15 1 Tf
23 0 TD
(S)Tj
/T14 1 Tf
46 0 TD
(j)Tj
/T12 1 Tf
117 -57 TD
(;)Tj
ET
endstream
endobj
40 0 obj
<<
/ProcSet [/PDF /Text /ImageB ]
/Font <<
/T2 41 0 R
/T3 42 0 R
/T5 26 0 R
/T6 27 0 R
/T7 20 0 R
/T8 21 0 R
/T9 22 0 R
/T12 6 0 R
/T13 7 0 R
/T14 8 0 R
/T15 9 0 R
>>
/ExtGState <<
/GS1 12 0 R
>>
/ColorSpace <<
/CS1 13 0 R
>>
>>
endobj
44 0 obj
<<
/Length 12003
>>
stream
BT
/T15 1 Tf
0.12 0 0 -0.12 72 708.7 Tm
/CS1 cs 0 sc
BX /GS1 gs EX
0 Tc
(whic)Tj
164 0 TD
[(h)-28000(equals)]TJ
1157 165 TD
(1)Tj
/T14 1 Tf
60 0 TD
(\000)Tj
/T5 1 Tf
93 -136 TD
(P)Tj
/T7 1 Tf
87 21 TD
(m)Tj
0 66 TD
(j)Tj
/T8 1 Tf
30 0 TD
(=1)Tj
/T5 1 Tf
131 -87 TD
(P)Tj
/T8 1 Tf
88 87 TD
[(for)-27000(eac)]TJ
198 0 TD
27 Tc
(he)Tj
/T3 1 Tf
121 0 TD
(2)Tj
/T8 1 Tf
72 0 TD
(P)Tj
/T2 1 Tf
45 8 TD
(j)Tj
/T15 1 Tf
68 -33 TD
(m)Tj
/T8 1 Tf
69 12 TD
(j)Tj
/T15 1 Tf
25 -12 TD
0 Tc
(\(S\))Tj
ET
1 i
248.88 691.18 125.4 0.48 re
f
BT
/T14 1 Tf
0.12 0 0 -0.12 303.72 682.06 Tm
(j)Tj
/T15 1 Tf
23 0 TD
(S)Tj
/T14 1 Tf
46 0 TD
(j)Tj
/T8 1 Tf
23 -24 TD
(2)Tj
/T12 1 Tf
533 -33 TD
(:)Tj
/T15 1 Tf
-2431 245 TD
[(The)-27000(whole)-27000(expression)-27000(can)-27000(no)]TJ
1044 0 TD
[(w)-27000(b)-2000(e)-28000(simpli\014ed)-28000(to)]TJ
94 244 TD
(1)Tj
/T14 1 Tf
60 0 TD
(\000)Tj
/T5 1 Tf
93 -136 TD
(P)Tj
/T7 1 Tf
87 21 TD
(m)Tj
0 66 TD
(j)Tj
/T8 1 Tf
30 0 TD
(=1)Tj
/T5 1 Tf
131 -87 TD
(P)Tj
/T8 1 Tf
87 87 TD
[(for)-28000(eac)]TJ
199 0 TD
26 Tc
(he)Tj
/T3 1 Tf
121 0 TD
(2)Tj
/T8 1 Tf
72 0 TD
(P)Tj
/T2 1 Tf
45 8 TD
(j)Tj
/T14 1 Tf
68 -33 TD
(j)Tj
/T15 1 Tf
23 0 TD
(S)Tj
/T14 1 Tf
46 0 TD
18 Tc
(j\000j)Tj
/T15 1 Tf
147 0 TD
(P)Tj
/T8 1 Tf
57 12 TD
(j)Tj
/T14 1 Tf
25 -12 TD
(j)Tj
ET
241.92 632.5 139.44 0.48 re
f
BT
0.12 0 0 -0.12 303.72 623.38 Tm
(j)Tj
/T15 1 Tf
23 0 TD
(S)Tj
/T14 1 Tf
46 0 TD
(j)Tj
/T8 1 Tf
23 -24 TD
(2)Tj
/T12 1 Tf
591 -33 TD
(:)Tj
/T15 1 Tf
-2489 281 TD
0 Tc
(Moreo)Tj
228 0 TD
(v)Tj
42 0 TD
[(er,)-24000(the)-27000(measure)-26000(can)-26000(b)-2000(e)-27000(extended)-27000(from)-26000(a)-27000(single)-26000(k)]TJ
1733 0 TD
[(ey)-26000(equiv)]TJ
296 0 TD
[(alence)-26000(class)-26000(to)-27000(a)-26000(set)-27000(T)-27000(=)]TJ
/T14 1 Tf
918 0 TD
(f)Tj
/T15 1 Tf
42 0 TD
(S)Tj
/T8 1 Tf
46 12 TD
(1)Tj
/T12 1 Tf
37 -12 TD
(;)Tj
/T15 1 Tf
37 0 TD
(S)Tj
/T8 1 Tf
46 12 TD
(2)Tj
/T12 1 Tf
37 -12 TD
14 Tc
(;:::;)Tj
/T15 1 Tf
185 0 TD
(S)Tj
/T8 1 Tf
46 12 TD
(n)Tj
/T14 1 Tf
40 -12 TD
(g)Tj
/T15 1 Tf
-3858 100 TD
0 Tc
[(of)-28000(equiv)]TJ
284 0 TD
[(alence)-27000(classes.)]TJ
545 0 TD
[(Therefore,)-27000(the)-28000(recall)-26000(R)]TJ
/T8 1 Tf
825 12 TD
(i)Tj
/T15 1 Tf
51 -12 TD
[(for)-27000(an)-27000(equiv)]TJ
431 0 TD
[(alence)-28000(class)-26000(S)]TJ
/T8 1 Tf
490 12 TD
(i)Tj
/T15 1 Tf
51 -12 TD
(equals)Tj
-1536 244 TD
(R)Tj
/T8 1 Tf
61 12 TD
(i)Tj
/T15 1 Tf
46 -12 TD
23 Tc
(=1)Tj
/T14 1 Tf
148 0 TD
(\000)Tj
/T5 1 Tf
93 -136 TD
(P)Tj
/T8 1 Tf
87 20 TD
(m)Tj
0 67 TD
0 Tc
(j=1)Tj
/T5 1 Tf
151 -87 TD
(P)Tj
/T8 1 Tf
87 87 TD
[(for)-28000(eac)]TJ
199 0 TD
26 Tc
(he)Tj
/T3 1 Tf
121 0 TD
(2)Tj
/T8 1 Tf
72 0 TD
(P)Tj
/T2 1 Tf
45 8 TD
0 Tc
(ij)Tj
/T14 1 Tf
84 -33 TD
(j)Tj
/T15 1 Tf
23 0 TD
(S)Tj
/T8 1 Tf
46 12 TD
(i)Tj
/T14 1 Tf
23 -12 TD
19 Tc
[(j\000)1000(j)]TJ
/T15 1 Tf
148 0 TD
(P)Tj
/T8 1 Tf
57 12 TD
0 Tc
(ij)Tj
/T14 1 Tf
43 -12 TD
(j)Tj
ET
250.68 557.5 145.2 0.48 re
f
BT
0.12 0 0 -0.12 314.04 548.38 Tm
(j)Tj
/T15 1 Tf
23 0 TD
(S)Tj
/T8 1 Tf
46 12 TD
(i)Tj
/T14 1 Tf
23 -12 TD
(j)Tj
/T8 1 Tf
23 -24 TD
(2)Tj
/T12 1 Tf
604 -33 TD
(;)Tj
/T15 1 Tf
-2736 227 TD
[(where)-27000(P)]TJ
/T8 1 Tf
297 12 TD
(ij)Tj
/T15 1 Tf
71 -12 TD
[(is)-27000(the)]TJ
/T12 1 Tf
226 0 TD
(j)Tj
/T15 1 Tf
39 0 TD
[(th)-28000(elemen)]TJ
353 0 TD
[(t)-28000(o)0(f)-28000(the)-28000(partition)-27000(p\(S)]TJ
/T8 1 Tf
768 12 TD
(i)Tj
/T15 1 Tf
23 -12 TD
[(\),)-28000(and,)-28000(hence,)-27000(is)-28000(a)-27000(subset)-28000(of)-27000(S)]TJ
/T8 1 Tf
1069 12 TD
(i)Tj
/T15 1 Tf
23 -12 TD
(.)Tj
-2744 154 TD
[(The)-29000(recall)-29000(n)]TJ
440 0 TD
(um)Tj
113 0 TD
[(b)-2000(ers)-30000(calculated)-29000(for)-29000(eac)]TJ
818 0 TD
[(h)-29000(class)-29000(can)-30000(no)]TJ
513 0 TD
[(w)-29000(b)-2000(e)-30000(com)]TJ
350 0 TD
[(bined)-30000(in)-30000(v)]TJ
366 0 TD
[(arious)-28000(w)]TJ
305 0 TD
(a)Tj
40 0 TD
[(ys)-29000(to)-30000(pro)-2000(duce)-29000(the)-30000(\014nal)]TJ
-3070 99 TD
(recall.)Tj
267 0 TD
(Di\013eren)Tj
285 0 TD
(t)Tj
65 0 TD
(v)Tj
42 0 TD
[(ersions)-30000(of)-32000(the)]TJ
524 0 TD
[(algorithm)-30000(are)-32000(obtained)-31000(b)]TJ
915 0 TD
[(y)-32000(using)-32000(di\013eren)]TJ
566 0 TD
[(t)-32000(com)]TJ
210 0 TD
[(bination)-31000(strategies.)]TJ
747 0 TD
(If)Tj
88 0 TD
(equal)Tj
-3709 100 TD
(w)Tj
58 0 TD
(eigh)Tj
146 0 TD
[(ts)-27000(are)-27000(assigned)-27000(to)-28000(eac)]TJ
776 0 TD
[(h)-27000(class,)-27000(the)-29000(v)]TJ
477 0 TD
[(ersion)-26000(of)-28000(the)-29000(algorithm)-27000(pro)-2000(duced)-28000(is)-27000(exactly)-28000(the)-28000(same)-28000(as)-27000(the)-29000(v)]TJ
2231 0 TD
(ersion)Tj
-3688 99 TD
[(of)-27000(the)-28000(informal)-26000(algorithm)-26000(whic)]TJ
1108 0 TD
[(h)-27000(assigns)-26000(equal)-26000(w)]TJ
627 0 TD
(eigh)Tj
146 0 TD
[(ts)-27000(to)-27000(eac)]TJ
307 0 TD
[(h)-26000(class,)-26000(as)-27000(describ)-2000(ed)-27000(in)-27000(the)-28000(previous)-26000(section.)]TJ
1636 0 TD
(In)Tj
-3824 100 TD
[(other)-27000(w)]TJ
275 0 TD
[(ords,)-26000(the)-28000(\014nal)-28000(recall)-26000(is)-28000(an)-27000(a)]TJ
990.999 0 TD
(v)Tj
42 0 TD
[(erage)-25000(of)-27000(the)-28000(recall)-27000(n)]TJ
719 0 TD
(um)Tj
113 0 TD
[(b)-2000(ers)-27000(for)-27000(eac)]TJ
419 0 TD
[(h)-27000(equiv)]TJ
262 0 TD
[(alence)-27000(class,)-27000(i.e.,)]TJ
-1185 241 TD
(R)Tj
/T8 1 Tf
61 12 TD
(T)Tj
/T15 1 Tf
75 -12 TD
(=)Tj
100 -56 TD
(1)Tj
ET
296.4 435.1 5.64 0.48 re
f
BT
0.12 0 0 -0.12 296.4 425.98 Tm
(n)Tj
/T8 1 Tf
139 -161 TD
(n)Tj
/T5 1 Tf
-42 25 TD
(X)Tj
/T8 1 Tf
9 177 TD
(i=1)Tj
/T15 1 Tf
153 -98 TD
(R)Tj
/T8 1 Tf
61 12 TD
(i)Tj
/T12 1 Tf
51 -12 TD
(:)Tj
/T15 1 Tf
-2241 252 TD
(T)Tj
53 0 TD
[(o)-30000(obtain)-31000(the)-31000(v)]TJ
521.999 0 TD
[(ersion)-29000(of)-31000(the)-31000(informal)-30000(algorithm)-30000(whic)]TJ
1366 0 TD
[(h)-31000(assigns)-29000(equal)-30000(w)]TJ
638 0 TD
(eigh)Tj
146 0 TD
[(ts)-30000(to)-31000(eac)]TJ
314 0 TD
30 Tc
[(he)30000(n)]TJ
157 0 TD
0 Tc
(tit)Tj
85 0 TD
(y)Tj
37 0 TD
[(,)-31000(the)-31000(\014nal)-31000(recall)]TJ
-3317.999 100 TD
[(is)-28000(computed)-29000(b)]TJ
512 0 TD
[(y)-29000(calculating)-27000(the)-29000(w)]TJ
694.999 0 TD
(eigh)Tj
146 0 TD
[(ted)-28000(a)]TJ
183 0 TD
(v)Tj
42 0 TD
[(erage)-26000(of)-28000(the)-29000(recall)-28000(n)]TJ
723 0 TD
(um)Tj
113 0 TD
[(b)-2000(ers)-28000(for)-28000(eac)]TJ
421 0 TD
[(h)-28000(equiv)]TJ
263 0 TD
[(alence)-28000(class)-28000(where)-28000(the)]TJ
-3097.999 99 TD
(w)Tj
58 0 TD
(eigh)Tj
146 0 TD
[(ts)-27000(are)-26000(decided)-28000(b)]TJ
574 0 TD
[(y)-27000(the)-28000(n)]TJ
258 0 TD
(um)Tj
113 0 TD
2 Tc
[(be)2000(r)-26000(o)2000(f)-25000(e)2000(n)]TJ
321 0 TD
0 Tc
[(tities)-28000(in)-28000(eac)]TJ
419 0 TD
[(h)-27000(class,)-26000(i.e.,)]TJ
-400 241 TD
(R)Tj
/T8 1 Tf
61 12 TD
(T)Tj
/T15 1 Tf
75 -12 TD
(=)Tj
/T8 1 Tf
129 -104 TD
(n)Tj
/T5 1 Tf
-42 25 TD
(X)Tj
/T8 1 Tf
9 178 TD
(i=1)Tj
/T14 1 Tf
283 -155 TD
(j)Tj
/T15 1 Tf
23 0 TD
(S)Tj
/T8 1 Tf
46 12 TD
(i)Tj
/T14 1 Tf
23 -12 TD
(j)Tj
ET
298.08 352.06 42.6 0.48 re
f
BT
/T5 1 Tf
0.12 0 0 -0.12 298.08 350.38 Tm
(P)Tj
/T8 1 Tf
87 21 TD
(n)Tj
0 66 TD
(j=1)Tj
/T14 1 Tf
151 -24 TD
(j)Tj
/T15 1 Tf
23 0 TD
(S)Tj
/T8 1 Tf
46 12 TD
(j)Tj
/T14 1 Tf
25 -12 TD
(j)Tj
/T15 1 Tf
60 -58 TD
(R)Tj
/T8 1 Tf
61 12 TD
(i)Tj
/T12 1 Tf
51 -12 TD
(:)Tj
/T15 1 Tf
-2263 306 TD
(Finally)Tj
248 0 TD
[(,)-31000(a)0(s)-30000(i)0(n)-31000(the)-31000(case)-29000(of)-31000(the)-31000(Vilain)-30000(et.)]TJ
1214 0 TD
[(al)-31000(algorithm,)-30000(the)-31000(precision)-29000(n)]TJ
1040 0 TD
(um)Tj
113 0 TD
[(b)-2000(ers)-30000(are)-30000(calculated)-30000(b)]TJ
762.001 0 TD
[(y)-30000(rev)]TJ
186 0 TD
(ersing)Tj
-3688 99 TD
[(the)-28000(roles)-26000(of)-28000(the)-28000(k)]TJ
617 0 TD
[(ey)-27000(and)-27000(the)-28000(resp)-2000(onse)-27000(in)-28000(the)-28000(ab)-2000(o)]TJ
1118 0 TD
(v)Tj
42 0 TD
[(e)-26000(form)]TJ
230 0 TD
(ulation.)Tj
/T13 1 Tf
-2007 233 TD
(T)Tj
69 0 TD
(ask)Tj
195 0 TD
(Relativ)Tj
353 0 TD
(e)Tj
86 0 TD
(Strengths)Tj
509 0 TD
(and)Tj
216 0 TD
(W)Tj
107 0 TD
(eaknesses)Tj
499 0 TD
(of)Tj
128 0 TD
(the)Tj
194 0 TD
(Tw)Tj
156 0 TD
(o)Tj
93 0 TD
(Algorithms)Tj
/T15 1 Tf
-2605 232 TD
(The)Tj
176 0 TD
(Vilain)Tj
252 0 TD
(et.)Tj
145 0 TD
(al)Tj
98 0 TD
[(algorithm)-32000(is)]TJ
473 0 TD
(useful)Tj
243 0 TD
(for)Tj
133 0 TD
[(applications/tasks)-30000(that)]TJ
877 0 TD
[(use)-32000(single)]TJ
385 0 TD
[(coreference)-31000(relations)-32000(at)]TJ
882 0 TD
(a)Tj
75 0 TD
(time)Tj
-3738.999 100 TD
[(rather)-30000(than)-30000(resulting)-30000(equiv)]TJ
983 0 TD
[(alence)-31000(classes.)]TJ
558 0 TD
(F)Tj
47 0 TD
[(or)-31000(our)-29000(dev)]TJ
381 0 TD
(elopmen)Tj
298 0 TD
[(t)-30000(i)0(n)-31000(the)-31000(coreference)-30000(task,)-31000(the)-31000(t)]TJ
1120.001 0 TD
(w)Tj
58 0 TD
[(o)-29000(algorithms)]TJ
-3445 99 TD
(pro)Tj
119 0 TD
(vide)Tj
183 0 TD
(distinct)Tj
307 0 TD
[(p)-2000(ersp)-2000(ectiv)]TJ
370 0 TD
(es)Tj
104 0 TD
(on)Tj
122 0 TD
(system)Tj
283 0 TD
[(p)-2000(erformance.)]TJ
528 0 TD
(Vilain)Tj
254 0 TD
(et.)Tj
149 0 TD
(al)Tj
99 0 TD
(pro)Tj
119 0 TD
(vide)Tj
184 0 TD
(a)Tj
76 0 TD
(strong)Tj
261 0 TD
(diagnostic)Tj
399 0 TD
(for)Tj
134 0 TD
(errors)Tj
-3691 100 TD
[(that)-27000(re\015ect)-26000(pairwise)-25000(decisions)-26000(done)-26000(b)]TJ
1336 0 TD
[(y)-26000(the)-27000(system.)]TJ
519 0 TD
[(Our)-26000(visual)-26000(displa)]TJ
618 0 TD
[(y)-26000(tec)]TJ
174 0 TD
[(hniques)-26000(emphasize)-26000(just)-27000(this)-27000(sort)-25000(of)]TJ
-2647 100 TD
[(pro)-2000(cessing.)]TJ
125 153 TD
(Our)Tj
179 0 TD
(total)Tj
207 0 TD
(score)Tj
217 0 TD
(under)Tj
244 0 TD
(the)Tj
152 0 TD
(Vilain)Tj
255 0 TD
(algorithm,)Tj
413 0 TD
(with)Tj
197 0 TD
(a)Tj
78 0 TD
(somewhat)Tj
397 0 TD
(fuzzier)Tj
274 0 TD
(exten)Tj
194 0 TD
(t)Tj
68 0 TD
(requiremen)Tj
403 0 TD
(t)Tj
68 0 TD
(and)Tj
170 0 TD
(stricter)Tj
-3641 100 TD
(requiremen)Tj
403 0 TD
[(t)-27000(for)-27000(links)-27000(is)-28000(81%)-27000(precision)-26000(and)-28000(45%)-27000(recall.)]TJ
-278 153 TD
[(The)-28000(same)-27000(\014les)-28000(using)-27000(the)-29000(B3)-27000(algorithm)-27000(resulted)-27000(in)-29000(78%)-27000(precision)-27000(and)-28000(31%)-27000(recall.)]TJ
2950 0 TD
[(The)-28000(precision)-27000(n)]TJ
562 0 TD
(um)Tj
113 0 TD
[(b)-2000(ers)]TJ
-3750 99 TD
[(are)-30000(comparable)-29000(whic)]TJ
752 0 TD
[(h)-31000(indicates)-30000(that)-31000(our)-30000(goal)-30000(of)-31000(high)-30000(precision)-30000(is)-31000(supp)-2000(orted)-30000(under)-31000(b)-2000(oth)-31000(views)-31000(of)-30000(the)-32000(data.)]TJ
-752 100 TD
[(The)-32000(14%)-31000(drop)-31000(in)-32000(recall)-30000(w)]TJ
941 0 TD
[(as)-31000(ho)]TJ
192 0 TD
(w)Tj
58 0 TD
(ev)Tj
79 0 TD
[(er)-30000(unexp)-2000(ected.)]TJ
583 0 TD
[(The)-31000(reason)-30000(is)-32000(fairly)-31000(straigh)]TJ
995 0 TD
[(t)-31000(forw)]TJ
221 0 TD
[(ard{)-29000(our)-31000(system)-31000(is)-32000(not)]TJ
-3069 100 TD
(doing)Tj
235 0 TD
(a)Tj
78 0 TD
2 Tc
[(g)2000(ood)]TJ
212 0 TD
0 Tc
(job)Tj
150 0 TD
(of)Tj
103 0 TD
(relating)Tj
314 0 TD
(large)Tj
212 0 TD
(equiv)Tj
189 0 TD
(alence)Tj
258 0 TD
(classes.)Tj
324 0 TD
(This)Tj
198 0 TD
(is)Tj
93 0 TD
(the)Tj
152 0 TD
(con)Tj
123 0 TD
(v)Tj
42 0 TD
(erse)Tj
174 0 TD
(of)Tj
104 0 TD
[(p)-2000(enalizing)]TJ
403 0 TD
(the)Tj
152 0 TD
(system)Tj
285 0 TD
(for)Tj
-3801 99 TD
[(p)-2000(ositing)-27000(incorrect)-27000(links)-27000(that)-28000(result)-27000(in)-28000(larger)-26000(equiv)]TJ
1792 0 TD
(alen)Tj
146 0 TD
[(t)-27000(classes)-27000(than)-27000(smaller)-27000(ones.)]TJ
-1812.999 154 TD
[(The)-26000(drop)-26000(in)-28000(recall)-25000(in)-27000(the)-28000(B3)-26000(scorer)-25000(also)-25000(suggests)-26000(a)-26000(distinct)-28000(class)-25000(of)-27000(coreference)-25000(resolution)-26000(pro)-2000(cedure)-26000(that)]TJ
-125 99 TD
(w)Tj
58 0 TD
[(e)-31000(could)-31000(in)]TJ
360 0 TD
(v)Tj
42 0 TD
(estigate{gro)Tj
435 0 TD
[(wing)-28000(of)-32000(large)-30000(equiv)]TJ
694 0 TD
[(alence)-31000(classes)-30000(via)-31000(an)-31000(en)]TJ
861 0 TD
(tit)Tj
85 0 TD
[(y)-32000(merging)-30000(mo)-2000(del)-32000(whic)]TJ
813 0 TD
[(h)-31000(esc)]TJ
182 0 TD
(hew)Tj
141 0 TD
[(ed)-31000(the)]TJ
ET
endstream
endobj
45 0 obj
<<
/ProcSet [/PDF /Text /ImageB ]
/Font <<
/T2 41 0 R
/T3 42 0 R
/T5 26 0 R
/T7 20 0 R
/T8 21 0 R
/T12 6 0 R
/T13 7 0 R
/T14 8 0 R
/T15 9 0 R
>>
/ExtGState <<
/GS1 12 0 R
>>
/ColorSpace <<
/CS1 13 0 R
>>
>>
endobj
47 0 obj
<<
/Length 10911
>>
stream
BT
/T15 1 Tf
0.12 0 0 -0.12 72 708.7 Tm
/CS1 cs 0 sc
BX /GS1 gs EX
0 Tc
[(standard)-32000(left-to-righ)]TJ
741 0 TD
(t)Tj
65 0 TD
[(pro)-2000(cessing)-32000(strategy)-32000(of)]TJ
833 0 TD
(most)Tj
209 0 TD
[(coreference)-32000(resolution)-32000(systems.)]TJ
1180 0 TD
(If)Tj
88 0 TD
(suc)Tj
114 0 TD
(h)Tj
79 0 TD
(a)Tj
75 0 TD
[(pro)-2000(cedure)-32000(can)]TJ
-3384 100 TD
[(reliably)-24000(gro)]TJ
410 0 TD
[(w)-23000(medium)-26000(sized)-24000(equiv)]TJ
788 0 TD
[(alence)-25000(classes)-23000(in)]TJ
575 0 TD
[(to)-25000(large)-23000(ones,)-25000(then)-25000(the)-26000(recall)-24000(\014gures)-23000(will)-26000(impro)]TJ
1677 0 TD
(v)Tj
42 0 TD
[(e)-23000(under)-25000(the)]TJ
-3492 99 TD
[(B3)-28000(scorer.)]TJ
407 0 TD
[(The)-29000(Vilain)-29000(et.)]TJ
554 0 TD
[(al)-28000(scorer)-28000(notes)-28000(no)-29000(di\013erence)-29000(b)-2000(e)0(t)]TJ
1159 0 TD
(w)Tj
58 0 TD
[(een)-29000(correctly)-28000(relating)-28000(t)]TJ
831 0 TD
(w)Tj
58 0 TD
[(o)-28000(singleton)-29000(equiv)]TJ
612 0 TD
(alence)Tj
-3678.999 100 TD
[(classes)-26000(and)-28000(correctly)-26000(relating)-26000(t)]TJ
1104 0 TD
(w)Tj
58 0 TD
[(o)-27000(large)-26000(equiv)]TJ
461 0 TD
[(alence)-28000(classes.)]TJ
-1497.999 153 TD
[(Since)-25000(large)-23000(equiv)]TJ
603 0 TD
[(alence)-24000(classes)-24000(tend)-25000(to)-25000(include)-25000(topically)-24000(signi\014can)]TJ
1748 0 TD
24 Tc
[(te)24000(n)]TJ
137 0 TD
0 Tc
[(tities)-25000(for)-24000(do)-2000(cumen)]TJ
652 0 TD
[(ts,)-26000(correctly)-23000(iden-)]TJ
-3265 100 TD
[(tifying)-28000(them)-28000(is)-28000(p)-2000(erhaps)-27000(crucial)-26000(to)-28000(applications)-26000(lik)]TJ
1788.999 0 TD
[(e)-28000(summarization)-26000(and)-28000(information)-26000(extraction.)]TJ
/T13 1 Tf
-1788.999 235 TD
(Dev)Tj
193 0 TD
(eloping)Tj
387 0 TD
(with)Tj
256 0 TD
(the)Tj
194 0 TD
(Vilain)Tj
332 0 TD
(et)Tj
132 0 TD
(al)Tj
123 0 TD
(algorithm)Tj
/T15 1 Tf
-1492 235 TD
[(The)-26000(b)-2000(elo)]TJ
317 0 TD
[(w)-26000(analysis)-25000(re\015ects)-26000(ho)]TJ
764 0 TD
26 Tc
(ww)Tj
144 0 TD
0 Tc
[(e)-26000(assesed)-25000(the)-27000(individual)-27000(con)]TJ
1003 0 TD
[(tributions)-26000(of)-26000(the)-27000(comp)-2000(onen)]TJ
982 0 TD
[(ts)-26000(during)-26000(dev)]TJ
478 0 TD
(el-)Tj
-3813 99 TD
(opmen)Tj
238 0 TD
(t.)Tj
92 0 TD
[(Since)-28000(the)-28000(B3)-27000(algorithm)-26000(w)]TJ
924 0 TD
[(as)-27000(not)-27000(y)]TJ
291 0 TD
[(et)-28000(implemen)]TJ
445 0 TD
[(ted,)-28000(w)]TJ
224 0 TD
[(e)-27000(did)-28000(not)-28000(use)-27000(it)-28000(for)-27000(dev)]TJ
833 0 TD
(elopmen)Tj
298 0 TD
(t.)Tj
-3219.999 154 TD
[(Our)-28000(explicit)-29000(goal)-28000(w)]TJ
700.999 0 TD
[(as)-28000(to)-29000(maximize)-29000(recall)-28000(at)-29000(a)-29000(precision)-28000(lev)]TJ
1425.999 0 TD
[(el)-29000(of)-29000(80%.)]TJ
402 0 TD
(W)Tj
78 0 TD
[(e)-29000(feel)-30000(that)-29000(this)-29000(lev)]TJ
664 0 TD
[(el)-29000(of)-29000(precision)]TJ
-3395.999 99 TD
(pro)Tj
119 0 TD
[(vides)-25000(enough)-25000(accuracy)-24000(to)-26000(driv)]TJ
1078 0 TD
[(e)-25000(a)-26000(range)-24000(of)-26000(coreference)-25000(dep)-2000(enden)]TJ
1214 0 TD
[(t)-27000(applications{most)-24000(imp)-2000(ortan)]TJ
1069 0 TD
[(t)-26000(for)-25000(us)-26000(w)]TJ
346 0 TD
(as)Tj
-3826 100 TD
(query)Tj
238 0 TD
(sensitiv)Tj
269 0 TD
(e)Tj
72 0 TD
(text)Tj
181 0 TD
(summarization.)Tj
618 0 TD
(Our)Tj
178 0 TD
(o)Tj
40 0 TD
(v)Tj
42 0 TD
(erall)Tj
191 0 TD
(approac)Tj
286 0 TD
(h)Tj
80 0 TD
(w)Tj
58 0 TD
(as)Tj
109 0 TD
(to)Tj
109 0 TD
(break)Tj
236 0 TD
(do)Tj
86 0 TD
(wn)Tj
141 0 TD
(coreference)Tj
434 0 TD
(resolution)Tj
391 0 TD
(in)Tj
67 0 TD
(to)Tj
-3826 99 TD
(concrete)Tj
335 0 TD
(subprograms)Tj
498 0 TD
(that)Tj
188 0 TD
(resolv)Tj
210 0 TD
(ed)Tj
117 0 TD
(a)Tj
77 0 TD
(limited)Tj
289 0 TD
(class)Tj
202 0 TD
(of)Tj
102 0 TD
(coreference)Tj
435 0 TD
(w)Tj
58 0 TD
(ell.)Tj
166 0 TD
(Eac)Tj
134 0 TD
(h)Tj
80 0 TD
[(comp)-2000(onen)]TJ
365 0 TD
(t)Tj
67 0 TD
(could)Tj
230 0 TD
2 Tc
(be)Tj
120 0 TD
0 Tc
(scored)Tj
-3673 100 TD
[(separately)-25000(b)]TJ
438 0 TD
[(y)-26000(either)-26000(running)-26000(it)-27000(in)-27000(isolation,)-25000(or)-26000(b)]TJ
1288.999 0 TD
[(y)-26000(blo)-2000(c)]TJ
218 0 TD
[(king)-25000(coreference)-25000(from)-26000(subsequen)]TJ
1167 0 TD
[(t)-26000(pro)-2000(cesses.)]TJ
449 0 TD
(Belo)Tj
159 0 TD
25 Tc
(ww)Tj
143 0 TD
(e)Tj
-3863 100 TD
0 Tc
[(discuss)-27000(eac)]TJ
392 0 TD
[(h)-27000(comp)-2000(onen)]TJ
438 0 TD
[(t)-27000(i)0(n)-28000(the)-28000(order)-26000(of)-28000(execution.)]TJ
/T13 1 Tf
-830 235 TD
(Genre)Tj
336 0 TD
[(Sp)-3000(eci\014c)]TJ
409 0 TD
(Coreference)Tj
/T15 1 Tf
-620 234 TD
[(A)-25000(problematic)-25000(asp)-2000(ect)-25000(of)-26000(an)]TJ
975 0 TD
[(y)-24000(new)-26000(genre)-24000(of)-26000(data)-25000(is)-25000(the)-26000(existence)-25000(of)-26000(idiosyncratic)-24000(classes)-24000(of)-25000(coreference)-24000(and)]TJ
-1100 100 TD
[(the)-31000(MUC-7)-31000(data)-31000(w)]TJ
695.999 0 TD
[(as)-30000(particularly)-29000(troubling)-31000(since)-31000(v)]TJ
1171 0 TD
[(ery)-30000(o)-2000(ddly)-31000(formatted)-31000(text)-31000(w)]TJ
1001 0 TD
[(as)-30000(fair)-31000(game)-30000(for)-31000(coreference.)]TJ
-2867.999 100 TD
(F)Tj
47 0 TD
[(or)-27000(example,)-27000(the)-28000(strings)-27000(`HUGHES')-28000(and)-28000(`F)]TJ
1527 0 TD
[(CC')-27000(in)-28000(`)]TJ
/T1 1 Tf
290 0 TD
()Tj
436 0 TD
(BC-HUGHES-FCC-B)Tj
659 0 TD
(LO)Tj
87 0 TD
(OM)Tj
125 0 TD
()Tj
/T15 1 Tf
308 0 TD
[(')-25000(are)]TJ
-3740.999 99 TD
(coreferen)Tj
325 0 TD
[(t)-27000(with)-30000(the)-29000(same)-28000(strings)-28000(in)-29000(`)]TJ
/T1 1 Tf
994 0 TD
(BC-HUG)Tj
703 0 TD
(HE)Tj
87 0 TD
(S-)Tj
87 0 TD
(FCC)Tj
131 0 TD
(-B)Tj
87 0 TD
(LOO)Tj
131 0 TD
(M.)Tj
87 0 TD
(..)Tj
/T15 1 Tf
87 0 TD
(')Tj
58 0 TD
(whic)Tj
164 0 TD
28 Tc
(hw)Tj
132 0 TD
0 Tc
[(as)-28000(outside)-29000(the)-29000(scop)-2000(e)-28000(o)0(f)]TJ
-3073 100 TD
[(our)-24000(linguistic)-25000(to)-2000(ols.)]TJ
731 0 TD
[(Simple)-26000(programs)-22000(w)]TJ
690 0 TD
[(ere)-24000(written)-26000(to)-25000(recognize)-23000(this)-25000(sort)-25000(of)-25000(coreference.)]TJ
1750 0 TD
[(The)-25000(p)-2000(erformance)-24000(b)]TJ
685 0 TD
(y)Tj
-3856 100 TD
[(the)-28000(Vilain)-28000(scorer)-26000(is)-27000(4.2%)-27000(recall)-27000(67.5%)-26000(precision.)]TJ
125 153 TD
[(This)-24000(p)-2000(erformance)-24000(is)-25000(w)]TJ
798 0 TD
[(ell)-25000(b)-2000(elo)]TJ
256 0 TD
[(w)-24000(what)-25000(w)]TJ
347 0 TD
[(e)-24000(observ)]TJ
294 0 TD
[(ed)-24000(in)-25000(training)-24000(data{the)-24000(precision)-24000(w)]TJ
1257 0 TD
[(as)-24000(85-90%)-23000(for)-24000(similarly)]TJ
-3077 99 TD
[(sized)-28000(collections.)]TJ
641 0 TD
(P)Tj
55 0 TD
[(erhaps)-27000(part)-28000(of)-29000(the)-29000(problem)-28000(w)]TJ
1067 0 TD
[(as)-28000(that)-28000(w)]TJ
341 0 TD
[(e)-29000(nev)]TJ
191 0 TD
[(er)-27000(quite)-29000(grasp)-2000(ed)-28000(wh)]TJ
721 0 TD
[(y)-28000(some)-28000(but)-29000(not)-29000(all)-28000(these)]TJ
-3016 100 TD
[(all)-27000(CAPS)-28000(strings)-27000(w)]TJ
695 0 TD
[(ere)-26000(not)-28000(coreferen)]TJ
606 0 TD
(t.)Tj
/T13 1 Tf
-1301 235 TD
(La)Tj
159 0 TD
(Hac)Tj
189 0 TD
(k)Tj
97 0 TD
(3)Tj
/T15 1 Tf
-320 235 TD
[(La)-24000(Hac)]TJ
257 0 TD
[(k)-23000(i)0(s)-25000(a)-24000(carry)-23000(o)]TJ
466 0 TD
(v)Tj
42 0 TD
[(er)-23000(from)-24000(our)-24000(original)-22000(MUC-6)-25000(system,)-25000(and)-24000(it)-25000(is)-24000(resp)-2000(onsible)-24000(for)-24000(iden)]TJ
2333 0 TD
[(ti\014cation)-25000(of)-24000(prop)-2000(er)]TJ
-3223 99 TD
(noun)Tj
214 0 TD
(coreference.)Tj
479 0 TD
(This)Tj
196 0 TD
[(comp)-2000(onen)]TJ
365 0 TD
(t)Tj
65 0 TD
(is)Tj
90 0 TD
(indirectly)Tj
378 0 TD
[(help)-2000(ed)]TJ
272 0 TD
(b)Tj
44 0 TD
(y)Tj
78 0 TD
(IBM's)Tj
254 0 TD
(named)Tj
274 0 TD
(en)Tj
81 0 TD
(tit)Tj
85 0 TD
(y)Tj
78 0 TD
[(to)-2000(ol)]TJ
175 0 TD
('T)Tj
76 0 TD
(extract')Tj
314 0 TD
(whic)Tj
164 0 TD
(h)Tj
80 0 TD
(\014nd)Tj
-3762 100 TD
(exten)Tj
194 0 TD
(ts)Tj
98 0 TD
(of)Tj
101 0 TD
(named)Tj
273 0 TD
(en)Tj
81 0 TD
(tities)Tj
214 0 TD
(in)Tj
103 0 TD
(addition)Tj
333 0 TD
(to)Tj
107 0 TD
[(assigning)-32000(them)]TJ
580 0 TD
[(prop)-2000(erties)]TJ
397 0 TD
(lik)Tj
88 0 TD
(e)Tj
70 0 TD
('is)Tj
ET
1 i
398.76 257.26 3 0.48 re
f
BT
0.12 0 0 -0.12 401.76 257.26 Tm
[(p)-2000(erson',)]TJ
319 0 TD
('is)Tj
ET
450.12 257.26 3 0.48 re
f
BT
0.12 0 0 -0.12 453.12 257.26 Tm
(compan)Tj
280 0 TD
(y'.)Tj
143 0 TD
(It)Tj
96 0 TD
(is)Tj
90 0 TD
(the)Tj
-3785 100 TD
[(foundation)-29000(up)-2000(on)-29000(whic)]TJ
793.999 0 TD
[(h)-29000(our)-28000(coreference)-28000(annotation)-28000(is)-29000(built{mistak)]TJ
1612 0 TD
[(es)-28000(here)-29000(can)-29000(b)-2000(e)-29000(dev)]TJ
670.001 0 TD
[(astating)-29000(for)-28000(the)-30000(rest)-28000(of)]TJ
-3076 99 TD
[(the)-28000(system.)]TJ
451 0 TD
[(In)-27000(MUC-6,)-27000(La)-27000(Hac)]TJ
681 0 TD
[(k)-27000(p)-2000(erformed)-27000(at)-28000(29%)-27000(Recall)-27000(and)-27000(86%)-27000(precision,)-27000(but)-28000(it)-28000(faired)-27000(somewhat)-27000(w)]TJ
2625 0 TD
(orse)Tj
-3757 100 TD
[(in)-28000(MUC-7)-27000(with,)-28000(24.0%)-27000(precision)-26000(80.0%)-27000(recall.)]TJ
125 153 TD
(W)Tj
78 0 TD
[(e)-29000(observ)]TJ
299 0 TD
[(ed)-28000(that)-29000(the)-30000(New)-29000(Y)]TJ
680 0 TD
[(ork)-29000(Times)-29000(data)-28000(had)-29000(far)-29000(less)-29000(regular)-27000(honori\014c)-28000(use)-29000(and)-29000(corp)-2000(orate)-28000(designator)]TJ
-1181.999 100 TD
[(use)-29000(than)-29000(the)-30000(MUC-6)-28000(corpus)-28000(based)-29000(on)-29000(W)]TJ
1474 0 TD
[(all)-29000(Street)-29000(Journal.)]TJ
701 0 TD
[(As)-29000(a)-29000(result,)-29000(there)-29000(w)]TJ
723 0 TD
[(ere)-28000(few)]TJ
255 0 TD
[(er)-29000(reliable)-28000(indicators)]TJ
-3153 99 TD
[(of)-28000(prop)-2000(er)-26000(names.)]TJ
/T13 1 Tf
0 235 TD
(Highly)Tj
365 0 TD
(Syn)Tj
180 0 TD
(tactic)Tj
310 0 TD
(Coreference)Tj
/T15 1 Tf
-730 235 TD
[(This)-25000(comp)-2000(onen)]TJ
552 0 TD
[(t)-26000(asserts)-24000(coreference)-24000(b)-2000(e)0(t)]TJ
865 0 TD
(w)Tj
58 0 TD
[(een)-26000(phrases)-24000(that)-26000(are)-25000(in)-26000(app)-2000(ositiv)]TJ
1181 0 TD
[(e)-25000(relations)-24000(or)-25000(that)-26000(are)-25000(in)-26000(predi-)]TJ
-2781 100 TD
[(cate)-25000(nominal)-26000(relations.)]TJ
858.999 0 TD
(W)Tj
78 0 TD
26 Tc
(ew)Tj
121 0 TD
0 Tc
[(ere)-25000(quite)-26000(surprised)-24000(at)-26000(ho)]TJ
880 0 TD
[(w)-25000(p)-2000(o)-2000(orly)-25000(this)-26000(comp)-2000(onen)]TJ
869 0 TD
[(t)-26000(p)-2000(erformed)-25000(since)-25000(w)]TJ
712 0 TD
[(e)-26000(exp)-2000(ected)]TJ
-3519 99 TD
[(p)-2000(erformance)-27000(to)-27000(b)-2000(e)-28000(ab)-2000(o)]TJ
820 0 TD
(v)Tj
42 0 TD
[(e)-26000(the)-28000(80%)-27000(precision)-27000(cuto\013.)]TJ
998 0 TD
[(Our)-27000(actual)-27000(p)-2000(erformance)-26000(is)-28000(3.3%)-27000(precision)-26000(64.0%)-27000(recall.)]TJ
/T13 1 Tf
-1860 235 TD
(Quoted)Tj
397 0 TD
[(Sp)-3000(eec)]TJ
276 0 TD
(h)Tj
ET
endstream
endobj
48 0 obj
<<
/ProcSet [/PDF /Text /ImageB ]
/Font <<
/T1 49 0 R
/T13 7 0 R
/T15 9 0 R
>>
/ExtGState <<
/GS1 12 0 R
>>
/ColorSpace <<
/CS1 13 0 R
>>
>>
endobj
51 0 obj
<<
/Length 11282
>>
stream
BT
/T15 1 Tf
0.12 0 0 -0.12 87 708.7 Tm
/CS1 cs 0 sc
BX /GS1 gs EX
0 Tc
[(Quoted)-24000(sp)-2000(eec)]TJ
482 0 TD
[(h)-25000(has)-25000(idiosyncratic)-24000(patterns)-24000(of)-25000(use)-25000(that)-26000(are)-24000(b)-2000(etter)-25000(solv)]TJ
1958 0 TD
[(ed)-24000(out)-25000(side)-25000(the)-26000(scop)-2000(e)-25000(o)0(f)-25000(our)-24000(standard)]TJ
-2565 100 TD
[(coreference)-27000(resolution)-28000(mo)-2000(dule.)]TJ
1141 0 TD
(W)Tj
78 0 TD
[(e)-30000(exp)-2000(ected)-29000(p)-2000(erformance)-27000(to)-29000(b)-2000(e)-29000(ab)-2000(o)]TJ
1237 0 TD
(v)Tj
42 0 TD
[(e)-27000(90%)-28000(precision)-28000(and)-29000(w)]TJ
814 0 TD
[(ere)-27000(pleased)-29000(with)]TJ
-3312 99 TD
(2.6%)Tj
210 0 TD
(recall)Tj
228 0 TD
(and)Tj
168 0 TD
(86.8%)Tj
252 0 TD
(precision.)Tj
399 0 TD
(This)Tj
196 0 TD
[(mo)-2000(dule)]TJ
300 0 TD
(is)Tj
90 0 TD
(a)Tj
76 0 TD
2 Tc
[(g)2000(ood)]TJ
210 0 TD
0 Tc
(example)Tj
332 0 TD
(of)Tj
101 0 TD
(ho)Tj
86 0 TD
(w)Tj
94 0 TD
(the)Tj
150 0 TD
(coreference)Tj
434 0 TD
(problem)Tj
330 0 TD
(can)Tj
159 0 TD
2 Tc
(be)Tj
-3815 100 TD
0 Tc
[(fruitfully)-28000(brok)]TJ
511 0 TD
[(en)-27000(up)-28000(in)]TJ
297 0 TD
[(to)-27000(sub-parts)-27000(of)-27000(individually)-28000(high)-27000(precision.)]TJ
/T13 1 Tf
-808 236 TD
(CogNIA)Tj
405 0 TD
(C)Tj
117 0 TD
[(Prop)-3000(er)]TJ
378 0 TD
(Noun)Tj
306 0 TD
(Resolution)Tj
/T15 1 Tf
-1081 237 TD
(CogNIA)Tj
296 0 TD
(C)Tj
95 0 TD
(is)Tj
92 0 TD
(the)Tj
152 0 TD
(most)Tj
212 0 TD
(general)Tj
295 0 TD
[(purp)-2000(ose)]TJ
321 0 TD
(coreference)Tj
436 0 TD
(resolution)Tj
393 0 TD
[(comp)-2000(onen)]TJ
365 0 TD
(t)Tj
68 0 TD
(of)Tj
103 0 TD
(the)Tj
152 0 TD
(system.)Tj
334 0 TD
(It)Tj
99 0 TD
(features)Tj
320 0 TD
(a)Tj
-3858 99 TD
(fairly)Tj
225 0 TD
(sophisticated)Tj
507 0 TD
(salience)Tj
313 0 TD
[(mo)-2000(del)]TJ
255 0 TD
(and)Tj
170 0 TD
[(prop)-2000(ert)]TJ
269 0 TD
(y)Tj
79 0 TD
(con\014dence)Tj
409 0 TD
[(mo)-2000(del)]TJ
255 0 TD
(to)Tj
110 0 TD
(preorder)Tj
340 0 TD
(order)Tj
226 0 TD
(the)Tj
151 0 TD
(set)Tj
138 0 TD
(of)Tj
102 0 TD
(candidate)Tj
-3549 100 TD
(an)Tj
86 0 TD
(teceden)Tj
270 0 TD
(ts.)Tj
130 0 TD
[(The)-30000(imp)-2000(ortance)-29000(of)-30000(the)-30000(preorder)-28000(is)-30000(that)-30000(it)-30000(allo)]TJ
1669 0 TD
[(ws)-28000(ties)-30000(b)-2000(e)0(t)]TJ
391 0 TD
(w)Tj
58 0 TD
[(een)-29000(equally)-29000(salien)]TJ
639 0 TD
30 Tc
[(ta)30000(n)]TJ
148 0 TD
0 Tc
(teceden)Tj
270 0 TD
(ts{and)Tj
-3661 99 TD
[(in)-28000(the)-28000(case)-27000(of)-27000(ties)-28000(the)-28000(anaphor)-26000(is)-27000(not)-28000(resolv)]TJ
1570 0 TD
(ed.)Tj
-1445 154 TD
[(When)-24000(de\014ciencies)-24000(w)]TJ
722 0 TD
[(ere)-23000(noted)-24000(with)-24000(the)-24000(output)-25000(of)-24000(LaHac)]TJ
1274 0 TD
[(k,)-23000(the)-25000(simplest)-24000(solution)-23000(w)]TJ
918 0 TD
[(as)-23000(to)-24000(add)-24000(a)-23000(prop)-2000(er)-23000(noun)]TJ
-3039 99 TD
[(resolution)-21000(comp)-2000(onen)]TJ
743 0 TD
[(t)-22000(t)0(o)-23000(CogNIA)]TJ
447 0 TD
[(C.)-21000(In)-23000(the)-23000(end)-22000(this)-23000(addition)-22000(added)-22000(a)-22000(bit)-23000(of)-22000(recall)-22000(but)-23000(with)-23000(fairly)-21000(lo)]TJ
2309 0 TD
[(w)-22000(precision)]TJ
-3498.999 100 TD
[(with)-28000(1.2%)-27000(recall)-27000(and)-27000(65.2%)-27000(precision.)]TJ
/T13 1 Tf
0 236 TD
(CogNIA)Tj
405 0 TD
(C)Tj
117 0 TD
(Common)Tj
480 0 TD
(Noun)Tj
305 0 TD
(Resolution)Tj
/T15 1 Tf
-1182 237 TD
[(Common)-20000(noun)-21000(coreference)-19000(is)-21000(an)-21000(imp)-2000(ortan)]TJ
1488 0 TD
[(t)-21000(part)-20000(of)-21000(coreference,)-21000(but)-22000(it)-21000(is)-21000(v)]TJ
1100 0 TD
[(ery)-20000(di\016cult)-21000(to)-21000(accurately)-20000(resolv)]TJ
1129 0 TD
(e.)Tj
-3842 99 TD
[(Our)-25000(MUC-6)-26000(system)-25000(had)-26000(fairly)-25000(p)-2000(o)-2000(o)0(r)-26000(p)-2000(erformance)-25000(with)-26000(10%)-25000(recall)-25000(and)-26000(a)-25000(precision)-25000(of)-26000(48%.)]TJ
3239.999 0 TD
(W)Tj
78 0 TD
26 Tc
(ew)Tj
121 0 TD
0 Tc
[(ere)-25000(surprised)]TJ
-3438.999 100 TD
[(with)-28000(an)-27000(increase)-27000(in)-28000(p)-2000(erformance)-26000(o)]TJ
1231 0 TD
(v)Tj
42 0 TD
[(er)-26000(training)-27000(data)-27000(\(78%)-27000(precision\))-27000(with)-28000(7.1%)-27000(recall)-27000(90.7%)-26000(precision.)]TJ
-1148 153 TD
(Common)Tj
367 0 TD
(noun)Tj
220 0 TD
(anaphora)Tj
377 0 TD
(is)Tj
96 0 TD
(probably)Tj
361 0 TD
(one)Tj
165 0 TD
(of)Tj
107 0 TD
(the)Tj
155 0 TD
(most)Tj
216 0 TD
(trying)Tj
259 0 TD
(classes)Tj
277 0 TD
(of)Tj
107 0 TD
(coreference)Tj
440 0 TD
(to)Tj
114 0 TD
(annotate)Tj
358 0 TD
(as)Tj
114 0 TD
(a)Tj
-3858 100 TD
(h)Tj
44 0 TD
(uman.)Tj
262 0 TD
[(This)-26000(is)-26000(due)-25000(to)-26000(man)]TJ
679 0 TD
[(y)-25000(di\016cult)-27000(judgmen)]TJ
681 0 TD
[(t)-26000(calls)-25000(required)-25000(on)-25000(the)-27000(part)-25000(of)-26000(the)-26000(h)]TJ
1276 0 TD
[(uman)-25000(judges,)-26000(and)-26000(this)-26000(w)]TJ
884 0 TD
(as)Tj
-3826 99 TD
[(re\015ected)-32000(in)-32000(the)-32000(consistency)-31000(of)-31000(annotation)-31000(in)-32000(the)]TJ
1797 0 TD
[(training)-31000(data.)]TJ
552 0 TD
(W)Tj
78 0 TD
[(e)-32000(found)-32000(it)-32000(c)]TJ
428 0 TD
[(hallenging)-31000(to)-31000(dev)]TJ
631 0 TD
[(elop)-31000(on)-32000(the)]TJ
-3485.999 100 TD
[(training)-31000(data)-30000(b)-2000(ecause)-31000(the)-32000(system)-31000(w)]TJ
1305 0 TD
[(ould)-31000(\014nd)-32000(what)-31000(w)]TJ
627 0 TD
[(e)-31000(considered)-30000(to)-31000(b)-2000(e)-32000(reasonable)-29000(instances)-31000(of)-32000(coreference)]TJ
-1932 100 TD
[(that)-32000(the)-31000(annotator)-30000(had)-31000(not)-31000(made)-31000(coreferen)]TJ
1583 0 TD
(t.)Tj
102 0 TD
(W)Tj
78 0 TD
[(e)-31000(b)-2000(eliev)]TJ
278 0 TD
[(e)-32000(that)-31000(common)-31000(noun)-31000(anaphora)-29000(is)-31000(a)-31000(large)-30000(source)-30000(of)]TJ
-2040.999 99 TD
(in)Tj
67 0 TD
[(ter-annotator)-25000(disagreemen)]TJ
955 0 TD
(t.)Tj
/T13 1 Tf
-1022 236 TD
(CogNIA)Tj
405 0 TD
(C)Tj
117 0 TD
(Pronouns)Tj
/T15 1 Tf
-397 237 TD
[(The)-28000(pronominal)-28000(system)-28000(p)-2000(erformed)-28000(under)-29000(our)-28000(goal)-27000(of)-29000(80%)-28000(precision.)]TJ
2506 0 TD
[(In)-29000(training,)-28000(w)]TJ
501 0 TD
[(e)-29000(found)-29000(that)-29000(w)]TJ
539 0 TD
28 Tc
(ew)Tj
123 0 TD
0 Tc
(ere)Tj
-3794 99 TD
(constan)Tj
276 0 TD
[(tly)-32000(balancing)-32000(the)]TJ
657.999 0 TD
(abilit)Tj
187 0 TD
(y)Tj
77 0 TD
(of)Tj
100 0 TD
[(pronouns)-32000(to)]TJ
473 0 TD
(i\))Tj
88 0 TD
[(refer)-32000(uniquely)]TJ
499 0 TD
(,)Tj
58 0 TD
[(and)-32000(ii\))]TJ
278 0 TD
(ha)Tj
86 0 TD
(v)Tj
42 0 TD
[(e)-32000(all)-32000(en)]TJ
270 0 TD
(tities)Tj
213 0 TD
(ha)Tj
86 0 TD
(v)Tj
42 0 TD
[(e)-32000(the)]TJ
217 0 TD
(correct)Tj
-3649.999 100 TD
[(prop)-2000(ert)]TJ
269 0 TD
(y)Tj
37 0 TD
(.)Tj
58 0 TD
(W)Tj
78 0 TD
[(e)-25000(adopted)-25000(a)-24000(prop)-2000(ert)]TJ
713 0 TD
[(y)-24000(con\014dence)-25000(mo)-2000(del)-25000(that)-25000(encouraged)-23000(recall)-24000(o)]TJ
1578 0 TD
(v)Tj
42 0 TD
[(er)-23000(precision.)]TJ
471 0 TD
[(This)-25000(mean)]TJ
379 0 TD
[(t)-24000(that)-25000(a)]TJ
-3625 100 TD
[(prop)-2000(er)-25000(noun)-25000(lik)]TJ
557 0 TD
[(e)-25000('Mrs.)]TJ
286 0 TD
[(Fields')-26000(w)]TJ
323 0 TD
[(ould)-25000(b)-2000(e)-26000(b)-2000(oth)-26000(p)-2000(oten)]TJ
690 0 TD
[(tially)-25000(an)-25000(an)]TJ
411 0 TD
(teceden)Tj
270 0 TD
[(t)-25000(t)0(o)-26000(feminine)-26000(pronouns,)-25000(and)-25000(pronouns)]TJ
-2537 99 TD
[(that)-28000(referred)-27000(to)-28000(companies.)]TJ
1026 0 TD
[(A)-28000(salience)-28000(mo)-2000(del)-28000(w)]TJ
701 0 TD
[(as)-27000(then)-28000(applied)-29000(to)-27000(these)-29000(o)]TJ
938 0 TD
(v)Tj
42 0 TD
[(erloaded)-25000(en)]TJ
412 0 TD
[(tities)-28000(and)-28000(pronominal)]TJ
-3119 100 TD
[(resolution)-22000(serv)]TJ
524 0 TD
[(ed)-21000(to)-23000(b)-2000(e)-23000(a)-23000(w)]TJ
432 0 TD
[(ord-sense)-21000(disam)]TJ
567 0 TD
[(biguation)-21000(problem)-22000(in)-24000(addition)-22000(to)-23000(a)-22000(coreference)-22000(resolution)-22000(problem.)]TJ
-1523 99 TD
[(Our)-27000(p)-2000(erformance)-26000(w)]TJ
704 0 TD
[(as)-27000(4.5%)-27000(recall)-27000(and)-27000(70.0%)-26000(precision.)]TJ
/T13 1 Tf
-704 237 TD
(Conclusions)Tj
/T15 1 Tf
125 236 TD
[(One)-27000(of)-28000(the)-28000(stronger)-26000(conclusions)-27000(that)-28000(w)]TJ
1410 0 TD
28 Tc
[(eh)28000(a)]TJ
151 0 TD
(v)Tj
42 0 TD
0 Tc
[(e)-27000(come)-27000(to)-28000(regarding)-26000(coreference)-26000(is)-28000(that)-28000(there)-28000(is)-28000(an)-27000(apparen)]TJ
2141.001 0 TD
(t)Tj
-3869 100 TD
[(linear)-20000(trade-o\013)-20000(b)-2000(e)0(t)]TJ
667 0 TD
(w)Tj
58 0 TD
[(een)-20000(precision)-20000(and)-21000(recall)-19000(giv)]TJ
956 0 TD
[(en)-20000(the)-21000(p)-2000(erformance)-20000(of)-21000(other)-20000(systems)-20000(with)-21000(the)-21000(coreference)-20000(task.)]TJ
-1680.999 99 TD
[(Our)-27000(suspicion)-28000(is)-28000(that)-28000(the)-28000(same)-28000(can)-27000(b)-2000(e)-28000(said)-28000(with)-28000(the)-29000(B3)-27000(scorer)-26000(but)-29000(that)-28000(will)-28000(ha)]TJ
2859 0 TD
(v)Tj
42 0 TD
27 Tc
[(et)27000(o)-1000(a)]TJ
206 0 TD
(w)Tj
58 0 TD
0 Tc
[(ait)-27000(exp)-2000(erimen)]TJ
496 0 TD
(tation.)Tj
-3661 100 TD
(This)Tj
195 0 TD
(is)Tj
89 0 TD
(a)Tj
75 0 TD
[(p)-2000(ositiv)]TJ
243 0 TD
(e)Tj
70 0 TD
[(result)-32000(in)]TJ
339 0 TD
(its)Tj
121 0 TD
(self)Tj
152 0 TD
[(b)-2000(ecause)-32000(w)]TJ
370 0 TD
(e)Tj
70 0 TD
(no)Tj
86 0 TD
(w)Tj
93 0 TD
[(can)-32000(c)]TJ
192 0 TD
[(ho)-2000(ose)-32000(from)]TJ
436 0 TD
(m)Tj
67 0 TD
(ultiple)Tj
264 0 TD
(t)Tj
30 0 TD
[(yp)-2000(es)]TJ
195 0 TD
(of)Tj
100 0 TD
[(coreference)-32000(systems)]TJ
-3187 99 TD
[(dep)-2000(ending)-29000(on)-28000(our)-28000(task.)]TJ
878 0 TD
(W)Tj
78 0 TD
[(e)-28000(consider)-28000(high)-28000(precision)-28000(systems)-28000(to)-28000(b)-2000(e)-29000(more)-27000(useful)-29000(for)-28000(the)-29000(t)]TJ
2197.001 0 TD
[(yp)-2000(es)-28000(of)-29000(systems)-28000(that)]TJ
-3153 100 TD
(w)Tj
58 0 TD
[(e)-27000(build,)-28000(but,)-29000(it)-28000(has)-27000(not)-27000(b)-2000(een)-28000(clear)-27000(that)-28000(high)-27000(precision)-27000(systems)-27000(w)]TJ
2325 0 TD
[(ere)-27000(p)-2000(ossible.)]TJ
-2258 153 TD
(W)Tj
78 0 TD
[(e)-25000(also)-25000(b)-2000(eliev)]TJ
437 0 TD
[(e)-25000(that)-26000(the)-26000(space)-25000(of)-25000(high)-26000(precision)-24000('con)]TJ
1366 0 TD
[(tributors')-25000(to)-25000(coreference)-24000(is)-26000(not)-25000(exhausted.)]TJ
1541 0 TD
(W)Tj
78 0 TD
[(e)-26000(doubt)]TJ
-3625 100 TD
[(that)-32000(there)-31000(are)-30000(an)]TJ
628 0 TD
[(y)-31000(10%)-30000(recall/80%)-29000(precision)-31000(sub)-2000(comp)-2000(onen)]TJ
1520 0 TD
[(ts)-31000(that)-32000(w)]TJ
338 0 TD
31 Tc
[(eh)31000(a)]TJ
154 0 TD
(v)Tj
42 0 TD
0 Tc
[(e)-30000(not)-31000(already)-30000(explored,)-31000(but)-32000(there)]TJ
-2682 100 TD
[(are)-22000(certainly)-23000(1-5%)-22000(recall)-22000(opp)-2000(ortunities.)]TJ
1434.999 0 TD
(Ho)Tj
102 0 TD
23 Tc
(ww)Tj
141 0 TD
0 Tc
[(ell)-23000(they)-23000(will)-23000(sum)-23000(to)-24000(the)-23000(recall)-22000(of)-23000(the)-24000(en)]TJ
1374 0 TD
[(tire)-23000(system)-22000(is)-23000(unkno)]TJ
719 0 TD
(wn,)Tj
-3771 99 TD
[(but)-28000(there)-28000(is)-27000(ro)-2000(om)-27000(for)-27000(impro)]TJ
1001 0 TD
(v)Tj
42 0 TD
(emen)Tj
187 0 TD
(t.)Tj
ET
endstream
endobj
52 0 obj
<<
/ProcSet [/PDF /Text /ImageB ]
/Font <<
/T13 7 0 R
/T15 9 0 R
>>
/ExtGState <<
/GS1 12 0 R
>>
/ColorSpace <<
/CS1 13 0 R
>>
>>
endobj
54 0 obj
<<
/Length 1534
>>
stream
BT
/T13 1 Tf
0.12 0 0 -0.12 72 708.7 Tm
/CS1 cs 0 sc
BX /GS1 gs EX
0 Tc
(REFERENCES)Tj
/T15 1 Tf
0 183 TD
[([Bagga,)-26000(98a])]TJ
578.999 0 TD
(Bagga,)Tj
292 0 TD
(Amit.)Tj
251 0 TD
(Ho)Tj
102 0 TD
(w)Tj
100 0 TD
(Muc)Tj
157 0 TD
(h)Tj
86 0 TD
[(Pro)-2000(cessing)]TJ
424 0 TD
(Is)Tj
103 0 TD
(Required)Tj
367 0 TD
(for)Tj
141 0 TD
[(Cross-Do)-2000(cumen)]TJ
569 0 TD
(t)Tj
71 0 TD
(Coreference?,)Tj
/T6 1 Tf
529 0 TD
(this)Tj
-3191.999 99 TD
(volume.)Tj
/T15 1 Tf
-579 118 TD
[([Vilain,)-28000(95])]TJ
579 0 TD
[(Vilain,)-25000(Marc,)-24000(et)-24000(al.)-23000(A)-24000(M)0(o)-2000(del-Theoretic)-23000(Coreference)-23000(Scoring)-22000(Sc)]TJ
2234.999 0 TD
(heme,)Tj
/T6 1 Tf
237 0 TD
(Pr)Tj
87 0 TD
(o)Tj
38 0 TD
(c)Tj
34 0 TD
(e)Tj
34 0 TD
(e)Tj
34 0 TD
[(dings)-27000(of)-27000(the)-26000(Sixth)]TJ
-2699 100 TD
[(Message)-31000(Understanding)-30000(Confer)]TJ
1128 0 TD
(enc)Tj
119 0 TD
[(e)-31000(\(MUC-6\))]TJ
/T15 1 Tf
404 0 TD
[(,)-28000(pp.)-28000(45-52,)-26000(No)]TJ
541 0 TD
(v)Tj
42 0 TD
(em)Tj
104 0 TD
[(b)-2000(e)0(r)-26000(1995.)]TJ
-2917 117 TD
[([MUC-6,)-27000(95])]TJ
/T6 1 Tf
579 0 TD
(Pr)Tj
87 0 TD
(o)Tj
38 0 TD
(c)Tj
34 0 TD
(e)Tj
34 0 TD
(e)Tj
34 0 TD
[(dings)-29000(of)-29000(the)-29000(Sixth)-28000(Message)-30000(Understanding)-28000(Confer)]TJ
1781.999 0 TD
(enc)Tj
119 0 TD
[(e)-30000(\(MUC-6\))]TJ
/T15 1 Tf
403 0 TD
27 Tc
[(,N)27000(o)]TJ
152 0 TD
(v)Tj
42 0 TD
0 Tc
(em)Tj
104 0 TD
[(b)-2000(e)0(r)-25000(1995,)-24000(San)]TJ
-2828.999 100 TD
(Mateo:)Tj
288 0 TD
[(Morgan)-26000(Kaufmann.)]TJ
ET
endstream
endobj
55 0 obj
<<
/ProcSet [/PDF /Text /ImageB ]
/Font <<
/T6 27 0 R
/T13 7 0 R
/T15 9 0 R
>>
/ExtGState <<
/GS1 12 0 R
>>
/ColorSpace <<
/CS1 13 0 R
>>
>>
endobj
56 0 obj
<<
/Type /Halftone
/HalftoneType 1
/HalftoneName (Default)
/Frequency 60
/Angle 45
/SpotFunction /Round
>>
endobj
12 0 obj
<<
/Type /ExtGState
/SA false
/OP false
/HT /Default
>>
endobj
57 0 obj
<<
/ProcSet [/PDF /ImageB ]
>>
endobj
49 0 obj
<<
/Name /T1
/Type /Font
/Subtype /Type3
/Resources 57 0 R
/FontBBox [0 -7 43 46]
/FontMatrix [1 0 0 -1 0 0]
/FirstChar 45
/LastChar 122
/Encoding 58 0 R
/CharProcs 59 0 R
/Widths [44 44 44 0 0 0 0 0 0 0 0 0 0 0 0 44
44 44 0 0 44 44 44 0 44 44 44 44 0 0 0 44
44 0 44 44 0 44 44 0 44 0 0 0 0 0 0 0
0 0 0 0 44 0 0 0 0 44 0 0 44 0 0 0
0 0 0 0 0 0 0 44 0 44 0 0 0 44 ]
>>
endobj
58 0 obj
<<
/Type /Encoding
/Differences [45/- /. /#232F 60/#233C
/= /#233E 65/A /B /C 69/E /F /G /H 76/L
/M 79/O /P 82/R /S 85/U 97/a 102/f 105/i 116/t 118/v 122/z ]
>>
endobj
59 0 obj
<<
/- 60 0 R
/. 61 0 R
/#232F 62 0 R
/#233C 63 0 R
/= 64 0 R
/#233E 65 0 R
/A 66 0 R
/B 67 0 R
/C 68 0 R
/E 69 0 R
/F 70 0 R
/G 71 0 R
/H 72 0 R
/L 73 0 R
/M 74 0 R
/O 75 0 R
/P 76 0 R
/R 77 0 R
/S 78 0 R
/U 79 0 R
/a 80 0 R
/f 81 0 R
/i 82 0 R
/t 83 0 R
/v 84 0 R
/z 85 0 R
>>
endobj
60 0 obj
<<
/Length 124
>>
stream
44 0 5 22 38 28 d1
q
33 0 0 6 5.1 21.9 cm
BI
/W 33
/H 6
/BPC 1
/IM true
/F /CCF
/DP <>
ID #M?ڀ
EI
Q
endstream
endobj
61 0 obj
<<
/Length 129
>>
stream
44 0 16 0 27 11 d1
q
11 0 0 11 16.1 -0.1 cm
BI
/W 11
/H 11
/BPC 1
/IM true
/F /CCF
/DP <>
ID 0tjը
EI
Q
endstream
endobj
62 0 obj
<<
/Length 160
>>
stream
44 0 5 -7 38 58 d1
q
33 0 0 65 5.1 -7.1 cm
BI
/W 33
/H 65
/BPC 1
/IM true
/F /CCF
/DP <>
ID U5֫꺮u]WUu]z꺮Uu]WU꺮
EI
Q
endstream
endobj
63 0 obj
<<
/Length 166
>>
stream
44 0 5 4 38 46 d1
q
33 0 0 42 5.1 3.9 cm
BI
/W 33
/H 42
/BPC 1
/IM true
/F /CCF
/DP <>
ID T&B*P*
*
*_v7av7av7a0
EI
Q
endstream
endobj
64 0 obj
<<
/Length 133
>>
stream
44 0 3 15 40 35 d1
q
37 0 0 20 3.1 14.9 cm
BI
/W 37
/H 20
/BPC 1
/IM true
/F /CCF
/DP <>
ID #?Q.P@ @
EI
Q
endstream
endobj
65 0 obj
<<
/Length 166
>>
stream
44 0 5 4 38 46 d1
q
33 0 0 42 5.1 3.9 cm
BI
/W 33
/H 42
/BPC 1
/IM true
/F /CCF
/DP <>
ID #0}BT%ABT%ABT%*P
EI
Q
endstream
endobj
66 0 obj
<<
/Length 168
>>
stream
44 0 2 0 41 52 d1
q
39 0 0 52 2.1 -0.1 cm
BI
/W 39
/H 52
/BPC 1
/IM true
/F /CCF
/DP <>
ID 5ɏIM7<