filters

transform.cpp
1#include "transform.h"
2
3#include <kdebug.h>
4
5
6namespace PDFImport
7{
8
9// classification of some Unicode characters
10static const uint TABLE_SIZE = 0x0100;
11static const uint NB_TABLES = 5;
12static const uint OFFSET[NB_TABLES] = {
13 0x00, 0x01, 0x20, 0x21, 0x22
14};
15static const char TABLE[NB_TABLES][TABLE_SIZE] = {
16#define U Unknown
17#define P Punctuation
18#define S SymbolChar
19#define D Digit
20#define L Letter
21
22#define H Hyphen
23#define B Bullet
24#define I SuperScript
25#define Y SpecialSymbol
26#define G Ligature
27
28#define A Accent
29#define E Punctuation_Accent
30#define C Letter_CanHaveAccent
31
32#define X LatexSpecial
33//-------------------------------------------------------------------
34{ // 0x0000 to 0x00FF
35 U,U,U,U,U,U,U,U,U,P,P,U,U,U,U,U, U,U,U,U,U,U,U,U,U,U,U,U,U,U,U,U,
36//
37 P,P,E,S,S,S,S,P,P,P,S,S,E,H,E,S, D,D,D,D,D,D,D,D,D,D,P,P,S,S,S,P,
38// ! " # $ % & ' ( ) * + , - . / 0 1 2 3 4 5 6 7 8 9 : ; < = > ?
39 S,C,L,C,L,C,L,C,C,C,C,C,C,L,C,C, L,L,C,C,C,C,L,C,L,C,C,S,X,S,A,A,
40// @ A B C D E F G H I J K L M N O P Q R S T U V W X Y Z [ \ ] ^ _
41 E,C,L,C,L,C,L,C,C,C,C,C,C,L,C,C, L,L,C,C,C,C,L,C,L,C,C,S,S,S,A,U,
42// ` a b c d e f g h i j k l m n o p q r s t u v w x y z { | } ~
43 U,U,U,U,U,U,U,U,U,U,U,U,U,U,U,U, U,U,U,U,U,U,U,U,U,U,U,U,U,U,U,U,
44//
45 U,U,S,S,S,S,S,S,A,S,U,P,U,H,S,A, A,S,I,I,E,U,U,S,A,I,U,P,S,S,S,P,
46// ¢ £ ¤ § ¨ « ¬ ° ± ² ³ µ ¶ · ¹ » ¼ ¿
47 L,L,L,L,L,L,C,L,L,L,L,L,L,L,L,L, L,L,L,L,L,L,L,S,C,L,L,L,L,L,L,L,
48// Â Ä Ê Ë Î Ï Ô Ö Û Ü ß
49 L,L,L,L,L,L,C,L,L,L,L,L,L,L,L,L, L,L,L,L,L,L,L,S,C,L,L,L,L,L,L,L
50// à â ä æ ç è é ê ë î ï ð ô ö ø û ü þ ÿ
51},
52//-------------------------------------------------------------------
53{ // 0x0100 to 0x01FF
54 L,L,L,L,L,L,L,L,L,L,L,L,L,L,L,L, L,L,L,L,L,L,L,L,L,L,L,L,L,L,L,L,
55 L,L,L,L,L,L,L,L,L,L,L,L,L,L,L,L, L,C,L,L,L,L,L,L,L,L,L,L,L,L,L,L,
56 L,L,L,L,L,L,L,L,L,L,L,L,L,L,L,L, L,L,L,L,L,L,L,L,L,L,L,L,L,L,L,L,
57 L,L,L,L,L,L,L,L,L,L,L,L,L,L,L,L, L,L,L,L,L,L,L,L,L,L,L,L,L,L,L,U,
58 U,U,U,U,U,U,U,U,U,U,U,U,U,U,S,U, U,U,U,U,U,U,U,U,U,U,U,U,U,U,U,U,
59 L,L,U,U,U,U,U,L,L,S,U,U,U,U,L,L, L,U,U,U,U,L,L,U,U,U,U,U,U,U,U,U,
60 U,U,S,P,U,U,U,U,U,U,U,U,U,L,L,L, L,L,L,L,L,L,L,L,L,L,L,L,L,L,L,L,
61 L,L,L,L,L,L,L,L,L,L,L,L,L,L,U,U, L,U,U,U,L,L,U,U,L,L,L,L,L,L,L,L
62},
63//-------------------------------------------------------------------
64{ // 0x2000 to 0x20FF
65 U,U,U,U,U,U,U,U,U,U,U,U,U,U,U,U, U,U,U,U,U,U,U,U,P,P,P,P,P,P,P,P,
66 S,S,B,U,U,U,P,U,U,U,U,U,U,U,U,U, S,U,P,P,U,U,U,U,U,Y,Y,U,U,U,U,U,
67 U,U,U,U,Y,U,U,U,U,U,U,U,U,U,U,U, U,U,U,U,U,U,U,U,U,U,U,U,U,U,U,U,
68 U,U,U,U,U,U,U,U,U,U,U,U,U,U,U,U, U,U,U,U,U,U,U,U,U,U,U,U,U,U,U,U,
69 U,U,U,U,U,U,U,U,U,U,U,U,U,U,U,U, U,U,U,U,U,U,U,U,U,U,U,U,U,U,U,U,
70 U,U,U,U,U,U,U,U,U,U,U,U,S,U,U,U, U,U,U,U,U,U,U,U,U,U,U,U,U,U,U,U,
71 U,U,U,U,U,U,U,U,U,U,U,U,U,U,U,U, U,U,U,U,U,U,U,U,U,U,U,U,U,U,U,U,
72 U,U,U,U,U,U,U,U,U,U,U,U,U,U,U,U, U,U,U,U,U,U,U,U,U,U,U,U,U,U,U,U
73},
74//-------------------------------------------------------------------
75{ // 0x2100 to 0x21FF
76 U,U,U,G,U,U,U,U,U,G,U,U,U,U,U,U, U,U,U,U,U,U,U,U,U,U,U,U,U,U,U,U,
77 U,U,S,U,U,U,U,U,U,U,Y,Y,U,U,U,U, U,U,U,U,U,U,U,U,U,U,U,U,U,U,U,U,
78 U,U,U,U,U,U,U,U,U,U,U,U,U,U,U,U, U,U,U,U,U,U,U,U,U,U,U,U,U,U,U,U,
79 U,U,U,U,U,U,U,U,U,U,U,U,U,U,U,U, U,U,U,U,U,U,U,U,U,U,U,U,U,U,U,U,
80 U,U,U,U,U,U,U,U,U,U,U,U,U,U,U,U, Y,Y,Y,Y,Y,U,U,U,U,U,Y,Y,U,U,U,U,
81 U,U,U,U,U,U,U,U,U,U,U,U,U,U,Y,U, U,U,U,U,U,Y,U,U,U,U,U,U,U,U,U,U,
82 U,U,U,U,U,U,U,U,U,U,U,U,U,Y,Y,Y, Y,Y,Y,Y,Y,U,U,U,U,U,U,U,U,U,U,U,
83 U,U,U,U,U,U,U,U,U,U,U,U,U,U,U,U, U,U,U,U,U,U,U,U,U,U,U,U,U,U,U,U
84},
85//-------------------------------------------------------------------
86{ // 0x2200 to 0x22FF
87 Y,U,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,U,Y, U,Y,Y,U,U,Y,U,Y,U,Y,Y,U,U,Y,Y,U,
88 Y,U,U,U,U,U,U,Y,Y,Y,Y,Y,U,U,U,U, U,U,U,U,Y,Y,Y,U,U,U,U,U,Y,U,U,U,
89 U,Y,U,U,U,Y,U,Y,Y,Y,U,U,U,U,U,U, U,U,U,U,U,U,U,U,U,Y,Y,U,U,U,U,U,
90 Y,Y,Y,U,Y,Y,U,U,U,U,U,U,U,U,Y,Y, Y,Y,U,U,U,U,U,U,U,U,U,U,U,U,U,U,
91 U,U,Y,Y,Y,Y,Y,Y,Y,Y,U,U,U,U,U,U, U,U,U,U,U,Y,U,Y,U,U,U,U,U,U,U,U,
92 U,U,U,U,Y,Y,U,U,U,U,U,U,U,U,U,U, U,U,U,U,U,U,U,U,U,U,U,U,U,U,U,U,
93 U,U,U,U,U,Y,U,U,U,U,U,U,U,U,U,U, U,U,U,U,U,U,U,U,U,U,U,U,U,U,U,U,
94 U,U,U,U,U,U,U,U,U,U,U,U,U,U,U,U, U,U,U,U,U,U,U,U,U,U,U,U,U,U,U,U
95}
96//-------------------------------------------------------------------
97#undef U
98#undef P
99#undef S
100#undef D
101#undef L
102
103#undef H
104#undef B
105#undef I
106#undef Y
107
108#undef A
109#undef E
110#undef C
111#undef X
112};
113
114CharType type(Unicode u)
115{
116 uint offset = u / TABLE_SIZE;
117 uint index = u % TABLE_SIZE;
118 for (uint i=0; i<NB_TABLES; i++) {
119 if ( offset==OFFSET[i] ) return (CharType)TABLE[i][index];
120 if ( offset<OFFSET[i] ) break;
121 }
122 if ( u>=0xFB00 && u<=0xFB06 ) return Ligature;
123 return Unknown;
124}
125
126//-----------------------------------------------------------------------------
127static const Unicode LIGATURE_DATA[][MaxLigatureLength+1] = {
128 { 0xFB00, 0x0066, 0x0066, 0x0000 }, // ff
129 { 0xFB01, 0x0066, 0x0069, 0x0000 }, // fi
130 { 0xFB02, 0x0066, 0x006C, 0x0000 }, // fl
131 { 0xFB03, 0x0066, 0x0066, 0x0069 }, // ffi
132 { 0xFB04, 0x0066, 0x0066, 0x006c }, // ffl
133 // 0xFB05, 0xFB06 ??
134 // latex does not generate a ligature for ffe...
135 { 0x0000, 0x0000, 0x0000, 0x0000 }
136};
137
138uint checkLigature(Unicode u, Unicode res[MaxLigatureLength])
139{
140 if ( type(u)==Unknown ) kdDebug(30516) << "unknown char " << u << endl;
141 if ( type(u)!=Ligature ) {
142 res[0] = u;
143 return 1;
144 }
145
146 uint i = 0;
147 while ( LIGATURE_DATA[i][0]!=0 ) {
148 if ( LIGATURE_DATA[i][0]==u ) {
149 uint k = 0;
150 for (; k<MaxLigatureLength; k++) {
151 if ( LIGATURE_DATA[i][k+1]==0 ) break;
152 res[k] = LIGATURE_DATA[i][k+1];
153 }
154 return k;
155 }
156 i++;
157 }
158 kdDebug(30516) << "undefined ligature !! " << u <<endl;
159 res[0] = u;
160 return 1;
161}
162
163
164//-----------------------------------------------------------------------------
165static const Unicode SUPER_DATA[][2] = {
166 { 0x00B9, 0x0031 }, // 1
167 { 0x00B2, 0x0032 }, // 2
168 { 0x00B3, 0x0033 }, // 3
169 { 0x0000, 0x0000 }
170};
171
172static const Unicode BULLET_DATA[][2] = {
173 { 0x2022, 0x00B7 }, // full circle
174 { 0x0000, 0x0000 }
175};
176
177//static const SpecialData SPECIAL_DATA[] = {
178// #### FIXME : add missing characters (table 0x20 to 0x22)
179// { 0x2190, SpecialSymbol, 0x00AC }, // <-
180// { 0x2192, SpecialSymbol, 0x00AE }, // ->
181// { 0x2212, SpecialSymbol, 0x002D }, // -
182// { 0x2217, SpecialSymbol, 0x002A }, // *
183// { 0x221A, SpecialSymbol, 0x00D6 }, // squareroot
184// { 0x2229, SpecialSymbol, 0x00C7 }, // intersection
185// { 0x222A, SpecialSymbol, 0x00C8 }, // union
186// { 0x0000, Unknown, 0x0000 }
187//};
188
189CharType checkSpecial(Unicode u, Unicode &res)
190{
191 CharType t = type(u);
192
193 // special mapping
194 switch (t) {
195 case Unknown:
196 kdDebug(30516) << "unknown special " << TQString(TQChar(u))
197 << " (" << u << ")" << endl;
198 break;
199 case SuperScript: {
200 uint i = 0;
201 for (;;) {
202 if ( SUPER_DATA[i][0]==0 ) {
203 kdDebug(30516) << "undefined superscript !!" << endl;
204 break;
205 }
206 if ( SUPER_DATA[i][0]==u ) {
207 res = SUPER_DATA[i][1];
208 break;
209 }
210 i++;
211 }
212 break;
213 }
214 case Bullet:{
215 uint i = 0;
216 for (;;) {
217 if ( BULLET_DATA[i][0]==0 ) {
218 kdDebug(30516) << "undefined bullet !!" << endl;
219 break;
220 }
221 if ( BULLET_DATA[i][0]==u ) {
222 res = BULLET_DATA[i][1];
223 break;
224 }
225 i++;
226 }
227 break;
228 }
229
230 default:
231 break;
232 }
233
234 return t;
235}
236
237
238//-----------------------------------------------------------------------------
239enum AccentType {
240 NoAccent,
241 Grave, Acute, Circumflex, Tilde, Diaeresis, Degree,
242 Macron, LowLine, Dot, Comma, DQuote, Cedilla
243};
244struct AccentData {
245 Unicode u;
246 AccentType type;
247};
248static const AccentData ACCENT_DATA[] = {
249 { 0x0060, Grave }, // `
250 { 0x00B4, Acute }, // '
251 { 0x005E, Circumflex }, // ^
252 { 0x007E, Tilde }, // ~
253 { 0x00A8, Diaeresis }, // ¨
254 { 0x00B0, Degree }, // °
255 { 0x00AF, Macron }, // macron
256 { 0x005F, LowLine }, // _
257 { 0x002E, Dot }, // .
258// { 0x00B7, Dot }, ????
259 { 0x002C, Comma }, // ,
260 { 0x0022, DQuote }, // "
261 { 0x00B8, Cedilla }, // cedilla
262 { 0x0000, NoAccent }
263};
264
265struct CombiData {
266 AccentType type;
267 Unicode upper, lower;
268};
269static const CombiData A_DATA[] = {
270 { Grave, 0x00C0, 0x00E0 },
271 { Acute, 0x00C1, 0x00E1 },
272 { Circumflex, 0x00C2, 0x00E2 },
273 { Tilde, 0x00C3, 0x00E3 },
274 { Diaeresis, 0x00C4, 0x00E4 },
275 { Degree, 0x00C5, 0x00E5 },
276 { Macron, 0x0100, 0x0101 },
277 { Dot, 0x0226, 0x0227 },
278 { NoAccent, 0x0000, 0x0000 }
279};
280static const CombiData C_DATA[] = {
281 { Acute, 0x0106, 0x0107 },
282 { Circumflex, 0x0108, 0x0109 },
283 { Dot, 0x010A, 0x010B },
284 { Cedilla, 0x00C7, 0x00E7 },
285 { NoAccent, 0x0000, 0x0000 }
286};
287static const CombiData E_DATA[] = {
288 { Grave, 0x00C8, 0x00E8 },
289 { Acute, 0x00C9, 0x00E9 },
290 { Circumflex, 0x00CA, 0x00EA },
291 { Diaeresis, 0x00CB, 0x00EB },
292 { Macron, 0x0112, 0x0113 },
293 { Dot, 0x0116, 0x0117 },
294 { Cedilla, 0x0228, 0x0229 },
295 { NoAccent, 0x0000, 0x0000 }
296};
297static const CombiData G_DATA[] = {
298 { Acute, 0x01F4, 0x01F5 },
299 { Circumflex, 0x011C, 0x011D },
300 { Comma, 0x0122, 0x0000 },
301 { NoAccent, 0x0000, 0x0000 }
302 };
303static const CombiData H_DATA[] = {
304 { Circumflex, 0x0124, 0x0125 },
305 { NoAccent, 0x0000, 0x0000 }
306};
307static const CombiData I_DATA[] = {
308 { Grave, 0x00CC, 0x0000 },
309 { Acute, 0x00CD, 0x0000 },
310 { Circumflex, 0x00CE, 0x0000 },
311 { Tilde, 0x0128, 0x0000 },
312 { Diaeresis, 0x00CF, 0x0000 },
313 { Macron, 0x012A, 0x0000 },
314 { Dot, 0x0130, 0x0000 },
315 { Cedilla, 0x012E, 0x0000 },
316 { NoAccent, 0x0000, 0x0000 }
317};
318static const CombiData J_DATA[] = {
319 { Circumflex, 0x0134, 0x0135 },
320 { NoAccent, 0x0000, 0x0000 }
321};
322static const CombiData K_DATA[] = {
323 { Comma, 0x0136, 0x0137 },
324 { NoAccent, 0x0000, 0x0000 }
325};
326static const CombiData L_DATA[] = {
327 { Acute, 0x0139, 0x013A },
328 { Comma, 0x013B, 0x013C },
329 { NoAccent, 0x0000, 0x0000 }
330};
331static const CombiData N_DATA[] = {
332 { Acute, 0x0143, 0x0144 },
333 { Comma, 0x0145, 0x0146 },
334 { Cedilla, 0x00D1, 0x00F1 },
335 { NoAccent, 0x0000, 0x0000 }
336};
337static const CombiData O_DATA[] = {
338 { Grave, 0x00D2, 0x00F2 },
339 { Acute, 0x00D3, 0x00F3 },
340 { Circumflex, 0x00D4, 0x00F4 },
341 { Tilde, 0x00D5, 0x00F5 },
342 { Diaeresis, 0x00D6, 0x00F6 },
343 { Macron, 0x014C, 0x014D },
344 { DQuote, 0x0150, 0x0151 },
345 { Dot, 0x022E, 0x022F },
346 { NoAccent, 0x0000, 0x0000 },
347};
348static const CombiData R_DATA[] = {
349 { Acute, 0x0154, 0x0155 },
350 { Comma, 0x0156, 0x0157 },
351 { NoAccent, 0x0000, 0x0000 }
352};
353static const CombiData S_DATA[] = {
354 { Acute, 0x015A, 0x015B },
355 { Circumflex, 0x015C, 0x015D },
356 { Comma, 0x0218, 0x0219 },
357 { Cedilla, 0x015E, 0x015F },
358 { NoAccent, 0x0000, 0x0000 }
359};
360static const CombiData T_DATA[] = {
361 { Comma, 0x021A, 0x021B },
362 { Cedilla, 0x0162, 0x0163 },
363 { NoAccent, 0x0000, 0x0000 }
364};
365static const CombiData U_DATA[] = {
366 { Grave, 0x00D9, 0x00F9 },
367 { Acute, 0x00DA, 0x00FA },
368 { Circumflex, 0x00DB, 0x00FB },
369 { Tilde, 0x0168, 0x0169 },
370 { Diaeresis, 0x00DC, 0x00FC },
371 { Macron, 0x016A, 0x016B },
372 { Dot, 0x016E, 0x016F },
373 { DQuote, 0x0170, 0x0171 },
374 { NoAccent, 0x0000, 0x0000 }
375};
376static const CombiData W_DATA[] = {
377 { Circumflex, 0x0174, 0x0175 },
378 { NoAccent, 0x0000, 0x0000 }
379};
380static const CombiData Y_DATA[] = {
381 { Acute, 0x00DD, 0x00FD },
382 { Circumflex, 0x0176, 0x0177 },
383 { Diaeresis, 0x0178, 0x00FF },
384 { Macron, 0x0232, 0x0233 },
385 { NoAccent, 0x0000, 0x0000 }
386};
387static const CombiData Z_DATA[] = {
388 { Acute, 0x0179, 0x017A },
389 { Dot, 0x017B, 0x017C },
390 { NoAccent, 0x0000, 0x0000 }
391};
392
393static const CombiData *LETTER_DATA[26] = {
394 A_DATA, 0, C_DATA, 0, E_DATA, 0, G_DATA, H_DATA, I_DATA, J_DATA,
395 K_DATA, L_DATA, 0, N_DATA, O_DATA, 0, 0, R_DATA, S_DATA, T_DATA,
396 U_DATA, 0, W_DATA, 0, Y_DATA, Z_DATA
397};
398
399static const CombiData AE_DATA[] = {
400 { Acute, 0x01FC, 0x01FD },
401 { Macron, 0x01E2, 0x01E3 },
402 { NoAccent, 0x0000, 0x0000 }
403};
404static const CombiData NULL_DATA[] = {
405 { Acute, 0x01FE, 0x01FF },
406 { NoAccent, 0x0000, 0x0000 }
407};
408static const CombiData I_LOWER_DATA[] = {
409 { Grave, 0x0000, 0x00EC },
410 { Acute, 0x0000, 0x00ED },
411 { Circumflex, 0x0000, 0x00EE },
412 { Tilde, 0x0000, 0x0129 },
413 { Diaeresis, 0x0000, 0x00EF },
414 { Macron, 0x0000, 0x012B },
415 { Dot, 0x0000, 0x0045 },
416 { Cedilla, 0x0000, 0x012F },
417 { NoAccent, 0x0000, 0x0000 }
418};
419
420struct SpecialCombiData {
421 Unicode upper, lower;
422 const CombiData *data;
423};
424static const SpecialCombiData SPECIAL_COMBI_DATA[] = {
425 { 0x00C6, 0x00E6, AE_DATA },
426 { 0x00D8, 0x00E8, NULL_DATA },
427 { 0x0000, 0x0131, I_LOWER_DATA },
428 { 0x0000, 0x0000, 0 }
429};
430
431Unicode checkCombi(Unicode letter, Unicode accent)
432{
433 // quick check
434 if ( !isAccent( type(accent) ) ) return 0;
435 if ( type(letter)!=Letter_CanHaveAccent ) return 0;
436
437 // find accent
438 uint i = 0;
439 for (;;) {
440 if ( ACCENT_DATA[i].u==0 ) return 0;
441 if ( ACCENT_DATA[i].u==accent ) break;
442 i++;
443 }
444
445 // find letter
446 const CombiData *data = 0;
447 bool upper = true;
448 if ( letter>='A' && letter<='Z' )
449 data = LETTER_DATA[letter-'A'];
450 else if ( letter>='a' && letter<='z' ) {
451 data = LETTER_DATA[letter-'a'];
452 upper = false;
453 } else {
454 uint k = 0;
455 for (;;) {
456 if ( SPECIAL_COMBI_DATA[k].data==0 ) return 0;
457 if ( letter==SPECIAL_COMBI_DATA[k].upper ) {
458 data = SPECIAL_COMBI_DATA[k].data;
459 break;
460 } else if ( letter==SPECIAL_COMBI_DATA[k].lower ) {
461 data = SPECIAL_COMBI_DATA[k].data;
462 upper = false;
463 break;
464 }
465 k++;
466 }
467 }
468 if ( data==0 ) return 0;
469
470 // find combi
471 uint l = 0;
472 while ( data[l].type!=NoAccent ) {
473 if ( data[l].type==ACCENT_DATA[i].type )
474 return (upper ? data[l].upper : data[l].lower);
475 l++;
476 }
477 return 0;
478}
479
480}