rtfimport_tokenizer.cpp

/*
   This file is part of the KDE project
   Copyright (C) 2001 Ewald Snel <ewald@rambo.its.tudelft.nl>
   Copyright (C) 2001 Tomasz Grobelny <grotk@poczta.onet.pl>
   Copyright (C) 2005 Tommi Rantala <tommi.rantala@cs.helsinki.fi>
 
   This library is free software; you can redistribute it and/or
   modify it under the terms of the GNU Library General Public
   License as published by the Free Software Foundation; either
   version 2 of the License, or (at your option) any later version.
*/
 
#include <kdebug.h>
 
#include "rtfimport_tokenizer.h"
 
 
RTFTokenizer::RTFTokenizer()
{
    tokenText.resize( 4113 );
    fileBuffer.resize( 4096 );
    infile = 0L;
}

void RTFTokenizer::open( TQFile *in )
{
    fileBufferPtr = 0L;
    fileBufferEnd = 0L;
    infile = in;
    type = RTFTokenizer::PlainText;
}
 
int RTFTokenizer::nextChar()
{
    if ( fileBufferPtr == fileBufferEnd ) {
        int n = infile->readBlock( fileBuffer.data(), fileBuffer.size() );
        fileBufferPtr = ( uchar* ) fileBuffer.data();
        fileBufferEnd = fileBufferPtr;
 
        if ( n <= 0 )
            return -1;
 
        fileBufferEnd = fileBufferPtr + n;
    }
    return *fileBufferPtr++;
}
 

void RTFTokenizer::next()
{
    int ch;
    value=0;
    if (!infile)
    return;
 
    do {
        int n = nextChar();
 
        if ( n <= 0 ) {
            ch = '}';
            break;
        }
 
        ch = n;
    }
    while (ch == '\n' || ch == '\r' && ch != 0);
 
    // Skip one byte for prepend '@' to destinations
    text = (tokenText.data() + 1);
    hasParam = false;
 
    uchar *_text = (uchar *)text;
 
 
    if (ch == '{')
    type = RTFTokenizer::OpenGroup;
    else if (ch == '}')
    type = RTFTokenizer::CloseGroup;
    else if (ch == '\\')
    {
    type = RTFTokenizer::ControlWord;
 
        int n = nextChar();
 
        if ( n <= 0 ) {
            // Return CloseGroup on EOF
            type = RTFTokenizer::CloseGroup;
            return;
        }
    ch = n;
 
    // Type is either control word or control symbol
    if ((ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z'))
    {
        int v = 0;
 
        // Read alphabetic string (command)
        while (_text < ( uchar* )tokenText.data()+tokenText.size()-3 && 
                  ((ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z')) )
        {
        *_text++ = ch;
 
                int n = nextChar();
                if ( n <= 0 ) {
                    ch = ' ';
                    break;
                }
                ch = n;
        }
 
        // Read numeric parameter (param)
        bool isneg = (ch == '-');
 
        if (isneg) {
                int n = nextChar();
                if ( n <= 0 ) {
                    type = RTFTokenizer::CloseGroup;
                    return;
                }
        ch = n;
        }
 
        while (ch >= '0' && ch <= '9') {
        v    = (10 * v) + ch - '0';
        hasParam = true;
 
                int n = nextChar();
 
                if ( n <= 0 )
                    n = ' ';
                ch = n;
            }
        value = isneg ? -v : v;
 
        // If delimiter is a space, it's part of the control word
        if (ch != ' ')
        {
        --fileBufferPtr;
        }
 
            *_text = 0; // Just put an end of string for the test, it can then be over-written again
            if ( !memcmp( tokenText.data()+1, "bin", 4 ) )
            {   // We have \bin, so we need to read the bytes
                kdDebug(30515) << "Token:" << tokenText << endl;
                if (value > 0)
                {
                    kdDebug(30515) << "\\bin" << value << endl;
                    type = RTFTokenizer::BinaryData;
                    binaryData.resize(value);
                    for (int i=0; i<value; i++)
                    {
                        int n = nextChar();
                        if ( n <= 0 ) {
                            type = RTFTokenizer::CloseGroup;
                            break;
                        }
 
                        binaryData[i] = n;
                    }
                }
            }
 
    }
    else if (ch=='\'')
    {
        // Got hex value, for example \'2d
 
        type = RTFTokenizer::ControlWord;
        *_text++ = ch;
 
        for(int i=0;i<2;i++)
        {
        int n = nextChar();
 
        if ( n <= 0 ) {
            if ( i == 0 ) {
                type = RTFTokenizer::CloseGroup;
                return;
            } else {
                        ch = ' ';
            break;
            }
        }
 
        ch = n;
 
        hasParam = true;
        value<<=4;
        value=value|((ch + ((ch & 16) ? 0 : 9)) & 0xf);
        }
        }
    else
    {
        type = RTFTokenizer::ControlWord;
        *_text++ = ch;
    }
    }
    else
    {
    type = RTFTokenizer::PlainText;
 
    // Everything until next backslash, opener or closer
    while ( ch != '\\' && ch != '{' && ch != '}' && ch != '\n' &&
        ch != '\r')
    {
        *_text++ = ch;
            if(fileBufferPtr >= fileBufferEnd)
                break;
        ch = *fileBufferPtr++;
    }
        if(fileBufferPtr < fileBufferEnd)
          --fileBufferPtr; // give back the last char
    }
    *_text++ = 0;
 
}