Tanl Linguistic Pipeline

split/Tokenizer/Tokenizer.cpp

/*
**  Tanl
**  split/Tokenizer/Tokenizer.cpp
**  ----------------------------------------------------------------------
**  Copyright (c) 2008  Giuseppe Attardi (attardi@di.unipi.it).
**  ----------------------------------------------------------------------
**
**  This file is part of Tanl.
**
**  Tanl is free software; you can redistribute it and/or modify it
**  under the terms of the GNU General Public License, version 3,
**  as published by the Free Software Foundation.
**
**  Tanl is distributed in the hope that it will be useful,
**  but WITHOUT ANY WARRANTY; without even the implied warranty of
**  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
**  GNU General Public License for more details.
**
**  You should have received a copy of the GNU General Public License
**  along with this program.  If not, see <http://www.gnu.org/licenses/>.
**  ----------------------------------------------------------------------
*/

#include "Tokenizer.h"

using namespace std;

namespace Tanl {


TokenizerPipe::TokenizerPipe() :
  qlex((QUEX_TYPE_CHARACTER*)0, 0, 0, "UTF-8"),
  len(0),
  currContext(0)
{
  qlex.token_p_switch(&token);
}

TokenizerPipe::~TokenizerPipe()
{
  if (currContext)
    currContext->decRef();
}

Enumerator<Token*>* Tokenizer::pipe(istream& is)
{
  return new TokenizerPipe(*this, is);
}

Enumerator<Token*>* Tokenizer::pipe(Enumerator<std::string*>& se)
{
  return new TokenizerPipeEnum(*this, se);
}

Enumerator<Token*>* Tokenizer::pipe(PyObject* pit)
{
  return new TokenizerPipePython(*this, pit);
}

TokenizerPipe::TokenizerPipe(Tokenizer& tokenizer, istream& is) :
  qlex(&is, "UTF-8"),
  len(0),
  currContext(0)
{
  qlex.token_p_switch(&token);
}

bool TokenizerPipe::MoveNext()
{
  // (*) get next token from the token stream
  const QUEX_TYPE_TOKEN_ID TokenID = qlex.receive();

  // (*) check against 'termination'
  if (TokenID == QUEX_TKN_TERMINATION)
      return false;
  if (TokenID == QUEX_TKN_EOS) {
    len = 0;
  } else if (TokenID == QUEX_TKN_XML) {
    string xmlTag = token.utf8_text();
    if (xmlTag[1] == '/') { // closing tag
      if (currContext) {
        Context* prev = currContext->parent;
        if (prev)
          prev->incRef();       // we are going to reference it, preserve from deletion
        currContext->decRef();
        currContext = prev;
      }
      // else
      // std::cerr << "XML tag balancing error!" << std::endl;
    } else { // opening tag
      Context* prevContext = currContext; // delay decRef()
      currContext = new Context(xmlTag, currContext);
      currContext->incRef();    // we keep a reference
      if (prevContext)
        prevContext->decRef();  // free after storing in new Context
    }
    return MoveNext();
  } else
    len++;
  return true;
}

// Create an AttrtibuteIndex with just the FORM attribute.
static AttributeIndex* createSimpleAI()
{
  AttributeIndex* ai = new AttributeIndex();
  ai->insert("FORM");
  return ai;
}

AttributeIndex* simpleAI = createSimpleAI();

Token* TokenizerPipe::Current()
{
  string form = token.utf8_text();
  Token* t =  new Token(form, simpleAI, currContext);
  t->set("FORM", form);         // FIXME
  return t;
}

// ======================================================================
// Reading from a string Enumerator

static const QUEX_TYPE_CHARACTER eos[] = { (QUEX_TYPE_CHARACTER)'\n', 0 };

TokenizerPipeEnum::TokenizerPipeEnum(Tokenizer& tokenizer, Enumerator<std::string*>& senum) :
  senum(senum)
{
  char const* null = "";
  qlex.buffer_fill_region_append((void*)null, (void*)null);
  qlex.token_p_switch(&token);
}

bool TokenizerPipeEnum::MoveNext()
{
  while (true) {
    const QUEX_TYPE_TOKEN_ID TokenID = qlex.receive();
    // reload if previous item was consumed
    if (TokenID == QUEX_TKN_TERMINATION) {
      if (len) { len=0; token.set(QUEX_TKN_EOS,eos); return true; } // hack to handle EOS
      if (senum.MoveNext()) {
        string& line = *senum.Current();
        qlex.buffer_fill_region_append_conversion(line.begin().base(), line.end().base());
        continue;
      } else
        return false;
    }
    if (TokenID == QUEX_TKN_EOS) {
      len = 0;
    } else if (TokenID == QUEX_TKN_XML) {
      string xmlTag = token.utf8_text();
      if (xmlTag[1] == '/') { // closing tag
        if (currContext) {
          Context* prev = currContext->parent;
          if (prev)
            prev->incRef();     // we are going to reference it, preserve from deletion
          currContext->decRef();
          currContext = prev;
        }
        // else
        // std::cerr << "XML tag balancing error!" << std::endl;
      } else { // opening tag
        Context* prevContext = currContext; // delay decRef()
        currContext = new Context(xmlTag, currContext);
        currContext->incRef();  // we keep a reference
        if (prevContext)
          prevContext->decRef();        // free after storing in new Context
      }
      continue;
    } else
      len++;
    return true;
  }
}

// ======================================================================
// Reading from a Python iterator

TokenizerPipePython::TokenizerPipePython(Tokenizer& tokenizer, PyObject* pit) :
  pit(pit)
{
  Py_INCREF(pit);
  char const* null = "";
  qlex.buffer_fill_region_append((void*)null, (void*)null);
  qlex.token_p_switch(&token);
}

TokenizerPipePython::~TokenizerPipePython()
{
  Py_DECREF(pit);
}

bool TokenizerPipePython::MoveNext()
{
  while (true) {
    const QUEX_TYPE_TOKEN_ID TokenID = qlex.receive();
    // reload if previous item was consumed
    if (TokenID == QUEX_TKN_TERMINATION) {
      if (len) { len=0; token.set(QUEX_TKN_EOS,eos); return true; } // hack to handle EOS
      PyObject* next = PyIter_Next(pit);
      if (next) {
        char* line = PyString_AsString(next);
        qlex.buffer_fill_region_append_conversion(line, line + strlen(line));
        continue;
      } else
        return false;
    }
    if (TokenID == QUEX_TKN_EOS) {
      len = 0;
    } else if (TokenID == QUEX_TKN_XML) {
      string xmlTag = token.utf8_text();
      if (xmlTag[1] == '/') { // closing tag
        if (currContext) {
          Context* prev = currContext->parent;
          if (prev)
            prev->incRef();     // we are going to reference it, preserve from deletion
          currContext->decRef();
          currContext = prev;
        }
        // else
        // std::cerr << "XML tag balancing error!" << std::endl;
      } else { // opening tag
        Context* prevContext = currContext; // delay decRef()
        currContext = new Context(xmlTag, currContext);
        currContext->incRef();  // we keep a reference
        if (prevContext)
          prevContext->decRef();        // free after storing in new Context
      }
      continue;
    } else
      len++;
    return true;
  }
}

} // namespace Tanl

#ifdef UNIT_TEST

using namespace Tanl;

static char* lines[] = { "la prima riga", "la seconda", "ultima riga", 0 };

struct EnumTest : public Enumerator<string*>
{
  EnumTest() :
    current(-1)
  {
    for (char** scan = lines; *scan; ++scan)
      strings.push_back(*scan);
  }
  bool MoveNext() { return ++current < strings.size(); }
  string* Current() { return &strings[current]; }

  int current;
  vector<string>        strings;
};

int main(int argc, char* argv[])
{
  EnumTest et;
  Tokenizer l;
  Enumerator<Token*>* lpe = l.pipe(et);
  while (lpe->MoveNext())
    cout << *lpe->Current() << endl;
  // test python
  Py_Initialize();
  PyObject* list = PyList_New(0);
  for (char** scan = lines; *scan; ++scan)
    PyList_Append(list, PyString_FromString(*scan));
  PyObject* pit = PyObject_GetIter(list);
  Enumerator<Token*>* lpp = l.pipe(pit);
  while (lpp->MoveNext())
    cout << *lpp->Current() << endl;
  Py_DECREF(pit);
  Py_DECREF(list);
  Py_Finalize();
  return 0;
}
#endif

 

Copyright © 2005-2011 G. Attardi. Generated on 3 Mar 2011 by doxygen 1.6.1.