Tanl Linguistic Pipeline

Corpus/SentenceReader.cpp

/*
**  Tanl
**  Corpus/SentenceReader.cpp
**  ----------------------------------------------------------------------
**  Copyright (c) 2005  Giuseppe Attardi (attardi@di.unipi.it).
**  ----------------------------------------------------------------------
**
**  This file is part of Tanl.
**
**  Tanl is free software; you can redistribute it and/or modify it
**  under the terms of the GNU General Public License, version 3,
**  as published by the Free Software Foundation.
**
**  Tanl is distributed in the hope that it will be useful,
**  but WITHOUT ANY WARRANTY; without even the implied warranty of
**  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
**  GNU General Public License for more details.
**
**  You should have received a copy of the GNU General Public License
**  along with this program.  If not, see <http://www.gnu.org/licenses/>.
**  ----------------------------------------------------------------------
*/

#include "SentenceReader.h"

// standard
#include <iostream>
#include <fstream>
#include <iomanip>

// library
#include "include/unordered_map.h"
#include "text/Utf8Utils.h"

// local
#include "Corpus.h"

using namespace std;
using namespace Tanl::Text;

namespace Tanl {

//======================================================================
// SentenceReader

SentenceReader::SentenceReader(istream* is, Corpus* corpus) :
  is(is),
  corpus(corpus)
{ }

void SentenceReader::reset()
{
  line.clear();
}

// Generic tab format reader
static RegExp::Pattern reTab("([^ \t\n]+)");

bool SentenceReader::MoveNext()
{
  sentence = new Sentence();
  vector<char const*>& names = corpus->index.names;
  vector<int> preds;            // ids of predicates in sentence
  int id = 1;
  RegExp::MatchGroups match(2);

  while (getline(*is, line) && line.size()) {
    Attributes attributes(&corpus->index);
    string form;
    int head = 0;
    string deprel;
    int fields = corpus->tokenFields.size();
    TokenLinks links;
    unordered_map<string, int> linkMap;
    int argNo = 0;
    int i = 0;
    char const* cur = line.c_str();
    char const* end = cur + line.size();
    while (reTab.match(cur, end, match) > 0) {
      TokenField const& tf = corpus->tokenFields[i];
      char const* fieldStart = cur + match[1].first;
      int fieldLen = match[1].second - match[1].first;
      string field(fieldStart, fieldLen);
      // clear empty fields
      if (field == tf.default_)
        field = "";
      // discard IGNORE fields
      if (tf.use != TokenField::ignore) {
        if (!tf.link.empty()) {
          // Got target of link: create even when value is missing
          int head = field.empty() ? -1 : atoi(field.c_str());
          if (linkMap.find(tf.link) == linkMap.end()) {
            // Create link and assign position in LinkMap 
            linkMap[tf.link] = links.size();
            links.push_back(TokenLink(head));
          } else
            // fill head in previously created link
            links[linkMap[tf.link]].head = head;
        } else if (!tf.label.empty()) {
          if (field.empty())
            --argNo;
          else {
            // Got label of link
            if (linkMap.find(tf.label) == linkMap.end()) {
              // Create link with dummy target and assign position in LinkMap 
              linkMap[tf.label] = links.size();
              // use negative value to distinguish from real target
              links.push_back(TokenLink(--argNo, field.c_str()));
            } else
              // fill label in previously created link
              links[linkMap[tf.label]].label = field;
          }
        } else {
          attributes[i] = field;
#         ifdef CONLL08
          if (useGold && i == 7 && !attributes[3].empty())
            attributes[7] = attributes[3];
#         endif
          switch (tf.role) {
          case TokenField::form:
            form = field; break;
          case TokenField::predicate:
            if (!field.empty())
              preds.push_back(id);
            break;
          }
        }
      }
      i++;
      cur += match[0].second;
      if (i == fields || cur == end) {
        // skip extra fields
        break;
      }
    }
    Token* token = new Token(form, attributes, links);
    sentence->push_back(token);
  }
  if (sentence->empty()) {
    delete sentence;
    sentence = 0;
    return false;
  }
  // fix ARG links
  for (Sentence::const_iterator sit = sentence->begin();
       sit != sentence->end(); ++sit) {
    for (TokenLinks::iterator tit = (*sit)->links.begin();
         tit != (*sit)->links.end(); ++tit) {
      if (tit->head < 0)
        tit->head = preds[-tit->head - 1];
    }
  }
  return true;
}

Sentence* SentenceReader::Current()
{
  return sentence;
}

} // namespace Tanl

 

Copyright © 2005-2011 G. Attardi. Generated on 3 Mar 2011 by doxygen 1.6.1.