Tanl Linguistic Pipeline

Corpus/Corpus.cpp

/*
**  Tanl
**  Corpus/Corpus.cpp
** ----------------------------------------------------------------------
**  Copyright (c) 2008  Giuseppe Attardi (attardi@di.unipi.it).
** ----------------------------------------------------------------------
**
**  This file is part of Tanl.
**
**  Tanl is free software; you can redistribute it and/or modify it
**  under the terms of the GNU General Public License, version 3,
**  as published by the Free Software Foundation.
**
**  Tanl is distributed in the hope that it will be useful,
**  but WITHOUT ANY WARRANTY; without even the implied warranty of
**  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
**  GNU General Public License for more details.
**
**  You should have received a copy of the GNU General Public License
**  along with this program.  If not, see <http://www.gnu.org/licenses/>.
**  ----------------------------------------------------------------------
*/

#include "Corpus.h"
#include "text/XmlReader.h"

// standard
#include <sstream>

// library
#include "io/File.h"

using namespace std;

namespace Tanl {

// ======================================================================
// Corpus

Corpus::Corpus(Language const& lang, char const* formatFile) :
  language(lang)
{
  CorpusFormat* cf = parseFormat(formatFile);
  if (cf) {
    tokenFields = cf->tokenFields;
    index = cf->index;
    delete cf;          // save index from destruction
  } else
    throw CorpusFormatError(string("Reading file: ") + formatFile);
}

Corpus* Corpus::create(Language const& language, char const* inputFormat)
{
  Corpus* corpus;
  CorpusFactory* cfactory = CorpusMap::get(inputFormat);
  if (cfactory)
    corpus = cfactory(language, 0); // builtin corpus
  else if (IXE::io::File(inputFormat).exists()) {
    CorpusFormat* inFormat = Corpus::parseFormat(inputFormat);
    if (!inFormat) {
      cerr << "Bad format file: " << *inputFormat << endl;
      return 0;
    }
    cfactory = CorpusMap::get(inFormat->name.c_str());
    if (cfactory)
      corpus = cfactory(language, inFormat);
    else
      corpus = new Corpus(language, *inFormat); // default tab format
  }
  return corpus;
}

Corpus* Corpus::create(char const* language, char const* inputFormat)
{
  Language const* lang = Language::get(language);
  if (!lang)
    lang = Language::get("en");
  return create(*lang, inputFormat);
}

CorpusFormat* Corpus::parseFormat(char const* formatFile)
{
  ifstream fmt(formatFile);
  return parseFormat(fmt);
}

CorpusFormat* Corpus::parseFormat(istream& fmt)
{
  Tanl::XML::XmlReader reader(fmt);
  CorpusFormat* format = new CorpusFormat();
  AttributeIndex& index = format->index;
  while (reader.Read()) {
    switch (reader.NodeType) {
    case Tanl::XML::Element:
      string& name = reader.Name;
      if (name == "CorpusFormat") {
        if (reader.MoveToFirstAttribute()) {
          do {
            string& name = reader.Name;
            string& value = reader.Value;
            if (name == "name") {
              format->name = value;
            }
          } while (reader.MoveToNextAttribute());
        }
      } else if (name == "field") {
        // read field
        TokenField field;
        if (reader.MoveToFirstAttribute()) {
          do {
            string& name = reader.Name;
            string& value = reader.Value;
            if (name == "name") {
              field.name = value;
              index.insert(value.c_str()); // names of attributes go into index
            } else if (name == "use")
              field.use = (value == "INPUT" ? TokenField::input :
                           (value == "OUTPUT" ? TokenField::output :
                            (value == "ECHO" ? TokenField::echo :
                             (value == "IGNORE" ? TokenField::ignore :
                              TokenField::input))));
            else if (name == "value")
              field.value = (value == "STRING" ? TokenField::string :
                             (value == "INTEGER" ? TokenField::integer : TokenField::string));
            else if (name == "role")
              field.role = (value == "FORM" ? TokenField::form :
                            (value == "HEAD" ? TokenField::head :
                             (value == "DEPREL" ? TokenField::deprel :
                              (value == "PREDICATE" ? TokenField::predicate :
                               TokenField::none))));
            else if (name == "link")
              field.link = value;
            else if (name == "label")
              field.label = value;
            else if (name == "default")
              field.default_ = value;
          } while (reader.MoveToNextAttribute());
        }
        format->tokenFields.push_back(field);
      } else {
        delete format;
        return 0;
      }
    }
  }
  return format;
}

SentenceReader* Corpus::sentenceReader(istream* is) {
  return new SentenceReader(is, this);
}

void Corpus::print(ostream& os, Sentence const& sent) const
{
  for (Sentence::const_iterator tit = sent.begin(); tit != sent.end(); ++tit) {
    for (Attributes::const_iterator fit = (*tit)->attributes.begin();
         fit != (*tit)->attributes.end(); ++fit)
      os << ((*(*fit).second == "") ? "_" : *(*fit).second) << "\t";
    os << endl;
  }
}

std::string Corpus::toString(Sentence const& sent) const
{
  std::ostringstream ss;
  FOR_EACH (Sentence, sent, tit) {
    (*tit)->printTab(ss, *this);
    ss << endl;
  }
  return ss.str().c_str();
}

// ======================================================================
// Corpus Factory

std::map<char const*, CorpusFactory*>& CorpusMap::get()
{
  static std::map<char const*, CorpusFactory*> corpusFor;
  return corpusFor;
}

CorpusFactory* CorpusMap::get(char const* type)
{
  std::map<char const*, CorpusFactory*>& corpusMap = get();
  map<char const*, CorpusFactory*>::const_iterator rit = corpusMap.find(type);
  return (rit == corpusMap.end()) ? 0 : rit->second;
}

} // namespace Tanl

 

Copyright © 2005-2011 G. Attardi. Generated on 3 Mar 2011 by doxygen 1.6.1.