Tanl Linguistic Pipeline

morph/MorphSplitter/MorphSplitter.cpp

/*
**  Tanl
**  MorphSplitter/MorphSplitter.cpp
** ----------------------------------------------------------------------
**  Authors: Antonio Fuschetto (fuschett@di.unipi.it), University of Pisa
**           Giuseppe Attardi (attardi@di.unipi.it), University of Pisa
** ----------------------------------------------------------------------
**  Copyright (c) 2009  Giuseppe Attardi (attardi@di.unipi.it).
** ----------------------------------------------------------------------
**
**  This file is part of Tanl.
**
**  Tanl is free software; you can redistribute it and/or modify it
**  under the terms of the GNU General Public License, version 3,
**  as published by the Free Software Foundation.
**
**  Tanl is distributed in the hope that it will be useful,
**  but WITHOUT ANY WARRANTY; without even the implied warranty of
**  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
**  GNU General Public License for more details.
**
**  You should have received a copy of the GNU General Public License
**  along with this program.  If not, see <http://www.gnu.org/licenses/>.
**  ----------------------------------------------------------------------
*/

// Local
#include "MorphSplitter.h"

// Standard
#include <sstream>

using namespace std;

namespace Tanl {

map<string, string> MorphSplitter::clitics;

MorphSplitter::MorphSplitter()
{
  if (!clitics.size()) {
    clitics["ce"]   = "PC1np";
    clitics["ci"]   = "PC1np";
    clitics["gli"]  = "PC3ms";
    clitics["glie"] = "PC3ms";
    clitics["la"]   = "PC3fs";
    clitics["le"]   = "PC3fn";
    clitics["li"]   = "PC3mp";
    clitics["lo"]   = "PC3ms";
    clitics["me"]   = "PC1ns";
    clitics["mi"]   = "PC1ns";
    clitics["se"]   = "PC3nn";
    clitics["si"]   = "PC3nn";
    clitics["te"]   = "PC2ns";
    clitics["ti"]   = "PC2ns";
    clitics["ve"]   = "PC2np";
    clitics["vi"]   = "PC2np";
    clitics["ne"]   = "PCnn";
  }
}

// ======================================================================

Enumerator<vector<Token*>*>* MorphSplitter::pipe(istream& is)
{
  return new MorphStream(is);
}

Enumerator<vector<Token*>*>* MorphSplitter::pipe(Enumerator<vector<Token*>*>& tve)
{
  return new MorphPipe(tve);
}

// ======================================================================
// Auxiliary functions.

static void splitPos(const string& pos, string& postag, string& feats)
{
  postag.clear();

  string morph;
  stringstream oss;
  for (string::size_type i = 0; i < pos.size(); ++i)
    if (isupper(pos[i]))
      postag += pos[i];
    else
      morph += pos[i];

  if (morph.size()) {
    if (postag[0] == 'V') { // verb
      if (morph[0] == 'p') { // participle
        if (morph[1] == 'p') // present participle
          oss << "num=" << morph[2] << "|mod=" << morph[0];
        else // past participle
          oss << "num=" << morph[3] << "|mod=" << morph[0] << "|gen=" << morph[2];
      } else if (morph[0] == 'm' || morph[0] == 'd') // imperative or conditional
        oss << "num=" << morph[2] << "|per=" << morph[1] << "|mod=" << morph[0] << "|ten=p";
      else if (morph[0] == 'f' || morph[0] == 'g') // infinite or gerundive
        oss << "mod=" << morph[0];
      else // other (indicative or conjunctive)
        oss << "num=" << morph[3] << "|per=" << morph[2] << "|mod=" << morph[0] << "|ten=" << morph[1];
    } else if ((postag == "PC" || postag == "PE") && isdigit(morph[0])) // personal pronoun
      oss << "num=" << morph[2] << "|per=" << morph[0] << "|gen=" << morph[1];
    else // other
      oss << "num=" << morph[1] << "|gen=" << morph[0];
  }
  feats = oss.str();
}

static void tokenizeText(const string& text, vector<string>& tokens,
                         const char* delimiters)
{
  // FIXME: revwrite using strtok_r().
  tokens.clear();

  string::size_type lastPos = text.find_first_not_of(delimiters, 0);
  string::size_type pos = text.find_first_of(delimiters, lastPos);

  while (string::npos != pos || string::npos != lastPos) {
    tokens.push_back(text.substr(lastPos, pos - lastPos));
    lastPos = text.find_first_not_of(delimiters, pos);
    pos = text.find_first_of(delimiters, lastPos);
  }
}

vector<Token*>* MorphSplitter::split(vector<Token*>* inSentence)
{
  vector<Token*>* outSentence = new vector<Token*>;
  unsigned tokenNo = 1;
  FOR_EACH (vector<Token*>, *inSentence, it) {
    Token* token = *it;

    string form = *token->get("FORM");
    const string& pos = *token->get("POS"); // input tag
    string lemma = *token->get("LEMMA");

    string postag, feats;

    // Watch for unknown lemma.
    // e.g.: de-condizionarlo   Vfc   <unknown>
    bool unknown = false;
    if (lemma == "<unknown>" || (form.find('-') && form == lemma)) {
      unknown = true;
      lemma = form;
    }

    // Check presence of clitics.
    if (pos[pos.size() - 1] != 'c' || unknown) {
      splitPos(pos, postag, feats);

      token->set("ID", tokenNo++);
      token->set("FORM", form);
      token->set("CPOSTAG", postag.substr(0, 1));
      token->set("POSTAG", postag);
      token->set("FEATS", feats);

      outSentence->push_back(token);
      continue;
    }

    // Watch for multiple lemmas.
    // e.g.: rifugiatisi   Vpsmpc   rifugiare-si|rifugiarsi-si
    vector<string> tokens;
    vector<string> lemmas;
    tokenizeText(lemma, lemmas, "|");
    tokenizeText(lemmas[0], tokens, "-");

    // False clitic, assume it is a proper name.
    // e.g.: Zevi   Vm2pc   <unknown>
    if (tokens.size() == 1) {
      token->set("ID", tokenNo++);
      token->set("FORM", form);
      token->set("LEMMA", lemma);
      token->set("CPOSTAG", "S");
      token->set("POSTAG", "SP");
      token->set("FEATS", "_");
      outSentence->push_back(token);
      continue;
    }

    // Output verbal root.
    // e.g.: leggervela   Vfc   leggere-ve-la
    for (int i = 1; i < tokens.size(); ++i)
      form = form.substr(0, form.size() - tokens[i].size());
    splitPos(pos.substr(0, pos.size() - 1), postag, feats);

    token->set("ID", tokenNo++);
    token->set("FORM", form + "-");
    token->set("LEMMA", tokens[0]);
    token->set("CPOSTAG", postag.substr(0, 1));
    token->set("POSTAG", postag);
    token->set("FEATS", feats);

    outSentence->push_back(token);

    // Output all clitics.
    for (int i = 1; i < tokens.size(); ++i) {
      form = i < tokens.size() - 1 ? tokens[i] + "-" : tokens[i];

      string& pos = MorphSplitter::clitics[tokens[i]];
      splitPos(tokens[i], postag, feats);

      AttributeIndex* ai = new AttributeIndex;
      Token* otherToken = new Token(form, ai);
      otherToken->set("ID", tokenNo++);
      otherToken->set("FORM", form);
      otherToken->set("LEMMA", pos);
      // FIXME: only for Tanl has single char corase POS
      otherToken->set("CPOSTAG", postag.substr(0, 1));
      otherToken->set("POSTAG", postag);
      otherToken->set("FEATS", feats);

      outSentence->push_back(otherToken);
    }
  }
  return outSentence;
}

// ======================================================================

MorphStream::MorphStream(std::istream& is) :
  is(is)
{ }

bool MorphStream::MoveNext()
{
  sentence.clear();

  string line;
  vector<string> attributes;

  while (!is.eof()) {
    getline(is, line);

    tokenizeText(line, attributes, "\t ");
    if (!attributes.size()) {
      if (sentence.size())
        return true;
    } else {
      AttributeIndex* ai = new AttributeIndex;
      ai->insert("FORM");

      Token* token = new Token(attributes[0], ai);
      token->set("FORM", attributes[0]);
      token->set("POS", attributes[1]);
      token->set("LEMMA", attributes[2]);

      sentence.push_back(token);
    }
  }
  return false;
}

vector<Token*>* MorphStream::Current()
{
  return MorphSplitter::split(&sentence);
}

// ======================================================================

MorphPipe::MorphPipe(Enumerator<vector<Token*>*>& tve) :
  tve(tve)
{ }

bool MorphPipe::MoveNext()
{
  return tve.MoveNext();
}

vector<Token*>* MorphPipe::Current()
{
  vector<Token*>* inSentence = tve.Current();
  vector<Token*>* sentence = MorphSplitter::split(inSentence);
  delete inSentence;
  return sentence;
}

} // namespace Tanl

using namespace Tanl;

int main(int argc, char* argv[])
{
  if (argc > 1) {
    cerr << "Usage: MorphSplitter < inputFile > outputFile" << endl
         << "   splits morphed pos tags and clitics." << endl
         << "   inputFile       file in Tanl POS format, one token per line:" << endl
         << "                   form\\tlemma\\tpos+morph" << endl
         << "   outputFile      file in Tanl POS format, one token per line:" << endl
         << "                   form\\tlemma\\tcpos\\tpos\\tfeats" << endl;
    return -1;
  }

  Enumerator<vector<Token*>*>* pipe = MorphSplitter().pipe();
  while (pipe->MoveNext()) {
    int id = 1;
    vector<Token*>* sentence = pipe->Current();
    for (vector<Token*>::const_iterator it = sentence->begin(); it != sentence->end(); ++it) {
      Token* token = *it;
      cout << id++ << "\t" << *token->get("FORM") << "\t"
           << *token->get("LEMMA") << "\t" << *token->get("CPOSTAG") << "\t"
           << *token->get("POSTAG") << "\t" << *token->get("FEATS") << endl;

      delete token;
    }
    cout << endl;
    delete sentence;
  }
  delete pipe;
  return 0;
}

 

Copyright © 2005-2011 G. Attardi. Generated on 3 Mar 2011 by doxygen 1.6.1.