Tanl Linguistic Pipeline

tag/SST/SST.cpp

/*
**  Tanl
**  SST/SST.cpp
**  ----------------------------------------------------------------------
**  Copyright (c) 2005  Giuseppe Attardi (attardi@di.unipi.it).
**  ----------------------------------------------------------------------
**
**  This file is part of Tanl.
**
**  Tanl is free software; you can redistribute it and/or modify it
**  under the terms of the GNU General Public License, version 3,
**  as published by the Free Software Foundation.
**
**  Tanl is distributed in the hope that it will be useful,
**  but WITHOUT ANY WARRANTY; without even the implied warranty of
**  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
**  GNU General Public License for more details.
**
**  You should have received a copy of the GNU General Public License
**  along with this program.  If not, see <http://www.gnu.org/licenses/>.
**  ----------------------------------------------------------------------
*/

// IXE
#include "conf/conf_bool.h"
#include "conf/conf_int.h"
#include "conf/conf_float.h"
#include "conf/conf_string.h"
#include "io/File.h"

#include "SST.h"
#include "SstEventStream.h"

// MaxEntropy
#define MODEL GIS
#if (MODEL == GIS)
#include "classifier/GIS.h"
#else
#include "classifier/LBFGS.h"
#endif

// standard
#include <assert.h>
#include <math.h>
#include <limits>

using namespace std;
using namespace Tanl::Text;
using namespace Tanl::Text::RegExp;
using namespace Tanl::Classifier;
using namespace IXE;            // for conf<>

namespace Tanl {
namespace SST {

// Configuration Parameters

conf<string>    resourceDir("ResourceDir", "Resource");
conf<string>    language("Language", "italian"); //Changed from english to to italian
conf<int>       cutoff("Cutoff", 2);
conf<int>       iter("Iterations", 100);
conf<float>     alpha("Alpha", 0.0);
conf<bool>      verbose("Verbose", false);

SST::SST(char const* modelFile, char const* configFile) :
  model(0),
  resources(*language)
{
  // read configuration file
  if (configFile && IXE::io::File(configFile).exists())
    IXE::Configuration::load(configFile);
  if (IXE::io::File(modelFile).exists())
    model = new MaxEnt(modelFile);
  resources.load(*resourceDir);
}

SST::~SST() { delete model; }

Text::RegExp::Pattern datePat("[12][0-9][0-9][0-9]-[0-9]+-[0-9]+");

bool checkDate(std::vector<Token*>& sent)
{
  // look for a line with a date
  for (int i = 0; i < sent.size(); i++)
    if (datePat.test(sent[i]->form))
      return true;
  return false;
}

void SST::train(SentenceReader* sentenceReader, char const* modelFile)
{
  MODEL trainer(iter, cutoff, alpha);
  trainer.verbose = verbose;

  // One SstEventStream for each document, since it maintains
  // per document information:
  SstEventStream eventStream(resources);

  while (sentenceReader->MoveNext()) {
    std::vector<Token*>* sent = sentenceReader->Current();
    if (sent->size() == 1 && (*sent)[0]->form == "-DOCSTART-") { // CoNLL03
      FOR_EACH (std::vector<Token*>, *sent, sit)
        delete *sit;
      delete sent;
      eventStream.reset();
      continue;
    }
    eventStream.analyze(sent);
    trainer.read(eventStream);
    // delete sent's tokens
    FOR_EACH (std::vector<Token*>, *sent, sit)
      delete *sit;
    delete sent;
  }

  trainer.train();

  // dump the model
  trainer.save(modelFile);
}

Enumerator<std::vector<Token*>*>*       SST::pipe(Enumerator<std::vector<Token*>*>& sen)
{
  return new SSTPipe(*this, sen);
}

// ======================================================================

static char const* superSenses[] = {
  "adj.all", "adj.pert", "adj.ppl", "adv.all", "noun.Tops", "noun.act",
  "noun.animal", "noun.artifact", "noun.attribute", "noun.body",
  "noun.cognition", "noun.communication", "noun.event", "noun.feeling",
  "noun.food", "noun.group", "noun.location", "noun.motive",
  "noun.object", "noun.other", "noun.person", "noun.phenomenon",
  "noun.plant", "noun.possession", "noun.process", "noun.quantity",
  "noun.relation", "noun.shape", "noun.state", "noun.substance",
  "noun.time", "verb.body", "verb.change", "verb.cognition",
  "verb.communication", "verb.competition", "verb.consumption",
  "verb.contact", "verb.creation", "verb.emotion", "verb.motion",
  "verb.perception", "verb.possession", "verb.social",
  "verb.stative", "verb.weather"
};

static const int nSenses = sizeof(superSenses)/sizeof(char const*);
static const int ssTags = 1 + 2 * nSenses;
static char const* tags[ssTags];

static int initTags()
{
  tags[0] = "O";
  // create IOB tags
  char buffer[256];
  for (int i = 0; i < nSenses; i++) {
    sprintf(buffer, "B-%s", superSenses[i]);
    tags[1 + i] = strdup(buffer);
    sprintf(buffer, "I-%s", superSenses[i]);
    tags[1 + nSenses + i] = strdup(buffer);
  }
  return 0;
}

static int dummyInitVar = initTags();

// ======================================================================

SSTPipe::SSTPipe(SST& sst, Enumerator<std::vector<Token*>*>& se) :
  sst(sst),
  se(se),
  eventStream(sst.resources)
{
  // create map of states to tags (classifier orders them differently).
  int nTags = sst.model->NumOutcomes();
  for (int i = 0; i < nTags; ++i) {
    char const* tag = sst.model->OutcomeName(i);
    for (int j = 0; j < ssTags; ++j)
      if (!strcmp(tag, tags[j])) {
        outcomeId[j] = i;
        break;
      }
  }
}

bool SSTPipe::MoveNext()
{
  while (se.MoveNext()) {
    std::vector<Token*>* sent = se.Current();
    if (sent->size() == 1 && (*sent)[0]->form == "-DOCSTART-") { // CoNLL03
      eventStream.reset();
      delete sent;
    } else
      return true;
  }
  return false;
}

// dynamic programming
std::vector<Token*>* SSTPipe::Current()
{
  std::vector<Token*>* sentence = se.Current();
  int len = sentence->size();
  // drop gold tags
  for (int i = 0; i < len; i++)
    (*sentence)[i]->set("SSTAG", "");
  eventStream.analyze(sentence);
  // collect HMM parameters
  int nTags = sst.model->NumOutcomes();
  string predTags[len];
  double best[len];
  best[0] = -numeric_limits<double>::infinity(); // value to beat
  ClassID O = sst.model->OutcomeID("O"); // Outside
  static const int beam = 5;             // beam size
  string tag[len];
  double p[beam][nTags];
  int n = 0;                    // current token

  while (eventStream.hasNext()) {
    Classifier::Event* event = eventStream.next();
# ifndef NO_BEAM_SEARCH
    sst.model->estimate(event->features, p[n % beam]);
    delete event;
    // accumulate enough results
    if (n < beam-1 && n != len-1) {
      n++;
      continue;
    }

    // handle first block, up to beam, with a cycle, the others individually
    int iFirst = (n < beam) ? 1 : n+1;
    for (int i = iFirst; i <= n+1; i++) {
      // consider ngrams up to 5 long
      // compute best[i] = best[i-5] best[i-4] ... best[i-1]
      for (int j = i - 1; j >= 0 && j >= i-5; j--) {
        int w = i - j;          // ngram size

        // ngram start: j, ngram last: i-1
        for (int k = j; k < i; k++) {
          if (k == j)           // left boundary
            tag[k] = "B";       // Begin
          else
            tag[k] = "I";       // Inside
        }
        // Baseline: Product{prob("O") in ngram}
        double probStart = (j == 0) ? 0.0 : best[j-1]; // up to previous token
        double probO = probStart;
        for (int k = j; k < i; k++) { // scan ngram [j, i-1]
          probO += log(p[k % beam][O]); // log(p) is negative
          if (j != i-1 && probO < best[i-1])
            break;
        }
        if (j == i-1 || probO >= best[i-1]) { // j==i-1: set initial value
          best[i-1] = probO;
          predTags[i-1] = "O";
        }

        // for each supersense
        for (int q = 0; q < nSenses; q++) {
          char const* classTag = superSenses[q];
          // compute Product{prob(Class) in ngram}
          double prob = probStart;
          for (int k = j; k < i; k++) { // scan ngrams [j, i-1]
            string tagk = tag[k] + '-' + classTag;
            ClassID cid = sst.model->OutcomeID(tagk.c_str());
            if (cid == (ClassID)-1) // tag not seen in learning
              prob = -numeric_limits<double>::infinity(); // lowest probability
            else
              prob += log(p[k % beam][cid]);
            if (prob < best[i-1]) // optimization, already worse
              break;
          }
          if (prob < probStart && prob > best[i-1]) { // improved
            best[i-1] = prob;
            for (int k = j; k < i; k++)
              predTags[k] = tag[k] + '-' + classTag;
          }
        }
      }
    }
    n++;
# else // NO_BEAM_SEARCH
    sst.model->estimate(event->features, p[0]);
    predTags[n++] = sst.model->OutcomeName(sst.model->BestOutcome(p[0]));
    delete event;
# endif
  }
  for (int i = 0; i < len; ++i) {
    // feedback
    eventStream.predicted(predTags[i].c_str(), i);
  }
  return sentence;
}

} // namespace SST
} // namespace Tanl

 

Copyright © 2005-2011 G. Attardi. Generated on 3 Mar 2011 by doxygen 1.6.1.