Tanl Linguistic Pipeline

tag/NER/NER.cpp

/*
**  Tanl
**  NER/NER.cpp
**  ----------------------------------------------------------------------
**  Copyright (c) 2005  Giuseppe Attardi (attardi@di.unipi.it).
**  ----------------------------------------------------------------------
**
**  This file is part of Tanl.
**
**  Tanl is free software; you can redistribute it and/or modify it
**  under the terms of the GNU General Public License, version 3,
**  as published by the Free Software Foundation.
**
**  Tanl is distributed in the hope that it will be useful,
**  but WITHOUT ANY WARRANTY; without even the implied warranty of
**  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
**  GNU General Public License for more details.
**
**  You should have received a copy of the GNU General Public License
**  along with this program.  If not, see <http://www.gnu.org/licenses/>.
**  ----------------------------------------------------------------------
*/

// IXE
#include "io/File.h"

#include "NER.h"

// MaxEntropy
#define MODEL GIS
#if (MODEL == GIS)
#include "classifier/GIS.h"
#else
#include "classifier/LBFGS.h"
#endif

// standard
#include <assert.h>
#include <math.h>
#include <limits>

using namespace std;
using namespace Tanl::Text;
using namespace Tanl::Classifier;

namespace Tanl {
namespace NER {

// Configuration Parameters

extern IXE::conf<bool>  oldIOB;
extern IXE::conf<bool>  refine;

NER::NER(char const* modelFile, char const* configFile,
         char const* POStag, char const* NEtag) :
  language("Language", "english"),
  resourceDir("ResourceDir", "Resource"),
  cutoff("Cutoff", 2),
  iter("Iterations", 100),
  alpha("Alpha", 0.0),
  verbose("Verbose", false),
  POStag(POStag),
  NEtag(NEtag),
  model(0),
  resources(POStag, NEtag)
{
  // read configuration file
  if (configFile && IXE::io::File(configFile).exists())
    IXE::Configuration::load(configFile);
  if (IXE::io::File(modelFile).exists())
    model = new MaxEnt(modelFile);
  resources.language = language->c_str(); // after loading configFile.
  resources.load(*resourceDir);
}

NER::~NER() { delete model; }

Text::RegExp::Pattern datePat("[12][0-9][0-9][0-9]-[0-9]+-[0-9]+");

bool checkDate(vector<Token*>& sent)
{
  // look for a line with a date
  for (int i = 0; i < sent.size(); i++)
    if (datePat.test(sent[i]->form))
      return true;
  return false;
}

#ifdef ZONES
void NER::train(SentenceReader* sentenceReader, char const* modelFile)
{
  MODEL trainer(iter, cutoff, alpha);
  trainer.verbose = verbose;

  // One NerEventStream for each document, since it maintains
  // per document information:
  NerEventStream eventStream(resources);
  int sentNum = 0;
  vector<Token*>* prev = 0;
  int zone = 0;

  while (sentenceReader->MoveNext()) {
    vector<Token*>* sent = sentenceReader->Current();
    if (sent->size() == 1 && (*sent)[0]->form == "-DOCSTART-") { // CoNLL03
      FOR_EACH (vector<Token*>, *sent, sit)
        delete *sit;
      delete sent;
      eventStream.reset();
      sentNum = 0;
      continue;
    }
    switch (sentNum) {
    case 0:
      zone = 0;
      break;
    case 1:
      if (checkDate(*sent)) {
        zone = 2;               // Date
        break;
      } else {
        prev = sent;            // must wait until next to decide
        sentNum++;
        continue;
      }
    case 2:
      if (prev) {
        if (checkDate(*sent)) {
          // process prev as Author
          zone = 1;             // Author
        } else {
          // no date, process prev as Text
          zone = 3;
        }
        eventStream.analyze(prev, zone);
        trainer.read(eventStream);
        FOR_EACH (vector<Token*>, *prev, sit)
          delete *sit;
        delete prev;
        prev = 0;
        if (zone == 1)
          zone = 2;             // Date
      } else {
        zone = 3;
      }
      break;
    default:
      zone = 3;                 // Text
    }
    eventStream.analyze(sent, zone);
    trainer.read(eventStream);
    // delete sent's tokens
    FOR_EACH (vector<Token*>, *sent, sit)
      delete *sit;
    delete sent;
    sentNum++;
  }

  trainer.train();

  // dump the model
  trainer.save(modelFile);
}
#else

void NER::train(SentenceReader* sentenceReader, char const* modelFile)
{
  MODEL trainer(iter, cutoff, alpha);
  trainer.verbose = verbose;

  // One NerEventStream for each document, since it maintains
  // per document information:
  NerEventStream eventStream(resources);

  while (sentenceReader->MoveNext()) {
    vector<Token*>* sent = sentenceReader->Current();
    if (sent->size() == 1 && (*sent)[0]->form == "-DOCSTART-") { // CoNLL03
      FOR_EACH (vector<Token*>, *sent, sit)
        delete *sit;
      delete sent;
      eventStream.reset();
      continue;
    }
    eventStream.analyze(sent);
    trainer.read(eventStream);
    // delete sent's tokens
    FOR_EACH (vector<Token*>, *sent, sit)
      delete *sit;
    delete sent;
  }

  trainer.train();

  // dump the model
  trainer.save(modelFile);
}
#endif // ZONES

vector<Token*>* NER::tag(std::vector<Token*>* sentence,
                         NerEventStream* eventStream)
{
  // dynamic programming
  int len = sentence->size();
  // drop gold tags
  for (int i = 0; i < len; i++)
    (*sentence)[i]->set(NEtag, "");
  bool newEventStream = false;
  if (!eventStream) {
    // create a temporary one, just for this sentence.
    eventStream = new NerEventStream(resources);
    newEventStream = true;
  }
  eventStream->analyze(sentence);
  // collect HMM parameters
  int nTags = model->NumOutcomes();
  string predTags[len];
  double best[len];
  best[0] = -numeric_limits<double>::infinity(); // value to beat
# ifdef RL
  int bestLength[len];
# endif
  static int nCateg = 4;
  ClassID O = model->OutcomeID("O"); // Outside
  const int beam = 5;                    // beam size
  string tag[len];              // tag[beam] would suffice
  double p[beam][nTags];
  int n = 0;                    // current token

  while (eventStream->hasNext()) {
    Classifier::Event* event = eventStream->next();
    model->estimate(event->features, p[n % beam]);
    delete event;

    // accumulate enough results
    if (n < beam-1 && n != len-1) {
      n++;
      continue;
    }

    // handle first block, up to beam, with a cycle, the others individually
    int iFirst = (n < beam) ? 1 : n+1;
    for (int i = iFirst; i <= n+1; i++) {
      // consider ngrams up to 5 long
      // compute best[i] = best[i-5] best[i-4] ... best[i-1]
      for (int j = i - 1; j >= 0 && j >= i-5; j--) {
        int w = i - j;          // ngram size

        // ngram start: j, ngram last: i-1
        if (refine) {
          for (int k = j; k < i; k++) {
            if (k == j) {       // left boundary
              if (k == i - 1)   // last char, right boundary
                tag[k] = "U";   // Unique
              else
                tag[k] = "B";   // Begin
            } else {
              if (k == i - 1) 
                tag[k] = "E";   // End
              else
                tag[k] = "I";   // Inside
            }
          }
        } else {
          for (int k = j; k < i; k++) {
            if (k == j)         // left boundary
              tag[k] = "B";     // Begin
            else
              tag[k] = "I";     // Inside
          }
        }
        // Baseline: Product{prob("O") in ngram}
        double probStart = (j == 0) ? 0.0 : best[j-1]; // up to previous token
        double probO = probStart;
        for (int k = j; k < i; k++) { // scan ngram [j, i-1]
          probO += log(p[k % beam][O]); // log(p) is negative
          if (j != i-1 && probO < best[i-1])
            break;
        }
        if (j == i-1 || probO >= best[i-1]) { // j==i-1: set initial value
          best[i-1] = probO;
#ifdef RL
          bestLength[i-1] = i-j;
#endif
          predTags[i-1] = "O";
        }

        // for each category
        FOR_EACH (set<string>, *Resources::entityTypes, it) {
          string const& classTag = *it;
          // compute Product{prob(Class) in ngram}
          double prob = probStart;
          for (int k = j; k < i; k++) { // scan ngrams [j, i-1]
            string tagk = tag[k] + '-' + classTag;
            ClassID cid = model->OutcomeID(tagk.c_str());
            if (cid == (ClassID)-1) // tag not seen in learning
              prob = -numeric_limits<double>::infinity(); // lowest probability
            else
              prob += log(p[k % beam][cid]);
            if (prob < best[i-1]) // optimization, already worse
              break;
          }
          if (prob < probStart && prob > best[i-1]) { // improved
            best[i-1] = prob;
#           ifdef RL
            bestLength[i-1] = i-j;
            predTags[i-1] = tag[i-1] + '-' + classTag;
#           else
            for (int k = j; k < i; k++)
              predTags[k] = tag[k] + '-' + classTag;
#           endif
          }
        }
      }
    }
    n++;
  }
# ifdef RL
  // assign tags to groups from right to left.
  for (int i = len-1; i >= 0; ) {
    string predTag = predTags[i];
    if (predTag[0] != 'O')
      predTag[0] = 'I';   // FIXME: should introduce B tags where appropriate.
    for (int j = i; j > i - bestLength[i]; --j)
      predTags[j] = predTag;
    i -= bestLength[i];
  }
# endif
  for (int i = 0; i < len; ++i) {
    // feedback
    eventStream->predicted(predTags[i].c_str(), i);
  }
  // fix tags
  if (oldIOB) {
    // restore old IOB convention
    int i = 0;
    int len = sentence->size();
    for (Sentence::const_iterator sit = sentence->begin();
         sit != sentence->end(); ++sit, ++i) {
      Token* tok = *sit;
      string const* tag = tok->get(NEtag);
      if (!tag)
        continue;
      string const* prevTag = (i > 0) ? (*sentence)[i-1]->get(NEtag) : 0;
      string& nt = const_cast<string&>(*tag);
      if (nt[0] == 'B' && (!prevTag || (*prevTag)[0] == 'O')) {
        nt[0] = 'I';            // begin
      }
    }
  }
  if (refine) {
    // restore original IOB tags
    int i = 0;
    int len = sentence->size();
    for (Sentence::const_iterator sit = sentence->begin();
         sit != sentence->end(); ++sit, ++i) {
      Token* tok = *sit;
      string const* tag = tok->get(NEtag);
      if (!tag)
        continue;
      string& nt = const_cast<string&>(*tag);
      switch (nt[0]) {
      case 'U':
        if (oldIOB) {
          if (i > 0) {
            string const* prevTag = (*sentence)[i-1]->get(NEtag);
            if (!strcmp(prevTag->c_str()+1, tag->c_str()+1))
              nt[0] = 'B';
            else
              nt[0] = 'I';
          } else
            nt[0] = 'I';
        } else
          nt[0] = 'B';
        break;
      case 'E':
        nt[0] = 'I';
        break;
      }
    }
  }
  if (newEventStream)
    delete eventStream;
  return sentence;
}

Enumerator<vector<Token*>*>*    NER::pipe(Enumerator<vector<Token*>*>& se)
{
  return new NerPipe(*this, se);
}

// ======================================================================

NerPipe::NerPipe(NER& ner, Enumerator<vector<Token*>*>& se) :
  ner(ner),
  eventStream(ner.resources),
  se(se)
{
  vector<string> iobTags;
  int entities = ner.resources.entityTypes->size();
  int neTags = 1 + 2 * entities;
  iobTags.resize(neTags);
  iobTags[0] = "O";
  // create IOB tags
  int i = 0;
  FOR_EACH (set<string>, *ner.resources.entityTypes, it) {
    iobTags[1 + i] = "B-" + *it;
    iobTags[1 + entities + i] = "I-" + *it;
    ++i;
  }

  // create map of states to tags (classifier orders them differently).
  int nTags = ner.model->NumOutcomes();
  for (int i = 0; i < nTags; ++i) {
    char const* tag = ner.model->OutcomeName(i);
    for (int j = 0; j < iobTags.size(); ++j)
      if (tag == iobTags[j]) {
        outcomeId[j] = i;
        break;
      }
  }
}

void NerPipe::Dispose()
{
  ner.decRef();
  delete this;
}

bool NerPipe::MoveNext()
{
  while (se.MoveNext()) {
    sent = se.Current();
    if (sent->size() == 1 && (*sent)[0]->form == "-DOCSTART-") { // CoNLL03
      eventStream.reset();
      delete sent;
    } else 
      return true;
  }
  return false;
}

vector<Token*>* NerPipe::Current()
{
  return ner.tag(sent, &eventStream);
}

} // namespace NER
} // namespace Tanl

 

Copyright © 2005-2011 G. Attardi. Generated on 3 Mar 2011 by doxygen 1.6.1.