Tanl Linguistic Pipeline

tag/SST/SstFeatureExtractor.cpp

/*
**  Tanl
**  SST/SstFeatureExtractor.cpp
**  ----------------------------------------------------------------------
**  Copyright (c) 2005  Giuseppe Attardi (attardi@di.unipi.it).
**  ----------------------------------------------------------------------
**
**  This file is part of Tanl.
**
**  Tanl is free software; you can redistribute it and/or modify it
**  under the terms of the GNU General Public License, version 3,
**  as published by the Free Software Foundation.
**
**  Tanl is distributed in the hope that it will be useful,
**  but WITHOUT ANY WARRANTY; without even the implied warranty of
**  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
**  GNU General Public License for more details.
**
**  You should have received a copy of the GNU General Public License
**  along with this program.  If not, see <http://www.gnu.org/licenses/>.
**  ----------------------------------------------------------------------
*/

// Local
#include "SstFeatureExtractor.h"

// standard
#include <sstream>

// library
#include "conf/conf_bool.h"
#include "conf/conf_string.h"

using namespace Tanl::Classifier;
using namespace Tanl::Text;

namespace Tanl {
namespace SST {

// Default feature model
IXE::conf<string>       FORM("FORM", "0");
IXE::conf<string>       POSTAG("POSTAG", "-2 -1 0 1 2");
IXE::conf<string>       CPOSTAG("CPOSTAG", "-1 0");
IXE::conf<string>       SSTAG("SSTAG", "");

FeatureSpec::FeatureSpec(char const* name, char const* list) :
  name(name)
{
  stringstream ss(list);
  int n;
  while (ss >> n)
    tokens.push_back(n);
}

FeatureSpecs::FeatureSpecs()
{
  // FORM
  push_back(FeatureSpec("FORM", FORM));
  // POS
  push_back(FeatureSpec("POSTAG", POSTAG));
  // CPOSTAG
  push_back(FeatureSpec("CPOSTAG", CPOSTAG));
  // SSTAG
  push_back(FeatureSpec("SSTAG", SSTAG));
}

// ======================================================================

  static string const AllAlpha          = "W1";
  static string const AllDigits         = "W2";
  static string const CapPeriod         = "initCapPeriod";
  static string const SingleCap         = "oneCap";
  static string const SingleS           = "W4";
  static string const ContainsApo       = "W5";
  static string const ContainsDot       = "W6";
  static string const ContainsComma     = "W7";
  static string const DigitSlash        = "digitSlash";
  static string const ContainsPercent   = "percent";
  static string const DigitsDot         = "digitPeriod";
  static string const ContainsDollar    = "dollar";
  static string const ContainsDigit     = "containDigit";
  static string const TwoDigits         = "twoDigit";
  static string const FourDigits        = "fourDigit";
  static string const AllQuoting        = "Wf";
/*
  static char const*  dictTags[] = { "UNI-LOC", "UNI-MISC", "UNI-ORG", "UNI-PER" };
  static string const Money     = "D1";
  static string const Name      = "D2";
  static string const Time      = "D4";
  static string const PrevName  = "D5";
  static string const NextPerson= "D6";*/

// ======================================================================

  /*
   * Token Types (not used).
   */
  // These must be aligned with EntityType::Type
  /*char const* TokenTypes[] = {
    "UNKNOWN",
    "WORD",
    "QUANTITY",
    "NAME",
    "TIME",
    "SYMBOL",
    "PUNCT",
    "ADDRESS",
    "PERSON",
    "ORGANIZATION",
    "LOCATION",
    "PRODUCT",
    "NUMBER",
    "MONEY",
    "MEASURE",
    "DURATION",
    "DATE",
    "TIME",
    "URL",
    "EMAIL"
  };*/

// ======================================================================

  // Global lists
  //
  // FWL (Frequent Word List): words that occur in more than 10 documents
  //
  // CPW (Common Preceding Words): 20 words that most often precede
  //    a certain class
  //
  // CPB (Common Preceding Bigrams): bigrams that often precede
  //    a certain class
  //
  // SUF (Suffix for Class): common 3-4 letter suffix for a certain class
  //    (-ian, -ish)
  //
  // NLW (Name Last Words): list of words terminating a Name sequence
  //    Organization: Inc, Org, Co
  //    Locations: center, museum, square, street
  //    Person: Jr, II, III
  //
  // LIW (Lowercase Intermediate Words): list of lowercase words appearing in a
  //    Name sequence
  //    Organization: al, in, zonder, vor, for
  //    Person: "van der", "de", "of"
  //

  // Lexical Features

  char const* AllUpper[4]       = {"allCaps_HL", "allCaps_AU", "allCaps_DL",
                                   "allCaps_TXT"};
  char const* Capitalized[4]    = {"initCaps_HL", "initCaps_AU", "initCaps_DL",
                                   "initCaps_TXT"};

  // first word of sentence is capitalized
  char const* firstWordCap[4]   = {"firstWordInitCaps_HL",
                                   "firstWordInitCaps_AU",
                                   "firstWordInitCaps_DL",
                                   "firstWordInitCaps_TXT"};

  // first word but not capitalized
  char const* firstWordLow[4]   = {"firstWordNotInitCaps_HL",
                                   "firstWordNotInitCaps_AU",
                                   "firstWordNotInitCaps_DL",
                                   "firstWordNotInitCaps_TXT"};

  char const* MixedCase[4]      = {"mixedCaps_HL", "mixedCaps_AU",
                                   "mixedCaps_DL", "mixedCaps_TXT"};

  string const HyphenCapCap     = "U-U"; // Str1-Str2
  string const HyphenLowCap     = "L-U"; // str1-Str2
  string const HyphenCapLow     = "U-L"; // Str1-str2

  // Local Features

  // previous is capitalized
  char const* prevCap[4]        = { "initCapsPrev_HL", "initCapsPrev_AU",
                                    "initCapsPrev_DL", "initCapsPrev_TXT"};
  // next is capitalized
  char const* nextCap[4]        = { "initCapsNext_HL", "initCapsNext_AU",
                                    "initCapsNext_DL", "initCapsNext_TXT"};

  string const seqCap           = "I";  // previous, current and next are Capitalized
  string const notSeqCap        = "NI"; // not SeqCap
  string const CapNext          = "initCaps_Next";      // current is Capitalized, append next
  string const LowNext          = "nonInitCaps_Next";   // current is not Capitalized, append next
  string const CapNoNext        = "initCaps__Next";     // current is Capitalized, no next
  string const LowNoNext        = "nonInitCaps__Next";  // current is not Capitalized, no next
  string const CapPrev          = "initCaps_Prev";      // current and previous are Capitalized
  string const LowPrev          = "nonInitCaps_Prev";   // current is Capitalized but previous is not
  string const CapNoPrev        = "initCaps__Prev";     // current is Capitalized, no previous
  string const LowNoPrev        = "nonInitCaps__Prev";  // current is not Capitalized, no previous is not

  string const Cap              = "initCaps_";
  string const Low      = "nonInitCaps_";


  string const withinQuotes     = "quote";      // word is in sequence within quotes
  string const rare             = "rare";       // word not present in FWL

  /*// w-2, w-1 appear in CPB list
  char const* bigramTags[]      = { "BI-LOC", "BI-MISC", "BI-ORG", "BI-PER" };

  // 3-letter suffix present in SUF list
  char const* suffixTags[]      = { "Suf_LOC", "Suf_MISC", "Suf_ORG", "Suf_PER" };

  // present in List of Last words
  string const lastWordTags[]   = { "NCS-LOC", "NCS-LISC", "NCS-ORG", "NCS-PER" };

  // word in List NLW appears within sequence of Caps
  string const lowerSeqTags[]   = { "lSL", "lSM", "lSO", "lSP" };

  // word appearing in list Lower Words appearing within names for given class
  string const lowerIntermTags[]= { "lL", "lM", "lO", "lP" };*/

  // Global features

  // another occurrence of current word was tagged as given class
char const* prevTags[]  = {
  "NCPrevOccur-adj.all", "NCPrevOccur-adj.pert", "NCPrevOccur-adj.ppl",
  "adv.all", "NCPrevOccur-noun.Tops", "NCPrevOccur-noun.act",
  "NCPrevOccur-noun.animal", "NCPrevOccur-noun.artifact",
  "NCPrevOccur-noun.attribute", "NCPrevOccur-noun.body",
  "NCPrevOccur-noun.cognition", "NCPrevOccur-noun.communication",
  "NCPrevOccur-noun.event", "NCPrevOccur-noun.feeling",
  "NCPrevOccur-noun.food", "NCPrevOccur-noun.group",
  "NCPrevOccur-noun.location", "NCPrevOccur-noun.motive",
  "NCPrevOccur-noun.object", "NCPrevOccur-noun.other",
  "NCPrevOccur-noun.person", "NCPrevOccur-noun.phenomenon",
  "NCPrevOccur-noun.plant", "NCPrevOccur-noun.possession",
  "NCPrevOccur-noun.process", "NCPrevOccur-noun.quantity",
  "NCPrevOccur-noun.relation", "NCPrevOccur-noun.shape",
  "NCPrevOccur-noun.state", "NCPrevOccur-noun.substance",
  "NCPrevOccur-noun.time", "NCPrevOccur-verb.body",
  "NCPrevOccur-verb.change", "NCPrevOccur-verb.cognition",
  "NCPrevOccur-verb.communication", "NCPrevOccur-verb.competition",
  "NCPrevOccur-verb.consumption", "NCPrevOccur-verb.contact",
  "NCPrevOccur-verb.creation", "NCPrevOccur-verb.emotion",
  "NCPrevOccur-verb.motion", "NCPrevOccur-verb.perception",
  "NCPrevOccur-verb.possession", "NCPrevOccur-verb.social",
  "NCPrevOccur-verb.stative", "NCPrevOccur-verb.weather"
};

// another occurrence of current word was last word
char const* otherLastTags[]     = {
  "OtherNCS-adj.all", "OtherNCS-adj.pert", "OtherNCS-adj.ppl", "adv.all",
  "OtherNCS-noun.Tops", "OtherNCS-noun.act", "OtherNCS-noun.animal",
  "OtherNCS-noun.artifact", "OtherNCS-noun.attribute", "OtherNCS-noun.body",
  "OtherNCS-noun.cognition", "OtherNCS-noun.communication",
  "OtherNCS-noun.event", "OtherNCS-noun.feeling", "OtherNCS-noun.food",
  "OtherNCS-noun.group", "OtherNCS-noun.location", "OtherNCS-noun.motive",
  "OtherNCS-noun.object", "OtherNCS-noun.other", "OtherNCS-noun.person",
  "OtherNCS-noun.phenomenon", "OtherNCS-noun.plant",
  "OtherNCS-noun.possession", "OtherNCS-noun.process",
  "OtherNCS-noun.quantity", "OtherNCS-noun.relation", "OtherNCS-noun.shape",
  "OtherNCS-noun.state", "OtherNCS-noun.substance", "OtherNCS-noun.time",
  "OtherNCS-verb.body", "OtherNCS-verb.change", "OtherNCS-verb.cognition",
  "OtherNCS-verb.communication", "OtherNCS-verb.competition",
  "OtherNCS-verb.consumption", "OtherNCS-verb.contact",
  "OtherNCS-verb.creation", "OtherNCS-verb.emotion", "OtherNCS-verb.motion",
  "OtherNCS-verb.perception", "OtherNCS-verb.possession",
  "OtherNCS-verb.social", "OtherNCS-verb.stative", "OtherNCS-verb.weather"
  };

  string const otherCapitalized = "otherInitCap";// an occurrence not as first word was
                                // Capitalized
  string const otherNotCapitalized      = "otherNotInitCap";// an occurrence not as first word was
                                // not Capitalized

  // Acronyms
  // An AllUpper word is stored as an acronym.
  // Caps sequences with those initials will be given the following features:
  string const Acronym          = "acronym";    // e.g. FCC
  string const AcronymBegin     = "cI"; // e.g. Federal
  string const AcronymContinue  = "cJ"; // e.g. Communication
  string const AcronymEnd       = "cK"; // e.g. Committee

 /*
    Not used: we rely on POS tagger for disambiguating these cases.

  // Longest Common Caps Sequence
  // To avoid confusing "Also Texas Instruments Corp." as a whole company name,
  // the longest common Caps sequence occurring in a document is noted.
  // So if "Xerox Corp." also appears, the following features are set:
  string const LongestB         = "cL"; // Texas
  string const LongestC         = "cM"; // Instruments
  string const LongestE         = "cN"; // Corp.
  */

bool allUpper(char const* s)
{
  char* scan = (char*)s;
  while (*scan)
    if (!isupper(*scan++))
      return false;
  return true;
}

Text::RegExp::Pattern patMix("^[a-z]+[A-Z]([a-zA-Z])*$");
bool mixedCase(char const* s)
{
  return patMix.test(s);
}

bool noLetter(char const* s)
{
  char* scan = (char*)s;
  while (*scan)
    if (isalpha(*scan++))
      return false;
  return true;
}

bool containsDigit(char const* s)
{
  return ::strpbrk(s, "0123456789");
}

bool allDigits(char const* s)
{
  return ::strspn(s, "0123456789") == :: strlen(s);
}

bool digitSlash(char const* s)
{
  return containsDigit(s) && strchr(s, '/');
}

bool allQuotes(char const* s)
{
  return (::strspn(s, "\"") == ::strlen(s));
}

// ======================================================================

char const* Resources::classNames[] = {
  "adj.all", "adj.pert", "adj.ppl", "adv.all", "noun.Tops", "noun.act",
  "noun.animal", "noun.artifact", "noun.attribute", "noun.body",
  "noun.cognition", "noun.communication", "noun.event", "noun.feeling",
  "noun.food", "noun.group", "noun.location", "noun.motive", "noun.object",
  "noun.other", "noun.person", "noun.phenomenon", "noun.plant",
  "noun.possession", "noun.process", "noun.quantity", "noun.relation",
  "noun.shape", "noun.state", "noun.substance", "noun.time", "verb.body",
  "verb.change", "verb.cognition", "verb.communication", "verb.competition",
  "verb.consumption", "verb.contact", "verb.creation", "verb.emotion",
  "verb.motion", "verb.perception", "verb.possession", "verb.social",
  "verb.stative", "verb.weather"
};

Resources::Resources(string& resourceDir, string& locale) :
  language(locale.c_str())
{
  load(resourceDir);
}

void Resources::load(string& resourceDir)
{
  for (int i = 0; i < sizeof(classNames)/sizeof(char*); i++)
    classId[classNames[i]] = i;

  // Dictionaries
  /*load(dict, (resourceDir + "/Dict.lst").c_str());
  moneyDict.load((resourceDir + "/money.lst").c_str());
  namesDict.load((resourceDir + "/names.lst").c_str());
  timeDict.load((resourceDir + "/time.lst").c_str());

  // Frequent Word List
  FWL.load((resourceDir + "/FrequentWords.lst").c_str());

  // Designators
  load(designators, (resourceDir + "/Designators.lst").c_str());

  // Preceding Bigrams
  load(preBigrams, (resourceDir + "/Bigrams.lst").c_str());

  // Suffix
  load(suffixes, (resourceDir + "/Suffix.lst").c_str());

  // Last word
  load(lastWords, (resourceDir + "/Last.lst").c_str());

  // Lower intermediate word
  load(lowerInterm, (resourceDir + "/LowerIn.lst").c_str());*/

}

template<class WordSet>
void Resources::load(WordSet* sets, char const* file)
{
  ifstream ifs(file);
  char line[MAX_LINE_LEN];

  while (ifs.getline(line, MAX_LINE_LEN)) {
    // split line
    char* next = line;
    char* className = strtok_r(0, " \t", &next);
    if (!className)
      continue;
    char* word = next + strspn(next, " \t");
    if (strlen(word) == 0)
      continue;
    // determine class
    int cid = classId[className];
    sets[cid].insert(word);
  }
}

/*
char const* Resources::typeName(EntityType et)
{
  if (et.subtype != (EntityType::Subtype)EntityType::NoType)
    return TokenTypes[et.subtype];
  else if (et.type != EntityType::NoType)
    return TokenTypes[et.type];
  else
    return 0;
}
*/

Text::RegExp::Pattern number("\\d+");
string numRep("+NUM+");

// ======================================================================

SstFeatureExtractor::SstFeatureExtractor(Resources& resources) :
    resources(resources),
    tokenCategorizer(resources.language),
    insideQuotes(false)
{ }

void SstFeatureExtractor::extract(Classifier::Features& features, const int& position)
{
  Token* token = (*sentence)[position];
  char const* word = token->form.c_str();
  char const* prevWord = (position > 0) ? (*sentence)[position-1]->form.c_str() : 0;
  char const* nextWord = (position+1 < sentence->size()) ? (*sentence)[position+1]->form.c_str() : 0;

  //======================================================================
  // Lexical Features

  size_t len = strlen(word);
  size_t sentLen = sentence->size();

  // 1st word and initCaps
  if (!prevWord && isupper(word[0]))
    features.add(firstWordCap[zone]);

  // initCaps and not 1st word
  if (prevWord && isupper(word[0]))
      features.add(Capitalized[zone]);

  // 1st word and not initCaps
  if (!prevWord && !isupper(word[0]))
    features.add(firstWordLow[zone]);

  // AllUpper
  if (allUpper(word)) {
    features.add(AllUpper[zone]);
    if (acronyms.contains(word))
      features.add(Acronym);
  } else if (mixedCase(word))
    features.add(MixedCase[zone]);

  // Cap Period
  if (isupper(word[0]) && word[len-1] == '.')
    features.add(CapPeriod);

  // Single Cap
  if (len == 1 && isupper(word[0]))
    features.add(SingleCap);

  // Contains Digit
  if (containsDigit(word))
    features.add(ContainsDigit);

  if (allDigits(word)) {
    // TwoDigits
    if (len == 2)
      features.add(TwoDigits);
    // TwoDigits
    else if (len == 4)
      features.add(FourDigits);
  }

  // Digits and /
  if (digitSlash(word))
    features.add(DigitSlash);
    
  // Contains $
  if (strchr(word, '$'))
    features.add(ContainsDollar);
    
  // Contains %
  if (strchr(word, '%'))
    features.add(ContainsDollar);
    
  // Digits and .
  if (strpbrk(word, "1234567890") && strchr(word, '.'))
    features.add(DigitsDot);

  //======================================================================
  // Local Features
  string upNext(nextWord ? nextWord : "");
  to_upper(upNext);
  upNext = number.replace(upNext, numRep, true);
  string upPrev(prevWord ? prevWord : "");
  to_upper(upPrev);
  upPrev = number.replace(upPrev, numRep, true);

  // next word
  if (nextWord) {
    if (isupper(word[0]))
        features.add(Cap + upNext + "Next");
    else
      features.add(Low + upNext + "Next");
  } else if (isupper(word[0]))
    features.add(CapNoNext);
  else
    features.add(LowNoNext);

  // previous word
  if (prevWord) {
    if (isupper(word[0]))
        features.add(Cap + upPrev + "Prev");
     else
      features.add(Low + upPrev + "Prev");
  } else if (isupper(word[0]))
    features.add(CapNoPrev);
  else
    features.add(LowNoPrev);

  // initCapsNext
  if (nextWord && isupper(nextWord[0]))
    features.add(nextCap[zone]);

  // initCapsPrev
  if (prevWord && isupper(prevWord[0]))
    features.add(prevCap[zone]);

  // Case Sequence
  if (prevWord && nextWord &&
      isupper(prevWord[0]) && isupper(word[0]) && isupper(nextWord[0]))
    features.add(seqCap);
  else
    features.add(notSeqCap);

  // hyphenate words
  if (strchr(word, '-')) {
    char copy[800];
    strncpy(copy, word, sizeof(copy));
    char* next = copy;
    char* word1 = strtok_r(0, "-", &next);
    char* word2 = strtok_r(0, "-", &next);
    if (word1 && word2) {
      bool up1 = isupper(word1[0]);
      bool up2 = isupper(word2[0]);
      if (next[0] == '\0') {    // just two
        if (up1 && up2)
          features.add(HyphenCapCap);
        else {
          to_upper(word1);
          to_upper(word2);
          if (up1) {
            features.add(HyphenCapLow);
            features.add(string("U-") + word1);
            features.add(string("L_") + word2);
          } else if (up2) {
            features.add(HyphenLowCap);
            features.add(string("L-") + word1);
            features.add(string("U_") + word2);
          }
        }
      }
    }
  }

  string upper(word);
  to_upper(upper);
  upper = number.replace(upper, numRep, true);

  /*// rare words
  if (!resources.FWL.contains(upper))
    features.add(rare);*/

  /*// bigrams
  if (position > 1) {
    string bigram = (*sentence)[position-2]->form + '_' + prevWord;
    to_upper(bigram);
    for (int i = 0; i < Resources::nClasses; ++i) {
      if (resources.preBigrams[i].contains(bigram))
        features.add(bigramTags[i]);
    }
  }*/
/*
#ifdef EXTRA
  for (int i = 0; i < Resources::nClasses; ++i) {
    if (resources.lastWords[i].contains(word))
      features.add(lastWordTags[i]);
  }
#endif
*/
/*
  for (int i = 0; i < Resources::nClasses; ++i) {
    if (resources.lowerInterm[i].contains(word))
      features.add(lowerIntermTags[i]);
  }*/

  //FIXME: Word Shape


  //======================================================================
  // Dictionary Features
/*
  if (position > 0) {
    for (int i = 0; i < Resources::nClasses; ++i) {
      if (resources.dict[i].contains(word)) 
        features.add(dictTags[i]);
    }
  }
*/
  // features according to feature model
  char feature[1024];
  FOR_EACH (FeatureSpecs, featureSpecs, fit) {
    char const* attrName = fit->name;
    int attrIndex = token->attrIndex(attrName);
    char featId = 'A' + attrIndex;              // feature type identifier
    FOR_EACH (vector<int>, fit->tokens, tit) {
      // find token
      int dx = *tit;
      int idx = position + dx;
      Token* tok = (0 <= idx && idx < sentLen) ? (*sentence)[idx] : 0;
      if (tok) {
        string const* item = tok->get(attrName);
        if (item && !item->empty()) {
          // skip empty attributes
          if (dx < 0)
            sprintf(feature, "%d%c%s", -dx, featId, item->c_str());
          else
            sprintf(feature, "%c%d%s", featId, dx, item->c_str());
          features.add(feature);
        }
      }
    }
  }

  // Global features
  unordered_map<string, bool>::iterator it = capitalized.find(upper);
  if (it != capitalized.end()) {
    if (it->second)
      features.add(otherCapitalized);
    else
      features.add(otherNotCapitalized);
  } else if (position > 0)      // non ambiguous position
    capitalized[upper] = isupper(word[0]); // FIXME: only first occurrence counts

  // Acronyms
  if (nextWord && isupper(word[0])) {
    string acronym;
    for (int i = position; i < sentLen; i++) {
      char c = (*sentence)[i]->form[0];
      if (isupper(c))
        acronym.push_back(c);
      else
        break;
    }
    if (acronym.size() > 1)
      acronyms.insert(acronym.c_str());
  }

  // Global Class of word in Cap sequence that occurred as last word
  for (int i = 0; i < Resources::nClasses; i++) {
    if (otherLast[i].contains(word))
      features.add(otherLastTags[i]);
  }

  // Global Name Class of previous occurences
  for (int i = 0; i < Resources::nClasses; i++) {
    if (prevClass[i].contains(word))
      features.add(prevTags[i]);
  }

  /*// char suffixes
  for (int i = 0; i < Resources::nClasses; ++i) {
    if (resources.suffixes[i].match(upper))
      features.add(suffixTags[i]);
  }*/

  // inside quotes
  /*
  if (insideQuotes)
    features.add(withinQuotes);
  */
  if (prevWord && nextWord &&
      strpbrk(prevWord, "\"()[]{}'<>") &&
      strpbrk(nextWord, "\"()[]{}'<>"))
    features.add(withinQuotes);
  // prepare for next position
  if (allQuotes(word))
    insideQuotes = !insideQuotes;
}

void SstFeatureExtractor::analyze(Sentence* sent, int zone)
{
  sentence = sent;
  this->zone = zone;
  int len = sentence->size();
  tokenTypes.resize(len);
  insideQuotes = false;
  int i = 0;
  for (Sentence::const_iterator sit = sentence->begin();
       sit != sentence->end(); ++sit, ++i) {
    Token* tok = *sit;
    tokenTypes[i] = tokenCategorizer.analyze(tok->form.c_str());
  }
}

void SstFeatureExtractor::reset()
{
  insideQuotes = false;

  // global document features
  for (int i = 0; i < Resources::nClasses; ++i) {
    prevClass[i].clear();
    otherLast[i].clear();
  }
  acronyms.clear();
  capitalized.clear();
}

void SstFeatureExtractor::classified(int position, char const* className)
{
  Token* token = (*sentence)[position];
  char const* word = token->form.c_str();
  designated(word, className);
  // set the SSTAG of token
  token->set("SSTAG", className);
}

void SstFeatureExtractor::designated(char const* word, char const* ssTag)
{
  if (strlen(ssTag) < 2)
    return;
  for (int i = 0; i < Resources::nClasses; ++i) {
    if (!strcmp(ssTag+2, resources.classNames[i])) // skip X- from tag
      prevClass[i].insert(word);
  }
}

} // namespace SST
} // namespace Tanl

 

Copyright © 2005-2011 G. Attardi. Generated on 3 Mar 2011 by doxygen 1.6.1.