Tanl Linguistic Pipeline

tag/SST/TokenCategorizer.cpp

/*
**  Tanl
**  SST/TokenCategorizer.cpp
**  ----------------------------------------------------------------------
**  Copyright (c) 2005  Giuseppe Attardi (attardi@di.unipi.it).
**  ----------------------------------------------------------------------
**
**  This file is part of Tanl.
**
**  Tanl is free software; you can redistribute it and/or modify it
**  under the terms of the GNU General Public License, version 3,
**  as published by the Free Software Foundation.
**
**  Tanl is distributed in the hope that it will be useful,
**  but WITHOUT ANY WARRANTY; without even the implied warranty of
**  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
**  GNU General Public License for more details.
**
**  You should have received a copy of the GNU General Public License
**  along with this program.  If not, see <http://www.gnu.org/licenses/>.
**  ----------------------------------------------------------------------
*/

#include "TokenCategorizer.h"

using namespace Tanl::Text;

namespace Tanl {
namespace SST {

//
// Apostrophes in english tokens
char const* engSplit[] = {"'s", "n't", "'ll", "'d", 0 };

//
// In all the patterns below, we avoid initial '^', since we
// use the Anchored option in RegExp constructor
//
char const WORD_EN[] = "[[:alnum:]]+-?[[:alnum:]'/@\\.#&]*";

/*
 * European languages word can only have apostrophe only at end
 */
char const WORD_EU[] = "[[:alnum:]]+-?[[:alnum:]/@\\.#&]*'?";

/*
 * English languages use '.' as decimal separator and ',' as spacing
 */
// (?: ...) means non capturing subpattern.
char const QUANTITY_EN[] = "[+-]?[0-9]+(?:,[0-9]{3})*(?:\\.[0-9]+)?(?:(?:[eE](?:[+-]?[0-9]+))|(?:\\s*%))?";

/*
 * European languages use ',' as decimal separator and '.' as spacing
 */
char const QUANTITY_EU[] = "[+-]?[0-9]+(?:\\.[0-9]{3})*(?:,[0-9]+)?(?:(?:[eE](?:[+-]?[0-9]+))|(?:\\s*%))?";

/*
 * Currency
 */
char const CURRENCY_US[] = "$[\\s]?([0-9]+(?:\\,[0-9]{3})+(?:\\.[0-9]+)?)";

/*
 * Matches
 * dd/dd[/dd]
 * dd/dd[/dddd]
 * dd-dd[-dd]
 * dd-dd[-dddd]
 * dddd-d[d]-d[d]
 */
char const DATE_EN[] = "(?:\\d{1,2}/\\d{1,2}(?:/\\d{2}\\d{2}?))|(?:\\d{1,2}-\\d{1,2}(?:-\\d{2}\\d{2}?))|(?:\\d{4}-\\d{1,2}-\\d{1,2})";

/*
 * Matches
 * dd:dd
 */
char const TIME_EN[] = "\\d{1,2}:\\d{1,2}";

/*
 * URL
 */
char const URL[] = "(?:((http:)//)|www.)([\\w+\\-]\\.)+[[:alpha:]]{2,3}(:\\d{1,4})?(/[\\w.%~!#?+\\-])*";

/*
 * Email
 */
char const EMAIL[] = "[[:alpha:]\\.]+@[[:alpha:]\\.]+";

/*
 * MUC-6 annotations
 */
char const MUC_NAME[] = "<ENAMEX\\s(?:ALT=.*?\\s)?TYPE='(\\w+)'>";
char const MUC_NUM[] = "<NUMEX\\sTYPE='(\\w+)'>";
char const MUC_TIME[] = "<TIMEX\\sTYPE='(\\w+)'>";
char const MUC_OPT[] = "<..MEX\\STATUS='OPT'\\sTYPE='(\\w+)'>";

/*
 * Generic tag
 */
char const XML_TAG[] = "</?\\w+[^<]*?>";

static map<string, Language>            localeLang;

static int NameTypeBuilder()
{
  struct LocaleLang
  {
    char const* locale;
    Language    language;
  };

  static LocaleLang const localeLangTable[] = {
    { "english",        en },
    { "czech",          cz },
    { "deutsch",        de },
    { "french",         fr },
    { "italian",        it },
    { "spanish",        es },
    { "portuguese",     pt }
  };

  int i = 0;
  for (register LocaleLang const* e = localeLangTable;
       i < sizeof(localeLangTable)/sizeof(LocaleLang);
       ++i, ++e)
    localeLang[string(e->locale)] = e->language;
  return 0;
}

static int ntBuilt = NameTypeBuilder();

TokenCategorizer::TokenCategorizer(char const* locale) :
  language(localeLang[locale])
{
  switch (language) {
  case it:
  case de:
  case es:
  case fr:
    wordPattern = RegExp::Pattern(WORD_EU, RegExp::Anchored);
    quantityPattern = RegExp::Pattern(QUANTITY_EU, RegExp::Anchored);
    break;
  case en:
  default:
    wordPattern = RegExp::Pattern(WORD_EN, RegExp::Anchored);
    quantityPattern = RegExp::Pattern(QUANTITY_EN, RegExp::Anchored);
    break;
  }
  currencyPattern = RegExp::Pattern(CURRENCY_US, RegExp::Anchored);
  datePattern = RegExp::Pattern(DATE_EN, RegExp::Anchored);
  timePattern = RegExp::Pattern(TIME_EN, RegExp::Anchored);
  urlPattern = RegExp::Pattern(URL, RegExp::Anchored);
  emailPattern = RegExp::Pattern(EMAIL, RegExp::Anchored);

  // MUC-6
  MUC_name = RegExp::Pattern(MUC_NAME, RegExp::Anchored);
  MUC_num = RegExp::Pattern(MUC_NUM, RegExp::Anchored);
  MUC_time = RegExp::Pattern(MUC_TIME, RegExp::Anchored);
  MUC_opt = RegExp::Pattern(MUC_OPT, RegExp::Anchored);

  tagPattern = RegExp::Pattern(XML_TAG, RegExp::Anchored);
}

EntityType TokenCategorizer::analyze(char const* token)
{
  EntityType    res;
  RegExp::MatchGroups matches(3);

  if (quantityPattern.match(token, matches)) {
    res.type = EntityType::Quantity;
    res.subtype = EntityType::Number;
  } else if (currencyPattern.match(token, matches)) {
    res.type = EntityType::Quantity;
    res.subtype = EntityType::Money;
  } else if (datePattern.match(token, matches)) {
    res.type = EntityType::Time;
    res.subtype = EntityType::Date;
  } else if (timePattern.match(token, matches)) {
    res.type = EntityType::Time;
    res.subtype = EntityType::Instant;
  } else if (urlPattern.match(token, matches)) {
    res.type = EntityType::Address;
    res.subtype = EntityType::Url;
  } else if (emailPattern.match(token, matches)) {
    res.type = EntityType::Address;
    res.subtype = EntityType::Email;
  } else if (wordPattern.match(token, matches)) {
    res.type = EntityType::Word;
    res.subtype = (EntityType::Subtype)EntityType::NoType;
  } else if (tagPattern.match(token, matches)) {
    // skip closing or unknown tags
    res.type = EntityType::NoType;
    res.subtype = (EntityType::Subtype)EntityType::NoType;
  } else if (!strcmp(token, "'s")) {
    // special case of genitive after tag: </NAMEX>'s
    res.type = EntityType::Word;
    res.subtype = (EntityType::Subtype)EntityType::NoType;
  }
  return res;
}

} // SST
} // Tanl

 

Copyright © 2005-2011 G. Attardi. Generated on 3 Mar 2011 by doxygen 1.6.1.