Tanl Linguistic Pipeline

tag/NER/NerFeatureExtractor.h

/*
**  Tanl
**  NER/NerFeatureExtractor.h
**  ----------------------------------------------------------------------
**  Copyright (c) 2005  Giuseppe Attardi (attardi@di.unipi.it).
**  ----------------------------------------------------------------------
**
**  This file is part of Tanl.
**
**  Tanl is free software; you can redistribute it and/or modify it
**  under the terms of the GNU General Public License, version 3,
**  as published by the Free Software Foundation.
**
**  Tanl is distributed in the hope that it will be useful,
**  but WITHOUT ANY WARRANTY; without even the implied warranty of
**  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
**  GNU General Public License for more details.
**
**  You should have received a copy of the GNU General Public License
**  along with this program.  If not, see <http://www.gnu.org/licenses/>.
**  ----------------------------------------------------------------------
*/

#ifndef Tanl_NER_NerFeatureExtractor_H
#define Tanl_NER_NerFeatureExtractor_H

// Library
#include "conf/conf_set.h"
#include "text/WordIndex.h"
#include "text/WordSet.h"
#include "text/Suffixes.h"

// Local
#include "TokenCategorizer.h"
#include "FeatureSpecs.h"
#include "Corpus/Token.h"
#include "classifier/Classifier.h"
#include "classifier/FeatureExtractor.h"

// Standard
#include <cstring>
#include <string>
#include <vector>

namespace Tanl {
namespace NER {

//======================================================================

// Used to get the encoding of a tag corresponding to previous/next positions.
typedef unordered_map<char const*, std::string> TagSet;

struct Resources
{
  static IXE::conf_set<std::string>     entityTypes;

  Text::WordIndex       classId;

  Resources(char const* POStag, char const* NEtag) :
    POStag(POStag),
    NEtag(NEtag)
  { resize(); }

  Resources(std::string& locale, char const* POStag, char const* NEtag) :
    language(locale.c_str()),
    POStag(POStag),
    NEtag(NEtag)
  { resize(); }

  Resources(std::string& resourceDir, std::string& locale,
            char const* POStag, char const* NEtag);

  char const*           language;
  char const*           POStag;
  char const*           NEtag;

  size_t        typesCount() { return entityTypes->size(); }

  // Token Types
  char const*   typeName(EntityType et);
  TagSet        prevTokenType; // feature Type for prev word
  TagSet        nextTokenType; // feature Type for next word

  // Dictionaries
  std::vector<Tanl::Text::NormWordSet>  dict; // uni
  Tanl::Text::NormWordSet       moneyDict;
  Tanl::Text::NormWordSet       namesDict;
  Tanl::Text::NormWordSet       timeDict;
  Tanl::Text::NormWordSet       prodDict;

  Tanl::Text::NormWordSet       FWL;            // Frequent Word List

  std::vector<Tanl::Text::NormWordSet>  designators; // func

  std::vector<Tanl::Text::NormWordSet>  preBigrams; // ubi

  std::vector<Tanl::Text::NormWordSet>  prefixes; // wordPre

  std::vector<Tanl::Text::Suffixes>     suffixes; // wordSuf

  std::vector<Tanl::Text::NormWordSet> firstWords; // classPref

  std::vector<Tanl::Text::NormWordSet> lastWords; // classSuf

  //std::vector<Tanl::Text::NormWordSet>        afterWords;

  std::vector<Tanl::Text::NormWordSet> lowerInterm;

  void load(std::string& resourceDir);

  template<class WordSet>
  void          load(std::vector<WordSet>& sets, char const* file);

private:
  void          resize();

};

//======================================================================
//======================================================================

class NerFeatureExtractor :
 public Classifier::FeatureExtractor<Classifier::Features, const int>
{
public:
  NerFeatureExtractor(Resources& resources);

  ~NerFeatureExtractor() { reset(); }

  void          analyze(Sentence* sent, int zone);

  void  extract(Classifier::Features& feats, const int& pos);

  void  reset();

  void  classified(int position, char const* className);

  Resources&            resources;

protected:
  bool                  insideQuotes;
  TokenCategorizer      tokenCategorizer;       
  std::vector<EntityType>       tokenTypes; 

  Sentence*             sentence; 

  // Global Maps

  unordered_map<string, bool>   capitalized; // firstInitCap

  std::vector<Tanl::Text::NormWordSet>  prevClass; // NCPrevOccur

  std::vector<Tanl::Text::NormWordSet>  otherFirst; // otherNCF

  std::vector<Tanl::Text::NormWordSet>  otherLast; // otherNCL

  Tanl::Text::NormWordSet       acronyms;

private:
  void designated(char const* word, char const* neTag);
  int           zone;   // zone within document
};

extern char const* UnknownTag;

} // namespace NER
} // namespace Tanl

#endif // Tanl_NER_NerFeatureExtractor_H

 

Copyright © 2005-2011 G. Attardi. Generated on 4 Mar 2011 by doxygen 1.6.1.