Tanl Linguistic Pipeline

Corpus/Token.h

/*
**  Tanl
**  Corpus/Token.h
**  ----------------------------------------------------------------------
**  Copyright (c) 2005 Giuseppe Attardi (attardi@di.unipi.it).
**  ----------------------------------------------------------------------
**
**  This file is part of Tanl.
**
**  Tanl is free software; you can redistribute it and/or modify it
**  under the terms of the GNU General Public License, version 3,
**  as published by the Free Software Foundation.
**
**  Tanl is distributed in the hope that it will be useful,
**  but WITHOUT ANY WARRANTY; without even the implied warranty of
**  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
**  GNU General Public License for more details.
**
**  You should have received a copy of the GNU General Public License
**  along with this program. If not, see <http://www.gnu.org/licenses/>.
**  ----------------------------------------------------------------------
*/

#ifndef Tanl_Token_H
#define Tanl_Token_H

// standard
#include <string>
#include <vector>
#include <sstream>
#include <map>

#include <iostream>

// library
#include "text/WordIndex.h"
#include "text/RegExp.h"

// local
#include "include/RefCountable.h"

namespace Tanl {

typedef unsigned        AttributeId;

struct AttributeIndex : public Text::WordIndex
{
  std::vector<char const*>      names;

  AttributeId   insert(const char* name) {
    AttributeId id = Text::WordIndex::insert(name);
    if (id >= names.size()) {
      // add new name to names
      names.resize(id + 1);
      names[id] = ::strdup(name);
    }
    return id;
  }

  AttributeIndex& operator =(AttributeIndex const& ai) {
    Text::WordIndex::operator=(ai);
    for (unsigned i = 0; i < names.size(); i++)
      free((void*)names[i]);
    names.resize(ai.size());
    for (unsigned i = 0; i < ai.names.size(); i++)
      names[i] = strdup(ai.names[i]);
    return *this;
  }

  ~AttributeIndex() {
    for (unsigned i = 0; i < names.size(); i++)
      free((void*)names[i]);
  }
};

std::ostream& operator <<(std::ostream& os, AttributeIndex const& ai);

struct Attributes
{
  typedef std::string           Attribute;

  AttributeIndex*               attributeIndex;
  std::vector<Attribute>        values;

  static AttributeIndex*        emptyAttrIndex;

  Attributes(AttributeIndex* attributeIndex) :
    attributeIndex(attributeIndex)
  {
    if (attributeIndex)
      values.resize(attributeIndex->size());
    else
      this->attributeIndex = emptyAttrIndex;
  }

  Attributes(AttributeIndex* attributeIndex, std::vector<Attribute>& values) :
    attributeIndex(attributeIndex),
    values(values)
  { }

  Attribute& operator [](int i) { return values[i]; }

  bool operator ==(const Attributes& other) {
    return attributeIndex == other.attributeIndex &&
      values == other.values;
  }

  bool operator !=(const Attributes& other) { return !(*this == other); }

  struct const_iterator {
    const_iterator(Attributes const& attributes, int fit = 0) :
      attributes(attributes),
      fit(fit)
    { }

    bool operator ==(const const_iterator& other) {
      return &attributes == &other.attributes && fit == other.fit;
    }
    bool operator !=(const const_iterator& other) { return !(*this == other); }
    const_iterator& operator ++() { fit++; return *this; }
    const_iterator operator ++(int) { // postincrement
      const_iterator tmp = *this;
      ++*this;
      return tmp;
    }

    // can't use string& since pair<> constructor uses reference parameters.
    std::pair<char const*, std::string const*> operator*() {
      std::string const& val = attributes.values[fit];
      return std::make_pair(attributes.attributeIndex->names[fit], &val);
    }

    Attributes const& attributes;
    int fit;
  };

  const_iterator begin() const { return const_iterator(*this); }
  const_iterator end() const { return const_iterator(*this, values.size()); }

  std::string const* get(std::string const& name) const {
    AttributeId id = attributeIndex->index(name.c_str());
    return (id == Text::WordIndex::None) ? 0 : &values[id];
  }

  std::string const* get(char const* name) const {
    AttributeId id = attributeIndex->index(name);
    return (id == Text::WordIndex::None) ? 0 : &values[id];
  }

  AttributeId   index(char const* key) const {
    return attributeIndex->index(key);
  }

  void insert(char const* key, char const* value) {
    AttributeId id = attributeIndex->insert(key); // get an ID for key
    if (id >= values.size())
      values.resize(id + 1);
    values[id] = value;
  }

  void insert(char const* key, std::string const& value) {
    AttributeId id = attributeIndex->insert(key); // get an ID for key
    if (id >= values.size())
      values.resize(id + 1);
    values[id] = value;
  }

  void insert(char const* key, int value) {
    AttributeId id = attributeIndex->insert(key); // get an ID for key
    if (id >= values.size())
      values.resize(id + 1);
    std::ostringstream oss;
    oss << value;
    values[id] = oss.str();
  }
};

struct TokenLink
{
  TokenLink(int head = -1, char const* label = "") :
    head(head),
    label(label)
  { }

  bool operator ==(const TokenLink& other) const {
    return head == other.head && label == other.label;
  }
  bool operator !=(const TokenLink& other) { return !(*this == other); }

  int           head;
  std::string   label;
};

typedef std::vector<TokenLink> TokenLinks;

struct Tag
{
  Tag(const std::string& xmlTag);

  std::string& operator[](const std::string& key) {
    return attributes[key];
  }

  const std::string open() const {
    std::string tag = "<" + name;
    for (std::map<std::string, std::string>::const_iterator it = attributes.begin();
         it != attributes.end(); ++it)
      tag += ' ' + it->first + "=\"" + it->second + '"';
    return tag + ">";
  }

  const std::string close() const {
    return "</" + name + ">";
  }

  std::string                           name;
  std::map<std::string, std::string>    attributes;
};

struct Context : public RefCountable
{
  Context(const std::string& xmlTag, Context* parent = 0) :
    tag(xmlTag),
    parent(parent)
  {
    if (parent)
      parent->incRef();
  }

  ~Context() {
    if (parent)
      parent->decRef();
  }

  void Dispose() {
    decRef();
  }

  Tag           tag;
  Context*      parent;

};

class Corpus;

struct Token
{
  struct Morpho {
    std::string         morph;  
    char                Case;   
    char                gender; 
    char                mode;   
    char                negative; 
    char                number; 
    char                person; 
    char                tense;  
    char                aspect; 
    char                transitive; 
    char                extra[20]; 

    Morpho(const char* morph = "", size_t morphSize = 0) :
      morph(morph, morphSize),
      Case(0),
      gender(0),
      mode(0),
      negative(0),
      number(0),
      person(0),
      tense(0),
      transitive(0)
    {
      extra[0] = '\0';
    }
  };

  Token(std::string& form, Attributes& attributes, Context* context = 0) :
    form(form),
    attributes(attributes),
    links(1, TokenLink(-1)),
    context(context)
  {
    if (context)
      context->incRef();
  }

  Token(std::string& form, Attributes& attributes, TokenLinks& links, Context* context = 0) :
    form(form),
    attributes(attributes),
    links(links),
    context(context)
  {
    if (context)
      context->incRef();
  }

  Token(Attributes& attributes, Context* context = 0) :
    attributes(attributes),
    links(1, TokenLink(-1)),
    context(context)
  {
    if (context)
      context->incRef();
  }

  Token(std::string& form, AttributeIndex* attributeIndex, Context* context = 0) :
    form(form),
    attributes(attributeIndex),
    links(1, TokenLink(-1)),
    context(context)
  {
    if (context)
      context->incRef();
  }

  Token(char const* form, AttributeIndex* attributeIndex, Context* context = 0) :
    form(form),
    attributes(attributeIndex),
    links(1, TokenLink(-1)),
    context(context)
  {
    if (context)
      context->incRef();
  }

  Token(AttributeIndex* attributeIndex, Context* context = 0) :
    attributes(attributeIndex),
    links(1, TokenLink(-1)),
    context(context)
  {
    if (context)
      context->incRef();
  }

  ~Token() {
    if (context)
      context->decRef();
  }

  std::string const& operator [](char const* key) {
    return *attributes.get(key);
  }

  std::string const*    get(char const* key) const {
    return attributes.get(key);
  }

  void          set(char const* key, char const* value) {
    attributes.insert(key, value);
  }

  void          set(char const* key, std::string const& value) {
    attributes.insert(key, value);
  }

  void          set(char const* key, int value) {
    attributes.insert(key, value);
  }

  int           attrIndex(char const* name) const {
    std::vector<char const*> const& names = attributes.attributeIndex->names;
    unsigned idx = 0;
    for (std::vector<char const*>::const_iterator it = names.begin();
         it != names.end(); ++it) {
      if (!::strcmp(*it, name))
        break;
      idx++;
    }
    return (idx >= names.size()) ? -1 : (int)idx;
  }

  bool operator ==(const Token& other) {
    return attributes == other.attributes &&
      links == other.links;
  }
  bool operator !=(const Token& other) { return !(*this == other); }

  void          print(std::ostream& os, int indent = 0) const;

  void          printTab(std::ostream& os, Corpus const& corpus);

  std::string   form;           
  Morpho        morpho;         
  Attributes    attributes;     
  TokenLinks    links;          
  Context*      context;        
};

std::ostream& operator <<(std::ostream& os, Token const& tok);

typedef std::vector<Token*> Sentence;

} // namespace Tanl

#endif // Tanl_Token_H

 

Copyright © 2005-2011 G. Attardi. Generated on 4 Mar 2011 by doxygen 1.6.1.