Tanl Linguistic Pipeline

text/WordSet.h

/*
**      Tanl
**      text/WordSet.h
** ----------------------------------------------------------------------
**      Copyright (c) 2003  Giuseppe Attardi (attardi@di.unipi.it).
** ----------------------------------------------------------------------
**
**  This file is part of Tanl.
**
**  Tanl is free software; you can redistribute it and/or modify it
**  under the terms of the GNU General Public License, version 3,
**  as published by the Free Software Foundation.
**
**  Tanl is distributed in the hope that it will be useful,
**  but WITHOUT ANY WARRANTY; without even the implied warranty of
**  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
**  GNU General Public License for more details.
**
**  You should have received a copy of the GNU General Public License
**  along with this program.  If not, see <http://www.gnu.org/licenses/>.
**  ----------------------------------------------------------------------
*/

#ifndef Tanl_Text_WordSet_H
#define Tanl_Text_WordSet_H

#include "include/unordered_set.h"
#include "text/strings.h"

// standard
#include <fstream>
#include <stdlib.h>

namespace Tanl {
namespace Text {

template <class _Hash = hash<const char *>,
          class _Pred = std::equal_to<const char*> >
class WordSetBase :
    public unordered_set<const char*, _Hash, _Pred>
{
protected:
  typedef unordered_set<const char*, _Hash, _Pred> _Base;

public:

  WordSetBase() { }

  WordSetBase(char const* file);

  WordSetBase(std::string& file);

  ~WordSetBase() { clear(); }

  void clear() {
    for (typename _Base::iterator it = this->begin(); it != this->end(); ++it) {
      free((void*)*it);         // allocated by strdup()
    }
    _Base::clear();
  }

  bool  contains(char const* word) {
    return this->find(word) != this->end();
  }

  bool  contains(std::string const& word) {
    return this->find(word.c_str()) != this->end();
  }

  std::pair<typename _Base::iterator, bool> insert(const char* word) {
    typename _Base::iterator fit = this->find(word);
    return (fit != this->end()) ?
      std::pair<typename _Base::iterator, bool>(fit, false) :
      _Base::insert(::strdup(word));
  }

  void  store(char const* file);

  void  load(char const* file);

protected:
  void  load(std::ifstream& ifs);
};

class WordSet : public WordSetBase<>
{
public:
  WordSet() { }
  WordSet(char const* file) :  WordSetBase<>(file) { }
  WordSet(std::string& file) : WordSetBase<>(file) { }
};

struct NormEqual
{
  bool operator()(const char* s1, const char* s2) const
  {
    char n1, n2;
    while (n1 = *s1, n2 = *s2, n1 || n2) {
      if (n1 && n2 && tolower(n1) == tolower(n2)) {
        ++s1;
        ++s2;
      } else if (n1 == '.') {
        ++s1;
      } else if (n2 == '.') {
        ++s2;
      } else
        break;
    }
    return (n1 == '\0') && (n2 == '\0');
  }
};

struct NormHash
{
  size_t operator()(char const* str) const {
    std::string s = str;
    to_lower(s);
    return hash<char const *>()(s.c_str());
  }
};

class NormWordSet : public WordSetBase<NormHash, NormEqual>
{
public:
  NormWordSet() { }
  NormWordSet(char const* file) : WordSetBase<NormHash, NormEqual>(file) { }
  NormWordSet(std::string& file) : WordSetBase<NormHash, NormEqual>(file) { }

  std::pair<_Base::iterator, bool> insert(const char* word) {
    char* lower = (char*)malloc(strlen(word)+1);
    to_lower(lower, word);
    _Base::iterator fit = this->find(lower);
    return (fit != this->end()) ?
      std::pair<_Base::iterator, bool>(fit, false) :
      _Base::insert(lower);
  }

  bool  contains(char const* word) {
    return this->find(word) != this->end();
  }

  bool  contains(std::string const& word) {
    return this->find(word.c_str()) != this->end();
  }

};

} // Text
} // Tanl

// Mechanism to keep .cpp file separate
#ifndef NON_TEMPLATE
# include "WordSet.cpp"
#endif

#endif // Tanl_Text_WordSet_H

 

Copyright © 2005-2011 G. Attardi. Generated on 4 Mar 2011 by doxygen 1.6.1.