Tanl Linguistic Pipeline

parse/desr/src/WordCounts.h

/*
**  DeSR
**  src/WordCounts.h
**  ----------------------------------------------------------------------
**  Copyright (c) 2008  Giuseppe Attardi (attardi@di.unipi.it).
**  ----------------------------------------------------------------------
**
**  This file is part of DeSR.
**
**  DeSR is free software; you can redistribute it and/or modify it
**  under the terms of the GNU General Public License, version 3,
**  as published by the Free Software Foundation.
**
**  DeSR is distributed in the hope that it will be useful,
**  but WITHOUT ANY WARRANTY; without even the implied warranty of
**  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
**  GNU General Public License for more details.
**
**  You should have received a copy of the GNU General Public License
**  along with this program.  If not, see <http://www.gnu.org/licenses/>.
**  ----------------------------------------------------------------------
*/

#ifndef DeSR_WordCounts_H
#define DeSR_WordCounts_H

// settings
#include "include/unordered_map.h"

// IXE library
#include "lib/strtok_r.h"

// standard
#include <string>

namespace Parser {

struct WordCounts : public unordered_map<std::string, int> {
  int add(std::string const& w) {
    iterator wcit = find(w);
    // increment # of occurrences
    int count;
    if (wcit == end())
      count = operator[](w) = 1;
    else
      count = ++wcit->second;
    return count;
  }
  int count(std::string const& w) {
    iterator wcit = find(w);
    return (wcit == end()) ? 0 : wcit->second;
  }
};

class WordFreq : public unordered_map<const char*, float, hash<const char*>, eqstr>
{
public:

  typedef unordered_map<const char*, float, hash<const char*>, eqstr> HashMap;

  WordFreq(char const* file) { load(file); }
  WordFreq(std::string& file) { load(file.c_str()); }

  ~WordFreq() {
    for (iterator it = this->begin(); it != this->end(); ++it) 
      free((void*)it->first);           // allocated by strdup()
  }

  bool  contains(char const* ngram) {
    return this->find(ngram) != this->end();
  }

  bool  contains(std::string const& ngram) {
    return this->find(ngram.c_str()) != this->end();
  }

  std::pair<iterator,bool> insert(const value_type& __obj) {
    iterator _Where = this->find(__obj.first);
    key_type _Keyval = (_Where == this->end()) ?
      ::strdup(__obj.first) :
      _Where->first;
    return HashMap::insert(value_type(_Keyval, __obj.second));
  }

  mapped_type& operator[](const key_type& _Keyval) {
    // find element matching _Keyval or insert with default mapped
    iterator _Where = this->find(_Keyval);
    if (_Where == this->end())
      _Where = this->insert(value_type(_Keyval, mapped_type())).first;
    return _Where->second;
  }

  void  load(char const* file) {
    std::ifstream ifs(file);
    load(ifs);
  }

  void  load(std::ifstream& ifs) {
# define MAX_LINE_LEN 4096
    char line[MAX_LINE_LEN];

    while (ifs.getline(line, MAX_LINE_LEN)) {
      char* next = line;
      char const* ngram = strtok_r(0, "\t", &next);
      float count = atof(strtok_r(0, " ", &next));
      insert(std::make_pair(ngram, count));
    }
#  undef MAX_LINE_LEN
  }
};

} // namespace Parser

#endif // DeSR_WordCounts_H

 

Copyright © 2005-2011 G. Attardi. Generated on 4 Mar 2011 by doxygen 1.6.1.