#!/usr/bin/env ruby
  
  #--
  # This file is a part of the mirimiri library
  #
  # Copyright (C) 2010-2011 Romain Deveaud <romain.deveaud@gmail.com>
  #
  # This program is free software: you can redistribute it and/or modify
  # it under the terms of the GNU General Public License as published by
  # the Free Software Foundation, either version 3 of the License, or
  # (at your option) any later version.
  #
  # This program is distributed in the hope that it will be useful,
  # but WITHOUT ANY WARRANTY; without even the implied warranty of
  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  # GNU General Public License for more details.
  #
  # You should have received a copy of the GNU General Public License
  # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  #++
  
  module Mirimiri
  
    # These are the default stopwords provided by Lemur.
    Stoplist = [
    "a", "anything", "anyway", "anywhere", "apart", "are", "around", "as", "at", "av",
    "be", "became", "because", "become", "becomes", "becoming", "been", "before", "beforehand",
    "behind", "being", "below", "beside", "besides", "between", "beyond", "both", "but", "by",
    "can", "cannot", "canst", "certain", "cf", "choose", "contrariwise", "cos", "could", "cu",
    "day", "do", "does", "doesn't", "doing", "dost", "doth", "double", "down", "dual", "during",
    "each", "either", "else", "elsewhere", "enough", "et", "etc", "even", "ever", "every",
    "everybody", "everyone", "everything", "everywhere", "except", "excepted", "excepting",
    "exception", "exclude", "excluding", "exclusive", "far", "farther", "farthest", "few", "ff",
    "first", "for", "formerly", "forth", "forward", "from", "front", "further", "furthermore",
    "furthest", "get", "go", "had", "halves", "hardly", "has", "hast", "hath", "have", "he",
    "hence", "henceforth", "her", "here", "hereabouts", "hereafter", "hereby", "herein", "hereto",
    "hereupon", "hers", "herself", "him", "himself", "hindmost", "his", "hither", "hitherto",
    "how", "however", "howsoever", "i", "ie", "if", "in", "inasmuch", "inc", "include",
    "included", "including", "indeed", "indoors", "inside", "insomuch", "instead", "into",
    "inward", "inwards", "is", "it", "its", "itself", "just", "kind", "kg", "km", "last",
    "latter", "latterly", "less", "lest", "let", "like", "little", "ltd", "many", "may", "maybe",
    "me", "meantime", "meanwhile", "might", "moreover", "most", "mostly", "more", "mr", "mrs",
    "ms", "much", "must", "my", "myself", "namely", "need", "neither", "never", "nevertheless",
    "next", "no", "nobody", "none", "nonetheless", "noone", "nope", "nor", "not", "nothing",
    "notwithstanding", "now", "nowadays", "nowhere", "of", "off", "often", "ok", "on", "once",
    "one", "only", "onto", "or", "other", "others", "otherwise", "ought", "our", "ours",
    "ourselves", "out", "outside", "over", "own", "per", "perhaps", "plenty", "provide", "quite",
    "rather", "really", "round", "said", "sake", "same", "sang", "save", "saw", "see", "seeing",
    "seem", "seemed", "seeming", "seems", "seen", "seldom", "selves", "sent", "several", "shalt",
    "she", "should", "shown", "sideways", "since", "slept", "slew", "slung", "slunk", "smote",
    "so", "some", "somebody", "somehow", "someone", "something", "sometime", "sometimes",
    "somewhat", "somewhere", "spake", "spat", "spoke", "spoken", "sprang", "sprung", "stave",
    "staves", "still", "such", "supposing", "than", "that", "the", "thee", "their", "them",
    "themselves", "then", "thence", "thenceforth", "there", "thereabout", "thereabouts",
    "thereafter", "thereby", "therefore", "therein", "thereof", "thereon", "thereto", "thereupon",
    "these", "they", "this", "those", "thou", "though", "thrice", "through", "throughout", "thru",
    "thus", "thy", "thyself", "till", "to", "together", "too", "toward", "towards", "ugh",
    "unable", "under", "underneath", "unless", "unlike", "until", "up", "upon", "upward",
    "upwards", "us", "use", "used", "using", "very", "via", "vs", "want", "was", "we", "week",
    "well", "were", "what", "whatever", "whatsoever", "when", "whence", "whenever", "whensoever",
    "where", "whereabouts", "whereafter", "whereas", "whereat", "whereby", "wherefore",
    "wherefrom", "wherein", "whereinto", "whereof", "whereon", "wheresoever", "whereto",
    "whereunto", "whereupon", "wherever", "wherewith", "whether", "whew", "which", "whichever",
    "whichsoever", "while", "whilst", "whither", "who", "whoa", "whoever", "whole", "whom",
    "whomever", "whomsoever", "whose", "whosoever", "why", "will", "wilt", "with", "within",
    "without", "worse", "worst", "would", "wow", "ye", "yet", "year", "yippee", "you", "your",
    "yours", "yourself", "yourselves" 
    ]

    Transmap = { 
    "\xC3\x80" => "A", "\xC3\x81" => "A", "\xC3\x82" => "A", "\xC3\x83" => "A", 
    "\xC3\x84" => "A", "\xC3\x85" => "A", "\xC3\x86" => "AE","\xC3\x87" => "C", 
    "\xC3\x88" => "E", "\xC3\x89" => "E", "\xC3\x8A" => "E", "\xC3\x8B" => "E", 
    "\xC3\x8C" => "I", "\xC3\x8D" => "I", "\xC3\x8E" => "I", "\xC3\x8F" => "I", 
    "\xC3\x90" => "D", "\xC3\x91" => "N", "\xC3\x92" => "O", "\xC3\x93" => "O", 
    "\xC3\x94" => "O", "\xC3\x95" => "O", "\xC3\x96" => "O", "\xC3\x98" => "O", 
    "\xC3\x99" => "U", "\xC3\x9A" => "U", "\xC3\x9B" => "U", "\xC3\x9C" => "U", 
    "\xC3\x9D" => "Y", "\xC3\x9E" => "P", "\xC3\x9F" => "ss", 
    "\xC3\xA0" => "a", "\xC3\xA1" => "a", "\xC3\xA2" => "a", "\xC3\xA3" => "a", 
    "\xC3\xA4" => "a", "\xC3\xA5" => "a", "\xC3\xA6" => "ae","\xC3\xA7" => "c", 
    "\xC3\xA8" => "e", "\xC3\xA9" => "e", "\xC3\xAA" => "e", "\xC3\xAB" => "e", 
    "\xC3\xAC" => "i", "\xC3\xAD" => "i", "\xC3\xAE" => "i", "\xC3\xAF" => "i", 
    "\xC3\xB0" => "o", "\xC3\xB1" => "n", "\xC3\xB2" => "o", "\xC3\xB3" => "o",
    "\xC3\xB4" => "o", "\xC3\xB5" => "o", "\xC3\xB6" => "o", "\xC3\xB8" => "o", 
    "\xC3\xB9" => "u", "\xC3\xBA" => "u", "\xC3\xBB" => "u", "\xC3\xBC" => "u", 
    "\xC3\xBD" => "y", "\xC3\xBE" => "p", "\xC3\xBF" => "y",     
    "\xC4\x80" => "A", "\xC4\x81" => "a", "\xC4\x82" => "A", "\xC4\x83" => "a",
    "\xC4\x84" => "A", "\xC4\x85" => "a", "\xC4\x86" => "C", "\xC4\x87" => "c", 
    "\xC4\x88" => "C", "\xC4\x89" => "c", "\xC4\x8A" => "C", "\xC4\x8B" => "c", 
    "\xC4\x8C" => "C", "\xC4\x8D" => "c", "\xC4\x8E" => "D", "\xC4\x8F" => "d", 
    "\xC4\x90" => "D", "\xC4\x91" => "d", "\xC4\x92" => "E", "\xC4\x93" => "e", 
    "\xC4\x94" => "E", "\xC4\x95" => "e", "\xC4\x96" => "E", "\xC4\x97" => "e", 
    "\xC4\x98" => "E", "\xC4\x99" => "e", "\xC4\x9A" => "E", "\xC4\x9B" => "e", 
    "\xC4\x9C" => "G", "\xC4\x9D" => "g", "\xC4\x9E" => "G", "\xC4\x9F" => "g", 
    "\xC4\xA0" => "G", "\xC4\xA1" => "g", "\xC4\xA2" => "G", "\xC4\xA3" => "g", 
    "\xC4\xA4" => "H", "\xC4\xA5" => "h", "\xC4\xA6" => "H", "\xC4\xA7" => "h", 
    "\xC4\xA8" => "I", "\xC4\xA9" => "i", "\xC4\xAA" => "I", "\xC4\xAB" => "i", 
    "\xC4\xAC" => "I", "\xC4\xAD" => "i", "\xC4\xAE" => "I", "\xC4\xAF" => "i",
    "\xC4\xB0" => "I", "\xC4\xB1" => "i", "\xC4\xB2" => "IJ","\xC4\xB3" => "ij", 
    "\xC4\xB4" => "J", "\xC4\xB5" => "j", "\xC4\xB6" => "K", "\xC4\xB7" => "k", 
    "\xC4\xB8" => "k", "\xC4\xB9" => "L", "\xC4\xBA" => "l", "\xC4\xBB" => "L", 
    "\xC4\xBC" => "l", "\xC4\xBD" => "L", "\xC4\xBE" => "l", "\xC4\xBF" => "L", 
    "\xC5\x80" => "l", "\xC5\x81" => "L", "\xC5\x82" => "l", "\xC5\x83" => "N", 
    "\xC5\x84" => "n", "\xC5\x85" => "N", "\xC5\x86" => "n", "\xC5\x87" => "N", 
    "\xC5\x88" => "n", "\xC5\x89" => "n", "\xC5\x8A" => "N", "\xC5\x8B" => "n", 
    "\xC5\x8C" => "O", "\xC5\x8D" => "o", "\xC5\x8E" => "O", "\xC5\x8F" => "o", 
    "\xC5\x90" => "O", "\xC5\x91" => "o", "\xC5\x92" => "CE","\xC5\x93" => "ce", 
    "\xC5\x94" => "R", "\xC5\x95" => "r", "\xC5\x96" => "R", "\xC5\x97" => "r", 
    "\xC5\x98" => "R", "\xC5\x99" => "r", "\xC5\x9A" => "S", "\xC5\x9B" => "s", 
    "\xC5\x9C" => "S", "\xC5\x9D" => "s", "\xC5\x9E" => "S", "\xC5\x9F" => "s", 
    "\xC5\xA0" => "S", "\xC5\xA1" => "s", "\xC5\xA2" => "T", "\xC5\xA3" => "t", 
    "\xC5\xA4" => "T", "\xC5\xA5" => "t", "\xC5\xA6" => "T", "\xC5\xA7" => "t", 
    "\xC5\xA8" => "U", "\xC5\xA9" => "u", "\xC5\xAA" => "U", "\xC5\xAB" => "u", 
    "\xC5\xAC" => "U", "\xC5\xAD" => "u", "\xC5\xAE" => "U", "\xC5\xAF" => "u", 
    "\xC5\xB0" => "U", "\xC5\xB1" => "u", "\xC5\xB2" => "U", "\xC5\xB3" => "u", 
    "\xC5\xB4" => "W", "\xC5\xB5" => "w", "\xC5\xB6" => "Y", "\xC5\xB7" => "y", 
    "\xC5\xB8" => "Y", "\xC5\xB9" => "Z", "\xC5\xBA" => "z", "\xC5\xBB" => "Z", 
    "\xC5\xBC" => "z", "\xC5\xBD" => "Z", "\xC5\xBE" => "z", "\xC6\x8F" => "E", 
    "\xC6\xA0" => "O", "\xC6\xA1" => "o", "\xC6\xAF" => "U", "\xC6\xB0" => "u", 
    "\xC7\x8D" => "A", "\xC7\x8E" => "a", "\xC7\x8F" => "I",     
    "\xC7\x90" => "i", "\xC7\x91" => "O", "\xC7\x92" => "o", "\xC7\x93" => "U", 
    "\xC7\x94" => "u", "\xC7\x95" => "U", "\xC7\x96" => "u", "\xC7\x97" => "U", 
    "\xC7\x98" => "u", "\xC7\x99" => "U", "\xC7\x9A" => "u", "\xC7\x9B" => "U", 
    "\xC7\x9C" => "u", 
    "\xC7\xBA" => "A", "\xC7\xBB" => "a", "\xC7\xBC" => "AE","\xC7\xBD" => "ae", 
    "\xC7\xBE" => "O", "\xC7\xBF" => "o", 
    "\xC9\x99" => "e", 
    "\xC2\x82" => ",",        # High code comma 
    "\xC2\x84" => ",,",       # High code double comma 
    "\xC2\x85" => "...",      # Tripple dot 
    "\xC2\x88" => "^",        # High carat 
    "\xC2\x91" => "\x27",     # Forward single quote 
    "\xC2\x92" => "\x27",     # Reverse single quote 
    "\xC2\x93" => "\x22",     # Forward double quote 
    "\xC2\x94" => "\x22",     # Reverse double quote 
    "\xC2\x96" => "-",        # High hyphen 
    "\xC2\x97" => "--",       # Double hyphen 
    "\xC2\xA6" => "|",        # Split vertical bar 
    "\xC2\xAB" => "<<",       # Double less than 
    "\xC2\xBB" => ">>",       # Double greater than 
    "\xC2\xBC" => "1/4",      # one quarter 
    "\xC2\xBD" => "1/2",      # one half 
    "\xC2\xBE" => "3/4",      # three quarters 
    "\xCA\xBF" => "\x27",     # c-single quote 
    "\xCC\xA8" => "",         # modifier - under curve 
    "\xCC\xB1" => "",         # modifier - under line 
    /\W/ => ""
    } 

  
  end
  
  # Extention of the standard class String with useful function.
  class String
    include Mirimiri

    def unaccent
      # force_encoding is needed with ruby1.9
      Transmap.inject(self.force_encoding("ASCII-8BIT")) { |str, (utf8, asc)| str.gsub(utf8, asc) }
    end

    # Returns +true+ if +self+ belongs to Rir::Stoplist, +false+ otherwise.
    def is_stopword?
      Stoplist.include?(self.downcase)
    end
  
    # Do not use.
    # TODO: rewamp. find why this function is here.
    def remove_special_characters
      self.split.collect { |w| w.gsub(/\W/,' ').split.collect { |w| w.gsub(/\W/,' ').strip.sub(/\A.\z/, '')}.join(' ').strip.sub(/\A.\z/, '')}.join(' ')
    end
  
    # Removes all XML-like tags from +self+.
    #
    #   s = "<html><body>test</body></html>"
    #   s.strip_xml_tags!                 
    #   s                                     #=> "test"
    def strip_xml_tags!
      replace strip_with_pattern /<\/?[^>]*>/
    end
  
    # Removes all XML-like tags from +self+.
    #
    #   s = "<html><body>test</body></html>"
    #   s.strip_xml_tags                      #=> "test"
    #   s                                     #=> "<html><body>test</body></html>"
    def strip_xml_tags
      dup.strip_xml_tags!
    end
  
    # Removes all Javascript sources from +self+.
    #
    #   s = "<script type='text/javascript'> 
    #         var skin='vector',
    #         stylepath='http://bits.wikimedia.org/skins-1.5'
    #        </script>
    #
    #        test"
    #   s.strip_javascripts!                                  
    #   s                                     #=> "test"
    def strip_javascripts!
      replace strip_with_pattern /<script type="text\/javascript">(.+?)<\/script>/m 
    end
  
    # Removes all Javascript sources from +self+.
    #
    #   s = "<script type='text/javascript'> 
    #         var skin='vector',
    #         stylepath='http://bits.wikimedia.org/skins-1.5'
    #        </script>
    #
    #        test"
    #   s.strip_javascripts                   #=> "test"                                  
    def strip_javascripts
      dup.strip_javascripts!
    end
  
    def strip_stylesheets!
    # TODO: rewamp. dunno what is it.
      replace strip_with_pattern /<style type="text\/css">(.+?)<\/style>/m 
    end
  
    def strip_stylesheets
      dup.strip_stylesheets!
    end
  
    # Removes punctuation from +self+.
    #
    #   s = "hello, world. how are you?!"
    #   s.strip_punctuation!
    #   s                                 # => "hello world how are you"
    def strip_punctuation!
      replace strip_with_pattern /[^a-zA-Z0-9\-\s]/
    end
  
    # Removes punctuation from +self+.
    #
    #   s = "hello, world. how are you?!"
    #   s.strip_punctuation               # => "hello world how are you"
    def strip_punctuation
      dup.strip_punctuation!
    end
  
    # Returns the text values inside all occurences of a XML tag in +self+
    #
    #   s = "four-piece in <a href='#'>Indianapolis</a>, <a href='#'>Indiana</a> at the Murat Theatre"
    #   s.extract_xmltags_values 'a' #=> ["Indianapolis", "Indiana"]
    def extract_xmltags_values(tag_name)
      self.scan(/<#{tag_name}.*?>(.+?)<\/#{tag_name}>/).flatten
    end
  
    def strip_with_pattern(pattern)
      require 'cgi'
      require 'kconv'
      CGI::unescapeHTML(self.gsub(pattern,"")).toutf8
    end
  
    private :strip_with_pattern
  end