-This file is a part of an Information Retrieval oriented Ruby library -
-
-Copyright (C) 2010-2011 Romain Deveaud
-This program is free software: you can redistribute it and/or modify it -under the terms of the GNU General Public License as published by the Free -Software Foundation, either version 3 of the License, or (at your option) -any later version. -
--This program is distributed in the hope that it will be useful, but WITHOUT -ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -more details. -
--You should have received a copy of the GNU General Public License along -with this program. If not, see <www.gnu.org/licenses/>. +General module for many purposes related to Information Retrieval.
@@ -180,34 +160,14 @@ href="http://www.gnu.org/licenses/">www.gnu.org/licenses/>.| Stoplist | -= | -[ "a", "anything", "anyway", "anywhere", "apart", "are", "around", "as", "at", "av", "be", "became", "because", "become", "becomes", "becoming", "been", "before", "beforehand", "behind", "being", "below", "beside", "besides", "between", "beyond", "both", "but", "by", "can", "cannot", "canst", "certain", "cf", "choose", "contrariwise", "cos", "could", "cu", "day", "do", "does", "doesn't", "doing", "dost", "doth", "double", "down", "dual", "during", "each", "either", "else", "elsewhere", "enough", "et", "etc", "even", "ever", "every", "everybody", "everyone", "everything", "everywhere", "except", "excepted", "excepting", "exception", "exclude", "excluding", "exclusive", "far", "farther", "farthest", "few", "ff", "first", "for", "formerly", "forth", "forward", "from", "front", "further", "furthermore", "furthest", "get", "go", "had", "halves", "hardly", "has", "hast", "hath", "have", "he", "hence", "henceforth", "her", "here", "hereabouts", "hereafter", "hereby", "herein", "hereto", "hereupon", "hers", "herself", "him", "himself", "hindmost", "his", "hither", "hitherto", "how", "however", "howsoever", "i", "ie", "if", "in", "inasmuch", "inc", "include", "included", "including", "indeed", "indoors", "inside", "insomuch", "instead", "into", "inward", "inwards", "is", "it", "its", "itself", "just", "kind", "kg", "km", "last", "latter", "latterly", "less", "lest", "let", "like", "little", "ltd", "many", "may", "maybe", "me", "meantime", "meanwhile", "might", "moreover", "most", "mostly", "more", "mr", "mrs", "ms", "much", "must", "my", "myself", "namely", "need", "neither", "never", "nevertheless", "next", "no", "nobody", "none", "nonetheless", "noone", "nope", "nor", "not", "nothing", "notwithstanding", "now", "nowadays", "nowhere", "of", "off", "often", "ok", "on", "once", "one", "only", "onto", "or", "other", "others", "otherwise", "ought", "our", "ours", "ourselves", "out", "outside", "over", "own", "per", "perhaps", "plenty", "provide", "quite", "rather", "really", "round", "said", "sake", "same", "sang", "save", "saw", "see", "seeing", "seem", "seemed", "seeming", "seems", "seen", "seldom", "selves", "sent", "several", "shalt", "she", "should", "shown", "sideways", "since", "slept", "slew", "slung", "slunk", "smote", "so", "some", "somebody", "somehow", "someone", "something", "sometime", "sometimes", "somewhat", "somewhere", "spake", "spat", "spoke", "spoken", "sprang", "sprung", "stave", "staves", "still", "such", "supposing", "than", "that", "the", "thee", "their", "them", "themselves", "then", "thence", "thenceforth", "there", "thereabout", "thereabouts", "thereafter", "thereby", "therefore", "therein", "thereof", "thereon", "thereto", "thereupon", "these", "they", "this", "those", "thou", "though", "thrice", "through", "throughout", "thru", "thus", "thy", "thyself", "till", "to", "together", "too", "toward", "towards", "ugh", "unable", "under", "underneath", "unless", "unlike", "until", "up", "upon", "upward", "upwards", "us", "use", "used", "using", "very", "via", "vs", "want", "was", "we", "week", "well", "were", "what", "whatever", "whatsoever", "when", "whence", "whenever", "whensoever", "where", "whereabouts", "whereafter", "whereas", "whereat", "whereby", "wherefore", "wherefrom", "wherein", "whereinto", "whereof", "whereon", "wheresoever", "whereto", "whereunto", "whereupon", "wherever", "wherewith", "whether", "whew", "which", "whichever", "whichsoever", "while", "whilst", "whither", "who", "whoa", "whoever", "whole", "whom", "whomever", "whomsoever", "whose", "whosoever", "why", "will", "wilt", "with", "within", "without", "worse", "worst", "would", "wow", "ye", "yet", "year", "yippee", "you", "your", "yours", "yourself", "yourselves" ] | - -- | -These are the default stopwords provided by Lemur. - - | - -
- entropy("guitar") #=> 0.00389919463243839
+ entropy("guitar") #=> 0.00432114812727959
+ entropy("dillinger escape plan") #=> 0.265862076325102
+Computes the term frequency of a given word s. +
+
+ tf("guitar") #=> 0.000380372765310004
+
+
+ # File lib/rir/document.rb, line 31 + def format_words + wo = [] + + @doc_content.split.each do |w| + w.split(/\W/).each do |sw| + wo.push(sw.downcase) if sw =~ /[a-zA-Z]/ + end + end + + wo + end+ + diff --git a/doc/classes/RIR/Document.src/M000009.html b/doc/classes/RIR/Document.src/M000009.html new file mode 100644 index 0000000..6257629 --- /dev/null +++ b/doc/classes/RIR/Document.src/M000009.html @@ -0,0 +1,26 @@ + + + +
# File lib/rir/document.rb, line 46 + def ngrams(n) + window = [] + ngrams_array = [] + + @words.each do |w| + window.push(w) + if window.size == n + ngrams_array.push window.join(" ") + window.delete_at(0) + end + end + + ngrams_array.uniq + end+ + diff --git a/doc/classes/RIR/Document.src/M000010.html b/doc/classes/RIR/Document.src/M000010.html index 5056008..e8ddeec 100644 --- a/doc/classes/RIR/Document.src/M000010.html +++ b/doc/classes/RIR/Document.src/M000010.html @@ -2,22 +2,17 @@ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> -
# File lib/rir/document.rb, line 31 - def format_words - wo = [] +# File lib/rir/document.rb, line 64 + def count_words + counts = Hash.new { |h,k| h[k] = 0 } + @words.each { |w| counts[w] += 1 } - @doc_content.split.each do |w| - w.split(/\W/).each do |sw| - wo.push(sw) if sw =~ /[a-zA-Z]/ - end - end - - wo + counts enddiff --git a/doc/classes/RIR/Document.src/M000011.html b/doc/classes/RIR/Document.src/M000011.html index 6257629..50f6db7 100644 --- a/doc/classes/RIR/Document.src/M000011.html +++ b/doc/classes/RIR/Document.src/M000011.html @@ -2,25 +2,23 @@ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> -ngrams (RIR::Document) +entropy (RIR::Document) -# File lib/rir/document.rb, line 46 - def ngrams(n) - window = [] - ngrams_array = [] +# File lib/rir/document.rb, line 78 + def entropy(s) + en = 0.0 + counts = self.count_words - @words.each do |w| - window.push(w) - if window.size == n - ngrams_array.push window.join(" ") - window.delete_at(0) - end + s.split.each do |w| + p_wi = counts[w].to_f/@words.count.to_f + en += p_wi*Math.log2(p_wi) end - ngrams_array.uniq + en *= -1 + en enddiff --git a/doc/classes/RIR/Document.src/M000012.html b/doc/classes/RIR/Document.src/M000012.html index 978234a..eb4436b 100644 --- a/doc/classes/RIR/Document.src/M000012.html +++ b/doc/classes/RIR/Document.src/M000012.html @@ -2,17 +2,14 @@ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> -count_words (RIR::Document) +tf (RIR::Document) -# File lib/rir/document.rb, line 64 - def count_words - counts = Hash.new { |h,k| h[k] = 0 } - @words.each { |w| counts[w.downcase] += 1 } - - counts +# File lib/rir/document.rb, line 94 + def tf(s) + self.count_words[s].to_f/@words.size.to_f enddiff --git a/doc/classes/RIR/Document.src/M000013.html b/doc/classes/RIR/Document.src/M000013.html index 5694971..1ef96d5 100644 --- a/doc/classes/RIR/Document.src/M000013.html +++ b/doc/classes/RIR/Document.src/M000013.html @@ -2,23 +2,15 @@ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> -entropy (RIR::Document) +new (RIR::Document) -# File lib/rir/document.rb, line 77 - def entropy(s) - en = 0.0 - counts = self.count_words - - s.split.each do |w| - p_wi = counts[w].to_f/@words.count.to_f - en += p_wi*Math.log2(p_wi) - end - - en *= -1 - en +# File lib/rir/document.rb, line 99 + def initialize(content) + @doc_content = content + @words = format_words enddiff --git a/doc/classes/RIR/Indri/IndriQuery.html b/doc/classes/RIR/Indri/IndriQuery.html index 5bd769e..922525d 100644 --- a/doc/classes/RIR/Indri/IndriQuery.html +++ b/doc/classes/RIR/Indri/IndriQuery.html @@ -95,9 +95,9 @@
# File lib/rir/query.rb, line 62 + def initialize(id,query,params) + @params = params + # Here we set the default retrieval model as Language Modeling + # with a Dirichlet smoothing at 2500. + # TODO: maybe a Rule class... + @params.rule = 'method:dirichlet,mu:2500' if @params.rule.nil? + + @id = id + @query = query + end+ + diff --git a/doc/classes/RIR/Indri/IndriQuery.src/M000007.html b/doc/classes/RIR/Indri/IndriQuery.src/M000007.html new file mode 100644 index 0000000..e237f9a --- /dev/null +++ b/doc/classes/RIR/Indri/IndriQuery.src/M000007.html @@ -0,0 +1,22 @@ + + + +
# File lib/rir/query.rb, line 73 + def to_s + h = @params.to_s + h += "<query>\n" + h += "<number>#{@id}</number>\n" + h += "<text>#{@query}</text>\n" + h += "</query>\n" + h += "</parameters>" + + h + end+ + diff --git a/doc/classes/RIR/Indri/Parameters.html b/doc/classes/RIR/Indri/Parameters.html index 686469e..28e11cb 100644 --- a/doc/classes/RIR/Indri/Parameters.html +++ b/doc/classes/RIR/Indri/Parameters.html @@ -91,9 +91,9 @@
# File lib/rir/query.rb, line 30 + def initialize(corpus,mem="1g",count="1000",offset="1",run_id="default",print_query=false,print_docs=false) + @index_path = corpus + @memory = mem + @count = count + @offset = offset + @run_id = run_id + @print_query = print_query ? "true" : "false" + @print_docs = print_docs ? "true" : "false" + end+ + diff --git a/doc/classes/RIR/Indri/Parameters.src/M000005.html b/doc/classes/RIR/Indri/Parameters.src/M000005.html new file mode 100644 index 0000000..3a235ca --- /dev/null +++ b/doc/classes/RIR/Indri/Parameters.src/M000005.html @@ -0,0 +1,29 @@ + + + +
# File lib/rir/query.rb, line 40 + def to_s + h = "<parameters>\n" + h += "<memory>#{@memory}</memory>\n" + h += "<index>#{@index_path}</index>\n" + h += "<count>#{@count}</count>\n" + unless @baseline.nil? + h += "<baseline>#{@baseline}</baseline>\n" + else + h += "<rule>#{@rule}</rule>\n" + end + h += "<queryOffset>#{@offset}</queryOffset>\n" + h += "<runID>#{@run_id}</runID>\n" + h += "<printQuery>#{@print_query}</printQuery>\n" + h += "<printDocuments>#{@print_docs}</printDocuments>\n" + + h + end+ + diff --git a/doc/classes/RIR/TreeTagger.html b/doc/classes/RIR/TreeTagger.html new file mode 100644 index 0000000..a46f3bd --- /dev/null +++ b/doc/classes/RIR/TreeTagger.html @@ -0,0 +1,123 @@ + + + +
| Module | +RIR::TreeTagger | +
| In: | +
+
+
+
+
+ lib/rir/ttagger.rb
+
+
+
+
+ + + |
+
+TreeTagger-related stuff module. +
++See www.ims.uni-stuttgart.de/projekte/corplex/TreeTagger/DecisionTreeTagger.html +
+ +| Class | +RIR::TreeTagger::Chunk | +
| In: | +
+
+
+
+
+ lib/rir/ttagger.rb
+
+
+
+
+ + + |
+
| Parent: | ++ + Object + + | +
+Represents a Chunk extracted when parsing a TaggerChunker file. +
+ +| tag | + +[R] | + ++ |
| words | + +[R] | + ++ |
+str are whitespace-separated terms. tag see : ftp.ims.uni-stuttgart.de/pub/corpora/chunker-tagset-english.txt +
+ +# File lib/rir/ttagger.rb, line 86 + def initialize str,tag + @words = str.split + @tag = tag[1..-2] + end+ + diff --git a/doc/classes/RIR/TreeTagger/TaggerChunker.html b/doc/classes/RIR/TreeTagger/TaggerChunker.html new file mode 100644 index 0000000..57ae1ae --- /dev/null +++ b/doc/classes/RIR/TreeTagger/TaggerChunker.html @@ -0,0 +1,216 @@ + + + +
| Class | +RIR::TreeTagger::TaggerChunker | +
| In: | +
+
+
+
+
+ lib/rir/ttagger.rb
+
+
+
+
+ + + |
+
| Parent: | ++ + Object + + | +
+This class handles generic parsing of tagger-chunker outputs. +
+ +| chunks | + +[R] | + ++ |
| file | + +[R] | + ++ |
+Initializes parsing. chunk_file is the output of +tagger-chunker- and must be a valid path to the file. +
+
+ TaggerChunker.new("ttout/2010020") #=> #<RIR::TreeTagger::TaggerChunker:0x92fd088 @chunks=[#<RIR::TreeTagger::Chunk:0x8ec5a10 @words=["robert", "schumann"], @tag="NC">, ...] ...>
+
+
+ +Parses a tagger-chunker output and returns an Array of Chunk. +
+ +# File lib/rir/ttagger.rb, line 33 + def self.parse chunk_lines + open = false + tag = nil + + chunks = [] + words = [] + + chunk_lines.each do |l| + l.chomp! + if l =~ /^<\w+>$/ + open = true + tag = l + elsif l =~ /^<\/\w+>$/ + if !words.empty? && open && l == tag.sub(/</, '</') + open = false + chunks.push Chunk.new(words.join(" "), tag) + words.clear + else + next + end + else + words.push(l.split.first) + end + end + + chunks + end+ + diff --git a/doc/classes/RIR/TreeTagger/TaggerChunker.src/M000002.html b/doc/classes/RIR/TreeTagger/TaggerChunker.src/M000002.html new file mode 100644 index 0000000..c33487c --- /dev/null +++ b/doc/classes/RIR/TreeTagger/TaggerChunker.src/M000002.html @@ -0,0 +1,15 @@ + + + +
# File lib/rir/ttagger.rb, line 65 + def initialize chunk_file + @chunks = TaggerChunker.parse File.open(chunk_file).readlines + end+ + diff --git a/doc/classes/RIR/TreeTagger/TaggerChunkerEnglish.html b/doc/classes/RIR/TreeTagger/TaggerChunkerEnglish.html new file mode 100644 index 0000000..f5a878f --- /dev/null +++ b/doc/classes/RIR/TreeTagger/TaggerChunkerEnglish.html @@ -0,0 +1,114 @@ + + + +
| Class | +RIR::TreeTagger::TaggerChunkerEnglish | +
| In: | +
+
+
+
+
+ lib/rir/ttagger.rb
+
+
+
+
+ + + |
+
| Parent: | ++ + + + RIR::TreeTagger::TaggerChunker + + + + | +
| Class | +RIR::TreeTagger::TaggerChunkerFrench | +
| In: | +
+
+
+
+
+ lib/rir/ttagger.rb
+
+
+
+
+ + + |
+
| Parent: | ++ + + + RIR::TreeTagger::TaggerChunker + + + + | +
| Class | +RIR::TreeTagger::TaggerChunkerGerman | +
| In: | +
+
+
+
+
+ lib/rir/ttagger.rb
+
+
+
+
+ + + |
+
| Parent: | ++ + + + RIR::TreeTagger::TaggerChunker + + + + | +
# File lib/rir/document.rb, line 112 + def self.get_content(url) + require 'net/http' + Net::HTTP.get(URI.parse(url)) + end+ + diff --git a/doc/classes/RIR/WebDocument.src/M000015.html b/doc/classes/RIR/WebDocument.src/M000015.html index 54776b5..dd6da25 100644 --- a/doc/classes/RIR/WebDocument.src/M000015.html +++ b/doc/classes/RIR/WebDocument.src/M000015.html @@ -2,15 +2,15 @@ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> -
# File lib/rir/document.rb, line 105 - def self.get_content(url) - require 'net/http' - Net::HTTP.get(URI.parse(url)) +.# File lib/rir/document.rb, line 119 + def initialize(url) + @url = url + super WebDocument.get_content(url).strip_javascripts.strip_stylesheets.strip_xml_tags enddiff --git a/doc/classes/RIR/WikipediaPage.html b/doc/classes/RIR/WikipediaPage.html index 4464b3e..02b496f 100644 --- a/doc/classes/RIR/WikipediaPage.html +++ b/doc/classes/RIR/WikipediaPage.html @@ -103,11 +103,11 @@ href="WebDocument.html">WebDocument
# File lib/rir/document.rb, line 132 + def self.search_wikipedia_titles(name) + raise ArgumentError, "Bad encoding", name unless name.isutf8 + + res = REXML::Document.new(Net::HTTP.get( URI.parse "http://en.wikipedia.org/w/api.php?action=query&list=search&srsearch=#{URI.escape name}&format=xml" ).toutf8).elements['api/query/search'] + + res.collect { |e| e.attributes['title'] } unless res.nil? + end+ + diff --git a/doc/classes/RIR/WikipediaPage.src/M000017.html b/doc/classes/RIR/WikipediaPage.src/M000017.html new file mode 100644 index 0000000..0b6b98b --- /dev/null +++ b/doc/classes/RIR/WikipediaPage.src/M000017.html @@ -0,0 +1,19 @@ + + + +
# File lib/rir/document.rb, line 140 + def self.get_url(name) + raise ArgumentError, "Bad encoding", name unless name.isutf8 + + atts = REXML::Document.new(Net::HTTP.get( URI.parse "http://en.wikipedia.org/w/api.php?action=query&titles=#{URI.escape name}&inprop=url&prop=info&format=xml" ).toutf8).elements['api/query/pages/page'].attributes + + atts['fullurl'] if atts['missing'].nil? + end+ + diff --git a/doc/classes/RIR/WikipediaPage.src/M000018.html b/doc/classes/RIR/WikipediaPage.src/M000018.html new file mode 100644 index 0000000..d93d8db --- /dev/null +++ b/doc/classes/RIR/WikipediaPage.src/M000018.html @@ -0,0 +1,17 @@ + + + +
# File lib/rir/document.rb, line 148 + def self.search_homepage(name) + title = WikipediaPage.search_wikipedia_titles name + + WikipediaPage.new(WikipediaPage.get_url title[0]) unless title.nil? || title.empty? + end+ + diff --git a/doc/created.rid b/doc/created.rid index 5d2582c..5367b38 100644 --- a/doc/created.rid +++ b/doc/created.rid @@ -1 +1 @@ -Tue, 23 Nov 2010 18:20:46 +0100 +Thu, 25 Nov 2010 17:01:52 +0100 diff --git a/doc/files/lib/rir/document_rb.html b/doc/files/lib/rir/document_rb.html index 767c904..d5a6ac5 100644 --- a/doc/files/lib/rir/document_rb.html +++ b/doc/files/lib/rir/document_rb.html @@ -53,7 +53,7 @@
| Path: | +lib/rir/ttagger.rb + + | +
| Last Update: | +2010-11-25 17:01:46 +0100 | +
+This file is a part of an Information Retrieval oriented Ruby library +
+
+Copyright (C) 2010-2011 Romain Deveaud
+This program is free software: you can redistribute it and/or modify it +under the terms of the GNU General Public License as published by the Free +Software Foundation, either version 3 of the License, or (at your option) +any later version. +
++This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +more details. +
++You should have received a copy of the GNU General Public License along +with this program. If not, see <www.gnu.org/licenses/>. +
++This file is a part of an Information Retrieval oriented Ruby library +
+
+Copyright (C) 2010-2011 Romain Deveaud
+This program is free software: you can redistribute it and/or modify it +under the terms of the GNU General Public License as published by the Free +Software Foundation, either version 3 of the License, or (at your option) +any later version. +
++This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +more details. +
++You should have received a copy of the GNU General Public License along +with this program. If not, see <www.gnu.org/licenses/>. +
++General module for many purposes related to Information Retrieval. +
+ +