an entropy computation example. words in an RIR::Document are now lowercased.

Romain Deveaud
1 parent 145387519e
Showing 3 changed files with 12 additions and 9 deletions Side-by-side Diff
examples/entropy.rb
lib/rir/document.rb
main.rb
+require 'rir'
+
+# Concatenates all lines from one file, without \n
+readme = File.open('README.markdown').readlines.collect { |l| l.chomp }.join(" ")
+
+# Creates the document with a string
+doc = RIR::Document.new readme
+
+# Outputs all the unique words of the document with their entropy scores
+p doc.words.collect { |w| "#{w} => #{doc.entropy w}" }
@@ -33,7 +33,7 @@
  
       @doc_content.split.each do |w|
         w.split(/\W/).each do |sw| 
-          wo.push(sw) if sw =~ /[a-zA-Z]/ 
+          wo.push(sw.downcase) if sw =~ /[a-zA-Z]/ 
         end
       end
  
@@ -63,7 +63,7 @@
     #   count_words #=> { "guitar"=>1, "bass"=>3, "album"=>20, ... } 
     def count_words
       counts = Hash.new { |h,k| h[k] = 0 }
-      @words.each { |w| counts[w.downcase] += 1 }
+      @words.each { |w| counts[w] += 1 }
  
       counts
     end
@@ -4,11 +4,4 @@
  
 w = RIR::WikipediaPage.new("http://en.wikipedia.org/wiki/The_Dillinger_Escape_Plan")
 p w.entropy("guitar")
-
-params = RIR::Indri::Parameters.new("path_vers_mon_index")
-q = RIR::Indri::IndriQuery.new("pouet", "bla", params)
-puts q
-
-c = RIR::Corpus.new "/home/romain/INEX/BookTrack/corpus/"
-puts c.files.size
	1	+require 'rir'
	2	+
	3	+# Concatenates all lines from one file, without \n
	4	+readme = File.open('README.markdown').readlines.collect { \|l\| l.chomp }.join(" ")
	5	+
	6	+# Creates the document with a string
	7	+doc = RIR::Document.new readme
	8	+
	9	+# Outputs all the unique words of the document with their entropy scores
	10	+p doc.words.collect { \|w\| "#{w} => #{doc.entropy w}" }
...	...	@@ -33,7 +33,7 @@
33	33
34	34	@doc_content.split.each do \|w\|
35	35	w.split(/\W/).each do \|sw\|
36		- wo.push(sw) if sw =~ /[a-zA-Z]/
	36	+ wo.push(sw.downcase) if sw =~ /[a-zA-Z]/
37	37	end
38	38	end
39	39
...	...	@@ -63,7 +63,7 @@
63	63	# count_words #=> { "guitar"=>1, "bass"=>3, "album"=>20, ... }
64	64	def count_words
65	65	counts = Hash.new { \|h,k\| h[k] = 0 }
66		- @words.each { \|w\| counts[w.downcase] += 1 }
	66	+ @words.each { \|w\| counts[w] += 1 }
67	67
68	68	counts
69	69	end
...	...	@@ -4,11 +4,4 @@
4	4
5	5	w = RIR::WikipediaPage.new("http://en.wikipedia.org/wiki/The_Dillinger_Escape_Plan")
6	6	p w.entropy("guitar")
7		-
8		-params = RIR::Indri::Parameters.new("path_vers_mon_index")
9		-q = RIR::Indri::IndriQuery.new("pouet", "bla", params)
10		-puts q
11		-
12		-c = RIR::Corpus.new "/home/romain/INEX/BookTrack/corpus/"
13		-puts c.files.size