Blame view
bin/scoring.rb
3.7 KB
362b552ee
upload system
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 |
#!/usr/bin/env ruby
require "rubygems"
require "text-table"
class ConfusionMatrix
def initialize
@h = Hash.new()
@total = 0
end
def keys
@h.each do |key, value|
yield key
end
end
def store(actual, truth)
@h[ actual ] ||= {"tp" => 0, "tn" => 0, "fp" => 0, "fn" => 0}
@h[ truth ] ||= {"tp" => 0, "tn" => 0, "fp" => 0, "fn" => 0}
if actual == truth
@h[ actual ]["tp"] += 1
else
@h[ actual ]["fp"] += 1
@h[ truth ]["fn"] += 1
@h[ truth ]["tn"] += 1
end
@total += 1
end
def recall(name)
t = @h[ name ]["tp"].to_f + @h[name]["fn"].to_f
return 0 if t == 0
return (@h[ name]["tp"].to_f / ( @h[ name]["tp"] + @h[name]["fn"] ).to_f )
end
def precision(name)
t = @h[ name ]["tp"].to_f + @h[name]["fp"].to_f
return 0 if t == 0
return (@h[ name]["tp"].to_f / ( @h[ name]["tp"] + @h[name]["fp"] ).to_f )
end
def fscore(name)
return (2 * precision(name) * recall(name) ) / ( precision(name) + recall(name) )
end
def score_semeval_2016
return ( fscore("positive") + fscore("negative") ) / 2
end
def score_semeval_2017
return ( recall("positive") + recall("negative") + recall("neutral") ) / 3
end
def accuracy
somme = 0
@h.each do |key, values|
somme += values["tp"]
end
return somme.to_f / @total.to_f
end
def macro_fscore
counter = 0
p = 0
keys do |key|
p += fscore(key)
counter += 1
end
return p.to_f/counter.to_f
end
def macro_precision
counter = 0
p = 0
keys do |key|
p += precision(key)
counter += 1
end
return p.to_f/counter.to_f
end
def macro_recall
counter = 0
p = 0
keys do |key|
p += recall(key)
counter += 1
end
return p.to_f/counter.to_f
end
end
def read_gold(file)
h = Hash.new
f = File.open(file)
f.each do |line|
line.chomp!
line = line.split("\t")
h[ line[0] ] = line[1]
end
f.close
return h
end
def launch(gold, predict)
h = read_gold(gold)
score = ConfusionMatrix.new
#convert = ["negative", "positive", "neutral"]
convert_task1 = ["negative", "positive", "objective", "mixed"]
convert_task2 = ["figurative", "nonfigurative"]
f = File.open(predict)
f.each do |line|
line.chomp!
line = line.split("\t")
if line[1].split(" ").size == 1
score.store( line[1], h[ line[0] ] )
else
s = line[1].split(" ").map{ |x| x.to_f }
if s.size == 2
score.store( convert_task2[ s.index(s.max) ], h[ line[0] ] )
else
score.store( convert_task1[ s.index(s.max) ], h[line[0]] )
end
end
end
f.close
table = Text::Table.new()
table.head = ["Classe", "Rappel", "Precision", "FMesure"]
score.keys do |key|
r = score.recall(key).round(4)
p = score.precision(key).round(4)
f = score.fscore(key).round(4)
table.rows << [key, r, p, f]
end
puts table.to_s
puts "Accuracy: #{score.accuracy.round(6)}"
puts "Macro-fscore: #{score.macro_fscore.round(6)}"
puts "Macro-recall: #{score.macro_recall.round(6)}"
puts "Macro-precision: #{score.macro_precision.round(6)}"
puts "
"
end
def errarg
puts "Usage : ./programm.rb"
puts "Mickael Rouvier <mickael.rouvier@univ-avignon.fr>"
end
if ARGV.size == 2
launch(ARGV[0], ARGV[1])
else
errarg
end
|