scoring.rb
3.7 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
#!/usr/bin/env ruby
require "rubygems"
require "text-table"
class ConfusionMatrix
def initialize
@h = Hash.new()
@total = 0
end
def keys
@h.each do |key, value|
yield key
end
end
def store(actual, truth)
@h[ actual ] ||= {"tp" => 0, "tn" => 0, "fp" => 0, "fn" => 0}
@h[ truth ] ||= {"tp" => 0, "tn" => 0, "fp" => 0, "fn" => 0}
if actual == truth
@h[ actual ]["tp"] += 1
else
@h[ actual ]["fp"] += 1
@h[ truth ]["fn"] += 1
@h[ truth ]["tn"] += 1
end
@total += 1
end
def recall(name)
t = @h[ name ]["tp"].to_f + @h[name]["fn"].to_f
return 0 if t == 0
return (@h[ name]["tp"].to_f / ( @h[ name]["tp"] + @h[name]["fn"] ).to_f )
end
def precision(name)
t = @h[ name ]["tp"].to_f + @h[name]["fp"].to_f
return 0 if t == 0
return (@h[ name]["tp"].to_f / ( @h[ name]["tp"] + @h[name]["fp"] ).to_f )
end
def fscore(name)
return (2 * precision(name) * recall(name) ) / ( precision(name) + recall(name) )
end
def score_semeval_2016
return ( fscore("positive") + fscore("negative") ) / 2
end
def score_semeval_2017
return ( recall("positive") + recall("negative") + recall("neutral") ) / 3
end
def accuracy
somme = 0
@h.each do |key, values|
somme += values["tp"]
end
return somme.to_f / @total.to_f
end
def macro_fscore
counter = 0
p = 0
keys do |key|
p += fscore(key)
counter += 1
end
return p.to_f/counter.to_f
end
def macro_precision
counter = 0
p = 0
keys do |key|
p += precision(key)
counter += 1
end
return p.to_f/counter.to_f
end
def macro_recall
counter = 0
p = 0
keys do |key|
p += recall(key)
counter += 1
end
return p.to_f/counter.to_f
end
end
def read_gold(file)
h = Hash.new
f = File.open(file)
f.each do |line|
line.chomp!
line = line.split("\t")
h[ line[0] ] = line[1]
end
f.close
return h
end
def launch(gold, predict)
h = read_gold(gold)
score = ConfusionMatrix.new
#convert = ["negative", "positive", "neutral"]
convert_task1 = ["negative", "positive", "objective", "mixed"]
convert_task2 = ["figurative", "nonfigurative"]
f = File.open(predict)
f.each do |line|
line.chomp!
line = line.split("\t")
if line[1].split(" ").size == 1
score.store( line[1], h[ line[0] ] )
else
s = line[1].split(" ").map{ |x| x.to_f }
if s.size == 2
score.store( convert_task2[ s.index(s.max) ], h[ line[0] ] )
else
score.store( convert_task1[ s.index(s.max) ], h[line[0]] )
end
end
end
f.close
table = Text::Table.new()
table.head = ["Classe", "Rappel", "Precision", "FMesure"]
score.keys do |key|
r = score.recall(key).round(4)
p = score.precision(key).round(4)
f = score.fscore(key).round(4)
table.rows << [key, r, p, f]
end
puts table.to_s
puts "Accuracy: #{score.accuracy.round(6)}"
puts "Macro-fscore: #{score.macro_fscore.round(6)}"
puts "Macro-recall: #{score.macro_recall.round(6)}"
puts "Macro-precision: #{score.macro_precision.round(6)}"
puts "\n\n\n"
end
def errarg
puts "Usage : ./programm.rb"
puts "Mickael Rouvier <mickael.rouvier@univ-avignon.fr>"
end
if ARGV.size == 2
launch(ARGV[0], ARGV[1])
else
errarg
end