preprocess.regex
3.11 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
# ======== PREPROCESS REGEX ===========
#
# Author:
# ------
# Emmanuel FERREIRA
#
# Warning :
# --------
# this regex are first writted to process
# Bdlex formatted data
#
# How can use it ?
# ----------------
# You can comment, un-comment, correct
#
# =======================================
# to process already cleaned corpus
s/<\/s> <s>/\./g
s/<s>//g
s/<\/s>/\./g
# remove all characters between [ ] and [ ]
s/\[[^\[]*\]/ /g
# remove all characters between ( ) and ( )
s/([^)]*)/ /g
# remove all characters between < > and < >
s/<[^<]*>/ /g
# remove url (TODO convert url to txt)
s/\bhttp:[A-Za-z0-9\/_-\.=&?]\+//g
# preformat roman value XVe => XV e / Ier => I er
#s/\bIer\b/ I er /g
#s/\bIere\b/ I e2re /g
#s/\bIe2re\b/ I e2re /g
#s/\b\([IVX]\+\)e\b/ \1 e /g
# remove ponctuation _ \ * - #"
s/_\|\*\|\\\||\|-\|"\|#/ /g
# replace j'etais => j' etais
s/\([A-Za-z0-9]\)'\([A-Za-z0-9]\)/\1' \2/g
# replace 08:32 par 08h32
s/\(\b[0-9]\{2\}\):\([0-9]\{2\}\b\)/\1h\2/g
# replace : by .
s/:/\./g
# replace ! by .
s/\!/\./g
# replace ? by .
s/?/\./g
# replace break line sequence by " "
s/\n+/ /g
# replace . sequence by .
s/\.\+/\./g
# 2e2me => 2 e2me & 1e2re => 1 e2me
s/\b1e2re\b/ 1 e2re /g
s/\b\([0-9]\+\)e2me\b/ \1 e2me /g
# hours
s/\b00\(H\|h\)00\b/ minuit /g
s/\b00\(H\|h\)/ minuit /g
s/\b12\(H\|h\)00\b/ midi /g
s/\b12\(H\|h\)/ midi /g
s/\b0\([1-9]\)\(H\|h\)/ \1H/g
# 1/2 => 1 demi
s/\([0-9]\)\/2\b/\1 demi /g
# 1/4 => 4 quart
s/\([0-9]\)\/4\b/\1 quart /g
# 1/16 => 1 16 e
s/\b\([0-9]\)\/\([0-9][0-9]*\)\b/ \1 \2 e /g
# abbreviation
s/\./\. /g
s/\b[Mm]r\. / mister /g
s/\b[Mm]me\. / madame /g
s/\b[Mm]lle\. / mademoiselle /g
s/\b[Ll]t\. / lieutenant /g
s/\bM\. / monsieur /g
s/\b[Mm]rs\. / Messieurs /g
s/\bMgr\. / Monseigneur/g
s/\bDr\. / docteur /g
s/\bPr\. / professeur /g
s/\bPrs\. / professeurs /g
s/\bJ\. \?C\. / Je1sus Christ /g
s/\bSt\.\? / saint /g
s/\bSte\.\? / sainte /g
s/\bapr\?\. J\. \?C\. / apre2s Je1sus Christ /g
s/\bav\. J\. \?C\. / avant Je1sus Christ /g
# =><Months> (like avr. => avr)
s/\b[aA][vp]r\. / avril /g
s/\b[aA]ug\. / aou3t /g
s/\b[dD]\(e\|e1\)c\. / de1cembre /g
s/\b[fF]e[bv]\. / fe1vrier /g
s/\b[jJ]an\. / janvier /g
s/\b[jJ]ul\. / juillet /g
s/\b[jJ]un\. / juin /g
s/\b[mM]ar\. / mars /g
s/\b[nN]ov\. / novembre /g
s/\b[oO]ct\. / octobre /g
s/\b[sS]ept\. / septembre /g
#=></Months>
s/\b[Ss]te1\. / socie1te1 /g
s/\b[Nn]um\. / nume1ro /g
s/\b[Tt]\(e1\|e\)l\. / te1le1phone /g
s/\bart\. / article /g
s/\bann\. / annexe /g
s/\bappt\. / appartement /g
s/\bapr\. / apre2s /g
s/\bav\. / avant /g
s/\bba3t\. / ba3timent /g
s/\bbd\. / boulevard /g
s/\bboul\. / boulevard /g
s/\bBd\. / boulevard /g
s/\bchap\. / chapitre /g
s/\b[Cc]ie\. / compagnie /g
s/\bdir\. / directeur /g
s/\be2d\. / e2dition /g
s/\be2ds\. / e2ditions /g
s/\be\. \?g\. / par exemple /g
s/\bet al\. / et autres /g
s/\betc\. / et caetera /g
s/\bfe1m\. / fe1minin /g
s/\bfig\. / figure /g
s/\bi\. \?e\. / c' est a2 dire /g
s/\bmasc\. / masculin/g
s/\bN\.D\. / notre Dame /g
s/\bp\.\(\d+\)/ page \1/g
s/\bvs\. / versus /g
s/\bc\.\? \?a\.\? \?d\.\? / c' est a2 dire /g
# 10% => 10 pour cent
s/\([0-9]\)%/\1 pour cent /g
# +1 => plus 1
s/+\([0-9]\)/ plus \1/g
s/x\([0-9]\)/ fois \1/g