validate_text.pl
4.39 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
#!/usr/bin/env perl
#
#===============================================================================
# Copyright 2017 Johns Hopkins University (author: Yenda Trmal <jtrmal@gmail.com>)
# Johns Hopkins University (author: Daniel Povey)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
# WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
# MERCHANTABLITY OR NON-INFRINGEMENT.
# See the Apache 2 License for the specific language governing permissions and
# limitations under the License.
#===============================================================================
# validation script for data/<dataset>/text
# to be called (preferably) from utils/validate_data_dir.sh
use strict;
use warnings;
use utf8;
use Fcntl qw< SEEK_SET >;
# this function reads the opened file (supplied as a first
# parameter) into an array of lines. For each
# line, it tests whether it's a valid utf-8 compatible
# line. If all lines are valid utf-8, it returns the lines
# decoded as utf-8, otherwise it assumes the file's encoding
# is one of those 1-byte encodings, such as ISO-8859-x
# or Windows CP-X.
# Please recall we do not really care about
# the actually encoding, we just need to
# make sure the length of the (decoded) string
# is correct (to make the output formatting looking right).
sub get_utf8_or_bytestream {
use Encode qw(decode encode);
my $is_utf_compatible = 1;
my @unicode_lines;
my @raw_lines;
my $raw_text;
my $lineno = 0;
my $file = shift;
while (<$file>) {
$raw_text = $_;
last unless $raw_text;
if ($is_utf_compatible) {
my $decoded_text = eval { decode("UTF-8", $raw_text, Encode::FB_CROAK) } ;
$is_utf_compatible = $is_utf_compatible && defined($decoded_text);
push @unicode_lines, $decoded_text;
} else {
#print STDERR "WARNING: the line $raw_text cannot be interpreted as UTF-8: $decoded_text\n";
;
}
push @raw_lines, $raw_text;
$lineno += 1;
}
if (!$is_utf_compatible) {
return (0, @raw_lines);
} else {
return (1, @unicode_lines);
}
}
# check if the given unicode string contain unicode whitespaces
# other than the usual four: TAB, LF, CR and SPACE
sub validate_utf8_whitespaces {
my $unicode_lines = shift;
use feature 'unicode_strings';
for (my $i = 0; $i < scalar @{$unicode_lines}; $i++) {
my $current_line = $unicode_lines->[$i];
if ((substr $current_line, -1) ne "\n"){
print STDERR "$0: The current line (nr. $i) has invalid newline\n";
return 1;
}
my @A = split(" ", $current_line);
my $utt_id = $A[0];
# we replace TAB, LF, CR, and SPACE
# this is to simplify the test
if ($current_line =~ /\x{000d}/) {
print STDERR "$0: The line for utterance $utt_id contains CR (0x0D) character\n";
return 1;
}
$current_line =~ s/[\x{0009}\x{000a}\x{0020}]/./g;
if ($current_line =~/\s/) {
print STDERR "$0: The line for utterance $utt_id contains disallowed Unicode whitespaces\n";
return 1;
}
}
return 0;
}
# checks if the text in the file (supplied as the argument) is utf-8 compatible
# if yes, checks if it contains only allowed whitespaces. If no, then does not
# do anything. The function seeks to the original position in the file after
# reading the text.
sub check_allowed_whitespace {
my $file = shift;
my $filename = shift;
my $pos = tell($file);
(my $is_utf, my @lines) = get_utf8_or_bytestream($file);
seek($file, $pos, SEEK_SET);
if ($is_utf) {
my $has_invalid_whitespaces = validate_utf8_whitespaces(\@lines);
if ($has_invalid_whitespaces) {
print STDERR "$0: ERROR: text file '$filename' contains disallowed UTF-8 whitespace character(s)\n";
return 0;
}
}
return 1;
}
if(@ARGV != 1) {
die "Usage: validate_text.pl <text-file>\n" .
"e.g.: validate_text.pl data/train/text\n";
}
my $text = shift @ARGV;
if (-z "$text") {
print STDERR "$0: ERROR: file '$text' is empty or does not exist\n";
exit 1;
}
if(!open(FILE, "<$text")) {
print STDERR "$0: ERROR: failed to open $text\n";
exit 1;
}
check_allowed_whitespace(\*FILE, $text) or exit 1;
close(FILE);