Blame view
egs/wsj/s5/utils/create_split_dir.pl
2.43 KB
8dcb6dfcb
first commit
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 |
#!/usr/bin/env perl
# Copyright 2013 Guoguo Chen
# Apache 2.0.
#
# This script creates storage directories on different file systems, and creates
# symbolic links to those directories. For example, a command
#
# utils/create_split_dir.pl /export/gpu-0{3,4,5}/egs/storage egs/storage
#
# will mkdir -p all of those directories, and will create links
#
# egs/storage/1 -> /export/gpu-03/egs/storage
# egs/storage/2 -> /export/gpu-03/egs/storage
# ...
#
use strict;
use warnings;
use File::Spec;
use Getopt::Long;
my $Usage = <<EOU;
create_split_dir.pl:
This script creates storage directories on different file systems, and creates
symbolic links to those directories.
Usage: utils/create_split_dir.pl <actual_storage_dirs> <pseudo_storage_dir>
e.g.: utils/create_split_dir.pl /export/gpu-0{3,4,5}/egs/storage egs/storage
Allowed options:
--suffix : Common suffix to <actual_storage_dirs> (string, default = "")
See also create_data_link.pl, which is intended to work with the resulting
directory structure, and remove_data_links.sh
EOU
my $suffix="";
GetOptions('suffix=s' => \$suffix);
if (@ARGV < 2) {
die $Usage;
}
my $ans = 1;
my $dir = pop(@ARGV);
system("mkdir -p $dir 2>/dev/null");
my @all_actual_storage = ();
foreach my $file (@ARGV) {
push @all_actual_storage, File::Spec->rel2abs($file . "/" . $suffix);
}
my $index = 1;
foreach my $actual_storage (@all_actual_storage) {
my $pseudo_storage = "$dir/$index";
# If the symbolic link already exists, delete it.
if (-l $pseudo_storage) {
print STDERR "$0: link $pseudo_storage already exists, not overwriting.
";
$index++;
next;
}
# Create the destination directory and make the link.
system("mkdir -p $actual_storage 2>/dev/null");
if ($? != 0) {
print STDERR "$0: error creating directory $actual_storage
";
exit(1);
}
{ # create a README file for easier deletion.
open(R, ">$actual_storage/README.txt");
my $storage_dir = File::Spec->rel2abs($dir);
print R "# This directory is linked from $storage_dir, as part of Kaldi striped data
";
print R "# The full list of directories where this data resides is:
";
foreach my $d (@all_actual_storage) {
print R "$d
";
}
close(R);
}
my $ret = symlink($actual_storage, $pseudo_storage);
# Process the returned values
$ans = $ans && $ret;
if (! $ret) {
print STDERR "Error linking $actual_storage to $pseudo_storage
";
}
$index++;
}
exit($ans == 1 ? 0 : 1);
|