mapped-file.cc
3.79 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
// See www.openfst.org for extensive documentation on this weighted
// finite-state transducer library.
//
#include <fst/mapped-file.h>
#include <errno.h>
#include <fcntl.h>
#include <sys/mman.h>
#include <unistd.h>
#include <algorithm>
#include <ios>
#include <memory>
#include <fst/log.h>
namespace fst {
MappedFile::MappedFile(const MemoryRegion ®ion) : region_(region) {}
MappedFile::~MappedFile() {
if (region_.size != 0) {
if (region_.mmap) {
VLOG(1) << "munmap'ed " << region_.size << " bytes at " << region_.mmap;
if (munmap(region_.mmap, region_.size) != 0) {
LOG(ERROR) << "Failed to unmap region: " << strerror(errno);
}
} else {
if (region_.data) {
operator delete(static_cast<char *>(region_.data) - region_.offset);
}
}
}
}
MappedFile *MappedFile::Map(std::istream *istrm, bool memorymap,
const string &source, size_t size) {
const auto spos = istrm->tellg();
VLOG(1) << "memorymap: " << (memorymap ? "true" : "false") << " source: \""
<< source << "\""
<< " size: " << size << " offset: " << spos;
if (memorymap && spos >= 0 && spos % kArchAlignment == 0) {
const size_t pos = spos;
int fd = open(source.c_str(), O_RDONLY);
if (fd != -1) {
const int pagesize = sysconf(_SC_PAGESIZE);
const off_t offset = pos % pagesize;
const off_t upsize = size + offset;
void *map =
mmap(nullptr, upsize, PROT_READ, MAP_SHARED, fd, pos - offset);
auto *data = reinterpret_cast<char *>(map);
if (close(fd) == 0 && map != MAP_FAILED) {
MemoryRegion region;
region.mmap = map;
region.size = upsize;
region.data = reinterpret_cast<void *>(data + offset);
region.offset = offset;
std::unique_ptr<MappedFile> mmf(new MappedFile(region));
istrm->seekg(pos + size, std::ios::beg);
if (istrm) {
VLOG(1) << "mmap'ed region of " << size << " at offset " << pos
<< " from " << source << " to addr " << map;
return mmf.release();
}
} else {
LOG(INFO) << "Mapping of file failed: " << strerror(errno);
}
}
}
// If all else fails, reads from the file into the allocated buffer.
if (memorymap) {
LOG(WARNING) << "File mapping at offset " << spos << " of file " << source
<< " could not be honored, reading instead";
}
// Reads the file into the buffer in chunks not larger than kMaxReadChunk.
std::unique_ptr<MappedFile> mf(Allocate(size));
auto *buffer = reinterpret_cast<char *>(mf->mutable_data());
while (size > 0) {
const auto next_size = std::min(size, kMaxReadChunk);
const auto current_pos = istrm->tellg();
if (!istrm->read(buffer, next_size)) {
LOG(ERROR) << "Failed to read " << next_size << " bytes at offset "
<< current_pos << "from \"" << source << "\"";
return nullptr;
}
size -= next_size;
buffer += next_size;
VLOG(2) << "Read " << next_size << " bytes. " << size << " remaining";
}
return mf.release();
}
MappedFile *MappedFile::Allocate(size_t size, int align) {
MemoryRegion region;
region.data = nullptr;
region.offset = 0;
if (size > 0) {
char *buffer = static_cast<char *>(operator new(size + align));
size_t address = reinterpret_cast<size_t>(buffer);
region.offset = kArchAlignment - (address % align);
region.data = buffer + region.offset;
}
region.mmap = nullptr;
region.size = size;
return new MappedFile(region);
}
MappedFile *MappedFile::Borrow(void *data) {
MemoryRegion region;
region.data = data;
region.mmap = data;
region.size = 0;
region.offset = 0;
return new MappedFile(region);
}
constexpr int MappedFile::kArchAlignment;
constexpr size_t MappedFile::kMaxReadChunk;
} // namespace fst