1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
|
// file : libbuild2/file-cache.cxx -*- C++ -*-
// license : MIT; see accompanying LICENSE file
#include <libbuild2/file-cache.hxx>
#include <libbutl/lz4.hxx>
#include <libbutl/filesystem.mxx> // entry_stat, path_entry()
#include <libbuild2/filesystem.hxx> // exists(), try_rmfile()
#include <libbuild2/diagnostics.hxx>
using namespace butl;
namespace build2
{
// file_cache::entry
//
file_cache::write file_cache::entry::
init_new ()
{
assert (state_ == uninit);
// Remove stale compressed file if exists. While not strictly necessary
// (since the presence of the new uncompressed file will render the
// compressed one invalid), this makes things cleaner in case we don't get
// to compressing the new file (for example, if we fail and leave the
// uncompressed file behind for troubleshooting).
//
if (!comp_path_.empty ())
try_rmfile_ignore_error (comp_path_);
pin ();
return write (*this);
}
void file_cache::entry::
init_existing ()
{
assert (state_ == uninit);
bool c (!comp_path_.empty ());
// Determine the cache state from the filesystem state.
//
// First check for the uncompressed file. Its presence means that the
// compressed file, if exists, is invalid and we clean it up, similar to
// init_new().
//
if (exists (path_))
{
if (c)
try_rmfile_ignore_error (comp_path_);
state_ = uncomp;
}
else if (c && exists (comp_path_))
{
state_ = comp;
}
else
fail << path_ << (c ? " (or its compressed variant)" : "")
<< " does not exist" <<
info << "consider cleaning the build state";
}
void file_cache::entry::
preempt ()
{
// Note that this function is called from destructors so it's best if it
// doesn't throw.
//
switch (state_)
{
case uncomp:
{
if (!compress ())
break;
state_ = decomp; // We now have both.
}
// Fall through.
case decomp:
{
if (try_rmfile_ignore_error (path_))
state_ = comp;
break;
}
default:
assert (false);
}
}
bool file_cache::entry::
compress ()
{
tracer trace ("file_cache::entry::compress");
pair<bool, entry_stat> st (
path_entry (path_,
true /* follow_symlinks */,
true /* ignore_error */));
if (!st.first)
return false;
try
{
ifdstream ifs (path_, fdopen_mode::binary, ifdstream::badbit);
ofdstream ofs (comp_path_, fdopen_mode::binary);
// Experience shows that for the type of content we typically cache
// using 1MB blocks results in almost the same comression as for 4MB.
//
uint64_t comp_size (
lz4::compress (ofs, ifs,
1 /* compression_level (fastest) */,
6 /* block_size_id (1MB) */,
st.second.size));
ofs.close ();
l6 ([&]{trace << "compressed " << path_ << " to "
<< (comp_size * 100 / st.second.size) << '%';});
}
catch (const std::exception& e)
{
l5 ([&]{trace << "unable to compress " << path_ << ": " << e;});
try_rmfile_ignore_error (comp_path_);
return false;
}
return true;
}
void file_cache::entry::
decompress ()
{
try
{
ifdstream ifs (comp_path_, fdopen_mode::binary, ifdstream::badbit);
ofdstream ofs (path_, fdopen_mode::binary);
lz4::decompress (ofs, ifs);
ofs.close ();
}
catch (const std::exception& e)
{
fail << "unable to decompress " << comp_path_ << ": " << e <<
info << "consider cleaning the build state";
}
}
void file_cache::entry::
remove ()
{
switch (state_)
{
case uninit:
{
// In this case we are cleaning the filesystem without having any idea
// about its state. As a result, if we couldn't remove the compressed
// file, then we don't attempt to remove the uncompressed file either
// since it could be an indicator that the compressed file is invalid.
//
if (comp_path_.empty () || try_rmfile_ignore_error (comp_path_))
try_rmfile_ignore_error (path_);
break;
}
case uncomp:
{
try_rmfile_ignore_error (path_);
break;
}
case comp:
{
try_rmfile_ignore_error (comp_path_);
break;
}
case decomp:
{
// Both are valid so we are ok with failing to remove either.
//
try_rmfile_ignore_error (comp_path_);
try_rmfile_ignore_error (path_);
break;
}
case null:
assert (false);
}
}
}
|
