1
// Copyright (c) 2016-2020 The Bitcoin Core developers
2
// Distributed under the MIT software license, see the accompanying
3
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
4

            
5
#include <support/lockedpool.h>
6
#include <support/cleanse.h>
7

            
8
#if defined(HAVE_CONFIG_H)
9
#include <config/bitcoin-config.h>
10
#endif
11

            
12
#ifdef WIN32
13
#ifndef NOMINMAX
14
#define NOMINMAX
15
#endif
16
#include <windows.h>
17
#else
18
#include <sys/mman.h> // for mmap
19
#include <sys/resource.h> // for getrlimit
20
#include <limits.h> // for PAGESIZE
21
#include <unistd.h> // for sysconf
22
#endif
23

            
24
#include <algorithm>
25
#ifdef ARENA_DEBUG
26
#include <iomanip>
27
#include <iostream>
28
#endif
29

            
30
LockedPoolManager* LockedPoolManager::_instance = nullptr;
31

            
32
/*******************************************************************************/
33
// Utilities
34
//
35
/** Align up to power of 2 */
36
static inline size_t align_up(size_t x, size_t align)
37
259
{
38
259
    return (x + align - 1) & ~(align - 1);
39
259
}
40

            
41
/*******************************************************************************/
42
// Implementation: Arena
43

            
44
Arena::Arena(void *base_in, size_t size_in, size_t alignment_in):
45
    base(static_cast<char*>(base_in)), end(static_cast<char*>(base_in) + size_in), alignment(alignment_in)
46
1
{
47
    // Start with one free chunk that covers the entire arena
48
1
    auto it = size_to_free_chunk.emplace(size_in, base);
49
1
    chunks_free.emplace(base, it);
50
1
    chunks_free_end.emplace(base + size_in, it);
51
1
}
52

            
53
Arena::~Arena()
54
{
55
}
56

            
57
void* Arena::alloc(size_t size)
58
258
{
59
    // Round to next multiple of alignment
60
258
    size = align_up(size, alignment);
61

            
62
    // Don't handle zero-sized chunks
63
258
    if (size == 0)
64
        return nullptr;
65

            
66
    // Pick a large enough free-chunk. Returns an iterator pointing to the first element that is not less than key.
67
    // This allocation strategy is best-fit. According to "Dynamic Storage Allocation: A Survey and Critical Review",
68
    // Wilson et. al. 1995, https://www.scs.stanford.edu/14wi-cs140/sched/readings/wilson.pdf, best-fit and first-fit
69
    // policies seem to work well in practice.
70
258
    auto size_ptr_it = size_to_free_chunk.lower_bound(size);
71
258
    if (size_ptr_it == size_to_free_chunk.end())
72
        return nullptr;
73

            
74
    // Create the used-chunk, taking its space from the end of the free-chunk
75
258
    const size_t size_remaining = size_ptr_it->first - size;
76
258
    auto allocated = chunks_used.emplace(size_ptr_it->second + size_remaining, size).first;
77
258
    chunks_free_end.erase(size_ptr_it->second + size_ptr_it->first);
78
258
    if (size_ptr_it->first == size) {
79
        // whole chunk is used up
80
256
        chunks_free.erase(size_ptr_it->second);
81
256
    } else {
82
        // still some memory left in the chunk
83
2
        auto it_remaining = size_to_free_chunk.emplace(size_remaining, size_ptr_it->second);
84
2
        chunks_free[size_ptr_it->second] = it_remaining;
85
2
        chunks_free_end.emplace(size_ptr_it->second + size_remaining, it_remaining);
86
2
    }
87
258
    size_to_free_chunk.erase(size_ptr_it);
88

            
89
258
    return reinterpret_cast<void*>(allocated->first);
90
258
}
91

            
92
void Arena::free(void *ptr)
93
257
{
94
    // Freeing the nullptr pointer is OK.
95
257
    if (ptr == nullptr) {
96
        return;
97
    }
98

            
99
    // Remove chunk from used map
100
257
    auto i = chunks_used.find(static_cast<char*>(ptr));
101
257
    if (i == chunks_used.end()) {
102
        throw std::runtime_error("Arena: invalid or double free");
103
    }
104
257
    std::pair<char*, size_t> freed = *i;
105
257
    chunks_used.erase(i);
106

            
107
    // coalesce freed with previous chunk
108
257
    auto prev = chunks_free_end.find(freed.first);
109
257
    if (prev != chunks_free_end.end()) {
110
        freed.first -= prev->second->first;
111
        freed.second += prev->second->first;
112
        size_to_free_chunk.erase(prev->second);
113
        chunks_free_end.erase(prev);
114
    }
115

            
116
    // coalesce freed with chunk after freed
117
257
    auto next = chunks_free.find(freed.first + freed.second);
118
257
    if (next != chunks_free.end()) {
119
        freed.second += next->second->first;
120
        size_to_free_chunk.erase(next->second);
121
        chunks_free.erase(next);
122
    }
123

            
124
    // Add/set space with coalesced free chunk
125
257
    auto it = size_to_free_chunk.emplace(freed.second, freed.first);
126
257
    chunks_free[freed.first] = it;
127
257
    chunks_free_end[freed.first + freed.second] = it;
128
257
}
129

            
130
Arena::Stats Arena::stats() const
131
{
132
    Arena::Stats r{ 0, 0, 0, chunks_used.size(), chunks_free.size() };
133
    for (const auto& chunk: chunks_used)
134
        r.used += chunk.second;
135
    for (const auto& chunk: chunks_free)
136
        r.free += chunk.second->first;
137
    r.total = r.used + r.free;
138
    return r;
139
}
140

            
141
#ifdef ARENA_DEBUG
142
static void printchunk(void* base, size_t sz, bool used) {
143
    std::cout <<
144
        "0x" << std::hex << std::setw(16) << std::setfill('0') << base <<
145
        " 0x" << std::hex << std::setw(16) << std::setfill('0') << sz <<
146
        " 0x" << used << std::endl;
147
}
148
void Arena::walk() const
149
{
150
    for (const auto& chunk: chunks_used)
151
        printchunk(chunk.first, chunk.second, true);
152
    std::cout << std::endl;
153
    for (const auto& chunk: chunks_free)
154
        printchunk(chunk.first, chunk.second->first, false);
155
    std::cout << std::endl;
156
}
157
#endif
158

            
159
/*******************************************************************************/
160
// Implementation: Win32LockedPageAllocator
161

            
162
#ifdef WIN32
163
/** LockedPageAllocator specialized for Windows.
164
 */
165
class Win32LockedPageAllocator: public LockedPageAllocator
166
{
167
public:
168
    Win32LockedPageAllocator();
169
    void* AllocateLocked(size_t len, bool *lockingSuccess) override;
170
    void FreeLocked(void* addr, size_t len) override;
171
    size_t GetLimit() override;
172
private:
173
    size_t page_size;
174
};
175

            
176
Win32LockedPageAllocator::Win32LockedPageAllocator()
177
{
178
    // Determine system page size in bytes
179
    SYSTEM_INFO sSysInfo;
180
    GetSystemInfo(&sSysInfo);
181
    page_size = sSysInfo.dwPageSize;
182
}
183
void *Win32LockedPageAllocator::AllocateLocked(size_t len, bool *lockingSuccess)
184
{
185
    len = align_up(len, page_size);
186
    void *addr = VirtualAlloc(nullptr, len, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
187
    if (addr) {
188
        // VirtualLock is used to attempt to keep keying material out of swap. Note
189
        // that it does not provide this as a guarantee, but, in practice, memory
190
        // that has been VirtualLock'd almost never gets written to the pagefile
191
        // except in rare circumstances where memory is extremely low.
192
        *lockingSuccess = VirtualLock(const_cast<void*>(addr), len) != 0;
193
    }
194
    return addr;
195
}
196
void Win32LockedPageAllocator::FreeLocked(void* addr, size_t len)
197
{
198
    len = align_up(len, page_size);
199
    memory_cleanse(addr, len);
200
    VirtualUnlock(const_cast<void*>(addr), len);
201
}
202

            
203
size_t Win32LockedPageAllocator::GetLimit()
204
{
205
    // TODO is there a limit on Windows, how to get it?
206
    return std::numeric_limits<size_t>::max();
207
}
208
#endif
209

            
210
/*******************************************************************************/
211
// Implementation: PosixLockedPageAllocator
212

            
213
#ifndef WIN32
214
/** LockedPageAllocator specialized for OSes that don't try to be
215
 * special snowflakes.
216
 */
217
class PosixLockedPageAllocator: public LockedPageAllocator
218
{
219
public:
220
    PosixLockedPageAllocator();
221
    void* AllocateLocked(size_t len, bool *lockingSuccess) override;
222
    void FreeLocked(void* addr, size_t len) override;
223
    size_t GetLimit() override;
224
private:
225
    size_t page_size;
226
};
227

            
228
PosixLockedPageAllocator::PosixLockedPageAllocator()
229
1
{
230
    // Determine system page size in bytes
231
#if defined(PAGESIZE) // defined in limits.h
232
    page_size = PAGESIZE;
233
#else                   // assume some POSIX OS
234
1
    page_size = sysconf(_SC_PAGESIZE);
235
1
#endif
236
1
}
237

            
238
void *PosixLockedPageAllocator::AllocateLocked(size_t len, bool *lockingSuccess)
239
1
{
240
1
    void *addr;
241
1
    len = align_up(len, page_size);
242
1
    addr = mmap(nullptr, len, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
243
1
    if (addr == MAP_FAILED) {
244
        return nullptr;
245
    }
246
1
    if (addr) {
247
1
        *lockingSuccess = mlock(addr, len) == 0;
248
1
#if defined(MADV_DONTDUMP) // Linux
249
1
        madvise(addr, len, MADV_DONTDUMP);
250
#elif defined(MADV_NOCORE) // FreeBSD
251
        madvise(addr, len, MADV_NOCORE);
252
#endif
253
1
    }
254
1
    return addr;
255
1
}
256
void PosixLockedPageAllocator::FreeLocked(void* addr, size_t len)
257
{
258
    len = align_up(len, page_size);
259
    memory_cleanse(addr, len);
260
    munlock(addr, len);
261
    munmap(addr, len);
262
}
263
size_t PosixLockedPageAllocator::GetLimit()
264
1
{
265
1
#ifdef RLIMIT_MEMLOCK
266
1
    struct rlimit rlim;
267
1
    if (getrlimit(RLIMIT_MEMLOCK, &rlim) == 0) {
268
1
        if (rlim.rlim_cur != RLIM_INFINITY) {
269
1
            return rlim.rlim_cur;
270
1
        }
271
1
    }
272
#endif
273
    return std::numeric_limits<size_t>::max();
274
1
}
275
#endif
276

            
277
/*******************************************************************************/
278
// Implementation: LockedPool
279

            
280
LockedPool::LockedPool(std::unique_ptr<LockedPageAllocator> allocator_in, LockingFailed_Callback lf_cb_in):
281
    allocator(std::move(allocator_in)), lf_cb(lf_cb_in), cumulative_bytes_locked(0)
282
1
{
283
1
}
284

            
285
LockedPool::~LockedPool()
286
{
287
}
288
void* LockedPool::alloc(size_t size)
289
258
{
290
258
    std::lock_guard<std::mutex> lock(mutex);
291

            
292
    // Don't handle impossible sizes
293
258
    if (size == 0 || size > ARENA_SIZE)
294
        return nullptr;
295

            
296
    // Try allocating from each current arena
297
258
    for (auto &arena: arenas) {
298
257
        void *addr = arena.alloc(size);
299
257
        if (addr) {
300
257
            return addr;
301
257
        }
302
257
    }
303
    // If that fails, create a new one
304
1
    if (new_arena(ARENA_SIZE, ARENA_ALIGN)) {
305
1
        return arenas.back().alloc(size);
306
1
    }
307
    return nullptr;
308
1
}
309

            
310
void LockedPool::free(void *ptr)
311
257
{
312
257
    std::lock_guard<std::mutex> lock(mutex);
313
    // TODO we can do better than this linear search by keeping a map of arena
314
    // extents to arena, and looking up the address.
315
257
    for (auto &arena: arenas) {
316
257
        if (arena.addressInArena(ptr)) {
317
257
            arena.free(ptr);
318
257
            return;
319
257
        }
320
257
    }
321
    throw std::runtime_error("LockedPool: invalid address not pointing to any arena");
322
257
}
323

            
324
LockedPool::Stats LockedPool::stats() const
325
{
326
    std::lock_guard<std::mutex> lock(mutex);
327
    LockedPool::Stats r{0, 0, 0, cumulative_bytes_locked, 0, 0};
328
    for (const auto &arena: arenas) {
329
        Arena::Stats i = arena.stats();
330
        r.used += i.used;
331
        r.free += i.free;
332
        r.total += i.total;
333
        r.chunks_used += i.chunks_used;
334
        r.chunks_free += i.chunks_free;
335
    }
336
    return r;
337
}
338

            
339
bool LockedPool::new_arena(size_t size, size_t align)
340
1
{
341
1
    bool locked;
342
    // If this is the first arena, handle this specially: Cap the upper size
343
    // by the process limit. This makes sure that the first arena will at least
344
    // be locked. An exception to this is if the process limit is 0:
345
    // in this case no memory can be locked at all so we'll skip past this logic.
346
1
    if (arenas.empty()) {
347
1
        size_t limit = allocator->GetLimit();
348
1
        if (limit > 0) {
349
1
            size = std::min(size, limit);
350
1
        }
351
1
    }
352
1
    void *addr = allocator->AllocateLocked(size, &locked);
353
1
    if (!addr) {
354
        return false;
355
    }
356
1
    if (locked) {
357
1
        cumulative_bytes_locked += size;
358
1
    } else if (lf_cb) { // Call the locking-failed callback if locking failed
359
        if (!lf_cb()) { // If the callback returns false, free the memory and fail, otherwise consider the user warned and proceed.
360
            allocator->FreeLocked(addr, size);
361
            return false;
362
        }
363
    }
364
1
    arenas.emplace_back(allocator.get(), addr, size, align);
365
1
    return true;
366
1
}
367

            
368
LockedPool::LockedPageArena::LockedPageArena(LockedPageAllocator *allocator_in, void *base_in, size_t size_in, size_t align_in):
369
    Arena(base_in, size_in, align_in), base(base_in), size(size_in), allocator(allocator_in)
370
1
{
371
1
}
372
LockedPool::LockedPageArena::~LockedPageArena()
373
{
374
    allocator->FreeLocked(base, size);
375
}
376

            
377
/*******************************************************************************/
378
// Implementation: LockedPoolManager
379
//
380
LockedPoolManager::LockedPoolManager(std::unique_ptr<LockedPageAllocator> allocator_in):
381
    LockedPool(std::move(allocator_in), &LockedPoolManager::LockingFailed)
382
1
{
383
1
}
384

            
385
bool LockedPoolManager::LockingFailed()
386
{
387
    // TODO: log something but how? without including util.h
388
    return true;
389
}
390

            
391
void LockedPoolManager::CreateInstance()
392
1
{
393
    // Using a local static instance guarantees that the object is initialized
394
    // when it's first needed and also deinitialized after all objects that use
395
    // it are done with it.  I can think of one unlikely scenario where we may
396
    // have a static deinitialization order/problem, but the check in
397
    // LockedPoolManagerBase's destructor helps us detect if that ever happens.
398
#ifdef WIN32
399
    std::unique_ptr<LockedPageAllocator> allocator(new Win32LockedPageAllocator());
400
#else
401
1
    std::unique_ptr<LockedPageAllocator> allocator(new PosixLockedPageAllocator());
402
1
#endif
403
1
    static LockedPoolManager instance(std::move(allocator));
404
1
    LockedPoolManager::_instance = &instance;
405
1
}