Commit 517d0756 authored by Samuel Debionne's avatar Samuel Debionne

Merge branch...

Merge branch '92-memutils-softbufferallocmgr-fix-issues-when-allocating-more-hw-buffers' into 'master'

Resolve "MemUtils/SoftBufferAllocMgr: Fix issues when allocating more HW buffers"

Closes #92

See merge request !122
parents 1b49e27e 2f1835b1
Pipeline #10697 passed with stages
in 13 minutes and 49 seconds
Subproject commit 23069964bcd1bf1b729672ecc3250c4786cb4b8c
Subproject commit 1b91c2da5eebba5be3de2219d6d2a0f21e11aa97
......@@ -26,6 +26,8 @@
#include "lima/SizeUtils.h"
#include "lima/Debug.h"
#include <cassert>
namespace lima
{
......@@ -38,129 +40,174 @@ int LIMACORE_API GetDefMaxNbBuffers(const FrameDim& frame_dim);
void LIMACORE_API ClearBuffer(void *ptr, int nb_concat_frames, const FrameDim& frame_dim);
class LIMACORE_API MemBuffer
struct LIMACORE_API Allocator
{
public:
enum {
Alignment = 16,
struct Data {
virtual ~Data() = default;
};
typedef AutoPtr<Data> DataPtr;
MemBuffer();
MemBuffer(int size);
~MemBuffer();
MemBuffer(const MemBuffer&);
MemBuffer& operator=(const MemBuffer&);
// MemBuffer are move-constructible or move-assignable.
MemBuffer(MemBuffer&&);
MemBuffer& operator=(MemBuffer&&) = default;
Allocator() : m_ref_count(0)
{}
#ifdef LIMA_USE_NUMA
void setCPUAffinityMask(unsigned long cpu_mask);
#endif
Allocator(const Allocator& o) : m_ref_count(0)
{}
void alloc(int size);
void deepCopy(const MemBuffer& buffer);
void release();
Allocator(Allocator&& o) : m_ref_count(0)
{ assert(o.m_ref_count == 0); }
int getSize() const;
void *getPtr();
const void *getConstPtr() const;
~Allocator()
{ assert(m_ref_count == 0); }
void clear();
// Allocate a buffer of a given size and eventually return
// the associated allocator data and potentially modified size
virtual DataPtr alloc(void* &ptr, size_t& size, size_t alignment = 16);
// Fill buffer with zeros (hot page)
virtual void init(void* ptr, size_t size);
// Free a buffer
virtual void release(void* ptr, size_t size, DataPtr alloc_data);
operator void *();
operator const void *() const;
// Returns a static instance of the default allocator
static Allocator *defaultAllocator();
private:
class Allocator
// All references to Allocators should be kept with this class
class Ref
{
public:
// Allocate a buffer of a given size
virtual void alloc(MemBuffer& buffer, int& size);
// Fill buffer with zeros (hot page)
virtual void init(MemBuffer& buffer);
// Copy a buffer from src to dst
virtual void copy(MemBuffer& dst, const MemBuffer& src);
// Fill buffer with zeros
virtual void clear(MemBuffer& buffer);
//
virtual void release(MemBuffer& buffer);
// Returns a Singleton
static Allocator *getAllocator();
// Returns the size of a page aligned buffer (multiple of page size)
static int getPageAlignedSize(int size);
#ifdef __unix
// Returns true if mmap is available
static bool useMmap(int size);
// Allocate a buffer with mmap (virtual address mapping)
static void *allocMmap(int& size);
#endif
protected:
Allocator() {}
Ref(Allocator *alloc) : m_alloc(alloc->get())
{}
Ref(const Ref& o) : m_alloc(o.m_alloc->get())
{}
~Ref()
{ m_alloc->put(); }
Ref& operator =(const Ref& o)
{
if (m_alloc != o.m_alloc) {
m_alloc->put();
m_alloc = o.m_alloc->get();
}
return *this;
}
operator Allocator *() const
{ return m_alloc; }
Allocator *operator ->() const
{ return m_alloc; }
private:
Allocator *m_alloc;
};
friend class Allocator;
void init();
void allocMemory(int& size);
void initMemory();
protected:
friend class Ref;
int m_size;
void *m_ptr;
Allocator *m_allocator;
// The real resource management counter, triggered by Ref
Allocator *get()
{ return ++m_ref_count, this; }
void put()
{ --m_ref_count; }
// Keep track of allocated buffers pointing to this Allocator:
// if greather than 0 this object cannot be moved
unsigned m_ref_count;
};
inline bool operator ==(const Allocator::Ref& a, const Allocator::Ref& b)
{ return (Allocator *) a == (Allocator *) b; }
#ifdef __unix
// Allocator for virtual address mapping
class LIMACORE_API MMapAllocator : public Allocator
{
public:
// Allocate a buffer of a given size
virtual DataPtr alloc(void* &ptr, size_t& size, size_t alignment = 16)
override;
// Free a buffer
virtual void release(void* ptr, size_t size, DataPtr alloc_data)
override;
// Returns the size of a page aligned buffer (multiple of page size)
static int getPageAlignedSize(int size);
protected:
// Allocate a buffer with mmap (virtual address mapping)
// The real, page-aligned buffer size, is returned in size arg
void *allocMmap(size_t& size);
};
#endif //__unix
#ifdef LIMA_USE_NUMA
class NumaAllocator : public Allocator
{
public:
virtual void alloc(MemBuffer& buffer, int& size);
virtual void init(MemBuffer& buffer);
virtual void copy(MemBuffer& dst, const MemBuffer& src);
virtual void clear(MemBuffer& buffer);
virtual void release(MemBuffer& buffer);
// Returns a Singleton
static NumaAllocator *getAllocator();
// Given a cpu_mask, returns the memory node mask
// used by alloc to bind memory with the proper socket
void getNUMANodeMask(unsigned long cpu_mask,
unsigned long& node_mask,
int& max_node);
};
friend class NumaAllocator;
class LIMACORE_API NumaAllocator : public MMapAllocator
{
public:
NumaAllocator(unsigned long cpu_mask) : m_cpu_mask(cpu_mask) {}
unsigned long getCPUAffinityMask()
{ return m_cpu_mask; }
// Allocate a buffer and sets the NUMA memory policy with mbind
virtual DataPtr alloc(void* &ptr, size_t& size, size_t alignment = 16)
override;
private:
// Given a cpu_mask, returns the memory node mask
// used by alloc to bind memory with the proper socket
void getNUMANodeMask(unsigned long cpu_mask,
unsigned long& node_mask,
int& max_node);
unsigned long m_cpu_mask; //<! if NUMA is used, keep the cpu_mask for later use
#endif
};
#endif
inline int MemBuffer::getSize() const
{
return m_size;
}
inline void *MemBuffer::getPtr()
class LIMACORE_API MemBuffer
{
return m_ptr;
}
public:
//By default, construct a MemBuffer with the default constructor
MemBuffer(Allocator *allocator = Allocator::defaultAllocator());
MemBuffer(int size, Allocator *allocator =
Allocator::defaultAllocator());
~MemBuffer();
inline const void *MemBuffer::getConstPtr() const
{
return m_ptr;
}
// MemBuffer are copy constructible (deep copy, no aliasing)
MemBuffer(const MemBuffer& buffer);
MemBuffer& operator =(const MemBuffer& buffer);
inline MemBuffer::operator void *()
{
return getPtr();
}
// MemBuffer are move-constructible or move-assignable.
MemBuffer(MemBuffer&& rhs);
MemBuffer& operator =(MemBuffer&& rhs);
inline MemBuffer::operator const void *() const
{
return getConstPtr();
}
/// Allocate and initialized memory
void alloc(size_t size);
void deepCopy(const MemBuffer& buffer);
void release();
size_t getSize() const { return m_size; }
void *getPtr() { return m_ptr; }
const void *getConstPtr() const { return m_ptr; }
void clear();
operator void *() { return getPtr(); }
operator const void *() const { return getConstPtr(); }
/// Returns the allocator currently associated with MemBuffer
Allocator *getAllocator() const { return m_allocator; }
private:
/// Call the allocator to (eventually) free the current buffer then allocate a new buffer
void uninitializedAlloc(size_t size);
/// Initialize the memory
void initMemory();
size_t m_size; //!< The size of the buffer in bytes
void *m_ptr; //!< The pointer ot the buffer
Allocator::Ref m_allocator; //!< The allocator used to alloc and free the buffer
Allocator::DataPtr m_alloc_data;
};
} // namespace lima
......
......@@ -22,6 +22,7 @@
#include "lima/MemUtils.h"
#include "lima/Exceptions.h"
#include <cassert>
#include <cstdlib>
#include <sstream>
#ifdef __unix
......@@ -63,7 +64,7 @@ void lima::GetSystemMem(int& mem_unit, int& system_mem)
GlobalMemoryStatusEx(&statex);
long long tot_mem = (long long) statex.ullAvailPhys;
if (mem_unit == 0)
mem_unit = 1;
mem_unit = 1;
#endif
const bool platform_32 = (sizeof(void *) == 4);
......@@ -107,98 +108,46 @@ void lima::ClearBuffer(void *ptr, int nb_concat_frames,
memset(ptr, 0, nb_concat_frames * frame_dim.getMemSize());
}
MemBuffer::Allocator *MemBuffer::Allocator::getAllocator()
Allocator *Allocator::defaultAllocator()
{
static Allocator allocator;
return &allocator;
}
void MemBuffer::Allocator::alloc(MemBuffer& buffer, int& size)
Allocator::DataPtr Allocator::alloc(void* &ptr, size_t& size, size_t alignment)
{
void *ptr;
#ifdef __unix
if (useMmap(size)) {
int real_size = size;
ptr = allocMmap(real_size);
} else {
int ret = posix_memalign(&ptr, Alignment, size);
if (ret != 0)
throw LIMA_COM_EXC(Error, "Error in posix_memalign: ")
<< strerror(ret);
}
int ret = posix_memalign(&ptr, alignment, size);
if (ret != 0)
throw LIMA_COM_EXC(Error, "Error in posix_memalign: ")
<< strerror(ret);
#else
ptr = _aligned_malloc(size, Alignment);
ptr = _aligned_malloc(size, alignment);
if (!ptr)
throw LIMA_COM_EXC(Error, "Error in _aligned_malloc: ")
<< "NULL pointer return";
#endif
buffer.m_ptr = ptr;
buffer.m_size = size;
return DataPtr();
}
void MemBuffer::Allocator::init(MemBuffer& buffer)
void Allocator::init(void* ptr, size_t size)
{
char *ptr = (char *) buffer.getPtr();
int size = buffer.getSize();
#ifdef __unix
int page_size;
GetPageSize(page_size);
#ifdef __SSE2__
if (!((long) ptr & 15)) { // aligned to 128 bits
__m128i zero = _mm_setzero_si128();
for (long i = 0; i < size; i += page_size, ptr += page_size) {
if (size_t(size - i) >= sizeof(__m128i))
_mm_store_si128((__m128i *) ptr, zero);
else
*ptr = 0;
}
_mm_empty();
} else {
#endif
for (long i = 0; i < size; i += page_size, ptr += page_size)
*ptr = 0;
#ifdef __SSE2__
}
#endif
#else
// memset implementation is already vectorized
memset(ptr, 0, size);
#endif
}
void MemBuffer::Allocator::copy(MemBuffer& buffer, const MemBuffer& src)
{
memcpy(buffer.getPtr(), src.getConstPtr(), src.getSize());
}
void MemBuffer::Allocator::clear(MemBuffer& buffer)
{
ClearBuffer(buffer.getPtr(), 1, FrameDim(buffer.getSize(), 1, Bpp8));
}
void MemBuffer::Allocator::release(MemBuffer& buffer)
void Allocator::release(void* ptr, size_t /*size*/, DataPtr /*alloc_data*/)
{
void *ptr = buffer.getPtr();
int size = buffer.getSize();
#ifdef __unix
if (useMmap(size)) {
int real_size = getPageAlignedSize(size);
if (munmap(ptr, real_size) != 0)
throw LIMA_COM_EXC(Error, "Error in munmap: ")
<< strerror(errno);
} else {
free(ptr);
}
free(ptr);
#else
_aligned_free(ptr);
#endif
buffer.m_ptr = NULL;
buffer.m_size = 0;
}
int MemBuffer::Allocator::getPageAlignedSize(int size)
#ifdef __unix
int MMapAllocator::getPageAlignedSize(int size)
{
int page_size;
GetPageSize(page_size);
......@@ -208,14 +157,22 @@ int MemBuffer::Allocator::getPageAlignedSize(int size)
return size;
}
#ifdef __unix
bool MemBuffer::Allocator::useMmap(int size)
// Allocate a buffer of a given size
Allocator::DataPtr MMapAllocator::alloc(void* &ptr, size_t& size,
size_t alignment/* = 16*/)
{
ptr = allocMmap(size);
return DataPtr();
}
// Free a buffer
void MMapAllocator::release(void* ptr, size_t size, DataPtr alloc_data)
{
// Use MMap if size is greater than 128KB
return size >= 128 * 1024;
size = getPageAlignedSize(size);
munmap(ptr, size);
}
void *MemBuffer::Allocator::allocMmap(int& size)
void *MMapAllocator::allocMmap(size_t& size)
{
size = getPageAlignedSize(size);
void *ptr = (char *) mmap(0, size, PROT_READ | PROT_WRITE,
......@@ -225,45 +182,66 @@ void *MemBuffer::Allocator::allocMmap(int& size)
<< strerror(errno);
return ptr;
}
#endif
#endif //__unix
inline void MemBuffer::init()
MemBuffer::MemBuffer(Allocator *allocator /*= Allocator::defaultAllocator()*/) :
m_ptr(nullptr),
m_size(0),
m_allocator(allocator)
{
m_size = 0;
m_ptr = NULL;
m_allocator = NULL;
#ifdef LIMA_USE_NUMA
m_cpu_mask = 0;
#endif
}
MemBuffer::MemBuffer()
MemBuffer::MemBuffer(int size, Allocator *allocator /*=
Allocator::defaultAllocator()*/):
m_ptr(nullptr),
m_size(0),
m_allocator(allocator)
{
init();
alloc(size);
}
MemBuffer::MemBuffer(int size)
MemBuffer::MemBuffer(const MemBuffer& buffer) :
m_ptr(nullptr),
m_size(0),
m_allocator(buffer.m_allocator)
{
init();
alloc(size);
deepCopy(buffer);
}
MemBuffer::MemBuffer(const MemBuffer& buffer)
MemBuffer& MemBuffer::operator =(const MemBuffer& buffer)
{
init();
if (m_allocator != buffer.m_allocator) {
release();
m_allocator = buffer.m_allocator;
}
deepCopy(buffer);
return *this;
}
MemBuffer::MemBuffer(MemBuffer&& rhs)
// Steal buffer ressource
MemBuffer::MemBuffer(MemBuffer&& rhs) :
m_ptr(move(rhs.m_ptr)),
m_size(move(rhs.m_size)),
m_allocator(move(rhs.m_allocator))
{
// Stealing buffer ressource
m_ptr = std::move(rhs.m_ptr);
m_size = std::move(rhs.m_size);
m_allocator = std::move(rhs.m_allocator);
// Finish resource transfer: remove it from rhs so
// it is not deallocated twice
rhs.m_ptr = nullptr;
rhs.m_size = 0;
}
// Repare rhs (we don't it to be deallocated twice)
MemBuffer& MemBuffer::operator =(MemBuffer&& rhs)
{
// First release previous contents
release();
// Steal buffer ressource
m_ptr = move(rhs.m_ptr);
m_size = move(rhs.m_size);
m_allocator = move(rhs.m_allocator);
// Finish transfer
rhs.m_ptr = nullptr;
rhs.m_size = 0;
return *this;
}
MemBuffer::~MemBuffer()
......@@ -271,104 +249,78 @@ MemBuffer::~MemBuffer()
release();
}
void MemBuffer::alloc(int size)
void MemBuffer::alloc(size_t size)
{
allocMemory(size);
uninitializedAlloc(size);
initMemory();
}
void MemBuffer::allocMemory(int& size)
void MemBuffer::release()
{
if (m_size) {
m_allocator->release(m_ptr, m_size, m_alloc_data);
m_ptr = nullptr;
m_size = 0;
}
}
void MemBuffer::clear()
{
ClearBuffer(getPtr(), 1, FrameDim((int) getSize(), 1, Bpp8));
}
void MemBuffer::uninitializedAlloc(size_t size)
{
assert(m_allocator);
if (m_size == size)
return;
release();
if (!m_allocator)
m_allocator = Allocator::getAllocator();
size_t real_size = size;
m_alloc_data = m_allocator->alloc(m_ptr, real_size);
m_allocator->alloc(*this, size);
m_size = size;
}
void MemBuffer::initMemory()
{
m_allocator->init(*this);
m_allocator->init(m_ptr, m_size);
}
void MemBuffer::deepCopy(const MemBuffer& buffer)
{
int size = buffer.getSize();
allocMemory(size);
m_allocator->copy(*this, buffer);
}
MemBuffer& MemBuffer::operator =(const MemBuffer& buffer)
{
deepCopy(buffer);
return *this;
if (buffer.m_ptr) {
uninitializedAlloc(buffer.getSize());
memcpy(getPtr(), buffer.getConstPtr(), buffer.getSize());
} else {
release();
}
}
void MemBuffer::release()
{
if (m_size)
m_allocator->release(*this);
}
void MemBuffer::clear()
{
if (m_size)
m_allocator->clear(*this);
}
#ifdef LIMA_USE_NUMA
void MemBuffer::setCPUAffinityMask(unsigned long cpu_mask)
{
m_cpu_mask = cpu_mask;
if (m_cpu_mask != 0)
m_allocator = NumaAllocator::getAllocator();
}
void MemBuffer::NumaAllocator::alloc(MemBuffer& buffer, int& size)
Allocator::DataPtr NumaAllocator::alloc(void* &ptr, size_t& size,
size_t alignment)
{
Allocator::alloc(buffer, size);
DataPtr alloc_data = MMapAllocator::alloc(ptr, size, alignment);
if (!useMmap(size) || !buffer.m_cpu_mask)
return;
if (!m_cpu_mask)
return alloc_data;
void *ptr = buffer.getPtr();
unsigned long node_mask;
int max_node;
getNUMANodeMask(buffer.m_cpu_mask, node_mask, max_node);
mbind(ptr, size, MPOL_BIND, &node_mask, max_node, 0);
}
void MemBuffer::NumaAllocator::init(MemBuffer& buffer)
{
Allocator::init(buffer);
}
void MemBuffer::NumaAllocator::copy(MemBuffer& buffer, const MemBuffer& src)
{
Allocator::copy(buffer, src);
}
void MemBuffer::NumaAllocator::clear(MemBuffer& buffer)
{
Allocator::clear(buffer);
}
void MemBuffer::NumaAllocator::release(MemBuffer& buffer)
{
Allocator::release(buffer);
}
getNUMANodeMask(m_cpu_mask, node_mask, max_node);
if (mbind(ptr, size, MPOL_BIND, &node_mask, max_node, 0) != 0)
throw LIMA_COM_EXC(Error, "Error in mbind: ")
<< strerror(errno);
MemBuffer::NumaAllocator *MemBuffer::NumaAllocator::getAllocator()
{
static NumaAllocator allocator;
return &allocator;
return alloc_data;
}
void MemBuffer::NumaAllocator::getNUMANodeMask(unsigned long cpu_mask,
void NumaAllocator::getNUMANodeMask(unsigned long cpu_mask,
unsigned long& node_mask,
int& max_node)
{
......@@ -383,4 +335,4 @@ void MemBuffer::NumaAllocator::getNUMANodeMask(unsigned long cpu_mask,
}
}
}
#endif
#endif //LIMA_USE_NUMA
......@@ -91,12 +91,84 @@ void test_alloc()
assert(g.getSize() == 1);
}
struct MockAllocator : lima::Allocator
{
virtual DataPtr alloc(void* &ptr, size_t& size, size_t alignment = 16) override
{
ptr = malloc(size);
return DataPtr();
}
virtual void init(void* ptr, size_t size) override
{
memset(ptr, 0, size);
}
virtual void release(void* ptr, size_t /*size*/,
DataPtr /*alloc_data*/) override
{
assert(ptr);
free(ptr);
}
static MockAllocator *getAllocator()
{
static MockAllocator instance;
return &instance;
}
};
void test_custom_allocator()
{
MockAllocator allocator;
//Default construction
MemBuffer b(1, &allocator);
assert(b.getSize() == 1);
assert(b.getAllocator() == &allocator);
//Copy construction
MemBuffer c(b);
assert(c.getSize() == 1);
assert(c.getConstPtr() != b.getConstPtr());
assert(c.getAllocator() == &allocator);