34 #ifndef EIGEN_MEMORY_H
35 #define EIGEN_MEMORY_H
44 #if defined(__GLIBC__) && ((__GLIBC__>=2 && __GLIBC_MINOR__ >= 8) || __GLIBC__>2) \
46 #define EIGEN_GLIBC_MALLOC_ALREADY_ALIGNED 1
48 #define EIGEN_GLIBC_MALLOC_ALREADY_ALIGNED 0
55 #if defined(__FreeBSD__) && !defined(__arm__) && !defined(__mips__)
56 #define EIGEN_FREEBSD_MALLOC_ALREADY_ALIGNED 1
58 #define EIGEN_FREEBSD_MALLOC_ALREADY_ALIGNED 0
61 #if defined(__APPLE__) \
63 || EIGEN_GLIBC_MALLOC_ALREADY_ALIGNED \
64 || EIGEN_FREEBSD_MALLOC_ALREADY_ALIGNED
65 #define EIGEN_MALLOC_ALREADY_ALIGNED 1
67 #define EIGEN_MALLOC_ALREADY_ALIGNED 0
70 #if ((defined __QNXNTO__) || (defined _GNU_SOURCE) || ((defined _XOPEN_SOURCE) && (_XOPEN_SOURCE >= 600))) \
71 && (defined _POSIX_ADVISORY_INFO) && (_POSIX_ADVISORY_INFO > 0)
72 #define EIGEN_HAS_POSIX_MEMALIGN 1
74 #define EIGEN_HAS_POSIX_MEMALIGN 0
77 #ifdef EIGEN_VECTORIZE_SSE
78 #define EIGEN_HAS_MM_MALLOC 1
80 #define EIGEN_HAS_MM_MALLOC 0
89 #ifdef EIGEN_EXCEPTIONS
90 throw std::bad_alloc();
92 std::size_t huge = -1;
108 void *original = std::malloc(size+16);
109 if (original == 0)
return 0;
110 void *aligned =
reinterpret_cast<void*
>((
reinterpret_cast<size_t>(original) & ~(
size_t(15))) + 16);
111 *(
reinterpret_cast<void**
>(aligned) - 1) = original;
118 if (ptr) std::free(*(reinterpret_cast<void**>(ptr) - 1));
129 void *original = *(
reinterpret_cast<void**
>(ptr) - 1);
130 original = std::realloc(original,size+16);
131 if (original == 0)
return 0;
132 void *aligned =
reinterpret_cast<void*
>((
reinterpret_cast<size_t>(original) & ~(
size_t(15))) + 16);
133 *(
reinterpret_cast<void**
>(aligned) - 1) = original;
163 #ifdef EIGEN_HAS_ERRNO
171 std::memcpy(newptr, ptr, (std::min)(size,old_size));
182 #ifdef EIGEN_NO_MALLOC
185 eigen_assert(
false &&
"heap allocation is forbidden (EIGEN_NO_MALLOC is defined)");
187 #elif defined EIGEN_RUNTIME_NO_MALLOC
188 inline bool is_malloc_allowed_impl(
bool update,
bool new_value =
false)
190 static bool value =
true;
195 inline bool is_malloc_allowed() {
return is_malloc_allowed_impl(
false); }
196 inline bool set_is_malloc_allowed(
bool new_value) {
return is_malloc_allowed_impl(
true, new_value); }
199 eigen_assert(is_malloc_allowed() &&
"heap allocation is forbidden (EIGEN_RUNTIME_NO_MALLOC is defined and g_is_malloc_allowed is false)");
215 result = std::malloc(size);
216 #elif EIGEN_MALLOC_ALREADY_ALIGNED
217 result = std::malloc(size);
218 #elif EIGEN_HAS_POSIX_MEMALIGN
219 if(posix_memalign(&result, 16, size)) result = 0;
220 #elif EIGEN_HAS_MM_MALLOC
221 result = _mm_malloc(size, 16);
222 #elif (defined _MSC_VER)
223 result = _aligned_malloc(size, 16);
239 #elif EIGEN_MALLOC_ALREADY_ALIGNED
241 #elif EIGEN_HAS_POSIX_MEMALIGN
243 #elif EIGEN_HAS_MM_MALLOC
245 #elif defined(_MSC_VER)
263 result = std::realloc(ptr,new_size);
264 #elif EIGEN_MALLOC_ALREADY_ALIGNED
265 result = std::realloc(ptr,new_size);
266 #elif EIGEN_HAS_POSIX_MEMALIGN
268 #elif EIGEN_HAS_MM_MALLOC
272 #if defined(_MSC_VER) && defined(_mm_free)
273 result = _aligned_realloc(ptr,new_size,16);
277 #elif defined(_MSC_VER)
278 result = _aligned_realloc(ptr,new_size,16);
283 if (!result && new_size)
305 void *result = std::malloc(size);
329 return std::realloc(ptr, new_size);
341 for (
size_t i=0; i < size; ++i) ::
new (ptr + i) T;
352 while(size) ptr[--size].~T();
362 if(size >
size_t(-1) /
sizeof(T))
372 check_size_for_overflow<T>(size);
379 check_size_for_overflow<T>(size);
380 T *result =
reinterpret_cast<T*
>(conditional_aligned_malloc<Align>(
sizeof(T)*size));
389 destruct_elements_of_array<T>(ptr, size);
398 destruct_elements_of_array<T>(ptr, size);
399 conditional_aligned_free<Align>(ptr);
404 check_size_for_overflow<T>(new_size);
405 check_size_for_overflow<T>(old_size);
406 if(new_size < old_size)
408 T *result =
reinterpret_cast<T*
>(conditional_aligned_realloc<Align>(
reinterpret_cast<void*
>(pts),
sizeof(T)*new_size,
sizeof(T)*old_size));
409 if(new_size > old_size)
417 check_size_for_overflow<T>(size);
418 T *result =
reinterpret_cast<T*
>(conditional_aligned_malloc<Align>(
sizeof(T)*size));
426 check_size_for_overflow<T>(new_size);
427 check_size_for_overflow<T>(old_size);
430 T *result =
reinterpret_cast<T*
>(conditional_aligned_realloc<Align>(
reinterpret_cast<void*
>(pts),
sizeof(T)*new_size,
sizeof(T)*old_size));
439 destruct_elements_of_array<T>(ptr, size);
440 conditional_aligned_free<Align>(ptr);
461 template<
typename Scalar,
typename Index>
462 static inline Index first_aligned(
const Scalar* array, Index size)
464 typedef typename packet_traits<Scalar>::type Packet;
465 enum { PacketSize = packet_traits<Scalar>::size,
466 PacketAlignedMask = PacketSize-1
475 else if(
size_t(array) & (
sizeof(Scalar)-1))
483 return std::min<Index>( (PacketSize - (Index((
size_t(array)/
sizeof(Scalar))) & PacketAlignedMask))
484 & PacketAlignedMask, size);
491 template<
typename T,
bool UseMemcpy>
struct smart_copy_helper;
493 template<
typename T>
void smart_copy(
const T* start,
const T* end, T* target)
495 smart_copy_helper<T,!NumTraits<T>::RequireInitialization>::run(start, end, target);
498 template<
typename T>
struct smart_copy_helper<T,true> {
499 static inline void run(
const T* start,
const T* end, T* target)
500 { memcpy(target, start, std::ptrdiff_t(end)-std::ptrdiff_t(start)); }
503 template<
typename T>
struct smart_copy_helper<T,false> {
504 static inline void run(
const T* start,
const T* end, T* target)
505 { std::copy(start, end, target); }
516 #if (defined __linux__)
517 #define EIGEN_ALLOCA alloca
518 #elif defined(_MSC_VER)
519 #define EIGEN_ALLOCA _alloca
525 template<
typename T>
class aligned_stack_memory_handler
534 aligned_stack_memory_handler(T* ptr,
size_t size,
bool dealloc)
535 : m_ptr(ptr), m_size(size), m_deallocate(dealloc)
537 if(NumTraits<T>::RequireInitialization && m_ptr)
540 ~aligned_stack_memory_handler()
542 if(NumTraits<T>::RequireInitialization && m_ptr)
543 Eigen::internal::destruct_elements_of_array<T>(m_ptr, m_size);
573 #define EIGEN_ALIGNED_ALLOCA(SIZE) reinterpret_cast<void*>((reinterpret_cast<size_t>(EIGEN_ALLOCA(SIZE+16)) & ~(size_t(15))) + 16)
575 #define EIGEN_ALIGNED_ALLOCA EIGEN_ALLOCA
578 #define ei_declare_aligned_stack_constructed_variable(TYPE,NAME,SIZE,BUFFER) \
579 Eigen::internal::check_size_for_overflow<TYPE>(SIZE); \
580 TYPE* NAME = (BUFFER)!=0 ? (BUFFER) \
581 : reinterpret_cast<TYPE*>( \
582 (sizeof(TYPE)*SIZE<=EIGEN_STACK_ALLOCATION_LIMIT) ? EIGEN_ALIGNED_ALLOCA(sizeof(TYPE)*SIZE) \
583 : Eigen::internal::aligned_malloc(sizeof(TYPE)*SIZE) ); \
584 Eigen::internal::aligned_stack_memory_handler<TYPE> EIGEN_CAT(NAME,_stack_memory_destructor)((BUFFER)==0 ? NAME : 0,SIZE,sizeof(TYPE)*SIZE>EIGEN_STACK_ALLOCATION_LIMIT)
588 #define ei_declare_aligned_stack_constructed_variable(TYPE,NAME,SIZE,BUFFER) \
589 Eigen::internal::check_size_for_overflow<TYPE>(SIZE); \
590 TYPE* NAME = (BUFFER)!=0 ? BUFFER : reinterpret_cast<TYPE*>(Eigen::internal::aligned_malloc(sizeof(TYPE)*SIZE)); \
591 Eigen::internal::aligned_stack_memory_handler<TYPE> EIGEN_CAT(NAME,_stack_memory_destructor)((BUFFER)==0 ? NAME : 0,SIZE,true)
601 #ifdef EIGEN_EXCEPTIONS
602 #define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_NOTHROW(NeedsToAlign) \
603 void* operator new(size_t size, const std::nothrow_t&) throw() { \
604 try { return Eigen::internal::conditional_aligned_malloc<NeedsToAlign>(size); } \
605 catch (...) { return 0; } \
609 #define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_NOTHROW(NeedsToAlign) \
610 void* operator new(size_t size, const std::nothrow_t&) throw() { \
611 return Eigen::internal::conditional_aligned_malloc<NeedsToAlign>(size); \
615 #define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF(NeedsToAlign) \
616 void *operator new(size_t size) { \
617 return Eigen::internal::conditional_aligned_malloc<NeedsToAlign>(size); \
619 void *operator new[](size_t size) { \
620 return Eigen::internal::conditional_aligned_malloc<NeedsToAlign>(size); \
622 void operator delete(void * ptr) throw() { Eigen::internal::conditional_aligned_free<NeedsToAlign>(ptr); } \
623 void operator delete[](void * ptr) throw() { Eigen::internal::conditional_aligned_free<NeedsToAlign>(ptr); } \
627 static void *operator new(size_t size, void *ptr) { return ::operator new(size,ptr); } \
628 void operator delete(void * memory, void *ptr) throw() { return ::operator delete(memory,ptr); } \
630 EIGEN_MAKE_ALIGNED_OPERATOR_NEW_NOTHROW(NeedsToAlign) \
631 void operator delete(void *ptr, const std::nothrow_t&) throw() { \
632 Eigen::internal::conditional_aligned_free<NeedsToAlign>(ptr); \
634 typedef void eigen_aligned_operator_new_marker_type;
636 #define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF(NeedsToAlign)
639 #define EIGEN_MAKE_ALIGNED_OPERATOR_NEW EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF(true)
640 #define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF_VECTORIZABLE_FIXED_SIZE(Scalar,Size) \
641 EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF(bool(((Size)!=Eigen::Dynamic) && ((sizeof(Scalar)*(Size))%16==0)))
708 return (std::numeric_limits<size_type>::max)();
714 internal::check_size_for_overflow<T>(num);
720 ::new( p ) T( value );
724 #if (__cplusplus >= 201103L)
725 template<
typename... Args>
728 ::new(p) T(std::forward<Args>(args)...);
751 #if !defined(EIGEN_NO_CPUID)
752 # if defined(__GNUC__) && ( defined(__i386__) || defined(__x86_64__) )
753 # if defined(__PIC__) && defined(__i386__)
755 # define EIGEN_CPUID(abcd,func,id) \
756 __asm__ __volatile__ ("xchgl %%ebx, %%esi;cpuid; xchgl %%ebx,%%esi": "=a" (abcd[0]), "=S" (abcd[1]), "=c" (abcd[2]), "=d" (abcd[3]) : "a" (func), "c" (id));
759 # define EIGEN_CPUID(abcd,func,id) \
760 __asm__ __volatile__ ("cpuid": "=a" (abcd[0]), "=b" (abcd[1]), "=c" (abcd[2]), "=d" (abcd[3]) : "a" (func), "c" (id) );
762 # elif defined(_MSC_VER)
763 # if (_MSC_VER > 1500)
764 # define EIGEN_CPUID(abcd,func,id) __cpuidex((int*)abcd,func,id)
773 inline bool cpuid_is_vendor(
int abcd[4],
const char* vendor)
775 return abcd[1]==(
reinterpret_cast<const int*
>(vendor))[0] && abcd[3]==(
reinterpret_cast<const int*
>(vendor))[1] && abcd[2]==(
reinterpret_cast<const int*
>(vendor))[2];
778 inline void queryCacheSizes_intel_direct(
int& l1,
int& l2,
int& l3)
785 abcd[0] = abcd[1] = abcd[2] = abcd[3] = 0;
786 EIGEN_CPUID(abcd,0x4,cache_id);
787 cache_type = (abcd[0] & 0x0F) >> 0;
788 if(cache_type==1||cache_type==3)
790 int cache_level = (abcd[0] & 0xE0) >> 5;
791 int ways = (abcd[1] & 0xFFC00000) >> 22;
792 int partitions = (abcd[1] & 0x003FF000) >> 12;
793 int line_size = (abcd[1] & 0x00000FFF) >> 0;
794 int sets = (abcd[2]);
796 int cache_size = (ways+1) * (partitions+1) * (line_size+1) * (sets+1);
800 case 1: l1 = cache_size;
break;
801 case 2: l2 = cache_size;
break;
802 case 3: l3 = cache_size;
break;
807 }
while(cache_type>0 && cache_id<16);
810 inline void queryCacheSizes_intel_codes(
int& l1,
int& l2,
int& l3)
813 abcd[0] = abcd[1] = abcd[2] = abcd[3] = 0;
815 EIGEN_CPUID(abcd,0x00000002,0);
816 unsigned char * bytes =
reinterpret_cast<unsigned char *
>(abcd)+2;
817 bool check_for_p2_core2 =
false;
818 for(
int i=0; i<14; ++i)
822 case 0x0A: l1 = 8;
break;
823 case 0x0C: l1 = 16;
break;
824 case 0x0E: l1 = 24;
break;
825 case 0x10: l1 = 16;
break;
826 case 0x15: l1 = 16;
break;
827 case 0x2C: l1 = 32;
break;
828 case 0x30: l1 = 32;
break;
829 case 0x60: l1 = 16;
break;
830 case 0x66: l1 = 8;
break;
831 case 0x67: l1 = 16;
break;
832 case 0x68: l1 = 32;
break;
833 case 0x1A: l2 = 96;
break;
834 case 0x22: l3 = 512;
break;
835 case 0x23: l3 = 1024;
break;
836 case 0x25: l3 = 2048;
break;
837 case 0x29: l3 = 4096;
break;
838 case 0x39: l2 = 128;
break;
839 case 0x3A: l2 = 192;
break;
840 case 0x3B: l2 = 128;
break;
841 case 0x3C: l2 = 256;
break;
842 case 0x3D: l2 = 384;
break;
843 case 0x3E: l2 = 512;
break;
844 case 0x40: l2 = 0;
break;
845 case 0x41: l2 = 128;
break;
846 case 0x42: l2 = 256;
break;
847 case 0x43: l2 = 512;
break;
848 case 0x44: l2 = 1024;
break;
849 case 0x45: l2 = 2048;
break;
850 case 0x46: l3 = 4096;
break;
851 case 0x47: l3 = 8192;
break;
852 case 0x48: l2 = 3072;
break;
853 case 0x49:
if(l2!=0) l3 = 4096;
else {check_for_p2_core2=
true; l3 = l2 = 4096;}
break;
854 case 0x4A: l3 = 6144;
break;
855 case 0x4B: l3 = 8192;
break;
856 case 0x4C: l3 = 12288;
break;
857 case 0x4D: l3 = 16384;
break;
858 case 0x4E: l2 = 6144;
break;
859 case 0x78: l2 = 1024;
break;
860 case 0x79: l2 = 128;
break;
861 case 0x7A: l2 = 256;
break;
862 case 0x7B: l2 = 512;
break;
863 case 0x7C: l2 = 1024;
break;
864 case 0x7D: l2 = 2048;
break;
865 case 0x7E: l2 = 256;
break;
866 case 0x7F: l2 = 512;
break;
867 case 0x80: l2 = 512;
break;
868 case 0x81: l2 = 128;
break;
869 case 0x82: l2 = 256;
break;
870 case 0x83: l2 = 512;
break;
871 case 0x84: l2 = 1024;
break;
872 case 0x85: l2 = 2048;
break;
873 case 0x86: l2 = 512;
break;
874 case 0x87: l2 = 1024;
break;
875 case 0x88: l3 = 2048;
break;
876 case 0x89: l3 = 4096;
break;
877 case 0x8A: l3 = 8192;
break;
878 case 0x8D: l3 = 3072;
break;
883 if(check_for_p2_core2 && l2 == l3)
890 inline void queryCacheSizes_intel(
int& l1,
int& l2,
int& l3,
int max_std_funcs)
893 queryCacheSizes_intel_direct(l1,l2,l3);
895 queryCacheSizes_intel_codes(l1,l2,l3);
898 inline void queryCacheSizes_amd(
int& l1,
int& l2,
int& l3)
901 abcd[0] = abcd[1] = abcd[2] = abcd[3] = 0;
902 EIGEN_CPUID(abcd,0x80000005,0);
903 l1 = (abcd[2] >> 24) * 1024;
904 abcd[0] = abcd[1] = abcd[2] = abcd[3] = 0;
905 EIGEN_CPUID(abcd,0x80000006,0);
906 l2 = (abcd[2] >> 16) * 1024;
907 l3 = ((abcd[3] & 0xFFFC000) >> 18) * 512 * 1024;
919 EIGEN_CPUID(abcd,0x0,0);
920 int max_std_funcs = abcd[1];
921 if(cpuid_is_vendor(abcd,
"GenuineIntel"))
922 queryCacheSizes_intel(l1,l2,l3,max_std_funcs);
923 else if(cpuid_is_vendor(abcd,
"AuthenticAMD") || cpuid_is_vendor(abcd,
"AMDisbetter!"))
924 queryCacheSizes_amd(l1,l2,l3);
927 queryCacheSizes_intel(l1,l2,l3,max_std_funcs);
958 int l1, l2(-1), l3(-1);
960 return (std::max)(l2,l3);
967 #endif // EIGEN_MEMORY_H