clean up csize functions

Author: daanx

CMakeLists.txt

@@ -1151,7 +1151,7 @@ if (MI_BUILD_TESTS)
   endforeach()
 
   # static override test
-  if(MI_BUILD_OBJECT AND NOT (MI_CYGWIN OR (MI_TRACK STREQUAL "ASAN") OR MI_DEBUG_TSAN OR MI_DEBUG_UBSAN))
+  if(MI_BUILD_OBJECT AND MI_OVERRIDE AND NOT (MI_CYGWIN OR (MI_TRACK STREQUAL "ASAN") OR MI_DEBUG_TSAN OR MI_DEBUG_UBSAN))
     add_executable(mimalloc-test-stress-static test/test-stress.c)
     target_compile_definitions(mimalloc-test-stress-static PRIVATE ${mi_defines} "USE_STD_MALLOC=1")
     target_compile_options(mimalloc-test-stress-static PRIVATE ${mi_cflags})
@@ -1161,7 +1161,7 @@ if (MI_BUILD_TESTS)
   endif()
 
   # dynamic override test
-  if(MI_BUILD_SHARED AND NOT (MI_CYGWIN OR (MI_TRACK STREQUAL "ASAN") OR MI_DEBUG_TSAN OR MI_DEBUG_UBSAN)) # AND NOT (APPLE AND MI_USE_CXX))
+  if(MI_BUILD_SHARED AND MI_OVERRIDE AND NOT (MI_CYGWIN OR (MI_TRACK STREQUAL "ASAN") OR MI_DEBUG_TSAN OR MI_DEBUG_UBSAN)) # AND NOT (APPLE AND MI_USE_CXX))
     add_executable(mimalloc-test-stress-dynamic test/test-stress.c)
     target_compile_definitions(mimalloc-test-stress-dynamic PRIVATE ${mi_defines} "USE_STD_MALLOC=1")
     target_compile_options(mimalloc-test-stress-dynamic PRIVATE ${mi_cflags})

include/mimalloc-new-delete.h

@@ -37,7 +37,7 @@ terms of the MIT license. A copy of the license can be found in the file
   void operator delete  (void* p, const std::nothrow_t&) noexcept { mi_free(p); }
   void operator delete[](void* p, const std::nothrow_t&) noexcept { mi_free(p); }
 
-  mi_decl_new(n) void* operator new(std::size_t n) noexcept(false) { return mi_new(n); }
+  mi_decl_new(n) void* operator new(std::size_t n) noexcept(false)   { return mi_new(n); }
   mi_decl_new(n) void* operator new[](std::size_t n) noexcept(false) { return mi_new(n); }
 
   mi_decl_new_nothrow(n) void* operator new  (std::size_t n, const std::nothrow_t& tag) noexcept { (void)(tag); return mi_new_nothrow(n); }

include/mimalloc.h

@@ -396,43 +396,6 @@ mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_zalloc_aligned(
 mi_decl_nodiscard mi_decl_export                  void* mi_theap_realloc(mi_theap_t* theap, void* p, size_t newsize)              mi_attr_noexcept mi_attr_alloc_size(3);
 mi_decl_nodiscard mi_decl_export                  void* mi_theap_rezalloc(mi_theap_t* theap, void* p, size_t newsize)             mi_attr_noexcept mi_attr_alloc_size(3);
 
-// ------------------------------------------------------
-// Fast constant size allocations.
-// ------------------------------------------------------
-
-// Machine word size allocation. `wsize` is the allocation size in machine words (`sizeof(size_t)`)
-mi_decl_nodiscard mi_decl_restrict void* mi_wzalloc_small(size_t wsize) mi_attr_noexcept;
-mi_decl_nodiscard mi_decl_restrict void* mi_wmalloc_small(size_t wsize) mi_attr_noexcept;
-mi_decl_nodiscard mi_decl_restrict void* mi_theap_wmalloc_small(mi_theap_t* theap, size_t wsize) mi_attr_noexcept;
-mi_decl_nodiscard mi_decl_restrict void* mi_theap_wzalloc_small(mi_theap_t* theap, size_t wsize) mi_attr_noexcept;
-
-// get the machine word size from a byte size.
-static inline size_t mi_wsize_from_size(size_t size) { 
-  return ((size + sizeof(size_t) - 1) / sizeof(size_t));  
-}
-
-static inline mi_decl_restrict void* mi_malloc_csize(size_t size) mi_attr_noexcept {
-  if (size <= MI_SMALL_SIZE_MAX) { return mi_wmalloc_small(mi_wsize_from_size(size)); } else { return mi_malloc(size); }
-}
-static inline mi_decl_restrict void* mi_zalloc_csize(size_t size) mi_attr_noexcept {
-  if (size <= MI_SMALL_SIZE_MAX) { return mi_wzalloc_small(mi_wsize_from_size(size)); } else { return mi_zalloc(size); }
-}
-static inline mi_decl_restrict void* mi_theap_malloc_csize(mi_theap_t* theap, size_t size) mi_attr_noexcept {  
-  assert(theap!=NULL);
-  if (size <= MI_SMALL_SIZE_MAX) { return mi_theap_wmalloc_small(theap,mi_wsize_from_size(size)); } else { return mi_theap_malloc(theap,size); }
-}
-static inline mi_decl_restrict void* mi_theap_zalloc_csize(mi_theap_t* theap, size_t size) mi_attr_noexcept {
-  assert(theap!=NULL);
-  if (size <= MI_SMALL_SIZE_MAX) { return mi_theap_wzalloc_small(theap,mi_wsize_from_size(size)); } else { return mi_theap_zalloc(theap,size); }
-}
-static inline void mi_free_csize(void* p, size_t size) mi_attr_noexcept {
-  if (size <= MI_SMALL_SIZE_MAX) { mi_free_small(p); } else { mi_free(p); }
-}
-static inline void mi_free_csize_nonnull(void* p, size_t size) mi_attr_noexcept {
-  assert(p!=NULL);
-  if (size <= MI_SMALL_SIZE_MAX) { mi_free_small_nonnull(p); } else { mi_free(p); }
-}
-
 // ------------------------------------------------------
 // Experimental
 // ------------------------------------------------------
@@ -588,8 +551,13 @@ mi_decl_export int mi_wdupenv_s(wchar_t** buf, size_t* size, const wchar_t* name
 mi_decl_nodiscard mi_decl_export mi_decl_restrict wchar_t* mi_wcsdup(const wchar_t* s)  mi_attr_noexcept mi_attr_malloc;
 mi_decl_nodiscard mi_decl_export mi_decl_restrict unsigned char* mi_mbsdup(const unsigned char* s)  mi_attr_noexcept mi_attr_malloc;
 
-// The `mi_new` wrappers implement C++ semantics on out-of-memory instead of directly returning `NULL`.
-// (and call `std::get_new_handler` and potentially raise a `std::bad_alloc` exception).
+// --------------------------------------------------------
+// C++ wrappers
+// The `mi_new` wrappers implement C++ semantics on out-of-memory 
+// instead of directly returning `NULL`. (and call `std::get_new_handler` 
+// and potentially raise a `std::bad_alloc` exception).
+// --------------------------------------------------------
+
 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new(size_t size)                   mi_attr_malloc mi_attr_alloc_size(1);
 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_aligned(size_t size, size_t alignment) mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2);
 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_nothrow(size_t size)           mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
@@ -601,6 +569,63 @@ mi_decl_nodiscard mi_decl_export void* mi_new_reallocn(void* p, size_t newcount,
 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_alloc_new(mi_heap_t* heap, size_t size)                 mi_attr_malloc mi_attr_alloc_size(2);
 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_alloc_new_n(mi_heap_t* heap, size_t count, size_t size) mi_attr_malloc mi_attr_alloc_size2(2, 3);
 
+mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_alloc_new(mi_theap_t* theap, size_t size) mi_attr_malloc mi_attr_alloc_size(2);
+mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_alloc_new_n(mi_theap_t* theap, size_t count, size_t size) mi_attr_malloc mi_attr_alloc_size2(2, 3);
+mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_alloc_new_nothrow(mi_theap_t* theap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
+
+
+// --------------------------------------------------------------------------
+// Inlined constant size allocations.
+// These are meant for runtime systems, or overrides where we need the best 
+// performance for small, constant-size allocations.
+// These are only better than the regular functions if the size or alignment 
+// are indeed constant at the call site.
+// --------------------------------------------------------------------------
+
+// Internal machine word size allocation. `wsize` is the allocation size in machine words (`sizeof(size_t)`)
+mi_decl_nodiscard mi_decl_restrict void* mi_wzalloc_small(size_t wsize) mi_attr_noexcept mi_attr_malloc;
+mi_decl_nodiscard mi_decl_restrict void* mi_wmalloc_small(size_t wsize) mi_attr_noexcept mi_attr_malloc;
+mi_decl_nodiscard mi_decl_restrict void* mi_theap_wmalloc_small(mi_theap_t* theap, size_t wsize) mi_attr_noexcept mi_attr_malloc;
+mi_decl_nodiscard mi_decl_restrict void* mi_theap_wzalloc_small(mi_theap_t* theap, size_t wsize) mi_attr_noexcept mi_attr_malloc;
+
+static inline size_t mi_wsize_from_size(size_t size) mi_attr_noexcept {
+  return (size + sizeof(size_t) - 1) / sizeof(size_t);
+}
+
+static inline mi_decl_restrict void* mi_malloc_csize(size_t size) mi_attr_noexcept {
+  if (size <= MI_SMALL_SIZE_MAX) { return mi_wmalloc_small(mi_wsize_from_size(size)); } else { return mi_malloc(size); }
+}
+static inline mi_decl_restrict void* mi_zalloc_csize(size_t size) mi_attr_noexcept {
+  if (size <= MI_SMALL_SIZE_MAX) { return mi_wzalloc_small(mi_wsize_from_size(size)); } else { return mi_zalloc(size); }
+}
+static inline mi_decl_restrict void* mi_theap_malloc_csize(mi_theap_t* theap, size_t size) mi_attr_noexcept {  
+  assert(theap!=NULL);
+  if (size <= MI_SMALL_SIZE_MAX) { return mi_theap_wmalloc_small(theap,mi_wsize_from_size(size)); } else { return mi_theap_malloc(theap,size); }
+}
+static inline mi_decl_restrict void* mi_theap_zalloc_csize(mi_theap_t* theap, size_t size) mi_attr_noexcept {
+  assert(theap!=NULL);
+  if (size <= MI_SMALL_SIZE_MAX) { return mi_theap_wzalloc_small(theap,mi_wsize_from_size(size)); } else { return mi_theap_zalloc(theap,size); }
+}
+
+static inline void mi_free_csize(void* p, size_t size) mi_attr_noexcept {
+  if (size <= MI_SMALL_SIZE_MAX) { mi_free_small(p); } else { mi_free(p); }
+}
+static inline void mi_free_csize_nonnull(void* p, size_t size) mi_attr_noexcept {
+  assert(p!=NULL);
+  if (size <= MI_SMALL_SIZE_MAX) { mi_free_small_nonnull(p); } else { mi_free(p); }
+}
+static inline void mi_free_csize_aligned(void* p, size_t size, size_t aligned) mi_attr_noexcept {
+  if (aligned <= size && size <= MI_SMALL_SIZE_MAX) { mi_free_small(p); } else { mi_free(p); }
+}
+static inline void mi_free_csize_aligned_nonnull(void* p, size_t size, size_t aligned) mi_attr_noexcept {
+  assert(p!=NULL);
+  if (aligned <= size && size <= MI_SMALL_SIZE_MAX) { mi_free_small_nonnull(p); } else { mi_free(p); }
+}
+
+// ------------------------------------------------------
+// C++ standard library allocator interface.
+// ------------------------------------------------------
+
 #ifdef __cplusplus
 }
 #endif

include/mimalloc/types.h

@@ -535,6 +535,13 @@ typedef struct mi_page_s {
 #define MI_MAX_SINGLETON_BIN   MI_BIN_HUGE
 #endif
 
+// The small object max size must be larger than twice the small size max minus one,
+// such that any aligned allocation with `alignment <= size <= MI_SMALL_SIZE_MAX` will
+// be allocated in a small page (so `mi_free_csize` can delegate correctly to `mi_free_small`.
+#if MI_SMALL_MAX_OBJ_SIZE <= (2*(MI_SMALL_WSIZE_MAX * MI_SIZE_SIZE)-1)
+#error "mimalloc internal: the small object max size is too small"
+#endif
+
 // ------------------------------------------------------
 // Page kinds
 // ------------------------------------------------------

src/alloc.c

@@ -818,15 +818,15 @@ static mi_decl_noinline void* mi_theap_try_new(mi_theap_t* theap, size_t size, b
   return p;
 }
 
-static mi_decl_noinline void* mi_try_new(size_t size, bool nothrow) {
-  return mi_theap_try_new(_mi_theap_default(), size, nothrow);
-}
+// static mi_decl_noinline void* mi_try_new(size_t size, bool nothrow) {
+//   return mi_theap_try_new(_mi_theap_default(), size, nothrow);
+// }
 
 static mi_decl_noinline void* mi_heap_try_new(mi_heap_t* heap, size_t size, bool nothrow) {
   return mi_theap_try_new(_mi_heap_theap(heap), size, nothrow);
 }
 
-mi_decl_nodiscard static mi_decl_restrict void* mi_theap_alloc_new(mi_theap_t* theap, size_t size) {
+mi_decl_nodiscard mi_decl_restrict void* mi_theap_alloc_new(mi_theap_t* theap, size_t size) {
   void* p = mi_theap_malloc(theap,size);
   if mi_unlikely(p == NULL) return mi_theap_try_new(theap, size, false);
   return p;
@@ -842,7 +842,7 @@ mi_decl_nodiscard mi_decl_restrict void* mi_heap_alloc_new(mi_heap_t* heap, size
   return p;
 }
 
-mi_decl_nodiscard static mi_decl_restrict void* mi_theap_alloc_new_n(mi_theap_t* theap, size_t count, size_t size) {
+mi_decl_nodiscard mi_decl_restrict void* mi_theap_alloc_new_n(mi_theap_t* theap, size_t count, size_t size) {
   size_t total;
   if mi_unlikely(mi_count_size_overflow(count, size, &total)) {
     mi_try_new_handler(false);  // on overflow we invoke the try_new_handler once to potentially throw std::bad_alloc
@@ -861,12 +861,17 @@ mi_decl_nodiscard mi_decl_restrict void* mi_heap_alloc_new_n(mi_heap_t* heap, si
   return mi_theap_alloc_new_n(_mi_heap_theap(heap), count, size);
 }
 
-mi_decl_nodiscard mi_decl_restrict void* mi_new_nothrow(size_t size) mi_attr_noexcept {
-  void* p = mi_malloc(size);
-  if mi_unlikely(p == NULL) return mi_try_new(size, true);
+
+mi_decl_nodiscard mi_decl_restrict void* mi_theap_alloc_new_nothrow(mi_theap_t* theap, size_t size) mi_attr_noexcept {
+  void* p = mi_theap_malloc(theap,size);
+  if mi_unlikely(p == NULL) return mi_theap_try_new(theap, size, true);
   return p;
 }
 
+mi_decl_nodiscard mi_decl_restrict void* mi_new_nothrow(size_t size) mi_attr_noexcept {
+  return mi_theap_alloc_new_nothrow(_mi_theap_default(), size);
+}
+
 static mi_decl_noinline void* mi_try_new_aligned(size_t size, size_t alignment, bool nothrow) {
   void* p = NULL;
   for(int i = 0; i < MI_TRY_NEW_MAX && p==NULL && mi_try_new_handler(nothrow); i++) {

src/arena.c

@@ -889,11 +889,15 @@ mi_decl_maybe_unused static size_t mi_page_block_start(size_t block_size, bool o
   if (os_align) {
     offset = MI_PAGE_ALIGN;
   }
-  else if (_mi_is_power_of_two(block_size) && block_size <= MI_PAGE_MAX_START_BLOCK_ALIGN2) {
+  else if (block_size != 0 && _mi_is_power_of_two(block_size) && block_size <= MI_PAGE_MAX_START_BLOCK_ALIGN2) {
     // naturally align power-of-2 blocks up to MI_PAGE_MAX_START_BLOCK_ALIGN2 size (4KiB)
     offset = _mi_align_up(mi_page_info_size(), block_size);
     if (block_size < 64) { offset += 3*block_size; }
   }
+  else if (block_size != 0 && block_size <= MI_SMALL_SIZE_MAX) {
+    // align small blocks to their size
+    offset = _mi_align_up(mi_page_info_size(), block_size);
+  }
   else if (block_size != 0 && (block_size % MI_PAGE_OSPAGE_BLOCK_ALIGN2) == 0) {
     // also align large pages that are a multiple of MI_PAGE_OSPAGE_BLOCK_ALIGN2 (4KiB)
     offset = _mi_align_up(mi_page_info_size(), MI_PAGE_OSPAGE_BLOCK_ALIGN2);

src/free.c

@@ -345,7 +345,7 @@ void mi_free_size(void* p, size_t size) mi_attr_noexcept {
     if (page==NULL) return;
     mi_assert(p!=NULL);
     const mi_block_t* block = _mi_page_ptr_unalign(page, p);      
-    const size_t usable = mi_page_usable_size(page,p);
+    const size_t usable = _mi_page_usable_size(page,p);
     if mi_unlikely(size > usable) { 
       const bool is_guarded = mi_block_ptr_is_guarded(block,p);
       if (!is_guarded) {

test/test-api.c

@@ -73,26 +73,6 @@ static bool mem_is_zero(const void* p, size_t size) {
 int main(void) {
   mi_option_disable(mi_option_verbose);
 
-  #if 1
-  #if defined(__cplusplus) && !defined(_MSC_VER)
-  CHECK_BODY("c++ new-handler") {
-    std::set_new_handler([]{ throw std::bad_alloc(); });
-    void* p = mi_new_nothrow(SIZE_MAX/2);
-    result = (p==NULL);
-  }
-  CHECK_BODY("c++ new handler2") {
-    try {
-      void* p = mi_new_n(SIZE_MAX/2, 4);
-      (void)(p);
-      result = false;
-    }
-    catch(std::bad_alloc) {
-      result = true;
-    }
-  }
-  #endif
-  #endif
-
   // ---------------------------------------------------
   // Malloc
   // ---------------------------------------------------
@@ -319,12 +299,22 @@ int main(void) {
         for(int i = 0; i < 10 && ok; i++) {
           mi_free(p[i]);
         }
-        /*
-        if (ok && align <= size && ((size + MI_PADDING_SIZE) & (align-1)) == 0) {
-          size_t bsize = mi_good_size(size);
-          ok = (align <= bsize && (bsize & (align-1)) == 0);
+      }
+    }
+    result = ok;
+  }
+  CHECK_BODY("mimalloc-size-aligned14") {
+    bool ok = true;
+    for( size_t size = 1; size <= (MI_SMALL_SIZE_MAX * 2) && ok; size++ ) {
+      for(size_t align = 1; align <= 16*size && ok; align *= 2) {
+        void* p[10];
+        for(int i = 0; i < 10 && ok; i++) {
+          p[i] = mi_malloc_aligned(size,align);;
+          ok = (p[i] != NULL && ((uintptr_t)(p[i]) % align) == 0);
+        }
+        for(int i = 0; i < 10 && ok; i++) {
+          mi_free_size_aligned(p[i],size,align);
         }
-        */
       }
     }
     result = ok;
@@ -355,9 +345,7 @@ int main(void) {
       memset(junk, 0xAB, size);
       mi_free(junk);
       uint8_t* z = (uint8_t*)mi_theap_zalloc_csize(theap, size);
-      for (size_t i = 0; i < size; i++) {
-        if (z[i] != 0) { ok = false; break; }
-      }
+      ok = mem_is_zero(z, size);
       mi_free(z);
     }
     result = ok;
@@ -496,6 +484,29 @@ int main(void) {
   }
   #endif
 
+
+  // ---------------------------------------------------
+  // C++
+  // ---------------------------------------------------
+  
+  #if defined(__cplusplus) && !defined(_MSC_VER)  
+  CHECK_BODY("c++ new-handler") {
+    std::set_new_handler([]{ throw std::bad_alloc(); });
+    void* p = mi_new_nothrow(SIZE_MAX/2);
+    result = (p==NULL);
+  }
+  CHECK_BODY("c++ new handler2") {
+    try {
+      void* p = mi_new_n(SIZE_MAX/2, 4);
+      (void)(p);
+      result = false;
+    }
+    catch(std::bad_alloc) {
+      result = true;
+    }
+  }
+  #endif
+
   // ---------------------------------------------------
   // Heaps
   // ---------------------------------------------------