Merge pull request #2 from Zia-Rashid/dev

cleaned up dead/unnecessary code, somehow improved performance.

Author: Zia-Rashid

Makefile

@@ -43,7 +43,6 @@ BIN_DIR   := bin
 ZIALLOC_MAIN := allocators/zialloc/alloc.cpp
 ifeq ($(ALLOCATOR),$(ZIALLOC_MAIN))
 ALLOC_CPP_SRCS := allocators/zialloc/alloc.cpp \
-				  allocators/zialloc/free.cpp \
 				  allocators/zialloc/os.cpp \
 				  allocators/zialloc/segments.cpp
 ALLOC_OBJ := $(patsubst %.cpp,$(BUILD_DIR)/%.o,$(ALLOC_CPP_SRCS))

allocators/zialloc/free.cpp

@@ -86,12 +86,6 @@ constexpr size_t page_kind_size(int kind) {
                               SEGMENT_SIZE;
 }
 
-typedef enum segment_kind_e {
-  SEGMENT_NORM, // most allocations
-  SEGMENT_HUGE, // if page kind is "huge"
-  SEGMENT_GUARD
-} segment_kind_t;
-
 typedef enum page_status_e { FULL, ACTIVE, EMPTY } page_status_t;
 
 // all power aligned (i think), we can fill unused space w/ guard chunks.
@@ -107,46 +101,6 @@ typedef enum chunk_max_e {
   CHUNK_XL             // whatever it wants to be
 } chunk_max_t;
 
-// Memory can reside in arena's, direct OS allocated, or statically allocated.
-// The memid keeps track of this.
-typedef enum memkind_e {
-  MEM_NONE,     // not allocated
-  MEM_EXTERNAL, // not owned by mimalloc but provided externally (via
-                // `mi_manage_os_memory` for example)
-  MEM_STATIC,   // allocated in a static area and should not be freed (for arena
-                // meta data for example)
-  MEM_OS,       // allocated from the OS
-  MEM_OS_HUGE,  // allocated as huge OS pages (usually 1GiB, pinned to physical
-                // memory)
-  MEM_OS_REMAP, // allocated in a remapable area (i.e. using `mremap`)
-  MEM_ARENA     // allocated from an arena (the usual case)
-} memkind_t;
-
-constexpr static inline bool memkind_is_os(memkind_t memkind) {
-  return (memkind >= MEM_OS && memkind <= MEM_OS_REMAP);
-}
-
-typedef struct memid_os_info {
-  void *base;  // actual base address of the block (used for offset aligned
-               // allocations)
-  size_t size; // full allocation size
-} memid_os_info_t;
-typedef struct memid_arena_info {
-  size_t block_index; // index in the arena
-  uint8_t id;         // arena id (>= 1)
-  bool is_exclusive;  // this arena can only be used for specific arena
-                      // allocations
-} memid_arena_info_t;
-typedef struct memid_s {
-  union {
-    memid_os_info_t os;       // MI_MEM_OS
-    memid_arena_info_t arena; // MI_MEM_ARENA
-  } mem;
-  bool is_pinned; // `true` if we cannot decommit/reset/protect in this memory
-  bool initially_committed; 
-  bool initially_zero; 
-} memid_t;
-
 static inline pid_t current_tid() { return syscall(SYS_gettid); }
 
 static inline uint64_t generate_canary() {

allocators/zialloc/mem.h

@@ -415,8 +415,6 @@ class Page {
   page_status_t get_status() const { return status; }
   size_t get_chunk_usable() const { return chunk_usable; }
   pid_t get_owner_tid() const { return owner_tid; }
-  uintptr_t get_base_addr() const { return reinterpret_cast<uintptr_t>(base); }
-  size_t get_span_size() const { return page_span; }
   size_t get_segment_index() const { return owner_segment_idx; }
   Segment *get_owner_segment() const { return owner_segment; }
 };
@@ -512,45 +510,32 @@ class Segment {
       void *out = nullptr;
       page_status_t before = EMPTY;
       page_status_t after = EMPTY;
-      if (mt) {
-        std::lock_guard<std::mutex> lk(page_lock_for(&page));
+      auto alloc_from_page = [&]() -> void * {
         const size_t target_req =
             (fixed_chunk_set.load(std::memory_order_relaxed))
                 ? fixed_chunk_usable.load(std::memory_order_relaxed)
                 : req;
         if (!page.is_initialized()) {
           void *page_base = static_cast<void *>(static_cast<char *>(base) + idx * page_size);
           if (!page.init(page_base, size_class, target_req))
-            continue;
+            return nullptr;
           if (!fixed_chunk_set.load(std::memory_order_relaxed)) {
             fixed_chunk_usable.store(page.get_chunk_usable(), std::memory_order_relaxed);
             fixed_chunk_set.store(true, std::memory_order_relaxed);
           }
         }
         if (!page.can_hold(req)) {
-          continue;
+          return nullptr;
         }
 
-        out = page.allocate(req, &before, &after); // dispatch to page lvl alloc
-      } else {
-        const size_t target_req =
-            (fixed_chunk_set.load(std::memory_order_relaxed))
-                ? fixed_chunk_usable.load(std::memory_order_relaxed)
-                : req;
-        if (!page.is_initialized()) {
-          void *page_base = static_cast<void *>(static_cast<char *>(base) + idx * page_size);
-          if (!page.init(page_base, size_class, target_req))
-            continue;
-          if (!fixed_chunk_set.load(std::memory_order_relaxed)) {
-            fixed_chunk_usable.store(page.get_chunk_usable(), std::memory_order_relaxed);
-            fixed_chunk_set.store(true, std::memory_order_relaxed);
-          }
-        }
-        if (!page.can_hold(req)) {
-          continue;
-        }
+        return page.allocate(req, &before, &after);
+      };
 
-        out = page.allocate(req, &before, &after);
+      if (mt) {
+        std::lock_guard<std::mutex> lk(page_lock_for(&page));
+        out = alloc_from_page();
+      } else {
+        out = alloc_from_page();
       }
 
       if (!out)
@@ -592,30 +577,19 @@ class Segment {
   }
 };
 
-typedef struct tc_page_s {
-  Page *loc;
-  page_kind_t kind;
-  size_t size;
-  void *freelist;
-} tc_page_t;
-
 class ThreadCache {
 private:
   static std::atomic<uint32_t> live_threads;
   pid_t tid;
   bool is_active;
-  std::vector<tc_page_t *> pages;
   Page *cached_pages[3];
-  uintptr_t cached_page_bases[3];
-  uintptr_t cached_page_ends[3];
   size_t preferred_seg_idx[3];
   bool preferred_seg_valid[3];
 
 public:
   ThreadCache()
-      : tid(current_tid()), is_active(true), pages(),
-        cached_pages{nullptr, nullptr, nullptr}, cached_page_bases{0, 0, 0},
-        cached_page_ends{0, 0, 0}, preferred_seg_idx{0, 0, 0},
+      : tid(current_tid()), is_active(true), cached_pages{nullptr, nullptr, nullptr},
+        preferred_seg_idx{0, 0, 0},
         preferred_seg_valid{false, false, false} {
     live_threads.fetch_add(1, std::memory_order_relaxed);
     g_live_threads.fetch_add(1, std::memory_order_relaxed);
@@ -644,13 +618,11 @@ class ThreadCache {
     return cached_pages[class_index_for_kind(kind)];
   }
 
-  void cache_page(page_kind_t kind, Page *page, uintptr_t page_base, size_t page_size) {
+  void cache_page(page_kind_t kind, Page *page) {
     if (kind > PAGE_LG)
       return;
     const size_t idx = class_index_for_kind(kind);
     cached_pages[idx] = page;
-    cached_page_bases[idx] = page_base;
-    cached_page_ends[idx] = page_base + page_size;
   }
 
   void clear_cached_page(page_kind_t kind, Page *page) {
@@ -659,8 +631,6 @@ class ThreadCache {
     const size_t idx = class_index_for_kind(kind);
     if (cached_pages[idx] == page) {
       cached_pages[idx] = nullptr;
-      cached_page_bases[idx] = 0;
-      cached_page_ends[idx] = 0;
     }
   }
 
@@ -695,15 +665,11 @@ struct ClassShard {
 
 class HeapState {
 private:
-  memid_t memid;
   void *base;
   size_t reserved_size;
   uint32_t num_segments;
   std::vector<std::unique_ptr<Segment>> layout;
-  std::vector<segment_kind_t> seg_kind;
   std::vector<void *> seg_bases;
-  std::vector<page_kind_t> seg_page_kind;
-  memkind_t mem_kind;
   uint64_t canary;
   size_t reserved_cursor;
   std::mutex heap_mu;
@@ -727,7 +693,7 @@ class HeapState {
     shard.non_full_segments.push_back(seg_idx);
   }
 
-  bool add_segment_nolock(void *segment_base, segment_kind_t kind, page_kind_t page_kind) {
+  bool add_segment_nolock(void *segment_base, page_kind_t page_kind) {
     if (!segment_base)
       return false;
 
@@ -737,9 +703,7 @@ class HeapState {
       return false;
 
     layout.push_back(std::move(seg));
-    seg_kind.push_back(kind);
     seg_bases.push_back(segment_base);
-    seg_page_kind.push_back(page_kind);
     num_segments = static_cast<uint32_t>(layout.size());
 
     ClassShard &shard = shard_for(page_kind);
@@ -755,7 +719,7 @@ class HeapState {
     return true;
   }
 
-  bool add_segment_from_reserved_nolock(segment_kind_t kind, page_kind_t page_kind) {
+  bool add_segment_from_reserved_nolock(page_kind_t page_kind) {
     if (!base || reserved_size == 0)
       return false;
     if (reserved_cursor + SEGMENT_SIZE > reserved_size)
@@ -766,7 +730,7 @@ class HeapState {
       return false;
 
     reserved_cursor += SEGMENT_SIZE;
-    return add_segment_nolock(seg_base, kind, page_kind);
+    return add_segment_nolock(seg_base, page_kind);
   }
 
   void *alloc_xl(size_t size) {
@@ -819,8 +783,8 @@ class HeapState {
 
 public:
   HeapState()
-      : memid(), base(nullptr), reserved_size(0), num_segments(0), layout(),
-        seg_kind(), seg_bases(), seg_page_kind(), mem_kind(MEM_NONE), canary(0),
+      : base(nullptr), reserved_size(0), num_segments(0), layout(),
+        seg_bases(), canary(0),
         reserved_cursor(0), heap_mu(), class_shards() {}
 
   static HeapState &instance() {
@@ -837,13 +801,10 @@ class HeapState {
     reserved_size = size;
     reserved_cursor = 0;
     canary = generate_canary();
-    mem_kind = MEM_OS;
 
     const size_t cap = size / SEGMENT_SIZE;
     layout.reserve(cap);
-    seg_kind.reserve(cap);
     seg_bases.reserve(cap);
-    seg_page_kind.reserve(cap);
 
     for (ClassShard &shard : class_shards) {
       std::lock_guard<std::mutex> shard_lk(shard.mu);
@@ -854,14 +815,9 @@ class HeapState {
     return true;
   }
 
-  bool add_segment(void *segment_base, segment_kind_t kind, page_kind_t page_kind) {
+  bool add_segment_from_reserved(page_kind_t page_kind) {
     std::lock_guard<std::mutex> lk(heap_mu);
-    return add_segment_nolock(segment_base, kind, page_kind);
-  }
-
-  bool add_segment_from_reserved(segment_kind_t kind, page_kind_t page_kind) {
-    std::lock_guard<std::mutex> lk(heap_mu);
-    return add_segment_from_reserved_nolock(kind, page_kind);
+    return add_segment_from_reserved_nolock(page_kind);
   }
 
   void *allocate(size_t size) {
@@ -930,7 +886,7 @@ class HeapState {
       if (tc->get_active()) {
         tc->set_preferred_segment(kind, seg_idx);
         if (page)
-          tc->cache_page(kind, page, page->get_base_addr(), page->get_span_size());
+          tc->cache_page(kind, page);
       }
       if (page)
         g_last_alloc_usable = page->get_chunk_usable();
@@ -992,7 +948,7 @@ class HeapState {
     // grow from reserved heap (ideal) instead of mmaping more mem to expand.
     {
       std::lock_guard<std::mutex> lk(heap_mu);
-      if (add_segment_from_reserved_nolock(SEGMENT_NORM, kind)) {
+      if (add_segment_from_reserved_nolock(kind)) {
         return try_segment(layout.size() - 1);
       }
     }
@@ -1003,7 +959,7 @@ class HeapState {
 
     {
       std::lock_guard<std::mutex> lk(heap_mu);
-      if (!add_segment_nolock(seg_mem, SEGMENT_NORM, kind)) {
+      if (!add_segment_nolock(seg_mem, kind)) {
         free_segment(seg_mem, SEGMENT_SIZE);
         return nullptr;
       }
@@ -1050,7 +1006,7 @@ class HeapState {
       }
 
       if (tc->get_active()) {
-        tc->cache_page(kind, page, page->get_base_addr(), page->get_span_size());
+        tc->cache_page(kind, page);
         if (page->get_status() == EMPTY)
           tc->clear_cached_page(kind, page);
       }
@@ -1076,7 +1032,6 @@ class HeapState {
     return usable_xl(ptr);
   }
 
-  std::vector<segment_kind_t> get_segment_kinds() { return seg_kind; }
   uint32_t get_num_segments() { return num_segments; }
 
   bool is_corrupted() {
@@ -1111,9 +1066,7 @@ class HeapState {
     }
 
     layout.clear();
-    seg_kind.clear();
     seg_bases.clear();
-    seg_page_kind.clear();
 
     for (ClassShard &shard : class_shards) {
       std::lock_guard<std::mutex> shard_lk(shard.mu);
@@ -1125,29 +1078,20 @@ class HeapState {
     reserved_size = 0;
     num_segments = 0;
     canary = 0;
-    mem_kind = MEM_NONE;
     reserved_cursor = 0;
   }
 };
 
 } // namespace
 
-bool heap_register_segment(void *segment_base) {
-  return HeapState::instance().add_segment(segment_base, SEGMENT_NORM, PAGE_SM);
-}
-
 void heap_clear_metadata() { HeapState::instance().clear_metadata(); }
 
 bool heap_init_reserved(void *reserved_base, size_t size) {
   return HeapState::instance().init_reserved(reserved_base, size);
 }
 
-bool heap_add_segment_from_reserved(segment_kind_t kind) {
-  return HeapState::instance().add_segment_from_reserved(kind, PAGE_SM);
-}
-
 bool heap_add_segment_for_class(page_kind_t kind) {
-  return HeapState::instance().add_segment_from_reserved(SEGMENT_NORM, kind);
+  return HeapState::instance().add_segment_from_reserved(kind);
 }
 
 void *heap_alloc(size_t size) { return HeapState::instance().allocate(size); }