Sanity Checks for DB

CMakeLists.txt

@@ -253,6 +253,7 @@ message(STATUS "Binary name: ${NDD_BINARY_NAME}")
 # Add new src/*.cpp files here when they should be compiled into ndd.
 set(NDD_CORE_SOURCES
     src/sparse/inverted_index.cpp
+    src/utils/system_sanity/system_sanity.cpp
 )
 
 # Build non-main project sources separately so they can be compiled in parallel

docs/logs.md

@@ -68,6 +68,8 @@ The same overload shapes apply to `LOG_WARN` and `LOG_ERROR`.
 ## Rules
 
 - Explicit numeric codes are preferred for stable operational logs.
+- Each numeric code must map to exactly one production log site. Do not reuse a code for a
+  different message or path.
 - Code-less logs are valid and must never receive synthesized IDs.
 - Prefer logging at request boundaries, lifecycle transitions, and rare failure paths.
 - Do not add logs in hot loops or per-vector/per-result paths.
@@ -85,7 +87,7 @@ The same overload shapes apply to `LOG_WARN` and `LOG_ERROR`.
   - `1400s` WAL logs
   - `1500s` metadata logs
   - `1600s` vector storage logs
-  - `1700s` CPU compatibility logs
+  - `1700s` system sanity checks (CPU compatibility, disk, memory, ulimits)
   - `2000s` index manager logs
   - `2100s` HNSW load/cache logs
 

src/core/ndd.hpp

@@ -626,7 +626,7 @@ class IndexManager {
                     LOG_INFO(2017, index_id, "Saving dirty index during shutdown");
                     saveIndex(index_id);
                 } catch(const std::exception& e) {
-                    LOG_ERROR(2017,
+                    LOG_ERROR(2015,
                                     index_id,
                                     "Failed to save index during shutdown: " << e.what());
                 }
@@ -909,7 +909,7 @@ class IndexManager {
                 std::shared_lock<std::shared_mutex> lock(indices_mutex_);
                 auto it = indices_.find(index_id);
                 if(it != indices_.end() && it->second && it->second->is_dirty) {
-                    LOG_INFO(2023, index_id, "Saving dirty index before reload");
+                    LOG_INFO(2055, index_id, "Saving dirty index before reload");
                     saveIndex(index_id);
                 }
             }
@@ -1167,7 +1167,7 @@ class IndexManager {
             throw;
         } catch(const std::exception& e) {
             LOG_ERROR(2027, index_id, "Batch insertion failed: " << e.what());
-            return false;
+            throw std::runtime_error(std::string("Batch insertion failed: ") + e.what());
         }
     }
 
@@ -1983,7 +1983,7 @@ inline void IndexManager::executeBackupJob(const std::string& index_id, const st
 
         // Check stop_token before expensive operations
         if (st.stop_requested()) {
-            LOG_INFO(2046, index_id, "Backup cancelled");
+            LOG_INFO(2056, index_id, "Backup cancelled before backup work started");
             backup_store_.clearActiveBackup(username);
             return;
         }
@@ -2004,7 +2004,7 @@ inline void IndexManager::executeBackupJob(const std::string& index_id, const st
 
             // Check again after acquiring lock (shutdown may have been requested while waiting)
             if (st.stop_requested()) {
-                LOG_INFO(2047, index_id, "Backup cancelled");
+                LOG_INFO(2057, index_id, "Backup cancelled");
                 backup_store_.clearActiveBackup(username);
                 return;
             }

src/main.cpp

@@ -37,8 +37,7 @@
 #include "core/ndd.hpp"
 #include "auth.hpp"
 #include "quant/common.hpp"
-#include "cpu_compat_check/check_avx_compat.hpp"
-#include "cpu_compat_check/check_arm_compat.hpp"
+#include "system_sanity/system_sanity.hpp"
 
 using ndd::quant::quantLevelToString;
 using ndd::quant::stringToQuantLevel;
@@ -142,32 +141,6 @@ inline nlohmann::ordered_json make_index_info_payload(const IndexInfo& info) {
     return payload;
 }
 
-/**
- * Checks if the CPU is compatible with all
- * the instruction sets being used for x86, ARM and MAC Mxx
- */
-bool is_cpu_compatible() {
-    bool ret = true;
-
-#if defined(USE_AVX2) && (defined(__x86_64__) || defined(_M_X64))
-    ret &= is_avx2_compatible();
-#endif  //AVX2 checks
-
-#if defined(USE_AVX512) && (defined(__x86_64__) || defined(_M_X64))
-    ret &= is_avx512_compatible();
-#endif  //AVX512 checks
-
-#if defined(USE_NEON)
-    ret &= is_neon_compatible();
-#endif
-
-#if defined(USE_SVE2)
-    ret &= is_sve2_compatible();
-#endif
-
-    return ret;
-}
-
 // Read file contents
 std::string read_file(const std::string& path) {
     std::ifstream file(path, std::ios::binary);
@@ -233,9 +206,9 @@ int main(int argc, char** argv) {
         return 1;
     }
 
-    if(!is_cpu_compatible()) {
-        LOG_ERROR(1004, "CPU is not compatible; server startup aborted");
-        return 0;
+    if(!run_startup_sanity_checks()) {
+        LOG_ERROR(1799, "Server startup aborted due to failed sanity checks");
+        return 1;
     }
 
     LOG_INFO("SERVER_ID: " << settings::SERVER_ID);
@@ -253,6 +226,7 @@ int main(int argc, char** argv) {
     LOG_INFO("DEFAULT_MAX_ELEMENTS_INCREMENT: " << settings::DEFAULT_MAX_ELEMENTS_INCREMENT);
     LOG_INFO("DEFAULT_MAX_ELEMENTS_INCREMENT_TRIGGER: "
                 << settings::DEFAULT_MAX_ELEMENTS_INCREMENT_TRIGGER);
+    LOG_INFO("MINIMUM_REQUIRED_DRAM_MB: " << settings::MINIMUM_REQUIRED_DRAM_MB);
 
     // Path to React build directory
     // Get the executable's directory and resolve frontend/dist relative to it
@@ -263,7 +237,6 @@ int main(int argc, char** argv) {
 
     // Initialize index manager with persistence config
     std::string data_dir = settings::DATA_DIR;
-    std::filesystem::create_directories(data_dir);
 
     PersistenceConfig persistence_config{
             settings::SAVE_EVERY_N_UPDATES,                        // Save every n updates
@@ -282,10 +255,11 @@ int main(int argc, char** argv) {
 
     // ========== GENERAL ==========
     // Health check endpoint (no auth required)
-    CROW_ROUTE(app, "/api/v1/health").methods("GET"_method)([](const crow::request& req) {
+    // CROW_ROUTE(app, "/api/v1/health").methods("GET"_method)([](const crow::request& req) {
+    CROW_ROUTE(app, "/api/v1/health").methods("GET"_method)([]() {
         crow::json::wvalue response(
                 {{"status", "ok"},
-                 {"timestamp", std::chrono::system_clock::now().time_since_epoch().count()}});
+                {"timestamp", (std::int64_t)std::chrono::system_clock::now().time_since_epoch().count()}});
         PRINT_LOG_TIME();
         ndd::printSparseSearchDebugStats();
         ndd::printSparseUpdateDebugStats();
@@ -450,7 +424,7 @@ int main(int argc, char** argv) {
                         body.has("sparse_model") ? std::string(body["sparse_model"].s()) : "None";
                 const auto sparse_model = ndd::sparseScoringModelFromString(sparse_model_str);
                 if(!sparse_model.has_value()) {
-                    LOG_WARN(1019, index_id, "Invalid sparse_model: " << sparse_model_str);
+                    LOG_WARN(1025, index_id, "Invalid sparse_model: " << sparse_model_str);
                     return json_error(
                         400,
                         "Invalid sparse_model. Must be one of: None, default, endee_bm25");
@@ -470,7 +444,7 @@ int main(int argc, char** argv) {
                     index_manager.createIndex(index_id, config, UserType::Admin, size_in_millions);
                     return crow::response(200, "Index created successfully");
                 } catch(const std::runtime_error& e) {
-                    LOG_WARN(1019, index_id, "Create-index request failed: " << e.what());
+                    LOG_WARN(1026, index_id, "Create-index request failed: " << e.what());
                     return json_error(409, e.what());
                 } catch(const std::exception& e) {
                     return json_error_500(
@@ -936,6 +910,10 @@ int main(int argc, char** argv) {
                 // Verify content type is application/msgpack or application/json
                 auto content_type = req.get_header_value("Content-Type");
 
+                if(is_disk_full()){
+                    return json_error(400, "Batch insertion aborted: Not enough storage space");
+                }
+
                 if(content_type == "application/json") {
                     auto body = crow::json::load(req.body);
                     if(!body) {
@@ -999,7 +977,14 @@ int main(int argc, char** argv) {
 
                     try {
                         bool success = index_manager.addVectors(index_id, vectors);
-                        return crow::response(success ? 200 : 400);
+                        if(!success) {
+                            LOG_WARN(1066,
+                                     ctx.username,
+                                     index_name,
+                                     "Insert request failed without detailed error from addVectors");
+                            return json_error(400, "Batch insertion failed");
+                        }
+                        return crow::response(200);
                     } catch(const std::runtime_error& e) {
                         LOG_WARN(1041, ctx.username, index_name, "Insert request rejected: " << e.what());
                         return json_error(400, e.what());
@@ -1017,13 +1002,27 @@ int main(int argc, char** argv) {
                             auto vectors = obj.as<std::vector<ndd::HybridVectorObject>>();
                             LOG_DEBUG("Batch size (Hybrid): " << vectors.size());
                             bool success = index_manager.addVectors(index_id, vectors);
-                            return crow::response(success ? 200 : 400);
+                            if(!success) {
+                                LOG_WARN(1067,
+                                         ctx.username,
+                                         index_name,
+                                         "Insert request failed without detailed error from addVectors");
+                                return json_error(400, "Batch insertion failed");
+                            }
+                            return crow::response(200);
                         } catch(...) {
                             // Fallback to VectorObject
                             auto vectors = obj.as<std::vector<ndd::VectorObject>>();
                             LOG_DEBUG("Batch size (Dense): " << vectors.size());
                             bool success = index_manager.addVectors(index_id, vectors);
-                            return crow::response(success ? 200 : 400);
+                            if(!success) {
+                                LOG_WARN(1068,
+                                         ctx.username,
+                                         index_name,
+                                         "Insert request failed without detailed error from addVectors");
+                                return json_error(400, "Batch insertion failed");
+                            }
+                            return crow::response(200);
                         }
                     } catch(const std::runtime_error& e) {
                         LOG_WARN(1042, ctx.username, index_name, "Insert request rejected: " << e.what());

src/storage/vector_storage.hpp

@@ -280,6 +280,9 @@ class VectorStore {
                 MDBX_val data{const_cast<uint8_t*>(vector_bytes.data()), vector_bytes.size()};
 
                 int rc = mdbx_put(txn, dbi_, &key, &data, MDBX_UPSERT);
+                if(rc != MDBX_SUCCESS) {
+                    return rc;
+                }
             }
             return MDBX_SUCCESS;
         };
@@ -449,7 +452,7 @@ class MetaStore {
             }
         };
 
-        auto write_batch = [&](MDBX_txn* txn) {
+        auto write_batch = [&](MDBX_txn* txn) -> int {
             for(const auto& [numeric_id, meta] : batch) {
                 msgpack::sbuffer sbuf;
                 msgpack::pack(sbuf, meta);
@@ -459,6 +462,7 @@ class MetaStore {
 
                 int rc = mdbx_put(txn, dbi_, &key, &data, MDBX_UPSERT);
                 if(rc != MDBX_SUCCESS) {
+                    return rc;
                 }
             }
             return MDBX_SUCCESS;

src/utils/settings.hpp

@@ -70,6 +70,15 @@ namespace settings {
     // Maximum number of elements in the index
     constexpr size_t MAX_VECTORS_ADMIN = 1'000'000'000;
 
+
+    //minimum bytes in filesystem before triggering out of storage sequence
+    constexpr size_t MINIMUM_REQUIRED_FS_BYTES = (10 * GB);
+
+    // System sanity check thresholds
+    constexpr size_t DEFAULT_MINIMUM_REQUIRED_DRAM_MB = (4 * 1024); //GB in MB
+    constexpr size_t MINIMUM_OPEN_FILES = 5000;
+    constexpr size_t DEFAULT_MINIMUM_CPU_CORES = 2;
+
     // Buffer for early exit in search base layer
     constexpr int EARLY_EXIT_BUFFER_INSERT = 16;
     constexpr int EARLY_EXIT_BUFFER_QUERY = 8;
@@ -192,6 +201,12 @@ namespace settings {
         return env ? std::stoull(env) : DEFAULT_NUM_RECOVERY_THREADS;
     }();
 
+    // Minimum available DRAM in MB (configurable via NDD_MIN_DRAM_MB)
+    inline static size_t MINIMUM_REQUIRED_DRAM_MB = [] {
+        const char* env = std::getenv("NDD_MIN_DRAM_MB");
+        return env ? std::stoull(env) : DEFAULT_MINIMUM_REQUIRED_DRAM_MB;
+    }();
+
     inline static bool ENABLE_DEBUG_LOG = [] {
         const char* env = std::getenv("NDD_DEBUG_LOG");
         return env ? (std::string(env) == "1" || std::string(env) == "true")
@@ -384,6 +399,8 @@ namespace settings {
         oss << "ENABLE_DEBUG_LOG: " << (ENABLE_DEBUG_LOG ? "true" : "false") << "\n";
         oss << "AUTH_ENABLED: " << (AUTH_ENABLED ? "true" : "false") << "\n";
         oss << "DEFAULT_USERNAME: " << DEFAULT_USERNAME << "\n";
+        oss << "MINIMUM_REQUIRED_DRAM_MB: " << MINIMUM_REQUIRED_DRAM_MB << "\n";
+        oss << "MINIMUM_OPEN_FILES: " << MINIMUM_OPEN_FILES << "\n";
         oss << "\n=== MDBX Map Sizes (bit shifts) ===\n";
         oss << "INDEX_META_MAP_SIZE_BITS: " << INDEX_META_MAP_SIZE_BITS << "\n";
         oss << "INDEX_META_MAP_SIZE_MAX_BITS: " << INDEX_META_MAP_SIZE_MAX_BITS << "\n";