[Linux] Implement KSM Kernel Samepage Merging with Maps (#4601)

* KSM work

* Windows fixes

* Add KSM logging, cleanup

* Cleanup raycast logging

Author: Akkadius

.gitignore

@@ -68,3 +68,4 @@ compile_flags.txt
 
 # CMake Files
 cmake-build-relwithdebinfo/*
+skill-caps.diff

common/CMakeLists.txt

@@ -98,6 +98,7 @@ SET(common_sources
     json/json.hpp
     json/jsoncpp.cpp
     zone_store.cpp
+    memory/ksm.hpp
     net/console_server.cpp
     net/console_server_connection.cpp
     net/crc32.cpp

common/eqemu_logsys.h

@@ -144,6 +144,7 @@ namespace Logs {
 		XTargets,
 		EvolveItem,
 		PositionUpdate,
+		KSM,
 		MaxCategoryID /* Don't Remove this */
 	};
 
@@ -246,7 +247,8 @@ namespace Logs {
 		"Corpses",
 		"XTargets",
 		"EvolveItem",
-		"PositionUpdate"
+		"PositionUpdate",
+		"KSM" // Kernel Samepage Merging
 	};
 }
 

common/eqemu_logsys_log_aliases.h

@@ -861,7 +861,17 @@
 
 #define LogPositionUpdateDetail(message, ...) do {\
     if (LogSys.IsLogEnabled(Logs::Detail, Logs::PositionUpdate))\
-        OutF(LogSys, Logs::Detail, Logs::PositionUpdate, __FILE__, __func__, __LINE__, message, ##__VA_ARGS__);\
+        OutF(LogSys, Logs::Detail, Logs::PositionUpdate, __FILE__, __func__, __LINE__, message, ##__VA_ARGS__); \
+} while (0)
+
+#define LogKSM(message, ...) do {\
+    if (LogSys.IsLogEnabled(Logs::General, Logs::KSM))\
+        OutF(LogSys, Logs::General, Logs::KSM, __FILE__, __func__, __LINE__, message, ##__VA_ARGS__);\
+} while (0)
+
+#define LogKSMDetail(message, ...) do {\
+    if (LogSys.IsLogEnabled(Logs::Detail, Logs::KSM))\
+        OutF(LogSys, Logs::Detail, Logs::KSM, __FILE__, __func__, __LINE__, message, ##__VA_ARGS__);\
 } while (0)
 
 #define Log(debug_level, log_category, message, ...) do {\

common/memory/ksm.hpp

@@ -0,0 +1,220 @@
+#ifndef EQEMU_KSM_HPP
+#define EQEMU_KSM_HPP
+
+#include "../eqemu_logsys.h"
+#include <iostream>
+#include <vector>
+#include <cstring>
+#ifdef _WIN32
+#include <malloc.h>  // For _aligned_malloc, _aligned_free
+#include <windows.h>
+#else
+#include <sys/mman.h>  // For madvise
+#include <unistd.h>    // For sysconf, sbrk
+#endif
+
+
+// Page-aligned allocator for std::vector
+template <typename T>
+class PageAlignedAllocator {
+public:
+	using value_type = T;
+
+	PageAlignedAllocator() noexcept = default;
+	template <typename U>
+	PageAlignedAllocator(const PageAlignedAllocator<U>&) noexcept {}
+
+	T* allocate(std::size_t n) {
+		void* ptr = nullptr;
+		size_t size = n * sizeof(T);
+
+#ifdef _WIN32
+		// Simply allocate memory without alignment
+        ptr = malloc(size);
+        if (!ptr) throw std::bad_alloc();
+#else
+		size_t alignment = getPageSize(); // Get the system's page size
+		if (posix_memalign(&ptr, alignment, size) != 0) {
+			throw std::bad_alloc();
+		}
+#endif
+		return static_cast<T*>(ptr);
+	}
+
+	void deallocate(T* p, std::size_t) noexcept {
+		free(p);
+	}
+
+private:
+	size_t getPageSize() const
+	{
+#ifdef _WIN32
+		SYSTEM_INFO sysInfo;
+		GetSystemInfo(&sysInfo);
+		return sysInfo.dwPageSize; // Page size in bytes
+#else
+		return static_cast<size_t>(sysconf(_SC_PAGESIZE));
+#endif
+	};
+};
+
+template <typename T, typename U>
+bool operator==(const PageAlignedAllocator<T>&, const PageAlignedAllocator<U>&) noexcept {
+	return true;
+}
+
+template <typename T, typename U>
+bool operator!=(const PageAlignedAllocator<T>&, const PageAlignedAllocator<U>&) noexcept {
+	return false;
+}
+
+// Kernel Samepage Merging (KSM) functionality
+namespace KSM {
+
+#ifdef _WIN32
+	// Windows-specific placeholder functions (no-op)
+    inline void CheckPageAlignment(void* ptr) {
+    }
+
+    inline void* AllocatePageAligned(size_t size) {
+        return memset(malloc(size), 0, size);
+    }
+
+    inline void MarkMemoryForKSM(void* start, size_t size) {
+    }
+
+    inline void AlignHeapToPageBoundary() {
+    }
+
+    inline void* MarkHeapStart() {
+        return nullptr;
+    }
+
+    inline size_t MeasureHeapUsage(void* start) {
+        return 0;
+    }
+#else
+	// Linux-specific functionality
+	inline void CheckPageAlignment(void* ptr) {
+		size_t page_size = sysconf(_SC_PAGESIZE);
+		if (reinterpret_cast<uintptr_t>(ptr) % page_size == 0) {
+			LogKSMDetail("Memory is page-aligned [{}]", ptr);
+		} else {
+			LogKSMDetail("Memory is NOT page-aligned [{}]", ptr);
+		}
+	}
+
+	inline void* AllocatePageAligned(size_t size) {
+		size_t page_size = sysconf(_SC_PAGESIZE);
+		void* aligned_ptr = nullptr;
+		if (posix_memalign(&aligned_ptr, page_size, size) != 0) {
+			LogKSM("Failed to allocate page-aligned memory on Linux. page_size [{}] size [{}] bytes", page_size, size);
+		}
+		std::memset(aligned_ptr, 0, size);
+		return aligned_ptr;
+	}
+
+	inline void MarkMemoryForKSM(void* start, size_t size) {
+		if (madvise(start, size, MADV_MERGEABLE) == 0) {
+			LogKSM("Marked memory for KSM | start [{}] size [{}] bytes", start, size);
+		} else {
+			perror("madvise failed");
+		}
+	}
+
+	inline void AlignHeapToPageBoundary() {
+		size_t page_size = sysconf(_SC_PAGESIZE);
+		if (page_size == 0) {
+			LogKSM("Failed to retrieve page size SC_PAGESIZE [{}]", page_size);
+			return;
+		}
+
+		void* current_break = sbrk(0);
+		if (current_break == (void*)-1) {
+			LogKSM("Failed to retrieve the current program break");
+			return;
+		}
+
+		uintptr_t current_address = reinterpret_cast<uintptr_t>(current_break);
+		size_t misalignment = current_address % page_size;
+
+		if (misalignment != 0) {
+			size_t adjustment = page_size - misalignment;
+			if (sbrk(adjustment) == (void*)-1) {
+				LogKSM("Failed to align heap to page boundary. adjustment [{}] bytes", adjustment);
+				return;
+			}
+		}
+
+		LogKSMDetail("Heap aligned to next page boundary. Current break [{}]", sbrk(0));
+	}
+
+	inline void* MarkHeapStart() {
+		void* current_pos = sbrk(0);
+		AlignHeapToPageBoundary();
+		return current_pos;
+	}
+
+	inline size_t MeasureHeapUsage(void* start) {
+		void* current_break = sbrk(0);
+		return static_cast<char*>(current_break) - static_cast<char*>(start);
+	}
+#endif
+
+
+	inline size_t getPageSize()
+	{
+#ifdef _WIN32
+		SYSTEM_INFO sysInfo;
+		GetSystemInfo(&sysInfo);
+		return sysInfo.dwPageSize; // Page size in bytes
+#else
+		return static_cast<size_t>(sysconf(_SC_PAGESIZE)); // POSIX page size
+#endif
+	};
+
+	template <typename T>
+	inline void PageAlignVectorAligned(std::vector<T, PageAlignedAllocator<T>>& vec) {
+		if (vec.empty()) {
+			return;
+		}
+
+		size_t page_size = getPageSize();
+		void* start = vec.data();
+		size_t size = vec.size() * sizeof(T);
+
+		// Check if the memory is page-aligned
+		if (reinterpret_cast<std::uintptr_t>(start) % page_size != 0) {
+			// Allocate a new aligned vector
+			std::vector<T, PageAlignedAllocator<T>> aligned_vec(vec.get_allocator());
+			aligned_vec.reserve(vec.capacity()); // Match capacity to avoid reallocation during copy
+
+			// Copy elements from the original vector
+			aligned_vec.insert(aligned_vec.end(), vec.begin(), vec.end());
+
+			// Swap the aligned vector with the original vector
+			vec.swap(aligned_vec);
+
+			// Clear the temporary aligned vector to free its memory
+			aligned_vec.clear();
+
+			// Verify the new alignment
+			start = vec.data();
+			if (reinterpret_cast<std::uintptr_t>(start) % page_size != 0) {
+				throw std::runtime_error("Failed to align vector memory to page boundaries.");
+			}
+
+			LogKSMDetail("Vector reallocated to ensure page alignment. start [{}] size [{}] bytes", start, size);
+		} else {
+			LogKSMDetail("Vector is already page-aligned. start [{}] size [{}] bytes", start, size);
+		}
+
+#ifndef _WIN32
+		// Mark memory for KSM (only on non-Windows systems)
+		MarkMemoryForKSM(start, size);
+#endif
+	}
+
+}
+
+#endif // EQEMU_KSM_HPP

world/cli/test.cpp

@@ -1,6 +1,8 @@
 #include <cereal/archives/json.hpp>
 #include <cereal/types/vector.hpp>
+#include <iomanip>
 #include "../../common/events/player_events.h"
+#include "../../common/memory/ksm.hpp"
 
 void WorldserverCLI::TestCommand(int argc, char **argv, argh::parser &cmd, std::string &description)
 {
@@ -10,5 +12,21 @@ void WorldserverCLI::TestCommand(int argc, char **argv, argh::parser &cmd, std::
 		return;
 	}
 
-	
+	void* start_marker = KSM::MarkHeapStart();
+	std::cout << "Start marker: " << start_marker << "\n";
+
+	std::vector<std::string> vec = {};
+	for (int i = 0; i < 100000; i++) {
+		vec.push_back("Some random string");
+	}
+
+	// Measure allocated memory size
+	size_t allocated_size = KSM::MeasureHeapUsage(start_marker);
+	// Convert to MB as a float and output with precision
+	double allocated_size_mb = static_cast<double>(allocated_size) / (1024 * 1024);
+	std::cout << std::fixed << std::setprecision(3)
+			  << "Allocated size: " << allocated_size_mb << " MB\n";
+
+	// Mark memory for KSM
+	KSM::MarkMemoryForKSM(start_marker, allocated_size);
 }

zone/gm_commands/loc.cpp

@@ -9,6 +9,19 @@ void command_loc(Client *c, const Seperator *sep)
 
 	auto target_position = target->GetPosition();
 
+	// check los benchmark
+	BenchTimer timer;
+	for (int i = 0; i < 1000; i++) {
+		zone->zonemap->CheckLoS(c->GetPosition(), target_position);
+	}
+	c->Message(
+		Chat::White,
+		fmt::format(
+			"CheckLoS benchmark took [{}]",
+			timer.elapsed()
+		).c_str()
+	);
+
 	c->Message(
 		Chat::White,
 		fmt::format(

zone/map.cpp

@@ -7,6 +7,7 @@
 #include "raycast_mesh.h"
 #include "zone.h"
 #include "../common/file.h"
+#include "../common/memory/ksm.hpp"
 
 #include <algorithm>
 #include <map>
@@ -953,6 +954,7 @@ bool Map::LoadV2(FILE *f) {
 	return true;
 }
 
+
 void Map::RotateVertex(glm::vec3 &v, float rx, float ry, float rz) {
 	glm::vec3 nv = v;