Allow database backend to optimize group-by-XZ operation

.gitignore

@@ -13,3 +13,4 @@ Makefile
 cmake_install.cmake
 cmake_config.h
 compile_commands.json
+.vscode/

src/TileGenerator.cpp

@@ -464,26 +464,20 @@ void TileGenerator::loadBlocks()
 	const int16_t yMin = mod16(m_yMin);
 
 	if (m_exhaustiveSearch == EXH_NEVER || m_exhaustiveSearch == EXH_Y) {
-		std::vector<BlockPos> vec = m_db->getBlockPos(
+		std::vector<BlockPos> vec = m_db->getBlockPosXZ(
 			BlockPos(m_geomX, yMin, m_geomY),
 			BlockPos(m_geomX2, yMax, m_geomY2)
 		);
 
 		for (auto pos : vec) {
 			assert(pos.x >= m_geomX && pos.x < m_geomX2);
-			assert(pos.y >= yMin && pos.y < yMax);
 			assert(pos.z >= m_geomY && pos.z < m_geomY2);
 
 			// Adjust minimum and maximum positions to the nearest block
-			if (pos.x < m_xMin)
-				m_xMin = pos.x;
-			if (pos.x > m_xMax)
-				m_xMax = pos.x;
-
-			if (pos.z < m_zMin)
-				m_zMin = pos.z;
-			if (pos.z > m_zMax)
-				m_zMax = pos.z;
+			m_xMin = mymin<int>(m_xMin, pos.x);
+			m_xMax = mymax<int>(m_xMax, pos.x);
+			m_zMin = mymin<int>(m_zMin, pos.z);
+			m_zMax = mymax<int>(m_zMax, pos.z);
 
 			m_positions[pos.z].emplace(pos.x);
 		}

src/db-leveldb.cpp

@@ -1,5 +1,6 @@
 #include <stdexcept>
 #include <sstream>
+#include <algorithm>
 #include "db-leveldb.h"
 #include "types.h"
 
@@ -18,14 +19,20 @@ static inline std::string i64tos(int64_t i)
 	return os.str();
 }
 
+// finds the first position in the list where it.x >= x
+#define lower_bound_x(container, find_x) \
+	std::lower_bound((container).begin(), (container).end(), (find_x), \
+		[] (const vec2 &left, int16_t right) { \
+			return left.x < right; \
+	})
 
 DBLevelDB::DBLevelDB(const std::string &mapdir)
 {
 	leveldb::Options options;
 	options.create_if_missing = false;
 	leveldb::Status status = leveldb::DB::Open(options, mapdir + "map.db", &db);
 	if (!status.ok()) {
-		throw std::runtime_error(std::string("Failed to open Database: ") + status.ToString());
+		throw std::runtime_error(std::string("Failed to open database: ") + status.ToString());
 	}
 
 	/* LevelDB is a dumb key-value store, so the only optimization we can do
@@ -41,18 +48,24 @@ DBLevelDB::~DBLevelDB()
 }
 
 
-std::vector<BlockPos> DBLevelDB::getBlockPos(BlockPos min, BlockPos max)
+std::vector<BlockPos> DBLevelDB::getBlockPosXZ(BlockPos min, BlockPos max)
 {
 	std::vector<BlockPos> res;
 	for (const auto &it : posCache) {
-		if (it.first < min.z || it.first >= max.z)
+		const int16_t zpos = it.first;
+		if (zpos < min.z || zpos >= max.z)
 			continue;
-		for (auto pos2 : it.second) {
-			if (pos2.first < min.x || pos2.first >= max.x)
+		auto it2 = lower_bound_x(it.second, min.x);
+		for (; it2 != it.second.end(); it2++) {
+			const auto &pos2 = *it2;
+			if (pos2.x >= max.x)
+				break; // went past
+			if (pos2.y < min.y || pos2.y >= max.y)
 				continue;
-			if (pos2.second < min.y || pos2.second >= max.y)
+			// skip duplicates
+			if (!res.empty() && res.back().x == pos2.x && res.back().z == zpos)
 				continue;
-			res.emplace_back(pos2.first, pos2.second, it.first);
+			res.emplace_back(pos2.x, pos2.y, zpos);
 		}
 	}
 	return res;
@@ -61,14 +74,17 @@ std::vector<BlockPos> DBLevelDB::getBlockPos(BlockPos min, BlockPos max)
 
 void DBLevelDB::loadPosCache()
 {
-	leveldb::Iterator * it = db->NewIterator(leveldb::ReadOptions());
+	leveldb::Iterator *it = db->NewIterator(leveldb::ReadOptions());
 	for (it->SeekToFirst(); it->Valid(); it->Next()) {
 		int64_t posHash = stoi64(it->key().ToString());
 		BlockPos pos = decodeBlockPos(posHash);
 
 		posCache[pos.z].emplace_back(pos.x, pos.y);
 	}
 	delete it;
+
+	for (auto &it : posCache)
+		std::sort(it.second.begin(), it.second.end());
 }
 
 
@@ -81,13 +97,18 @@ void DBLevelDB::getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
 	auto it = posCache.find(z);
 	if (it == posCache.cend())
 		return;
-	for (auto pos2 : it->second) {
-		if (pos2.first != x)
-			continue;
-		if (pos2.second < min_y || pos2.second >= max_y)
+	auto it2 = lower_bound_x(it->second, x);
+	if (it2 == it->second.end() || it2->x != x)
+		return;
+	// it2 is now pointing to a contigous part where it2->x == x
+	for (; it2 != it->second.end(); it2++) {
+		const auto &pos2 = *it2;
+		if (pos2.x != x)
+			break; // went past
+		if (pos2.y < min_y || pos2.y >= max_y)
 			continue;
 
-		BlockPos pos(x, pos2.second, z);
+		BlockPos pos(x, pos2.y, z);
 		status = db->Get(leveldb::ReadOptions(), i64tos(encodeBlockPos(pos)), &datastr);
 		if (status.ok()) {
 			blocks.emplace_back(

src/db-leveldb.h

@@ -8,7 +8,7 @@
 class DBLevelDB : public DB {
 public:
 	DBLevelDB(const std::string &mapdir);
-	std::vector<BlockPos> getBlockPos(BlockPos min, BlockPos max) override;
+	std::vector<BlockPos> getBlockPosXZ(BlockPos min, BlockPos max) override;
 	void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
 			int16_t min_y, int16_t max_y) override;
 	void getBlocksByPos(BlockList &blocks,
@@ -18,11 +18,24 @@ class DBLevelDB : public DB {
 	bool preferRangeQueries() const override { return false; }
 
 private:
-	using pos2d = std::pair<int16_t, int16_t>;
+	struct vec2 {
+		int16_t x, y;
+		constexpr vec2() : x(0), y(0) {}
+		constexpr vec2(int16_t x, int16_t y) : x(x), y(y) {}
+
+		inline bool operator<(const vec2 &p) const
+		{
+			if (x < p.x)
+				return true;
+			if (x > p.x)
+				return false;
+			return y < p.y;
+		}
+	};
 
 	void loadPosCache();
 
 	// indexed by Z, contains all (x,y) position pairs
-	std::unordered_map<int16_t, std::vector<pos2d>> posCache;
-	leveldb::DB *db;
+	std::unordered_map<int16_t, std::vector<vec2>> posCache;
+	leveldb::DB *db = NULL;
 };

src/db-postgresql.cpp

@@ -9,28 +9,82 @@
 
 #define ARRLEN(x) (sizeof(x) / sizeof((x)[0]))
 
+/* PostgreSQLBase */
+
+PostgreSQLBase::~PostgreSQLBase()
+{
+	if (db)
+		PQfinish(db);
+}
+
+void PostgreSQLBase::openDatabase(const char *connect_string)
+{
+	if (db)
+		throw std::logic_error("Database already open");
+
+	db = PQconnectdb(connect_string);
+	if (PQstatus(db) != CONNECTION_OK) {
+		throw std::runtime_error(std::string("PostgreSQL database error: ") +
+			PQerrorMessage(db)
+		);
+	}
+}
+
+PGresult *PostgreSQLBase::checkResults(PGresult *res, bool clear)
+{
+	ExecStatusType statusType = PQresultStatus(res);
+
+	switch (statusType) {
+	case PGRES_COMMAND_OK:
+	case PGRES_TUPLES_OK:
+		break;
+	case PGRES_FATAL_ERROR:
+		throw std::runtime_error(
+			std::string("PostgreSQL database error: ") +
+			PQresultErrorMessage(res)
+		);
+	default:
+		throw std::runtime_error(
+			std::string("Unhandled PostgreSQL result code ") +
+			std::to_string(statusType)
+		);
+	}
+
+	if (clear)
+		PQclear(res);
+	return res;
+}
+
+PGresult *PostgreSQLBase::execPrepared(
+	const char *stmtName, const int paramsNumber,
+	const void **params,
+	const int *paramsLengths, const int *paramsFormats,
+	bool clear)
+{
+	return checkResults(PQexecPrepared(db, stmtName, paramsNumber,
+		(const char* const*) params, paramsLengths, paramsFormats,
+		1 /* binary output */), clear
+	);
+}
+
+/* DBPostgreSQL */
+
 DBPostgreSQL::DBPostgreSQL(const std::string &mapdir)
 {
 	std::ifstream ifs(mapdir + "world.mt");
 	if (!ifs.good())
 		throw std::runtime_error("Failed to read world.mt");
 	std::string connect_string = read_setting("pgsql_connection", ifs);
 	ifs.close();
-	db = PQconnectdb(connect_string.c_str());
 
-	if (PQstatus(db) != CONNECTION_OK) {
-		throw std::runtime_error(std::string(
-			"PostgreSQL database error: ") +
-			PQerrorMessage(db)
-		);
-	}
+	openDatabase(connect_string.c_str());
 
 	prepareStatement(
 		"get_block_pos",
-		"SELECT posX::int4, posY::int4, posZ::int4 FROM blocks WHERE"
+		"SELECT posX::int4, posZ::int4 FROM blocks WHERE"
 		" (posX BETWEEN $1::int4 AND $2::int4) AND"
 		" (posY BETWEEN $3::int4 AND $4::int4) AND"
-		" (posZ BETWEEN $5::int4 AND $6::int4)"
+		" (posZ BETWEEN $5::int4 AND $6::int4) GROUP BY posX, posZ"
 	);
 	prepareStatement(
 		"get_blocks",
@@ -56,11 +110,10 @@ DBPostgreSQL::~DBPostgreSQL()
 	} catch (const std::exception& caught) {
 		std::cerr << "could not finalize: " << caught.what() << std::endl;
 	}
-	PQfinish(db);
 }
 
 
-std::vector<BlockPos> DBPostgreSQL::getBlockPos(BlockPos min, BlockPos max)
+std::vector<BlockPos> DBPostgreSQL::getBlockPosXZ(BlockPos min, BlockPos max)
 {
 	int32_t const x1 = htonl(min.x);
 	int32_t const x2 = htonl(max.x - 1);
@@ -83,11 +136,14 @@ std::vector<BlockPos> DBPostgreSQL::getBlockPos(BlockPos min, BlockPos max)
 	std::vector<BlockPos> positions;
 	positions.reserve(numrows);
 
-	for (int row = 0; row < numrows; ++row)
-		positions.emplace_back(pg_to_blockpos(results, row, 0));
+	BlockPos pos;
+	for (int row = 0; row < numrows; ++row) {
+		pos.x = pg_binary_to_int(results, row, 0);
+		pos.z = pg_binary_to_int(results, row, 1);
+		positions.push_back(pos);
+	}
 
 	PQclear(results);
-
 	return positions;
 }
 
@@ -166,61 +222,8 @@ void DBPostgreSQL::getBlocksByPos(BlockList &blocks,
 	}
 }
 
-
-PGresult *DBPostgreSQL::checkResults(PGresult *res, bool clear)
-{
-	ExecStatusType statusType = PQresultStatus(res);
-
-	switch (statusType) {
-	case PGRES_COMMAND_OK:
-	case PGRES_TUPLES_OK:
-		break;
-	case PGRES_FATAL_ERROR:
-		throw std::runtime_error(
-			std::string("PostgreSQL database error: ") +
-			PQresultErrorMessage(res)
-		);
-	default:
-		throw std::runtime_error(
-			"Unhandled PostgreSQL result code"
-		);
-	}
-
-	if (clear)
-		PQclear(res);
-
-	return res;
-}
-
-void DBPostgreSQL::prepareStatement(const std::string &name, const std::string &sql)
-{
-	checkResults(PQprepare(db, name.c_str(), sql.c_str(), 0, NULL));
-}
-
-PGresult *DBPostgreSQL::execPrepared(
-	const char *stmtName, const int paramsNumber,
-	const void **params,
-	const int *paramsLengths, const int *paramsFormats,
-	bool clear
-)
-{
-	return checkResults(PQexecPrepared(db, stmtName, paramsNumber,
-		(const char* const*) params, paramsLengths, paramsFormats,
-		1 /* binary output */), clear
-	);
-}
-
 int DBPostgreSQL::pg_binary_to_int(PGresult *res, int row, int col)
 {
 	int32_t* raw = reinterpret_cast<int32_t*>(PQgetvalue(res, row, col));
 	return ntohl(*raw);
 }
-
-BlockPos DBPostgreSQL::pg_to_blockpos(PGresult *res, int row, int col)
-{
-	BlockPos result;
-	result.x = pg_binary_to_int(res, row, col);
-	result.y = pg_binary_to_int(res, row, col + 1);
-	result.z = pg_binary_to_int(res, row, col + 2);
-	return result;
-}