Skip to content

Commit

Permalink
Remove cached MMDB files (#4235)
Browse files Browse the repository at this point in the history 
This makes sure that our cache directory doesn't grow quite as quickly.
  • Loading branch information
@dominiklohmann
dominiklohmann committed May 22, 2024
2 parents 6f90fa2 + 888f47e commit 870928a
Show file tree
Hide file tree
Showing 2 changed files with 38 additions and 31 deletions.
3 changes: 3 additions & 0 deletions .clang-tidy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ Checks: >
-abseil-*,
-altera-*,
-bugprone-lambda-function-name,
-bugprone-easily-swappable-parameters,
-bugprone-switch-missing-default-case,
-cert-dcl16-c,
-cert-err58-cpp,
-clang-analyzer-alpha*,
Expand All @@ -19,6 +21,7 @@ Checks: >
-hicpp-*,
-llvm-header-guard,
-llvmlibc-*,
-misc-no-recursion,
-modernize-use-nodiscard,
-modernize-use-ranges,
-readability-braces-around-statements,
Expand Down
66 changes: 35 additions & 31 deletions libtenzir/builtins/contexts/geoip.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,7 @@

#include <fmt/format.h>

#include <chrono>
#include <cstdint>
#include <cstdio>
#include <maxminddb.h>
#include <memory>
#include <string>
Expand All @@ -41,6 +39,7 @@ struct mmdb_deleter final {
auto operator()(MMDB_s* ptr) noexcept -> void {
if (ptr) {
MMDB_close(ptr);
// NOLINTNEXTLINE(cppcoreguidelines-owning-memory)
delete ptr;
}
}
Expand All @@ -55,9 +54,11 @@ auto make_mmdb(const std::string& path) -> caf::expected<mmdb_ptr> {
.note("failed to find path `{}`", path)
.to_error();
}
auto ptr = new MMDB_s;
// NOLINTNEXTLINE(cppcoreguidelines-owning-memory)
auto* ptr = new MMDB_s;
const auto status = MMDB_open(path.c_str(), MMDB_MODE_MMAP, ptr);
if (status != MMDB_SUCCESS) {
// NOLINTNEXTLINE(cppcoreguidelines-owning-memory)
delete ptr;
return diagnostic::error("{}", MMDB_strerror(status))
.note("failed to open MaxMind database at `{}`", path)
Expand Down Expand Up @@ -117,8 +118,8 @@ class ctx final : public virtual context {
case MMDB_DATA_TYPE_MAP: {
auto size = entry_data_list->entry_data.data_size;
auto sub_r = record{};
for (entry_data_list = entry_data_list->next; size && entry_data_list;
size--) {
for (entry_data_list = entry_data_list->next;
size > 0 && entry_data_list; size--) {
if (MMDB_DATA_TYPE_UTF8_STRING != entry_data_list->entry_data.type) {
*status = MMDB_INVALID_DATA_ERROR;
return entry_data_list;
Expand All @@ -140,8 +141,8 @@ class ctx final : public virtual context {
case MMDB_DATA_TYPE_ARRAY: {
auto sub_l = list{};
auto size = entry_data_list->entry_data.data_size;
for (entry_data_list = entry_data_list->next; size && entry_data_list;
size--) {
for (entry_data_list = entry_data_list->next;
size > 0 && entry_data_list; size--) {
entry_data_list
= entry_data_list_to_list(entry_data_list, status, sub_l);
if (*status != MMDB_SUCCESS) {
Expand Down Expand Up @@ -207,16 +208,16 @@ class ctx final : public virtual context {
return entry_data_list;
}

auto
entry_data_list_to_record(MMDB_entry_data_list_s* entry_data_list,
int* status, record& r, std::string key = {}) const
auto entry_data_list_to_record(MMDB_entry_data_list_s* entry_data_list,
int* status, record& r,
const std::string& key = {}) const
-> MMDB_entry_data_list_s* {
switch (entry_data_list->entry_data.type) {
case MMDB_DATA_TYPE_MAP: {
auto size = entry_data_list->entry_data.data_size;

for (entry_data_list = entry_data_list->next; size && entry_data_list;
size--) {
for (entry_data_list = entry_data_list->next;
size > 0 && entry_data_list; size--) {
if (MMDB_DATA_TYPE_UTF8_STRING != entry_data_list->entry_data.type) {
*status = MMDB_INVALID_DATA_ERROR;
return entry_data_list;
Expand Down Expand Up @@ -245,8 +246,8 @@ class ctx final : public virtual context {
auto l = list{};
auto size = entry_data_list->entry_data.data_size;
auto sub_r = record{};
for (entry_data_list = entry_data_list->next; size && entry_data_list;
size--) {
for (entry_data_list = entry_data_list->next;
size > 0 && entry_data_list; size--) {
entry_data_list = entry_data_list_to_list(entry_data_list, status, l);
if (*status != MMDB_SUCCESS) {
return entry_data_list;
Expand Down Expand Up @@ -395,31 +396,31 @@ class ctx final : public virtual context {
}

auto dump_recurse(uint64_t node_number, uint8_t type, MMDB_entry_s* entry,
current_dump& current_dump) -> generator<table_slice> {
if (current_dump.visited.contains(node_number)) {
current_dump* current_dump) -> generator<table_slice> {
if (current_dump->visited.contains(node_number)) {
co_return;
}
current_dump.visited.emplace(node_number);
current_dump->visited.emplace(node_number);
switch (type) {
case MMDB_RECORD_TYPE_SEARCH_NODE: {
MMDB_search_node_s search_node{};
current_dump.status
current_dump->status
= MMDB_read_node(mmdb_.get(), node_number, &search_node);
if (current_dump.status != MMDB_SUCCESS) {
if (current_dump->status != MMDB_SUCCESS) {
break;
}
for (auto&& x :
dump_recurse(search_node.left_record, search_node.left_record_type,
&search_node.left_record_entry, current_dump)) {
if (current_dump.status != MMDB_SUCCESS) {
if (current_dump->status != MMDB_SUCCESS) {
break;
}
co_yield x;
}
for (auto&& x : dump_recurse(
search_node.right_record, search_node.right_record_type,
&search_node.right_record_entry, current_dump)) {
if (current_dump.status != MMDB_SUCCESS) {
if (current_dump->status != MMDB_SUCCESS) {
break;
}
co_yield x;
Expand All @@ -433,8 +434,9 @@ class ctx final : public virtual context {
case MMDB_RECORD_TYPE_DATA: {
TENZIR_ASSERT(entry != nullptr);
MMDB_entry_data_list_s* entry_data_list = nullptr;
current_dump.status = MMDB_get_entry_data_list(entry, &entry_data_list);
if (current_dump.status != MMDB_SUCCESS) {
current_dump->status
= MMDB_get_entry_data_list(entry, &entry_data_list);
if (current_dump->status != MMDB_SUCCESS) {
break;
}
auto free_entry_data_list = caf::detail::make_scope_guard([&] {
Expand All @@ -443,14 +445,15 @@ class ctx final : public virtual context {
}
});
auto output = list{};
entry_data_list_to_list(entry_data_list, &current_dump.status, output);
if (current_dump.status != MMDB_SUCCESS) {
entry_data_list_to_list(entry_data_list, &current_dump->status, output);
if (current_dump->status != MMDB_SUCCESS) {
break;
}
for (auto& x : output) {
current_dump.builder.data(x);
if (current_dump.builder.length() >= context::dump_batch_size_limit) {
for (auto&& slice : current_dump.builder.finish_as_table_slice(
current_dump->builder.data(x);
if (current_dump->builder.length()
>= context::dump_batch_size_limit) {
for (auto&& slice : current_dump->builder.finish_as_table_slice(
fmt::format("tenzir.{}.info", context_type()))) {
co_yield std::move(slice);
}
Expand All @@ -459,7 +462,7 @@ class ctx final : public virtual context {
break;
}
case MMDB_RECORD_TYPE_INVALID: {
current_dump.status = MMDB_INVALID_DATA_ERROR;
current_dump->status = MMDB_INVALID_DATA_ERROR;
break;
}
}
Expand All @@ -468,8 +471,8 @@ class ctx final : public virtual context {
auto dump() -> generator<table_slice> override {
TENZIR_ASSERT(mmdb_);
current_dump current_dump;
for (auto&& slice :
dump_recurse(0, MMDB_RECORD_TYPE_SEARCH_NODE, nullptr, current_dump)) {
for (auto&& slice : dump_recurse(0, MMDB_RECORD_TYPE_SEARCH_NODE, nullptr,
&current_dump)) {
co_yield slice;
}
// Dump all remaining entries that did not reach the size limit.
Expand Down Expand Up @@ -598,6 +601,7 @@ struct v2_loader : public context_loader {
fmt::format("failed close the temp file: {}",
detail::describe_errno()));
}
std::filesystem::remove(temp_file_name);
return std::make_unique<ctx>(std::move(*mmdb),
std::move(mapped_mmdb.value()));
}
Expand Down

0 comments on commit 870928a

Please sign in to comment.