Merge pull request #20 from hoytech/master

[pull] master from hoytech:master

CHANGES

@@ -1,3 +1,21 @@
+0.9.7
+  * `--fried` support from 1.0.0 was back-ported, allowing export of DBs
+    in a fried format for efficient import by 1.0.0+ relays.
+  * Bugfix: The cron thread was incorrectly removing expireable events prior
+    to their expiry. Reported and fixed by KoalaSat.
+  * A `limitation` entry is now included in the NIP-11 output. This exposes
+    configured relay limits such as max message size. Added by Alex Gleason.
+  * Node info support added: The relay now replies to requests to /nodeinfo/2.1
+    and /.well-known/nodeinfo . Added by Alex Gleason.
+  * NIP-70 support: Events with a "_" tag are considered protected and relays
+    should only allow them to be posted by their author. Since strfry does
+    not yet support AUTH, these events are always rejected. Added by fiatjaf.
+  * NIP-11 icon support added by zappityzap.
+  * Preliminary FreeBSD support added by cosmicpsyop.
+  * Switch import to use WriterPipeline, allowing strfry import be used as a
+    general-purpose non-relay event ingester. To do so, users must ensure that
+    the stdout of their process they pipe into import is line buffered.
+
 0.9.6
   * Bugfix: Sometimes malformed or old-format negentropy messages would throw
     uncaught exceptions which would cause the relay to crash. Now it properly

src/PackedEvent.h

@@ -0,0 +1,91 @@
+#pragma once
+
+#include <string_view>
+
+#include "golpe.h"
+
+
+// PackedEvent (summary of indexable data in a nostr event)
+//   0: id (32)
+//  32: pubkey (32)
+//  64: created_at (8)
+//  72: kind (8)
+//  80: expiration (8)
+//  88: tags[] (variable)
+//
+// each tag:
+//   0: tag char (1)
+//   1: length (1)
+//   2: value (variable)
+
+struct PackedEventView {
+    std::string_view buf;
+
+    PackedEventView(const std::string &str) : buf(std::string_view(str)) {
+        if (buf.size() < 88) throw hoytech::error("PackedEventView too short");
+    }
+
+    PackedEventView(std::string_view sv) : buf(sv) {
+        if (buf.size() < 88) throw hoytech::error("PackedEventView too short");
+    }
+
+    std::string_view id() const {
+        return buf.substr(0, 32);
+    }
+
+    std::string_view pubkey() const {
+        return buf.substr(32, 32);
+    }
+
+    uint64_t created_at() const {
+        return lmdb::from_sv<uint64_t>(buf.substr(64, 8));
+    }
+
+    uint64_t kind() const {
+        return lmdb::from_sv<uint64_t>(buf.substr(72, 8));
+    }
+
+    uint64_t expiration() const {
+        return lmdb::from_sv<uint64_t>(buf.substr(80, 8));
+    }
+
+    void foreachTag(const std::function<bool(char, std::string_view)> &cb) {
+        std::string_view b = buf.substr(88);
+
+        while (b.size()) {
+            bool ret = cb(b[0], b.substr(2, (size_t)b[1]));
+            if (!ret) break;
+            b = b.substr(2 + b[1]);
+        }
+    }
+};
+
+struct PackedEventTagBuilder {
+    std::string buf;
+
+    void add(char tagKey, std::string_view tagVal) {
+        if (tagVal.size() > 255) throw hoytech::error("tagVal too long");
+
+        buf += tagKey;
+        buf += (unsigned char) tagVal.size();
+        buf += tagVal;
+    }
+};
+
+struct PackedEventBuilder {
+    std::string buf;
+
+    PackedEventBuilder(std::string_view id, std::string_view pubkey, uint64_t created_at, uint64_t kind, uint64_t expiration, const PackedEventTagBuilder &tagBuilder) {
+        if (id.size() != 32) throw hoytech::error("unexpected id size");
+        if (pubkey.size() != 32) throw hoytech::error("unexpected pubkey size");
+
+        buf.reserve(88 + tagBuilder.buf.size());
+
+        buf += id;
+        buf += pubkey;
+        buf += lmdb::to_sv<uint64_t>(created_at);
+        buf += lmdb::to_sv<uint64_t>(kind);
+        buf += lmdb::to_sv<uint64_t>(expiration);
+        buf += tagBuilder.buf;
+    }
+};

src/apps/dbutils/cmd_export.cpp

@@ -4,22 +4,50 @@
 #include "golpe.h"
 
 #include "events.h"
+#include "PackedEvent.h"
 
 
 static const char USAGE[] =
 R"(
     Usage:
-      export [--since=<since>] [--until=<until>] [--reverse]
+      export [--since=<since>] [--until=<until>] [--reverse] [--fried]
 )";
 
 
+std::string getPackedEventBuf(lmdb::txn &txn, uint64_t levId) {
+    auto ev = lookupEventByLevId(txn, levId);
+    auto *flat = ev.flat_nested();
+
+    PackedEventTagBuilder tagBuilder;
+
+    // Unfortunately we lose the original ordering...
+
+    for (const auto &tagPair : *(flat->tagsGeneral())) {
+        auto tagName = (char)tagPair->key();
+        auto tagVal = sv(tagPair->val());
+        tagBuilder.add(tagName, tagVal);
+    }
+
+    for (const auto &tagPair : *(flat->tagsFixed32())) {
+        auto tagName = (char)tagPair->key();
+        auto tagVal = sv(tagPair->val());
+        tagBuilder.add(tagName, tagVal);
+    }
+
+    PackedEventBuilder builder(sv(flat->id()), sv(flat->pubkey()), flat->created_at(), flat->kind(), flat->expiration(), tagBuilder);
+
+    return std::move(builder.buf);
+}
+
+
 void cmd_export(const std::vector<std::string> &subArgs) {
     std::map<std::string, docopt::value> args = docopt::docopt(USAGE, subArgs, true, "");
 
     uint64_t since = 0, until = MAX_U64;
     if (args["--since"]) since = args["--since"].asLong();
     if (args["--until"]) until = args["--until"].asLong();
     bool reverse = args["--reverse"].asBool();
+    bool fried = args["--fried"].asBool();
 
     Decompressor decomp;
 
@@ -34,6 +62,8 @@ void cmd_export(const std::vector<std::string> &subArgs) {
 
     exitOnSigPipe();
 
+    std::string o;
+
     env.generic_foreachFull(txn, env.dbi_Event__created_at, lmdb::to_sv<uint64_t>(start), lmdb::to_sv<uint64_t>(startDup), [&](auto k, auto v) {
         if (reverse) {
             if (lmdb::from_sv<uint64_t>(k) < since) return false;
@@ -42,8 +72,23 @@ void cmd_export(const std::vector<std::string> &subArgs) {
         }
 
         auto levId = lmdb::from_sv<uint64_t>(v);
+        std::string_view json = getEventJson(txn, decomp, levId);
+
+        if (fried) {
+            std::string friedBuf = getPackedEventBuf(txn, levId);
 
-        std::cout << getEventJson(txn, decomp, levId) << "\n";
+            o.clear();
+            o.reserve(json.size() + friedBuf.size() * 2 + 100);
+            o = json;
+            o.resize(o.size() - 1);
+            o += ",\"fried\":\"";
+            o += to_hex(friedBuf);
+            o += "\"}\n";
+
+            std::cout << o;
+        } else {
+            std::cout << json << "\n";
+        }
 
         return true;
     }, reverse);

src/apps/relay/RelayCron.cpp

@@ -85,7 +85,8 @@ void RelayServer::runCron() {
                 auto expiration = lmdb::from_sv<uint64_t>(k);
                 auto levId =  lmdb::from_sv<uint64_t>(v);
 
-                if (levId == mostRecent) return true;
+                if (expiration > now) return false;
+                if (levId == mostRecent) return true; // don't delete because it could cause levId re-use
 
                 if (expiration == 1) { // Ephemeral event
                     auto view = env.lookup_Event(txn, levId);
@@ -101,7 +102,7 @@ void RelayServer::runCron() {
                     expiredLevIds.emplace_back(levId);
                 }
 
-                return expiration <= now;
+                return true;
             });
         }