Event log

.gitignore

@@ -13,9 +13,11 @@ try.js
 # default location for levelDB data storage in a non-browser env
 MESSAGESTORE
 DATASTORE
+EVENTLOG
 # location for levelDB data storage for non-browser tests
 TEST-DATASTORE
 TEST-MESSAGESTORE
+TEST-EVENTLOG
 
 # default location for index specific levelDB data storage in a non-browser env
 INDEX

README.md

@@ -3,7 +3,7 @@
 # Decentralized Web Node (DWN) SDK
 
 Code Coverage
-![Statements](https://img.shields.io/badge/statements-93.68%25-brightgreen.svg?style=flat) ![Branches](https://img.shields.io/badge/branches-92.29%25-brightgreen.svg?style=flat) ![Functions](https://img.shields.io/badge/functions-90.8%25-brightgreen.svg?style=flat) ![Lines](https://img.shields.io/badge/lines-93.68%25-brightgreen.svg?style=flat)
+![Statements](https://img.shields.io/badge/statements-92.86%25-brightgreen.svg?style=flat) ![Branches](https://img.shields.io/badge/branches-92.34%25-brightgreen.svg?style=flat) ![Functions](https://img.shields.io/badge/functions-90.34%25-brightgreen.svg?style=flat) ![Lines](https://img.shields.io/badge/lines-92.86%25-brightgreen.svg?style=flat)
 
 ## Introduction
 

package.json

@@ -52,6 +52,7 @@
     "@js-temporal/polyfill": "0.4.3",
     "@noble/ed25519": "1.7.1",
     "@noble/secp256k1": "1.7.1",
+    "@scure/base": "1.1.1",
     "@swc/helpers": "0.3.8",
     "@types/eccrypto": "1.1.3",
     "@types/ms": "0.7.31",
@@ -75,6 +76,7 @@
     "multiformats": "11.0.2",
     "randombytes": "2.1.0",
     "readable-stream": "4.3.0",
+    "ulid": "2.3.0",
     "uuid": "8.3.2",
     "varint": "6.0.0"
   },

src/dwn.ts

@@ -1,5 +1,6 @@
 import type { BaseMessage } from './core/types.js';
 import type { DataStore } from './store/data-store.js';
+import type { EventLog } from './event-log/event-log.js';
 import type { MessageStore } from './store/message-store.js';
 import type { MethodHandler } from './interfaces/types.js';
 import type { Readable } from 'readable-stream';
@@ -8,6 +9,7 @@ import type { TenantGate } from './core/tenant-gate.js';
 import { AllowAllTenantGate } from './core/tenant-gate.js';
 import { DataStoreLevel } from './store/data-store-level.js';
 import { DidResolver } from './did/did-resolver.js';
+import { EventLogLevel } from './event-log/event-log-level.js';
 import { MessageReply } from './core/message-reply.js';
 import { MessageStoreLevel } from './store/message-store-level.js';
 import { PermissionsRequestHandler } from './interfaces/permissions/handlers/permissions-request.js';
@@ -24,22 +26,30 @@ export class Dwn {
   private didResolver: DidResolver;
   private messageStore: MessageStore;
   private dataStore: DataStore;
+  private eventLog: EventLog;
   private tenantGate: TenantGate;
 
   private constructor(config: DwnConfig) {
+    this.didResolver = config.didResolver!;
+    this.messageStore = config.messageStore!;
+    this.dataStore = config.dataStore!;
+    this.eventLog = config.eventLog!;
+    this.tenantGate = config.tenantGate!;
     this.didResolver = config.didResolver!;
     this.messageStore = config.messageStore!;
     this.dataStore = config.dataStore!;
     this.tenantGate = config.tenantGate!;
 
     this.methodHandlers = {
       [DwnInterfaceName.Permissions + DwnMethodName.Request] : new PermissionsRequestHandler(this.didResolver, this.messageStore, this.dataStore),
-      [DwnInterfaceName.Protocols + DwnMethodName.Configure] : new ProtocolsConfigureHandler(this.didResolver, this.messageStore, this.dataStore),
-      [DwnInterfaceName.Protocols + DwnMethodName.Query]     : new ProtocolsQueryHandler(this.didResolver, this.messageStore, this.dataStore),
-      [DwnInterfaceName.Records + DwnMethodName.Delete]      : new RecordsDeleteHandler(this.didResolver, this.messageStore, this.dataStore),
-      [DwnInterfaceName.Records + DwnMethodName.Query]       : new RecordsQueryHandler(this.didResolver, this.messageStore, this.dataStore),
-      [DwnInterfaceName.Records + DwnMethodName.Read]        : new RecordsReadHandler(this.didResolver, this.messageStore, this.dataStore),
-      [DwnInterfaceName.Records + DwnMethodName.Write]       : new RecordsWriteHandler(this.didResolver, this.messageStore, this.dataStore),
+      [DwnInterfaceName.Protocols + DwnMethodName.Configure] : new ProtocolsConfigureHandler(
+        this.didResolver, this.messageStore, this.dataStore, this.eventLog),
+      [DwnInterfaceName.Protocols + DwnMethodName.Query] : new ProtocolsQueryHandler(this.didResolver, this.messageStore, this.dataStore),
+      [DwnInterfaceName.Records + DwnMethodName.Delete]  : new RecordsDeleteHandler(
+        this.didResolver, this.messageStore, this.dataStore, this.eventLog),
+      [DwnInterfaceName.Records + DwnMethodName.Query] : new RecordsQueryHandler(this.didResolver, this.messageStore, this.dataStore),
+      [DwnInterfaceName.Records + DwnMethodName.Read]  : new RecordsReadHandler(this.didResolver, this.messageStore, this.dataStore),
+      [DwnInterfaceName.Records + DwnMethodName.Write] : new RecordsWriteHandler(this.didResolver, this.messageStore, this.dataStore, this.eventLog),
     };
   }
 
@@ -52,6 +62,7 @@ export class Dwn {
     config.tenantGate ??= new AllowAllTenantGate();
     config.messageStore ??= new MessageStoreLevel();
     config.dataStore ??= new DataStoreLevel();
+    config.eventLog ??= new EventLogLevel();
 
     const dwn = new Dwn(config);
     await dwn.open();
@@ -62,11 +73,13 @@ export class Dwn {
   private async open(): Promise<void> {
     await this.messageStore.open();
     await this.dataStore.open();
+    await this.eventLog.open();
   }
 
   public async close(): Promise<void> {
     this.messageStore.close();
     this.dataStore.close();
+    this.eventLog.close();
   }
 
   /**
@@ -119,6 +132,10 @@ export class Dwn {
     // @ts-ignore
     await this.dataStore['dump']?.();
     console.groupEnd();
+
+    console.group('eventLog');
+    await this.eventLog['dump']?.();
+    console.groupEnd();
   }
 };
 
@@ -127,4 +144,5 @@ export type DwnConfig = {
   messageStore?: MessageStore;
   dataStore?: DataStore;
   tenantGate?: TenantGate;
+  eventLog?: EventLog
 };

src/event-log/event-log-level.ts

@@ -0,0 +1,95 @@
+import type { ULID } from 'ulid';
+import type { Event, EventLog, GetEventsOptions } from './event-log.js';
+
+import { monotonicFactory } from 'ulid';
+import { createLevelDatabase, LevelWrapper } from '../store/level-wrapper.js';
+
+
+type EventLogLevelConfig = {
+ /**
+   * must be a directory path (relative or absolute) where
+   *  LevelDB will store its files, or in browsers, the name of the
+   * {@link https://developer.mozilla.org/en-US/docs/Web/API/IDBDatabase IDBDatabase} to be opened.
+  */
+  location: string,
+  createLevelDatabase?: typeof createLevelDatabase,
+};
+
+export class EventLogLevel implements EventLog {
+  config: EventLogLevelConfig;
+  db: LevelWrapper<string>;
+  ulid: ULID;
+
+  constructor(config?: EventLogLevelConfig) {
+    this.config = {
+      location: 'EVENTLOG',
+      createLevelDatabase,
+      ...config,
+    };
+
+    this.db = new LevelWrapper<string>({ ...this.config, valueEncoding: 'utf8' });
+    this.ulid = monotonicFactory();
+  }
+
+  async open(): Promise<void> {
+    return this.db.open();
+  }
+
+  async close(): Promise<void> {
+    return this.db.close();
+  }
+
+  async clear(): Promise<void> {
+    return this.db.clear();
+  }
+
+  async append(tenant: string, messageCid: string): Promise<string> {
+    const tenantEventLog = await this.db.partition(tenant);
+
+    const watermark = this.ulid();
+    await tenantEventLog.put(watermark, messageCid);
+
+    return watermark;
+  }
+
+  async getEvents(tenant: string, options?: GetEventsOptions): Promise<Event[]> {
+    const tenantEventLog = await this.db.partition(tenant);
+    const events: Array<Event> = [];
+
+    for await (const [key, value] of tenantEventLog.iterator(options)) {
+      const event = { watermark: key, messageCid: value };
+      events.push(event);
+    }
+
+    return events;
+  }
+
+  async deleteEventsByCid(tenant: string, cids: Array<string>): Promise<number> {
+    if (cids.length === 0) {
+      return 0;
+    }
+
+    const cidSet = new Set(cids);
+    const tenantEventLog = await this.db.partition(tenant);
+    const ops = [];
+
+    let numEventsDeleted = 0;
+
+    for await (const [key, value] of tenantEventLog.iterator()) {
+      if (cidSet.has(value)) {
+        ops.push({ type: 'del', key });
+        numEventsDeleted += 1;
+      }
+    }
+
+    await tenantEventLog.batch(ops);
+
+    return numEventsDeleted;
+  }
+
+  async dump(): Promise<void> {
+    console.group('db');
+    await this.db['dump']?.();
+    console.groupEnd();
+  }
+}

src/event-log/event-log.ts

@@ -0,0 +1,45 @@
+export type Event = {
+  watermark: string,
+  messageCid: string
+};
+
+
+export type GetEventsOptions = {
+  gt: string
+};
+
+export interface EventLog {
+ /**
+  * opens a connection to the underlying store
+  */
+  open(): Promise<void>;
+
+  /**
+   * closes the connection to the underlying store
+   */
+  close(): Promise<void>;
+
+  /**
+   * adds an event to a tenant's event log
+   * @param tenant - the tenant's DID
+   * @param messageCid - the CID of the message
+   * @returns {Promise<string>} watermark
+   */
+  append(tenant: string, messageCid: string): Promise<string>
+
+  /**
+   * retrieves all of a tenant's events that occurred after the watermark provided.
+   * If no watermark is provided, all events for a given tenant will be returned.
+   *
+   * @param tenant
+   * @param watermark
+   */
+  getEvents(tenant: string, options?: GetEventsOptions): Promise<Array<Event>>
+
+  /**
+   * deletes any events that have any of the cids provided
+   * @param tenant
+   * @param cids
+   */
+  deleteEventsByCid(tenant: string, cids: Array<string>): Promise<number>
+}

src/interfaces/protocols/handlers/protocols-configure.ts

@@ -1,3 +1,4 @@
+import type { EventLog } from '../../../event-log/event-log.js';
 import type { MethodHandler } from '../../types.js';
 import type { ProtocolsConfigureMessage } from '../types.js';
 import type { DataStore, DidResolver, MessageStore } from '../../../index.js';
@@ -11,7 +12,7 @@ import { DwnInterfaceName, DwnMethodName, Message } from '../../../core/message.
 
 export class ProtocolsConfigureHandler implements MethodHandler {
 
-  constructor(private didResolver: DidResolver, private messageStore: MessageStore,private dataStore: DataStore) { }
+  constructor(private didResolver: DidResolver, private messageStore: MessageStore, private dataStore: DataStore, private eventLog: EventLog) { }
 
   public async handle({
     tenant,
@@ -57,9 +58,10 @@ export class ProtocolsConfigureHandler implements MethodHandler {
         author,
         ... message.descriptor
       };
+
       // FIXME: indexes, Property 'dataSize' is incompatible with index signature.
       // Type 'number' is not assignable to type 'string'.
-      await StorageController.put(this.messageStore, this.dataStore, tenant, message, indexes as any, dataStream);
+      await StorageController.put(this.messageStore, this.dataStore, this.eventLog, tenant, message, indexes as any, dataStream);
 
       messageReply = new MessageReply({
         status: { code: 202, detail: 'Accepted' }