diff --git a/deno.lock b/deno.lock index 1cb0ec2..0e5c9a7 100644 --- a/deno.lock +++ b/deno.lock @@ -181,6 +181,8 @@ "https://deno.land/x/dnt@0.39.0/lib/utils.ts": "224f15f33e7226a2fd991e438d0291d7ed8c7889807efa2e1ecb67d2d1db6720", "https://deno.land/x/dnt@0.39.0/mod.ts": "9df36a862161d9eb376472b699f6cb08ba0ad1704e0826fbe13be766bd3c01da", "https://deno.land/x/dnt@0.39.0/transform.ts": "f68743a14cf9bf53bfc9c81073871d69d447a7f9e3453e0447ca2fb78926bb1d", + "https://deno.land/x/sleep@v1.3.0/mod.ts": "e9955ecd3228a000e29d46726cd6ab14b65cf83904e9b365f3a8d64ec61c1af3", + "https://deno.land/x/sleep@v1.3.0/sleep.ts": "b6abaca093b094b0c2bba94f287b19a60946a8d15764d168f83fcf555f5bb59e", "https://deno.land/x/ts_morph@20.0.0/bootstrap/mod.ts": "b53aad517f106c4079971fcd4a81ab79fadc40b50061a3ab2b741a09119d51e9", "https://deno.land/x/ts_morph@20.0.0/bootstrap/ts_morph_bootstrap.js": "6645ac03c5e6687dfa8c78109dc5df0250b811ecb3aea2d97c504c35e8401c06", "https://deno.land/x/ts_morph@20.0.0/common/DenoRuntime.ts": "6a7180f0c6e90dcf23ccffc86aa8271c20b1c4f34c570588d08a45880b7e172d", diff --git a/package.json b/package.json index be5be27..7cf8263 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@ucla-irl/ndnts-aux", - "version": "1.1.0", + "version": "1.1.1", "description": "NDNts Auxiliary Package for Web and Deno", "scripts": { "test": "deno test --no-check", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c389139..3dc166d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -755,7 +755,7 @@ packages: '@ndnts-nightly.ndn.today/endpoint.tgz': resolution: {tarball: https://ndnts-nightly.ndn.today/endpoint.tgz} name: '@ndn/endpoint' - version: 0.0.20240208-nightly-2a9e3be + version: 0.0.20240209-nightly-9b234f4 dependencies: '@ndn/fw': '@ndnts-nightly.ndn.today/fw.tgz' '@ndn/packet': '@ndnts-nightly.ndn.today/packet.tgz' @@ -768,7 +768,7 @@ packages: '@ndnts-nightly.ndn.today/fw.tgz': resolution: {tarball: https://ndnts-nightly.ndn.today/fw.tgz} name: '@ndn/fw' - version: 0.0.20240208-nightly-2a9e3be + version: 0.0.20240209-nightly-9b234f4 dependencies: '@ndn/packet': '@ndnts-nightly.ndn.today/packet.tgz' '@ndn/util': '@ndnts-nightly.ndn.today/util.tgz' @@ -785,7 +785,7 @@ packages: '@ndnts-nightly.ndn.today/keychain.tgz': resolution: {tarball: https://ndnts-nightly.ndn.today/keychain.tgz} name: '@ndn/keychain' - version: 0.0.20240208-nightly-2a9e3be + version: 0.0.20240209-nightly-9b234f4 dependencies: '@ndn/naming-convention2': '@ndnts-nightly.ndn.today/naming-convention2.tgz' '@ndn/packet': '@ndnts-nightly.ndn.today/packet.tgz' @@ -803,7 +803,7 @@ packages: '@ndnts-nightly.ndn.today/l3face.tgz': resolution: {tarball: https://ndnts-nightly.ndn.today/l3face.tgz} name: '@ndn/l3face' - version: 0.0.20240208-nightly-2a9e3be + version: 0.0.20240209-nightly-9b234f4 dependencies: '@ndn/fw': '@ndnts-nightly.ndn.today/fw.tgz' '@ndn/lp': '@ndnts-nightly.ndn.today/lp.tgz' @@ -823,7 +823,7 @@ packages: '@ndnts-nightly.ndn.today/lp.tgz': resolution: {tarball: https://ndnts-nightly.ndn.today/lp.tgz} name: '@ndn/lp' - version: 0.0.20240208-nightly-2a9e3be + version: 0.0.20240209-nightly-9b234f4 dependencies: '@ndn/packet': '@ndnts-nightly.ndn.today/packet.tgz' '@ndn/tlv': '@ndnts-nightly.ndn.today/tlv.tgz' @@ -835,7 +835,7 @@ packages: '@ndnts-nightly.ndn.today/naming-convention2.tgz': resolution: {tarball: https://ndnts-nightly.ndn.today/naming-convention2.tgz} name: '@ndn/naming-convention2' - version: 0.0.20240208-nightly-2a9e3be + version: 0.0.20240209-nightly-9b234f4 dependencies: '@ndn/packet': '@ndnts-nightly.ndn.today/packet.tgz' '@ndn/tlv': '@ndnts-nightly.ndn.today/tlv.tgz' @@ -846,7 +846,7 @@ packages: '@ndnts-nightly.ndn.today/ndncert.tgz': resolution: {tarball: https://ndnts-nightly.ndn.today/ndncert.tgz} name: '@ndn/ndncert' - version: 0.0.20240208-nightly-2a9e3be + version: 0.0.20240209-nightly-9b234f4 dependencies: '@ndn/endpoint': '@ndnts-nightly.ndn.today/endpoint.tgz' '@ndn/keychain': '@ndnts-nightly.ndn.today/keychain.tgz' @@ -870,7 +870,7 @@ packages: '@ndnts-nightly.ndn.today/ndnsec.tgz': resolution: {tarball: https://ndnts-nightly.ndn.today/ndnsec.tgz} name: '@ndn/ndnsec' - version: 0.0.20240208-nightly-2a9e3be + version: 0.0.20240209-nightly-9b234f4 dependencies: '@ndn/keychain': '@ndnts-nightly.ndn.today/keychain.tgz' '@ndn/packet': '@ndnts-nightly.ndn.today/packet.tgz' @@ -885,7 +885,7 @@ packages: '@ndnts-nightly.ndn.today/nfdmgmt.tgz': resolution: {tarball: https://ndnts-nightly.ndn.today/nfdmgmt.tgz} name: '@ndn/nfdmgmt' - version: 0.0.20240208-nightly-2a9e3be + version: 0.0.20240209-nightly-9b234f4 dependencies: '@ndn/endpoint': '@ndnts-nightly.ndn.today/endpoint.tgz' '@ndn/fw': '@ndnts-nightly.ndn.today/fw.tgz' @@ -901,7 +901,7 @@ packages: '@ndnts-nightly.ndn.today/node-transport.tgz': resolution: {tarball: https://ndnts-nightly.ndn.today/node-transport.tgz} name: '@ndn/node-transport' - version: 0.0.20240208-nightly-2a9e3be + version: 0.0.20240209-nightly-9b234f4 dependencies: '@ndn/l3face': '@ndnts-nightly.ndn.today/l3face.tgz' event-iterator: 2.0.0 @@ -915,7 +915,7 @@ packages: '@ndnts-nightly.ndn.today/packet.tgz': resolution: {tarball: https://ndnts-nightly.ndn.today/packet.tgz} name: '@ndn/packet' - version: 0.0.20240208-nightly-2a9e3be + version: 0.0.20240209-nightly-9b234f4 dependencies: '@ndn/tlv': '@ndnts-nightly.ndn.today/tlv.tgz' '@ndn/util': '@ndnts-nightly.ndn.today/util.tgz' @@ -928,7 +928,7 @@ packages: '@ndnts-nightly.ndn.today/psync.tgz': resolution: {tarball: https://ndnts-nightly.ndn.today/psync.tgz} name: '@ndn/psync' - version: 0.0.20240208-nightly-2a9e3be + version: 0.0.20240209-nightly-9b234f4 dependencies: '@ndn/endpoint': '@ndnts-nightly.ndn.today/endpoint.tgz' '@ndn/naming-convention2': '@ndnts-nightly.ndn.today/naming-convention2.tgz' @@ -951,7 +951,7 @@ packages: '@ndnts-nightly.ndn.today/rdr.tgz': resolution: {tarball: https://ndnts-nightly.ndn.today/rdr.tgz} name: '@ndn/rdr' - version: 0.0.20240208-nightly-2a9e3be + version: 0.0.20240209-nightly-9b234f4 dependencies: '@ndn/endpoint': '@ndnts-nightly.ndn.today/endpoint.tgz' '@ndn/naming-convention2': '@ndnts-nightly.ndn.today/naming-convention2.tgz' @@ -963,7 +963,7 @@ packages: '@ndnts-nightly.ndn.today/repo-api.tgz': resolution: {tarball: https://ndnts-nightly.ndn.today/repo-api.tgz} name: '@ndn/repo-api' - version: 0.0.20240208-nightly-2a9e3be + version: 0.0.20240209-nightly-9b234f4 dependencies: '@ndn/l3face': '@ndnts-nightly.ndn.today/l3face.tgz' '@ndn/naming-convention2': '@ndnts-nightly.ndn.today/naming-convention2.tgz' @@ -984,7 +984,7 @@ packages: '@ndnts-nightly.ndn.today/segmented-object.tgz': resolution: {tarball: https://ndnts-nightly.ndn.today/segmented-object.tgz} name: '@ndn/segmented-object' - version: 0.0.20240208-nightly-2a9e3be + version: 0.0.20240209-nightly-9b234f4 dependencies: '@ndn/endpoint': '@ndnts-nightly.ndn.today/endpoint.tgz' '@ndn/fw': '@ndnts-nightly.ndn.today/fw.tgz' @@ -1005,7 +1005,7 @@ packages: '@ndnts-nightly.ndn.today/svs.tgz': resolution: {tarball: https://ndnts-nightly.ndn.today/svs.tgz} name: '@ndn/svs' - version: 0.0.20240208-nightly-2a9e3be + version: 0.0.20240209-nightly-9b234f4 dependencies: '@ndn/endpoint': '@ndnts-nightly.ndn.today/endpoint.tgz' '@ndn/naming-convention2': '@ndnts-nightly.ndn.today/naming-convention2.tgz' @@ -1024,7 +1024,7 @@ packages: '@ndnts-nightly.ndn.today/sync-api.tgz': resolution: {tarball: https://ndnts-nightly.ndn.today/sync-api.tgz} name: '@ndn/sync-api' - version: 0.0.20240208-nightly-2a9e3be + version: 0.0.20240209-nightly-9b234f4 dependencies: '@ndn/packet': '@ndnts-nightly.ndn.today/packet.tgz' '@ndn/util': '@ndnts-nightly.ndn.today/util.tgz' @@ -1035,7 +1035,7 @@ packages: '@ndnts-nightly.ndn.today/sync.tgz': resolution: {tarball: https://ndnts-nightly.ndn.today/sync.tgz} name: '@ndn/sync' - version: 0.0.20240208-nightly-2a9e3be + version: 0.0.20240209-nightly-9b234f4 dependencies: '@ndn/endpoint': '@ndnts-nightly.ndn.today/endpoint.tgz' '@ndn/naming-convention2': '@ndnts-nightly.ndn.today/naming-convention2.tgz' @@ -1044,6 +1044,7 @@ packages: '@ndn/repo-api': '@ndnts-nightly.ndn.today/repo-api.tgz' '@ndn/segmented-object': '@ndnts-nightly.ndn.today/segmented-object.tgz' '@ndn/svs': '@ndnts-nightly.ndn.today/svs.tgz' + '@ndn/sync-api': '@ndnts-nightly.ndn.today/sync-api.tgz' '@ndn/tlv': '@ndnts-nightly.ndn.today/tlv.tgz' '@ndn/util': '@ndnts-nightly.ndn.today/util.tgz' '@yoursunny/psync-bloom': github.com/yoursunny/PSyncBloom-wasm/d4f18babd44f142021046d5a76ae92efbc5a49d3 @@ -1061,7 +1062,7 @@ packages: '@ndnts-nightly.ndn.today/tlv.tgz': resolution: {tarball: https://ndnts-nightly.ndn.today/tlv.tgz} name: '@ndn/tlv' - version: 0.0.20240208-nightly-2a9e3be + version: 0.0.20240209-nightly-9b234f4 dependencies: '@ndn/util': '@ndnts-nightly.ndn.today/util.tgz' mnemonist: 0.39.8 @@ -1072,7 +1073,7 @@ packages: '@ndnts-nightly.ndn.today/util.tgz': resolution: {tarball: https://ndnts-nightly.ndn.today/util.tgz} name: '@ndn/util' - version: 0.0.20240208-nightly-2a9e3be + version: 0.0.20240209-nightly-9b234f4 engines: {node: ^18.18.0 || ^20.10.0 || ^21.0.0} dependencies: '@shigen/polyfill-symbol-dispose': 1.0.1 @@ -1087,7 +1088,7 @@ packages: '@ndnts-nightly.ndn.today/ws-transport.tgz': resolution: {tarball: https://ndnts-nightly.ndn.today/ws-transport.tgz} name: '@ndn/ws-transport' - version: 0.0.20240208-nightly-2a9e3be + version: 0.0.20240209-nightly-9b234f4 dependencies: '@ndn/l3face': '@ndnts-nightly.ndn.today/l3face.tgz' '@ndn/util': '@ndnts-nightly.ndn.today/util.tgz' diff --git a/src/adaptors/bundler.test.ts b/src/adaptors/bundler.test.ts new file mode 100644 index 0000000..9a9d001 --- /dev/null +++ b/src/adaptors/bundler.test.ts @@ -0,0 +1,76 @@ +import { assert as assertMod, sleep } from '../dep.ts'; +import { Bundler } from './bundler.ts'; + +const { assertEquals } = assertMod; + +export const b = ([value]: TemplateStringsArray) => new TextEncoder().encode(value); + +const joinMerger = (updates: Uint8Array[]) => { + const totLength = updates.reduce((acc, cur) => acc + cur.length, 0); + const ret = new Uint8Array(totLength); + updates.reduce((offset, cur) => { + ret.set(cur, offset); + return offset + cur.length; + }, 0); + return ret; +}; + +Deno.test('Bundler.1', async () => { + const result: Array = []; + const bundler = new Bundler( + joinMerger, + (update) => { + result.push(update); + return Promise.resolve(); + }, + { + thresholdSize: 100, + delayMs: 100, + }, + ); + + await bundler.produce(b`Hello `); + await bundler.produce(b`World.`); + await sleep(0.1); + assertEquals(result, [b`Hello World.`]); +}); + +Deno.test('Bundler.2', async () => { + const result: Array = []; + const bundler = new Bundler( + joinMerger, + (update) => { + result.push(update); + return Promise.resolve(); + }, + { + thresholdSize: 3, + delayMs: 100, + }, + ); + + await bundler.produce(b`Hello `); + await bundler.produce(b`World.`); + assertEquals(result, [b`Hello `, b`World.`]); +}); + +Deno.test('Bundler.3', async () => { + const result: Array = []; + const bundler = new Bundler( + joinMerger, + (update) => { + result.push(update); + return Promise.resolve(); + }, + { + thresholdSize: 10, + delayMs: 100, + }, + ); + + await bundler.produce(b`Hello `); + await bundler.produce(b`World,`); + await bundler.produce(b`Hello `); + await bundler.produce(b`World.`); + assertEquals(result, [b`Hello World,`, b`Hello World.`]); +}); diff --git a/src/adaptors/bundler.ts b/src/adaptors/bundler.ts new file mode 100644 index 0000000..9348cef --- /dev/null +++ b/src/adaptors/bundler.ts @@ -0,0 +1,49 @@ +export type BundlerOpts = { + thresholdSize?: number; + delayMs?: number; +}; + +export class Bundler { + protected _bundle: Array = []; + protected _timerId: number | undefined; + protected _cumulativeSize = 0; + protected _opts: Required; + + constructor( + public readonly merge: (updates: Array) => Uint8Array, + public readonly emit: (update: Uint8Array) => Promise, + opts: { + thresholdSize?: number; + delayMs?: number; + } = {}, + ) { + this._opts = { + thresholdSize: 3000, + delayMs: 200, + ...opts, + }; + } + + public async produce(update: Uint8Array) { + this._bundle.push(update); + this._cumulativeSize += update.byteLength; + if (this._cumulativeSize > this._opts.thresholdSize) { + // Emit immediately + await this.issue(); + } else if (!this._timerId) { + // Schedule emit + this._timerId = setTimeout(() => this.issue(), this._opts.delayMs); + } + } + + public async issue() { + if (this._timerId) { + clearTimeout(this._timerId); + } + this._timerId = undefined; + const output = this._bundle.length === 1 ? this._bundle[0] : this.merge(this._bundle); + this._bundle = []; + this._cumulativeSize = 0; + await this.emit(output); + } +} diff --git a/src/adaptors/yjs-ndn-adaptor.ts b/src/adaptors/yjs-ndn-adaptor.ts index d9d2dfb..90c6123 100644 --- a/src/adaptors/yjs-ndn-adaptor.ts +++ b/src/adaptors/yjs-ndn-adaptor.ts @@ -1,6 +1,7 @@ import { SyncAgent } from '../sync-agent/mod.ts'; import * as Y from 'yjs'; import { Awareness } from 'y-protocols/awareness.js'; +import { Bundler } from './bundler.ts'; /** * NDN SVS Provider for Yjs. Wraps update into `SyncAgent`'s `update` channel. @@ -23,13 +24,26 @@ export class NdnSvsAdaptor { return this.#awareness; } + #bundler: Bundler | undefined; + constructor( public syncAgent: SyncAgent, public readonly doc: Y.Doc, public readonly topic: string, + useBundler: boolean = false, ) { syncAgent.register('update', topic, (content) => this.handleSyncUpdate(content)); doc.on('update', this.callback); + if (useBundler) { + this.#bundler = new Bundler( + Y.mergeUpdates, + (content) => this.syncAgent.publishUpdate(this.topic, content), + { + thresholdSize: 3000, + delayMs: 200, + }, + ); + } } public bindAwareness(subDoc: Y.Doc, docId: string) { @@ -78,7 +92,11 @@ export class NdnSvsAdaptor { } private async produce(content: Uint8Array) { - await this.syncAgent.publishUpdate(this.topic, content); + if (this.#bundler) { + await this.#bundler.produce(content); + } else { + await this.syncAgent.publishUpdate(this.topic, content); + } } public handleSyncUpdate(content: Uint8Array) { diff --git a/src/dep.ts b/src/dep.ts index f50db33..788c61b 100644 --- a/src/dep.ts +++ b/src/dep.ts @@ -5,3 +5,4 @@ */ export * as assert from 'https://deno.land/std@0.212.0/assert/mod.ts'; export * as hex from 'https://deno.land/std@0.212.0/encoding/hex.ts'; +export { sleep } from 'https://deno.land/x/sleep@v1.3.0/mod.ts'; diff --git a/src/workspace/workspace.ts b/src/workspace/workspace.ts index 09f2d92..afafb61 100644 --- a/src/workspace/workspace.ts +++ b/src/workspace/workspace.ts @@ -26,6 +26,7 @@ export class Workspace implements AsyncDisposable { verifier: Verifier; onReset?: () => void; createNewDoc?: () => Promise; + useBundler?: boolean; }) { // Always init a new one, and then load. if (opts.createNewDoc) { @@ -50,6 +51,7 @@ export class Workspace implements AsyncDisposable { syncAgent, opts.rootDoc, 'doc', + opts.useBundler ?? false, ); const yjsSnapshotMgr = new YjsStateManager( () => encodeSyncState(syncAgent!.getUpdateSyncSV()),