diff --git a/package.json b/package.json index d169a04..ede82ba 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "url": "https://github.com/UCLA-IRL/ndnts-aux/issues" }, "dependencies": { + "@ndn/sync-api": "https://ndnts-nightly.ndn.today/sync-api.tgz", "event-iterator": "^2.0.0", "eventemitter3": "^5.0.1", "jose": "^5.2.2", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6e6a3c0..110177a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -5,6 +5,9 @@ settings: excludeLinksFromLockfile: false dependencies: + '@ndn/sync-api': + specifier: https://ndnts-nightly.ndn.today/sync-api.tgz + version: '@ndnts-nightly.ndn.today/sync-api.tgz' event-iterator: specifier: ^2.0.0 version: 2.0.0 @@ -111,7 +114,6 @@ packages: /@shigen/polyfill-symbol-dispose@1.0.1: resolution: {integrity: sha512-g8OSf0SiNNRojr+2PJT8j1yj+TPt7mDCYbjYWQMVOeFWcrDC9NEM87tElI/e6REpykSsKckoup8v0tcquQ6K5w==} - dev: true /@types/imap@0.8.40: resolution: {integrity: sha512-kWFwOc88CGwWZlHqCnZiceS6EralsAHdjpQyk1+fIA875NQdIHvLpdD5NU3Pi1yZ8FKFdOF81UDNAo8/XS6HiQ==} @@ -121,7 +123,6 @@ packages: /@types/minimalistic-assert@1.0.3: resolution: {integrity: sha512-Ku87cam4YxiXcEpeUemo+vO8QWGQ7U2CwEEcLcYFhxG8b4CK8gWxSX/oWjePWKwqPaWWxxVqXAdAjGdlJtVzDA==} - dev: true /@types/node@20.11.20: resolution: {integrity: sha512-7/rR21OS+fq8IyHTgtLkDK949uzsa6n8BkziAKtPVpugIkO6D+/ooXMvzXxDnZrmtXVfjb1bKQafYpb8s89LOg==} @@ -204,7 +205,6 @@ packages: /buffer-compare@1.1.1: resolution: {integrity: sha512-O6NvNiHZMd3mlIeMDjP6t/gPG75OqGPeiRZXoMQZJ6iy9GofCls4Ijs5YkPZZwoysizLiedhticmdyx/GyHghA==} - dev: true /bufferutil@4.0.8: resolution: {integrity: sha512-4T53u4PdgsXqKaIctwF8ifXlRTTmEPJ8iEPWFdGZvcf7sbwYo6FKFEX9eNNAnzFZ7EzJAQ3CJeOtCRA4rDp7Pw==} @@ -385,6 +385,11 @@ packages: engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} dev: true + /is-stream@4.0.1: + resolution: {integrity: sha512-Dnz92NInDqYckGEUJv689RbRiTSEHCQ7wOVeALbkOz999YpqT46yMRIGtSNl2iCL1waAZSx40+h59NV/EwzV/A==} + engines: {node: '>=18'} + dev: true + /isarray@0.0.1: resolution: {integrity: sha512-D2S+3GLxWH+uhrNEcoh/fnmYeP8E8/zHl644d/jdA0g2uyXvy3sb0qxotE+ne0LtccHknQzWwZEzhak7oJ0COQ==} dev: true @@ -499,13 +504,11 @@ packages: /minimalistic-assert@1.0.1: resolution: {integrity: sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A==} - dev: true /mnemonist@0.39.8: resolution: {integrity: sha512-vyWo2K3fjrUw8YeeZ1zF0fy6Mu59RHokURlld8ymdUPjMlD9EC9ov1/YPqTgqRvUN9nTr3Gqfz29LYAmu0PHPQ==} dependencies: obliterator: 2.0.4 - dev: true /node-gyp-build@4.8.0: resolution: {integrity: sha512-u6fs2AEUljNho3EYTJNBfImO5QTo/J/1Etd+NVdCj7qWKUSN/bSLkZwhDv7I+w/MSC6qJ4cknepkAYykDdK8og==} @@ -532,7 +535,6 @@ packages: /obliterator@2.0.4: resolution: {integrity: sha512-lgHwxlxV1qIg1Eap7LgIeoBWIMFibOjbrYPIPJZcI1mmGAI2m3lNYpK12Y+GBdPQ0U1hRwSord7GIaawz962qQ==} - dev: true /onetime@6.0.0: resolution: {integrity: sha512-1FlR+gjXK7X+AsAHso35MnyN5KqGwJRi/31ft6x0M194ht7S+rWAvd7PHss9xSKMzE0asv1pyIHaJYq+BbacAQ==} @@ -648,7 +650,6 @@ packages: /streaming-iterables@8.0.1: resolution: {integrity: sha512-yfQdmUB1b+rGLZkD/r6YisT/eNOjZxBAckXKlzYNmRJnwSzHaiScykD8gsQceFcShtK09qAbLhOqvzIpnBPoDQ==} engines: {node: '>=18'} - dev: true /string_decoder@0.10.31: resolution: {integrity: sha512-ev2QzSzWPYmy9GuqfIVildA4OdcGLeFZQrq5ys6RtiuF+RQQiZWr8TZNyAcuVXyQRYfEO+MsoB/1BuQVhOJuoQ==} @@ -677,7 +678,6 @@ packages: /typescript-event-target@1.1.0: resolution: {integrity: sha512-PMrzUVryhnUq2n8M7tjNHNRuIHlUqly5RfGltBTpPCdVpbytgALTRDegF/t6+mFmmtBVhOqEYlbjVNBxwabIug==} - dev: true /uc.micro@2.0.0: resolution: {integrity: sha512-DffL94LsNOccVn4hyfRe5rdKa273swqeA5DJpMOeFmEn1wCDc7nAbbB0gXlgBCL7TNzeTv6G7XVWzan7iJtfig==} @@ -713,7 +713,6 @@ packages: /wait-your-turn@1.0.1: resolution: {integrity: sha512-UejbIY32KXhghXGwH4J2pTKUNvgdrCjdDGrtrdfHHJUAwXCok1l9ptEp4n13lg6PuyQIgxPGkWyKeJvvKeAqsA==} - dev: false /which@2.0.2: resolution: {integrity: sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==} @@ -758,7 +757,7 @@ packages: '@ndnts-nightly.ndn.today/endpoint.tgz': resolution: {tarball: https://ndnts-nightly.ndn.today/endpoint.tgz} name: '@ndn/endpoint' - version: 0.0.20240209-nightly-9b234f4 + version: 0.0.20240222-nightly-a7c3335 dependencies: '@ndn/fw': '@ndnts-nightly.ndn.today/fw.tgz' '@ndn/packet': '@ndnts-nightly.ndn.today/packet.tgz' @@ -771,7 +770,7 @@ packages: '@ndnts-nightly.ndn.today/fw.tgz': resolution: {tarball: https://ndnts-nightly.ndn.today/fw.tgz} name: '@ndn/fw' - version: 0.0.20240209-nightly-9b234f4 + version: 0.0.20240222-nightly-a7c3335 dependencies: '@ndn/packet': '@ndnts-nightly.ndn.today/packet.tgz' '@ndn/util': '@ndnts-nightly.ndn.today/util.tgz' @@ -788,7 +787,7 @@ packages: '@ndnts-nightly.ndn.today/keychain.tgz': resolution: {tarball: https://ndnts-nightly.ndn.today/keychain.tgz} name: '@ndn/keychain' - version: 0.0.20240209-nightly-9b234f4 + version: 0.0.20240222-nightly-a7c3335 dependencies: '@ndn/naming-convention2': '@ndnts-nightly.ndn.today/naming-convention2.tgz' '@ndn/packet': '@ndnts-nightly.ndn.today/packet.tgz' @@ -801,12 +800,13 @@ packages: throat: 6.0.2 tslib: 2.6.2 type-fest: 4.10.3 + wait-your-turn: 1.0.1 dev: true '@ndnts-nightly.ndn.today/l3face.tgz': resolution: {tarball: https://ndnts-nightly.ndn.today/l3face.tgz} name: '@ndn/l3face' - version: 0.0.20240209-nightly-9b234f4 + version: 0.0.20240222-nightly-a7c3335 dependencies: '@ndn/fw': '@ndnts-nightly.ndn.today/fw.tgz' '@ndn/lp': '@ndnts-nightly.ndn.today/lp.tgz' @@ -826,7 +826,7 @@ packages: '@ndnts-nightly.ndn.today/lp.tgz': resolution: {tarball: https://ndnts-nightly.ndn.today/lp.tgz} name: '@ndn/lp' - version: 0.0.20240209-nightly-9b234f4 + version: 0.0.20240222-nightly-a7c3335 dependencies: '@ndn/packet': '@ndnts-nightly.ndn.today/packet.tgz' '@ndn/tlv': '@ndnts-nightly.ndn.today/tlv.tgz' @@ -838,7 +838,7 @@ packages: '@ndnts-nightly.ndn.today/naming-convention2.tgz': resolution: {tarball: https://ndnts-nightly.ndn.today/naming-convention2.tgz} name: '@ndn/naming-convention2' - version: 0.0.20240209-nightly-9b234f4 + version: 0.0.20240222-nightly-a7c3335 dependencies: '@ndn/packet': '@ndnts-nightly.ndn.today/packet.tgz' '@ndn/tlv': '@ndnts-nightly.ndn.today/tlv.tgz' @@ -849,7 +849,7 @@ packages: '@ndnts-nightly.ndn.today/ndncert.tgz': resolution: {tarball: https://ndnts-nightly.ndn.today/ndncert.tgz} name: '@ndn/ndncert' - version: 0.0.20240209-nightly-9b234f4 + version: 0.0.20240222-nightly-a7c3335 dependencies: '@ndn/endpoint': '@ndnts-nightly.ndn.today/endpoint.tgz' '@ndn/keychain': '@ndnts-nightly.ndn.today/keychain.tgz' @@ -873,7 +873,7 @@ packages: '@ndnts-nightly.ndn.today/ndnsec.tgz': resolution: {tarball: https://ndnts-nightly.ndn.today/ndnsec.tgz} name: '@ndn/ndnsec' - version: 0.0.20240209-nightly-9b234f4 + version: 0.0.20240222-nightly-a7c3335 dependencies: '@ndn/keychain': '@ndnts-nightly.ndn.today/keychain.tgz' '@ndn/packet': '@ndnts-nightly.ndn.today/packet.tgz' @@ -888,7 +888,7 @@ packages: '@ndnts-nightly.ndn.today/nfdmgmt.tgz': resolution: {tarball: https://ndnts-nightly.ndn.today/nfdmgmt.tgz} name: '@ndn/nfdmgmt' - version: 0.0.20240209-nightly-9b234f4 + version: 0.0.20240222-nightly-a7c3335 dependencies: '@ndn/endpoint': '@ndnts-nightly.ndn.today/endpoint.tgz' '@ndn/fw': '@ndnts-nightly.ndn.today/fw.tgz' @@ -904,7 +904,7 @@ packages: '@ndnts-nightly.ndn.today/node-transport.tgz': resolution: {tarball: https://ndnts-nightly.ndn.today/node-transport.tgz} name: '@ndn/node-transport' - version: 0.0.20240209-nightly-9b234f4 + version: 0.0.20240222-nightly-a7c3335 dependencies: '@ndn/l3face': '@ndnts-nightly.ndn.today/l3face.tgz' event-iterator: 2.0.0 @@ -918,7 +918,7 @@ packages: '@ndnts-nightly.ndn.today/packet.tgz': resolution: {tarball: https://ndnts-nightly.ndn.today/packet.tgz} name: '@ndn/packet' - version: 0.0.20240209-nightly-9b234f4 + version: 0.0.20240222-nightly-a7c3335 dependencies: '@ndn/tlv': '@ndnts-nightly.ndn.today/tlv.tgz' '@ndn/util': '@ndnts-nightly.ndn.today/util.tgz' @@ -926,12 +926,11 @@ packages: mnemonist: 0.39.8 tslib: 2.6.2 type-fest: 4.10.3 - dev: true '@ndnts-nightly.ndn.today/rdr.tgz': resolution: {tarball: https://ndnts-nightly.ndn.today/rdr.tgz} name: '@ndn/rdr' - version: 0.0.20240209-nightly-9b234f4 + version: 0.0.20240222-nightly-a7c3335 dependencies: '@ndn/endpoint': '@ndnts-nightly.ndn.today/endpoint.tgz' '@ndn/naming-convention2': '@ndnts-nightly.ndn.today/naming-convention2.tgz' @@ -943,7 +942,7 @@ packages: '@ndnts-nightly.ndn.today/repo-api.tgz': resolution: {tarball: https://ndnts-nightly.ndn.today/repo-api.tgz} name: '@ndn/repo-api' - version: 0.0.20240209-nightly-9b234f4 + version: 0.0.20240222-nightly-a7c3335 dependencies: '@ndn/l3face': '@ndnts-nightly.ndn.today/l3face.tgz' '@ndn/naming-convention2': '@ndnts-nightly.ndn.today/naming-convention2.tgz' @@ -951,7 +950,7 @@ packages: '@ndn/rdr': '@ndnts-nightly.ndn.today/rdr.tgz' '@ndn/tlv': '@ndnts-nightly.ndn.today/tlv.tgz' '@ndn/util': '@ndnts-nightly.ndn.today/util.tgz' - is-stream: 3.0.0 + is-stream: 4.0.1 it-pushable: 3.2.3 p-defer: 4.0.0 p-event: 6.0.0 @@ -959,12 +958,13 @@ packages: throat: 6.0.2 tslib: 2.6.2 typescript-event-target: 1.1.0 + wait-your-turn: 1.0.1 dev: true '@ndnts-nightly.ndn.today/segmented-object.tgz': resolution: {tarball: https://ndnts-nightly.ndn.today/segmented-object.tgz} name: '@ndn/segmented-object' - version: 0.0.20240209-nightly-9b234f4 + version: 0.0.20240222-nightly-a7c3335 dependencies: '@ndn/endpoint': '@ndnts-nightly.ndn.today/endpoint.tgz' '@ndn/fw': '@ndnts-nightly.ndn.today/fw.tgz' @@ -985,7 +985,7 @@ packages: '@ndnts-nightly.ndn.today/svs.tgz': resolution: {tarball: https://ndnts-nightly.ndn.today/svs.tgz} name: '@ndn/svs' - version: 0.0.20240209-nightly-9b234f4 + version: 0.0.20240222-nightly-a7c3335 dependencies: '@ndn/endpoint': '@ndnts-nightly.ndn.today/endpoint.tgz' '@ndn/naming-convention2': '@ndnts-nightly.ndn.today/naming-convention2.tgz' @@ -1004,29 +1004,27 @@ packages: '@ndnts-nightly.ndn.today/sync-api.tgz': resolution: {tarball: https://ndnts-nightly.ndn.today/sync-api.tgz} name: '@ndn/sync-api' - version: 0.0.20240209-nightly-9b234f4 + version: 0.0.20240222-nightly-a7c3335 dependencies: '@ndn/packet': '@ndnts-nightly.ndn.today/packet.tgz' '@ndn/util': '@ndnts-nightly.ndn.today/util.tgz' tslib: 2.6.2 typescript-event-target: 1.1.0 - dev: true '@ndnts-nightly.ndn.today/tlv.tgz': resolution: {tarball: https://ndnts-nightly.ndn.today/tlv.tgz} name: '@ndn/tlv' - version: 0.0.20240209-nightly-9b234f4 + version: 0.0.20240222-nightly-a7c3335 dependencies: '@ndn/util': '@ndnts-nightly.ndn.today/util.tgz' mnemonist: 0.39.8 tslib: 2.6.2 type-fest: 4.10.3 - dev: true '@ndnts-nightly.ndn.today/util.tgz': resolution: {tarball: https://ndnts-nightly.ndn.today/util.tgz} name: '@ndn/util' - version: 0.0.20240209-nightly-9b234f4 + version: 0.0.20240222-nightly-a7c3335 engines: {node: ^18.18.0 || ^20.10.0 || ^21.0.0} dependencies: '@shigen/polyfill-symbol-dispose': 1.0.1 @@ -1036,12 +1034,12 @@ packages: streaming-iterables: 8.0.1 tslib: 2.6.2 type-fest: 4.10.3 - dev: true + wait-your-turn: 1.0.1 '@ndnts-nightly.ndn.today/ws-transport.tgz': resolution: {tarball: https://ndnts-nightly.ndn.today/ws-transport.tgz} name: '@ndn/ws-transport' - version: 0.0.20240209-nightly-9b234f4 + version: 0.0.20240222-nightly-a7c3335 dependencies: '@ndn/l3face': '@ndnts-nightly.ndn.today/l3face.tgz' '@ndn/util': '@ndnts-nightly.ndn.today/util.tgz' diff --git a/src/sync-agent/deliveries.ts b/src/sync-agent/deliveries.ts index eaf3d23..8854dc4 100644 --- a/src/sync-agent/deliveries.ts +++ b/src/sync-agent/deliveries.ts @@ -311,7 +311,7 @@ export class AtLeastOnceDelivery extends SyncDelivery { for (; newSeq < lastSeen; newSeq++) { // This can be optimized with some data structure like C++ set, but not now. const dataName = updateBaseName.append(SequenceNum.create(newSeq + 1)); - if (!this.storage.has(dataName.toString())) { + if (!await this.storage.has(dataName.toString())) { break; } } diff --git a/src/sync-agent/delivery-alo.test.ts b/src/sync-agent/delivery-alo.test.ts index ba90d19..e663c2c 100644 --- a/src/sync-agent/delivery-alo.test.ts +++ b/src/sync-agent/delivery-alo.test.ts @@ -3,6 +3,7 @@ import { Forwarder } from '@ndn/fw'; import { Data, digestSigning, Name, type Signer, type Verifier } from '@ndn/packet'; import { GenericNumber } from '@ndn/naming-convention2'; import { Encoder } from '@ndn/tlv'; +import { SyncUpdate } from '@ndn/sync-api'; import { assert, hex } from '../dep.ts'; import { AtLeastOnceDelivery, SyncDelivery } from './deliveries.ts'; import { AsyncDisposableStack, name, Responder } from '../utils/mod.ts'; @@ -213,6 +214,126 @@ Deno.test('Alo.2 No missing due to out-of-order', async () => { }); }); +Deno.test('Alo.2.1 Concurrent onUpdates causing gap in the middle', async () => { + let eventSet; + { + const { promise: stopSignal1, resolve: stop1 } = Promise.withResolvers(); + const { promise: stopSignal2, resolve: stop2 } = Promise.withResolvers(); + const { promise: stopSignal3, resolve: stop3 } = Promise.withResolvers(); + await using tester = new DeliveryTester(2, () => { + if (tester.events.length === 4) { + stop1(); + } else if (tester.events.length === 6) { + stop2(); + } else if (tester.events.length === 8) { + stop3(); + } + return Promise.resolve(); + }); + await tester.start(3000); + + // Do not use alo1 for this time + await tester.alos[1].destroy(); + + // Then generate two normal updates. They are fetched first. (No 7, 8) + await tester.dispositData(1, 7, new TextEncoder().encode('G')); + await tester.dispositData(1, 8, new TextEncoder().encode('H')); + // Make some initial data (No 1, 2) + await tester.dispositData(1, 1, new TextEncoder().encode('A')); + await tester.dispositData(1, 2, new TextEncoder().encode('B')); + + // Call onUpdate for 7-8 and 1-2 + await tester.alos[0].handleSyncUpdate( + new SyncUpdate(tester.alos[0].syncInst!.get(name`/test/32=node/${1}`), 7, 8), + ); + await tester.alos[0].handleSyncUpdate( + new SyncUpdate(tester.alos[0].syncInst!.get(name`/test/32=node/${1}`), 1, 2), + ); + + await stopSignal1; + // For now, the state must be in the middle + assert.assertEquals(tester.alos[0].syncState.get(name`/test/32=node/${1}`), 2); + // But the data should be delivered + assert.assertEquals(tester.events.length, 4); + + // Make up some missing data. (No 3, 5) + await tester.dispositData(1, 3, new TextEncoder().encode('C')); + await tester.dispositData(1, 5, new TextEncoder().encode('E')); + // Call onUpdate on each of them + await tester.alos[0].handleSyncUpdate( + new SyncUpdate(tester.alos[0].syncInst!.get(name`/test/32=node/${1}`), 3, 3), + ); + await tester.alos[0].handleSyncUpdate( + new SyncUpdate(tester.alos[0].syncInst!.get(name`/test/32=node/${1}`), 5, 5), + ); + + await stopSignal2; + // For now, the state must move by 1 + assert.assertEquals(tester.alos[0].syncState.get(name`/test/32=node/${1}`), 3); + + // Finally make up all missing data. + await tester.dispositData(1, 4, new TextEncoder().encode('D')); + await tester.dispositData(1, 6, new TextEncoder().encode('F')); + // Call onUpdate on each of them + await tester.alos[0].handleSyncUpdate( + new SyncUpdate(tester.alos[0].syncInst!.get(name`/test/32=node/${1}`), 4, 4), + ); + await tester.alos[0].handleSyncUpdate( + new SyncUpdate(tester.alos[0].syncInst!.get(name`/test/32=node/${1}`), 6, 6), + ); + + await stopSignal3; + eventSet = tester.events; + + // At last, the state should be updated + assert.assertEquals(tester.alos[0].syncState.get(name`/test/32=node/${1}`), 8); + } + + // Since it is unordered, we have to sort + eventSet.sort(compareEvent); + assert.assertEquals(eventSet.length, 8); + assert.assertEquals(eventSet[0], { + content: new TextEncoder().encode('A'), + origin: 1, + receiver: 0, + }); + assert.assertEquals(eventSet[1], { + content: new TextEncoder().encode('B'), + origin: 1, + receiver: 0, + }); + assert.assertEquals(eventSet[2], { + content: new TextEncoder().encode('C'), + origin: 1, + receiver: 0, + }); + assert.assertEquals(eventSet[3], { + content: new TextEncoder().encode('D'), + origin: 1, + receiver: 0, + }); + assert.assertEquals(eventSet[4], { + content: new TextEncoder().encode('E'), + origin: 1, + receiver: 0, + }); + assert.assertEquals(eventSet[5], { + content: new TextEncoder().encode('F'), + origin: 1, + receiver: 0, + }); + assert.assertEquals(eventSet[6], { + content: new TextEncoder().encode('G'), + origin: 1, + receiver: 0, + }); + assert.assertEquals(eventSet[7], { + content: new TextEncoder().encode('H'), + origin: 1, + receiver: 0, + }); +}); + Deno.test('Alo.3 Recover after shutdown', async () => { let eventSet; {