From 02c5ab47aea1f09860cf1259e0b61eb898d7f5f7 Mon Sep 17 00:00:00 2001 From: Vin Kabuki Date: Mon, 15 May 2023 23:09:06 +0200 Subject: [PATCH 1/4] Working file deletion --- .../backend/src/storage/ipfsFileManager.ts | 72 ++++++------------- 1 file changed, 23 insertions(+), 49 deletions(-) diff --git a/packages/backend/src/storage/ipfsFileManager.ts b/packages/backend/src/storage/ipfsFileManager.ts index c30cc54ab8..7cddfd27b6 100644 --- a/packages/backend/src/storage/ipfsFileManager.ts +++ b/packages/backend/src/storage/ipfsFileManager.ts @@ -98,45 +98,25 @@ export class IpfsFilesManager extends EventEmitter { console.error(`downloading ${mid} has already been canceled or never started`) } }) - this.on(IpfsFilesManagerEvents.DELETE_FILE, async (fileMetadata: FileMetadata) => { - // Channel deletion WIP - - // Check if we have it in case we didnt downloaded file - // await this.deleteBlocks(fileMetadata) - }) } public async deleteBlocks(fileMetadata: FileMetadata) { - // console.log('deleting file in fileManager') - // const localBlocks = await this.getLocalBlocks() - // const hasBlockBeenDownloaded = localBlocks.includes(`z${fileMetadata.cid.toString()}`) - // console.log('has block been downlaoded ', hasBlockBeenDownloaded) - // if (!hasBlockBeenDownloaded) return - - // const les = this.ipfs.pin.ls({paths: CID.parse(fileMetadata.cid)}) - // for await (const l of les) { - // console.log('llllll', l) - // } - - // try { - // const result = await this.ipfs.pin.rm(CID.parse(fileMetadata.cid), {recursive: true}) - // console.log( - // 'unpinning result ', result - // ) - // } catch (e) { - // console.log('file removing error') - // console.log(e) - // } - // const gcresult = this.ipfs.repo.gc() - // for await (const res of gcresult) { - // console.log('garbage collector result', res) - // } - // const blocks = this.ipfs.get(CID.parse(fileMetadata.cid)) - // for await (const block of blocks) { - // // const decodedBlock = decode(block) - // console.log('bock', block) - // } - // Parse DAG blocks in case file is canceled mid download + const localBlocks = await this.getLocalBlocks() + const hasBlockBeenDownloaded = localBlocks.includes(`z${fileMetadata.cid.toString()}`) + if (!hasBlockBeenDownloaded) return + + try { + const result = await this.ipfs.pin.rm(fileMetadata.cid, {recursive: true}) + + } catch (e) { + console.log('file removing error') + console.log(e) + } + + const gcresult = this.ipfs.repo.gc() + for await (const res of gcresult) { + console.log('garbage collector result', res) + } } public async stop() { @@ -199,22 +179,18 @@ export class IpfsFilesManager extends EventEmitter { // Save copy to separate directory const filePath = this.copyFile(metadata.path, filename) console.time(`Writing ${filename} to ipfs`) - await this.ipfs.files.write(`/${dirname}/${filename}`, uploadedFileStreamIterable, { - create: true - }) + const newCid = await this.ipfs.add(uploadedFileStreamIterable) + + console.timeEnd(`Writing ${filename} to ipfs`) - // Get uploaded file information - const entries = this.ipfs.files.ls(`/${dirname}`) - for await (const entry of entries) { - if (entry.name === filename) { + this.emit(StorageEvents.REMOVE_DOWNLOAD_STATUS, { cid: metadata.cid }) - await this.ipfs.pin.add(entry.cid) const fileMetadata: FileMetadata = { ...metadata, path: filePath, - cid: entry.cid.toString(), - size: entry.size, + cid: newCid.cid.toString(), + size: newCid.size, width, height } @@ -233,9 +209,7 @@ export class IpfsFilesManager extends EventEmitter { if (metadata.path !== filePath) { this.emit(StorageEvents.UPDATE_MESSAGE_MEDIA, fileMetadata) } - break - } - } + } private cancelDownload = async (cid: string) => { From ea83c56be70875024c196349b5c7a09285350072 Mon Sep 17 00:00:00 2001 From: Vin Kabuki Date: Wed, 24 May 2023 11:46:11 +0200 Subject: [PATCH 2/4] Refactor file upload and deletion --- packages/backend/package-lock.json | 2 +- .../src/storage/ChannelsAccessController.ts | 27 +++++++-- .../backend/src/storage/ipfsFileManager.ts | 59 +++++++++---------- packages/backend/src/storage/storage.ts | 32 ++++++---- packages/common/package-lock.json | 2 +- packages/desktop/package-lock.json | 2 +- packages/e2e-tests/package-lock.json | 2 +- packages/identity/package-lock.json | 2 +- packages/integration-tests/package-lock.json | 2 +- packages/logger/package-lock.json | 2 +- packages/mobile/package-lock.json | 2 +- packages/state-manager/package-lock.json | 2 +- packages/types/package-lock.json | 2 +- 13 files changed, 83 insertions(+), 55 deletions(-) diff --git a/packages/backend/package-lock.json b/packages/backend/package-lock.json index cb6c1d1f72..99ac76b4c0 100644 --- a/packages/backend/package-lock.json +++ b/packages/backend/package-lock.json @@ -6,7 +6,7 @@ "packages": { "": { "name": "@quiet/backend", - "version": "1.2.1-alpha.3", + "version": "1.2.1-alpha.4", "license": "MIT", "dependencies": { "@chainsafe/libp2p-gossipsub": "6.1.0", diff --git a/packages/backend/src/storage/ChannelsAccessController.ts b/packages/backend/src/storage/ChannelsAccessController.ts index 13c731b7fd..7dbac34303 100644 --- a/packages/backend/src/storage/ChannelsAccessController.ts +++ b/packages/backend/src/storage/ChannelsAccessController.ts @@ -3,21 +3,40 @@ import { getCrypto } from 'pkijs' import { stringToArrayBuffer } from 'pvutils' import { ChannelMessage, PublicChannel } from '@quiet/types' import { keyObjectFromString, verifySignature } from '@quiet/identity' -import { IdentityProvider } from 'orbit-db-identity-provider' +import Identities from 'orbit-db-identity-provider' +// import Keystore from 'orbit-db-keystore' import OrbitDB from 'orbit-db' import PeerId from 'peer-id' +const Keystore = await import('orbit-db-keystore') + const type = 'channelsaccess' -export const createChannelAccessController = (peerId: PeerId) => { +export const createChannelAccessController = (peerId: PeerId, dir: string) => { // @ts-ignore class ChannelsAccessController extends AccessController { static get type() { return type } - async canAppend(entry: LogEntry, identityProvider: IdentityProvider) { - // Channel deletion WIP + async canAppend(entry: LogEntry, identityProvider: any) { + console.log('can append entry ', entry) + + // const keystore = identityProvider._keystore + + // const stringPeerId = 'QmTBYqK1qTXW9E6os3vc9phR9JWRs1jUTutgpLEGJVtubY' + + console.log('identityProvider', identityProvider) + // identityProvider.getId(peerId.toString()) + + // const identity = await Identities.createIdentity({ + // id: stringPeerId, + // keystore + // }) + + // console.log('identity ', identity) + + // console.log('access controller id ', identity) // console.log('peerId ', peerId.toString()) // console.log('entry ', entry) diff --git a/packages/backend/src/storage/ipfsFileManager.ts b/packages/backend/src/storage/ipfsFileManager.ts index 7cddfd27b6..f895b3a148 100644 --- a/packages/backend/src/storage/ipfsFileManager.ts +++ b/packages/backend/src/storage/ipfsFileManager.ts @@ -106,13 +106,13 @@ export class IpfsFilesManager extends EventEmitter { if (!hasBlockBeenDownloaded) return try { - const result = await this.ipfs.pin.rm(fileMetadata.cid, {recursive: true}) + const result = await this.ipfs.pin.rm(fileMetadata.cid, { recursive: true }) } catch (e) { console.log('file removing error') console.log(e) } - + const gcresult = this.ipfs.repo.gc() for await (const res of gcresult) { console.log('garbage collector result', res) @@ -148,7 +148,7 @@ export class IpfsFilesManager extends EventEmitter { let width: number | undefined let height: number | undefined if (imagesExtensions.includes(metadata.ext)) { - let imageSize: {width: number | undefined; height: number | undefined} | undefined // ISizeCalculationResult + let imageSize: { width: number | undefined; height: number | undefined } | undefined // ISizeCalculationResult try { imageSize = await sizeOfPromisified(metadata.path) } catch (e) { @@ -161,7 +161,7 @@ export class IpfsFilesManager extends EventEmitter { const stream = fs.createReadStream(metadata.path, { highWaterMark: 64 * 1024 * 10 }) const uploadedFileStreamIterable = { - async* [Symbol.asyncIterator]() { + async*[Symbol.asyncIterator]() { for await (const data of stream) { yield data } @@ -179,37 +179,34 @@ export class IpfsFilesManager extends EventEmitter { // Save copy to separate directory const filePath = this.copyFile(metadata.path, filename) console.time(`Writing ${filename} to ipfs`) - const newCid = await this.ipfs.add(uploadedFileStreamIterable) - + const newCid = await this.ipfs.add(uploadedFileStreamIterable) console.timeEnd(`Writing ${filename} to ipfs`) - - this.emit(StorageEvents.REMOVE_DOWNLOAD_STATUS, { cid: metadata.cid }) - const fileMetadata: FileMetadata = { - ...metadata, - path: filePath, - cid: newCid.cid.toString(), - size: newCid.size, - width, - height - } - - this.emit(StorageEvents.UPLOADED_FILE, fileMetadata) + this.emit(StorageEvents.REMOVE_DOWNLOAD_STATUS, { cid: metadata.cid }) + const fileMetadata: FileMetadata = { + ...metadata, + path: filePath, + cid: newCid.cid.toString(), + size: newCid.size, + width, + height + } - const statusReady: DownloadStatus = { - mid: fileMetadata.message.id, - cid: fileMetadata.cid, - downloadState: DownloadState.Hosted, - downloadProgress: undefined - } + this.emit(StorageEvents.UPLOADED_FILE, fileMetadata) - this.emit(StorageEvents.UPDATE_DOWNLOAD_PROGRESS, statusReady) + const statusReady: DownloadStatus = { + mid: fileMetadata.message.id, + cid: fileMetadata.cid, + downloadState: DownloadState.Hosted, + downloadProgress: undefined + } - if (metadata.path !== filePath) { - this.emit(StorageEvents.UPDATE_MESSAGE_MEDIA, fileMetadata) - } + this.emit(StorageEvents.UPDATE_DOWNLOAD_PROGRESS, statusReady) + if (metadata.path !== filePath) { + this.emit(StorageEvents.UPDATE_MESSAGE_MEDIA, fileMetadata) + } } private cancelDownload = async (cid: string) => { @@ -243,6 +240,8 @@ export class IpfsFilesManager extends EventEmitter { public downloadBlocks = async (fileMetadata: FileMetadata) => { const block = CID.parse(fileMetadata.cid) + await this.ipfs.pin.add(block, { recursive: true }) + const localBlocks = await this.getLocalBlocks() const processedBlocks: PBNode[] = [] // TODO: Should it be CID or PBNode? @@ -451,8 +450,8 @@ export class IpfsFilesManager extends EventEmitter { private updateStatus = async (cid: string, downloadState = DownloadState.Downloading) => { const metadata = this.files.get(cid) if (!metadata) { - // TODO: emit error? - return + // TODO: emit error? + return } const progress: DownloadProgress | undefined = downloadState !== DownloadState.Malicious ? { size: metadata.size, diff --git a/packages/backend/src/storage/storage.ts b/packages/backend/src/storage/storage.ts index 40df9c263a..432dddd02f 100644 --- a/packages/backend/src/storage/storage.ts +++ b/packages/backend/src/storage/storage.ts @@ -87,7 +87,7 @@ export class Storage extends EventEmitter { this.filesManager = new IpfsFilesManager(this.ipfs, this.quietDir) this.attachFileManagerEvents() - const channelsAccessController = createChannelAccessController(peerID) + const channelsAccessController = createChannelAccessController(peerID, this.orbitDbDir) AccessControllers.addAccessController({ AccessController: MessagesAccessController }) AccessControllers.addAccessController({ AccessController: channelsAccessController }) @@ -286,7 +286,7 @@ export class Storage extends EventEmitter { }) // @ts-expect-error - OrbitDB's type declaration of `load` lacks 'options' - await this.channels.load({ fetchEntryTimeout: 15000 }) + await this.channels.load({ fetchEntryTimeout: 1000 }) log('ALL CHANNELS COUNT:', Object.keys(this.channels.all).length) log('ALL CHANNELS COUNT:', Object.keys(this.channels.all)) Object.values(this.channels.all).forEach(async (channel: PublicChannel) => { @@ -498,18 +498,18 @@ export class Storage extends EventEmitter { this.publicChannelsRepos.set(data.address, { db, eventsAttached: false }) log(`Set ${data.address} to local channels`) // @ts-expect-error - OrbitDB's type declaration of `load` lacks 'options' - await db.load({ fetchEntryTimeout: 2000 }) + await db.load({ fetchEntryTimeout: 2000, }) log(`Created channel ${data.address}`) return db } - public async deleteChannel(payload: {channel: string}) { + public async deleteChannel(payload: { channel: string }) { console.log('deleting channel storage', payload) // @ts-expect-error - OrbitDB's type declaration of `load` lacks 'options' await this.channels.load({ fetchEntryTimeout: 15000 }) const channel = this.channels.get(payload.channel) if (channel) { - void this.channels.del(payload.channel) + await this.channels.del(payload.channel) } let repo = this.publicChannelsRepos.get(payload.channel) if (!repo) { @@ -535,8 +535,8 @@ export class Storage extends EventEmitter { const files = allEntries.map((e) => { return e.payload.value.media }).filter(isDefined) - await this.deleteChannelFiles(files) - await this.deleteChannelMessages(hashes) + // await this.deleteChannelFiles(files) + // await this.deleteChannelMessages(hashes) this.publicChannelsRepos.delete(payload.channel) this.emit(StorageEvents.CHANNEL_DELETION_RESPONSE, payload) } @@ -552,11 +552,21 @@ export class Storage extends EventEmitter { } public async deleteChannelMessages(hashes: CID[]) { - for await (const result of this.ipfs.block.rm(hashes)) { - if (result.error) { - console.error(`Failed to remove block ${result.cid} due to ${result.error.message}`) - } + console.log('hashes ', hashes) + const gcresult = this.ipfs.repo.gc() + for await (const res of gcresult) { + + // @ts-ignore + const ccc = base58.base58btc.encode(res.cid?.multihash.bytes) + + console.log('base58btc encoded', ccc) + console.log('garbage collector result', res) } + // for await (const result of this.ipfs.block.rm(hashes)) { + // if (result.error) { + // console.error(`Failed to remove block ${result.cid} due to ${result.error.message}`) + // } + // } } public async sendMessage(message: ChannelMessage) { diff --git a/packages/common/package-lock.json b/packages/common/package-lock.json index 5e90a582b2..d3acaa5794 100644 --- a/packages/common/package-lock.json +++ b/packages/common/package-lock.json @@ -6,7 +6,7 @@ "packages": { "": { "name": "@quiet/common", - "version": "1.2.1-alpha.2", + "version": "1.2.1-alpha.3", "license": "ISC", "dependencies": { "cross-env": "^5.2.0", diff --git a/packages/desktop/package-lock.json b/packages/desktop/package-lock.json index 3a7184d354..893f02e30a 100644 --- a/packages/desktop/package-lock.json +++ b/packages/desktop/package-lock.json @@ -6,7 +6,7 @@ "packages": { "": { "name": "quiet", - "version": "1.2.1-alpha.3", + "version": "1.2.1-alpha.4", "license": "ISC", "dependencies": { "@electron/remote": "^2.0.8", diff --git a/packages/e2e-tests/package-lock.json b/packages/e2e-tests/package-lock.json index 28c20cacdb..36046f0ac0 100644 --- a/packages/e2e-tests/package-lock.json +++ b/packages/e2e-tests/package-lock.json @@ -6,7 +6,7 @@ "packages": { "": { "name": "e2e-tests", - "version": "1.2.1-alpha.2", + "version": "1.2.1-alpha.3", "license": "ISC", "dependencies": { "cross-env": "^5.2.0", diff --git a/packages/identity/package-lock.json b/packages/identity/package-lock.json index 876da5ac12..f4d2c6c2f2 100644 --- a/packages/identity/package-lock.json +++ b/packages/identity/package-lock.json @@ -6,7 +6,7 @@ "packages": { "": { "name": "@quiet/identity", - "version": "1.2.1-alpha.1", + "version": "1.2.1-alpha.2", "license": "MIT", "dependencies": { "@peculiar/webcrypto": "1.4.3", diff --git a/packages/integration-tests/package-lock.json b/packages/integration-tests/package-lock.json index 813d358f96..729498fafb 100644 --- a/packages/integration-tests/package-lock.json +++ b/packages/integration-tests/package-lock.json @@ -6,7 +6,7 @@ "packages": { "": { "name": "integration-tests", - "version": "1.2.1-alpha.2", + "version": "1.2.1-alpha.3", "license": "ISC", "dependencies": { "@peculiar/webcrypto": "1.4.3", diff --git a/packages/logger/package-lock.json b/packages/logger/package-lock.json index 63ee683969..4b501d7c16 100644 --- a/packages/logger/package-lock.json +++ b/packages/logger/package-lock.json @@ -6,7 +6,7 @@ "packages": { "": { "name": "@quiet/logger", - "version": "1.2.1-alpha.1", + "version": "1.2.1-alpha.2", "license": "ISC", "dependencies": { "cross-env": "^5.2.0", diff --git a/packages/mobile/package-lock.json b/packages/mobile/package-lock.json index 7cf43f68da..78e4369da4 100644 --- a/packages/mobile/package-lock.json +++ b/packages/mobile/package-lock.json @@ -6,7 +6,7 @@ "packages": { "": { "name": "@quiet/mobile", - "version": "1.2.1-alpha.2", + "version": "1.2.1-alpha.6", "dependencies": { "@peculiar/webcrypto": "^1.4.3", "@react-native-clipboard/clipboard": "^1.11.2", diff --git a/packages/state-manager/package-lock.json b/packages/state-manager/package-lock.json index e3791c1012..45a1dac128 100644 --- a/packages/state-manager/package-lock.json +++ b/packages/state-manager/package-lock.json @@ -6,7 +6,7 @@ "packages": { "": { "name": "@quiet/state-manager", - "version": "1.2.1-alpha.2", + "version": "1.2.1-alpha.3", "license": "ISC", "dependencies": { "@reduxjs/toolkit": "^1.9.1", diff --git a/packages/types/package-lock.json b/packages/types/package-lock.json index 6b201dfc7c..34c0bf1970 100644 --- a/packages/types/package-lock.json +++ b/packages/types/package-lock.json @@ -6,7 +6,7 @@ "packages": { "": { "name": "@quiet/types", - "version": "1.2.1-alpha.1", + "version": "1.2.1-alpha.2", "license": "ISC", "dependencies": { "cross-env": "^5.2.0", From f6361d6bc8d70c0812457e660bfd61d5366ab92b Mon Sep 17 00:00:00 2001 From: Vin Kabuki Date: Wed, 24 May 2023 12:04:26 +0200 Subject: [PATCH 3/4] adjust ipfsFileManager tests to new logic --- .../src/storage/ipfsFileManager.test.ts | 28 +++++++++---------- .../backend/src/storage/ipfsFileManager.ts | 3 +- packages/backend/src/storage/storage.ts | 1 - packages/e2e-tests/package-lock.json | 17 +++++++++++ 4 files changed, 32 insertions(+), 17 deletions(-) diff --git a/packages/backend/src/storage/ipfsFileManager.test.ts b/packages/backend/src/storage/ipfsFileManager.test.ts index 6dedacaf0c..2bce64f8e2 100644 --- a/packages/backend/src/storage/ipfsFileManager.test.ts +++ b/packages/backend/src/storage/ipfsFileManager.test.ts @@ -65,12 +65,12 @@ describe('Ipfs file manager', () => { const newFilePath = copyFileSpy.mock.results[0].value as string metadata.path = newFilePath - const cid = 'QmPWwAxgGofmXZF5RqKE4K8rVeL6oAuCnAfoR4CZWTkJ5T' + const cid = 'QmSaK2joeTBYukh8L7besrvm56wSzMhn64nqLqtvxS3ths' await waitForExpect(() => { expect(eventSpy).toHaveBeenNthCalledWith(1, StorageEvents.REMOVE_DOWNLOAD_STATUS, { cid: 'uploading_id' }) }) await waitForExpect(() => { - expect(eventSpy).toHaveBeenNthCalledWith(2, StorageEvents.UPLOADED_FILE, expect.objectContaining({ cid: cid, ext: '.png', height: 44, message: { channelAddress: 'channelAddress', id: 'id' }, name: 'test-image', size: 15847, width: 824 }) + expect(eventSpy).toHaveBeenNthCalledWith(2, StorageEvents.UPLOADED_FILE, expect.objectContaining({ cid: cid, ext: '.png', height: 44, message: { channelAddress: 'channelAddress', id: 'id' }, name: 'test-image', size: 15858, width: 824 }) ) }) await waitForExpect(() => { @@ -98,12 +98,12 @@ describe('Ipfs file manager', () => { } await fileManager.uploadFile(metadata) - const cid = 'QmaA1C173ZDtoo7K6tLqq6o2eRce3kgwoVQpxsTfQgNjDZ' + const cid = 'QmR5NiFh2bTZCpdxZkYTMaceJFaYTPuxEt8J9BhKKdSv1o' await waitForExpect(() => { expect(eventSpy).toHaveBeenNthCalledWith(1, StorageEvents.REMOVE_DOWNLOAD_STATUS, { cid: 'uploading_id' }) }) await waitForExpect(() => { - expect(eventSpy).toHaveBeenNthCalledWith(2, StorageEvents.UPLOADED_FILE, expect.objectContaining({ cid: cid, ext: '.pdf', height: undefined, message: { channelAddress: 'channelAddress', id: 'id' }, name: 'test-file', size: 761797, width: undefined } + expect(eventSpy).toHaveBeenNthCalledWith(2, StorageEvents.UPLOADED_FILE, expect.objectContaining({ cid: cid, ext: '.pdf', height: undefined, message: { channelAddress: 'channelAddress', id: 'id' }, name: 'test-file', size: 761991, width: undefined } ) ) }) @@ -111,7 +111,7 @@ describe('Ipfs file manager', () => { expect(eventSpy).toHaveBeenNthCalledWith(3, StorageEvents.UPDATE_DOWNLOAD_PROGRESS, { cid: cid, downloadProgress: undefined, downloadState: 'hosted', mid: 'id' }) }) await waitForExpect(() => { - expect(eventSpy).toHaveBeenNthCalledWith(4, StorageEvents.UPDATE_MESSAGE_MEDIA, expect.objectContaining({ cid: cid, ext: '.pdf', height: undefined, message: { channelAddress: 'channelAddress', id: 'id' }, name: 'test-file', size: 761797, width: undefined }) + expect(eventSpy).toHaveBeenNthCalledWith(4, StorageEvents.UPDATE_MESSAGE_MEDIA, expect.objectContaining({ cid: cid, ext: '.pdf', height: undefined, message: { channelAddress: 'channelAddress', id: 'id' }, name: 'test-file', size: 761991, width: undefined }) ) }) }) @@ -162,18 +162,18 @@ describe('Ipfs file manager', () => { } await fileManager.uploadFile(metadata) - const cid = 'QmaA1C173ZDtoo7K6tLqq6o2eRce3kgwoVQpxsTfQgNjDZ' + const cid = 'QmR5NiFh2bTZCpdxZkYTMaceJFaYTPuxEt8J9BhKKdSv1o' await waitForExpect(() => { expect(eventSpy).toHaveBeenNthCalledWith(1, StorageEvents.REMOVE_DOWNLOAD_STATUS, { cid: 'uploading_id' }) }) await waitForExpect(() => { - expect(eventSpy).toHaveBeenNthCalledWith(2, StorageEvents.UPLOADED_FILE, expect.objectContaining({ cid: cid, ext: '.pdf', height: undefined, message: { channelAddress: 'channelAddress', id: 'id' }, name: 'test-file', size: 761797, width: undefined })) + expect(eventSpy).toHaveBeenNthCalledWith(2, StorageEvents.UPLOADED_FILE, expect.objectContaining({ cid: cid, ext: '.pdf', height: undefined, message: { channelAddress: 'channelAddress', id: 'id' }, name: 'test-file', size: 761991, width: undefined })) }) await waitForExpect(() => { expect(eventSpy).toHaveBeenNthCalledWith(3, StorageEvents.UPDATE_DOWNLOAD_PROGRESS, { cid: cid, downloadProgress: undefined, downloadState: 'hosted', mid: 'id' }) }) await waitForExpect(() => { - expect(eventSpy).toHaveBeenNthCalledWith(4, StorageEvents.UPDATE_MESSAGE_MEDIA, expect.objectContaining({ cid: cid, ext: '.pdf', height: undefined, message: { channelAddress: 'channelAddress', id: 'id' }, name: 'test-file', size: 761797, width: undefined })) + expect(eventSpy).toHaveBeenNthCalledWith(4, StorageEvents.UPDATE_MESSAGE_MEDIA, expect.objectContaining({ cid: cid, ext: '.pdf', height: undefined, message: { channelAddress: 'channelAddress', id: 'id' }, name: 'test-file', size: 761991, width: undefined })) }) // Downloading @@ -277,17 +277,17 @@ describe('Ipfs file manager', () => { } await fileManager.uploadFile(metadata) - const cid = 'QmPWwAxgGofmXZF5RqKE4K8rVeL6oAuCnAfoR4CZWTkJ5T' + const cid = 'QmSaK2joeTBYukh8L7besrvm56wSzMhn64nqLqtvxS3ths' expect(eventSpy).toHaveBeenNthCalledWith(1, StorageEvents.REMOVE_DOWNLOAD_STATUS, { cid: 'uploading_id' } ) - expect(eventSpy).toHaveBeenNthCalledWith(2, StorageEvents.UPLOADED_FILE, expect.objectContaining({ cid: cid, ext: '.png', height: 44, message: { channelAddress: 'channelAddress', id: 'id' }, name: 'test-image', size: 15847, width: 824 }) + expect(eventSpy).toHaveBeenNthCalledWith(2, StorageEvents.UPLOADED_FILE, expect.objectContaining({ cid: cid, ext: '.png', height: 44, message: { channelAddress: 'channelAddress', id: 'id' }, name: 'test-image', size: 15858, width: 824 }) ) expect(eventSpy).toHaveBeenNthCalledWith(3, StorageEvents.UPDATE_DOWNLOAD_PROGRESS, { cid: cid, downloadProgress: undefined, downloadState: 'hosted', mid: 'id' } ) - expect(eventSpy).toHaveBeenNthCalledWith(4, StorageEvents.UPDATE_MESSAGE_MEDIA, expect.objectContaining({ cid: cid, ext: '.png', height: 44, message: { channelAddress: 'channelAddress', id: 'id' }, name: 'test-image', size: 15847, width: 824 }) + expect(eventSpy).toHaveBeenNthCalledWith(4, StorageEvents.UPDATE_MESSAGE_MEDIA, expect.objectContaining({ cid: cid, ext: '.png', height: 44, message: { channelAddress: 'channelAddress', id: 'id' }, name: 'test-image', size: 15858, width: 824 }) ) // Downloading @@ -301,14 +301,14 @@ describe('Ipfs file manager', () => { }) await waitForExpect(() => { - expect(eventSpy).toHaveBeenNthCalledWith(6, StorageEvents.UPDATE_DOWNLOAD_PROGRESS, { cid: cid, downloadProgress: { downloaded: 15863, size: 15847, transferSpeed: 0 }, downloadState: 'downloading', mid: 'id' } + expect(eventSpy).toHaveBeenNthCalledWith(6, StorageEvents.UPDATE_DOWNLOAD_PROGRESS, { cid: cid, downloadProgress: { downloaded: 15855, size: 15858, transferSpeed: 0 }, downloadState: 'downloading', mid: 'id' } ) }, 20000) await waitForExpect(() => { - expect(eventSpy).toHaveBeenNthCalledWith(7, StorageEvents.UPDATE_DOWNLOAD_PROGRESS, { cid: cid, downloadProgress: { downloaded: 15863, size: 15847, transferSpeed: 0 }, downloadState: 'completed', mid: 'id' } + expect(eventSpy).toHaveBeenNthCalledWith(7, StorageEvents.UPDATE_DOWNLOAD_PROGRESS, { cid: cid, downloadProgress: { downloaded: 15855, size: 15858, transferSpeed: 0 }, downloadState: 'completed', mid: 'id' } ) }, 20000) - expect(eventSpy).toHaveBeenNthCalledWith(8, StorageEvents.UPDATE_MESSAGE_MEDIA, expect.objectContaining({ cid: cid, ext: '.png', height: 44, message: { channelAddress: 'channelAddress', id: 'id' }, name: 'test-image', size: 15847, width: 824 }) + expect(eventSpy).toHaveBeenNthCalledWith(8, StorageEvents.UPDATE_MESSAGE_MEDIA, expect.objectContaining({ cid: cid, ext: '.png', height: 44, message: { channelAddress: 'channelAddress', id: 'id' }, name: 'test-image', size: 15858, width: 824 }) ) }) it('downloaded file matches uploaded file', async () => { diff --git a/packages/backend/src/storage/ipfsFileManager.ts b/packages/backend/src/storage/ipfsFileManager.ts index 1794b1d0ba..7f61227477 100644 --- a/packages/backend/src/storage/ipfsFileManager.ts +++ b/packages/backend/src/storage/ipfsFileManager.ts @@ -107,7 +107,6 @@ export class IpfsFilesManager extends EventEmitter { try { const result = await this.ipfs.pin.rm(fileMetadata.cid, { recursive: true }) - } catch (e) { console.log('file removing error') console.log(e) @@ -164,7 +163,7 @@ export class IpfsFilesManager extends EventEmitter { const stream = fs.createReadStream(metadata.path, { highWaterMark: 64 * 1024 * 10 }) const uploadedFileStreamIterable = { - async*[Symbol.asyncIterator]() { + async* [Symbol.asyncIterator]() { for await (const data of stream) { yield data } diff --git a/packages/backend/src/storage/storage.ts b/packages/backend/src/storage/storage.ts index de63f0638f..82263aacfd 100644 --- a/packages/backend/src/storage/storage.ts +++ b/packages/backend/src/storage/storage.ts @@ -555,7 +555,6 @@ export class Storage extends EventEmitter { console.log('hashes ', hashes) const gcresult = this.ipfs.repo.gc() for await (const res of gcresult) { - // @ts-ignore const ccc = base58.base58btc.encode(res.cid?.multihash.bytes) diff --git a/packages/e2e-tests/package-lock.json b/packages/e2e-tests/package-lock.json index 36046f0ac0..180e11d68a 100644 --- a/packages/e2e-tests/package-lock.json +++ b/packages/e2e-tests/package-lock.json @@ -11,6 +11,7 @@ "dependencies": { "cross-env": "^5.2.0", "electron-chromedriver": "22.0.0", + "get-port": "^5.1.1", "jest": "^29.4.2", "path-browserify": "^1.0.1", "selenium-webdriver": "^4.8.0", @@ -3812,6 +3813,17 @@ "node": ">=8.0.0" } }, + "node_modules/get-port": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/get-port/-/get-port-5.1.1.tgz", + "integrity": "sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ==", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/get-stream": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz", @@ -10862,6 +10874,11 @@ "resolved": "https://registry.npmjs.org/get-package-type/-/get-package-type-0.1.0.tgz", "integrity": "sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==" }, + "get-port": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/get-port/-/get-port-5.1.1.tgz", + "integrity": "sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ==" + }, "get-stream": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz", From 1acf317660481945271e41b2b5828897d45ff51e Mon Sep 17 00:00:00 2001 From: Vin Kabuki Date: Wed, 24 May 2023 12:35:05 +0200 Subject: [PATCH 4/4] cleanup --- packages/backend/src/storage/ChannelsAccessController.ts | 5 ++--- packages/backend/src/storage/storage.ts | 6 +++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/packages/backend/src/storage/ChannelsAccessController.ts b/packages/backend/src/storage/ChannelsAccessController.ts index 7dbac34303..71c166d748 100644 --- a/packages/backend/src/storage/ChannelsAccessController.ts +++ b/packages/backend/src/storage/ChannelsAccessController.ts @@ -4,7 +4,6 @@ import { stringToArrayBuffer } from 'pvutils' import { ChannelMessage, PublicChannel } from '@quiet/types' import { keyObjectFromString, verifySignature } from '@quiet/identity' import Identities from 'orbit-db-identity-provider' -// import Keystore from 'orbit-db-keystore' import OrbitDB from 'orbit-db' import PeerId from 'peer-id' @@ -20,13 +19,13 @@ export const createChannelAccessController = (peerId: PeerId, dir: string) => { } async canAppend(entry: LogEntry, identityProvider: any) { - console.log('can append entry ', entry) + // console.log('can append entry ', entry) // const keystore = identityProvider._keystore // const stringPeerId = 'QmTBYqK1qTXW9E6os3vc9phR9JWRs1jUTutgpLEGJVtubY' - console.log('identityProvider', identityProvider) + // console.log('identityProvider', identityProvider) // identityProvider.getId(peerId.toString()) // const identity = await Identities.createIdentity({ diff --git a/packages/backend/src/storage/storage.ts b/packages/backend/src/storage/storage.ts index 82263aacfd..f3c11156ae 100644 --- a/packages/backend/src/storage/storage.ts +++ b/packages/backend/src/storage/storage.ts @@ -556,10 +556,10 @@ export class Storage extends EventEmitter { const gcresult = this.ipfs.repo.gc() for await (const res of gcresult) { // @ts-ignore - const ccc = base58.base58btc.encode(res.cid?.multihash.bytes) + // const ccc = base58.base58btc.encode(res.cid?.multihash.bytes) - console.log('base58btc encoded', ccc) - console.log('garbage collector result', res) + // console.log('base58btc encoded', ccc) + // console.log('garbage collector result', res) } // for await (const result of this.ipfs.block.rm(hashes)) { // if (result.error) {