Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Deletion/remove ipfs file blocks #1531

Merged
merged 5 commits into from
May 24, 2023
Merged

Deletion/remove ipfs file blocks #1531

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
02c5ab4
Working file deletion
vinkabuki May 15, 2023
ea83c56
Refactor file upload and deletion
vinkabuki May 24, 2023
1a31ac0
Merge branch 'develop' of github.com:TryQuiet/quiet into deletion/rem…
vinkabuki May 24, 2023
f6361d6
adjust ipfsFileManager tests to new logic
vinkabuki May 24, 2023
1acf317
cleanup
vinkabuki May 24, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 22 additions & 4 deletions packages/backend/src/storage/ChannelsAccessController.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,21 +3,39 @@ import { getCrypto } from 'pkijs'
import { stringToArrayBuffer } from 'pvutils'
import { ChannelMessage, PublicChannel } from '@quiet/types'
import { keyObjectFromString, verifySignature } from '@quiet/identity'
import { IdentityProvider } from 'orbit-db-identity-provider'
import Identities from 'orbit-db-identity-provider'
import OrbitDB from 'orbit-db'
import PeerId from 'peer-id'

const Keystore = await import('orbit-db-keystore')

const type = 'channelsaccess'

export const createChannelAccessController = (peerId: PeerId) => {
export const createChannelAccessController = (peerId: PeerId, dir: string) => {
// @ts-ignore
class ChannelsAccessController extends AccessController {
static get type() {
return type
}

async canAppend(entry: LogEntry<PublicChannel>, identityProvider: IdentityProvider) {
// Channel deletion WIP
async canAppend(entry: LogEntry<PublicChannel>, identityProvider: any) {
// console.log('can append entry ', entry)

// const keystore = identityProvider._keystore

// const stringPeerId = 'QmTBYqK1qTXW9E6os3vc9phR9JWRs1jUTutgpLEGJVtubY'

// console.log('identityProvider', identityProvider)
// identityProvider.getId(peerId.toString())

// const identity = await Identities.createIdentity({
// id: stringPeerId,
// keystore
// })

// console.log('identity ', identity)

// console.log('access controller id ', identity)

// console.log('peerId ', peerId.toString())
// console.log('entry ', entry)
Expand Down
28 changes: 14 additions & 14 deletions packages/backend/src/storage/ipfsFileManager.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,12 +65,12 @@ describe('Ipfs file manager', () => {
const newFilePath = copyFileSpy.mock.results[0].value as string
metadata.path = newFilePath

const cid = 'QmPWwAxgGofmXZF5RqKE4K8rVeL6oAuCnAfoR4CZWTkJ5T'
const cid = 'QmSaK2joeTBYukh8L7besrvm56wSzMhn64nqLqtvxS3ths'
await waitForExpect(() => {
expect(eventSpy).toHaveBeenNthCalledWith(1, StorageEvents.REMOVE_DOWNLOAD_STATUS, { cid: 'uploading_id' })
})
await waitForExpect(() => {
expect(eventSpy).toHaveBeenNthCalledWith(2, StorageEvents.UPLOADED_FILE, expect.objectContaining({ cid: cid, ext: '.png', height: 44, message: { channelAddress: 'channelAddress', id: 'id' }, name: 'test-image', size: 15847, width: 824 })
expect(eventSpy).toHaveBeenNthCalledWith(2, StorageEvents.UPLOADED_FILE, expect.objectContaining({ cid: cid, ext: '.png', height: 44, message: { channelAddress: 'channelAddress', id: 'id' }, name: 'test-image', size: 15858, width: 824 })
)
})
await waitForExpect(() => {
Expand Down Expand Up @@ -98,20 +98,20 @@ describe('Ipfs file manager', () => {
}

await fileManager.uploadFile(metadata)
const cid = 'QmaA1C173ZDtoo7K6tLqq6o2eRce3kgwoVQpxsTfQgNjDZ'
const cid = 'QmR5NiFh2bTZCpdxZkYTMaceJFaYTPuxEt8J9BhKKdSv1o'
await waitForExpect(() => {
expect(eventSpy).toHaveBeenNthCalledWith(1, StorageEvents.REMOVE_DOWNLOAD_STATUS, { cid: 'uploading_id' })
})
await waitForExpect(() => {
expect(eventSpy).toHaveBeenNthCalledWith(2, StorageEvents.UPLOADED_FILE, expect.objectContaining({ cid: cid, ext: '.pdf', height: undefined, message: { channelAddress: 'channelAddress', id: 'id' }, name: 'test-file', size: 761797, width: undefined }
expect(eventSpy).toHaveBeenNthCalledWith(2, StorageEvents.UPLOADED_FILE, expect.objectContaining({ cid: cid, ext: '.pdf', height: undefined, message: { channelAddress: 'channelAddress', id: 'id' }, name: 'test-file', size: 761991, width: undefined }
)
)
})
await waitForExpect(() => {
expect(eventSpy).toHaveBeenNthCalledWith(3, StorageEvents.UPDATE_DOWNLOAD_PROGRESS, { cid: cid, downloadProgress: undefined, downloadState: 'hosted', mid: 'id' })
})
await waitForExpect(() => {
expect(eventSpy).toHaveBeenNthCalledWith(4, StorageEvents.UPDATE_MESSAGE_MEDIA, expect.objectContaining({ cid: cid, ext: '.pdf', height: undefined, message: { channelAddress: 'channelAddress', id: 'id' }, name: 'test-file', size: 761797, width: undefined })
expect(eventSpy).toHaveBeenNthCalledWith(4, StorageEvents.UPDATE_MESSAGE_MEDIA, expect.objectContaining({ cid: cid, ext: '.pdf', height: undefined, message: { channelAddress: 'channelAddress', id: 'id' }, name: 'test-file', size: 761991, width: undefined })
)
})
})
Expand Down Expand Up @@ -162,18 +162,18 @@ describe('Ipfs file manager', () => {
}

await fileManager.uploadFile(metadata)
const cid = 'QmaA1C173ZDtoo7K6tLqq6o2eRce3kgwoVQpxsTfQgNjDZ'
const cid = 'QmR5NiFh2bTZCpdxZkYTMaceJFaYTPuxEt8J9BhKKdSv1o'
await waitForExpect(() => {
expect(eventSpy).toHaveBeenNthCalledWith(1, StorageEvents.REMOVE_DOWNLOAD_STATUS, { cid: 'uploading_id' })
})
await waitForExpect(() => {
expect(eventSpy).toHaveBeenNthCalledWith(2, StorageEvents.UPLOADED_FILE, expect.objectContaining({ cid: cid, ext: '.pdf', height: undefined, message: { channelAddress: 'channelAddress', id: 'id' }, name: 'test-file', size: 761797, width: undefined }))
expect(eventSpy).toHaveBeenNthCalledWith(2, StorageEvents.UPLOADED_FILE, expect.objectContaining({ cid: cid, ext: '.pdf', height: undefined, message: { channelAddress: 'channelAddress', id: 'id' }, name: 'test-file', size: 761991, width: undefined }))
})
await waitForExpect(() => {
expect(eventSpy).toHaveBeenNthCalledWith(3, StorageEvents.UPDATE_DOWNLOAD_PROGRESS, { cid: cid, downloadProgress: undefined, downloadState: 'hosted', mid: 'id' })
})
await waitForExpect(() => {
expect(eventSpy).toHaveBeenNthCalledWith(4, StorageEvents.UPDATE_MESSAGE_MEDIA, expect.objectContaining({ cid: cid, ext: '.pdf', height: undefined, message: { channelAddress: 'channelAddress', id: 'id' }, name: 'test-file', size: 761797, width: undefined }))
expect(eventSpy).toHaveBeenNthCalledWith(4, StorageEvents.UPDATE_MESSAGE_MEDIA, expect.objectContaining({ cid: cid, ext: '.pdf', height: undefined, message: { channelAddress: 'channelAddress', id: 'id' }, name: 'test-file', size: 761991, width: undefined }))
})

// Downloading
Expand Down Expand Up @@ -277,17 +277,17 @@ describe('Ipfs file manager', () => {
}

await fileManager.uploadFile(metadata)
const cid = 'QmPWwAxgGofmXZF5RqKE4K8rVeL6oAuCnAfoR4CZWTkJ5T'
const cid = 'QmSaK2joeTBYukh8L7besrvm56wSzMhn64nqLqtvxS3ths'
expect(eventSpy).toHaveBeenNthCalledWith(1, StorageEvents.REMOVE_DOWNLOAD_STATUS, { cid: 'uploading_id' }
)

expect(eventSpy).toHaveBeenNthCalledWith(2, StorageEvents.UPLOADED_FILE, expect.objectContaining({ cid: cid, ext: '.png', height: 44, message: { channelAddress: 'channelAddress', id: 'id' }, name: 'test-image', size: 15847, width: 824 })
expect(eventSpy).toHaveBeenNthCalledWith(2, StorageEvents.UPLOADED_FILE, expect.objectContaining({ cid: cid, ext: '.png', height: 44, message: { channelAddress: 'channelAddress', id: 'id' }, name: 'test-image', size: 15858, width: 824 })
)

expect(eventSpy).toHaveBeenNthCalledWith(3, StorageEvents.UPDATE_DOWNLOAD_PROGRESS, { cid: cid, downloadProgress: undefined, downloadState: 'hosted', mid: 'id' }
)

expect(eventSpy).toHaveBeenNthCalledWith(4, StorageEvents.UPDATE_MESSAGE_MEDIA, expect.objectContaining({ cid: cid, ext: '.png', height: 44, message: { channelAddress: 'channelAddress', id: 'id' }, name: 'test-image', size: 15847, width: 824 })
expect(eventSpy).toHaveBeenNthCalledWith(4, StorageEvents.UPDATE_MESSAGE_MEDIA, expect.objectContaining({ cid: cid, ext: '.png', height: 44, message: { channelAddress: 'channelAddress', id: 'id' }, name: 'test-image', size: 15858, width: 824 })
)

// Downloading
Expand All @@ -301,14 +301,14 @@ describe('Ipfs file manager', () => {
})

await waitForExpect(() => {
expect(eventSpy).toHaveBeenNthCalledWith(6, StorageEvents.UPDATE_DOWNLOAD_PROGRESS, { cid: cid, downloadProgress: { downloaded: 15863, size: 15847, transferSpeed: 0 }, downloadState: 'downloading', mid: 'id' }
expect(eventSpy).toHaveBeenNthCalledWith(6, StorageEvents.UPDATE_DOWNLOAD_PROGRESS, { cid: cid, downloadProgress: { downloaded: 15855, size: 15858, transferSpeed: 0 }, downloadState: 'downloading', mid: 'id' }
)
}, 20000)
await waitForExpect(() => {
expect(eventSpy).toHaveBeenNthCalledWith(7, StorageEvents.UPDATE_DOWNLOAD_PROGRESS, { cid: cid, downloadProgress: { downloaded: 15863, size: 15847, transferSpeed: 0 }, downloadState: 'completed', mid: 'id' }
expect(eventSpy).toHaveBeenNthCalledWith(7, StorageEvents.UPDATE_DOWNLOAD_PROGRESS, { cid: cid, downloadProgress: { downloaded: 15855, size: 15858, transferSpeed: 0 }, downloadState: 'completed', mid: 'id' }
)
}, 20000)
expect(eventSpy).toHaveBeenNthCalledWith(8, StorageEvents.UPDATE_MESSAGE_MEDIA, expect.objectContaining({ cid: cid, ext: '.png', height: 44, message: { channelAddress: 'channelAddress', id: 'id' }, name: 'test-image', size: 15847, width: 824 })
expect(eventSpy).toHaveBeenNthCalledWith(8, StorageEvents.UPDATE_MESSAGE_MEDIA, expect.objectContaining({ cid: cid, ext: '.png', height: 44, message: { channelAddress: 'channelAddress', id: 'id' }, name: 'test-image', size: 15858, width: 824 })
)
})
it('downloaded file matches uploaded file', async () => {
Expand Down
110 changes: 41 additions & 69 deletions packages/backend/src/storage/ipfsFileManager.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,45 +98,24 @@ export class IpfsFilesManager extends EventEmitter {
console.error(`downloading ${mid} has already been canceled or never started`)
}
})
this.on(IpfsFilesManagerEvents.DELETE_FILE, async (fileMetadata: FileMetadata) => {
// Channel deletion WIP

// Check if we have it in case we didnt downloaded file
// await this.deleteBlocks(fileMetadata)
})
}

public async deleteBlocks(fileMetadata: FileMetadata) {
// console.log('deleting file in fileManager')
// const localBlocks = await this.getLocalBlocks()
// const hasBlockBeenDownloaded = localBlocks.includes(`z${fileMetadata.cid.toString()}`)
// console.log('has block been downlaoded ', hasBlockBeenDownloaded)
// if (!hasBlockBeenDownloaded) return

// const les = this.ipfs.pin.ls({paths: CID.parse(fileMetadata.cid)})
// for await (const l of les) {
// console.log('llllll', l)
// }

// try {
// const result = await this.ipfs.pin.rm(CID.parse(fileMetadata.cid), {recursive: true})
// console.log(
// 'unpinning result ', result
// )
// } catch (e) {
// console.log('file removing error')
// console.log(e)
// }
// const gcresult = this.ipfs.repo.gc()
// for await (const res of gcresult) {
// console.log('garbage collector result', res)
// }
// const blocks = this.ipfs.get(CID.parse(fileMetadata.cid))
// for await (const block of blocks) {
// // const decodedBlock = decode(block)
// console.log('bock', block)
// }
// Parse DAG blocks in case file is canceled mid download
const localBlocks = await this.getLocalBlocks()
const hasBlockBeenDownloaded = localBlocks.includes(`z${fileMetadata.cid.toString()}`)
if (!hasBlockBeenDownloaded) return

try {
const result = await this.ipfs.pin.rm(fileMetadata.cid, { recursive: true })
} catch (e) {
console.log('file removing error')
console.log(e)
}

const gcresult = this.ipfs.repo.gc()
for await (const res of gcresult) {
console.log('garbage collector result', res)
}
}

public async stop() {
Expand Down Expand Up @@ -171,7 +150,7 @@ export class IpfsFilesManager extends EventEmitter {
throw new Error(`File metadata (cid ${metadata.cid}) does not contain path`)
}
if (imagesExtensions.includes(metadata.ext)) {
let imageSize: {width: number | undefined; height: number | undefined} | undefined // ISizeCalculationResult
let imageSize: { width: number | undefined; height: number | undefined } | undefined // ISizeCalculationResult
try {
imageSize = await sizeOfPromisified(metadata.path)
} catch (e) {
Expand Down Expand Up @@ -202,42 +181,33 @@ export class IpfsFilesManager extends EventEmitter {
// Save copy to separate directory
const filePath = this.copyFile(metadata.path, filename)
console.time(`Writing ${filename} to ipfs`)
await this.ipfs.files.write(`/${dirname}/${filename}`, uploadedFileStreamIterable, {
create: true
})
const newCid = await this.ipfs.add(uploadedFileStreamIterable)

console.timeEnd(`Writing ${filename} to ipfs`)

// Get uploaded file information
const entries = this.ipfs.files.ls(`/${dirname}`)
for await (const entry of entries) {
if (entry.name === filename) {
this.emit(StorageEvents.REMOVE_DOWNLOAD_STATUS, { cid: metadata.cid })
await this.ipfs.pin.add(entry.cid)
const fileMetadata: FileMetadata = {
...metadata,
path: filePath,
cid: entry.cid.toString(),
size: entry.size,
width,
height
}
this.emit(StorageEvents.REMOVE_DOWNLOAD_STATUS, { cid: metadata.cid })
const fileMetadata: FileMetadata = {
...metadata,
path: filePath,
cid: newCid.cid.toString(),
size: newCid.size,
width,
height
}

this.emit(StorageEvents.UPLOADED_FILE, fileMetadata)
this.emit(StorageEvents.UPLOADED_FILE, fileMetadata)

const statusReady: DownloadStatus = {
mid: fileMetadata.message.id,
cid: fileMetadata.cid,
downloadState: DownloadState.Hosted,
downloadProgress: undefined
}
const statusReady: DownloadStatus = {
mid: fileMetadata.message.id,
cid: fileMetadata.cid,
downloadState: DownloadState.Hosted,
downloadProgress: undefined
}

this.emit(StorageEvents.UPDATE_DOWNLOAD_PROGRESS, statusReady)
this.emit(StorageEvents.UPDATE_DOWNLOAD_PROGRESS, statusReady)

if (metadata.path !== filePath) {
this.emit(StorageEvents.UPDATE_MESSAGE_MEDIA, fileMetadata)
}
break
}
if (metadata.path !== filePath) {
this.emit(StorageEvents.UPDATE_MESSAGE_MEDIA, fileMetadata)
}
}

Expand Down Expand Up @@ -272,6 +242,8 @@ export class IpfsFilesManager extends EventEmitter {
public downloadBlocks = async (fileMetadata: FileMetadata) => {
const block = CID.parse(fileMetadata.cid)

await this.ipfs.pin.add(block, { recursive: true })

const localBlocks = await this.getLocalBlocks()
const processedBlocks: PBNode[] = [] // TODO: Should it be CID or PBNode?

Expand Down Expand Up @@ -480,8 +452,8 @@ export class IpfsFilesManager extends EventEmitter {
private updateStatus = async (cid: string, downloadState = DownloadState.Downloading) => {
const metadata = this.files.get(cid)
if (!metadata) {
// TODO: emit error?
return
// TODO: emit error?
return
}
const progress: DownloadProgress | undefined = downloadState !== DownloadState.Malicious ? {
size: metadata.size,
Expand Down
31 changes: 20 additions & 11 deletions packages/backend/src/storage/storage.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ export class Storage extends EventEmitter {
this.filesManager = new IpfsFilesManager(this.ipfs, this.quietDir)
this.attachFileManagerEvents()

const channelsAccessController = createChannelAccessController(peerID)
const channelsAccessController = createChannelAccessController(peerID, this.orbitDbDir)

AccessControllers.addAccessController({ AccessController: MessagesAccessController })
AccessControllers.addAccessController({ AccessController: channelsAccessController })
Expand Down Expand Up @@ -286,7 +286,7 @@ export class Storage extends EventEmitter {
})

// @ts-expect-error - OrbitDB's type declaration of `load` lacks 'options'
await this.channels.load({ fetchEntryTimeout: 15000 })
await this.channels.load({ fetchEntryTimeout: 1000 })
log('ALL CHANNELS COUNT:', Object.keys(this.channels.all).length)
log('ALL CHANNELS COUNT:', Object.keys(this.channels.all))
Object.values(this.channels.all).forEach(async (channel: PublicChannel) => {
Expand Down Expand Up @@ -498,18 +498,18 @@ export class Storage extends EventEmitter {
this.publicChannelsRepos.set(data.address, { db, eventsAttached: false })
log(`Set ${data.address} to local channels`)
// @ts-expect-error - OrbitDB's type declaration of `load` lacks 'options'
await db.load({ fetchEntryTimeout: 2000 })
await db.load({ fetchEntryTimeout: 2000, })
log(`Created channel ${data.address}`)
return db
}

public async deleteChannel(payload: {channel: string}) {
public async deleteChannel(payload: { channel: string }) {
console.log('deleting channel storage', payload)
// @ts-expect-error - OrbitDB's type declaration of `load` lacks 'options'
await this.channels.load({ fetchEntryTimeout: 15000 })
const channel = this.channels.get(payload.channel)
if (channel) {
void this.channels.del(payload.channel)
await this.channels.del(payload.channel)
}
let repo = this.publicChannelsRepos.get(payload.channel)
if (!repo) {
Expand All @@ -535,8 +535,8 @@ export class Storage extends EventEmitter {
const files = allEntries.map((e) => {
return e.payload.value.media
}).filter(isDefined)
await this.deleteChannelFiles(files)
await this.deleteChannelMessages(hashes)
// await this.deleteChannelFiles(files)
// await this.deleteChannelMessages(hashes)
this.publicChannelsRepos.delete(payload.channel)
this.emit(StorageEvents.CHANNEL_DELETION_RESPONSE, payload)
}
Expand All @@ -552,11 +552,20 @@ export class Storage extends EventEmitter {
}

public async deleteChannelMessages(hashes: CID[]) {
for await (const result of this.ipfs.block.rm(hashes)) {
if (result.error) {
console.error(`Failed to remove block ${result.cid} due to ${result.error.message}`)
}
console.log('hashes ', hashes)
const gcresult = this.ipfs.repo.gc()
for await (const res of gcresult) {
// @ts-ignore
// const ccc = base58.base58btc.encode(res.cid?.multihash.bytes)

// console.log('base58btc encoded', ccc)
// console.log('garbage collector result', res)
}
// for await (const result of this.ipfs.block.rm(hashes)) {
// if (result.error) {
// console.error(`Failed to remove block ${result.cid} due to ${result.error.message}`)
// }
// }
}

public async sendMessage(message: ChannelMessage) {
Expand Down
Loading