Skip to content

Commit

Permalink
Merge pull request #1531 from TryQuiet/deletion/remove-ipfs-file-blocks
Browse files Browse the repository at this point in the history 
Deletion/remove ipfs file blocks
  • Loading branch information
@vinkabuki
vinkabuki authored May 24, 2023
2 parents 27e940b + 1acf317 commit 459b308
Show file tree
Hide file tree
Showing 5 changed files with 114 additions and 98 deletions.
26 changes: 22 additions & 4 deletions packages/backend/src/storage/ChannelsAccessController.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,21 +3,39 @@ import { getCrypto } from 'pkijs'
import { stringToArrayBuffer } from 'pvutils'
import { ChannelMessage, PublicChannel } from '@quiet/types'
import { keyObjectFromString, verifySignature } from '@quiet/identity'
import { IdentityProvider } from 'orbit-db-identity-provider'
import Identities from 'orbit-db-identity-provider'
import OrbitDB from 'orbit-db'
import PeerId from 'peer-id'

const Keystore = await import('orbit-db-keystore')

const type = 'channelsaccess'

export const createChannelAccessController = (peerId: PeerId) => {
export const createChannelAccessController = (peerId: PeerId, dir: string) => {
// @ts-ignore
class ChannelsAccessController extends AccessController {
static get type() {
return type
}

async canAppend(entry: LogEntry<PublicChannel>, identityProvider: IdentityProvider) {
// Channel deletion WIP
async canAppend(entry: LogEntry<PublicChannel>, identityProvider: any) {
// console.log('can append entry ', entry)

// const keystore = identityProvider._keystore

// const stringPeerId = 'QmTBYqK1qTXW9E6os3vc9phR9JWRs1jUTutgpLEGJVtubY'

// console.log('identityProvider', identityProvider)
// identityProvider.getId(peerId.toString())

// const identity = await Identities.createIdentity({
// id: stringPeerId,
// keystore
// })

// console.log('identity ', identity)

// console.log('access controller id ', identity)

// console.log('peerId ', peerId.toString())
// console.log('entry ', entry)
Expand Down
28 changes: 14 additions & 14 deletions packages/backend/src/storage/ipfsFileManager.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,12 +65,12 @@ describe('Ipfs file manager', () => {
const newFilePath = copyFileSpy.mock.results[0].value as string
metadata.path = newFilePath

const cid = 'QmPWwAxgGofmXZF5RqKE4K8rVeL6oAuCnAfoR4CZWTkJ5T'
const cid = 'QmSaK2joeTBYukh8L7besrvm56wSzMhn64nqLqtvxS3ths'
await waitForExpect(() => {
expect(eventSpy).toHaveBeenNthCalledWith(1, StorageEvents.REMOVE_DOWNLOAD_STATUS, { cid: 'uploading_id' })
})
await waitForExpect(() => {
expect(eventSpy).toHaveBeenNthCalledWith(2, StorageEvents.UPLOADED_FILE, expect.objectContaining({ cid: cid, ext: '.png', height: 44, message: { channelAddress: 'channelAddress', id: 'id' }, name: 'test-image', size: 15847, width: 824 })
expect(eventSpy).toHaveBeenNthCalledWith(2, StorageEvents.UPLOADED_FILE, expect.objectContaining({ cid: cid, ext: '.png', height: 44, message: { channelAddress: 'channelAddress', id: 'id' }, name: 'test-image', size: 15858, width: 824 })
)
})
await waitForExpect(() => {
Expand Down Expand Up @@ -98,20 +98,20 @@ describe('Ipfs file manager', () => {
}

await fileManager.uploadFile(metadata)
const cid = 'QmaA1C173ZDtoo7K6tLqq6o2eRce3kgwoVQpxsTfQgNjDZ'
const cid = 'QmR5NiFh2bTZCpdxZkYTMaceJFaYTPuxEt8J9BhKKdSv1o'
await waitForExpect(() => {
expect(eventSpy).toHaveBeenNthCalledWith(1, StorageEvents.REMOVE_DOWNLOAD_STATUS, { cid: 'uploading_id' })
})
await waitForExpect(() => {
expect(eventSpy).toHaveBeenNthCalledWith(2, StorageEvents.UPLOADED_FILE, expect.objectContaining({ cid: cid, ext: '.pdf', height: undefined, message: { channelAddress: 'channelAddress', id: 'id' }, name: 'test-file', size: 761797, width: undefined }
expect(eventSpy).toHaveBeenNthCalledWith(2, StorageEvents.UPLOADED_FILE, expect.objectContaining({ cid: cid, ext: '.pdf', height: undefined, message: { channelAddress: 'channelAddress', id: 'id' }, name: 'test-file', size: 761991, width: undefined }
)
)
})
await waitForExpect(() => {
expect(eventSpy).toHaveBeenNthCalledWith(3, StorageEvents.UPDATE_DOWNLOAD_PROGRESS, { cid: cid, downloadProgress: undefined, downloadState: 'hosted', mid: 'id' })
})
await waitForExpect(() => {
expect(eventSpy).toHaveBeenNthCalledWith(4, StorageEvents.UPDATE_MESSAGE_MEDIA, expect.objectContaining({ cid: cid, ext: '.pdf', height: undefined, message: { channelAddress: 'channelAddress', id: 'id' }, name: 'test-file', size: 761797, width: undefined })
expect(eventSpy).toHaveBeenNthCalledWith(4, StorageEvents.UPDATE_MESSAGE_MEDIA, expect.objectContaining({ cid: cid, ext: '.pdf', height: undefined, message: { channelAddress: 'channelAddress', id: 'id' }, name: 'test-file', size: 761991, width: undefined })
)
})
})
Expand Down Expand Up @@ -162,18 +162,18 @@ describe('Ipfs file manager', () => {
}

await fileManager.uploadFile(metadata)
const cid = 'QmaA1C173ZDtoo7K6tLqq6o2eRce3kgwoVQpxsTfQgNjDZ'
const cid = 'QmR5NiFh2bTZCpdxZkYTMaceJFaYTPuxEt8J9BhKKdSv1o'
await waitForExpect(() => {
expect(eventSpy).toHaveBeenNthCalledWith(1, StorageEvents.REMOVE_DOWNLOAD_STATUS, { cid: 'uploading_id' })
})
await waitForExpect(() => {
expect(eventSpy).toHaveBeenNthCalledWith(2, StorageEvents.UPLOADED_FILE, expect.objectContaining({ cid: cid, ext: '.pdf', height: undefined, message: { channelAddress: 'channelAddress', id: 'id' }, name: 'test-file', size: 761797, width: undefined }))
expect(eventSpy).toHaveBeenNthCalledWith(2, StorageEvents.UPLOADED_FILE, expect.objectContaining({ cid: cid, ext: '.pdf', height: undefined, message: { channelAddress: 'channelAddress', id: 'id' }, name: 'test-file', size: 761991, width: undefined }))
})
await waitForExpect(() => {
expect(eventSpy).toHaveBeenNthCalledWith(3, StorageEvents.UPDATE_DOWNLOAD_PROGRESS, { cid: cid, downloadProgress: undefined, downloadState: 'hosted', mid: 'id' })
})
await waitForExpect(() => {
expect(eventSpy).toHaveBeenNthCalledWith(4, StorageEvents.UPDATE_MESSAGE_MEDIA, expect.objectContaining({ cid: cid, ext: '.pdf', height: undefined, message: { channelAddress: 'channelAddress', id: 'id' }, name: 'test-file', size: 761797, width: undefined }))
expect(eventSpy).toHaveBeenNthCalledWith(4, StorageEvents.UPDATE_MESSAGE_MEDIA, expect.objectContaining({ cid: cid, ext: '.pdf', height: undefined, message: { channelAddress: 'channelAddress', id: 'id' }, name: 'test-file', size: 761991, width: undefined }))
})

// Downloading
Expand Down Expand Up @@ -277,17 +277,17 @@ describe('Ipfs file manager', () => {
}

await fileManager.uploadFile(metadata)
const cid = 'QmPWwAxgGofmXZF5RqKE4K8rVeL6oAuCnAfoR4CZWTkJ5T'
const cid = 'QmSaK2joeTBYukh8L7besrvm56wSzMhn64nqLqtvxS3ths'
expect(eventSpy).toHaveBeenNthCalledWith(1, StorageEvents.REMOVE_DOWNLOAD_STATUS, { cid: 'uploading_id' }
)

expect(eventSpy).toHaveBeenNthCalledWith(2, StorageEvents.UPLOADED_FILE, expect.objectContaining({ cid: cid, ext: '.png', height: 44, message: { channelAddress: 'channelAddress', id: 'id' }, name: 'test-image', size: 15847, width: 824 })
expect(eventSpy).toHaveBeenNthCalledWith(2, StorageEvents.UPLOADED_FILE, expect.objectContaining({ cid: cid, ext: '.png', height: 44, message: { channelAddress: 'channelAddress', id: 'id' }, name: 'test-image', size: 15858, width: 824 })
)

expect(eventSpy).toHaveBeenNthCalledWith(3, StorageEvents.UPDATE_DOWNLOAD_PROGRESS, { cid: cid, downloadProgress: undefined, downloadState: 'hosted', mid: 'id' }
)

expect(eventSpy).toHaveBeenNthCalledWith(4, StorageEvents.UPDATE_MESSAGE_MEDIA, expect.objectContaining({ cid: cid, ext: '.png', height: 44, message: { channelAddress: 'channelAddress', id: 'id' }, name: 'test-image', size: 15847, width: 824 })
expect(eventSpy).toHaveBeenNthCalledWith(4, StorageEvents.UPDATE_MESSAGE_MEDIA, expect.objectContaining({ cid: cid, ext: '.png', height: 44, message: { channelAddress: 'channelAddress', id: 'id' }, name: 'test-image', size: 15858, width: 824 })
)

// Downloading
Expand All @@ -301,14 +301,14 @@ describe('Ipfs file manager', () => {
})

await waitForExpect(() => {
expect(eventSpy).toHaveBeenNthCalledWith(6, StorageEvents.UPDATE_DOWNLOAD_PROGRESS, { cid: cid, downloadProgress: { downloaded: 15863, size: 15847, transferSpeed: 0 }, downloadState: 'downloading', mid: 'id' }
expect(eventSpy).toHaveBeenNthCalledWith(6, StorageEvents.UPDATE_DOWNLOAD_PROGRESS, { cid: cid, downloadProgress: { downloaded: 15855, size: 15858, transferSpeed: 0 }, downloadState: 'downloading', mid: 'id' }
)
}, 20000)
await waitForExpect(() => {
expect(eventSpy).toHaveBeenNthCalledWith(7, StorageEvents.UPDATE_DOWNLOAD_PROGRESS, { cid: cid, downloadProgress: { downloaded: 15863, size: 15847, transferSpeed: 0 }, downloadState: 'completed', mid: 'id' }
expect(eventSpy).toHaveBeenNthCalledWith(7, StorageEvents.UPDATE_DOWNLOAD_PROGRESS, { cid: cid, downloadProgress: { downloaded: 15855, size: 15858, transferSpeed: 0 }, downloadState: 'completed', mid: 'id' }
)
}, 20000)
expect(eventSpy).toHaveBeenNthCalledWith(8, StorageEvents.UPDATE_MESSAGE_MEDIA, expect.objectContaining({ cid: cid, ext: '.png', height: 44, message: { channelAddress: 'channelAddress', id: 'id' }, name: 'test-image', size: 15847, width: 824 })
expect(eventSpy).toHaveBeenNthCalledWith(8, StorageEvents.UPDATE_MESSAGE_MEDIA, expect.objectContaining({ cid: cid, ext: '.png', height: 44, message: { channelAddress: 'channelAddress', id: 'id' }, name: 'test-image', size: 15858, width: 824 })
)
})
it('downloaded file matches uploaded file', async () => {
Expand Down
110 changes: 41 additions & 69 deletions packages/backend/src/storage/ipfsFileManager.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,45 +98,24 @@ export class IpfsFilesManager extends EventEmitter {
console.error(`downloading ${mid} has already been canceled or never started`)
}
})
this.on(IpfsFilesManagerEvents.DELETE_FILE, async (fileMetadata: FileMetadata) => {
// Channel deletion WIP

// Check if we have it in case we didnt downloaded file
// await this.deleteBlocks(fileMetadata)
})
}

public async deleteBlocks(fileMetadata: FileMetadata) {
// console.log('deleting file in fileManager')
// const localBlocks = await this.getLocalBlocks()
// const hasBlockBeenDownloaded = localBlocks.includes(`z${fileMetadata.cid.toString()}`)
// console.log('has block been downlaoded ', hasBlockBeenDownloaded)
// if (!hasBlockBeenDownloaded) return

// const les = this.ipfs.pin.ls({paths: CID.parse(fileMetadata.cid)})
// for await (const l of les) {
// console.log('llllll', l)
// }

// try {
// const result = await this.ipfs.pin.rm(CID.parse(fileMetadata.cid), {recursive: true})
// console.log(
// 'unpinning result ', result
// )
// } catch (e) {
// console.log('file removing error')
// console.log(e)
// }
// const gcresult = this.ipfs.repo.gc()
// for await (const res of gcresult) {
// console.log('garbage collector result', res)
// }
// const blocks = this.ipfs.get(CID.parse(fileMetadata.cid))
// for await (const block of blocks) {
// // const decodedBlock = decode(block)
// console.log('bock', block)
// }
// Parse DAG blocks in case file is canceled mid download
const localBlocks = await this.getLocalBlocks()
const hasBlockBeenDownloaded = localBlocks.includes(`z${fileMetadata.cid.toString()}`)
if (!hasBlockBeenDownloaded) return

try {
const result = await this.ipfs.pin.rm(fileMetadata.cid, { recursive: true })
} catch (e) {
console.log('file removing error')
console.log(e)
}

const gcresult = this.ipfs.repo.gc()
for await (const res of gcresult) {
console.log('garbage collector result', res)
}
}

public async stop() {
Expand Down Expand Up @@ -171,7 +150,7 @@ export class IpfsFilesManager extends EventEmitter {
throw new Error(`File metadata (cid ${metadata.cid}) does not contain path`)
}
if (imagesExtensions.includes(metadata.ext)) {
let imageSize: {width: number | undefined; height: number | undefined} | undefined // ISizeCalculationResult
let imageSize: { width: number | undefined; height: number | undefined } | undefined // ISizeCalculationResult
try {
imageSize = await sizeOfPromisified(metadata.path)
} catch (e) {
Expand Down Expand Up @@ -202,42 +181,33 @@ export class IpfsFilesManager extends EventEmitter {
// Save copy to separate directory
const filePath = this.copyFile(metadata.path, filename)
console.time(`Writing ${filename} to ipfs`)
await this.ipfs.files.write(`/${dirname}/${filename}`, uploadedFileStreamIterable, {
create: true
})
const newCid = await this.ipfs.add(uploadedFileStreamIterable)

console.timeEnd(`Writing ${filename} to ipfs`)

// Get uploaded file information
const entries = this.ipfs.files.ls(`/${dirname}`)
for await (const entry of entries) {
if (entry.name === filename) {
this.emit(StorageEvents.REMOVE_DOWNLOAD_STATUS, { cid: metadata.cid })
await this.ipfs.pin.add(entry.cid)
const fileMetadata: FileMetadata = {
...metadata,
path: filePath,
cid: entry.cid.toString(),
size: entry.size,
width,
height
}
this.emit(StorageEvents.REMOVE_DOWNLOAD_STATUS, { cid: metadata.cid })
const fileMetadata: FileMetadata = {
...metadata,
path: filePath,
cid: newCid.cid.toString(),
size: newCid.size,
width,
height
}

this.emit(StorageEvents.UPLOADED_FILE, fileMetadata)
this.emit(StorageEvents.UPLOADED_FILE, fileMetadata)

const statusReady: DownloadStatus = {
mid: fileMetadata.message.id,
cid: fileMetadata.cid,
downloadState: DownloadState.Hosted,
downloadProgress: undefined
}
const statusReady: DownloadStatus = {
mid: fileMetadata.message.id,
cid: fileMetadata.cid,
downloadState: DownloadState.Hosted,
downloadProgress: undefined
}

this.emit(StorageEvents.UPDATE_DOWNLOAD_PROGRESS, statusReady)
this.emit(StorageEvents.UPDATE_DOWNLOAD_PROGRESS, statusReady)

if (metadata.path !== filePath) {
this.emit(StorageEvents.UPDATE_MESSAGE_MEDIA, fileMetadata)
}
break
}
if (metadata.path !== filePath) {
this.emit(StorageEvents.UPDATE_MESSAGE_MEDIA, fileMetadata)
}
}

Expand Down Expand Up @@ -272,6 +242,8 @@ export class IpfsFilesManager extends EventEmitter {
public downloadBlocks = async (fileMetadata: FileMetadata) => {
const block = CID.parse(fileMetadata.cid)

await this.ipfs.pin.add(block, { recursive: true })

const localBlocks = await this.getLocalBlocks()
const processedBlocks: PBNode[] = [] // TODO: Should it be CID or PBNode?

Expand Down Expand Up @@ -480,8 +452,8 @@ export class IpfsFilesManager extends EventEmitter {
private updateStatus = async (cid: string, downloadState = DownloadState.Downloading) => {
const metadata = this.files.get(cid)
if (!metadata) {
// TODO: emit error?
return
// TODO: emit error?
return
}
const progress: DownloadProgress | undefined = downloadState !== DownloadState.Malicious ? {
size: metadata.size,
Expand Down
31 changes: 20 additions & 11 deletions packages/backend/src/storage/storage.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ export class Storage extends EventEmitter {
this.filesManager = new IpfsFilesManager(this.ipfs, this.quietDir)
this.attachFileManagerEvents()

const channelsAccessController = createChannelAccessController(peerID)
const channelsAccessController = createChannelAccessController(peerID, this.orbitDbDir)

AccessControllers.addAccessController({ AccessController: MessagesAccessController })
AccessControllers.addAccessController({ AccessController: channelsAccessController })
Expand Down Expand Up @@ -286,7 +286,7 @@ export class Storage extends EventEmitter {
})

// @ts-expect-error - OrbitDB's type declaration of `load` lacks 'options'
await this.channels.load({ fetchEntryTimeout: 15000 })
await this.channels.load({ fetchEntryTimeout: 1000 })
log('ALL CHANNELS COUNT:', Object.keys(this.channels.all).length)
log('ALL CHANNELS COUNT:', Object.keys(this.channels.all))
Object.values(this.channels.all).forEach(async (channel: PublicChannel) => {
Expand Down Expand Up @@ -498,18 +498,18 @@ export class Storage extends EventEmitter {
this.publicChannelsRepos.set(data.address, { db, eventsAttached: false })
log(`Set ${data.address} to local channels`)
// @ts-expect-error - OrbitDB's type declaration of `load` lacks 'options'
await db.load({ fetchEntryTimeout: 2000 })
await db.load({ fetchEntryTimeout: 2000, })
log(`Created channel ${data.address}`)
return db
}

public async deleteChannel(payload: {channel: string}) {
public async deleteChannel(payload: { channel: string }) {
console.log('deleting channel storage', payload)
// @ts-expect-error - OrbitDB's type declaration of `load` lacks 'options'
await this.channels.load({ fetchEntryTimeout: 15000 })
const channel = this.channels.get(payload.channel)
if (channel) {
void this.channels.del(payload.channel)
await this.channels.del(payload.channel)
}
let repo = this.publicChannelsRepos.get(payload.channel)
if (!repo) {
Expand All @@ -535,8 +535,8 @@ export class Storage extends EventEmitter {
const files = allEntries.map((e) => {
return e.payload.value.media
}).filter(isDefined)
await this.deleteChannelFiles(files)
await this.deleteChannelMessages(hashes)
// await this.deleteChannelFiles(files)
// await this.deleteChannelMessages(hashes)
this.publicChannelsRepos.delete(payload.channel)
this.emit(StorageEvents.CHANNEL_DELETION_RESPONSE, payload)
}
Expand All @@ -552,11 +552,20 @@ export class Storage extends EventEmitter {
}

public async deleteChannelMessages(hashes: CID[]) {
for await (const result of this.ipfs.block.rm(hashes)) {
if (result.error) {
console.error(`Failed to remove block ${result.cid} due to ${result.error.message}`)
}
console.log('hashes ', hashes)
const gcresult = this.ipfs.repo.gc()
for await (const res of gcresult) {
// @ts-ignore
// const ccc = base58.base58btc.encode(res.cid?.multihash.bytes)

// console.log('base58btc encoded', ccc)
// console.log('garbage collector result', res)
}
// for await (const result of this.ipfs.block.rm(hashes)) {
// if (result.error) {
// console.error(`Failed to remove block ${result.cid} due to ${result.error.message}`)
// }
// }
}

public async sendMessage(message: ChannelMessage) {
Expand Down
Loading

0 comments on commit 459b308

Please sign in to comment.