Merge pull request #115 from lighthouse-web3/v0.3.5

ravish1729 · web-flow · commit 3a4f1929f259 · 2024-07-25T16:40:36.000+05:30
V0.3.5
diff --git a/README.md b/README.md
@@ -1,4 +1,4 @@
-# Lighthouse <img src="https://img.shields.io/badge/BETA-v0.3.4-green"/>
+# Lighthouse <img src="https://img.shields.io/badge/BETA-v0.3.5-green"/>
 
 Lighthouse is a permanent decentralized file storage protocol that allows the ability to pay once and store forever. While traditionally, users need to repeatedly keep track and pay for their storage after every fixed amount of time, Lighthouse manages this for them and makes sure that user files are stored forever. The aim is to move users from a rent-based cost model where they are renting their own files on cloud storage to a permanent ownership model. It is built on top of IPFS, Filecoin, and Polygon. It uses the existing miner network and storage capacity of the filecoin network.
 
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@lighthouse-web3/sdk",
-  "version": "0.3.4",
+  "version": "0.3.5",
   "description": "NPM package and CLI tool to interact with lighthouse protocol",
   "main": "./dist/Lighthouse/index.js",
   "types": "./dist/Lighthouse/index.d.ts",
diff --git a/src/Commands/index.ts b/src/Commands/index.ts
@@ -72,7 +72,7 @@ Command.prototype.helpInformation = function (context: any) {
 }
 
 widgets.addHelpText('before', 'Welcome to lighthouse-web3')
-widgets.version('0.3.4')
+widgets.version('0.3.5')
 
 widgets
   .command('wallet')
diff --git a/src/Lighthouse/upload/files/browser.ts b/src/Lighthouse/upload/files/browser.ts
@@ -5,7 +5,7 @@ import {
   UploadFileReturnType,
   DealParameters,
 } from '../../../types'
-import { checkDuplicateFileNames } from '../../utils/util'
+import { checkDuplicateFileNames, retryFetch } from '../../utils/util'
 
 // eslint-disable-next-line @typescript-eslint/no-empty-function
 export default async <T extends boolean>(
@@ -35,10 +35,11 @@ export default async <T extends boolean>(
         : 'null',
     })
 
-    const response = await fetch(endpoint, {
+    const response = await retryFetch(endpoint, {
       method: 'POST',
       body: formData,
       headers: headers,
+      timeout: 7200000,
     })
 
     if (!response.ok) {
diff --git a/src/Lighthouse/upload/files/node.ts b/src/Lighthouse/upload/files/node.ts
@@ -1,6 +1,7 @@
 import basePathConvert from '../../utils/basePathConvert'
 import { lighthouseConfig } from '../../../lighthouse.config'
 import { UploadFileReturnType, DealParameters } from '../../../types'
+import { retryFetch } from '../../utils/util'
 
 export async function walk(dir: string) {
   const { readdir, stat } = eval(`require`)('fs-extra')
@@ -39,18 +40,19 @@ export default async <T extends boolean>(
     if (stats.isFile()) {
       const data = new FormData()
       const stream = createReadStream(sourcePath)
-      const buffers = []
+      const buffers: Buffer[] = []
       for await (const chunk of stream) {
         buffers.push(chunk)
       }
       const blob = new Blob(buffers)
 
-      data.set('file', blob, path.basename(sourcePath))
+      data.append('file', blob, path.basename(sourcePath))
 
-      const response = await fetch(endpoint, {
+      const response = await retryFetch(endpoint, {
         method: 'POST',
         body: data,
         credentials: 'include',
+        timeout: 7200000,
         headers: {
           Encryption: 'false',
           Authorization: token,
@@ -65,7 +67,6 @@ export default async <T extends boolean>(
       }
 
       let responseData = (await response.text()) as any
-      console.log(responseData)
       if (multi) {
         const temp = responseData.split('\n')
         responseData = JSON.parse(temp[temp.length - 2])
@@ -80,23 +81,24 @@ export default async <T extends boolean>(
 
       for (const file of files) {
         const stream = createReadStream(file)
-        const buffers: any = []
+        const buffers: Buffer[] = []
         for await (const chunk of stream) {
           buffers.push(chunk)
         }
         const blob = new Blob(buffers)
 
-        data.set(
+        data.append(
           'file',
           blob,
           multi ? path.basename(file) : basePathConvert(sourcePath, file)
         )
       }
 
-      const response = await fetch(endpoint, {
+      const response = await retryFetch(endpoint, {
         method: 'POST',
         body: data,
         credentials: 'include',
+        timeout: 7200000,
         headers: {
           Encryption: 'false',
           Authorization: token,
diff --git a/src/Lighthouse/upload/text/browser.ts b/src/Lighthouse/upload/text/browser.ts
@@ -1,4 +1,5 @@
 import { lighthouseConfig } from '../../../lighthouse.config'
+import { retryFetch } from '../../utils/util'
 
 export default async (text: string, apiKey: string, name: string) => {
   try {
@@ -8,11 +9,12 @@ export default async (text: string, apiKey: string, name: string) => {
     // Upload file
     const formData = new FormData()
     const blob = new Blob([text], { type: 'text/plain' })
-    formData.set('file', blob, name)
+    formData.append('file', blob, name)
 
-    const response = await fetch(endpoint, {
+    const response = await retryFetch(endpoint, {
       method: 'POST',
       body: formData,
+      timeout: 7200000,
       headers: {
         Encryption: 'false',
         'Mime-Type': 'text/plain',
diff --git a/src/Lighthouse/upload/text/node.ts b/src/Lighthouse/upload/text/node.ts
@@ -1,4 +1,5 @@
 import { lighthouseConfig } from '../../../lighthouse.config'
+import { retryFetch } from '../../utils/util'
 
 export default async (text: string, apiKey: string, name: string) => {
   try {
@@ -9,12 +10,13 @@ export default async (text: string, apiKey: string, name: string) => {
     const formData = new FormData()
     const blob = new Blob([Buffer.from(text)])
 
-    formData.set('file', blob, name)
+    formData.append('file', blob, name)
 
-    const response = await fetch(endpoint, {
+    const response = await retryFetch(endpoint, {
       method: 'POST',
       body: formData,
       credentials: 'include',
+      timeout: 7200000,
       headers: {
         Encryption: 'false',
         'Mime-Type': 'text/plain',
diff --git a/src/Lighthouse/uploadEncrypted/decrypt/browser.ts b/src/Lighthouse/uploadEncrypted/decrypt/browser.ts
@@ -1,16 +1,18 @@
 /* istanbul ignore file */
 import { decryptFile } from '../encryptionBrowser'
 import { lighthouseConfig } from '../../../lighthouse.config'
+import { retryFetch } from '../../utils/util'
 
 export default async (
   cid: string,
   fileEncryptionKey: string,
   mimeType: string
 ) => {
-  const response = await fetch(
+  const response = await retryFetch(
     lighthouseConfig.lighthouseGateway + '/api/v0/cat/' + cid,
     {
       method: 'POST',
+      timeout: 7200000,
     }
   )
 
diff --git a/src/Lighthouse/uploadEncrypted/decrypt/node.ts b/src/Lighthouse/uploadEncrypted/decrypt/node.ts
@@ -1,13 +1,15 @@
 /* istanbul ignore file */
 import { decryptFile } from '../encryptionNode'
 import { lighthouseConfig } from '../../../lighthouse.config'
+import { retryFetch } from '../../utils/util'
 
 export default async (cid: string, fileEncryptionKey: any) => {
   try {
-    const response = await fetch(
+    const response = await retryFetch(
       lighthouseConfig.lighthouseGateway + '/api/v0/cat/' + cid,
       {
         method: 'POST',
+        timeout: 7200000,
       }
     )
 
diff --git a/src/Lighthouse/uploadEncrypted/encrypt/file/browser.ts b/src/Lighthouse/uploadEncrypted/encrypt/file/browser.ts
@@ -6,7 +6,7 @@ import {
 } from '../../../../types'
 import { encryptFile } from '../../encryptionBrowser'
 import { lighthouseConfig } from '../../../../lighthouse.config'
-import { checkDuplicateFileNames } from '../../../utils/util'
+import { checkDuplicateFileNames, retryFetch } from '../../../utils/util'
 
 declare const FileReader: any
 
@@ -66,7 +66,7 @@ export default async (
       })
     )
     filesParam.forEach(function (item_) {
-      return formData.set(
+      return formData.append(
         'file',
         item_.data,
         item_.fileName ? item_.fileName : 'file'
@@ -75,17 +75,16 @@ export default async (
 
     const controller = new AbortController()
     const signal = controller.signal
-
-    const response = await fetch(endpoint, {
+    const response = await retryFetch(endpoint, {
       method: 'POST',
       body: formData,
+      timeout: 7200000,
       headers: {
         Encryption: `${true}`,
         Authorization: token,
       },
       signal,
     })
-
     if (!response.ok) {
       throw new Error(`HTTP error! status: ${response.status}`)
     }
diff --git a/src/Lighthouse/uploadEncrypted/encrypt/file/node.ts b/src/Lighthouse/uploadEncrypted/encrypt/file/node.ts
@@ -3,6 +3,7 @@ import { generate, saveShards } from '@lighthouse-web3/kavach'
 import { encryptFile } from '../../encryptionNode'
 import { walk } from '../../../upload/files/node'
 import { IFileUploadedResponse } from '../../../../types'
+import { retryFetch } from '../../../utils/util'
 
 export default async (
   sourcePath: any,
@@ -25,11 +26,12 @@ export default async (
       const fileData = fs.readFileSync(sourcePath)
       const encryptedData = await encryptFile(fileData, fileEncryptionKey)
       const blob = new Blob([Buffer.from(encryptedData)])
-      formData.set('file', blob, sourcePath.replace(/^.*[\\/]/, ''))
+      formData.append('file', blob, sourcePath.replace(/^.*[\\/]/, ''))
 
-      const response = await fetch(endpoint, {
+      const response = await retryFetch(endpoint, {
         method: 'POST',
         body: formData,
+        timeout: 7200000,
         headers: {
           Encryption: 'true',
           Authorization: token,
@@ -79,9 +81,10 @@ export default async (
       })
     )
 
-    const response = await fetch(endpoint, {
+    const response = await retryFetch(endpoint, {
       method: 'POST',
       body: formData,
+      timeout: 7200000,
       headers: {
         Encryption: 'true',
         Authorization: token,
diff --git a/src/Lighthouse/uploadEncrypted/encrypt/text/browser.ts b/src/Lighthouse/uploadEncrypted/encrypt/text/browser.ts
@@ -2,6 +2,7 @@
 import { encryptFile } from '../../encryptionBrowser'
 import { generate, saveShards } from '@lighthouse-web3/kavach'
 import { lighthouseConfig } from '../../../../lighthouse.config'
+import { retryFetch } from '../../../utils/util'
 
 export default async (
   text: string,
@@ -25,15 +26,16 @@ export default async (
       fileEncryptionKey
     )
 
-    formData.set(
+    formData.append(
       'file',
       new Blob([encryptedData], { type: 'text/plain' }),
       name
     )
 
-    const response = await fetch(endpoint, {
+    const response = await retryFetch(endpoint, {
       method: 'POST',
       body: formData,
+      timeout: 7200000,
       headers: {
         Encryption: 'true',
         'Mime-Type': 'text/plain',
diff --git a/src/Lighthouse/uploadEncrypted/encrypt/text/node.ts b/src/Lighthouse/uploadEncrypted/encrypt/text/node.ts
@@ -2,6 +2,7 @@
 import { encryptFile } from '../../encryptionNode'
 import { generate, saveShards } from '@lighthouse-web3/kavach'
 import { lighthouseConfig } from '../../../../lighthouse.config'
+import { retryFetch } from '../../../utils/util'
 
 export default async (
   text: string,
@@ -24,12 +25,13 @@ export default async (
       fileEncryptionKey
     )
     const blob = new Blob([Buffer.from(encryptedData)])
-    formData.set('file', blob, name)
+    formData.append('file', blob, name)
 
-    const response = await fetch(endpoint, {
+    const response = await retryFetch(endpoint, {
       method: 'POST',
       body: formData,
       credentials: 'include',
+      timeout: 7200000,
       headers: {
         Encryption: 'true',
         'Mime-Type': 'text/plain',
diff --git a/src/Lighthouse/utils/util.ts b/src/Lighthouse/utils/util.ts
@@ -1,5 +1,9 @@
 import { ethers } from 'ethers'
 
+interface FetchOptions extends RequestInit {
+  timeout?: number
+}
+
 const isCID = (cid: string) => {
   return /^(Qm[1-9A-HJ-NP-Za-km-z]{44}|b[A-Za-z2-7]{58}|B[A-Z2-7]{58}|z[1-9A-HJ-NP-Za-km-z]{48}|F[0-9A-F]{50})*$/.test(
     cid
@@ -33,4 +37,49 @@ function checkDuplicateFileNames(files: any[]) {
   }
 }
 
-export { isCID, isPrivateKey, addressValidator, checkDuplicateFileNames }
+async function fetchWithTimeout(
+  resource: string,
+  options: FetchOptions
+): Promise<Response> {
+  const { timeout = 8000, ...rest } = options
+
+  const controller = new AbortController()
+  const id = setTimeout(() => controller.abort(), timeout)
+  try {
+    const response = await fetch(resource, {
+      ...rest,
+      signal: controller.signal,
+    })
+    clearTimeout(id)
+    return response
+  } catch (error) {
+    clearTimeout(id)
+    throw error
+  }
+}
+
+async function retryFetch(
+  resource: string,
+  options: FetchOptions,
+  retries = 3
+): Promise<Response> {
+  for (let i = 0; i < retries; i++) {
+    try {
+      return await fetchWithTimeout(resource, options)
+    } catch (error) {
+      if (i < retries - 1) {
+      } else {
+        throw error
+      }
+    }
+  }
+  throw new Error('fetch failed')
+}
+
+export {
+  isCID,
+  isPrivateKey,
+  addressValidator,
+  checkDuplicateFileNames,
+  retryFetch,
+}

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-# Lighthouse <img src="https://img.shields.io/badge/BETA-v0.3.4-green"/>`
	`1`	`+# Lighthouse <img src="https://img.shields.io/badge/BETA-v0.3.5-green"/>`
`2`	`2`
`3`	`3`	Lighthouse is a permanent decentralized file storage protocol that allows the ability to pay once and store forever. While traditionally, users need to repeatedly keep track and pay for their storage after every fixed amount of time, Lighthouse manages this for them and makes sure that user files are stored forever. The aim is to move users from a rent-based cost model where they are renting their own files on cloud storage to a permanent ownership model. It is built on top of IPFS, Filecoin, and Polygon. It uses the existing miner network and storage capacity of the filecoin network.
`4`	`4`
Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`{`
`2`	`2`	`"name": "@lighthouse-web3/sdk",`
`3`		`- "version": "0.3.4",`
	`3`	`+ "version": "0.3.5",`
`4`	`4`	`"description": "NPM package and CLI tool to interact with lighthouse protocol",`
`5`	`5`	`"main": "./dist/Lighthouse/index.js",`
`6`	`6`	`"types": "./dist/Lighthouse/index.d.ts",`
Original file line number	Diff line number	Diff line change
`@@ -72,7 +72,7 @@ Command.prototype.helpInformation = function (context: any) {`
`72`	`72`	`}`
`73`	`73`
`74`	`74`	`widgets.addHelpText('before', 'Welcome to lighthouse-web3')`
`75`		`-widgets.version('0.3.4')`
	`75`	`+widgets.version('0.3.5')`
`76`	`76`
`77`	`77`	`widgets`
`78`	`78`	`.command('wallet')`
Original file line number	Diff line number	Diff line change
`@@ -1,16 +1,18 @@`
`1`	`1`	`/* istanbul ignore file */`
`2`	`2`	`import { decryptFile } from '../encryptionBrowser'`
`3`	`3`	`import { lighthouseConfig } from '../../../lighthouse.config'`
	`4`	`+import { retryFetch } from '../../utils/util'`
`4`	`5`
`5`	`6`	`export default async (`
`6`	`7`	`cid: string,`
`7`	`8`	`fileEncryptionKey: string,`
`8`	`9`	`mimeType: string`
`9`	`10`	`) => {`
`10`		`- const response = await fetch(`
	`11`	`+ const response = await retryFetch(`
`11`	`12`	`lighthouseConfig.lighthouseGateway + '/api/v0/cat/' + cid,`
`12`	`13`	`{`
`13`	`14`	`method: 'POST',`
	`15`	`+ timeout: 7200000,`
`14`	`16`	`}`
`15`	`17`	`)`
`16`	`18`
Original file line number	Diff line number	Diff line change
`@@ -1,13 +1,15 @@`
`1`	`1`	`/* istanbul ignore file */`
`2`	`2`	`import { decryptFile } from '../encryptionNode'`
`3`	`3`	`import { lighthouseConfig } from '../../../lighthouse.config'`
	`4`	`+import { retryFetch } from '../../utils/util'`
`4`	`5`
`5`	`6`	`export default async (cid: string, fileEncryptionKey: any) => {`
`6`	`7`	`try {`
`7`		`- const response = await fetch(`
	`8`	`+ const response = await retryFetch(`
`8`	`9`	`lighthouseConfig.lighthouseGateway + '/api/v0/cat/' + cid,`
`9`	`10`	`{`
`10`	`11`	`method: 'POST',`
	`12`	`+ timeout: 7200000,`
`11`	`13`	`}`
`12`	`14`	`)`
`13`	`15`