From ef6f04e19f4d3c4e97bf335f5339cfeb3b6c15a2 Mon Sep 17 00:00:00 2001 From: Alex Date: Mon, 20 Nov 2023 12:08:49 -0500 Subject: [PATCH 1/4] add websocket methods (#6479) * add getPendingRequestQueueSize * add safe disconnect * add getSentRequestsQueueSize and safeDisconnect * fix test * update test * eslint disable * fix * update import * test * add script * add eslint disable * Revert "add script" This reverts commit c80e25658a88ff38bf0806fe2997fc0b814935dc. * add try catch: * remove afterall * lint fix * remove failing testcase * fix up * add eslint disable * add clearqueues as public method * lint fix * address feedback * fix lint * update changelog --------- Co-authored-by: Muhammad Altabba <24407834+Muhammad-Altabba@users.noreply.github.com> --- packages/web3-utils/CHANGELOG.md | 4 +- packages/web3-utils/src/socket_provider.ts | 50 +++++++++++++++++++ .../test/unit/socket_provider.test.ts | 5 +- packages/web3/test/integration/ws.test.ts | 43 ++++++++++++++++ 4 files changed, 100 insertions(+), 2 deletions(-) create mode 100644 packages/web3/test/integration/ws.test.ts diff --git a/packages/web3-utils/CHANGELOG.md b/packages/web3-utils/CHANGELOG.md index e19848dac09..7a09eec979a 100644 --- a/packages/web3-utils/CHANGELOG.md +++ b/packages/web3-utils/CHANGELOG.md @@ -173,4 +173,6 @@ Documentation: ### Added -- Add `isContractInitOptions` method (#6455) \ No newline at end of file +- `SocketProvider` now contains public function `getPendingRequestQueueSize`, `getSentRequestsQueueSize` and `clearQueues` (#6479) +- Added `safeDisconnect` as a `SocketProvider` method to disconnect only when request queue size and send request queue size is 0 (#6479) +- Add `isContractInitOptions` method (#6555) diff --git a/packages/web3-utils/src/socket_provider.ts b/packages/web3-utils/src/socket_provider.ts index 380c230bd72..3c879a2df84 100644 --- a/packages/web3-utils/src/socket_provider.ts +++ b/packages/web3-utils/src/socket_provider.ts @@ -188,6 +188,24 @@ export abstract class SocketProvider< protected _validateProviderPath(path: string): boolean { return !!path; } + + /** + * + * @returns the pendingRequestQueue size + */ + // eslint-disable-next-line class-methods-use-this + public getPendingRequestQueueSize() { + return this._pendingRequestsQueue.size; + } + + /** + * + * @returns the sendPendingRequests size + */ + // eslint-disable-next-line class-methods-use-this + public getSentRequestsQueueSize() { + return this._sentRequestsQueue.size; + } /** * @@ -331,6 +349,34 @@ export abstract class SocketProvider< this._onDisconnect(disconnectCode, data); } + /** + * Safely disconnects the socket, async and waits for request size to be 0 before disconnecting + * @param forceDisconnect - If true, will clear queue after 5 attempts of waiting for both pending and sent queue to be 0 + * @param ms - Determines the ms of setInterval + * @param code - The code to be sent to the server + * @param data - The data to be sent to the server + */ + public async safeDisconnect(code?: number, data?: string, forceDisconnect = false,ms = 1000) { + let retryAttempt = 0; + const checkQueue = async () => + new Promise(resolve => { + const interval = setInterval(() => { + if (forceDisconnect && retryAttempt === 5) { + this.clearQueues(); + } + if (this.getPendingRequestQueueSize() === 0 && this.getSentRequestsQueueSize() === 0) { + clearInterval(interval); + resolve(true); + } + retryAttempt+=1; + }, ms) + }) + + await checkQueue(); + this.disconnect(code, data); + } + + /** * Removes all listeners for the specified event type. * @param type - The event type to remove the listeners for @@ -500,6 +546,10 @@ export abstract class SocketProvider< this._sentRequestsQueue.delete(requestId); } } + + public clearQueues(event?: ConnectionEvent) { + this._clearQueues(event); + } protected _clearQueues(event?: ConnectionEvent) { if (this._pendingRequestsQueue.size > 0) { diff --git a/packages/web3-utils/test/unit/socket_provider.test.ts b/packages/web3-utils/test/unit/socket_provider.test.ts index 9c20df7fdd3..b4b46221074 100644 --- a/packages/web3-utils/test/unit/socket_provider.test.ts +++ b/packages/web3-utils/test/unit/socket_provider.test.ts @@ -312,7 +312,7 @@ describe('SocketProvider', () => { }); // @ts-expect-error run protected method expect(provider._pendingRequestsQueue.size).toBe(1); - + expect(provider.getPendingRequestQueueSize()).toBe(1); const payload2 = { id: 2, method: 'some_rpc_method' }; provider.setStatus('connected'); const req2 = provider.request(payload2); @@ -323,6 +323,7 @@ describe('SocketProvider', () => { // @ts-expect-error run protected method expect(provider._sentRequestsQueue.size).toBe(1); + expect(provider.getSentRequestsQueueSize).toBe(1); provider.on('error', () => { // nothing @@ -331,8 +332,10 @@ describe('SocketProvider', () => { provider._clearQueues(); // @ts-expect-error run protected method expect(provider._pendingRequestsQueue.size).toBe(0); + expect(provider.getPendingRequestQueueSize()).toBe(0); // @ts-expect-error run protected method expect(provider._sentRequestsQueue.size).toBe(0); + expect(provider.getSentRequestsQueueSize).toBe(0); }); }); }); diff --git a/packages/web3/test/integration/ws.test.ts b/packages/web3/test/integration/ws.test.ts new file mode 100644 index 00000000000..7a11403fb32 --- /dev/null +++ b/packages/web3/test/integration/ws.test.ts @@ -0,0 +1,43 @@ +/* +This file is part of web3.js. + +web3.js is free software: you can redistribute it and/or modify +it under the terms of the GNU Lesser General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +web3.js is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public License +along with web3.js. If not, see . +*/ + +import { WebSocketProvider } from 'web3-providers-ws'; +import { + describeIf, getSystemTestProvider, isWs, +} from '../shared_fixtures/system_tests_utils'; +import Web3 from '../../src/index'; + + + +describe('Web3 instance', () => { + let web3: Web3; + + beforeEach(() => { + const provider = getSystemTestProvider(); + web3 = new Web3(provider); + }); + + describeIf(isWs)('web3 ws tests', () => { + it('should connect and disconnect using safe disconnect subscription successfully', async () => { + const subscription = await web3.eth.subscribe("newBlockHeaders"); + // eslint-disable-next-line + subscription.unsubscribe(); + // eslint-disable-next-line + await expect((web3.currentProvider as WebSocketProvider).safeDisconnect()).resolves.not.toThrow(); + }); + }); +}); \ No newline at end of file From be86e259ddcc735da00a3715b004e1ba0bc0d941 Mon Sep 17 00:00:00 2001 From: Muhammad Altabba <24407834+Muhammad-Altabba@users.noreply.github.com> Date: Mon, 20 Nov 2023 18:13:41 +0100 Subject: [PATCH 2/4] Write a migration guide from ethers to web3.js (#6603) * add `Migration from ethers.js` --- .../migration_from_other_libs/_category_.yml | 4 + .../migration_from_other_libs/ethers.md | 476 ++++++++++++++++++ 2 files changed, 480 insertions(+) create mode 100644 docs/docs/guides/migration_from_other_libs/_category_.yml create mode 100644 docs/docs/guides/migration_from_other_libs/ethers.md diff --git a/docs/docs/guides/migration_from_other_libs/_category_.yml b/docs/docs/guides/migration_from_other_libs/_category_.yml new file mode 100644 index 00000000000..ee8c672520c --- /dev/null +++ b/docs/docs/guides/migration_from_other_libs/_category_.yml @@ -0,0 +1,4 @@ +label: 'Migration from Other Libraries' +collapsible: true +collapsed: false +link: null \ No newline at end of file diff --git a/docs/docs/guides/migration_from_other_libs/ethers.md b/docs/docs/guides/migration_from_other_libs/ethers.md new file mode 100644 index 00000000000..96dfa9e5943 --- /dev/null +++ b/docs/docs/guides/migration_from_other_libs/ethers.md @@ -0,0 +1,476 @@ +--- +sidebar_position: 1 +sidebar_label: 'Migration from ethers.js' +title: 'Migration from ethers.js' +--- + +Follow this guide, if you're currently using the ethers.js library to interact with the Ethereum blockchain and want to migrate to web3.js. This guide is for ethers v5 and v6. And, if there are differences, code for both would be provided. And, if you find something missing, or worth adding, feel free to open a PR, please. + +However, migrating from a library to another would usually need careful changes. But, ethers.js have lots of similarities with web3.js and migration would usually be easy and straightforward. However, you still need to check your code for possible tweaks as needed. + +## Installation + +First, install the latest version of web3.js: + +```bash +npm install web3 +``` + +## Providers + +### Initialization and Calling RPC Methods + +With ethers.js, you would get the last block number from a provider like this: + +```typescript +import { ethers } from 'ethers'; + +// in v5: +const provider = new ethers.providers.JsonRpcProvider(url); + +// in v6: +const provider = new ethers.JsonRpcProvider(url); + +const blockNumber = provider.getBlockNumber(); + +// outputs something like: 18561956 +blockNumber.then(console.log); +``` + +With web3.js, you would get the last block number from a provider like this: + +```typescript +import { Web3 } from 'web3'; + +const web3 = new Web3(url); +const blockNumber = web3.eth.getBlockNumber(); + +// outputs something like: 18561956n +blockNumber.then(console.log); +``` + +:::tip +📝 web3.js uses `bigint` as the default type for all big numbers returned. For, this you see above the blocknumber has the `n` at its end (`18561956n`). However, you can change the returned type by passing an optional parameter like: +```ts +import { Web3, DEFAULT_RETURN_FORMAT, FMT_NUMBER } from 'web3'; + +const blockNumber = web3.eth.getBlockNumber({ + ...DEFAULT_RETURN_FORMAT, + number: FMT_NUMBER.HEX, // to get the block number in hex format +}); +// outputs something like: 0x11B3BA4 +blockNumber.then(console.log); + + +const blockNumber = web3.eth.getBlockNumber({ + ...DEFAULT_RETURN_FORMAT, + number: FMT_NUMBER.STR, // to get the block number as a string +}); +// the value would like: '18561956' +blockNumber.then(console.log); +``` +::: + +### Use browser-injected provider + +With ethers.js: + +```typescript +// in v5 +const provider = new ethers.providers.Web3Provider(window.ethereum); + +// in v6 +const provider = new ethers.BrowserProvider(window.ethereum); +``` + +With web3.js: + +```typescript +const web3 = new Web3(window.ethereum); +``` + + +## Wallets and Accounts + +### Generate Private Key + +With ethers.js: + +```typescript +// this would generate a private key similar to: +// '0x286f65c4191759fc5c7e6083b8c275ac2238cc7abb5915bd8c905ae4404215c9' +// (Be sure to store it encrypted in a safe place) +const privateKey = ethers.Wallet.createRandom().privateKey; +``` + +With web3.js: + +```typescript +// this would generate a private key similar to: +// '0x286f65c4191759fc5c7e6083b8c275ac2238cc7abb5915bd8c905ae4404215c9' +// (Be sure to store it encrypted in a safe place) +const privateKey = web3.eth.accounts.create().privateKey; +``` +### Create a wallet + +In ethers.js: + +```typescript +const wallet = new ethers.Wallet( + // A private key that you might had generated with: + // ethers.Wallet.createRandom().privateKey + privateKey, +); + +// outputs: 0x6f7D735dFB514AA1778E8D97EaCE72BfECE71865 +console.log(wallet.address); +``` + +With web3.js: + +```typescript +const web3 = new Web3(); +const wallet = web3.eth.accounts.wallet.add( + // you can generate a private key using web3.eth.accounts.create().privateKey + privateKey, +); + +// outputs: 0x6f7D735dFB514AA1778E8D97EaCE72BfECE71865 +console.log(wallet[0].address); +``` + +:::info + In web3.js, if you want to use a private key to later sign and send transactions, you first need to add this private key to the accounts with, for example, one of the methods: + `web3.eth.accounts.create()`, or `web3.eth.accounts.wallet.add(privateKey)`. + + And then whenever you provide the public address of that private key, web3.js will use that private key to sign. For example, you would pass the public key at `web3.eth.sendTransaction({from: publicAddress,...})` and web3.`eth.signTransaction({from: publicAddress,...})` then the privateKey of that publicAddress will be lookup and used to sign. + + However, it is not advised to use the privatekey directly. And you are advised to use a secret storage or a vault instead. +::: + +### Get unlocked account + +With ethers.js: + +```typescript +const signer = await provider.getSigner(); +``` + +With web3.js: + +```typescript +const account = (await web3.eth.getAccounts())[0]; +``` + + +### Signing a string message + +with ethers.js: + +```typescript +const signer = new ethers.Wallet(privateKey); + +const signature = await signer.signMessage('Some data'); +// Outputs something like: +// 0xb475e02218d7d6a16f3575de789996d0a57f900f240d73ed792672256d63913840c1da0dd3e7fe2e79485b7a1d81e8cc163f405c3df22d496f28f1dd148faebf1b +console.log(signature); + +``` + +With web3.js: + +```typescript + +// Sign with web3.js, using a private key: +const signature = web3.eth.accounts.sign('Some data', privateKey).signature; + +// Outputs something like: +// 0xb475e02218d7d6a16f3575de789996d0a57f900f240d73ed792672256d63913840c1da0dd3e7fe2e79485b7a1d81e8cc163f405c3df22d496f28f1dd148faebf1b +console.log(signature); + +// Sign using an account managed by the connected provider (for example the RPC client or a browser-injected provider) +const signature = await web3.eth.sign( + web3.utils.utf8ToHex('Some data'), // data to be signed (4.x only supports Hex Strings) + '0x6E599DA0bfF7A6598AC1224E4985430Bf16458a4', // the address that its private key would be used to sign +); + +// Outputs something like: +// 0xb475e02218d7d6a16f3575de789996d0a57f900f240d73ed792672256d63913840c1da0dd3e7fe2e79485b7a1d81e8cc163f405c3df22d496f28f1dd148faebf1b +console.log(signature); +``` + +## Signing and Sending Transactions + +### Sending Transactions + +Sending a transaction with ethers.js: + +```typescript +const signer = new ethers.Wallet(privateKey, provider); + +const tx = await signer.sendTransaction({ + to: '0x92d3267215Ec56542b985473E73C8417403B15ac', + value: ethers.parseUnits('0.001', 'ether'), +}); +console.log(tx); +``` + +With web3.js: + +:::info +The method `web3.eth.sendTransaction` will use the account that you pass the public address at `from` to sign the transaction. + +So, the `from` needs to be the public address of a private key that you added previously to the web3.eth.accounts. Or, else, it would pass it to the provider where an unlocked account would be used. + +And for the case when you did not add the private key early, and so the `from` was just passed to the provider. Then if the provider was a browser-injected provider like metamask, for example, it will ask the user to sign. And, if you are using a local dev node as a provider, it should be one of the accounts that were already unlocked at the node. However, note that it is highly risky and not recommended to unlock an account at a production or even a test node. +::: + +```typescript +const web3 = new Web3(url); + +// The method web3.eth.sendTransaction is helpful if you are using a browser-injected provider like metamask. +// Or, if you are using a local dev node like ganache; and you have some accounts already unlocked at the node. +// And this is how you would get the first unlocked account from a local node (not advised for production or even on test node to use unlock accounts on the node). +const account = (await web3.eth.getAccounts())[0]; + +// Alternative to the above, here is how to add wallet to be used as a signer later: +const wallet = web3.eth.accounts.wallet.add(privateKey); +const account = wallet[0].address; + +const tx = await web3.eth.sendTransaction({ + from: account, + to: '0x92d3267215Ec56542b985473E73C8417403B15ac', + value: web3.utils.toWei('0.00000000001', 'ether'), +}); +console.log(tx); +``` + +### Sending a Signed Transactions + +Posting a signed transaction to the node with ethers.js: + +```typescript +// in v5 +provider.sendTransaction(signedTx) + +// in v6 +provider.broadcastTransaction(signedTx); +``` + +With web3.js: + +```typescript +const transaction: Transaction = { + from: senderPublicAddress, + to: receiverPublicAddress, + value: 1, + gas: 21000, + type: 0, +}; + +// you might also use below `web3.eth.personal.signMessage`, depending on your use case. +const signedTransaction = await web3.eth.accounts.signTransaction( + transaction, + privateKey, +); + +const tx = await web3.eth.sendSignedTransaction( + signedTransaction.rawTransaction, +); + +console.log(tx); +``` + + + +## Contracts + +### Contracts Deployment + +To deploy a contract in ethers.js you might have something like: + +```typescript +const signer = provider.getSigner(); +const factory = new ethers.ContractFactory(abi, bytecode, signer); +const contract = await factory.deploy("constructor param"); +console.log('contract address', contract.address); + +// wait for contract creation transaction to be mined +await contract.deployTransaction.wait(); +``` + +In web3.js: + +```typescript +const contractObject = new web3.eth.Contract(abi); +const deployedContract = await contractObject.deploy({ + data: bytecode, + arguments: ["constructor param"] +}).send({ + from: "0x12598d2Fd88B420ED571beFDA8dD112624B5E730", + gas: '1000000', + // other transaction's params +}); + +console.log('contract address', deployedContract.options.address) +``` + +:::tip +📝 To get the smart contract ABI, you are advised to check: [Compile the Solidity code using the Solidity Compiler and get its ABI and Bytecode](/guides/smart_contracts/deploying_and_interacting_with_smart_contracts#step-4-compile-the-solidity-code-using-the-solidity-compiler-and-get-its-abi-and-bytecode) and [Infer Contract Types from JSON Artifact](/guides/smart_contracts/infer_contract_types_guide/) +::: + + +### Calling Contracts' Methods + +To interact with contracts in ethers.js: + +```typescript +const contract = new ethers.Contract(CONTRACT_ADDRESS, ABI, providerOrSigner); +const result = await contract.someFunction(); +``` + +In web3.js: + +```typescript +const web3 = new Web3(provider); +const contract = new web3.eth.Contract(ABI, CONTRACT_ADDRESS); + +// If the method was only to read form the Blockchain: +const result = await contract.methods.someFunction().call(); +// Or, if the method would need a transaction to be sent: +const result = await contract.methods.someFunction().send(); +``` + +#### Contracts Overloaded Methods + +In ethers.js: + +```typescript +// ethers +const abi = [ + "function getMessage(string) public view returns (string)", + "function getMessage() public view returns (string)" +] +const contract = new ethers.Contract(address, abi, signer); + +// for ambiguous functions (two functions with the same +// name), the signature must also be specified +message = await contract['getMessage(string)']('nice'); +// and to call the overladed method without a parameter: +message = await contract['getMessage()'](); + +// in v6 +contract.foo(Typed.string('nice')) +``` + +In web3.js: + +```typescript +// in web3.js the overloaded method implementation is automatically picked based on the passed datatype +message = await contract.methods.getMessage('nice').call(); +// To call the overladed method without a parameter: +message = await contract.methods.getMessage().call(); +``` + +### Gas Estimation + +To interact with contracts in ethers.js: + +```typescript +// Estimate the gas +contract.myMethod.estimateGas(123) +``` + +In web3.js: + +```typescript +// Estimate the gas +const gasAmount = await myContract.methods.myMethod(123).estimateGas( + { gas: 5000000, from: transactionSenderAddress } // optional +); +``` + + +### Handling Events + +Handling events with ethers.js: + +```typescript +contract.on('SomeEvent', (arg1, arg2, event) => { + // event handling +}); +``` + +With web3.js: + +```typescript +const event = contract.events.SomeEvent({ + filter: { + filter: { val: 100 }, + }, + fromBlock: 0, +}); + +event.on('data', resolve); +event.on('error', reject); +``` + + +## Utility methods + +### Hashing +Here is how to compute `keccak256` hash of a UTF-8 string with web3 and ethers. + + +With ethers.js: + +```typescript +// hash of a string +ethers.utils.id('hello world') +// hash of binary data +ethers.utils.keccak256('0x4242') +``` + +With web3.js: + +```typescript +// computes the Keccak-256 hash of the input and returns a hexstring: +// the `utils.sha3` accepts: string and Uint8Array +web3.utils.sha3('hello world'); +// the `utils.keccak256` accepts: string, Uint8Array, Numbers and ReadonlyArray +web3.utils.keccak256('hello world'); +``` + +### Ethers Conversion + +Here is how to convert from and to ether units. + +With ethers.js: + +```typescript +const fromWieToEther = ethers.formatEther('1000000000000000000'); +// outputs: 1.0 +console.log(fromWieToEther); + +const fromEtherToWie = ethers.parseEther('1.0'); +// outputs: 1000000000000000000n +console.log(fromEtherToWie); +``` + +With web3.js: + +```typescript +// the second parameter is "the unit to convert to" +const fromWieToEther = Web3.utils.fromWei('1000000000000000000', 'ether'); +// outputs: 1 +console.log(fromWieToEther); + +// the second parameter is "the unit of the number passed" +const fromEtherToWie = Web3.utils.toWei('1.0', 'ether'); +// outputs: 1000000000000000000 +console.log(fromEtherToWie); +``` + +## Conclusion + +This guide should provide a starting point for migrating from ethers.js to web3.js version 4.x. Remember to adapt the example code to your actual use case and verify the function arguments and setup as you migrate your application. And the official documentation of web3.js is your go-to resource for detailed information and updates. From d8b8e18f7e80ceb3dc0bfbdd9bcb7734dd1c0e94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marin=20Petruni=C4=87?= Date: Mon, 20 Nov 2023 19:08:58 +0100 Subject: [PATCH 3/4] fix: faster hex to byte implementation (#6596) * faster hex to byte implementation Signed-off-by: Marin Petrunic * Apply suggestions from code review Co-authored-by: Muhammad Altabba <24407834+Muhammad-Altabba@users.noreply.github.com> * fix tiny linting issue --------- Signed-off-by: Marin Petrunic Co-authored-by: Muhammad Altabba <24407834+Muhammad-Altabba@users.noreply.github.com> Co-authored-by: Alex --- packages/web3-validator/src/utils.ts | 51 ++++++++++++++++++++++------ 1 file changed, 40 insertions(+), 11 deletions(-) diff --git a/packages/web3-validator/src/utils.ts b/packages/web3-validator/src/utils.ts index 28270c99c3d..6be75256ce3 100644 --- a/packages/web3-validator/src/utils.ts +++ b/packages/web3-validator/src/utils.ts @@ -449,20 +449,49 @@ export function uint8ArrayToHexString(uint8Array: Uint8Array): string { return hexString; } +// for optimized technique for hex to bytes conversion +const charCodeMap = { + zero: 48, + nine: 57, + A: 65, + F: 70, + a: 97, + f: 102, + } as const + + function charCodeToBase16(char: number) { + if (char >= charCodeMap.zero && char <= charCodeMap.nine) + return char - charCodeMap.zero + if (char >= charCodeMap.A && char <= charCodeMap.F) + return char - (charCodeMap.A - 10) + if (char >= charCodeMap.a && char <= charCodeMap.f) + return char - (charCodeMap.a - 10) + return undefined + } + export function hexToUint8Array(hex: string): Uint8Array { - let value; - if (hex.toLowerCase().startsWith('0x')) { - value = hex.slice(2); - } else { - value = hex; + let offset = 0; + if (hex.startsWith('0') && (hex[1] === 'x' || hex[1] === 'X')) { + offset = 2; } - if (value.length % 2 !== 0) { + if (hex.length % 2 !== 0) { throw new InvalidBytesError(`hex string has odd length: ${hex}`); } - const bytes = new Uint8Array(Math.ceil(value.length / 2)); - for (let i = 0; i < bytes.length; i += 1) { - const byte = parseInt(value.substring(i * 2, i * 2 + 2), 16); - bytes[i] = byte; + const length = (hex.length - offset) / 2; + const bytes = new Uint8Array(length); + for (let index = 0, j = offset; index < length; index+=1) { + // eslint-disable-next-line no-plusplus + const nibbleLeft = charCodeToBase16(hex.charCodeAt(j++)) + // eslint-disable-next-line no-plusplus + const nibbleRight = charCodeToBase16(hex.charCodeAt(j++)) + if (nibbleLeft === undefined || nibbleRight === undefined) { + throw new InvalidBytesError( + `Invalid byte sequence ("${hex[j - 2]}${ + hex[j - 1] + }" in "${hex}").`, + ) + } + bytes[index] = nibbleLeft * 16 + nibbleRight } - return bytes; + return bytes } From 2c6068571268a41a5de530f9052f1de1faaafd24 Mon Sep 17 00:00:00 2001 From: Oleksii Kosynskyi Date: Mon, 20 Nov 2023 13:29:34 -0500 Subject: [PATCH 4/4] Stress Tests 2 - QA Tests (#6583) * add data stress test * fix unit tests * fix * try to fix validator test * fix ipc tests * fix load test * fix * move to tests folder * moved * fix * move validator test to web3 package * move to TS. remove time logs * fix start.sh * fix validation test run script * fix name --- package.json | 3 + .../web3-validator/test/unit/load.test.ts | 184 ------------------ packages/web3/test/stress/index.ts | 68 +++++++ packages/web3/test/stress/start.sh | 17 ++ packages/web3/test/stress/validator.ts | 126 ++++++++++++ scripts/geth_binary.sh | 17 +- 6 files changed, 229 insertions(+), 186 deletions(-) delete mode 100644 packages/web3-validator/test/unit/load.test.ts create mode 100644 packages/web3/test/stress/index.ts create mode 100755 packages/web3/test/stress/start.sh create mode 100644 packages/web3/test/stress/validator.ts diff --git a/package.json b/package.json index 8d5c00e2173..6adc68ec73b 100644 --- a/package.json +++ b/package.json @@ -85,6 +85,9 @@ "test:blackbox:geth:ws": "yarn pre-blackbox && yarn geth:start:background && ./scripts/verdaccio.sh startBackgroundAndPublish && lerna run test:blackbox:geth:ws --stream && yarn post-blackbox:geth", "test:blackbox:infura:http": "yarn pre-blackbox && ./scripts/verdaccio.sh startBackgroundAndPublish && lerna run test:blackbox:infura:http --stream && yarn post-blackbox", "test:blackbox:infura:ws": "yarn pre-blackbox && ./scripts/verdaccio.sh startBackgroundAndPublish && lerna run test:blackbox:infura:ws --stream && yarn post-blackbox", + "test:manual:stress:data": "packages/web3/test/stress/start.sh", + "test:manual:stress:validation": "npx ts-node packages/web3/test/stress/validator.ts", + "test:manual:stress": "yarn test:manual:stress:data && yarn test:manual:stress:validation", "husky:install": "husky install", "husky:uninstall": "husky uninstall", "postinstall": "yarn build", diff --git a/packages/web3-validator/test/unit/load.test.ts b/packages/web3-validator/test/unit/load.test.ts deleted file mode 100644 index 9053be7e8f0..00000000000 --- a/packages/web3-validator/test/unit/load.test.ts +++ /dev/null @@ -1,184 +0,0 @@ -/* -This file is part of web3.js. - -web3.js is free software: you can redistribute it and/or modify -it under the terms of the GNU Lesser General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -web3.js is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Lesser General Public License for more details. - -You should have received a copy of the GNU Lesser General Public License -along with web3.js. If not, see . -*/ - -import { Web3Validator } from '../../src/web3_validator'; -import { Json, JsonSchema, ValidationSchemaInput } from '../../src/types'; - -const abi = [ - { indexed: true, internalType: 'address', name: 'from', type: 'address' }, - { indexed: true, internalType: 'address', name: 'to', type: 'address' }, - { indexed: false, internalType: 'uint256', name: 'value', type: 'uint256' }, -]; - -const abiJsonSchema = { - type: 'array', - items: [ - { name: 'from', format: 'address' }, - { name: 'to', format: 'address' }, - { name: 'value', format: 'uint256' }, - ], -}; - -const abiData = [ - '0xCB00CDE33a7a0Fba30C63745534F1f7Ae607076b', - '0xCB00CDE33a7a0Fba30C63745534F1f7Ae607076b', - '0xCB00CDE33a7a0Fba30C63745534F1f7Ae607076b', -]; - -const simpleSchema = { - type: 'object', - required: ['blockHash', 'blockNumber', 'from', 'to', 'data'], - properties: { - blockHash: { - format: 'bytes32', - }, - blockNumber: { - format: 'uint', - }, - from: { - format: 'address', - }, - to: { - oneOf: [{ format: 'address' }, { type: 'null' }], - }, - data: { - format: 'bytes', - }, - }, -}; - -const simpleData = { - blockHash: '0x0dec0518fa672a70027b04c286582e543ab17319fbdd384fa7bc8f3d5a542c0b', - blockNumber: BigInt(2), - from: '0xCB00CDE33a7a0Fba30C63745534F1f7Ae607076b', - to: '0xCB00CDE33a7a0Fba30C63745534F1f7Ae607076b', - data: '0xafea', -} as unknown as ValidationSchemaInput; - -const createHugeSchema = ( - schema: JsonSchema, - data: Json, - n = 3, -): { schema: JsonSchema; data: Json } => { - if (n > 0) { - const { data: resultData, schema: resultSchema } = createHugeSchema( - { ...simpleSchema } as JsonSchema, - { ...simpleData } as Json, - n - 1, - ); - return { - data: { ...(data as unknown as object), simple: resultData }, - schema: { ...schema, properties: { ...schema.properties, simple: resultSchema } }, - }; - } - return { - schema, - data, - }; -}; - -const { schema: hugeSchema, data: hugeData } = createHugeSchema( - { ...simpleSchema } as JsonSchema, - { ...simpleData } as Json, - 500, -); - -const { schema: hugeSchema1000, data: hugeData1000 } = createHugeSchema( - { ...simpleSchema } as JsonSchema, - { ...simpleData } as Json, - 1000, -); -describe('instance of validator', () => { - let validator: Web3Validator; - beforeAll(() => { - validator = new Web3Validator(); - }); - - it('huge schema', () => { - let t = 0; - expect(() => { - const t1 = Number(new Date()); - validator.validateJSONSchema(hugeSchema, hugeData as object); - t = Number(new Date()) - t1; - }).not.toThrow(); - expect(t).toBeLessThan(6000); - expect(t).toBeGreaterThan(0); - }); - - it('huge schema 1000', () => { - let t = 0; - expect(() => { - const t1 = Number(new Date()); - validator.validateJSONSchema(hugeSchema1000, hugeData1000 as object); - t = Number(new Date()) - t1; - }).not.toThrow(); - expect(t).toBeLessThan(6000); - expect(t).toBeGreaterThan(0); - }); - - it('simple schema multiple times', () => { - let t = 0; - expect(() => { - const t1 = Number(new Date()); - for (let i = 0; i < 500; i += 1) { - validator.validateJSONSchema(simpleSchema, simpleData as object); - } - t = Number(new Date()) - t1; - }).not.toThrow(); - expect(t).toBeLessThan(3000); - expect(t).toBeGreaterThan(0); - }); - - it('simple schema 1000 times', () => { - let t = 0; - expect(() => { - const t1 = Number(new Date()); - for (let i = 0; i < 1000; i += 1) { - validator.validateJSONSchema(simpleSchema, simpleData as object); - } - t = Number(new Date()) - t1; - }).not.toThrow(); - expect(t).toBeLessThan(4000); - expect(t).toBeGreaterThan(0); - }); - - it('simple JSON schema 1000 times', () => { - let t = 0; - expect(() => { - const t1 = Number(new Date()); - for (let i = 0; i < 1000; i += 1) { - validator.validateJSONSchema(abiJsonSchema, abiData as object); - } - t = Number(new Date()) - t1; - }).not.toThrow(); - expect(t).toBeLessThan(4000); - expect(t).toBeGreaterThan(0); - }); - - it('simple ABI 1000 times', () => { - let t = 0; - expect(() => { - const t1 = Number(new Date()); - for (let i = 0; i < 1000; i += 1) { - validator.validate(abi, abiData); - } - t = Number(new Date()) - t1; - }).not.toThrow(); - expect(t).toBeLessThan(4000); - expect(t).toBeGreaterThan(0); - }); -}); diff --git a/packages/web3/test/stress/index.ts b/packages/web3/test/stress/index.ts new file mode 100644 index 00000000000..67471e136fb --- /dev/null +++ b/packages/web3/test/stress/index.ts @@ -0,0 +1,68 @@ +/* +This file is part of web3.js. + +web3.js is free software: you can redistribute it and/or modify +it under the terms of the GNU Lesser General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +web3.js is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public License +along with web3.js. If not, see . +*/ + +/* eslint-disable */ +import { Web3 } from 'web3'; +import { IpcProvider } from 'web3-providers-ipc'; +import accounts from '../shared_fixtures/accounts.json'; +import { BasicAbi, BasicBytecode } from '../shared_fixtures/build/Basic'; +import WebSocketProvider from 'web3-providers-ws'; +const DATA_AMOUNT = 50 * 1024; // 50 kB + +const sendAndGetData = async (web3: Web3, i: number) => { + const sendOptions = { from: accounts[i].address }; + const deployOptions = { + data: BasicBytecode, + arguments: [0, ''] as [number, string], + gasPrice: await web3.eth.getGasPrice(), + gas: BigInt(9000000000000), + gasLimit: BigInt(9000000000000), + type: BigInt(0), + }; + const c = new web3.eth.Contract(BasicAbi); + const contract = await c.deploy(deployOptions).send(sendOptions); + + await contract.methods + // @ts-ignore + .setValues(1, 'A'.repeat(DATA_AMOUNT), true) + .send({ from: accounts[i].address }); + + await contract.methods.getStringValue().call(); +}; + +const test = async () => { + const providerString = String(process.env.WEB3_SYSTEM_TEST_PROVIDER); + console.log(`Start test with provider: ${providerString}`); + const provider = providerString.includes('ipc') + ? new IpcProvider(providerString) + : providerString; + const web3 = new Web3(provider); + + for (const a of accounts) { + const acc = web3.eth.accounts.privateKeyToAccount(a.privateKey); + web3.eth.accounts.wallet.add(acc); + } + + const prs = []; + for (let i = 0; i < 15; i++) { + prs.push(sendAndGetData(web3, i)); + } + await Promise.all(prs); + (web3.provider as unknown as WebSocketProvider).disconnect(); +}; + +test().catch(console.error); diff --git a/packages/web3/test/stress/start.sh b/packages/web3/test/stress/start.sh new file mode 100755 index 00000000000..9de30d524d9 --- /dev/null +++ b/packages/web3/test/stress/start.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +. scripts/env.sh + +export WEB3_SYSTEM_TEST_BACKEND="geth" +export TS_NODE_PREFER_TS_EXTS=true + +./scripts/geth_binary.sh stressStart + +yarn generate:accounts + +export WEB3_SYSTEM_TEST_PROVIDER=$IPC_PATH +npx ts-node ./packages/web3/test/stress/index.ts + +export WEB3_SYSTEM_TEST_PROVIDER=ws://127.0.0.1:8545 +npx ts-node ./packages/web3/test/stress/index.ts + +./scripts/geth_binary.sh stop diff --git a/packages/web3/test/stress/validator.ts b/packages/web3/test/stress/validator.ts new file mode 100644 index 00000000000..076351820c0 --- /dev/null +++ b/packages/web3/test/stress/validator.ts @@ -0,0 +1,126 @@ +/* +This file is part of web3.js. + +web3.js is free software: you can redistribute it and/or modify +it under the terms of the GNU Lesser General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +web3.js is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public License +along with web3.js. If not, see . +*/ + +/* eslint-disable */ +import { Web3Validator, JsonSchema, Json } from 'web3-validator'; + +const abi = [ + { indexed: true, internalType: 'address', name: 'from', type: 'address' }, + { indexed: true, internalType: 'address', name: 'to', type: 'address' }, + { indexed: false, internalType: 'uint256', name: 'value', type: 'uint256' }, +]; + +const abiJsonSchema = { + type: 'array', + items: [ + { name: 'from', format: 'address' }, + { name: 'to', format: 'address' }, + { name: 'value', format: 'uint256' }, + ], +}; + +const abiData = [ + '0xCB00CDE33a7a0Fba30C63745534F1f7Ae607076b', + '0xCB00CDE33a7a0Fba30C63745534F1f7Ae607076b', + '0xCB00CDE33a7a0Fba30C63745534F1f7Ae607076b', +]; + +const simpleSchema = { + type: 'object', + required: ['blockHash', 'blockNumber', 'from', 'to', 'data'], + properties: { + blockHash: { + format: 'bytes32', + }, + blockNumber: { + format: 'uint', + }, + from: { + format: 'address', + }, + to: { + oneOf: [{ format: 'address' }, { type: 'null' }], + }, + data: { + format: 'bytes', + }, + }, +}; + +const simpleData = { + blockHash: '0x0dec0518fa672a70027b04c286582e543ab17319fbdd384fa7bc8f3d5a542c0b', + blockNumber: BigInt(2), + from: '0xCB00CDE33a7a0Fba30C63745534F1f7Ae607076b', + to: '0xCB00CDE33a7a0Fba30C63745534F1f7Ae607076b', + data: '0xafea', +}; + +const createHugeSchema = ( + schema: JsonSchema, + data: Json, + n = 3, +): { schema: JsonSchema; data: Json } => { + if (n > 0) { + const { data: resultData, schema: resultSchema } = createHugeSchema( + { ...simpleSchema }, + { ...simpleData } as unknown as Json, + n - 1, + ); + return { + data: { ...(typeof data === 'object' ? data : { data }), simple: resultData }, + schema: { ...schema, properties: { ...schema.properties, simple: resultSchema } }, + }; + } + return { + schema, + data, + }; +}; + +const { schema: hugeSchema, data: hugeData } = createHugeSchema( + { ...simpleSchema }, + { ...simpleData } as unknown as Json, + 500, +); + +const { schema: hugeSchema1000, data: hugeData1000 } = createHugeSchema( + { ...simpleSchema }, + { ...simpleData } as unknown as Json, + 1000, +); + +const validator = new Web3Validator(); + +validator.validateJSONSchema(hugeSchema, hugeData as object); + +validator.validateJSONSchema(hugeSchema1000, hugeData1000 as object); + +for (let i = 0; i < 500; i += 1) { + validator.validateJSONSchema(simpleSchema, simpleData); +} + +for (let i = 0; i < 1000; i += 1) { + validator.validateJSONSchema(simpleSchema, simpleData); +} + +for (let i = 0; i < 1000; i += 1) { + validator.validateJSONSchema(abiJsonSchema, abiData); +} + +for (let i = 0; i < 1000; i += 1) { + validator.validate(abi, abiData); +} diff --git a/scripts/geth_binary.sh b/scripts/geth_binary.sh index 59f9bfecaea..82f48622ec6 100755 --- a/scripts/geth_binary.sh +++ b/scripts/geth_binary.sh @@ -59,17 +59,29 @@ start() { download if [ -z "${ORIGARGS[1]}" ]; then echo "Starting geth..." - echo "geth --ipcpath $IPC_PATH --nodiscover --nousb --ws --ws.addr 0.0.0.0 --ws.port $WEB3_SYSTEM_TEST_PORT --http --http.addr 0.0.0.0 --http.port $WEB3_SYSTEM_TEST_PORT --allow-insecure-unlock --http.api personal,web3,eth,admin,debug,txpool,net --ws.api personal,web3,eth,admin,debug,miner,txpool,net --dev --mine --dev.period=0 " + echo "geth --ipcpath $IPC_PATH --nodiscover --nousb --ws --ws.addr 0.0.0.0 --ws.port $WEB3_SYSTEM_TEST_PORT --http --http.addr 0.0.0.0 --http.port $WEB3_SYSTEM_TEST_PORT --allow-insecure-unlock --http.api personal,web3,eth,admin,debug,txpool,net --ws.api personal,web3,eth,admin,debug,miner,txpool,net --dev --mine --dev.period=0" ${TMP_FOLDER}/geth --ipcpath $IPC_PATH --nodiscover --nousb --ws --ws.addr 0.0.0.0 --ws.port $WEB3_SYSTEM_TEST_PORT --http --http.addr 0.0.0.0 --http.port $WEB3_SYSTEM_TEST_PORT --allow-insecure-unlock --http.api personal,web3,eth,admin,debug,txpool,net --ws.api personal,web3,eth,admin,debug,miner,txpool,net --dev --mine --dev.period=0 --rpc.enabledeprecatedpersonal else echo "Starting geth..." - echo "geth --ipcpath $IPC_PATH --nodiscover --nousb --ws --ws.addr 0.0.0.0 --ws.port $WEB3_SYSTEM_TEST_PORT --http --http.addr 0.0.0.0 --http.port $WEB3_SYSTEM_TEST_PORT --allow-insecure-unlock --http.api personal,web3,eth,admin,debug,txpool,net --ws.api personal,web3,eth,admin,debug,miner,txpool,net --dev --mine --dev.period=0 &>/dev/null &" + echo "geth --ipcpath $IPC_PATH --nodiscover --nousb --ws --ws.addr 0.0.0.0 --ws.port $WEB3_SYSTEM_TEST_PORT --http --http.addr 0.0.0.0 --http.port $WEB3_SYSTEM_TEST_PORT --allow-insecure-unlock --http.api personal,web3,eth,admin,debug,txpool,net --ws.api personal,web3,eth,admin,debug,miner,txpool,net --dev --mine --dev.period=0 &>/dev/null &" ${TMP_FOLDER}/geth --ipcpath $IPC_PATH --nodiscover --nousb --ws --ws.addr 0.0.0.0 --ws.port $WEB3_SYSTEM_TEST_PORT --http --http.addr 0.0.0.0 --http.port $WEB3_SYSTEM_TEST_PORT --allow-insecure-unlock --http.api personal,web3,eth,admin,debug,txpool,net --ws.api personal,web3,eth,admin,debug,miner,txpool,net --dev --mine --dev.period=0 --rpc.enabledeprecatedpersonal &>/dev/null & echo "Waiting for geth..." npx wait-port -t 10000 "$WEB3_SYSTEM_TEST_PORT" fi } +startStress() { + download + + echo "Starting geth..." + echo "geth --ipcpath $IPC_PATH --nodiscover --nousb --ws --ws.addr 0.0.0.0 --ws.port $WEB3_SYSTEM_TEST_PORT --http --http.addr 0.0.0.0 --http.port $WEB3_SYSTEM_TEST_PORT --allow-insecure-unlock --http.api personal,web3,eth,admin,debug,txpool,net --ws.api personal,web3,eth,admin,debug,miner,txpool,net --dev --mine --dev.period=0 --dev.gaslimit 9000000000000000 --rpc.txfeecap=1000000 &>/dev/null &" + ${TMP_FOLDER}/geth --ipcpath $IPC_PATH --nodiscover --nousb --ws --ws.addr 0.0.0.0 --ws.port $WEB3_SYSTEM_TEST_PORT --http --http.addr 0.0.0.0 --http.port $WEB3_SYSTEM_TEST_PORT --allow-insecure-unlock --http.api personal,web3,eth,admin,debug,txpool,net --ws.api personal,web3,eth,admin,debug,miner,txpool,net --dev --mine --dev.period=0 --rpc.enabledeprecatedpersonal --dev.gaslimit 9000000000000000 --rpc.txfeecap=1000000 &>/dev/null & + echo "Waiting for geth..." + npx wait-port -t 10000 "$WEB3_SYSTEM_TEST_PORT" + +} + + startSync() { download @@ -95,6 +107,7 @@ stop() { } case $1 in +stressStart) startStress ;; syncStart) startSync ;; syncStop) syncStop ;; start) start ;;