feat: add integration tests for ao extra data from scripts repo

contracts/common/interfaces/ICSModule.sol

@@ -0,0 +1,97 @@
+// SPDX-FileCopyrightText: 2024 Lido <[email protected]>
+// SPDX-License-Identifier: GPL-3.0
+// For full version see: https://github.com/lidofinance/community-staking-module/blob/develop/src/interfaces/ICSModule.sol
+import { IStakingModule } from "../../0.8.9/interfaces/IStakingModule.sol";
+
+pragma solidity 0.8.9;
+
+
+struct NodeOperator {
+    // All the counters below are used together e.g. in the _updateDepositableValidatorsCount
+    /* 1 */ uint32 totalAddedKeys; // @dev increased and decreased when removed
+    /* 1 */ uint32 totalWithdrawnKeys; // @dev only increased
+    /* 1 */ uint32 totalDepositedKeys; // @dev only increased
+    /* 1 */ uint32 totalVettedKeys; // @dev both increased and decreased
+    /* 1 */ uint32 stuckValidatorsCount; // @dev both increased and decreased
+    /* 1 */ uint32 depositableValidatorsCount; // @dev any value
+    /* 1 */ uint32 targetLimit;
+    /* 1 */ uint8 targetLimitMode;
+    /* 2 */ uint32 totalExitedKeys; // @dev only increased except for the unsafe updates
+    /* 2 */ uint32 enqueuedCount; // Tracks how many places are occupied by the node operator's keys in the queue.
+    /* 2 */ address managerAddress;
+    /* 3 */ address proposedManagerAddress;
+    /* 4 */ address rewardAddress;
+    /* 5 */ address proposedRewardAddress;
+    /* 5 */ bool extendedManagerPermissions;
+}
+
+struct NodeOperatorManagementProperties {
+    address managerAddress;
+    address rewardAddress;
+    bool extendedManagerPermissions;
+}
+
+/// @title Lido's Community Staking Module interface
+interface ICSModule is IStakingModule
+{
+    error NodeOperatorDoesNotExist();
+    error ZeroRewardAddress();
+
+    /// @notice Gets node operator non-withdrawn keys
+    /// @param nodeOperatorId ID of the node operator
+    /// @return Non-withdrawn keys count
+    function getNodeOperatorNonWithdrawnKeys(
+        uint256 nodeOperatorId
+    ) external view returns (uint256);
+
+    /// @notice Returns the node operator by id
+    /// @param nodeOperatorId Node Operator id
+    function getNodeOperator(
+        uint256 nodeOperatorId
+    ) external view returns (NodeOperator memory);
+
+    /// @notice Gets node operator signing keys
+    /// @param nodeOperatorId ID of the node operator
+    /// @param startIndex Index of the first key
+    /// @param keysCount Count of keys to get
+    /// @return Signing keys
+    function getSigningKeys(
+        uint256 nodeOperatorId,
+        uint256 startIndex,
+        uint256 keysCount
+    ) external view returns (bytes memory);
+
+    /// @notice Gets node operator signing keys with signatures
+    /// @param nodeOperatorId ID of the node operator
+    /// @param startIndex Index of the first key
+    /// @param keysCount Count of keys to get
+    /// @return keys Signing keys
+    /// @return signatures Signatures of (deposit_message, domain) tuples
+    function getSigningKeysWithSignatures(
+        uint256 nodeOperatorId,
+        uint256 startIndex,
+        uint256 keysCount
+    ) external view returns (bytes memory keys, bytes memory signatures);
+
+    /// @notice Report node operator's key as slashed and apply initial slashing penalty.
+    /// @param nodeOperatorId Operator ID in the module.
+    /// @param keyIndex Index of the slashed key in the node operator's keys.
+    function submitInitialSlashing(
+        uint256 nodeOperatorId,
+        uint256 keyIndex
+    ) external;
+
+    /// @notice Report node operator's key as withdrawn and settle withdrawn amount.
+    /// @param nodeOperatorId Operator ID in the module.
+    /// @param keyIndex Index of the withdrawn key in the node operator's keys.
+    /// @param amount Amount of withdrawn ETH in wei.
+    /// @param isSlashed Validator is slashed or not
+    function submitWithdrawal(
+        uint256 nodeOperatorId,
+        uint256 keyIndex,
+        uint256 amount,
+        bool isSlashed
+    ) external;
+
+    function depositETH(uint256 nodeOperatorId) external payable;
+}

lib/oracle.ts

@@ -2,10 +2,12 @@ import { bigintToHex } from "bigint-conversion";
 import { keccak256, ZeroHash } from "ethers";
 import { ethers } from "hardhat";
 
-import { AccountingOracle, HashConsensus } from "typechain-types";
+import { AccountingOracle, HashConsensus, OracleReportSanityChecker } from "typechain-types";
 
 import { numberToHex } from "./string";
 
+import { ether, impersonate } from ".";
+
 function splitArrayIntoChunks<T>(inputArray: T[], maxItemsPerChunk: number): T[][] {
   const result: T[][] = [];
   for (let i = 0; i < inputArray.length; i += maxItemsPerChunk) {
@@ -194,7 +196,7 @@ export type OracleReportProps = {
   config?: ExtraDataConfig;
 };
 
-export function constructOracleReport({ reportFieldsWithoutExtraData, extraData, config }: OracleReportProps) {
+export function prepareExtraData(extraData: ExtraData, config?: ExtraDataConfig) {
   const extraDataItems: string[] = [];
 
   if (Array.isArray(extraData)) {
@@ -212,11 +214,17 @@ export function constructOracleReport({ reportFieldsWithoutExtraData, extraData,
   const extraDataChunks = packExtraDataItemsToChunksLinkedByHash(extraDataItems, maxItemsPerChunk);
   const extraDataChunkHashes = extraDataChunks.map((chunk) => calcExtraDataListHash(chunk));
 
+  return { extraDataItemsCount, extraDataChunks, extraDataChunkHashes };
+}
+
+export function constructOracleReport({ reportFieldsWithoutExtraData, extraData, config }: OracleReportProps) {
+  const { extraDataItemsCount, extraDataChunks, extraDataChunkHashes } = prepareExtraData(extraData, config);
+
   const report: OracleReport = {
     ...reportFieldsWithoutExtraData,
-    extraDataHash: extraDataItems.length ? extraDataChunkHashes[0] : ZeroHash,
-    extraDataItemsCount: extraDataItems.length,
-    extraDataFormat: extraDataItems.length ? EXTRA_DATA_FORMAT_LIST : EXTRA_DATA_FORMAT_EMPTY,
+    extraDataHash: extraDataItemsCount ? extraDataChunkHashes[0] : ZeroHash,
+    extraDataItemsCount,
+    extraDataFormat: extraDataItemsCount ? EXTRA_DATA_FORMAT_LIST : EXTRA_DATA_FORMAT_EMPTY,
   };
 
   const reportHash = calcReportDataHash(getReportDataItems(report));
@@ -239,3 +247,22 @@ export async function getSlotTimestamp(slot: bigint, consensus: HashConsensus) {
   const chainConfig = await consensus.getChainConfig();
   return chainConfig.genesisTime + chainConfig.secondsPerSlot * slot;
 }
+
+// Might be useful for tests on scratch where even reporting a single exited validator
+// is too much for the default limit
+export async function setAnnualBalanceIncreaseLimit(sanityChecker: OracleReportSanityChecker, limitBP: bigint) {
+  const adminRole = await sanityChecker.DEFAULT_ADMIN_ROLE();
+
+  const admin = await sanityChecker.getRoleMember(adminRole, 0);
+  const adminSigner = await impersonate(admin, ether("1"));
+
+  const setLimitRole = await sanityChecker.ANNUAL_BALANCE_INCREASE_LIMIT_MANAGER_ROLE();
+
+  // Grant the role to the current signer
+  await sanityChecker.connect(adminSigner).grantRole(setLimitRole, adminSigner.address);
+
+  await sanityChecker.connect(adminSigner).setAnnualBalanceIncreaseBPLimit(limitBP);
+
+  // Revoke the role after setting the limit
+  await sanityChecker.connect(adminSigner).revokeRole(setLimitRole, adminSigner.address);
+}

lib/protocol/context.ts

@@ -12,6 +12,10 @@ const getSigner = async (signer: Signer, balance = ether("100"), signers: Protoc
   return impersonate(signerAddress, balance);
 };
 
+export const withCSM = () => {
+  return process.env.INTEGRATION_WITH_CSM !== "off";
+};
+
 export const getProtocolContext = async (): Promise<ProtocolContext> => {
   if (hre.network.name === "hardhat") {
     const networkConfig = hre.config.networks[hre.network.name];
@@ -27,7 +31,7 @@ export const getProtocolContext = async (): Promise<ProtocolContext> => {
 
   // By default, all flags are "on"
   const flags = {
-    withCSM: process.env.INTEGRATION_WITH_CSM !== "off",
+    withCSM: withCSM(),
   } as ProtocolContextFlags;
 
   log.debug("Protocol context flags", {

lib/protocol/discover.ts

@@ -1,6 +1,14 @@
 import hre from "hardhat";
 
-import { AccountingOracle, Lido, LidoLocator, StakingRouter, WithdrawalQueueERC721 } from "typechain-types";
+import {
+  AccountingOracle,
+  ICSModule,
+  Lido,
+  LidoLocator,
+  NodeOperatorsRegistry,
+  StakingRouter,
+  WithdrawalQueueERC721,
+} from "typechain-types";
 
 import { batch, log } from "lib";
 
@@ -120,11 +128,18 @@ const getAragonContracts = async (lido: LoadedContract<Lido>, config: ProtocolNe
  * Load staking modules contracts registered in the staking router.
  */
 const getStakingModules = async (stakingRouter: LoadedContract<StakingRouter>, config: ProtocolNetworkConfig) => {
-  const [nor, sdvt] = await stakingRouter.getStakingModules();
-  return (await batch({
+  const [nor, sdvt, csm] = await stakingRouter.getStakingModules();
+
+  const promises: { [key: string]: Promise<LoadedContract<NodeOperatorsRegistry | ICSModule>> } = {
     nor: loadContract("NodeOperatorsRegistry", config.get("nor") || nor.stakingModuleAddress),
     sdvt: loadContract("NodeOperatorsRegistry", config.get("sdvt") || sdvt.stakingModuleAddress),
-  })) as StakingModuleContracts;
+  };
+
+  if (csm) {
+    promises.csm = loadContract("ICSModule", config.get("csm") || csm.stakingModuleAddress);
+  }
+
+  return (await batch(promises)) as StakingModuleContracts;
 };
 
 /**

lib/protocol/helpers/accounting.ts

@@ -12,11 +12,13 @@ import {
   certainAddress,
   ether,
   EXTRA_DATA_FORMAT_EMPTY,
+  EXTRA_DATA_FORMAT_LIST,
   getCurrentBlockTimestamp,
   HASH_CONSENSUS_FAR_FUTURE_EPOCH,
   impersonate,
   log,
   ONE_GWEI,
+  prepareExtraData,
 } from "lib";
 
 import { ProtocolContext } from "../types";
@@ -213,6 +215,56 @@ export const report = async (
   });
 };
 
+export async function reportWithoutExtraData(
+  ctx: ProtocolContext,
+  numExitedValidatorsByStakingModule: bigint[],
+  stakingModuleIdsWithNewlyExitedValidators: bigint[],
+  extraData: ReturnType<typeof prepareExtraData>,
+) {
+  const { accountingOracle } = ctx.contracts;
+
+  const { extraDataItemsCount, extraDataChunks, extraDataChunkHashes } = extraData;
+
+  const reportData: OracleReportParams = {
+    excludeVaultsBalances: true,
+    extraDataFormat: EXTRA_DATA_FORMAT_LIST,
+    extraDataHash: extraDataChunkHashes[0],
+    extraDataItemsCount: BigInt(extraDataItemsCount),
+    numExitedValidatorsByStakingModule,
+    stakingModuleIdsWithNewlyExitedValidators,
+    skipWithdrawals: true,
+  };
+
+  const { data } = await report(ctx, { ...reportData, dryRun: true });
+
+  const items = getReportDataItems(data);
+  const hash = calcReportDataHash(items);
+  const oracleVersion = await accountingOracle.getContractVersion();
+
+  const submitter = await reachConsensus(ctx, {
+    refSlot: BigInt(data.refSlot),
+    reportHash: hash,
+    consensusVersion: BigInt(data.consensusVersion),
+  });
+
+  const reportTx = await accountingOracle.connect(submitter).submitReportData(data, oracleVersion);
+  log.debug("Pushed oracle report main data", {
+    "Ref slot": data.refSlot,
+    "Consensus version": data.consensusVersion,
+    "Report hash": hash,
+  });
+
+  // Get processing state after main report is submitted
+  const processingStateAfterMainReport = await accountingOracle.getProcessingState();
+
+  // Verify that extra data is not yet submitted
+  expect(processingStateAfterMainReport.extraDataSubmitted).to.be.false;
+  expect(processingStateAfterMainReport.extraDataItemsCount).to.equal(extraDataItemsCount);
+  expect(processingStateAfterMainReport.extraDataItemsSubmitted).to.equal(0n);
+
+  return { reportTx, data, submitter, extraDataChunks, extraDataChunkHashes };
+}
+
 export const getReportTimeElapsed = async (ctx: ProtocolContext) => {
   const { hashConsensus } = ctx.contracts;
   const { slotsPerEpoch, secondsPerSlot, genesisTime } = await hashConsensus.getChainConfig();

lib/protocol/helpers/index.ts

@@ -16,8 +16,8 @@ export {
 } from "./accounting";
 
 export { ensureDsmGuardians } from "./dsm";
-export { norEnsureOperators } from "./nor";
-export { sdvtEnsureOperators } from "./sdvt";
+export { norSdvtEnsureOperators } from "./nor-sdvt";
+export { calcNodeOperatorRewards } from "./staking-module";
 export {
   createVaultProxy,
   createVaultsReportTree,