Skip to content

Commit 0de5672

Browse files
pingesFilter94
pinges
and
Filter94
authored
add an app test for discovery (#423)
* add an app test for discovery Signed-off-by: [email protected] <[email protected]> * job Signed-off-by: [email protected] <[email protected]> * nits Signed-off-by: [email protected] <[email protected]> * remove free port for tcp Signed-off-by: [email protected] <[email protected]> * make job reusable Signed-off-by: [email protected] <[email protected]> * make job reusable Signed-off-by: [email protected] <[email protected]> * make test less flaky Signed-off-by: [email protected] <[email protected]> * check initialized Signed-off-by: [email protected] <[email protected]> * make private Signed-off-by: [email protected] <[email protected]> * make ethTransactions private again Signed-off-by: [email protected] <[email protected]> * Fixed assertions * Removed findFreePort * Fixed node disconnects validator when requests time out test flakiness * Reverted fork transition * Removed transaction sending job * Removed not needed imports * Improved the log message --------- Signed-off-by: [email protected] <[email protected]> Co-authored-by: Roman <[email protected]>
1 parent e165017 commit 0de5672

File tree

4 files changed

+309
-128
lines changed

4 files changed

+309
-128
lines changed

‎app/src/integrationTest/kotlin/maru/app/MaruDiscoveryTest.kt‎

Lines changed: 219 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,219 @@
1+
/*
2+
* Copyright Consensys Software Inc.
3+
*
4+
* This file is dual-licensed under either the MIT license or Apache License 2.0.
5+
* See the LICENSE-MIT and LICENSE-APACHE files in the repository root for details.
6+
*
7+
* SPDX-License-Identifier: MIT OR Apache-2.0
8+
*/
9+
package maru.app
10+
11+
import kotlin.time.Duration.Companion.milliseconds
12+
import kotlin.time.Duration.Companion.seconds
13+
import kotlin.time.toJavaDuration
14+
import org.apache.logging.log4j.LogManager
15+
import org.assertj.core.api.Assertions.assertThat
16+
import org.awaitility.kotlin.await
17+
import org.hyperledger.besu.tests.acceptance.dsl.condition.net.NetConditions
18+
import org.hyperledger.besu.tests.acceptance.dsl.node.ThreadBesuNodeRunner
19+
import org.hyperledger.besu.tests.acceptance.dsl.node.cluster.Cluster
20+
import org.hyperledger.besu.tests.acceptance.dsl.node.cluster.ClusterConfigurationBuilder
21+
import org.hyperledger.besu.tests.acceptance.dsl.transaction.net.NetTransactions
22+
import org.junit.jupiter.api.AfterEach
23+
import org.junit.jupiter.api.Test
24+
import testutils.Checks.getBlockNumber
25+
import testutils.PeeringNodeNetworkStack
26+
import testutils.besu.BesuFactory
27+
import testutils.besu.BesuTransactionsHelper
28+
import testutils.maru.MaruFactory
29+
30+
/**
31+
* Test suite for Maru peer discovery with multiple nodes.
32+
* Tests that multiple Maru nodes can discover each other using discovery protocol.
33+
*/
34+
class MaruDiscoveryTest {
35+
private lateinit var besuCluster: Cluster
36+
private val networkStacks = mutableListOf<PeeringNodeNetworkStack>()
37+
private val maruApps = mutableListOf<MaruApp>()
38+
private lateinit var transactionsHelper: BesuTransactionsHelper
39+
private val log = LogManager.getLogger(this.javaClass)
40+
private val maruFactory = MaruFactory()
41+
42+
@AfterEach
43+
fun tearDown() {
44+
maruApps.forEach { app ->
45+
try {
46+
app.stop()
47+
app.close()
48+
} catch (e: Exception) {
49+
log.warn("Error stopping Maru app", e)
50+
}
51+
}
52+
maruApps.clear()
53+
54+
networkStacks.clear()
55+
56+
if (::besuCluster.isInitialized) {
57+
besuCluster.close()
58+
}
59+
}
60+
61+
@Test
62+
fun `ten nodes discover each other via bootnode`() {
63+
testMultiNodeDiscovery(numberOfNodes = 10, expectedPeers = 9u)
64+
}
65+
66+
/**
67+
* Tests peer discovery with multiple Maru nodes.
68+
*
69+
* Setup:
70+
* - Creates N Besu nodes (EL) and N Maru nodes (CL)
71+
* - First Maru node is the bootnode
72+
* - All other nodes use the bootnode's ENR for discovery
73+
*
74+
* @param numberOfNodes Total number of Maru nodes to create
75+
* @param expectedPeers Number of peers each node should discover
76+
*/
77+
private fun testMultiNodeDiscovery(
78+
numberOfNodes: Int,
79+
expectedPeers: UInt,
80+
) {
81+
require(numberOfNodes >= 2) { "Need at least 2 nodes for discovery test" }
82+
83+
log.info("Starting peer discovery test with $numberOfNodes nodes")
84+
85+
// Initialize test infrastructure
86+
transactionsHelper = BesuTransactionsHelper()
87+
besuCluster =
88+
Cluster(
89+
ClusterConfigurationBuilder().build(),
90+
NetConditions(NetTransactions()),
91+
ThreadBesuNodeRunner(),
92+
)
93+
94+
// Create and start all network stacks (Besu + Maru)
95+
repeat(numberOfNodes) { index ->
96+
val isValidator = index == 0 // First node is validator, rest are followers
97+
val stack =
98+
PeeringNodeNetworkStack(
99+
besuBuilder = {
100+
BesuFactory.buildTestBesu(validator = isValidator)
101+
},
102+
)
103+
networkStacks.add(stack)
104+
}
105+
106+
// Start all Besu nodes (they will automatically peer with each other at EL layer)
107+
log.info("Starting ${networkStacks.size} Besu nodes")
108+
PeeringNodeNetworkStack.startBesuNodes(besuCluster, *networkStacks.toTypedArray())
109+
110+
// Wait for Besu nodes to be ready
111+
await
112+
.atMost(100.seconds.toJavaDuration())
113+
.pollInterval(500.milliseconds.toJavaDuration())
114+
.untilAsserted {
115+
networkStacks.forEach { stack ->
116+
val blockNumber = stack.besuNode.getBlockNumber()
117+
assertThat(blockNumber).isNotNull
118+
}
119+
}
120+
121+
log.info("All Besu nodes are ready")
122+
123+
// Create and start the bootnode (validator)
124+
val bootnodeStack = networkStacks[0]
125+
126+
val bootnodeMaruApp =
127+
maruFactory.buildTestMaruValidatorWithDiscovery(
128+
ethereumJsonRpcUrl = bootnodeStack.besuNode.jsonRpcBaseUrl().get(),
129+
engineApiRpc = bootnodeStack.besuNode.engineRpcUrl().get(),
130+
dataDir = bootnodeStack.tmpDir,
131+
discoveryPort = 0u,
132+
allowEmptyBlocks = true,
133+
)
134+
135+
bootnodeStack.setMaruApp(bootnodeMaruApp)
136+
maruApps.add(bootnodeMaruApp)
137+
bootnodeMaruApp.start()
138+
139+
// Get bootnode ENR for other nodes to use
140+
val bootnodeEnr = bootnodeMaruApp.p2pNetwork.localNodeRecord?.asEnr()
141+
requireNotNull(bootnodeEnr) { "Bootnode ENR should not be null" }
142+
log.info("Bootnode ENR: $bootnodeEnr")
143+
144+
// Start block production on validator
145+
log.info("Starting block production on validator")
146+
147+
// Wait for some blocks to be produced
148+
await
149+
.atMost(20.seconds.toJavaDuration())
150+
.pollInterval(500.milliseconds.toJavaDuration())
151+
.untilAsserted {
152+
val blockNumber = bootnodeStack.besuNode.getBlockNumber()
153+
assertThat(blockNumber.toLong()).isGreaterThanOrEqualTo(5L)
154+
}
155+
156+
log.info("Validator is producing blocks")
157+
158+
// Create and start follower nodes
159+
for (i in 1 until numberOfNodes) {
160+
val stack = networkStacks[i]
161+
162+
val followerMaruApp =
163+
maruFactory.buildTestMaruFollowerWithDiscovery(
164+
ethereumJsonRpcUrl = stack.besuNode.jsonRpcBaseUrl().get(),
165+
engineApiRpc = stack.besuNode.engineRpcUrl().get(),
166+
dataDir = stack.tmpDir,
167+
bootnode = bootnodeEnr,
168+
discoveryPort = 0u,
169+
allowEmptyBlocks = true,
170+
)
171+
172+
stack.setMaruApp(followerMaruApp)
173+
maruApps.add(followerMaruApp)
174+
followerMaruApp.start()
175+
}
176+
177+
log.info("All $numberOfNodes Maru nodes started")
178+
179+
// Wait for all nodes to discover each other
180+
log.info("Waiting for all nodes to discover $expectedPeers peers")
181+
await
182+
.atMost(60.seconds.toJavaDuration())
183+
.pollInterval(2.seconds.toJavaDuration())
184+
.untilAsserted {
185+
maruApps.forEachIndexed { index, app ->
186+
val peerCount = app.peersConnected()
187+
log.info("Node $index has $peerCount peers (expected: $expectedPeers)")
188+
assertThat(peerCount).isGreaterThanOrEqualTo(expectedPeers)
189+
}
190+
}
191+
192+
log.info("All nodes have discovered their peers!")
193+
194+
// Verify each node can see the others
195+
maruApps.forEachIndexed { index, app ->
196+
val peers = app.p2pNetwork.getPeers()
197+
log.info("Node $index peers: ${peers.map { it.nodeId }}")
198+
assertThat(peers.size).isGreaterThanOrEqualTo(expectedPeers.toInt())
199+
}
200+
201+
log.info("Verifying followers sync EL blocks")
202+
val validatorBlockHeight =
203+
networkStacks[0].besuNode.getBlockNumber()
204+
205+
await
206+
.atMost(30.seconds.toJavaDuration())
207+
.pollInterval(1.seconds.toJavaDuration())
208+
.untilAsserted {
209+
networkStacks.forEachIndexed { i, stack ->
210+
val followerBlockHeight =
211+
stack.besuNode.getBlockNumber()
212+
log.info("Follower $i EL block height: $followerBlockHeight (validator: $validatorBlockHeight)")
213+
assertThat(followerBlockHeight).isGreaterThanOrEqualTo(validatorBlockHeight)
214+
}
215+
}
216+
217+
log.info("All followers have synced EL blocks successfully!")
218+
}
219+
}

0 commit comments

Comments
 (0)