[Devnet5] Weird "Error encountered during sync"

[tbenr]

We should analyze if there is something wrong here..
We logged a lot of this

2025-01-27 08:13:00.069 ERROR - Error encountered during sync
java.util.concurrent.CompletionException: tech.pegasys.teku.networking.eth2.rpc.core.RpcException: [Code 3] Requested historical blocks are currently unavailable
at java.base/java.util.concurrent.CompletableFuture.encodeRelay(Unknown Source) ~[?:?]
at java.base/java.util.concurrent.CompletableFuture.completeRelay(Unknown Source) ~[?:?]
at java.base/java.util.concurrent.CompletableFuture$UniRelay.tryFire(Unknown Source) ~[?:?]
at java.base/java.util.concurrent.CompletableFuture.postComplete(Unknown Source) ~[?:?]
at java.base/java.util.concurrent.CompletableFuture.completeExceptionally(Unknown Source) ~[?:?]
at java.base/java.util.Optional.ifPresentOrElse(Unknown Source) ~[?:?]
at tech.pegasys.teku.networking.eth2.rpc.core.Eth2RpcResponseHandler.onCompleted(Eth2RpcResponseHandler.java:118) ~[teku-networking-eth2-develop.jar:UNKNOWN+g28f69d3]
at tech.pegasys.teku.networking.p2p.rpc.RpcResponseHandler.onCompleted(RpcResponseHandler.java:28) ~[teku-networking-p2p-develop.jar:UNKNOWN+g28f69d3]
at tech.pegasys.teku.networking.eth2.rpc.core.Eth2OutgoingRequestHandler.lambda$abortRequest$5(Eth2OutgoingRequestHandler.java:247) ~[teku-networking-eth2-develop.jar:UNKNOWN+g28f69d3]
at tech.pegasys.teku.infrastructure.async.SafeFuture.lambda$always$25(SafeFuture.java:345) ~[teku-infrastructure-async-develop.jar:UNKNOWN+g28f69d3]
at tech.pegasys.teku.infrastructure.async.SafeFuture.lambda$finish$29(SafeFuture.java:363) ~[teku-infrastructure-async-develop.jar:UNKNOWN+g28f69d3]
at java.base/java.util.concurrent.CompletableFuture.uniHandle(Unknown Source) ~[?:?]
at java.base/java.util.concurrent.CompletableFuture.uniHandleStage(Unknown Source) ~[?:?]
at java.base/java.util.concurrent.CompletableFuture.handle(Unknown Source) ~[?:?]
at tech.pegasys.teku.infrastructure.async.SafeFuture.handle(SafeFuture.java:580) ~[teku-infrastructure-async-develop.jar:UNKNOWN+g28f69d3]
at tech.pegasys.teku.infrastructure.async.SafeFuture.finish(SafeFuture.java:358) ~[teku-infrastructure-async-develop.jar:UNKNOWN+g28f69d3]
at tech.pegasys.teku.infrastructure.async.SafeFuture.always(SafeFuture.java:345) ~[teku-infrastructure-async-develop.jar:UNKNOWN+g28f69d3]
at tech.pegasys.teku.networking.eth2.rpc.core.Eth2OutgoingRequestHandler.abortRequest(Eth2OutgoingRequestHandler.java:247) ~[teku-networking-eth2-develop.jar:UNKNOWN+g28f69d3]
at tech.pegasys.teku.networking.eth2.rpc.core.Eth2OutgoingRequestHandler.abortRequest(Eth2OutgoingRequestHandler.java:230) ~[teku-networking-eth2-develop.jar:UNKNOWN+g28f69d3]
at tech.pegasys.teku.networking.eth2.rpc.core.Eth2OutgoingRequestHandler.processData(Eth2OutgoingRequestHandler.java:139) ~[teku-networking-eth2-develop.jar:UNKNOWN+g28f69d3]
at tech.pegasys.teku.networking.p2p.libp2p.rpc.RpcHandler$Controller.lambda$channelRead0$0(RpcHandler.java:186) ~[teku-networking-p2p-develop.jar:UNKNOWN+g28f69d3]
at java.base/java.util.Optional.ifPresentOrElse(Unknown Source) ~[?:?]
at tech.pegasys.teku.networking.p2p.libp2p.rpc.RpcHandler$Controller.runHandler(RpcHandler.java:265) ~[teku-networking-p2p-develop.jar:UNKNOWN+g28f69d3]
at tech.pegasys.teku.networking.p2p.libp2p.rpc.RpcHandler$Controller.channelRead0(RpcHandler.java:186) ~[teku-networking-p2p-develop.jar:UNKNOWN+g28f69d3]
at tech.pegasys.teku.networking.p2p.libp2p.rpc.RpcHandler$Controller.channelRead0(RpcHandler.java:156) ~[teku-networking-p2p-develop.jar:UNKNOWN+g28f69d3]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.libp2p.etc.util.netty.mux.AbstractMuxHandler.childRead(AbstractMuxHandler.kt:71) ~[jvm-libp2p-1.2.2-RELEASE.jar:?]
at io.libp2p.mux.mplex.MplexHandler.channelRead(MplexHandler.kt:32) ~[jvm-libp2p-1.2.2-RELEASE.jar:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at tech.pegasys.teku.networking.p2p.libp2p.MuxFirewall$MuxFirewallHandler.channelRead(MuxFirewall.java:118) ~[teku-networking-p2p-develop.jar:UNKNOWN+g28f69d3]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageCodec.channelRead(ByteToMessageCodec.java:103) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:107) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.MessageToMessageCodec.channelRead(MessageToMessageCodec.java:120) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.CombinedChannelDuplexHandler$DelegatingChannelHandlerContext.fireChannelRead(CombinedChannelDuplexHandler.java:436) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:333) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:455) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.CombinedChannelDuplexHandler.channelRead(CombinedChannelDuplexHandler.java:251) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.base/java.lang.Thread.run(Unknown Source) [?:?]
Caused by: tech.pegasys.teku.networking.eth2.rpc.core.RpcException: [Code 3] Requested historical blocks are currently unavailable
at tech.pegasys.teku.networking.eth2.rpc.core.RpcResponseDecoder.lambda$decodeNextResponse$1(RpcResponseDecoder.java:128) ~[teku-networking-eth2-develop.jar:UNKNOWN+g28f69d3]
at java.base/java.util.Optional.map(Unknown Source) ~[?:?]
at tech.pegasys.teku.networking.eth2.rpc.core.RpcResponseDecoder.decodeNextResponse(RpcResponseDecoder.java:128) ~[teku-networking-eth2-develop.jar:UNKNOWN+g28f69d3]
at tech.pegasys.teku.networking.eth2.rpc.core.RpcResponseDecoder.decodeNextResponses(RpcResponseDecoder.java:70) ~[teku-networking-eth2-develop.jar:UNKNOWN+g28f69d3]
at tech.pegasys.teku.networking.eth2.rpc.core.Eth2OutgoingRequestHandler.processData(Eth2OutgoingRequestHandler.java:117) ~[teku-networking-eth2-develop.jar:UNKNOWN+g28f69d3]
... 55 more