diff --git a/chain/build.gradle b/chain/build.gradle index 8d9e5d254..b57657432 100644 --- a/chain/build.gradle +++ b/chain/build.gradle @@ -10,10 +10,20 @@ dependencies { implementation 'com.google.guava:guava' implementation 'io.projectreactor:reactor-core' + api "org.apache.commons:commons-collections4" testImplementation 'org.mockito:mockito-core' + testCompile 'io.projectreactor:reactor-test' // Gradle does not import test sources alongside with main sources // use a workaround until better solution will be found testImplementation project(':consensus').sourceSets.test.output + + testImplementation 'org.junit.jupiter:junit-jupiter:5.5.1' + + test { + useJUnitPlatform { + excludeTags 'FIX' + } + } } diff --git a/chain/src/main/java/org/ethereum/beacon/chain/BeaconChain.java b/chain/src/main/java/org/ethereum/beacon/chain/BeaconChain.java index 1dc675350..3fee37a00 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/BeaconChain.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/BeaconChain.java @@ -1,5 +1,6 @@ package org.ethereum.beacon.chain; +import org.ethereum.beacon.core.state.Checkpoint; import org.reactivestreams.Publisher; public interface BeaconChain { @@ -15,5 +16,22 @@ public interface BeaconChain { */ BeaconTuple getRecentlyProcessed(); + /** + * Returns the most recent justified checkpoint. + * + * @return a checkpoint. + */ + Publisher getJustifiedCheckpoints(); + + /** + * Returns the most recent finalized checkpoint. + * + *

Note: finalized checkpoints are published by {@link #getJustifiedCheckpoints()} + * either. + * + * @return a checkpoint. + */ + Publisher getFinalizedCheckpoints(); + void init(); } diff --git a/chain/src/main/java/org/ethereum/beacon/chain/DefaultBeaconChain.java b/chain/src/main/java/org/ethereum/beacon/chain/DefaultBeaconChain.java index a8636349d..3763c1c9d 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/DefaultBeaconChain.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/DefaultBeaconChain.java @@ -8,15 +8,16 @@ import org.apache.logging.log4j.Logger; import org.ethereum.beacon.chain.storage.BeaconChainStorage; import org.ethereum.beacon.chain.storage.BeaconTupleStorage; +import org.ethereum.beacon.consensus.BeaconChainSpec; import org.ethereum.beacon.consensus.BeaconStateEx; import org.ethereum.beacon.consensus.BlockTransition; -import org.ethereum.beacon.consensus.BeaconChainSpec; import org.ethereum.beacon.consensus.transition.EmptySlotTransition; import org.ethereum.beacon.consensus.verifier.BeaconBlockVerifier; import org.ethereum.beacon.consensus.verifier.BeaconStateVerifier; import org.ethereum.beacon.consensus.verifier.VerificationResult; import org.ethereum.beacon.core.BeaconBlock; import org.ethereum.beacon.core.BeaconState; +import org.ethereum.beacon.core.state.Checkpoint; import org.ethereum.beacon.core.types.SlotNumber; import org.ethereum.beacon.schedulers.Schedulers; import org.ethereum.beacon.stream.SimpleProcessor; @@ -36,6 +37,8 @@ public class DefaultBeaconChain implements MutableBeaconChain { private final BeaconTupleStorage tupleStorage; private final SimpleProcessor blockStream; + private final SimpleProcessor justifiedCheckpointStream; + private final SimpleProcessor finalizedCheckpointStream; private final Schedulers schedulers; private BeaconTuple recentlyProcessed; @@ -58,6 +61,10 @@ public DefaultBeaconChain( this.schedulers = schedulers; blockStream = new SimpleProcessor<>(schedulers.events(), "DefaultBeaconChain.block"); + justifiedCheckpointStream = + new SimpleProcessor<>(schedulers.events(), "DefaultBeaconChain.justifiedCheckpoint"); + finalizedCheckpointStream = + new SimpleProcessor<>(schedulers.events(), "DefaultBeaconChain.finalizedCheckpoint"); } @Override @@ -66,6 +73,8 @@ public void init() { throw new IllegalStateException("Couldn't start from empty storage"); } this.recentlyProcessed = fetchRecentTuple(); + justifiedCheckpointStream.onNext(fetchJustifiedCheckpoint()); + finalizedCheckpointStream.onNext(fetchFinalizedCheckpoint()); blockStream.onNext(new BeaconTupleDetails(recentlyProcessed)); } @@ -117,7 +126,7 @@ public synchronized ImportResult insert(BeaconBlock block) { BeaconTuple newTuple = BeaconTuple.of(block, postBlockState); tupleStorage.put(newTuple); - updateFinality(parentState, postBlockState); + updateFinality(postBlockState); chainStorage.commit(); @@ -144,15 +153,49 @@ public BeaconTuple getRecentlyProcessed() { return recentlyProcessed; } - private void updateFinality(BeaconState previous, BeaconState current) { - if (!previous.getFinalizedCheckpoint().equals(current.getFinalizedCheckpoint())) { + private void updateFinality(BeaconState current) { + boolean finalizedStorageUpdated = false; + boolean justifiedStorageUpdated = false; + if (current + .getFinalizedCheckpoint() + .getEpoch() + .greater(fetchFinalizedCheckpoint().getEpoch())) { chainStorage.getFinalizedStorage().set(current.getFinalizedCheckpoint()); + finalizedStorageUpdated = true; } - if (!previous.getCurrentJustifiedCheckpoint().equals(current.getCurrentJustifiedCheckpoint())) { + if (current + .getCurrentJustifiedCheckpoint() + .getEpoch() + .greater(fetchJustifiedCheckpoint().getEpoch())) { chainStorage.getJustifiedStorage().set(current.getCurrentJustifiedCheckpoint()); + justifiedStorageUpdated = true; + } + // publish updates after both storages have been updated + // the order can be important if a finalizedCheckpointStream subscriber will look + // into justified storage + // in general, it may be important to publish after commit has succeeded + if (finalizedStorageUpdated) { + finalizedCheckpointStream.onNext(current.getFinalizedCheckpoint()); + } + if (justifiedStorageUpdated) { + justifiedCheckpointStream.onNext(current.getCurrentJustifiedCheckpoint()); } } + private Checkpoint fetchJustifiedCheckpoint() { + return chainStorage + .getJustifiedStorage() + .get() + .orElseThrow(() -> new RuntimeException("Justified checkpoint not found")); + } + + private Checkpoint fetchFinalizedCheckpoint() { + return chainStorage + .getFinalizedStorage() + .get() + .orElseThrow(() -> new RuntimeException("Finalized checkpoint not found")); + } + private BeaconStateEx pullParentState(BeaconBlock block) { Optional parent = tupleStorage.get(block.getParentRoot()); checkArgument(parent.isPresent(), "No parent for block %s", block); @@ -187,4 +230,14 @@ private boolean rejectedByTime(BeaconBlock block) { public Publisher getBlockStatesStream() { return blockStream; } + + @Override + public Publisher getJustifiedCheckpoints() { + return justifiedCheckpointStream; + } + + @Override + public Publisher getFinalizedCheckpoints() { + return finalizedCheckpointStream; + } } diff --git a/chain/src/main/java/org/ethereum/beacon/chain/ForkChoiceProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/ForkChoiceProcessor.java new file mode 100644 index 000000000..375a48e3d --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/ForkChoiceProcessor.java @@ -0,0 +1,7 @@ +package org.ethereum.beacon.chain; + +import org.reactivestreams.Publisher; + +public interface ForkChoiceProcessor { + Publisher getChainHeads(); +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/LMDGhostProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/LMDGhostProcessor.java new file mode 100644 index 000000000..bccbe8cb5 --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/LMDGhostProcessor.java @@ -0,0 +1,183 @@ +package org.ethereum.beacon.chain; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import org.ethereum.beacon.chain.storage.BeaconChainStorage; +import org.ethereum.beacon.consensus.BeaconChainSpec; +import org.ethereum.beacon.consensus.spec.ForkChoice.LatestMessage; +import org.ethereum.beacon.consensus.spec.ForkChoice.Store; +import org.ethereum.beacon.core.BeaconBlock; +import org.ethereum.beacon.core.BeaconState; +import org.ethereum.beacon.core.operations.Attestation; +import org.ethereum.beacon.core.operations.attestation.AttestationData; +import org.ethereum.beacon.core.operations.slashing.IndexedAttestation; +import org.ethereum.beacon.core.state.Checkpoint; +import org.ethereum.beacon.core.types.SlotNumber; +import org.ethereum.beacon.core.types.ValidatorIndex; +import org.ethereum.beacon.schedulers.Schedulers; +import org.ethereum.beacon.stream.SimpleProcessor; +import org.reactivestreams.Publisher; +import reactor.core.publisher.Flux; +import reactor.core.scheduler.Scheduler; +import tech.pegasys.artemis.ethereum.core.Hash32; + +public class LMDGhostProcessor implements ForkChoiceProcessor { + + private final int SEARCH_LIMIT = Integer.MAX_VALUE; + + private final BeaconChainSpec spec; + private final BeaconChainStorage storage; + private final SimpleProcessor chainHeadStream; + + private final Map latestMessageStorage = new HashMap<>(); + private Checkpoint justifiedCheckpoint = Checkpoint.EMPTY; + private Hash32 currentHeadRoot = Hash32.ZERO; + + public LMDGhostProcessor( + BeaconChainSpec spec, + BeaconChainStorage storage, + Schedulers schedulers, + Publisher justifiedCheckpoints, + Publisher wireAttestations, + Publisher importedBlocks) { + this.spec = spec; + this.storage = storage; + + Scheduler scheduler = schedulers.newSingleThreadDaemon("lmd-ghost-processor").toReactor(); + this.chainHeadStream = new SimpleProcessor<>(scheduler, "LMDGhostProcessor.chainHeadStream"); + + Flux.from(justifiedCheckpoints).publishOn(scheduler).subscribe(this::onNewJustifiedCheckpoint); + Flux.from(wireAttestations).publishOn(scheduler).subscribe(this::onNewAttestation); + Flux.from(importedBlocks).publishOn(scheduler).subscribe(this::onNewImportedBlock); + } + + private void onNewImportedBlock(BeaconTuple tuple) { + if (!isJustifiedAncestor(tuple.getBlock())) { + return; + } + + for (Attestation attestation : tuple.getBlock().getBody().getAttestations()) { + List indices = + spec.get_attesting_indices( + tuple.getState(), attestation.getData(), attestation.getAggregationBits()); + processAttestation(indices, attestation.getData()); + } + + updateHead(); + } + + private boolean isJustifiedAncestor(BeaconBlock block) { + // genesis shortcut + if (justifiedCheckpoint.equals(Checkpoint.EMPTY) && block.getSlot().equals(SlotNumber.ZERO)) { + return true; + } + + BeaconBlock ancestor = block; + while (spec.compute_epoch_of_slot(ancestor.getSlot()) + .greaterEqual(justifiedCheckpoint.getEpoch())) { + Optional parent = storage.getBlockStorage().get(ancestor.getParentRoot()); + if (!parent.isPresent()) { + return false; + } + if (parent.get().getParentRoot().equals(justifiedCheckpoint.getRoot())) { + return true; + } + ancestor = parent.get(); + } + + return false; + } + + private void onNewAttestation(IndexedAttestation attestation) { + List indices = new ArrayList<>(attestation.getCustodyBit0Indices().listCopy()); + indices.addAll(attestation.getCustodyBit1Indices().listCopy()); + processAttestation(indices, attestation.getData()); + updateHead(); + } + + private void processAttestation(List indices, AttestationData data) { + LatestMessage message = + new LatestMessage(data.getTarget().getEpoch(), data.getBeaconBlockRoot()); + indices.forEach( + index -> { + latestMessageStorage.merge( + index, + message, + (oldMessage, newMessage) -> { + if (newMessage.getEpoch().greater(oldMessage.getEpoch())) { + return newMessage; + } else { + return oldMessage; + } + }); + }); + } + + private void updateHead() { + Hash32 newHeadRoot = getHeadRoot(); + if (!newHeadRoot.equals(currentHeadRoot)) { + BeaconTuple tuple = storage.getTupleStorage().get(newHeadRoot).get(); + currentHeadRoot = newHeadRoot; + chainHeadStream.onNext(new BeaconChainHead(tuple)); + } + } + + private Hash32 getHeadRoot() { + return spec.get_head( + new Store() { + + @Override + public Checkpoint getJustifiedCheckpoint() { + return storage.getJustifiedStorage().get().get(); + } + + @Override + public Checkpoint getFinalizedCheckpoint() { + return storage.getFinalizedStorage().get().get(); + } + + @Override + public Optional getBlock(Hash32 root) { + return storage.getBlockStorage().get(root); + } + + @Override + public Optional getState(Hash32 root) { + return storage.getStateStorage().get(root); + } + + @Override + public Optional getLatestMessage(ValidatorIndex index) { + return Optional.ofNullable(latestMessageStorage.get(index)); + } + + @Override + public List getChildren(Hash32 root) { + return storage.getBlockStorage().getChildren(root, SEARCH_LIMIT).stream() + .map(spec::signing_root) + .collect(Collectors.toList()); + } + }); + } + + private void onNewJustifiedCheckpoint(Checkpoint checkpoint) { + if (checkpoint.getEpoch().greater(justifiedCheckpoint.getEpoch())) { + justifiedCheckpoint = checkpoint; + resetLatestMessages(); + updateHead(); + } + } + + private void resetLatestMessages() { + latestMessageStorage.clear(); + } + + @Override + public Publisher getChainHeads() { + return chainHeadStream; + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/observer/ObservableStateProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/observer/ObservableStateProcessor.java index c4b69af58..d91eb083f 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/observer/ObservableStateProcessor.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/observer/ObservableStateProcessor.java @@ -1,14 +1,41 @@ package org.ethereum.beacon.chain.observer; import org.ethereum.beacon.chain.BeaconChainHead; +import org.ethereum.beacon.chain.pool.AttestationPool; +import org.ethereum.beacon.chain.pool.churn.AttestationChurn; +import org.ethereum.beacon.chain.pool.churn.AttestationChurnImpl; +import org.ethereum.beacon.consensus.BeaconChainSpec; +import org.ethereum.beacon.consensus.transition.EmptySlotTransition; +import org.ethereum.beacon.core.operations.Attestation; +import org.ethereum.beacon.core.state.Checkpoint; +import org.ethereum.beacon.core.types.SlotNumber; +import org.ethereum.beacon.schedulers.Schedulers; import org.reactivestreams.Publisher; public interface ObservableStateProcessor { void start(); - Publisher getHeadStream(); - Publisher getObservableStateStream(); - Publisher getPendingOperationsStream(); + static ObservableStateProcessor createNew( + BeaconChainSpec spec, + EmptySlotTransition emptySlotTransition, + Schedulers schedulers, + Publisher newSlots, + Publisher chainHeads, + Publisher justifiedCheckpoints, + Publisher finalizedCheckpoints, + Publisher validAttestations) { + AttestationChurn churn = new AttestationChurnImpl(spec, AttestationPool.ATTESTATION_CHURN_SIZE); + return new YetAnotherStateProcessor( + spec, + emptySlotTransition, + churn, + schedulers, + newSlots, + chainHeads, + justifiedCheckpoints, + finalizedCheckpoints, + validAttestations); + } } diff --git a/chain/src/main/java/org/ethereum/beacon/chain/observer/ObservableStateProcessorImpl.java b/chain/src/main/java/org/ethereum/beacon/chain/observer/ObservableStateProcessorImpl.java index 686caa61f..df3cacbe4 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/observer/ObservableStateProcessorImpl.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/observer/ObservableStateProcessorImpl.java @@ -14,7 +14,6 @@ import java.util.stream.Collectors; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.ethereum.beacon.chain.BeaconChainHead; import org.ethereum.beacon.chain.BeaconTuple; import org.ethereum.beacon.chain.BeaconTupleDetails; import org.ethereum.beacon.chain.LMDGhostHeadFunction; @@ -68,9 +67,7 @@ public class ObservableStateProcessorImpl implements ObservableStateProcessor { private final Map, Attestation> attestationCache = new HashMap<>(); private final Schedulers schedulers; - private final SimpleProcessor headStream; private final SimpleProcessor observableStateStream; - private final SimpleProcessor pendingOperationsStream; public ObservableStateProcessorImpl( BeaconChainStorage chainStorage, @@ -110,9 +107,7 @@ public ObservableStateProcessorImpl( this.schedulers = schedulers; this.maxEmptySlotTransitions = maxEmptySlotTransitions; - headStream = new SimpleProcessor<>(this.schedulers.events(), "ObservableStateProcessor.head"); observableStateStream = new SimpleProcessor<>(this.schedulers.events(), "ObservableStateProcessor.observableState"); - pendingOperationsStream = new SimpleProcessor<>(this.schedulers.events(), "PendingOperationsProcessor.pendingOperations"); } @Override @@ -230,7 +225,6 @@ private synchronized Map> copyAttestationCache private void newHead(BeaconTupleDetails head) { this.head = head; - headStream.onNext(new BeaconChainHead(this.head)); if (latestState == null) { latestState = head.getFinalState(); @@ -339,18 +333,8 @@ private void updateHead(BeaconState state) { newHead(tuple); } - @Override - public Publisher getHeadStream() { - return headStream; - } - @Override public Publisher getObservableStateStream() { return observableStateStream; } - - @Override - public Publisher getPendingOperationsStream() { - return pendingOperationsStream; - } } diff --git a/chain/src/main/java/org/ethereum/beacon/chain/observer/YetAnotherStateProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/observer/YetAnotherStateProcessor.java new file mode 100644 index 000000000..eba3fc2c4 --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/observer/YetAnotherStateProcessor.java @@ -0,0 +1,166 @@ +package org.ethereum.beacon.chain.observer; + +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; +import org.ethereum.beacon.chain.BeaconChainHead; +import org.ethereum.beacon.chain.BeaconTuple; +import org.ethereum.beacon.chain.pool.AttestationPool; +import org.ethereum.beacon.chain.pool.OffChainAggregates; +import org.ethereum.beacon.chain.pool.churn.AttestationChurn; +import org.ethereum.beacon.consensus.BeaconChainSpec; +import org.ethereum.beacon.consensus.BeaconStateEx; +import org.ethereum.beacon.consensus.transition.BeaconStateExImpl; +import org.ethereum.beacon.consensus.transition.EmptySlotTransition; +import org.ethereum.beacon.core.BeaconBlock; +import org.ethereum.beacon.core.operations.Attestation; +import org.ethereum.beacon.core.operations.ProposerSlashing; +import org.ethereum.beacon.core.operations.Transfer; +import org.ethereum.beacon.core.operations.VoluntaryExit; +import org.ethereum.beacon.core.operations.slashing.AttesterSlashing; +import org.ethereum.beacon.core.spec.SpecConstants; +import org.ethereum.beacon.core.state.Checkpoint; +import org.ethereum.beacon.core.types.SlotNumber; +import org.ethereum.beacon.schedulers.Schedulers; +import org.ethereum.beacon.stream.SimpleProcessor; +import org.reactivestreams.Publisher; +import reactor.core.publisher.Flux; +import reactor.core.scheduler.Scheduler; + +public class YetAnotherStateProcessor implements ObservableStateProcessor { + + private final AttestationChurn churn; + private final SimpleProcessor stateStream; + private final EmptySlotTransition emptySlotTransition; + private final BeaconChainSpec spec; + private final Scheduler scheduler; + private final Publisher newSlots; + private final Publisher chainHeads; + private final Publisher justifiedCheckpoints; + private final Publisher finalizedCheckpoints; + private final Publisher validAttestations; + + private SlotNumber recentSlot = SlotNumber.ZERO; + private BeaconStateEx recentState; + private BeaconBlock recentHead; + + public YetAnotherStateProcessor( + BeaconChainSpec spec, + EmptySlotTransition emptySlotTransition, + AttestationChurn churn, + Schedulers schedulers, + Publisher newSlots, + Publisher chainHeads, + Publisher justifiedCheckpoints, + Publisher finalizedCheckpoints, + Publisher validAttestations) { + this.spec = spec; + this.emptySlotTransition = emptySlotTransition; + this.churn = churn; + this.scheduler = schedulers.newSingleThreadDaemon("yet-another-state-processor").toReactor(); + this.stateStream = new SimpleProcessor<>(scheduler, "YetAnotherStateProcessor.stateStream"); + this.newSlots = newSlots; + this.justifiedCheckpoints = justifiedCheckpoints; + this.finalizedCheckpoints = finalizedCheckpoints; + this.validAttestations = validAttestations; + this.chainHeads = chainHeads; + } + + @Override + public void start() { + Flux.from(chainHeads).publishOn(scheduler).subscribe(this::onNewHead); + Flux.from(justifiedCheckpoints) + .publishOn(scheduler) + .subscribe(this.churn::feedJustifiedCheckpoint); + Flux.from(finalizedCheckpoints) + .publishOn(scheduler) + .subscribe(this.churn::feedFinalizedCheckpoint); + Flux.from(validAttestations) + .publishOn(scheduler) + .bufferTimeout( + AttestationPool.VERIFIER_BUFFER_SIZE, AttestationPool.VERIFIER_INTERVAL, scheduler) + .subscribe(this.churn::add); + Flux.from(newSlots).publishOn(scheduler).subscribe(this::onNewSlot); + } + + private void onNewHead(BeaconChainHead head) { + recentHead = head.getBlock(); + recentState = emptySlotTransition.apply(new BeaconStateExImpl(head.getState()), recentSlot); + publishObservableState(); + } + + private void onNewSlot(SlotNumber slot) { + if (slot.greater(recentSlot)) { + recentSlot = slot; + if (recentHead != null && recentState != null) { + recentState = emptySlotTransition.apply(recentState, slot); + publishObservableState(); + } + } + } + + private void publishObservableState() { + OffChainAggregates aggregates = churn.compute(BeaconTuple.of(recentHead, recentState)); + PendingOperations pendingOperations = new PendingOperationsImpl(aggregates); + stateStream.onNext(new ObservableBeaconState(recentHead, recentState, pendingOperations)); + } + + @Override + public Publisher getObservableStateStream() { + return stateStream; + } + + private final class PendingOperationsImpl implements PendingOperations { + + private final OffChainAggregates aggregates; + + private List attestations; + + public PendingOperationsImpl(OffChainAggregates aggregates) { + this.aggregates = aggregates; + } + + @Override + public List getAttestations() { + if (attestations == null) { + attestations = + aggregates.getAggregates().stream() + .map(aggregate -> aggregate.getAggregate(spec.getConstants())) + .collect(Collectors.toList()); + } + return attestations; + } + + @Override + public List peekProposerSlashings(int maxCount) { + return Collections.emptyList(); + } + + @Override + public List peekAttesterSlashings(int maxCount) { + return Collections.emptyList(); + } + + @Override + public List peekAggregateAttestations(int maxCount, SpecConstants specConstants) { + if (attestations == null) { + attestations = + aggregates.getAggregates().stream() + .limit(maxCount) + .map(aggregate -> aggregate.getAggregate(specConstants)) + .collect(Collectors.toList()); + } + return attestations.stream().limit(maxCount).collect(Collectors.toList()); + } + + @Override + public List peekExits(int maxCount) { + return Collections.emptyList(); + } + + @Override + public List peekTransfers(int maxCount) { + return Collections.emptyList(); + } + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationAggregate.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationAggregate.java new file mode 100644 index 000000000..15810e988 --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationAggregate.java @@ -0,0 +1,76 @@ +package org.ethereum.beacon.chain.pool; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import org.ethereum.beacon.core.operations.Attestation; +import org.ethereum.beacon.core.operations.attestation.AttestationData; +import org.ethereum.beacon.core.spec.SpecConstants; +import org.ethereum.beacon.core.types.BLSSignature; +import org.ethereum.beacon.crypto.BLS381; +import tech.pegasys.artemis.util.collections.Bitlist; + +public class AttestationAggregate { + + public static AttestationAggregate create(Attestation attestation) { + List signatures = new ArrayList<>(); + signatures.add(attestation.getSignature()); + return new AttestationAggregate( + attestation.getAggregationBits(), + attestation.getCustodyBits(), + attestation.getData(), + signatures); + } + + private Bitlist aggregationBits; + private Bitlist custodyBits; + private final AttestationData data; + private final List signatures; + + public AttestationAggregate( + Bitlist aggregationBits, + Bitlist custodyBits, + AttestationData data, + List signatures) { + this.aggregationBits = aggregationBits; + this.custodyBits = custodyBits; + this.data = data; + this.signatures = signatures; + } + + public boolean add(Attestation attestation) { + if (isAggregatable(attestation)) { + aggregationBits = aggregationBits.or(attestation.getAggregationBits()); + custodyBits = custodyBits.or(attestation.getCustodyBits()); + signatures.add(attestation.getSignature()); + + return true; + } else { + return false; + } + } + + private boolean isAggregatable(Attestation attestation) { + if (!data.equals(attestation.getData())) { + return false; + } + + if (!aggregationBits.and(attestation.getAggregationBits()).isEmpty()) { + return false; + } + + return true; + } + + public Attestation getAggregate(SpecConstants specConstants) { + BLSSignature signature = + BLSSignature.wrap( + BLS381.Signature.aggregate( + signatures.stream() + .map(BLS381.Signature::createWithoutValidation) + .collect(Collectors.toList())) + .getEncoded()); + + return new Attestation(aggregationBits, data, custodyBits, signature, specConstants); + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationPool.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationPool.java new file mode 100644 index 000000000..e9c588dcd --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationPool.java @@ -0,0 +1,139 @@ +package org.ethereum.beacon.chain.pool; + +import java.time.Duration; +import org.ethereum.beacon.chain.BeaconChain; +import org.ethereum.beacon.chain.pool.checker.SanityChecker; +import org.ethereum.beacon.chain.pool.checker.SignatureEncodingChecker; +import org.ethereum.beacon.chain.pool.checker.TimeFrameFilter; +import org.ethereum.beacon.chain.pool.churn.AttestationChurn; +import org.ethereum.beacon.chain.pool.registry.ProcessedAttestations; +import org.ethereum.beacon.chain.pool.registry.UnknownAttestationPool; +import org.ethereum.beacon.chain.pool.verifier.AttestationVerifier; +import org.ethereum.beacon.chain.pool.verifier.BatchVerifier; +import org.ethereum.beacon.chain.storage.BeaconChainStorage; +import org.ethereum.beacon.consensus.BeaconChainSpec; +import org.ethereum.beacon.consensus.transition.EmptySlotTransition; +import org.ethereum.beacon.core.BeaconBlock; +import org.ethereum.beacon.core.operations.Attestation; +import org.ethereum.beacon.core.operations.slashing.IndexedAttestation; +import org.ethereum.beacon.core.state.Checkpoint; +import org.ethereum.beacon.core.types.EpochNumber; +import org.ethereum.beacon.core.types.SlotNumber; +import org.ethereum.beacon.schedulers.Schedulers; +import org.reactivestreams.Publisher; + +/** + * Attestation pool API. + * + *

Along with {@link BeaconChain} attestation pool is one of the central components of the Beacon + * chain client. Its main responsibilities are to verify attestation coming from the wire and + * accumulate those of them which are not yet included on chain. + * + *

A list of attestation pool clients: + * + *

+ */ +public interface AttestationPool { + + /** A number of threads in the executor processing attestation pool. */ + int MAX_THREADS = 32; + + /** Discard attestations with target epoch greater than current epoch plus this number. */ + EpochNumber MAX_ATTESTATION_LOOKAHEAD = EpochNumber.of(1); + + /** + * Max number of attestations kept by processed attestations registry. An entry of this registry + * should be represented by a hash of registered attestation. + */ + int MAX_PROCESSED_ATTESTATIONS = 1_000_000; + + /** Max number of attestations made to not yet known block that could be kept in memory. */ + int MAX_UNKNOWN_ATTESTATIONS = 100_000; + + /** Max size of a buffer that collects attestations before passing them on the main verifier. */ + int VERIFIER_BUFFER_SIZE = 10_000; + + /** A throttling interval for verifier buffer. */ + Duration VERIFIER_INTERVAL = Duration.ofMillis(50); + + /** Max number of attestations held by {@link AttestationChurn}. */ + int ATTESTATION_CHURN_SIZE = 10_000; + + /** + * Valid attestations publisher. + * + * @return a publisher. + */ + Publisher getValid(); + + Publisher getValidUnboxed(); + + /** + * A publisher of valid indexed attestations. Publishes same attestations as {@link #getValid()} + * does. + * + * @return a publisher. + */ + Publisher getValidIndexed(); + + /** + * Invalid attestations publisher. + * + * @return a publisher. + */ + Publisher getInvalid(); + + /** + * Publishes attestations which block is not yet a known block. + * + *

These attestations should be passed to a wire module in order to request a block. + * + * @return a publisher. + */ + Publisher getUnknownAttestations(); + + /** Launches the pool. */ + void start(); + + static AttestationPool create( + Publisher source, + Publisher newSlots, + Publisher finalizedCheckpoints, + Publisher importedBlocks, + Schedulers schedulers, + BeaconChainSpec spec, + BeaconChainStorage storage, + EmptySlotTransition emptySlotTransition) { + + TimeFrameFilter timeFrameFilter = new TimeFrameFilter(spec, MAX_ATTESTATION_LOOKAHEAD); + SanityChecker sanityChecker = new SanityChecker(spec); + SignatureEncodingChecker encodingChecker = new SignatureEncodingChecker(); + ProcessedAttestations processedFilter = + new ProcessedAttestations(spec::hash_tree_root, MAX_PROCESSED_ATTESTATIONS); + UnknownAttestationPool unknownAttestationPool = + new UnknownAttestationPool( + storage.getBlockStorage(), spec, MAX_ATTESTATION_LOOKAHEAD, MAX_UNKNOWN_ATTESTATIONS); + BatchVerifier batchVerifier = + new AttestationVerifier(storage.getTupleStorage(), spec, emptySlotTransition); + + return new InMemoryAttestationPool( + source, + newSlots, + finalizedCheckpoints, + importedBlocks, + schedulers, + timeFrameFilter, + sanityChecker, + encodingChecker, + processedFilter, + unknownAttestationPool, + batchVerifier); + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPool.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPool.java new file mode 100644 index 000000000..95848c8d4 --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPool.java @@ -0,0 +1,164 @@ +package org.ethereum.beacon.chain.pool; + +import java.util.List; +import org.ethereum.beacon.chain.BeaconTuple; +import org.ethereum.beacon.chain.pool.checker.SanityChecker; +import org.ethereum.beacon.chain.pool.checker.SignatureEncodingChecker; +import org.ethereum.beacon.chain.pool.checker.TimeFrameFilter; +import org.ethereum.beacon.chain.pool.reactor.DoubleWorkProcessor; +import org.ethereum.beacon.chain.pool.reactor.IdentificationProcessor; +import org.ethereum.beacon.chain.pool.reactor.SanityProcessor; +import org.ethereum.beacon.chain.pool.reactor.SignatureEncodingProcessor; +import org.ethereum.beacon.chain.pool.reactor.TimeProcessor; +import org.ethereum.beacon.chain.pool.reactor.VerificationProcessor; +import org.ethereum.beacon.chain.pool.registry.ProcessedAttestations; +import org.ethereum.beacon.chain.pool.registry.UnknownAttestationPool; +import org.ethereum.beacon.chain.pool.verifier.BatchVerifier; +import org.ethereum.beacon.core.BeaconBlock; +import org.ethereum.beacon.core.operations.Attestation; +import org.ethereum.beacon.core.operations.slashing.IndexedAttestation; +import org.ethereum.beacon.core.state.Checkpoint; +import org.ethereum.beacon.core.types.SlotNumber; +import org.ethereum.beacon.schedulers.Schedulers; +import org.reactivestreams.Publisher; +import reactor.core.publisher.DirectProcessor; +import reactor.core.publisher.Flux; +import reactor.core.scheduler.Scheduler; + +/** + * An implementation of attestation pool based on Reactor + * library, one of the implementation of reactive streams. + */ +public class InMemoryAttestationPool implements AttestationPool { + + private final Publisher source; + private final Publisher newSlots; + private final Publisher finalizedCheckpoints; + private final Publisher importedBlocks; + private final Schedulers schedulers; + + private final TimeFrameFilter timeFrameFilter; + private final SanityChecker sanityChecker; + private final SignatureEncodingChecker encodingChecker; + private final ProcessedAttestations processedFilter; + private final UnknownAttestationPool unknownPool; + private final BatchVerifier verifier; + + private final DirectProcessor invalidAttestations = DirectProcessor.create(); + private final DirectProcessor validAttestations = DirectProcessor.create(); + private final DirectProcessor validIndexedAttestations = + DirectProcessor.create(); + private final DirectProcessor unknownAttestations = DirectProcessor.create(); + + public InMemoryAttestationPool( + Publisher source, + Publisher newSlots, + Publisher finalizedCheckpoints, + Publisher importedBlocks, + Schedulers schedulers, + TimeFrameFilter timeFrameFilter, + SanityChecker sanityChecker, + SignatureEncodingChecker encodingChecker, + ProcessedAttestations processedFilter, + UnknownAttestationPool unknownPool, + BatchVerifier batchVerifier) { + this.source = source; + this.newSlots = newSlots; + this.finalizedCheckpoints = finalizedCheckpoints; + this.importedBlocks = importedBlocks; + this.schedulers = schedulers; + this.timeFrameFilter = timeFrameFilter; + this.sanityChecker = sanityChecker; + this.encodingChecker = encodingChecker; + this.processedFilter = processedFilter; + this.unknownPool = unknownPool; + this.verifier = batchVerifier; + } + + @Override + public void start() { + org.ethereum.beacon.schedulers.Scheduler parallelExecutor = + schedulers.newParallelDaemon("attestation-pool-%d", AttestationPool.MAX_THREADS); + + // create sources + Flux sourceFx = Flux.from(source); + Flux newSlotsFx = Flux.from(newSlots); + Flux finalizedCheckpointsFx = Flux.from(finalizedCheckpoints); + Flux importedBlocksFx = Flux.from(importedBlocks); + + // check time frames + TimeProcessor timeProcessor = + new TimeProcessor( + timeFrameFilter, schedulers, sourceFx, finalizedCheckpointsFx, newSlotsFx); + + // run sanity check + SanityProcessor sanityProcessor = + new SanityProcessor(sanityChecker, schedulers, timeProcessor, finalizedCheckpointsFx); + + // discard already processed attestations + DoubleWorkProcessor doubleWorkProcessor = + new DoubleWorkProcessor(processedFilter, schedulers, sanityProcessor.getValid()); + + // check signature encoding + SignatureEncodingProcessor encodingProcessor = + new SignatureEncodingProcessor(encodingChecker, parallelExecutor, doubleWorkProcessor); + + // identify attestation target + IdentificationProcessor identificationProcessor = + new IdentificationProcessor( + unknownPool, schedulers, encodingProcessor.getValid(), newSlotsFx, importedBlocksFx); + + // verify attestations + Flux> verificationThrottle = + Flux.from(identificationProcessor.getIdentified()) + .publishOn(parallelExecutor.toReactor()) + .bufferTimeout(VERIFIER_BUFFER_SIZE, VERIFIER_INTERVAL, parallelExecutor.toReactor()); + VerificationProcessor verificationProcessor = + new VerificationProcessor(verifier, parallelExecutor, verificationThrottle); + + Scheduler outScheduler = schedulers.events().toReactor(); + // expose valid attestations + Flux.from(verificationProcessor.getValid()) + .publishOn(outScheduler) + .subscribe(validAttestations); + Flux.from(verificationProcessor.getValidIndexed()) + .publishOn(outScheduler) + .subscribe(validIndexedAttestations); + // expose not yet identified + Flux.from(identificationProcessor.getUnknown()) + .publishOn(outScheduler) + .subscribe(unknownAttestations); + // expose invalid attestations + Flux.merge( + sanityProcessor.getInvalid(), + encodingProcessor.getInvalid(), + verificationProcessor.getInvalid()) + .publishOn(outScheduler) + .subscribe(invalidAttestations); + } + + @Override + public Publisher getValid() { + return validAttestations; + } + + @Override + public Publisher getValidUnboxed() { + return validAttestations.map(ReceivedAttestation::getMessage); + } + + @Override + public Publisher getValidIndexed() { + return validIndexedAttestations; + } + + @Override + public Publisher getInvalid() { + return invalidAttestations; + } + + @Override + public Publisher getUnknownAttestations() { + return unknownAttestations; + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/OffChainAggregates.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/OffChainAggregates.java new file mode 100644 index 000000000..567eae21d --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/OffChainAggregates.java @@ -0,0 +1,35 @@ +package org.ethereum.beacon.chain.pool; + +import java.util.List; +import org.ethereum.beacon.core.types.SlotNumber; +import tech.pegasys.artemis.ethereum.core.Hash32; + +/** + * A DTO for aggregated attestations that are not yet included on chain. + * + *

Beacon block proposer should be fed with this data. + */ +public class OffChainAggregates { + private final Hash32 blockRoot; + private final SlotNumber slot; + private final List aggregates; + + public OffChainAggregates( + Hash32 blockRoot, SlotNumber slot, List aggregates) { + this.blockRoot = blockRoot; + this.slot = slot; + this.aggregates = aggregates; + } + + public Hash32 getBlockRoot() { + return blockRoot; + } + + public List getAggregates() { + return aggregates; + } + + public SlotNumber getSlot() { + return slot; + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/ReceivedAttestation.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/ReceivedAttestation.java new file mode 100644 index 000000000..371b8ee57 --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/ReceivedAttestation.java @@ -0,0 +1,30 @@ +package org.ethereum.beacon.chain.pool; + +import org.ethereum.beacon.core.operations.Attestation; +import org.ethereum.beacon.types.p2p.NodeId; + +/** An attestation received from the wire. */ +public class ReceivedAttestation { + + /** An id of a node sent this attestation. */ + private final NodeId sender; + /** An attestation message itself. */ + private final Attestation message; + + public ReceivedAttestation(NodeId sender, Attestation message) { + this.sender = sender; + this.message = message; + } + + public ReceivedAttestation(Attestation message) { + this(null, message); + } + + public NodeId getSender() { + return sender; + } + + public Attestation getMessage() { + return message; + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/StatefulProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/StatefulProcessor.java new file mode 100644 index 000000000..ea0d4d592 --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/StatefulProcessor.java @@ -0,0 +1,24 @@ +package org.ethereum.beacon.chain.pool; + +/** + * Stateful processor. + * + *

A processor that requires particular inner state to be initialized before it could be safely + * called by its clients. + * + *

{@link #isInitialized()} method indicates whether processor has already been initialized or + * not. It's a client responsibility to check {@link #isInitialized()} result before calling to the + * instance of this processor. + * + *

Implementor MAY throw an {@link AssertionError} if it's been called before inner state has + * been initialised. + */ +public interface StatefulProcessor { + + /** + * Checks whether processor state is initialized or not. + * + * @return {@code true} if processor is ready to work, {@link false}, otherwise. + */ + boolean isInitialized(); +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/checker/AttestationChecker.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/checker/AttestationChecker.java new file mode 100644 index 000000000..228acccfa --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/checker/AttestationChecker.java @@ -0,0 +1,21 @@ +package org.ethereum.beacon.chain.pool.checker; + +import org.ethereum.beacon.chain.pool.ReceivedAttestation; + +/** + * An interface of light weight attestation checker. + * + *

Implementations of this interface SHOULD NOT execute I/O operations or run checks that are in + * high demand to CPU resources. Usually, implementation of this interface runs quick checks that + * could be done with the attestation itself without involving any other data. + */ +public interface AttestationChecker { + + /** + * Given attestation runs a check. + * + * @param attestation an attestation to check. + * @return {@code true} if checks are passed successfully, {@code false} otherwise. + */ + boolean check(ReceivedAttestation attestation); +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/checker/SanityChecker.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/checker/SanityChecker.java new file mode 100644 index 000000000..e884ea9cd --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/checker/SanityChecker.java @@ -0,0 +1,71 @@ +package org.ethereum.beacon.chain.pool.checker; + +import org.ethereum.beacon.chain.pool.ReceivedAttestation; +import org.ethereum.beacon.chain.pool.StatefulProcessor; +import org.ethereum.beacon.consensus.BeaconChainSpec; +import org.ethereum.beacon.core.operations.attestation.AttestationData; +import org.ethereum.beacon.core.state.Checkpoint; +import tech.pegasys.artemis.ethereum.core.Hash32; + +/** + * Given attestation runs a number of sanity checks against it. + * + *

This is one of the first processors in attestation pool pipeline. Attestations that are not + * passing these checks SHOULD be considered invalid. + * + *

Note: this implementation is not thread-safe. + */ +public class SanityChecker implements AttestationChecker, StatefulProcessor { + + /** A beacon chain spec. */ + private final BeaconChainSpec spec; + /** Most recent finalized checkpoint. */ + private Checkpoint finalizedCheckpoint; + + public SanityChecker(BeaconChainSpec spec) { + this.spec = spec; + } + + @Override + public boolean check(ReceivedAttestation attestation) { + assert isInitialized(); + + final AttestationData data = attestation.getMessage().getData(); + + // sourceEpoch > targetEpoch + if (data.getSource().getEpoch().greater(data.getTarget().getEpoch())) { + return false; + } + + // finalizedEpoch == sourceEpoch && finalizedRoot != sourceRoot + // do not run this check for sourceRoot == ZERO + if (!data.getSource().getRoot().equals(Hash32.ZERO) + && data.getSource().getEpoch().equals(finalizedCheckpoint.getEpoch()) + && !finalizedCheckpoint.getRoot().equals(data.getSource().getRoot())) { + return false; + } + + // crosslinkShard >= SHARD_COUNT + if (data.getCrosslink().getShard().greaterEqual(spec.getConstants().getShardCount())) { + return false; + } + + return true; + } + + /** + * Update the most recent finalized checkpoint. + * + *

This method should be called each time new finalized checkpoint appears. + * + * @param checkpoint finalized checkpoint. + */ + public void feedFinalizedCheckpoint(Checkpoint checkpoint) { + this.finalizedCheckpoint = checkpoint; + } + + @Override + public boolean isInitialized() { + return finalizedCheckpoint != null; + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/checker/SignatureEncodingChecker.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/checker/SignatureEncodingChecker.java new file mode 100644 index 000000000..32daf0095 --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/checker/SignatureEncodingChecker.java @@ -0,0 +1,24 @@ +package org.ethereum.beacon.chain.pool.checker; + +import org.ethereum.beacon.chain.pool.ReceivedAttestation; +import org.ethereum.beacon.chain.pool.registry.ProcessedAttestations; +import org.ethereum.beacon.crypto.BLS381; + +/** + * Checks signature encoding format. + * + *

Attestations with invalid signature encoding SHOULD be considered as invalid. + * + *

This is relatively heavy check in terms of CPU cycles as it involves a few operations on a + * field numbers and one point multiplication. It's recommended to put this checker after {@link + * ProcessedAttestations} registry. + * + *

Note: this implementation is not thread-safe. + */ +public class SignatureEncodingChecker implements AttestationChecker { + + @Override + public boolean check(ReceivedAttestation attestation) { + return BLS381.Signature.validate(attestation.getMessage().getSignature()); + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/checker/TimeFrameFilter.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/checker/TimeFrameFilter.java new file mode 100644 index 000000000..2336552d1 --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/checker/TimeFrameFilter.java @@ -0,0 +1,88 @@ +package org.ethereum.beacon.chain.pool.checker; + +import org.ethereum.beacon.chain.pool.ReceivedAttestation; +import org.ethereum.beacon.chain.pool.StatefulProcessor; +import org.ethereum.beacon.consensus.BeaconChainSpec; +import org.ethereum.beacon.core.operations.attestation.AttestationData; +import org.ethereum.beacon.core.state.Checkpoint; +import org.ethereum.beacon.core.types.EpochNumber; +import org.ethereum.beacon.core.types.SlotNumber; + +/** + * Filters attestations by time frame of its target and source. + * + *

Attestations not passing these checks SHOULD NOT be considered as invalid, they rather SHOULD + * be considered as those which client is not interested in. Filtered out attestations SHOULD merely + * be discarded. + * + *

This is the first filter in attestation processors pipeline. + * + *

Note: this implementation is not thread-safe. + */ +public class TimeFrameFilter implements AttestationChecker, StatefulProcessor { + + /** A beacon chain spec. */ + private final BeaconChainSpec spec; + /** Accept attestations not older than current epoch plus this number. */ + private final EpochNumber maxAttestationLookahead; + + /** Most recent finalized checkpoint. */ + private Checkpoint finalizedCheckpoint; + /** Upper time frame boundary. */ + private EpochNumber maxAcceptableEpoch; + + public TimeFrameFilter(BeaconChainSpec spec, EpochNumber maxAttestationLookahead) { + this.spec = spec; + this.maxAttestationLookahead = maxAttestationLookahead; + } + + @Override + public boolean isInitialized() { + return finalizedCheckpoint != null && maxAcceptableEpoch != null; + } + + @Override + public boolean check(ReceivedAttestation attestation) { + assert isInitialized(); + + final AttestationData data = attestation.getMessage().getData(); + + // targetEpoch < finalizedEpoch + if (data.getTarget().getEpoch().less(finalizedCheckpoint.getEpoch())) { + return false; + } + + // sourceEpoch < finalizedEpoch + if (data.getSource().getEpoch().less(finalizedCheckpoint.getEpoch())) { + return false; + } + + // targetEpoch > maxAcceptableEpoch + if (data.getTarget().getEpoch().greater(maxAcceptableEpoch)) { + return false; + } + + return true; + } + + /** + * Update the most recent finalized checkpoint. + * + *

This method should be called each time new finalized checkpoint appears. + * + * @param checkpoint finalized checkpoint. + */ + public void feedFinalizedCheckpoint(Checkpoint checkpoint) { + this.finalizedCheckpoint = checkpoint; + this.maxAcceptableEpoch = checkpoint.getEpoch().plus(maxAttestationLookahead); + } + + /** + * This method should be called on each new slot. + * + * @param newSlot a new slot. + */ + public void feedNewSlot(SlotNumber newSlot) { + this.maxAcceptableEpoch = spec.compute_epoch_of_slot(newSlot).plus(maxAttestationLookahead); + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/churn/AttestationChurn.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/churn/AttestationChurn.java new file mode 100644 index 000000000..77500384c --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/churn/AttestationChurn.java @@ -0,0 +1,27 @@ +package org.ethereum.beacon.chain.pool.churn; + +import java.util.List; +import org.ethereum.beacon.chain.BeaconTuple; +import org.ethereum.beacon.chain.pool.OffChainAggregates; +import org.ethereum.beacon.chain.pool.StatefulProcessor; +import org.ethereum.beacon.consensus.BeaconChainSpec; +import org.ethereum.beacon.core.operations.Attestation; +import org.ethereum.beacon.core.state.Checkpoint; +import org.ethereum.beacon.core.types.SlotNumber; + +public interface AttestationChurn extends StatefulProcessor { + + OffChainAggregates compute(BeaconTuple tuple); + + void add(List attestation); + + void feedFinalizedCheckpoint(Checkpoint checkpoint); + + void feedJustifiedCheckpoint(Checkpoint checkpoint); + + void feedNewSlot(SlotNumber slotNumber); + + static AttestationChurn create(BeaconChainSpec spec, long size) { + return new AttestationChurnImpl(spec, size); + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/churn/AttestationChurnImpl.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/churn/AttestationChurnImpl.java new file mode 100644 index 000000000..3c0d4fb95 --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/churn/AttestationChurnImpl.java @@ -0,0 +1,178 @@ +package org.ethereum.beacon.chain.pool.churn; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.ethereum.beacon.chain.BeaconTuple; +import org.ethereum.beacon.chain.pool.AttestationAggregate; +import org.ethereum.beacon.chain.pool.OffChainAggregates; +import org.ethereum.beacon.consensus.BeaconChainSpec; +import org.ethereum.beacon.core.BeaconState; +import org.ethereum.beacon.core.MutableBeaconState; +import org.ethereum.beacon.core.operations.Attestation; +import org.ethereum.beacon.core.operations.attestation.AttestationData; +import org.ethereum.beacon.core.state.Checkpoint; +import org.ethereum.beacon.core.types.EpochNumber; +import org.ethereum.beacon.core.types.SlotNumber; +import tech.pegasys.artemis.util.collections.Bitlist; +import tech.pegasys.artemis.util.uint.UInt64s; + +public class AttestationChurnImpl implements AttestationChurn { + + /** A queue that maintains pooled attestations. */ + private final ChurnQueue queue; + /** A beacon chain spec. */ + private final BeaconChainSpec spec; + + private Checkpoint justifiedCheckpoint; + private EpochNumber lowerEpoch = EpochNumber.ZERO; + private EpochNumber upperEpoch = EpochNumber.ZERO; + + public AttestationChurnImpl(BeaconChainSpec spec, long size) { + this.spec = spec; + this.queue = new ChurnQueue(size); + + queue.updateEpochBoundaries(lowerEpoch, upperEpoch); + } + + @Override + public OffChainAggregates compute(BeaconTuple tuple) { + assert isInitialized(); + + // update epoch boundaries + updateEpochBoundaries( + spec.get_previous_epoch(tuple.getState()), spec.get_current_epoch(tuple.getState())); + + if (queue.isEmpty()) { + return new OffChainAggregates( + spec.signing_root(tuple.getBlock()), tuple.getState().getSlot(), Collections.emptyList()); + } + + // compute coverage + Map coverage = computeCoverage(tuple.getState()); + + // check attestations against coverage and state + MutableBeaconState state = tuple.getState().createMutableCopy(); + List offChainAttestations = + queue.stream() + .filter( + attestation -> { + Bitlist bits = coverage.get(attestation.getData()); + if (bits == null) { + return true; + } + + return bits.and(attestation.getAggregationBits()).isEmpty(); + }) + .sorted( + Comparator.comparing(attestation -> attestation.getData().getTarget().getEpoch())) + .filter( + attestation -> { + if (spec.verify_attestation_impl(state, attestation, false)) { + spec.process_attestation(state, attestation); + return true; + } else { + return false; + } + }) + .collect(Collectors.toList()); + + // compute aggregates + List aggregates = computeAggregates(offChainAttestations); + + return new OffChainAggregates( + spec.signing_root(tuple.getBlock()), tuple.getState().getSlot(), aggregates); + } + + @Override + public void feedFinalizedCheckpoint(Checkpoint checkpoint) { + // finalized checkpoint takes precedence + updateJustifiedCheckpoint(checkpoint); + } + + @Override + public void feedJustifiedCheckpoint(Checkpoint checkpoint) { + // discard forks if justified checkpoint is updated + if (justifiedCheckpoint == null + || checkpoint.getEpoch().greater(justifiedCheckpoint.getEpoch())) { + updateJustifiedCheckpoint(checkpoint); + } + } + + @Override + public void feedNewSlot(SlotNumber slotNumber) { + EpochNumber epoch = spec.compute_epoch_of_slot(slotNumber); + updateEpochBoundaries(epoch.equals(EpochNumber.ZERO) ? epoch : epoch.decrement(), epoch); + } + + private void updateJustifiedCheckpoint(Checkpoint checkpoint) { + updateEpochBoundaries(checkpoint.getEpoch(), UInt64s.max(checkpoint.getEpoch(), upperEpoch)); + this.justifiedCheckpoint = checkpoint; + } + + private void updateEpochBoundaries(EpochNumber newLower, EpochNumber newUpper) { + assert newLower.lessEqual(newUpper); + + // return if there is nothing to update + if (!(newLower.greater(lowerEpoch) || newUpper.greater(upperEpoch))) { + return; + } + + queue.updateEpochBoundaries(newLower, newUpper); + + lowerEpoch = newLower; + upperEpoch = newUpper; + } + + @Override + public boolean isInitialized() { + return true; + } + + @Override + public void add(List attestations) { + queue.add(attestations); + } + + private Map computeCoverage(BeaconState state) { + Map coverage = new HashMap<>(); + + Stream.concat( + state.getPreviousEpochAttestations().stream(), + state.getCurrentEpochAttestations().stream()) + .forEach( + pending -> { + Bitlist bitlist = coverage.get(pending.getData()); + if (bitlist == null) { + coverage.put(pending.getData(), pending.getAggregationBits()); + } else { + coverage.put(pending.getData(), bitlist.or(pending.getAggregationBits())); + } + }); + + return coverage; + } + + private List computeAggregates(List attestations) { + if (attestations.isEmpty()) { + return Collections.emptyList(); + } + + List aggregates = new ArrayList<>(); + AttestationAggregate current = null; + + for (Attestation attestation : attestations) { + if (current == null || !current.add(attestation)) { + current = AttestationAggregate.create(attestation); + aggregates.add(current); + } + } + + return aggregates; + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/churn/ChurnQueue.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/churn/ChurnQueue.java new file mode 100644 index 000000000..9b18501ad --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/churn/ChurnQueue.java @@ -0,0 +1,101 @@ +package org.ethereum.beacon.chain.pool.churn; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.LinkedList; +import java.util.List; +import java.util.stream.Stream; +import org.ethereum.beacon.core.operations.Attestation; +import org.ethereum.beacon.core.types.EpochNumber; + +final class ChurnQueue { + + private final long maxSize; + private final LinkedList buckets = new LinkedList<>(); + + private EpochNumber lowerEpoch; + private EpochNumber upperEpoch; + private long size = 0; + + public ChurnQueue(long maxSize) { + this.maxSize = maxSize; + } + + boolean isEmpty() { + return size == 0; + } + + Stream stream() { + return buckets.stream().map(Bucket::getAttestations).flatMap(List::stream); + } + + void add(List attestation) { + attestation.forEach(this::addImpl); + // heavy operation, don't wanna run it after each added attestation + purgeQueue(); + } + + private void addImpl(Attestation attestation) { + EpochNumber epoch = attestation.getData().getTarget().getEpoch(); + if (epoch.greaterEqual(lowerEpoch) && epoch.lessEqual(upperEpoch)) { + Bucket bucket = getOrCreateBucket(epoch); + bucket.attestations.add(attestation); + + size += 1; + } + } + + void updateEpochBoundaries(EpochNumber lower, EpochNumber upper) { + assert lower.lessEqual(upper); + + while (buckets.size() > 0 && buckets.getFirst().epoch.less(lower)) { + Bucket detached = buckets.removeFirst(); + size -= detached.attestations.size(); + } + + while (buckets.size() > 0 && buckets.getLast().epoch.greater(upper)) { + Bucket detached = buckets.removeLast(); + size -= detached.attestations.size(); + } + + this.lowerEpoch = lower; + this.upperEpoch = upper; + } + + Bucket getOrCreateBucket(EpochNumber epoch) { + for (Bucket bucket : buckets) { + if (bucket.epoch.equals(epoch)) { + return bucket; + } + } + + Bucket newBucket = new Bucket(epoch); + buckets.add(newBucket); + buckets.sort(Comparator.comparing(Bucket::getEpoch)); + + return newBucket; + } + + void purgeQueue() { + if (maxSize > size) { + // TODO calculate weights and sieve the attestations + } + } + + private static final class Bucket { + private final EpochNumber epoch; + private final List attestations = new ArrayList<>(); + + Bucket(EpochNumber epoch) { + this.epoch = epoch; + } + + EpochNumber getEpoch() { + return epoch; + } + + List getAttestations() { + return attestations; + } + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/ChurnProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/ChurnProcessor.java new file mode 100644 index 000000000..5e763ded9 --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/ChurnProcessor.java @@ -0,0 +1,62 @@ +package org.ethereum.beacon.chain.pool.reactor; + +import org.ethereum.beacon.chain.BeaconTuple; +import org.ethereum.beacon.chain.pool.AttestationPool; +import org.ethereum.beacon.chain.pool.OffChainAggregates; +import org.ethereum.beacon.chain.pool.ReceivedAttestation; +import org.ethereum.beacon.chain.pool.churn.AttestationChurn; +import org.ethereum.beacon.core.state.Checkpoint; +import org.ethereum.beacon.core.types.SlotNumber; +import org.ethereum.beacon.schedulers.Schedulers; +import org.ethereum.beacon.stream.SimpleProcessor; +import org.reactivestreams.Publisher; +import reactor.core.CoreSubscriber; +import reactor.core.publisher.Flux; +import reactor.core.scheduler.Scheduler; + +public class ChurnProcessor extends Flux { + + private final AttestationChurn churn; + private final SimpleProcessor out; + + public ChurnProcessor( + AttestationChurn churn, + Schedulers schedulers, + Publisher newSlots, + Publisher chainHeads, + Publisher justifiedCheckpoints, + Publisher finalizedCheckpoints, + Publisher source) { + this.churn = churn; + + Scheduler scheduler = schedulers.newSingleThreadDaemon("pool-churn-processor").toReactor(); + this.out = new SimpleProcessor<>(scheduler, "ChurnProcessor.out"); + + Flux.from(chainHeads).publishOn(scheduler).subscribe(this::hookOnNext); + Flux.from(newSlots).publishOn(scheduler).subscribe(this.churn::feedNewSlot); + Flux.from(justifiedCheckpoints) + .publishOn(scheduler) + .subscribe(this.churn::feedJustifiedCheckpoint); + Flux.from(finalizedCheckpoints) + .publishOn(scheduler) + .subscribe(this.churn::feedFinalizedCheckpoint); + Flux.from(source) + .publishOn(scheduler) + .map(ReceivedAttestation::getMessage) + .bufferTimeout( + AttestationPool.VERIFIER_BUFFER_SIZE, AttestationPool.VERIFIER_INTERVAL, scheduler) + .subscribe(this.churn::add); + } + + private void hookOnNext(BeaconTuple tuple) { + if (churn.isInitialized()) { + OffChainAggregates aggregates = churn.compute(tuple); + out.onNext(aggregates); + } + } + + @Override + public void subscribe(CoreSubscriber actual) { + out.subscribe(actual); + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/DoubleWorkProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/DoubleWorkProcessor.java new file mode 100644 index 000000000..9101b0dcb --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/DoubleWorkProcessor.java @@ -0,0 +1,55 @@ +package org.ethereum.beacon.chain.pool.reactor; + +import org.ethereum.beacon.chain.pool.ReceivedAttestation; +import org.ethereum.beacon.chain.pool.registry.ProcessedAttestations; +import org.ethereum.beacon.schedulers.Schedulers; +import org.ethereum.beacon.stream.SimpleProcessor; +import org.reactivestreams.Publisher; +import reactor.core.CoreSubscriber; +import reactor.core.publisher.Flux; +import reactor.core.scheduler.Scheduler; + +/** + * Passes attestations through {@link ProcessedAttestations} filter. + * + *

Input: + * + *

    + *
  • attestations + *
+ * + *

Output: + * + *

    + *
  • Not yet processed attestations + *
+ */ +public class DoubleWorkProcessor extends Flux { + + private final ProcessedAttestations processedAttestations; + private final SimpleProcessor out; + + public DoubleWorkProcessor( + ProcessedAttestations processedAttestations, + Schedulers schedulers, + Publisher source) { + this.processedAttestations = processedAttestations; + + Scheduler scheduler = + schedulers.newSingleThreadDaemon("pool-double-work-processor").toReactor(); + this.out = new SimpleProcessor<>(scheduler, "DoubleWorkProcessor.out"); + + Flux.from(source).publishOn(scheduler).subscribe(this::hookOnNext); + } + + private void hookOnNext(ReceivedAttestation attestation) { + if (processedAttestations.add(attestation)) { + out.onNext(attestation); + } + } + + @Override + public void subscribe(CoreSubscriber actual) { + out.subscribe(actual); + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/IdentificationProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/IdentificationProcessor.java new file mode 100644 index 000000000..9fd8e7b3a --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/IdentificationProcessor.java @@ -0,0 +1,77 @@ +package org.ethereum.beacon.chain.pool.reactor; + +import org.ethereum.beacon.chain.pool.ReceivedAttestation; +import org.ethereum.beacon.chain.pool.registry.UnknownAttestationPool; +import org.ethereum.beacon.core.BeaconBlock; +import org.ethereum.beacon.core.types.SlotNumber; +import org.ethereum.beacon.schedulers.Schedulers; +import org.ethereum.beacon.stream.SimpleProcessor; +import org.reactivestreams.Publisher; +import reactor.core.publisher.Flux; +import reactor.core.scheduler.Scheduler; + +/** + * Passes attestations through {@link UnknownAttestationPool}. + * + *

Input: + * + *

    + *
  • newly imported blocks + *
  • new slots + *
  • attestations + *
+ * + *

Output: + * + *

    + *
  • instantly identified attestations + *
  • attestations identified upon a new block come + *
  • attestations with not yet imported block + *
+ */ +public class IdentificationProcessor { + + private final UnknownAttestationPool pool; + private final SimpleProcessor identified; + private final SimpleProcessor unknown; + + public IdentificationProcessor( + UnknownAttestationPool pool, + Schedulers schedulers, + Publisher source, + Publisher newSlots, + Publisher newImportedBlocks) { + this.pool = pool; + + Scheduler scheduler = + schedulers.newSingleThreadDaemon("pool-identification-processor").toReactor(); + this.identified = new SimpleProcessor<>(scheduler, "IdentificationProcessor.identified"); + this.unknown = new SimpleProcessor<>(scheduler, "IdentificationProcessor.unknown"); + + Flux.from(newSlots).publishOn(scheduler).subscribe(this.pool::feedNewSlot); + Flux.from(newImportedBlocks).publishOn(scheduler).subscribe(this::hookOnNext); + Flux.from(source).publishOn(scheduler).subscribe(this::hookOnNext); + } + + private void hookOnNext(BeaconBlock block) { + pool.feedNewImportedBlock(block).forEach(identified::onNext); + } + + private void hookOnNext(ReceivedAttestation attestation) { + if (pool.isInitialized()) { + if (pool.add(attestation)) { + unknown.onNext(attestation); + } else { + identified.onNext(attestation); + } + } + } + + public Publisher getIdentified() { + return identified; + } + + public Publisher getUnknown() { + return unknown; + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SanityProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SanityProcessor.java new file mode 100644 index 000000000..5ea29efda --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SanityProcessor.java @@ -0,0 +1,72 @@ +package org.ethereum.beacon.chain.pool.reactor; + +import org.ethereum.beacon.chain.pool.ReceivedAttestation; +import org.ethereum.beacon.chain.pool.checker.SanityChecker; +import org.ethereum.beacon.core.state.Checkpoint; +import org.ethereum.beacon.schedulers.Schedulers; +import org.ethereum.beacon.stream.SimpleProcessor; +import org.reactivestreams.Publisher; +import reactor.core.publisher.Flux; +import reactor.core.scheduler.Scheduler; + +/** + * Passes attestations through a {@link SanityChecker}. + * + *

Input: + * + *

    + *
  • recently finalized checkpoints + *
  • attestations + *
+ * + *

Output: + * + *

    + *
  • attestations successfully passed sanity checks + *
  • invalid attestations + *
+ */ +public class SanityProcessor { + + private final SanityChecker checker; + + private final SimpleProcessor valid; + private final SimpleProcessor invalid; + + public SanityProcessor( + SanityChecker checker, + Schedulers schedulers, + Publisher source, + Publisher finalizedCheckpoints) { + this.checker = checker; + + Scheduler scheduler = schedulers.newSingleThreadDaemon("pool-sanity-processor").toReactor(); + this.valid = new SimpleProcessor<>(scheduler, "SanityProcessor.valid"); + this.invalid = new SimpleProcessor<>(scheduler, "SanityProcessor.invalid"); + + Flux.from(finalizedCheckpoints) + .publishOn(scheduler) + .subscribe(this.checker::feedFinalizedCheckpoint); + Flux.from(source).publishOn(scheduler).subscribe(this::hookOnNext); + } + + private void hookOnNext(ReceivedAttestation attestation) { + if (!checker.isInitialized()) { + return; + } + + if (checker.check(attestation)) { + valid.onNext(attestation); + } else { + invalid.onNext(attestation); + } + } + + public Publisher getValid() { + return valid; + } + + public Publisher getInvalid() { + return invalid; + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SignatureEncodingProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SignatureEncodingProcessor.java new file mode 100644 index 000000000..09b54e097 --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SignatureEncodingProcessor.java @@ -0,0 +1,59 @@ +package org.ethereum.beacon.chain.pool.reactor; + +import org.ethereum.beacon.chain.pool.ReceivedAttestation; +import org.ethereum.beacon.chain.pool.checker.SignatureEncodingChecker; +import org.ethereum.beacon.schedulers.Scheduler; +import org.ethereum.beacon.stream.SimpleProcessor; +import org.reactivestreams.Publisher; +import reactor.core.publisher.Flux; + +/** + * Passes attestations through {@link SignatureEncodingChecker}. + * + *

Input: + * + *

    + *
  • attestations + *
+ * + *

Output: + * + *

    + *
  • attestations with valid signature encoding + *
  • invalid attestations + *
+ */ +public class SignatureEncodingProcessor { + + private final SignatureEncodingChecker checker; + private final SimpleProcessor valid; + private final SimpleProcessor invalid; + + public SignatureEncodingProcessor( + SignatureEncodingChecker checker, + Scheduler scheduler, + Publisher source) { + this.checker = checker; + + this.valid = new SimpleProcessor<>(scheduler, "SignatureEncodingProcessor.valid"); + this.invalid = new SimpleProcessor<>(scheduler, "SignatureEncodingProcessor.invalid"); + + Flux.from(source).publishOn(scheduler.toReactor()).subscribe(this::hookOnNext); + } + + private void hookOnNext(ReceivedAttestation attestation) { + if (checker.check(attestation)) { + valid.onNext(attestation); + } else { + invalid.onNext(attestation); + } + } + + public Publisher getValid() { + return valid; + } + + public Publisher getInvalid() { + return invalid; + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/TimeProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/TimeProcessor.java new file mode 100644 index 000000000..a877a9de9 --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/TimeProcessor.java @@ -0,0 +1,67 @@ +package org.ethereum.beacon.chain.pool.reactor; + +import org.ethereum.beacon.chain.pool.ReceivedAttestation; +import org.ethereum.beacon.chain.pool.checker.TimeFrameFilter; +import org.ethereum.beacon.core.state.Checkpoint; +import org.ethereum.beacon.core.types.SlotNumber; +import org.ethereum.beacon.schedulers.Schedulers; +import org.ethereum.beacon.stream.SimpleProcessor; +import org.reactivestreams.Publisher; +import reactor.core.CoreSubscriber; +import reactor.core.publisher.Flux; +import reactor.core.scheduler.Scheduler; + +/** + * Passes attestations through a {@link TimeFrameFilter}. + * + *

Input: + * + *

    + *
  • recently finalized checkpoints + *
  • new slots + *
  • attestations + *
+ * + *

Output: + * + *

    + *
  • attestations satisfying time frames + *
+ */ +public class TimeProcessor extends Flux { + + private final TimeFrameFilter filter; + private final SimpleProcessor out; + + public TimeProcessor( + TimeFrameFilter filter, + Schedulers schedulers, + Publisher source, + Publisher finalizedCheckpoints, + Publisher newSlots) { + this.filter = filter; + + Scheduler scheduler = schedulers.newSingleThreadDaemon("pool-time-processor").toReactor(); + this.out = new SimpleProcessor<>(scheduler, "TimeProcessor.out"); + + Flux.from(source).publishOn(scheduler).subscribe(this::hookOnNext); + Flux.from(finalizedCheckpoints) + .publishOn(scheduler) + .subscribe(this.filter::feedFinalizedCheckpoint); + + Flux.from(newSlots) + .publishOn(scheduler) + .subscribe(this.filter::feedNewSlot); + } + + private void hookOnNext(ReceivedAttestation attestation) { + if (filter.isInitialized() && filter.check(attestation)) { + out.onNext(attestation); + } + } + + @Override + public void subscribe(CoreSubscriber actual) { + out.subscribe(actual); + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/VerificationProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/VerificationProcessor.java new file mode 100644 index 000000000..5401b9f2c --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/VerificationProcessor.java @@ -0,0 +1,66 @@ +package org.ethereum.beacon.chain.pool.reactor; + +import java.util.List; +import org.ethereum.beacon.chain.pool.ReceivedAttestation; +import org.ethereum.beacon.chain.pool.verifier.BatchVerifier; +import org.ethereum.beacon.chain.pool.verifier.VerificationResult; +import org.ethereum.beacon.core.operations.slashing.IndexedAttestation; +import org.ethereum.beacon.schedulers.Scheduler; +import org.ethereum.beacon.stream.SimpleProcessor; +import org.reactivestreams.Publisher; +import reactor.core.publisher.Flux; + +/** + * Passes attestations through {@link BatchVerifier}. + * + *

Input: + * + *

    + *
  • batches of {@link ReceivedAttestation} + *
+ * + *

Output: + * + *

    + *
  • attestations successfully passed verification + *
  • invalid attestations + *
+ */ +public class VerificationProcessor { + + private final BatchVerifier verifier; + + private final SimpleProcessor valid; + private final SimpleProcessor validIndexed; + private final SimpleProcessor invalid; + + public VerificationProcessor( + BatchVerifier verifier, Scheduler scheduler, Publisher> source) { + this.verifier = verifier; + + this.valid = new SimpleProcessor<>(scheduler, "VerificationProcessor.valid"); + this.validIndexed = new SimpleProcessor<>(scheduler, "VerificationProcessor.validIndexed"); + this.invalid = new SimpleProcessor<>(scheduler, "VerificationProcessor.invalid"); + + Flux.from(source).publishOn(scheduler.toReactor()).subscribe(this::hookOnNext); + } + + private void hookOnNext(List batch) { + VerificationResult result = verifier.verify(batch); + result.getValid().forEach(valid::onNext); + result.getValidIndexed().forEach(validIndexed::onNext); + result.getInvalid().forEach(invalid::onNext); + } + + public Publisher getValid() { + return valid; + } + + public SimpleProcessor getValidIndexed() { + return validIndexed; + } + + public Publisher getInvalid() { + return invalid; + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/AttestationRegistry.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/AttestationRegistry.java new file mode 100644 index 000000000..7122fd8c5 --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/AttestationRegistry.java @@ -0,0 +1,21 @@ +package org.ethereum.beacon.chain.pool.registry; + +import org.ethereum.beacon.chain.pool.ReceivedAttestation; + +/** + * An attestation registry interface. + * + *

Usually, an implementation of this interface tracks a set of attestation or identities of + * attestations passed on it. + */ +public interface AttestationRegistry { + + /** + * Adds attestation to the registry. + * + * @param attestation an attestation to be registered. + * @return {@link true} if an attestation is new to the registry, {@link false} if the attestation + * has been already added. + */ + boolean add(ReceivedAttestation attestation); +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/ProcessedAttestations.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/ProcessedAttestations.java new file mode 100644 index 000000000..82e0a4edb --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/ProcessedAttestations.java @@ -0,0 +1,37 @@ +package org.ethereum.beacon.chain.pool.registry; + +import java.util.function.Function; +import org.apache.commons.collections4.map.LRUMap; +import org.ethereum.beacon.chain.pool.ReceivedAttestation; +import org.ethereum.beacon.core.operations.Attestation; +import tech.pegasys.artemis.ethereum.core.Hash32; + +/** + * A registry that stores relatively big number of attestation hashes. + * + *

This particular implementation is based on LRU map. + * + *

It's recommended to place this registry prior to highly resource demanded processors in order + * to prevent double work. + * + *

Note: this implementation is not thread-safe. + */ +public class ProcessedAttestations implements AttestationRegistry { + + /** An entry of the map. */ + private static final Object ENTRY = new Object(); + /** LRU attestation cache. */ + private final LRUMap cache; + /** A function that given attestation returns its hash. */ + private final Function hasher; + + public ProcessedAttestations(Function hasher, int size) { + this.hasher = hasher; + this.cache = new LRUMap<>(size); + } + + @Override + public boolean add(ReceivedAttestation attestation) { + return null == cache.put(hasher.apply(attestation.getMessage()), ENTRY); + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/Queue.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/Queue.java new file mode 100644 index 000000000..0f9c1a360 --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/Queue.java @@ -0,0 +1,233 @@ +package org.ethereum.beacon.chain.pool.registry; + +import org.ethereum.beacon.chain.pool.ReceivedAttestation; +import org.ethereum.beacon.core.types.EpochNumber; +import tech.pegasys.artemis.ethereum.core.Hash32; + +import java.util.*; +import java.util.stream.Collectors; + +/** + * Maintains attestations parked by {@link UnknownAttestationPool}. + * + *

In terms of a number of parked attestations this queue holds LRU contract. Given a new + * attestation if overall attestation number exceeds {@link #maxSize} the earliest added attestation + * will be purged from the queue. + * + *

Note: this implementation is not thread-safe. + */ +final class Queue { + + /** A queue is a list of epoch buckets. See {@link EpochBucket}. */ + private final LinkedList queue = new LinkedList<>(); + /** A number of epochs held by the {@link #queue}. */ + private final EpochNumber trackedEpochs; + /** Max number of overall parked attestations. */ + private final long maxSize; + /** A lower time frame boundary of attestation queue. */ + private EpochNumber baseLine = EpochNumber.ZERO; + + Queue(EpochNumber trackedEpochs, long maxSize) { + assert maxSize > 0; + assert trackedEpochs.getValue() > 0; + + this.trackedEpochs = trackedEpochs; + this.maxSize = maxSize; + } + + /** + * Moves base line forward. + * + *

Removes attestations made to epochs standing behind new base line. + * + * @param newBaseLine a new base line. + */ + void moveBaseLine(EpochNumber newBaseLine) { + assert baseLine == null || newBaseLine.greater(baseLine); + + for (long i = 0; i < newBaseLine.minus(baseLine).getValue() && queue.size() > 0; i++) { + queue.removeFirst(); + } + + while (queue.size() < trackedEpochs.getValue()) { + queue.add(new EpochBucket()); + } + + this.baseLine = newBaseLine; + } + + /** + * Given a block hash evicts a list of attestations made to that block. + * + * @param root a block root. + * @return evicted attestations. + */ + List evict(Hash32 root) { + if (!isInitialized()) { + return Collections.emptyList(); + } + + List evictedFromAllEpochs = new ArrayList<>(); + for (EpochBucket epoch : queue) { + evictedFromAllEpochs.addAll(epoch.evict(root)); + } + + return evictedFromAllEpochs; + } + + /** + * Queues attestation. + * + * @param epoch target epoch. + * @param root beacon block root. + * @param attestation an attestation object. + */ + void add(EpochNumber epoch, Hash32 root, ReceivedAttestation attestation) { + if (!isInitialized() || epoch.less(baseLine)) { + return; + } + + EpochBucket epochBucket = getEpochBucket(epoch); + epochBucket.add(root, attestation); + + purgeQueue(); + } + + /** Purges queue by its {@link #maxSize}. */ + private void purgeQueue() { + for (EpochNumber e = baseLine; + computeSize() > maxSize && e.less(baseLine.plus(trackedEpochs)); + e = e.increment()) { + EpochBucket epochBucket = getEpochBucket(e); + while (epochBucket.size() > 0 && computeSize() > maxSize) { + epochBucket.removeEarliest(); + } + } + } + + /** @return a number of attestations in the queue. */ + private long computeSize() { + return queue.stream().map(EpochBucket::size).reduce(0L, Long::sum); + } + + /** + * Returns an epoch bucket. + * + * @param epoch an epoch number. + * @return a bucket corresponding to the epoch. + */ + private EpochBucket getEpochBucket(EpochNumber epoch) { + assert epoch.greaterEqual(baseLine); + return queue.get(epoch.minus(baseLine).getIntValue()); + } + + /** @return {@code true} if {@link #baseLine} is defined, {@code false} otherwise. */ + private boolean isInitialized() { + return baseLine != null; + } + + /** + * An epoch bucket. + * + *

Holds attestation with the same target epoch. + */ + static final class EpochBucket { + + /** A map of beacon block root on a list of attestations made to that root. */ + private final Map> bucket = new HashMap<>(); + /** An LRU index for the attestations. */ + private final LinkedList> lruIndex = new LinkedList<>(); + + /** A number of attestations in the bucket. */ + private long size = 0; + + /** + * Adds attestation to the bucket. + * + * @param root beacon block root. + * @param attestation an attestation. + */ + void add(Hash32 root, ReceivedAttestation attestation) { + LinkedList rootBucket = getOrInsert(root); + rootBucket.add(new RootBucketEntry(System.nanoTime(), attestation)); + updateIndex(); + size += 1; + } + + private LinkedList getOrInsert(Hash32 root) { + LinkedList rootBucket = bucket.get(root); + if (rootBucket == null) { + bucket.put(root, rootBucket = new LinkedList<>()); + lruIndex.add(rootBucket); + } + return rootBucket; + } + + private void updateIndex() { + lruIndex.sort(Comparator.comparing(b -> b.getFirst().timestamp)); + } + + /** + * Given beacon block root evicts attestations made to that root. + * + * @param root beacon block root. + * @return a list of evicted attestations. + */ + List evict(Hash32 root) { + List evicted = bucket.remove(root); + if (evicted != null) { + size -= evicted.size(); + lruIndex.remove(evicted); + updateIndex(); + } + return evicted != null + ? evicted.stream().map(RootBucketEntry::getAttestation).collect(Collectors.toList()) + : Collections.emptyList(); + } + + /** + * Removes an earliest attestation from the bucket. + * + * @return removed attestation. + */ + ReceivedAttestation removeEarliest() { + if (size > 0) { + LinkedList oldestBucket = lruIndex.getFirst(); + RootBucketEntry entry = oldestBucket.removeFirst(); + if (oldestBucket.isEmpty()) { + lruIndex.removeFirst(); + bucket.values().remove(oldestBucket); + } + updateIndex(); + + size -= 1; + return entry.attestation; + } else { + return null; + } + } + + /** @return size of a bucket. */ + long size() { + return size; + } + } + + /** Entry that stores LRU timestamp along with attestation. */ + static final class RootBucketEntry { + + /** Timestamp identifying a moment in time of when attestation was added to the queue. */ + private final long timestamp; + /** An attestation itself. */ + private final ReceivedAttestation attestation; + + public RootBucketEntry(long timestamp, ReceivedAttestation attestation) { + this.timestamp = timestamp; + this.attestation = attestation; + } + + ReceivedAttestation getAttestation() { + return attestation; + } + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/UnknownAttestationPool.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/UnknownAttestationPool.java new file mode 100644 index 000000000..29a9c0aa0 --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/UnknownAttestationPool.java @@ -0,0 +1,116 @@ +package org.ethereum.beacon.chain.pool.registry; + +import java.util.Collections; +import java.util.List; +import org.ethereum.beacon.chain.pool.ReceivedAttestation; +import org.ethereum.beacon.chain.pool.StatefulProcessor; +import org.ethereum.beacon.chain.storage.BeaconBlockStorage; +import org.ethereum.beacon.consensus.BeaconChainSpec; +import org.ethereum.beacon.core.BeaconBlock; +import org.ethereum.beacon.core.operations.attestation.AttestationData; +import org.ethereum.beacon.core.types.EpochNumber; +import org.ethereum.beacon.core.types.SlotNumber; +import tech.pegasys.artemis.ethereum.core.Hash32; + +/** + * Registers and manages attestation that were made to not yet known block. + * + *

There are two main use cases: + * + *

    + *
  • Pass attestation on {@link #add(ReceivedAttestation)} method. This method checks whether + * attestation block exists or not. If it exists then attestation is not added to the registry + * and the call will return {@code false}, otherwise, attestation will be added to the + * registry and {@link true} will be returned. + *
  • When new imported block comes attestations are checked against it. If there are + * attestations made to that block they are evicted from pool and should be forwarded to + * upstream processor. + *
+ * + *

Attestations that haven't been identified for a certain period of time are purged from the + * queue and eventually discarded. This part of the logic is based on {@link + * #feedNewSlot(SlotNumber)} calls. Effectively, queue contains attestation which target epoch lays + * between {@code previous_epoch} and {@code current_epoch + epoch_lookahead}. + * + *

Note: this implementation is not thread-safe. + */ +public class UnknownAttestationPool implements AttestationRegistry, StatefulProcessor { + + /** A number of tracked epochs: previous_epoch + current_epoch + epoch_lookahead. */ + private final EpochNumber trackedEpochs; + /** A queue that maintains pooled attestations. */ + private final Queue queue; + /** A block storage. */ + private final BeaconBlockStorage blockStorage; + /** A beacon chain spec. */ + private final BeaconChainSpec spec; + /** A lower time frame boundary of attestation queue. */ + private EpochNumber currentBaseLine = EpochNumber.ZERO; + + public UnknownAttestationPool( + BeaconBlockStorage blockStorage, BeaconChainSpec spec, EpochNumber lookahead, long size) { + this.blockStorage = blockStorage; + this.spec = spec; + this.trackedEpochs = EpochNumber.of(2).plus(lookahead); + this.queue = new Queue(trackedEpochs, size); + } + + @Override + public boolean add(ReceivedAttestation attestation) { + assert isInitialized(); + + AttestationData data = attestation.getMessage().getData(); + + // beacon block has not yet been imported + // it implies that source and target blocks might have not been imported too + if (blockStorage.get(data.getBeaconBlockRoot()).isPresent()) { + return false; + } else { + queue.add(data.getTarget().getEpoch(), data.getBeaconBlockRoot(), attestation); + return true; + } + } + + /** + * Processes recently imported block. + * + * @param block a block. + * @return a list of attestations that have been identified by the block. + */ + public List feedNewImportedBlock(BeaconBlock block) { + if (!isInitialized()) { + return Collections.emptyList(); + } + + EpochNumber blockEpoch = spec.compute_epoch_of_slot(block.getSlot()); + // blockEpoch < currentBaseLine || blockEpoch >= currentBaseLine + TRACKED_EPOCHS + if (blockEpoch.less(currentBaseLine) + || blockEpoch.greaterEqual(currentBaseLine.plus(trackedEpochs))) { + return Collections.emptyList(); + } + + Hash32 blockRoot = spec.hash_tree_root(block); + return queue.evict(blockRoot); + } + + /** + * Processes new slot. + * + * @param slotNumber a slot number. + */ + public void feedNewSlot(SlotNumber slotNumber) { + EpochNumber currentEpoch = spec.compute_epoch_of_slot(slotNumber); + EpochNumber baseLine = + currentEpoch.equals(spec.getConstants().getGenesisEpoch()) + ? currentEpoch + : currentEpoch.decrement(); + queue.moveBaseLine(baseLine); + + this.currentBaseLine = baseLine; + } + + @Override + public boolean isInitialized() { + return currentBaseLine != null; + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AggregateSignature.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AggregateSignature.java new file mode 100644 index 000000000..e72daf888 --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AggregateSignature.java @@ -0,0 +1,72 @@ +package org.ethereum.beacon.chain.pool.verifier; + +import java.util.ArrayList; +import java.util.List; +import org.ethereum.beacon.crypto.BLS381; +import org.ethereum.beacon.crypto.BLS381.PublicKey; +import org.ethereum.beacon.crypto.BLS381.Signature; +import tech.pegasys.artemis.util.collections.Bitlist; + +/** A helper class aiding signature aggregation in context of beacon chain attestations. */ +final class AggregateSignature { + private Bitlist bits; + private List key0s = new ArrayList<>(); + private List key1s = new ArrayList<>(); + private List sigs = new ArrayList<>(); + + /** + * Adds yet another attestation to the aggregation churn. + * + * @param bits an aggregate bitfield. + * @param key0 bit0 key. + * @param key1 bit1 key. + * @param sig a signature. + * @return {@code true} if it could be aggregated successfully, {@code false} if given bitfield + * has intersection with an accumulated one. + */ + boolean add(Bitlist bits, PublicKey key0, PublicKey key1, BLS381.Signature sig) { + // if bits has intersection it's not possible to get a viable aggregate + if (this.bits != null && !this.bits.and(bits).isEmpty()) { + return false; + } + + if (this.bits == null) { + this.bits = bits; + } else { + this.bits = this.bits.or(bits); + } + + key0s.add(key0); + key1s.add(key1); + sigs.add(sig); + + return true; + } + + /** + * Computes and returns aggregate bit0 public key. + * + * @return a public key. + */ + PublicKey getKey0() { + return PublicKey.aggregate(key0s); + } + + /** + * Computes and returns aggregate bit1 public key. + * + * @return a public key. + */ + PublicKey getKey1() { + return PublicKey.aggregate(key1s); + } + + /** + * Computes and returns aggregate signature. + * + * @return a signature. + */ + Signature getSignature() { + return Signature.aggregate(sigs); + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AggregateSignatureVerifier.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AggregateSignatureVerifier.java new file mode 100644 index 000000000..d2155a973 --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AggregateSignatureVerifier.java @@ -0,0 +1,151 @@ +package org.ethereum.beacon.chain.pool.verifier; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.ethereum.beacon.chain.pool.ReceivedAttestation; +import org.ethereum.beacon.consensus.BeaconChainSpec; +import org.ethereum.beacon.core.BeaconState; +import org.ethereum.beacon.core.operations.attestation.AttestationData; +import org.ethereum.beacon.core.operations.attestation.AttestationDataAndCustodyBit; +import org.ethereum.beacon.core.operations.slashing.IndexedAttestation; +import org.ethereum.beacon.crypto.BLS381; +import org.ethereum.beacon.crypto.BLS381.PublicKey; +import org.ethereum.beacon.crypto.BLS381.Signature; +import org.ethereum.beacon.crypto.MessageParameters; +import tech.pegasys.artemis.ethereum.core.Hash32; +import tech.pegasys.artemis.util.uint.UInt64; + +/** + * An implementation of aggregate-then-verify strategy. + * + *

In a few words this strategy looks as follows: + * + *

    + *
  1. Aggregate as much attestation as we can. + *
  2. Verify aggregate signature. + *
  3. Verify signatures of attestations that aren't aggregated in a one-by-one fashion. + *
+ * + *

If second step fails verification falls back to one-by-one strategy. + */ +public class AggregateSignatureVerifier { + + /** A beacon chain spec. */ + private final BeaconChainSpec spec; + /** A domain which attestation signatures has been created with. */ + private final UInt64 domain; + /** Verification churn. */ + private final List attestations = new ArrayList<>(); + + AggregateSignatureVerifier(BeaconChainSpec spec, UInt64 domain) { + this.spec = spec; + this.domain = domain; + } + + /** + * Adds an attestation to the verification churn. + * + * @param state a state attestation built upon. + * @param indexed an instance of corresponding {@link IndexedAttestation}. + * @param attestation an attestation itself. + */ + void add(BeaconState state, IndexedAttestation indexed, ReceivedAttestation attestation) { + attestations.add(VerifiableAttestation.create(spec, state, indexed, attestation)); + } + + /** + * Verifies previously added attestation. + * + *

First, attestations are grouped by {@link AttestationData} and then each group is passed + * onto {@link #verifyGroup(AttestationData, List)}. + * + * @return a result of singature verification. + */ + public VerificationResult verify() { + Map> signedMessageGroups = + attestations.stream().collect(Collectors.groupingBy(VerifiableAttestation::getData)); + + return signedMessageGroups.entrySet().stream() + .map(e -> verifyGroup(e.getKey(), e.getValue())) + .reduce(VerificationResult.EMPTY, VerificationResult::merge); + } + + /** + * Verifies a group of attestations signing the same {@link AttestationData}. + * + * @param data attestation data. + * @param group a group. + * @return a result of verification. + */ + private VerificationResult verifyGroup(AttestationData data, List group) { + final List valid = new ArrayList<>(); + final List validIndexed = new ArrayList<>(); + final List invalid = new ArrayList<>(); + + // for aggregation sake, smaller aggregates should go first + group.sort(Comparator.comparing(attestation -> attestation.getAggregationBits().size())); + + // try to aggregate as much as we can + List aggregated = new ArrayList<>(); + List notAggregated = new ArrayList<>(); + AggregateSignature aggregate = new AggregateSignature(); + for (VerifiableAttestation attestation : group) { + if (aggregate.add( + attestation.getAggregationBits(), + attestation.getBit0Key(), + attestation.getBit1Key(), + attestation.getSignature())) { + aggregated.add(attestation); + } else { + notAggregated.add(attestation); + } + } + + // verify aggregate and fall back to one-by-one verification if it has failed + if (verifySignature(data, aggregate.getKey0(), aggregate.getKey1(), aggregate.getSignature())) { + aggregated.forEach(attestation -> { + valid.add(attestation.getAttestation()); + validIndexed.add(attestation.getIndexed()); + }); + } else { + notAggregated = group; + } + + for (VerifiableAttestation attestation : notAggregated) { + if (verifySignature( + data, attestation.getBit0Key(), attestation.getBit1Key(), attestation.getSignature())) { + valid.add(attestation.getAttestation()); + validIndexed.add(attestation.getIndexed()); + } else { + invalid.add(attestation.getAttestation()); + } + } + + return new VerificationResult(valid, validIndexed, invalid); + } + + /** + * Verifies single signature. + * + * @param data a data being signed. + * @param bit0Key a bit0 public key. + * @param bit1Key a bit1 public key. + * @param signature a signature. + * @return {@code true} if signature is valid, {@code false} otherwise. + */ + private boolean verifySignature( + AttestationData data, PublicKey bit0Key, PublicKey bit1Key, Signature signature) { + Hash32 bit0Hash = spec.hash_tree_root(new AttestationDataAndCustodyBit(data, false)); + Hash32 bit1Hash = spec.hash_tree_root(new AttestationDataAndCustodyBit(data, true)); + + return BLS381.verifyMultiple( + Arrays.asList( + MessageParameters.create(bit0Hash, domain), MessageParameters.create(bit1Hash, domain)), + signature, + Arrays.asList(bit0Key, bit1Key)); + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AttestationVerifier.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AttestationVerifier.java new file mode 100644 index 000000000..e5e23d464 --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AttestationVerifier.java @@ -0,0 +1,215 @@ +package org.ethereum.beacon.chain.pool.verifier; + +import static org.ethereum.beacon.core.spec.SignatureDomains.ATTESTATION; + +import com.google.common.base.Objects; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import org.ethereum.beacon.chain.BeaconTuple; +import org.ethereum.beacon.chain.pool.ReceivedAttestation; +import org.ethereum.beacon.chain.storage.BeaconTupleStorage; +import org.ethereum.beacon.consensus.BeaconChainSpec; +import org.ethereum.beacon.consensus.BeaconStateEx; +import org.ethereum.beacon.consensus.transition.EmptySlotTransition; +import org.ethereum.beacon.core.BeaconState; +import org.ethereum.beacon.core.operations.Attestation; +import org.ethereum.beacon.core.operations.attestation.AttestationData; +import org.ethereum.beacon.core.operations.slashing.IndexedAttestation; +import org.ethereum.beacon.core.state.Checkpoint; +import org.ethereum.beacon.core.types.EpochNumber; +import tech.pegasys.artemis.ethereum.core.Hash32; +import tech.pegasys.artemis.util.uint.UInt64; + +/** + * An implementation of {@link BatchVerifier}. + * + *

There are three steps of batch verification: + * + *

    + *
  • Group batch by beacon block root and target. + *
  • Calculate state for each group and run checks against this state. + *
  • Pass group onto signature verifier. + *
+ * + *

Current implementation relies on {@link AggregateSignatureVerifier} which is pretty efficient. + * {@link AggregateSignatureVerifier} tries to first aggregate attestations and then verify a + * signature of that aggregate in a single operation instead of verifying signature of each + * standalone attestation. A group succeeded with state verification is next passed onto signature + * verifier. A nice part of it is that aggregatable attestation groups are subsets of groups made + * against attestation target. + * + *

Verification hierarchy can be represented with a diagram: + * + * 

+ *                        attestation_batch
+ *                       /                 \
+ * state:               target_1 ... target_N
+ *                     /        \
+ * signature:  aggregate_1 ... aggregate_N
+ *
+ */ +public class AttestationVerifier implements BatchVerifier { + + /** A beacon tuple storage. */ + private final BeaconTupleStorage tupleStorage; + /** A beacon chain spec. */ + private final BeaconChainSpec spec; + /** An empty slot transition. */ + private final EmptySlotTransition emptySlotTransition; + + public AttestationVerifier( + BeaconTupleStorage tupleStorage, + BeaconChainSpec spec, + EmptySlotTransition emptySlotTransition) { + this.tupleStorage = tupleStorage; + this.spec = spec; + this.emptySlotTransition = emptySlotTransition; + } + + @Override + public VerificationResult verify(List batch) { + Map> targetGroups = + batch.stream().collect(Collectors.groupingBy(AttestingTarget::from)); + + return targetGroups.entrySet().stream() + .map(e -> verifyGroup(e.getKey(), e.getValue())) + .reduce(VerificationResult.EMPTY, VerificationResult::merge); + } + + /** + * Verifies a group of attestations with the same target. + * + * @param target a target. + * @param group a group. + * @return result of verification. + */ + private VerificationResult verifyGroup(AttestingTarget target, List group) { + Optional rootTuple = tupleStorage.get(target.blockRoot); + + // it must be present, otherwise, attestation couldn't be here + // TODO keep assertion for a while, it might be useful to discover bugs + assert rootTuple.isPresent(); + + EpochNumber beaconBlockEpoch = spec.compute_epoch_of_slot(rootTuple.get().getState().getSlot()); + + // beaconBlockEpoch > targetEpoch + // it must either be equal or less than + if (beaconBlockEpoch.greater(target.checkpoint.getEpoch())) { + return VerificationResult.allInvalid(group); + } + + // beaconBlockEpoch < targetEpoch && targetRoot != beaconBlockRoot + // target checkpoint is built with empty slots upon a block root + // in that case target root and beacon block root must be equal + if (beaconBlockEpoch.less(target.checkpoint.getEpoch()) + && !target.checkpoint.getRoot().equals(target.blockRoot)) { + return VerificationResult.allInvalid(group); + } + + // compute state and domain, there must be the same state for all attestations + final BeaconState state = + computeState(rootTuple.get().getState(), target.checkpoint.getEpoch()); + final UInt64 domain = spec.get_domain(state, ATTESTATION, target.checkpoint.getEpoch()); + final AggregateSignatureVerifier signatureVerifier = + new AggregateSignatureVerifier(spec, domain); + final List invalid = new ArrayList<>(); + + for (ReceivedAttestation attestation : group) { + Optional result = verifyIndexed(state, attestation.getMessage()); + if (result.isPresent()) { + signatureVerifier.add(state, result.get(), attestation); + } else { + invalid.add(attestation); + } + } + + VerificationResult signatureResult = signatureVerifier.verify(); + return VerificationResult.allInvalid(invalid).merge(signatureResult); + } + + /** + * This method does two things: + * + *
    + *
  • Runs main checks defined in the spec; these checks verifies attestation against a state + * it's been made upon. + *
  • Computes {@link IndexedAttestation} and runs checks against it omitting signature + * verification. + *
+ * + * @param state a state attestation built upon. + * @param attestation an attestation. + * @return an optional filled with {@link IndexedAttestation} instance if verification passed + * successfully, empty optional box is returned otherwise. + */ + private Optional verifyIndexed(BeaconState state, Attestation attestation) { + // compute and verify indexed attestation + // skip signature verification + IndexedAttestation indexedAttestation = spec.get_indexed_attestation(state, attestation); + if (!spec.is_valid_indexed_attestation_impl(state, indexedAttestation, false)) { + return Optional.empty(); + } + + return Optional.of(indexedAttestation); + } + + /** + * Given epoch and beacon state computes a state that attestation built upon. + * + * @param state a state after attestation beacon block has been imported. + * @param targetEpoch target epoch of attestation. + * @return computed state. + */ + private BeaconState computeState(BeaconStateEx state, EpochNumber targetEpoch) { + EpochNumber beaconBlockEpoch = spec.compute_epoch_of_slot(state.getSlot()); + + // block is in the same epoch, no additional state is required to be built + if (beaconBlockEpoch.equals(targetEpoch)) { + return state; + } + + // build a state at epoch boundary, it must be enough to proceed + return emptySlotTransition.apply(state, spec.compute_start_slot_of_epoch(targetEpoch)); + } + + /** + * A wrapper for attestation target checkpoint and beacon block root. + * + *

This is the entity which initial verification groups are built around. + */ + private static final class AttestingTarget { + + static AttestingTarget from(ReceivedAttestation attestation) { + AttestationData data = attestation.getMessage().getData(); + return new AttestingTarget(data.getTarget(), data.getBeaconBlockRoot()); + } + + private final Checkpoint checkpoint; + private final Hash32 blockRoot; + + private AttestingTarget(Checkpoint checkpoint, Hash32 blockRoot) { + this.checkpoint = checkpoint; + this.blockRoot = blockRoot; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + AttestingTarget that = (AttestingTarget) o; + return Objects.equal(checkpoint, that.checkpoint) && Objects.equal(blockRoot, that.blockRoot); + } + + @Override + public int hashCode() { + return Objects.hashCode(checkpoint, blockRoot); + } + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/BatchVerifier.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/BatchVerifier.java new file mode 100644 index 000000000..854b9b5d4 --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/BatchVerifier.java @@ -0,0 +1,29 @@ +package org.ethereum.beacon.chain.pool.verifier; + +import java.util.List; +import org.ethereum.beacon.chain.pool.ReceivedAttestation; +import org.ethereum.beacon.chain.pool.checker.AttestationChecker; + +/** + * An interface of attestation verifier that processes attestations in a batch. + * + *

Opposed to {@link AttestationChecker}, verifier involves I/O operations and checks highly + * demanded to CPU resources. + * + *

Verifying attestations in a batch aids resource saving by grouping them by beacon block root + * and target they are attesting to. A state calculated for each of such groups is reused across the + * group. It saves a lot of resources as state calculation is pretty heavy operation. + * + *

It is highly recommended to place this verifier to the end of verification pipeline due to its + * high demand to computational resources. + */ +public interface BatchVerifier { + + /** + * Verifies a batch of attestations. + * + * @param batch a batch. + * @return result of verification. + */ + VerificationResult verify(List batch); +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/VerifiableAttestation.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/VerifiableAttestation.java new file mode 100644 index 000000000..89515f650 --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/VerifiableAttestation.java @@ -0,0 +1,116 @@ +package org.ethereum.beacon.chain.pool.verifier; + +import java.util.List; +import java.util.stream.Collectors; +import org.ethereum.beacon.chain.pool.ReceivedAttestation; +import org.ethereum.beacon.consensus.BeaconChainSpec; +import org.ethereum.beacon.core.BeaconState; +import org.ethereum.beacon.core.operations.Attestation; +import org.ethereum.beacon.core.operations.attestation.AttestationData; +import org.ethereum.beacon.core.operations.slashing.IndexedAttestation; +import org.ethereum.beacon.core.types.BLSPubkey; +import org.ethereum.beacon.crypto.BLS381; +import org.ethereum.beacon.crypto.BLS381.PublicKey; +import org.ethereum.beacon.crypto.BLS381.Signature; +import tech.pegasys.artemis.util.collections.Bitlist; + +/** + * An artificial entity exclusively related to signature verification process. + * + *

Being created from {@link IndexedAttestation}, {@link BeaconState} and {@link Attestation} + * itself contains all the information required to proceed with signature verification: + * + *

+ * + *

    + *
  • bit0 and bit1 aggregate public keys + *
  • signature + *
  • aggregation bits + *
  • attestation data + *
+ */ +final class VerifiableAttestation { + + static VerifiableAttestation create( + BeaconChainSpec spec, + BeaconState state, + IndexedAttestation indexed, + ReceivedAttestation attestation) { + + List bit0Keys = + indexed.getCustodyBit0Indices().stream() + .map(i -> state.getValidators().get(i).getPubKey()) + .collect(Collectors.toList()); + List bit1Keys = + indexed.getCustodyBit1Indices().stream() + .map(i -> state.getValidators().get(i).getPubKey()) + .collect(Collectors.toList()); + + // pre-process aggregated pubkeys + PublicKey bit0AggregateKey = spec.bls_aggregate_pubkeys_no_validate(bit0Keys); + PublicKey bit1AggregateKey = spec.bls_aggregate_pubkeys_no_validate(bit1Keys); + + return new VerifiableAttestation( + attestation, + indexed, + attestation.getMessage().getData(), + attestation.getMessage().getAggregationBits(), + bit0AggregateKey, + bit1AggregateKey, + BLS381.Signature.createWithoutValidation(attestation.getMessage().getSignature())); + } + + private final ReceivedAttestation attestation; + private final IndexedAttestation indexed; + + private final AttestationData data; + private final Bitlist aggregationBits; + private final PublicKey bit0Key; + private final PublicKey bit1Key; + private final BLS381.Signature signature; + + public VerifiableAttestation( + ReceivedAttestation attestation, + IndexedAttestation indexed, + AttestationData data, + Bitlist aggregationBits, + PublicKey bit0Key, + PublicKey bit1Key, + Signature signature) { + this.attestation = attestation; + this.indexed = indexed; + this.data = data; + this.aggregationBits = aggregationBits; + this.bit0Key = bit0Key; + this.bit1Key = bit1Key; + this.signature = signature; + } + + public IndexedAttestation getIndexed() { + return indexed; + } + + public AttestationData getData() { + return data; + } + + public Bitlist getAggregationBits() { + return aggregationBits; + } + + public ReceivedAttestation getAttestation() { + return attestation; + } + + public PublicKey getBit0Key() { + return bit0Key; + } + + public PublicKey getBit1Key() { + return bit1Key; + } + + public Signature getSignature() { + return signature; + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/VerificationResult.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/VerificationResult.java new file mode 100644 index 000000000..00dd075e5 --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/VerificationResult.java @@ -0,0 +1,54 @@ +package org.ethereum.beacon.chain.pool.verifier; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import org.ethereum.beacon.chain.pool.ReceivedAttestation; +import org.ethereum.beacon.core.operations.slashing.IndexedAttestation; + +/** Result of attestation batch verification. Contains a list of valid and invalid attestations. */ +public final class VerificationResult { + + static VerificationResult allInvalid(List attestations) { + return new VerificationResult(Collections.emptyList(), Collections.emptyList(), attestations); + } + + public static final VerificationResult EMPTY = + new VerificationResult(Collections.emptyList(), Collections.emptyList(), Collections.emptyList()); + + private final List valid; + private final List validIndexed; + private final List invalid; + + VerificationResult( + List valid, + List validIndexed, + List invalid) { + this.valid = valid; + this.validIndexed = validIndexed; + this.invalid = invalid; + } + + public List getValid() { + return valid; + } + + public List getValidIndexed() { + return validIndexed; + } + + public List getInvalid() { + return invalid; + } + + public VerificationResult merge(VerificationResult other) { + List valid = new ArrayList<>(this.valid); + List validIndexed = new ArrayList<>(this.validIndexed); + List invalid = new ArrayList<>(this.invalid); + valid.addAll(other.valid); + validIndexed.addAll(other.validIndexed); + invalid.addAll(other.invalid); + + return new VerificationResult(valid, validIndexed, invalid); + } +} diff --git a/chain/src/main/resources/doc-files/attestation-pool.svg b/chain/src/main/resources/doc-files/attestation-pool.svg new file mode 100644 index 000000000..89decf7ca --- /dev/null +++ b/chain/src/main/resources/doc-files/attestation-pool.svg @@ -0,0 +1,2 @@ + +
TimeFrameChecker

target_epoch < finalized_epoch
AND
target_epoch > curr_epoch + lookahead
[Not supported by viewer]
Discard
Discard
SanityChecker
SanityChecker
Invalid
Invalid
drop peer
drop peer
ProcessedAttestation
is new?
[Not supported by viewer]
Discard
Discard
SignatureEncodingChecker
is encoding valid?
[Not supported by viewer]
UnknownAttestationPool
is attested block imported?
[Not supported by viewer]
Finalized checkpoint
Finalized checkpoint<br>
Newly imported block
Newly imported block
New slot
New slot
AttestationVerifier
valid?
[Not supported by viewer]
timeout buffer
timeout buffer
Discard by timeout
[Not supported by viewer]
Valid
Valid
Propagate
Propagate
AttestationChurn
proved to be onchain?
[Not supported by viewer]
proposer
proposer
LMD GHOST
LMD GHOST
Head
Head
Discard by timeout
Discard by timeout
Wire
Wire \ No newline at end of file diff --git a/chain/src/test/java/org/ethereum/beacon/chain/DefaultBeaconChainTest.java b/chain/src/test/java/org/ethereum/beacon/chain/DefaultBeaconChainTest.java index 561e78255..ca16ed1b1 100644 --- a/chain/src/test/java/org/ethereum/beacon/chain/DefaultBeaconChainTest.java +++ b/chain/src/test/java/org/ethereum/beacon/chain/DefaultBeaconChainTest.java @@ -16,19 +16,13 @@ import org.ethereum.beacon.consensus.transition.InitialStateTransition; import org.ethereum.beacon.consensus.transition.PerEpochTransition; import org.ethereum.beacon.consensus.util.StateTransitionTestUtil; -import org.ethereum.beacon.consensus.verifier.BeaconBlockVerifier; -import org.ethereum.beacon.consensus.verifier.BeaconStateVerifier; -import org.ethereum.beacon.consensus.verifier.VerificationResult; -import org.ethereum.beacon.core.BeaconBlock; -import org.ethereum.beacon.core.BeaconBlockBody; -import org.ethereum.beacon.core.BeaconState; +import org.ethereum.beacon.consensus.verifier.*; +import org.ethereum.beacon.core.*; import org.ethereum.beacon.core.state.Eth1Data; -import org.ethereum.beacon.core.types.BLSSignature; -import org.ethereum.beacon.core.types.Time; +import org.ethereum.beacon.core.types.*; import org.ethereum.beacon.db.Database; import org.ethereum.beacon.schedulers.Schedulers; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.*; import tech.pegasys.artemis.ethereum.core.Hash32; import tech.pegasys.artemis.util.uint.UInt64; @@ -52,7 +46,7 @@ public void insertAChain() { beaconChain.init(); BeaconTuple initialTuple = beaconChain.getRecentlyProcessed(); - Assert.assertEquals( + Assertions.assertEquals( spec.getConstants().getGenesisSlot(), initialTuple.getBlock().getSlot()); IntStream.range(0, 10) @@ -61,8 +55,8 @@ public void insertAChain() { BeaconTuple recentlyProcessed = beaconChain.getRecentlyProcessed(); BeaconBlock aBlock = createBlock(recentlyProcessed, spec, schedulers.getCurrentTime(), perSlotTransition); - Assert.assertEquals(ImportResult.OK, beaconChain.insert(aBlock)); - Assert.assertEquals(aBlock, beaconChain.getRecentlyProcessed().getBlock()); + Assertions.assertEquals(ImportResult.OK, beaconChain.insert(aBlock)); + Assertions.assertEquals(aBlock, beaconChain.getRecentlyProcessed().getBlock()); System.out.println("Inserted block: " + (idx + 1)); }); diff --git a/chain/src/test/java/org/ethereum/beacon/chain/SlotTickerTests.java b/chain/src/test/java/org/ethereum/beacon/chain/SlotTickerTests.java index 9ae310644..ba9e45239 100644 --- a/chain/src/test/java/org/ethereum/beacon/chain/SlotTickerTests.java +++ b/chain/src/test/java/org/ethereum/beacon/chain/SlotTickerTests.java @@ -1,19 +1,14 @@ package org.ethereum.beacon.chain; import org.ethereum.beacon.consensus.BeaconChainSpec; -import org.ethereum.beacon.core.BeaconState; -import org.ethereum.beacon.core.MutableBeaconState; +import org.ethereum.beacon.core.*; import org.ethereum.beacon.core.spec.SpecConstants; -import org.ethereum.beacon.core.types.SlotNumber; -import org.ethereum.beacon.core.types.Time; +import org.ethereum.beacon.core.types.*; import org.ethereum.beacon.schedulers.Schedulers; -import org.junit.Test; +import org.junit.jupiter.api.*; import reactor.core.publisher.Flux; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import java.util.concurrent.*; public class SlotTickerTests { public static final int MILLIS_IN_SECOND = 1000; @@ -56,16 +51,16 @@ public void testSlotTicker() throws Exception { .subscribe( slotNumber -> { if (previousTick.greater(SlotNumber.ZERO)) { - assertEquals(previousTick.increment(), slotNumber); + Assertions.assertEquals(previousTick.increment(), slotNumber); bothAssertsRun.countDown(); } else { - assertTrue(slotNumber.greater(genesisSlot)); // first tracked tick + Assertions.assertTrue(slotNumber.greater(genesisSlot)); // first tracked tick bothAssertsRun.countDown(); } previousTick = slotNumber; }); - assertTrue( - String.format("%s assertion(s) was not correct or not tested", bothAssertsRun.getCount()), - bothAssertsRun.await(4, TimeUnit.SECONDS)); + Assertions.assertTrue( + (bothAssertsRun.await(4, TimeUnit.SECONDS)), + String.format("%s assertion(s) was not correct or not tested", bothAssertsRun.getCount())); } } diff --git a/chain/src/test/java/org/ethereum/beacon/chain/StreamTests.java b/chain/src/test/java/org/ethereum/beacon/chain/StreamTests.java index b1b551d21..e81f379c7 100644 --- a/chain/src/test/java/org/ethereum/beacon/chain/StreamTests.java +++ b/chain/src/test/java/org/ethereum/beacon/chain/StreamTests.java @@ -1,15 +1,13 @@ package org.ethereum.beacon.chain; -import java.time.Duration; -import org.junit.Ignore; -import org.junit.Test; +import org.junit.jupiter.api.*; import org.reactivestreams.Publisher; import reactor.core.Disposable; -import reactor.core.publisher.Flux; -import reactor.core.publisher.FluxSink; -import reactor.core.publisher.ReplayProcessor; +import reactor.core.publisher.*; import reactor.core.scheduler.Schedulers; +import java.time.Duration; + public class StreamTests { FluxSink sink; @@ -44,7 +42,7 @@ public void test1() throws InterruptedException { } @Test - @Ignore + @Disabled public void intervalTest1() throws InterruptedException { long initDelay = (System.currentTimeMillis() / 10000 + 1) * 10000 - System.currentTimeMillis(); diff --git a/chain/src/test/java/org/ethereum/beacon/chain/observer/ObservableStateProcessorTest.java b/chain/src/test/java/org/ethereum/beacon/chain/observer/ObservableStateProcessorTest.java index 5862bdc89..3ca2186b5 100644 --- a/chain/src/test/java/org/ethereum/beacon/chain/observer/ObservableStateProcessorTest.java +++ b/chain/src/test/java/org/ethereum/beacon/chain/observer/ObservableStateProcessorTest.java @@ -1,17 +1,14 @@ package org.ethereum.beacon.chain.observer; -import java.time.Duration; -import java.util.ArrayList; -import java.util.List; -import java.util.Random; import org.ethereum.beacon.chain.util.SampleObservableState; import org.ethereum.beacon.core.types.SlotNumber; -import org.ethereum.beacon.schedulers.ControlledSchedulers; -import org.ethereum.beacon.schedulers.Schedulers; -import org.junit.Assert; -import org.junit.Test; +import org.ethereum.beacon.schedulers.*; +import org.junit.jupiter.api.*; import reactor.core.publisher.Flux; +import java.time.Duration; +import java.util.*; + public class ObservableStateProcessorTest { @Test @@ -34,19 +31,19 @@ public void test1() throws Exception { System.out.println(states); System.out.println(slots); - Assert.assertEquals(1, states.size()); - Assert.assertEquals(0, slots.size()); + Assertions.assertEquals(1, states.size()); + Assertions.assertEquals(0, slots.size()); schedulers.addTime(Duration.ofSeconds(10)); System.out.println(states); System.out.println(slots); - Assert.assertEquals(2, states.size()); - Assert.assertEquals(1, slots.size()); + Assertions.assertEquals(2, states.size()); + Assertions.assertEquals(1, slots.size()); - Assert.assertEquals(genesisSlot.getValue() + 1, slots.get(0).getValue()); - Assert.assertEquals(genesisSlot.increment(), states.get(1).getLatestSlotState().getSlot()); + Assertions.assertEquals(genesisSlot.getValue() + 1, slots.get(0).getValue()); + Assertions.assertEquals(genesisSlot.increment(), states.get(1).getLatestSlotState().getSlot()); } @Test @@ -66,15 +63,15 @@ public void test2() throws Exception { List slots = new ArrayList<>(); Flux.from(sample.slotTicker.getTickerStream()).subscribe(slots::add); - Assert.assertEquals(1, states.size()); - Assert.assertEquals(0, slots.size()); + Assertions.assertEquals(1, states.size()); + Assertions.assertEquals(0, slots.size()); schedulers.addTime(Duration.ofSeconds(10)); - Assert.assertEquals(2, states.size()); - Assert.assertEquals(1, slots.size()); + Assertions.assertEquals(2, states.size()); + Assertions.assertEquals(1, slots.size()); - Assert.assertEquals(genesisSlot.getValue() + 61, slots.get(0).getValue()); - Assert.assertEquals(genesisSlot.plus(61), states.get(1).getLatestSlotState().getSlot()); + Assertions.assertEquals(genesisSlot.getValue() + 61, slots.get(0).getValue()); + Assertions.assertEquals(genesisSlot.plus(61), states.get(1).getLatestSlotState().getSlot()); } } diff --git a/chain/src/test/java/org/ethereum/beacon/chain/pool/PoolTestConfigurator.java b/chain/src/test/java/org/ethereum/beacon/chain/pool/PoolTestConfigurator.java new file mode 100644 index 000000000..42bbb4426 --- /dev/null +++ b/chain/src/test/java/org/ethereum/beacon/chain/pool/PoolTestConfigurator.java @@ -0,0 +1,190 @@ +package org.ethereum.beacon.chain.pool; + +import org.ethereum.beacon.chain.BeaconTuple; +import org.ethereum.beacon.chain.DefaultBeaconChain; +import org.ethereum.beacon.chain.MutableBeaconChain; +import org.ethereum.beacon.chain.storage.BeaconChainStorage; +import org.ethereum.beacon.chain.storage.impl.SSZBeaconChainStorageFactory; +import org.ethereum.beacon.chain.storage.impl.SerializerFactory; +import org.ethereum.beacon.chain.storage.util.StorageUtils; +import org.ethereum.beacon.consensus.BeaconChainSpec; +import org.ethereum.beacon.consensus.BeaconStateEx; +import org.ethereum.beacon.consensus.BlockTransition; +import org.ethereum.beacon.consensus.ChainStart; +import org.ethereum.beacon.consensus.StateTransition; +import org.ethereum.beacon.consensus.transition.BeaconStateExImpl; +import org.ethereum.beacon.consensus.transition.EmptySlotTransition; +import org.ethereum.beacon.consensus.transition.ExtendedSlotTransition; +import org.ethereum.beacon.consensus.transition.InitialStateTransition; +import org.ethereum.beacon.consensus.transition.PerEpochTransition; +import org.ethereum.beacon.consensus.transition.PerSlotTransition; +import org.ethereum.beacon.consensus.util.StateTransitionTestUtil; +import org.ethereum.beacon.consensus.verifier.BeaconBlockVerifier; +import org.ethereum.beacon.consensus.verifier.BeaconStateVerifier; +import org.ethereum.beacon.consensus.verifier.VerificationResult; +import org.ethereum.beacon.core.BeaconBlock; +import org.ethereum.beacon.core.BeaconBlockBody; +import org.ethereum.beacon.core.BeaconState; +import org.ethereum.beacon.core.operations.Attestation; +import org.ethereum.beacon.core.operations.attestation.AttestationData; +import org.ethereum.beacon.core.operations.attestation.Crosslink; +import org.ethereum.beacon.core.operations.deposit.DepositData; +import org.ethereum.beacon.core.spec.SpecConstants; +import org.ethereum.beacon.core.state.Checkpoint; +import org.ethereum.beacon.core.state.Eth1Data; +import org.ethereum.beacon.core.types.BLSPubkey; +import org.ethereum.beacon.core.types.BLSSignature; +import org.ethereum.beacon.core.types.EpochNumber; +import org.ethereum.beacon.core.types.ShardNumber; +import org.ethereum.beacon.core.types.SlotNumber; +import org.ethereum.beacon.core.types.Time; +import org.ethereum.beacon.crypto.BLS381; +import org.ethereum.beacon.crypto.Hashes; +import org.ethereum.beacon.crypto.MessageParameters; +import org.ethereum.beacon.db.Database; +import org.ethereum.beacon.schedulers.ControlledSchedulers; +import org.ethereum.beacon.schedulers.Schedulers; +import reactor.core.publisher.DirectProcessor; +import tech.pegasys.artemis.ethereum.core.Hash32; +import tech.pegasys.artemis.util.bytes.Bytes32; +import tech.pegasys.artemis.util.bytes.Bytes4; +import tech.pegasys.artemis.util.bytes.Bytes48; +import tech.pegasys.artemis.util.bytes.Bytes96; +import tech.pegasys.artemis.util.bytes.BytesValue; +import tech.pegasys.artemis.util.collections.Bitlist; +import tech.pegasys.artemis.util.uint.UInt64; + +import java.util.Collections; +import java.util.Random; + +import static org.ethereum.beacon.core.spec.SignatureDomains.DEPOSIT; + +public class PoolTestConfigurator { + + protected final SpecConstants specConstants = + new SpecConstants() { + @Override + public SlotNumber getGenesisSlot() { + return SlotNumber.of(12345); + } + + @Override + public Time getSecondsPerSlot() { + return Time.of(1); + } + }; + + protected final BeaconChainSpec spec = BeaconChainSpec.Builder.createWithDefaultParams() + .withConstants(new SpecConstants() { + @Override + public ShardNumber getShardCount() { + return ShardNumber.of(16); + } + + @Override + public SlotNumber.EpochLength getSlotsPerEpoch() { + return new SlotNumber.EpochLength(UInt64.valueOf(4)); + } + }) + .withComputableGenesisTime(false) + .withVerifyDepositProof(false) + .withBlsVerifyProofOfPossession(false) + .withBlsVerify(false) + .withCache(true) + .build(); + + protected StateTransition perSlotTransition = new PerSlotTransition(spec); + protected final DirectProcessor finalizedCheckpoints = DirectProcessor.create(); + protected final ControlledSchedulers schedulers = Schedulers.createControlled(); + protected final DirectProcessor source = DirectProcessor.create(); + + protected Attestation createAttestation(BytesValue someValue) { + return createAttestation(someValue, createAttestationData()); + } + + protected Attestation createAttestation(BytesValue someValue, AttestationData attestationData) { + final Random rnd = new Random(); + BLS381.KeyPair keyPair = BLS381.KeyPair.create(BLS381.PrivateKey.create(Bytes32.random(rnd))); + Hash32 withdrawalCredentials = Hash32.random(rnd); + DepositData depositDataWithoutSignature = new DepositData( + BLSPubkey.wrap(Bytes48.leftPad(keyPair.getPublic().getEncodedBytes())), + withdrawalCredentials, + spec.getConstants().getMaxEffectiveBalance(), + BLSSignature.wrap(Bytes96.ZERO) + ); + Hash32 msgHash = spec.signing_root(depositDataWithoutSignature); + UInt64 domain = spec.compute_domain(DEPOSIT, Bytes4.ZERO); + BLS381.Signature signature = BLS381 + .sign(MessageParameters.create(msgHash, domain), keyPair); + return new Attestation( + Bitlist.of(someValue.size() * 8, someValue, specConstants.getMaxValidatorsPerCommittee().getValue()), + attestationData, + Bitlist.of(8, BytesValue.fromHexString("bb"), specConstants.getMaxValidatorsPerCommittee().getValue()), + BLSSignature.wrap(signature.getEncoded()), + specConstants); + } + + private AttestationData createAttestationData() { + + return new AttestationData( + Hashes.sha256(BytesValue.fromHexString("aa")), + new Checkpoint(EpochNumber.of(125), Hashes.sha256(BytesValue.fromHexString("bb"))), + new Checkpoint(EpochNumber.of(126), Hashes.sha256(BytesValue.fromHexString("cc"))), + Crosslink.EMPTY); + } + + protected BeaconTuple createBlock( + BeaconTuple parent, + BeaconChainSpec spec, long currentTime, + StateTransition perSlotTransition) { + BeaconBlock block = + new BeaconBlock( + spec.get_current_slot(parent.getState(), currentTime), + spec.signing_root(parent.getBlock()), + Hash32.ZERO, + BeaconBlockBody.getEmpty(spec.getConstants()), + BLSSignature.ZERO); + BeaconState state = perSlotTransition.apply(new BeaconStateExImpl(parent.getState())); + + return BeaconTuple.of( + block.withStateRoot(spec.hash_tree_root(state)), new BeaconStateExImpl(state)); + } + + protected MutableBeaconChain createBeaconChain(BeaconChainSpec spec, StateTransition perSlotTransition, Schedulers schedulers) { + + final Time start = Time.castFrom(UInt64.valueOf(schedulers.getCurrentTime() / 1000)); + final ChainStart chainStart = new ChainStart(start, Eth1Data.EMPTY, Collections.emptyList()); + + final InitialStateTransition initialTransition = new InitialStateTransition(chainStart, spec); + final BlockTransition perBlockTransition = StateTransitionTestUtil.createPerBlockTransition(); + final StateTransition perEpochTransition = StateTransitionTestUtil.createStateWithNoTransition(); + + final BeaconBlockVerifier blockVerifier = (block, state) -> VerificationResult.PASSED; + final BeaconStateVerifier stateVerifier = (block, state) -> VerificationResult.PASSED; + + final Database database = Database.inMemoryDB(); + final SerializerFactory ssz = SerializerFactory.createSSZ(spec.getConstants()); + final BeaconChainStorage chainStorage = new SSZBeaconChainStorageFactory(spec.getObjectHasher(), ssz) + .create(database); + + final EmptySlotTransition preBlockTransition = new EmptySlotTransition( + new ExtendedSlotTransition(new PerEpochTransition(spec) { + @Override + public BeaconStateEx apply(BeaconStateEx stateEx) { + return perEpochTransition.apply(stateEx); + } + }, perSlotTransition, spec)); + + final BeaconStateEx initialState = initialTransition.apply(spec.get_empty_block()); + StorageUtils.initializeStorage(chainStorage, spec, initialState); + + return new DefaultBeaconChain( + spec, + preBlockTransition, + perBlockTransition, + blockVerifier, + stateVerifier, + chainStorage, + schedulers); + } +} diff --git a/chain/src/test/java/org/ethereum/beacon/chain/pool/checker/SignatureEncodingCheckerTest.java b/chain/src/test/java/org/ethereum/beacon/chain/pool/checker/SignatureEncodingCheckerTest.java new file mode 100644 index 000000000..679778e3a --- /dev/null +++ b/chain/src/test/java/org/ethereum/beacon/chain/pool/checker/SignatureEncodingCheckerTest.java @@ -0,0 +1,59 @@ +package org.ethereum.beacon.chain.pool.checker; + +import org.ethereum.beacon.chain.pool.PoolTestConfigurator; +import org.ethereum.beacon.chain.pool.ReceivedAttestation; +import org.ethereum.beacon.core.operations.Attestation; +import org.ethereum.beacon.core.operations.attestation.AttestationData; +import org.ethereum.beacon.core.operations.attestation.Crosslink; +import org.ethereum.beacon.core.state.Checkpoint; +import org.ethereum.beacon.core.types.BLSSignature; +import org.ethereum.beacon.core.types.EpochNumber; +import org.ethereum.beacon.crypto.Hashes; +import org.ethereum.beacon.types.p2p.NodeId; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import tech.pegasys.artemis.util.bytes.Bytes96; +import tech.pegasys.artemis.util.bytes.BytesValue; +import tech.pegasys.artemis.util.collections.Bitlist; + +import static org.assertj.core.api.Assertions.assertThat; + +class SignatureEncodingCheckerTest extends PoolTestConfigurator { + + private SignatureEncodingChecker checker; + + @BeforeEach + void setUp() { + checker = new SignatureEncodingChecker(); + assertThat(checker).isNotNull(); + } + + @Test + void testValidAttestation() { + final NodeId sender = new NodeId(new byte[100]); + final Attestation message = createAttestation(BytesValue.of(1, 2, 3)); + final ReceivedAttestation attestation = new ReceivedAttestation(sender, message); + + assertThat(checker.check(attestation)).isTrue(); + } + + @Test + void testInvalidAttestation() { + final NodeId sender = new NodeId(new byte[100]); + final AttestationData attestationData = new AttestationData( + Hashes.sha256(BytesValue.fromHexString("aa")), + new Checkpoint(EpochNumber.of(231), Hashes.sha256(BytesValue.fromHexString("bb"))), + new Checkpoint(EpochNumber.of(2), Hashes.sha256(BytesValue.fromHexString("cc"))), + Crosslink.EMPTY); + final BytesValue value = BytesValue.of(1, 2, 3); + final Attestation message = new Attestation( + Bitlist.of(value.size() * 8, value, specConstants.getMaxValidatorsPerCommittee().getValue()), + attestationData, + Bitlist.of(8, BytesValue.fromHexString("bb"), specConstants.getMaxValidatorsPerCommittee().getValue()), + BLSSignature.wrap(Bytes96.fromHexString("cc")), + specConstants); + final ReceivedAttestation attestation = new ReceivedAttestation(sender, message); + + assertThat(checker.check(attestation)).isFalse(); + } +} diff --git a/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/DoubleWorkProcessorTest.java b/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/DoubleWorkProcessorTest.java new file mode 100644 index 000000000..910e573d1 --- /dev/null +++ b/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/DoubleWorkProcessorTest.java @@ -0,0 +1,58 @@ +package org.ethereum.beacon.chain.pool.reactor; + +import org.ethereum.beacon.chain.pool.PoolTestConfigurator; +import org.ethereum.beacon.chain.pool.ReceivedAttestation; +import org.ethereum.beacon.chain.pool.registry.ProcessedAttestations; +import org.ethereum.beacon.core.operations.Attestation; +import org.ethereum.beacon.types.p2p.NodeId; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import tech.pegasys.artemis.util.bytes.BytesValue; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.ethereum.beacon.chain.pool.AttestationPool.MAX_PROCESSED_ATTESTATIONS; + +class DoubleWorkProcessorTest extends PoolTestConfigurator { + + private DoubleWorkProcessor doubleWorkProcessor; + private ReceivedAttestation attestation; + + @BeforeEach + void setUp() { + final ProcessedAttestations processedAttestations = new ProcessedAttestations(spec::hash_tree_root, MAX_PROCESSED_ATTESTATIONS); + + final NodeId sender = new NodeId(new byte[100]); + final Attestation message = createAttestation(BytesValue.of(1, 2, 3)); + attestation = new ReceivedAttestation(sender, message); + final boolean added = processedAttestations.add(attestation); + assertThat(added).isTrue(); + + doubleWorkProcessor = new DoubleWorkProcessor(processedAttestations, schedulers, source); + assertThat(doubleWorkProcessor).isNotNull(); + + doubleWorkProcessor.subscribe(s -> { + assertThat(s.getSender()) + .isNotNull() + .isEqualTo(sender); + + assertThat(s.getMessage()) + .isNotNull() + .isEqualTo(message); + }); + } + + @Test + void testAddAttestation() { + source.onNext(attestation); + + doubleWorkProcessor.subscribe(s -> { + assertThat(s.getSender()) + .isNotNull() + .isEqualTo(attestation.getSender()); + + assertThat(s.getMessage()) + .isNotNull() + .isEqualTo(attestation.getMessage()); + }); + } +} diff --git a/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/IdentificationProcessorTest.java b/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/IdentificationProcessorTest.java new file mode 100644 index 000000000..b09437752 --- /dev/null +++ b/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/IdentificationProcessorTest.java @@ -0,0 +1,57 @@ +package org.ethereum.beacon.chain.pool.reactor; + +import org.ethereum.beacon.chain.BeaconTuple; +import org.ethereum.beacon.chain.MutableBeaconChain; +import org.ethereum.beacon.chain.pool.PoolTestConfigurator; +import org.ethereum.beacon.chain.pool.ReceivedAttestation; +import org.ethereum.beacon.chain.pool.registry.UnknownAttestationPool; +import org.ethereum.beacon.chain.storage.BeaconChainStorage; +import org.ethereum.beacon.chain.storage.impl.SSZBeaconChainStorageFactory; +import org.ethereum.beacon.chain.storage.impl.SerializerFactory; +import org.ethereum.beacon.core.BeaconBlock; +import org.ethereum.beacon.core.operations.Attestation; +import org.ethereum.beacon.core.types.SlotNumber; +import org.ethereum.beacon.db.InMemoryDatabase; +import org.ethereum.beacon.types.p2p.NodeId; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import reactor.core.publisher.DirectProcessor; +import tech.pegasys.artemis.util.bytes.BytesValue; + +import static org.ethereum.beacon.chain.pool.AttestationPool.MAX_ATTESTATION_LOOKAHEAD; +import static org.ethereum.beacon.chain.pool.AttestationPool.MAX_UNKNOWN_ATTESTATIONS; + +class IdentificationProcessorTest extends PoolTestConfigurator { + + private IdentificationProcessor identificationProcessor; + private final DirectProcessor newSlots = DirectProcessor.create(); + private final DirectProcessor importedBlocks = DirectProcessor.create(); + + @BeforeEach + void setUp() { + final InMemoryDatabase db = new InMemoryDatabase(); + final BeaconChainStorage beaconChainStorage = + new SSZBeaconChainStorageFactory( + spec.getObjectHasher(), SerializerFactory.createSSZ(specConstants)) + .create(db); + final UnknownAttestationPool unknownAttestationPool = new UnknownAttestationPool(beaconChainStorage.getBlockStorage(), spec, MAX_ATTESTATION_LOOKAHEAD, MAX_UNKNOWN_ATTESTATIONS); + identificationProcessor = new IdentificationProcessor(unknownAttestationPool, schedulers, source, newSlots, importedBlocks); + newSlots.onNext(SlotNumber.of(500)); + + final MutableBeaconChain beaconChain = createBeaconChain(spec, perSlotTransition, schedulers); + beaconChain.init(); + final BeaconTuple recentlyProcessed = beaconChain.getRecentlyProcessed(); + final BeaconTuple aTuple = createBlock(recentlyProcessed, spec, + schedulers.getCurrentTime(), perSlotTransition); + final BeaconBlock aBlock = aTuple.getBlock(); + importedBlocks.onNext(aBlock); + } + + @Test + void testPublishValidAttestation() { + final NodeId sender = new NodeId(new byte[100]); + final Attestation message = createAttestation(BytesValue.fromHexString("aa")); + final ReceivedAttestation attestation = new ReceivedAttestation(sender, message); + source.onNext(attestation); + } +} diff --git a/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/MultiProcessorTest.java b/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/MultiProcessorTest.java new file mode 100644 index 000000000..e2f9afb53 --- /dev/null +++ b/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/MultiProcessorTest.java @@ -0,0 +1,94 @@ +package org.ethereum.beacon.chain.pool.reactor; + +import org.ethereum.beacon.chain.pool.AttestationPool; +import org.ethereum.beacon.chain.pool.PoolTestConfigurator; +import org.ethereum.beacon.chain.pool.ReceivedAttestation; +import org.ethereum.beacon.chain.pool.checker.SanityChecker; +import org.ethereum.beacon.chain.pool.checker.SignatureEncodingChecker; +import org.ethereum.beacon.chain.pool.checker.TimeFrameFilter; +import org.ethereum.beacon.chain.pool.registry.ProcessedAttestations; +import org.ethereum.beacon.core.operations.Attestation; +import org.ethereum.beacon.core.state.Checkpoint; +import org.ethereum.beacon.core.types.EpochNumber; +import org.ethereum.beacon.core.types.SlotNumber; +import org.ethereum.beacon.types.p2p.NodeId; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import reactor.core.publisher.DirectProcessor; +import tech.pegasys.artemis.util.bytes.BytesValue; + +import java.time.Duration; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.ethereum.beacon.chain.pool.AttestationPool.MAX_PROCESSED_ATTESTATIONS; + +class MultiProcessorTest extends PoolTestConfigurator { + + private final Checkpoint checkpoint = Checkpoint.EMPTY; + private final SlotNumber slotNumber = SlotNumber.of(100L); + private DirectProcessor newSlots = DirectProcessor.create(); + + private TimeProcessor timeProcessor; + private SanityProcessor sanityProcessor; + private DoubleWorkProcessor doubleWorkProcessor; + private SignatureEncodingProcessor signatureEncodingProcessor; + + @BeforeEach + void setUp() { + final TimeFrameFilter timeFrameFilter = new TimeFrameFilter(spec, EpochNumber.of(233)); + timeProcessor = new TimeProcessor(timeFrameFilter, schedulers, source, finalizedCheckpoints, newSlots); + + final SanityChecker sanityChecker = new SanityChecker(spec); + sanityProcessor = new SanityProcessor(sanityChecker, schedulers, timeProcessor, finalizedCheckpoints); + + final ProcessedAttestations processedAttestations = new ProcessedAttestations(spec::hash_tree_root, MAX_PROCESSED_ATTESTATIONS); + doubleWorkProcessor = new DoubleWorkProcessor(processedAttestations, schedulers, sanityProcessor.getValid()); + + final SignatureEncodingChecker checker = new SignatureEncodingChecker(); + org.ethereum.beacon.schedulers.Scheduler parallelExecutor = schedulers.newParallelDaemon("attestation-pool-%d", AttestationPool.MAX_THREADS); + signatureEncodingProcessor = new SignatureEncodingProcessor(checker, parallelExecutor, doubleWorkProcessor); + + finalizedCheckpoints.onNext(this.checkpoint); + newSlots.onNext(slotNumber); + + schedulers.addTime(Duration.ofSeconds(5)); + } + + @Test + @DisplayName("Process attestation by multiple processors") + void processAttestation() { + final NodeId sender = new NodeId(new byte[100]); + final Attestation message = createAttestation(BytesValue.fromHexString("aa")); + final ReceivedAttestation attestation = new ReceivedAttestation(sender, message); + source.onNext(attestation); + + schedulers.addTime(Duration.ofSeconds(5)); + + timeProcessor.subscribe(s -> { + System.out.println("TimeProcessor subscription"); + assertThat(s.getSender()) + .isNotNull() + .isEqualTo(sender); + + assertThat(s.getMessage()) + .isNotNull() + .isEqualTo(message); + }); + + //TODO: add assert to sanity checker + + doubleWorkProcessor.subscribe(s -> { + System.out.println("DoubleWorkProcessor subscription"); + assertThat(s.getSender()) + .isNotNull() + .isEqualTo(attestation.getSender()); + + assertThat(s.getMessage()) + .isNotNull() + .isEqualTo(attestation.getMessage()); + }); + + //TODO: add assert to signatureEncodingProcessor + } +} diff --git a/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/SanityProcessorTest.java b/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/SanityProcessorTest.java new file mode 100644 index 000000000..cf65e55e3 --- /dev/null +++ b/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/SanityProcessorTest.java @@ -0,0 +1,63 @@ +package org.ethereum.beacon.chain.pool.reactor; + +import org.ethereum.beacon.chain.pool.PoolTestConfigurator; +import org.ethereum.beacon.chain.pool.ReceivedAttestation; +import org.ethereum.beacon.chain.pool.checker.SanityChecker; +import org.ethereum.beacon.core.operations.Attestation; +import org.ethereum.beacon.core.operations.attestation.AttestationData; +import org.ethereum.beacon.core.operations.attestation.Crosslink; +import org.ethereum.beacon.core.state.Checkpoint; +import org.ethereum.beacon.core.types.EpochNumber; +import org.ethereum.beacon.crypto.Hashes; +import org.ethereum.beacon.types.p2p.NodeId; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import tech.pegasys.artemis.util.bytes.BytesValue; + +import java.time.Duration; + +class SanityProcessorTest extends PoolTestConfigurator { + + private SanityProcessor sanityProcessor; + + @BeforeEach + void setUp() { + final SanityChecker sanityChecker = new SanityChecker(spec); + sanityProcessor = new SanityProcessor(sanityChecker, schedulers, source, finalizedCheckpoints); + final Checkpoint checkpoint = Checkpoint.EMPTY; + finalizedCheckpoints.onNext(checkpoint); + + schedulers.addTime(Duration.ofSeconds(5)); + } + + @Test + void testValidAttestation() { + final NodeId sender = new NodeId(new byte[100]); + final Attestation message = createAttestation(BytesValue.fromHexString("aa")); + final ReceivedAttestation attestation = new ReceivedAttestation(sender, message); + source.onNext(attestation); +//TODO: assertThat +// schedulers.addTime(Duration.ofSeconds(5)); +// StepVerifier.create(sanityProcessor.getValid()) +// .expectNext(attestation) +// .verifyComplete(); + } + + @Test + void testInvalidAttestation() { + final NodeId sender = new NodeId(new byte[100]); + final AttestationData invalidAttestationData = new AttestationData( + Hashes.sha256(BytesValue.fromHexString("aa")), + new Checkpoint(EpochNumber.of(231), Hashes.sha256(BytesValue.fromHexString("bb"))), + new Checkpoint(EpochNumber.of(2), Hashes.sha256(BytesValue.fromHexString("cc"))), + Crosslink.EMPTY); + final Attestation message = createAttestation(BytesValue.fromHexString("aa"), invalidAttestationData); + final ReceivedAttestation attestation = new ReceivedAttestation(sender, message); + source.onNext(attestation); +//TODO: assertThat +// schedulers.addTime(Duration.ofSeconds(5)); +// StepVerifier.create(sanityProcessor.getInvalid()) +// .expectNext(attestation) +// .verifyComplete(); + } +} diff --git a/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/SignatureEncodingProcessorTest.java b/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/SignatureEncodingProcessorTest.java new file mode 100644 index 000000000..0c4bd3c24 --- /dev/null +++ b/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/SignatureEncodingProcessorTest.java @@ -0,0 +1,53 @@ +package org.ethereum.beacon.chain.pool.reactor; + +import org.ethereum.beacon.chain.pool.AttestationPool; +import org.ethereum.beacon.chain.pool.PoolTestConfigurator; +import org.ethereum.beacon.chain.pool.ReceivedAttestation; +import org.ethereum.beacon.chain.pool.checker.SignatureEncodingChecker; +import org.ethereum.beacon.core.operations.Attestation; +import org.ethereum.beacon.core.operations.attestation.AttestationData; +import org.ethereum.beacon.core.operations.attestation.Crosslink; +import org.ethereum.beacon.core.state.Checkpoint; +import org.ethereum.beacon.core.types.EpochNumber; +import org.ethereum.beacon.crypto.Hashes; +import org.ethereum.beacon.types.p2p.NodeId; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import tech.pegasys.artemis.util.bytes.BytesValue; + +class SignatureEncodingProcessorTest extends PoolTestConfigurator { + + private SignatureEncodingProcessor signatureEncodingProcessor; + + @BeforeEach + void setUp() { + final SignatureEncodingChecker checker = new SignatureEncodingChecker(); + org.ethereum.beacon.schedulers.Scheduler parallelExecutor = schedulers.newParallelDaemon("attestation-pool-%d", AttestationPool.MAX_THREADS); + signatureEncodingProcessor = new SignatureEncodingProcessor(checker, parallelExecutor, source); + } + + @Test + void testValidAttestation() { + final NodeId sender = new NodeId(new byte[100]); + final Attestation message = createAttestation(BytesValue.fromHexString("aa")); + final ReceivedAttestation attestation = new ReceivedAttestation(sender, message); + source.onNext(attestation); + + //TODO: assert signatureEncodingProcessor.getValid() + } + + @Test + void testInvalidAttestation() { + final NodeId sender = new NodeId(new byte[100]); + final AttestationData invalidAttestationData = new AttestationData( + Hashes.sha256(BytesValue.fromHexString("aa")), + new Checkpoint(EpochNumber.of(231), Hashes.sha256(BytesValue.fromHexString("bb"))), + new Checkpoint(EpochNumber.of(2), Hashes.sha256(BytesValue.fromHexString("cc"))), + Crosslink.EMPTY); + final Attestation message = createAttestation(BytesValue.fromHexString("aa"), invalidAttestationData); + final ReceivedAttestation attestation = new ReceivedAttestation(sender, message); + source.onNext(attestation); + + //TODO: assert signatureEncodingProcessor.getInvalid() + } +} diff --git a/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/TimeProcessorTest.java b/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/TimeProcessorTest.java new file mode 100644 index 000000000..4957dbdd9 --- /dev/null +++ b/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/TimeProcessorTest.java @@ -0,0 +1,56 @@ +package org.ethereum.beacon.chain.pool.reactor; + +import org.ethereum.beacon.chain.pool.PoolTestConfigurator; +import org.ethereum.beacon.chain.pool.ReceivedAttestation; +import org.ethereum.beacon.chain.pool.checker.TimeFrameFilter; +import org.ethereum.beacon.core.operations.Attestation; +import org.ethereum.beacon.core.state.Checkpoint; +import org.ethereum.beacon.core.types.EpochNumber; +import org.ethereum.beacon.core.types.SlotNumber; +import org.ethereum.beacon.types.p2p.NodeId; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import reactor.core.publisher.DirectProcessor; +import tech.pegasys.artemis.util.bytes.BytesValue; + +import java.time.Duration; + +import static org.assertj.core.api.Assertions.assertThat; + +class TimeProcessorTest extends PoolTestConfigurator { + + private final Checkpoint checkpoint = Checkpoint.EMPTY; + private final SlotNumber slotNumber = SlotNumber.of(100L); + private final DirectProcessor newSlots = DirectProcessor.create(); + + private TimeProcessor timeProcessor; + + @BeforeEach + void setUp() { + + final TimeFrameFilter timeFrameFilter = new TimeFrameFilter(spec, EpochNumber.of(233)); + timeProcessor = new TimeProcessor(timeFrameFilter, schedulers, source, finalizedCheckpoints, newSlots); + finalizedCheckpoints.onNext(this.checkpoint); + newSlots.onNext(slotNumber); + + schedulers.addTime(Duration.ofSeconds(5)); + } + + @Test + void testValidAttestation() { + final NodeId sender = new NodeId(new byte[100]); + final Attestation message = createAttestation(BytesValue.fromHexString("aa")); + final ReceivedAttestation attestation = new ReceivedAttestation(sender, message); + source.onNext(attestation); + + timeProcessor.subscribe(s -> { + assertThat(s.getSender()) + .isNotNull() + .isEqualTo(sender); + + assertThat(s.getMessage()) + .isNotNull() + .isEqualTo(message); + }); + } +} diff --git a/consensus/src/main/java/org/ethereum/beacon/consensus/spec/BlockProcessing.java b/consensus/src/main/java/org/ethereum/beacon/consensus/spec/BlockProcessing.java index 6468cb7c7..4254738ec 100644 --- a/consensus/src/main/java/org/ethereum/beacon/consensus/spec/BlockProcessing.java +++ b/consensus/src/main/java/org/ethereum/beacon/consensus/spec/BlockProcessing.java @@ -225,6 +225,11 @@ default void process_attester_slashing(MutableBeaconState state, AttesterSlashin } default boolean verify_attestation(BeaconState state, Attestation attestation) { + return verify_attestation_impl(state, attestation, true); + } + + default boolean verify_attestation_impl(BeaconState state, Attestation attestation, + boolean verify_indexed) { /* data = attestation.data assert data.crosslink.shard < SHARD_COUNT assert data.target.epoch in (get_previous_epoch(state), get_current_epoch(state)) */ @@ -294,6 +299,10 @@ assert len(attestation.aggregation_bits) == len(attestation.custody_bits) == len return false; } + if (!verify_indexed) { + return true; + } + return is_valid_indexed_attestation(state, get_indexed_attestation(state, attestation)); } diff --git a/consensus/src/main/java/org/ethereum/beacon/consensus/spec/EpochProcessing.java b/consensus/src/main/java/org/ethereum/beacon/consensus/spec/EpochProcessing.java index ac816272b..e007069c1 100644 --- a/consensus/src/main/java/org/ethereum/beacon/consensus/spec/EpochProcessing.java +++ b/consensus/src/main/java/org/ethereum/beacon/consensus/spec/EpochProcessing.java @@ -131,7 +131,7 @@ default Pair> get_winning_crosslink_and_attestin Gwei b2 = get_attesting_balance(state, attestations.stream().filter(a -> a.getData().getCrosslink().equals(c2)).collect(toList())); if (b1.equals(b2)) { - return c1.getDataRoot().toString().compareTo(c2.getDataRoot().toString()); + return c1.getDataRoot().compareTo(c2.getDataRoot()); } else { return b1.compareTo(b2); } diff --git a/consensus/src/main/java/org/ethereum/beacon/consensus/spec/ForkChoice.java b/consensus/src/main/java/org/ethereum/beacon/consensus/spec/ForkChoice.java index 4afb89087..bc3853bb8 100644 --- a/consensus/src/main/java/org/ethereum/beacon/consensus/spec/ForkChoice.java +++ b/consensus/src/main/java/org/ethereum/beacon/consensus/spec/ForkChoice.java @@ -10,6 +10,7 @@ import org.ethereum.beacon.core.types.Gwei; import org.ethereum.beacon.core.types.SlotNumber; import org.ethereum.beacon.core.types.ValidatorIndex; +import org.javatuples.Pair; import tech.pegasys.artemis.ethereum.core.Hash32; /** @@ -109,7 +110,7 @@ default Hash32 get_head(Store store) { } head = children.stream() - .max(Comparator.comparing(root -> get_latest_attesting_balance(store, root))) + .max(Comparator.comparing(root -> Pair.with(get_latest_attesting_balance(store, root), root))) .get(); } } diff --git a/consensus/src/main/java/org/ethereum/beacon/consensus/spec/HelperFunction.java b/consensus/src/main/java/org/ethereum/beacon/consensus/spec/HelperFunction.java index 15a6fc9ca..d597fa55b 100644 --- a/consensus/src/main/java/org/ethereum/beacon/consensus/spec/HelperFunction.java +++ b/consensus/src/main/java/org/ethereum/beacon/consensus/spec/HelperFunction.java @@ -1,6 +1,17 @@ package org.ethereum.beacon.consensus.spec; +import static java.util.Collections.emptyList; +import static java.util.stream.Collectors.toList; +import static org.ethereum.beacon.core.spec.SignatureDomains.ATTESTATION; + import com.google.common.collect.Ordering; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.ethereum.beacon.core.BeaconBlock; import org.ethereum.beacon.core.BeaconBlockBody; import org.ethereum.beacon.core.BeaconBlockHeader; @@ -37,18 +48,6 @@ import tech.pegasys.artemis.util.uint.UInt64; import tech.pegasys.artemis.util.uint.UInt64s; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static java.util.Collections.emptyList; -import static java.util.stream.Collectors.toList; -import static org.ethereum.beacon.core.spec.SignatureDomains.ATTESTATION; - /** * Helper functions. * @@ -897,7 +896,27 @@ default PublicKey bls_aggregate_pubkeys(List publicKeysBytes) { } List publicKeys = publicKeysBytes.stream().map(PublicKey::create).collect(toList()); - return PublicKey.aggregate(publicKeys); + + if (publicKeys.size() == 1) { + return publicKeys.get(0); + } else { + return PublicKey.aggregate(publicKeys); + } + } + + default PublicKey bls_aggregate_pubkeys_no_validate(List publicKeysBytes) { + if (!isBlsVerify()) { + return PublicKey.aggregate(Collections.emptyList()); + } + + List publicKeys = + publicKeysBytes.stream().map(PublicKey::createWithoutValidation).collect(toList()); + + if (publicKeys.size() == 1) { + return publicKeys.get(0); + } else { + return PublicKey.aggregate(publicKeys); + } } /* @@ -993,6 +1012,11 @@ def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: Indexe """ */ default boolean is_valid_indexed_attestation(BeaconState state, IndexedAttestation indexed_attestation) { + return is_valid_indexed_attestation_impl(state, indexed_attestation, true); + } + + default boolean is_valid_indexed_attestation_impl(BeaconState state, IndexedAttestation indexed_attestation, + boolean verify_signature) { /* bit_0_indices = indexed_attestation.custody_bit_0_indices bit_1_indices = indexed_attestation.custody_bit_1_indices @@ -1024,6 +1048,10 @@ default boolean is_valid_indexed_attestation(BeaconState state, IndexedAttestati return false; } + if (!verify_signature) { + return true; + } + /* return bls_verify_multiple( pubkeys=[ diff --git a/crypto/src/main/java/org/ethereum/beacon/crypto/BLS381.java b/crypto/src/main/java/org/ethereum/beacon/crypto/BLS381.java index 5977574f2..7d0c4d931 100644 --- a/crypto/src/main/java/org/ethereum/beacon/crypto/BLS381.java +++ b/crypto/src/main/java/org/ethereum/beacon/crypto/BLS381.java @@ -213,6 +213,31 @@ public static Signature create(Bytes96 encoded) { return new Signature(encoded); } + public static Signature createWithoutValidation(Bytes96 encoded) { + return new Signature(encoded); + } + + public static boolean validate(Bytes96 encoded) { + Validator.Result result = Validator.G2.validate(encoded); + if (!result.isValid()) { + return false; + } + + if (!Codec.G2.decode(encoded).isInfinity()) { + ECP2 point = G2.decode(encoded); + if (point.is_infinity()) { + return false; + } + + ECP2 orderCheck = point.mul(ORDER); + if (!orderCheck.is_infinity()) { + return false; + } + } + + return true; + } + /** * Aggregates a list of signatures into a single one. * diff --git a/start/common/src/main/java/org/ethereum/beacon/start/common/Launcher.java b/start/common/src/main/java/org/ethereum/beacon/start/common/Launcher.java index e75d9f627..6081b38a9 100644 --- a/start/common/src/main/java/org/ethereum/beacon/start/common/Launcher.java +++ b/start/common/src/main/java/org/ethereum/beacon/start/common/Launcher.java @@ -1,16 +1,22 @@ package org.ethereum.beacon.start.common; +import java.util.List; import org.ethereum.beacon.bench.BenchmarkController; import org.ethereum.beacon.bench.BenchmarkController.BenchmarkRoutine; +import org.ethereum.beacon.chain.BeaconTuple; import org.ethereum.beacon.chain.DefaultBeaconChain; +import org.ethereum.beacon.chain.ForkChoiceProcessor; +import org.ethereum.beacon.chain.LMDGhostProcessor; import org.ethereum.beacon.chain.MutableBeaconChain; import org.ethereum.beacon.chain.ProposedBlockProcessor; import org.ethereum.beacon.chain.ProposedBlockProcessorImpl; import org.ethereum.beacon.chain.SlotTicker; import org.ethereum.beacon.chain.observer.ObservableStateProcessor; -import org.ethereum.beacon.chain.observer.ObservableStateProcessorImpl; +import org.ethereum.beacon.chain.pool.AttestationPool; +import org.ethereum.beacon.chain.pool.ReceivedAttestation; import org.ethereum.beacon.chain.storage.BeaconChainStorage; import org.ethereum.beacon.chain.storage.BeaconChainStorageFactory; +import org.ethereum.beacon.chain.storage.util.StorageUtils; import org.ethereum.beacon.consensus.BeaconChainSpec; import org.ethereum.beacon.consensus.BeaconStateEx; import org.ethereum.beacon.consensus.ChainStart; @@ -29,7 +35,6 @@ import org.ethereum.beacon.db.InMemoryDatabase; import org.ethereum.beacon.pow.DepositContract; import org.ethereum.beacon.schedulers.Schedulers; -import org.ethereum.beacon.chain.storage.util.StorageUtils; import org.ethereum.beacon.util.stats.MeasurementsCollector; import org.ethereum.beacon.validator.BeaconChainProposer; import org.ethereum.beacon.validator.attester.BeaconChainAttesterImpl; @@ -41,8 +46,6 @@ import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; -import java.util.List; - public class Launcher { private final BeaconChainSpec spec; private final DepositContract depositContract; @@ -64,6 +67,8 @@ public class Launcher { private BeaconChainStorage beaconChainStorage; private MutableBeaconChain beaconChain; private SlotTicker slotTicker; + private AttestationPool attestationPool; + private ForkChoiceProcessor forkChoiceProcessor; private ObservableStateProcessor observableStateProcessor; private BeaconChainProposer beaconChainProposer; private BeaconChainAttesterImpl beaconChainAttester; @@ -150,14 +155,37 @@ void chainStarted(ChainStart chainStartEvent) { .publishOn(schedulers.events().toReactor()) .subscribe(allAttestations); - observableStateProcessor = new ObservableStateProcessorImpl( - beaconChainStorage, - slotTicker.getTickerStream(), - allAttestations, - beaconChain.getBlockStatesStream(), - spec, - emptySlotTransition, - schedulers); + attestationPool = + AttestationPool.create( + allAttestations.map(ReceivedAttestation::new), + slotTicker.getTickerStream(), + beaconChain.getFinalizedCheckpoints(), + Flux.from(beaconChain.getBlockStatesStream()).map(BeaconTuple::getBlock), + schedulers, + spec, + beaconChainStorage, + emptySlotTransition); + attestationPool.start(); + + forkChoiceProcessor = + new LMDGhostProcessor( + spec, + beaconChainStorage, + schedulers, + beaconChain.getJustifiedCheckpoints(), + attestationPool.getValidIndexed(), + beaconChain.getBlockStatesStream()); + + observableStateProcessor = + ObservableStateProcessor.createNew( + spec, + emptySlotTransition, + schedulers, + slotTicker.getTickerStream(), + forkChoiceProcessor.getChainHeads(), + beaconChain.getJustifiedCheckpoints(), + beaconChain.getFinalizedCheckpoints(), + attestationPool.getValidUnboxed()); observableStateProcessor.start(); if (validatorCred != null) { @@ -285,6 +313,10 @@ public ExtendedSlotTransition getExtendedSlotTransition() { return extendedSlotTransition; } + public EmptySlotTransition getEmptySlotTransition() { + return emptySlotTransition; + } + public BeaconBlockVerifier getBlockVerifier() { return blockVerifier; } @@ -344,4 +376,12 @@ public MeasurementsCollector getEpochCollector() { public MeasurementsCollector getBlockCollector() { return blockCollector; } + + public AttestationPool getAttestationPool() { + return attestationPool; + } + + public ForkChoiceProcessor getForkChoiceProcessor() { + return forkChoiceProcessor; + } } diff --git a/types/src/main/java/org/ethereum/beacon/types/p2p/NodeId.java b/types/src/main/java/org/ethereum/beacon/types/p2p/NodeId.java new file mode 100644 index 000000000..efba92b44 --- /dev/null +++ b/types/src/main/java/org/ethereum/beacon/types/p2p/NodeId.java @@ -0,0 +1,10 @@ +package org.ethereum.beacon.types.p2p; + +import tech.pegasys.artemis.util.bytes.ArrayWrappingBytesValue; +import tech.pegasys.artemis.util.bytes.BytesValue; + +public class NodeId extends ArrayWrappingBytesValue implements BytesValue { + public NodeId(byte[] bytes) { + super(bytes); + } +} diff --git a/types/src/main/java/tech/pegasys/artemis/util/collections/Bitlist.java b/types/src/main/java/tech/pegasys/artemis/util/collections/Bitlist.java index 074a98f51..4262d602d 100644 --- a/types/src/main/java/tech/pegasys/artemis/util/collections/Bitlist.java +++ b/types/src/main/java/tech/pegasys/artemis/util/collections/Bitlist.java @@ -210,6 +210,10 @@ public int size() { return size; } + public boolean isEmpty() { + return BytesValues.countZeros(wrapped) == wrapped.size(); + } + public long maxSize() { return maxSize; } diff --git a/types/src/test/java/tech/pegasys/artemis/util/bytes/BytesValueTest.java b/types/src/test/java/tech/pegasys/artemis/util/bytes/BytesValueTest.java index 762682236..27abc3fc0 100644 --- a/types/src/test/java/tech/pegasys/artemis/util/bytes/BytesValueTest.java +++ b/types/src/test/java/tech/pegasys/artemis/util/bytes/BytesValueTest.java @@ -16,7 +16,6 @@ import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; import io.vertx.core.buffer.Buffer; -import net.consensys.cava.bytes.MutableBytes; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; @@ -857,6 +856,49 @@ public void testBytesValuesComparatorReturnsMatchUnsignedValueByteValue() { assertThat(small.compareTo(otherSmall)).isEqualTo(0); } + @Test + public void testBytesValueComparator_lexicographical_order() { + checkOrder(h("0x00"), h("0x01")); + checkOrder(h("0x01"), h("0x7f")); + checkOrder(h("0x7f"), h("0x80")); + checkOrder(h("0x80"), h("0xff")); + + checkOrder(h("0x0000"), h("0x0001")); + checkOrder(h("0x0001"), h("0x007f")); + checkOrder(h("0x007f"), h("0x0080")); + checkOrder(h("0x0080"), h("0x00ff")); + + checkOrder(h("0x0000"), h("0x0100")); + checkOrder(h("0x0100"), h("0x7f00")); + checkOrder(h("0x7f00"), h("0x8000")); + checkOrder(h("0x8000"), h("0xff00")); + + checkOrder(h("0x00"), h("0x0100")); + checkOrder(h("0x01"), h("0x7f00")); + checkOrder(h("0x7f"), h("0x8000")); + checkOrder(h("0x80"), h("0xff00")); + + checkOrder(h("0x00"), h("0x01ff")); + checkOrder(h("0x01"), h("0x7fff")); + checkOrder(h("0x7f"), h("0x80ff")); + checkOrder(h("0x80"), h("0xffff")); + + checkOrder(h("0x0001"), h("0x0100")); + checkOrder(h("0x007f"), h("0x7f00")); + checkOrder(h("0x0080"), h("0x8000")); + checkOrder(h("0x00ff"), h("0xff00")); + + checkOrder(h("0x000001"), h("0x010000")); + checkOrder(h("0x00007f"), h("0x7f0000")); + checkOrder(h("0x000080"), h("0x800000")); + checkOrder(h("0x0000ff"), h("0xff0000")); + } + + static void checkOrder(BytesValue lesser, BytesValue greater) { + assertThat(lesser).isLessThan(greater); + assertThat(greater).isGreaterThan(lesser); + } + @Test public void testGetSetBit() { MutableBytesValue bytes = MutableBytesValue.create(4); diff --git a/util/src/main/java/org/ethereum/beacon/stream/Fluxes.java b/util/src/main/java/org/ethereum/beacon/stream/Fluxes.java new file mode 100644 index 000000000..2078182f6 --- /dev/null +++ b/util/src/main/java/org/ethereum/beacon/stream/Fluxes.java @@ -0,0 +1,68 @@ +package org.ethereum.beacon.stream; + +import java.util.function.Predicate; +import org.javatuples.Pair; +import org.reactivestreams.Publisher; +import reactor.core.Disposable; +import reactor.core.publisher.ConnectableFlux; +import reactor.core.publisher.Flux; + +/** Various utility methods to work with {@link Flux}. */ +public abstract class Fluxes { + private Fluxes() {} + + /** + * Given predicate creates a flux split. + * + * @param source a source. + * @param predicate a predicate. + * @param a kind of source data. + * @return a flux split. + */ + public static FluxSplit split(Publisher source, Predicate predicate) { + return new FluxSplit<>(source, predicate); + } + + /** + * A split of some publisher made upon a predicate. + * + *

Built atop of {@link ConnectableFlux}. + * + * @param a kind of source data. + */ + public static final class FluxSplit { + private final Flux satisfied; + private final Flux unsatisfied; + private final Disposable disposable; + + /** + * Given source and predicate creates a split. + * + * @param source a source publisher. + * @param predicate a predicate. + */ + FluxSplit(Publisher source, Predicate predicate) { + ConnectableFlux> splitter = + Flux.from(source).map(value -> Pair.with(predicate.test(value), value)).publish(); + + this.satisfied = splitter.filter(Pair::getValue0).map(Pair::getValue1); + this.unsatisfied = splitter.filter(pair -> !pair.getValue0()).map(Pair::getValue1); + this.disposable = splitter.connect(); + } + + /** @return a flux of data items which predicates with {@code true}. */ + public Flux getSatisfied() { + return satisfied; + } + + /** @return a flux of data items which predicates with {@code false}. */ + public Flux getUnsatisfied() { + return unsatisfied; + } + + /** @return a disposable for connection between source and outcomes. */ + public Disposable getDisposable() { + return disposable; + } + } +} diff --git a/validator/server/src/test/java/org/ethereum/beacon/validator/api/ServiceFactory.java b/validator/server/src/test/java/org/ethereum/beacon/validator/api/ServiceFactory.java index 50e60a05b..e4229c3aa 100644 --- a/validator/server/src/test/java/org/ethereum/beacon/validator/api/ServiceFactory.java +++ b/validator/server/src/test/java/org/ethereum/beacon/validator/api/ServiceFactory.java @@ -105,11 +105,6 @@ public static ObservableStateProcessor createObservableStateProcessor(SpecConsta @Override public void start() {} - @Override - public Publisher getHeadStream() { - return null; - } - @Override public Publisher getObservableStateStream() { return Mono.just( @@ -124,11 +119,6 @@ public Publisher getObservableStateStream() { BeaconStateEx.getEmpty(), new PendingOperationsState(Collections.emptyList()))); } - - @Override - public Publisher getPendingOperationsStream() { - return null; - } }; } @@ -138,11 +128,6 @@ public static ObservableStateProcessor createObservableStateProcessorGenesisTime @Override public void start() {} - @Override - public Publisher getHeadStream() { - return null; - } - @Override public Publisher getObservableStateStream() { MutableBeaconState state = BeaconStateEx.getEmpty().createMutableCopy(); @@ -159,11 +144,6 @@ public Publisher getObservableStateStream() { new BeaconStateExImpl(state.createImmutable()), new PendingOperationsState(Collections.emptyList()))); } - - @Override - public Publisher getPendingOperationsStream() { - return null; - } }; } @@ -173,11 +153,6 @@ public static ObservableStateProcessor createObservableStateProcessorWithValidat @Override public void start() {} - @Override - public Publisher getHeadStream() { - return null; - } - @Override public Publisher getObservableStateStream() { MutableBeaconState state = BeaconStateEx.getEmpty().createMutableCopy(); @@ -198,11 +173,6 @@ public Publisher getObservableStateStream() { new BeaconStateExImpl(state.createImmutable()), new PendingOperationsState(Collections.emptyList()))); } - - @Override - public Publisher getPendingOperationsStream() { - return null; - } }; } @@ -469,6 +439,16 @@ public BeaconTuple getRecentlyProcessed() { return null; } + @Override + public Publisher getJustifiedCheckpoints() { + return null; + } + + @Override + public Publisher getFinalizedCheckpoints() { + return null; + } + @Override public void init() {} }; diff --git a/versions.gradle b/versions.gradle index 7deb8ce43..4c9fae3e5 100644 --- a/versions.gradle +++ b/versions.gradle @@ -10,6 +10,7 @@ dependencyManagement { dependency "org.apache.logging.log4j:log4j-api:${log4j2Version}" dependency "org.apache.logging.log4j:log4j-core:${log4j2Version}" + dependency "org.apache.commons:commons-collections4:4.4" dependency 'org.ethereum:ethereumj-core:1+' dependency 'org.bouncycastle:bcprov-jdk15on:1.60'