Skip to content

[WIP]Indexer #524

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 44 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
44 commits
Select commit Hold shift + click to select a range
01b15ae
Draft APIs for v0.6
altafan Mar 20, 2025
f067afe
Add endpoint to get chain of txs for a vtxo
altafan Mar 21, 2025
5b8f528
Fixes after review
altafan Mar 21, 2025
216cd39
New iteration
altafan Mar 26, 2025
988f96e
Fixes
altafan Mar 26, 2025
2540395
Fix virtual tx apis
altafan Mar 28, 2025
6fc9c80
Add endpoint to explorer to subscribe for addresses
altafan Mar 28, 2025
e80f819
init
sekulicd Mar 28, 2025
1311785
Merge remote-tracking branch 'origin/new-api-spec' into explorer-api-…
sekulicd Mar 28, 2025
3c8b802
wip explorer
sekulicd Mar 28, 2025
d00b843
explorer get tx history
sekulicd Mar 31, 2025
5a47ad3
pr review
sekulicd Apr 2, 2025
4b44911
pr review
sekulicd Apr 2, 2025
ee11775
indexer.proto
sekulicd Apr 3, 2025
b512da3
Merge branch 'master' into explorer-api-impl
sekulicd Apr 3, 2025
57e2978
tmp
sekulicd Apr 4, 2025
97f6f3b
merge with master
sekulicd Apr 4, 2025
91d7e3f
fix lint
sekulicd Apr 7, 2025
cd50656
fix
sekulicd Apr 7, 2025
d8b6809
fix
sekulicd Apr 7, 2025
c28bb48
init
sekulicd Apr 8, 2025
eaa302a
sdk indexer impl
sekulicd Apr 8, 2025
fe47190
merge
sekulicd Apr 8, 2025
4576949
fix
sekulicd Apr 8, 2025
3010197
Merge branch 'master' into indexer
sekulicd Apr 11, 2025
845fd8f
pr review
sekulicd Apr 11, 2025
36effad
Add indexer to grpc-gateway
altafan Apr 18, 2025
ccbaaa1
Fix queries
altafan Apr 18, 2025
f7974f4
Fix app layer
altafan Apr 18, 2025
ef64ae4
Fix interface layer
altafan Apr 18, 2025
d119966
Update protos
altafan Apr 18, 2025
aea61bc
Update sdk
altafan Apr 18, 2025
b1978dc
Fix
altafan Apr 18, 2025
ae9e3d3
Add api to vtxo repo to get all vtxos with pubkey & Fixes
altafan Apr 22, 2025
85b3098
Add info to chained tx & Fix rest endpoint conflict
altafan Apr 22, 2025
0e8640c
Fixes
altafan Apr 22, 2025
6024b68
Updates and fixes to indexer svc
altafan Apr 22, 2025
144a46b
Update client sdk
altafan Apr 22, 2025
cb3a337
Doxkerize sqlc command
altafan Apr 22, 2025
d893ee7
Merge branch 'master' into indexer
altafan Apr 22, 2025
20aa611
Fix test
altafan Apr 22, 2025
49b51d5
Fix return txid in history endpoint
altafan Apr 22, 2025
1463fbe
Fix returning redeem txs with matching txid in sqlite db
altafan Apr 22, 2025
f6ff2dd
Fix spendable vtxos endpoint
altafan Apr 22, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
796 changes: 796 additions & 0 deletions api-spec/openapi/swagger/ark/v1/indexer.swagger.json

Large diffs are not rendered by default.

44 changes: 44 additions & 0 deletions api-spec/openapi/swagger/ark/v1/new_types.swagger.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
{
"swagger": "2.0",
"info": {
"title": "ark/v1/new_types.proto",
"version": "version not set"
},
"consumes": [
"application/json"
],
"produces": [
"application/json"
],
"paths": {},
"definitions": {
"protobufAny": {
"type": "object",
"properties": {
"@type": {
"type": "string"
}
},
"additionalProperties": {}
},
"rpcStatus": {
"type": "object",
"properties": {
"code": {
"type": "integer",
"format": "int32"
},
"message": {
"type": "string"
},
"details": {
"type": "array",
"items": {
"type": "object",
"$ref": "#/definitions/protobufAny"
}
}
}
}
}
}
8 changes: 4 additions & 4 deletions api-spec/protobuf/ark/v1/admin.proto
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ service AdminService {
post: "/v1/admin/rounds"
body: "*"
};
}
}
rpc CreateNote(CreateNoteRequest) returns (CreateNoteResponse) {
option (google.api.http) = {
post: "/v1/admin/note"
Expand Down Expand Up @@ -98,11 +98,11 @@ message CreateNoteResponse {

message GetMarketHourConfigRequest {}
message GetMarketHourConfigResponse {
MarketHourConfig config = 1;
MarketHourConfig config = 1;
}

message UpdateMarketHourConfigRequest {
MarketHourConfig config = 1;
MarketHourConfig config = 1;
}
message UpdateMarketHourConfigResponse {}

Expand Down Expand Up @@ -131,4 +131,4 @@ message WithdrawRequest {

message WithdrawResponse {
string txid = 1;
}
}
222 changes: 222 additions & 0 deletions api-spec/protobuf/ark/v1/indexer.proto
Original file line number Diff line number Diff line change
@@ -0,0 +1,222 @@
syntax = "proto3";

package ark.v1;

import "google/api/annotations.proto";

service IndexerService {
rpc GetCommitmentTx(GetCommitmentTxRequest) returns (GetCommitmentTxResponse) {
option (google.api.http) = {
get: "/v1/commitmentTx/{txid}"
};
};
rpc GetVtxoTree(GetVtxoTreeRequest) returns (GetVtxoTreeResponse) {
option (google.api.http) = {
get: "/v1/batch/{batch_outpoint.txid}/{batch_outpoint.vout}/tree"
};
};
rpc GetForfeitTxs(GetForfeitTxsRequest) returns (GetForfeitTxsResponse) {
option (google.api.http) = {
get: "/v1/batch/{batch_outpoint.txid}/{batch_outpoint.vout}/forfeitTxs"
};
};
rpc GetConnectors(GetConnectorsRequest) returns (GetConnectorsResponse) {
option (google.api.http) = {
get: "/v1/batch/{batch_outpoint.txid}/{batch_outpoint.vout}/connectors"
};
};
rpc GetSpendableVtxos(GetSpendableVtxosRequest) returns (GetSpendableVtxosResponse) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I personally find this call a bit limiting. Why not be GetVtxos and GetVtxosRequest would have extra filters ?

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we would have 2 lists in the response that would make the pagination tricky/impossible

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why 2 lists?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the IndexerVtxo holds enough data that you can return a flat list

option (google.api.http) = {
get: "/v1/spendableVtxos/{address}"
};
};
rpc GetTransactionHistory(GetTransactionHistoryRequest) returns (GetTransactionHistoryResponse) {
option (google.api.http) = {
get: "/v1/history/{address}"
};
};
rpc GetVtxoChain(GetVtxoChainRequest) returns (GetVtxoChainResponse) {
option (google.api.http) = {
get: "/v1/vtxo/{outpoint.txid}/{outpoint.vout}/chain"
};
}
rpc GetVirtualTxs(GetVirtualTxsRequest) returns (GetVirtualTxsResponse) {
option (google.api.http) = {
get: "/v1/virtualTx/{txids}"
};
}
rpc GetSweptCommitmentTx(GetSweptCommitmentTxRequest) returns (GetSweptCommitmentTxResponse) {
option (google.api.http) = {
get: "/v1/commitmentTx/{txid}/swept"
};
}
}

message GetCommitmentTxRequest {
string txid = 1;
}
message GetCommitmentTxResponse {
int64 started_at = 1;
int64 ended_at = 2;
map<uint32, IndexerBatch> batches = 3;
}

message GetVtxoTreeRequest {
IndexerOutpoint batch_outpoint = 1;
IndexerPageRequest page = 2;
}
message GetVtxoTreeResponse {
repeated IndexerNode vtxo_tree = 1;
IndexerPageResponse page = 2;
}

message GetForfeitTxsRequest {
IndexerOutpoint batch_outpoint = 1;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Would do on a commitment tx level

IndexerPageRequest page = 2;
}
message GetForfeitTxsResponse {
repeated string txs = 1;
IndexerPageResponse page = 2;
}

message GetConnectorsRequest {
IndexerOutpoint batch_outpoint = 1;
IndexerPageRequest page = 2;
}
message GetConnectorsResponse {
repeated IndexerNode connectors = 1;
IndexerPageResponse page = 2;
}

message GetSpendableVtxosRequest {
string address = 1;
IndexerPageRequest page = 2;
}
message GetSpendableVtxosResponse {
repeated IndexerVtxo vtxos = 1;
IndexerPageResponse page = 2;
}

message GetTransactionHistoryRequest {
string address = 1;
int64 start_time = 2;
int64 end_time = 3;
IndexerPageRequest page = 4;
}
message GetTransactionHistoryResponse {
repeated IndexerTxHistoryRecord history = 1;
IndexerPageResponse page = 2;
}

message GetVtxoChainRequest {
IndexerOutpoint outpoint = 1;
IndexerPageRequest page = 2;
}
message GetVtxoChainResponse {
map<string, IndexerTransactions> graph = 1;
IndexerPageResponse page = 2;
}

message GetVirtualTxsRequest {
repeated string txids = 1;
IndexerPageRequest page = 2;
}
message GetVirtualTxsResponse {
repeated string txs = 1;
IndexerPageResponse page = 2;
}

message GetSweptCommitmentTxRequest {
string txid = 1;
}
message GetSweptCommitmentTxResponse {
repeated string swept_by = 1;
}

message SubscribeForAddressesRequest {
repeated string addresses = 1;
}
message SubscribeForAddressesResponse {
string address = 1;
repeated IndexerVtxo new_vtxos = 2;
repeated IndexerVtxo spent_vtxos = 3;
}

message IndexerTransactions {
repeated IndexerChain txs = 1;
int64 expires_at = 2;
}

message IndexerBatch {
uint64 total_batch_amount = 1;
uint64 total_forfeit_amount = 2;
int32 total_input_vtxos = 3;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I thought we were removing this type (inputs) of data from the indexer? I would also move them from being batch specific to CommitmentTx level

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

total_input_vtxos here refers to the count of vtxos forfeited during the batch process

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i prefer redundancy here and have total_input_vtxos also in GetCommitmentTx as a cumulative sum

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let's say a multi batch tx happens and:
a user has a VTXO of 0.1 BTC but wants to get 2 VTXOs of 0.5 each.

These 0.5 VTXOs potentially end up in 2 different batches. There is only one forfeit. Only a total_input_vtxos on GetCommitmentTx would make sense.

When a round is being done, intents should be on the round, batches only dictate vtxo and tree signing, but the forfeiture is on a commitment tx scope?

int32 total_output_vtxos = 4;
int64 expires_at = 5;
bool swept = 6;
}

message IndexerOutpoint {
string txid = 1;
uint32 vout = 2;
}

message IndexerNode {
string txid = 1;
string parent_txid = 3;
int32 level = 4;
int32 level_index = 5;
}

message IndexerVtxo {
IndexerOutpoint outpoint = 1;
int64 created_at = 2;
int64 expires_at = 3;
uint64 amount = 4;
string script = 5;
bool is_leaf = 6;
bool is_swept = 7;
bool is_spent = 8;
string spent_by = 9;
}

message IndexerChain {
string txid = 1;
IndexerChainedTxType type = 2;
}

message IndexerTxHistoryRecord {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This message confuses me quite a bit:

  • Why would the key ever be a commitment tx?
  • What is amount specifically here? Is it a "balance delta"?
  • Unsure how the type works: a transactions that spends a vtxo from addressA and sends some funds to someone, and change back to addressA would mean the type of this tx is both INDEXER_TX_TYPE_RECEIVED and INDEXER_TX_TYPE_SENT. No clue what INDEXER_TX_TYPE_SWEEP means
  • created_at is actually a cosigning request timestamp right?
  • confirmed_at relates to its parent commitment tx(s)?
  • is_settled Unsure what it means

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would introduce a way to fetch all effected vtxos with each record, something similar to the SubscribeForAddressesResponse message. I guess fits with my suggestion to change GetSpendableVtxos to GetVtxos

oneof key {
string commitment_txid = 1;
string virtual_txid = 2;
}
IndexerTxType type = 3;
uint64 amount = 4;
int64 created_at = 5;
int64 confirmed_at = 6;
bool is_settled = 7;
}

enum IndexerTxType {
INDEXER_TX_TYPE_UNSPECIFIED = 0;
INDEXER_TX_TYPE_RECEIVED = 1;
INDEXER_TX_TYPE_SENT = 2;
INDEXER_TX_TYPE_SWEEP = 3;
}

enum IndexerChainedTxType {
INDEXER_CHAINED_TX_TYPE_UNSPECIFIED = 0;
INDEXER_CHAINED_TX_TYPE_VIRTUAL = 1;
INDEXER_CHAINED_TX_TYPE_COMMITMENT = 2;
}

message IndexerPageRequest {
int32 size = 1;
int32 index = 2;
}

message IndexerPageResponse {
int32 current = 1;
int32 next = 2;
int32 total = 3;
}
6 changes: 3 additions & 3 deletions api-spec/protobuf/ark/v1/service.proto
100755 → 100644
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@ message Musig2 {

message RegisterOutputsForNextRoundRequest {
string request_id = 1;
// List of receivers for to convert to leaves in the next VTXO tree.
// List of receivers for to convert to leaves in the next VTXO tree.
repeated Output outputs = 2;
optional Musig2 musig2 = 3;
}
Expand All @@ -159,7 +159,7 @@ message SubmitTreeSignaturesRequest {
message SubmitTreeSignaturesResponse {}

message SubmitSignedForfeitTxsRequest {
// Forfeit txs signed by the user.
// Forfeit txs signed by the user.
repeated string signed_forfeit_txs = 1;
// The user has to sign also the round tx if he registerd a boarding UTXO.
optional string signed_round_tx = 2;
Expand All @@ -173,7 +173,7 @@ message GetEventStreamResponse {
RoundFinalizedEvent round_finalized = 2;
RoundFailed round_failed = 3;
RoundSigningEvent round_signing = 4;
RoundSigningNoncesGeneratedEvent round_signing_nonces_generated = 5;
RoundSigningNoncesGeneratedEvent round_signing_nonces_generated = 5;
}
}

Expand Down
2 changes: 1 addition & 1 deletion api-spec/protobuf/ark/v1/wallet.proto
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ service WalletInitializerService {
rpc GetStatus(GetStatusRequest) returns (GetStatusResponse) {
option (google.api.http) = {
get: "/v1/admin/wallet/status"
};
};
}
}

Expand Down
Loading
Loading