forked from MystenLabs/sui
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[sui-tool] dump-packages uses GraphQL (MystenLabs#18337)
## Description Replace the original implementation of the dump-packages command (which requires access to an indexer database) with an implementation that reads from a GraphQL service. The former is not readily accessible, but the latter should be. The new tool is also able to run incrementally: Fetching only packages created before a certain checkpoint, or pick up where a previous invocation took off to fetch new packages that were introduced since. ## Test plan Ran a test invocation, on our experimental read replica. With a max page size of 200, I was able to fetch 17000 packages (all the packages at the time the read replica was created) in 3 minutes. ## Stack - MystenLabs#17543 - MystenLabs#17692 - MystenLabs#17693 - MystenLabs#17696 - MystenLabs#18287 - MystenLabs#18288 - MystenLabs#18336 --- ## Release notes Check each box that your changes affect. If none of the boxes relate to your changes, release notes aren't required. For each box you select, include information after the relevant heading that describes the impact of your changes that a user might notice and any actions they must take to implement updates. - [ ] Protocol: - [ ] Nodes (Validators and Full nodes): - [ ] Indexer: - [ ] JSON-RPC: - [ ] GraphQL: - [ ] CLI: - [ ] Rust SDK:
- Loading branch information
Showing
11 changed files
with
618 additions
and
147 deletions.
There are no files selected for viewing
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,22 @@ | ||
[package] | ||
name = "sui-package-dump" | ||
version.workspace = true | ||
authors = ["Mysten Labs <build@mystenlabs.com"] | ||
license = "Apache-2.0" | ||
publish = false | ||
edition = "2021" | ||
|
||
[dependencies] | ||
anyhow.workspace = true | ||
bcs.workspace = true | ||
cynic.workspace = true | ||
fastcrypto.workspace = true | ||
move-core-types.workspace = true | ||
reqwest.workspace = true | ||
serde.workspace = true | ||
serde_json.workspace = true | ||
sui-types.workspace = true | ||
tracing.workspace = true | ||
|
||
[build-dependencies] | ||
cynic-codegen.workspace = true |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,10 @@ | ||
// Copyright (c) Mysten Labs, Inc. | ||
// SPDX-License-Identifier: Apache-2.0 | ||
|
||
fn main() { | ||
cynic_codegen::register_schema("sui") | ||
.from_sdl_file("../sui-graphql-rpc/schema.graphql") | ||
.unwrap() | ||
.as_default() | ||
.unwrap(); | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,39 @@ | ||
// Copyright (c) Mysten Labs, Inc. | ||
// SPDX-License-Identifier: Apache-2.0 | ||
|
||
use anyhow::{anyhow, Context, Result}; | ||
use cynic::{http::ReqwestExt, Operation, QueryBuilder}; | ||
use reqwest::IntoUrl; | ||
use serde::{de::DeserializeOwned, Serialize}; | ||
|
||
pub(crate) struct Client { | ||
inner: reqwest::Client, | ||
url: reqwest::Url, | ||
} | ||
|
||
impl Client { | ||
/// Create a new GraphQL client, talking to a Sui GraphQL service at `url`. | ||
pub(crate) fn new(url: impl IntoUrl) -> Result<Self> { | ||
Ok(Self { | ||
inner: reqwest::Client::builder() | ||
.user_agent(concat!("sui-package-dump/", env!("CARGO_PKG_VERSION"))) | ||
.build() | ||
.context("Failed to create GraphQL client")?, | ||
url: url.into_url().context("Invalid RPC URL")?, | ||
}) | ||
} | ||
|
||
pub(crate) async fn query<Q, V>(&self, query: Operation<Q, V>) -> Result<Q> | ||
where | ||
V: Serialize, | ||
Q: DeserializeOwned + QueryBuilder<V> + 'static, | ||
{ | ||
self.inner | ||
.post(self.url.clone()) | ||
.run_graphql(query) | ||
.await | ||
.context("Failed to send GraphQL query")? | ||
.data | ||
.ok_or_else(|| anyhow!("Empty response to query")) | ||
} | ||
} |
Oops, something went wrong.