Skip to content

Commit

Permalink
Add workload_mix and realistic_env_graceful_overload, and cleanup con…
Browse files Browse the repository at this point in the history 
…tinuous forge test (#8601)
  • Loading branch information
@igor-aptos
igor-aptos authored Jun 15, 2023
1 parent 989e4ef commit 81bc224
Show file tree
Hide file tree
Showing 5 changed files with 314 additions and 234 deletions.
163 changes: 75 additions & 88 deletions .github/workflows/forge-stable.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,44 +108,84 @@ jobs:
fi
echo "IMAGE_TAG: [${IMAGE_TAG}](https://github.com/${{ github.repository }}/commit/${IMAGE_TAG})" >> $GITHUB_STEP_SUMMARY
### Performance Forge tests
run-forge-consensus-stress-test:
### Real-world-network tests.

run-forge-realistic-env-max-load-long:
if: ${{ github.event_name != 'pull_request' }}
needs: determine-test-metadata
uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
secrets: inherit
with:
IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_NAMESPACE: forge-consensus-stress-test-${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_RUNNER_DURATION_SECS: 2400
FORGE_TEST_SUITE: consensus_stress_test
POST_TO_SLACK: ${{ needs.determine-test-metadata.outputs.BRANCH == 'main' }} # only post to slack on main branch
FORGE_NAMESPACE: forge-realistic-env-max-load-long-${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_RUNNER_DURATION_SECS: 7200
FORGE_TEST_SUITE: realistic_env_max_load
POST_TO_SLACK: true

run-forge-account-creation-test:
run-forge-realistic-env-load-sweep:
if: ${{ github.event_name != 'pull_request' }}
needs: determine-test-metadata
uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
secrets: inherit
with:
IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_NAMESPACE: forge-account-creation-test-${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_RUNNER_DURATION_SECS: 900
FORGE_TEST_SUITE: account_creation
POST_TO_SLACK: ${{ needs.determine-test-metadata.outputs.BRANCH == 'main' }} # only post to slack on main branch
FORGE_NAMESPACE: forge-realistic-env-load-sweep-${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
# 5 tests, each 300s
FORGE_RUNNER_DURATION_SECS: 1500
FORGE_TEST_SUITE: realistic_env_load_sweep
POST_TO_SLACK: true

run-forge-performance-test:
run-forge-realistic-env-graceful-overload:
if: ${{ github.event_name != 'pull_request' }}
needs: determine-test-metadata
uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
secrets: inherit
with:
IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_NAMESPACE: forge-performance-${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_RUNNER_DURATION_SECS: 7200
FORGE_TEST_SUITE: land_blocking
FORGE_NAMESPACE: forge-realistic-env-graceful-overload-${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_RUNNER_DURATION_SECS: 1200
FORGE_TEST_SUITE: realistic_env_graceful_overload
POST_TO_SLACK: true

run-forge-realistic-network-tuned-for-throughput:
if: ${{ github.event_name != 'pull_request' }}
needs: determine-test-metadata
uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
secrets: inherit
with:
IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_NAMESPACE: forge-realistic-network-tuned-for-throughput-${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_RUNNER_DURATION_SECS: 900
FORGE_TEST_SUITE: realistic_network_tuned_for_throughput
FORGE_ENABLE_PERFORMANCE: true
POST_TO_SLACK: ${{ needs.determine-test-metadata.outputs.BRANCH == 'main' }} # only post to slack on main branch
POST_TO_SLACK: true

### Forge Correctness/Componenet/Stress tests

run-forge-consensus-stress-test:
if: ${{ github.event_name != 'pull_request' }}
needs: determine-test-metadata
uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
secrets: inherit
with:
IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_NAMESPACE: forge-consensus-stress-test-${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_RUNNER_DURATION_SECS: 2400
FORGE_TEST_SUITE: consensus_stress_test
POST_TO_SLACK: true

run-forge-workload-mix-test:
if: ${{ github.event_name != 'pull_request' }}
needs: determine-test-metadata
uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
secrets: inherit
with:
IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_NAMESPACE: forge-workload-mix-test-${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_RUNNER_DURATION_SECS: 900
FORGE_TEST_SUITE: workload_mix
POST_TO_SLACK: true

run-forge-single-vfn-perf:
if: ${{ github.event_name != 'pull_request' }}
Expand All @@ -158,7 +198,7 @@ jobs:
# Run for 8 minutes
FORGE_RUNNER_DURATION_SECS: 480
FORGE_TEST_SUITE: single_vfn_perf
POST_TO_SLACK: ${{ needs.determine-test-metadata.outputs.BRANCH == 'main' }} # only post to slack on main branch
POST_TO_SLACK: true

run-forge-haproxy:
if: ${{ github.event_name != 'pull_request' }}
Expand All @@ -171,7 +211,19 @@ jobs:
FORGE_RUNNER_DURATION_SECS: 600
FORGE_ENABLE_HAPROXY: true
FORGE_TEST_SUITE: land_blocking
POST_TO_SLACK: ${{ needs.determine-test-metadata.outputs.BRANCH == 'main' }} # only post to slack on main branch
POST_TO_SLACK: true

run-forge-fullnode-reboot-stress-test:
if: ${{ github.event_name != 'pull_request' }}
needs: determine-test-metadata
uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
secrets: inherit
with:
IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_NAMESPACE: forge-fullnode-reboot-stress-${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_RUNNER_DURATION_SECS: 1800
FORGE_TEST_SUITE: fullnode_reboot_stress_test
POST_TO_SLACK: true

### Compatibility Forge tests

Expand All @@ -188,33 +240,7 @@ jobs:
FORGE_TEST_SUITE: compat
IMAGE_TAG: testnet
GIT_SHA: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }} # this is the git ref to checkout
POST_TO_SLACK: ${{ needs.determine-test-metadata.outputs.BRANCH == 'main' }} # only post to slack on main branch

### Chaos Forge tests

run-forge-three-region:
if: ${{ github.event_name != 'pull_request' }}
needs: determine-test-metadata
uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
secrets: inherit
with:
IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_NAMESPACE: forge-three-region-${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_RUNNER_DURATION_SECS: 1800
FORGE_TEST_SUITE: three_region_simulation
POST_TO_SLACK: ${{ needs.determine-test-metadata.outputs.BRANCH == 'main' }} # only post to slack on main branch

run-forge-fullnode-reboot-stress-test:
if: ${{ github.event_name != 'pull_request' }}
needs: determine-test-metadata
uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
secrets: inherit
with:
IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_NAMESPACE: forge-fullnode-reboot-stress-${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_RUNNER_DURATION_SECS: 1800
FORGE_TEST_SUITE: fullnode_reboot_stress_test
POST_TO_SLACK: ${{ needs.determine-test-metadata.outputs.BRANCH == 'main' }} # only post to slack on main branch
POST_TO_SLACK: true

### Changing working quorum Forge tests

Expand All @@ -228,7 +254,7 @@ jobs:
FORGE_NAMESPACE: forge-changing-working-quorum-test-${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_RUNNER_DURATION_SECS: 1200
FORGE_TEST_SUITE: changing_working_quorum_test
POST_TO_SLACK: ${{ needs.determine-test-metadata.outputs.BRANCH == 'main' }} # only post to slack on main branch
POST_TO_SLACK: true
FORGE_ENABLE_FAILPOINTS: true

run-forge-changing-working-quorum-test-high-load:
Expand All @@ -241,7 +267,7 @@ jobs:
FORGE_NAMESPACE: forge-changing-working-quorum-test-high-load-${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_RUNNER_DURATION_SECS: 900
FORGE_TEST_SUITE: changing_working_quorum_test_high_load
POST_TO_SLACK: ${{ needs.determine-test-metadata.outputs.BRANCH == 'main' }} # only post to slack on main branch
POST_TO_SLACK: true
FORGE_ENABLE_FAILPOINTS: true

### State sync Forge tests
Expand Down Expand Up @@ -270,7 +296,7 @@ jobs:
# Run for 40 minutes
FORGE_RUNNER_DURATION_SECS: 2400
FORGE_TEST_SUITE: state_sync_perf_fullnodes_execute_transactions
POST_TO_SLACK: ${{ needs.determine-test-metadata.outputs.BRANCH == 'main' }} # only post to slack on main branch
POST_TO_SLACK: true

run-forge-state-sync-perf-fullnode-fast-sync-test:
if: ${{ github.event_name != 'pull_request' }}
Expand All @@ -283,7 +309,7 @@ jobs:
# Run for 40 minutes
FORGE_RUNNER_DURATION_SECS: 2400
FORGE_TEST_SUITE: state_sync_perf_fullnodes_fast_sync
POST_TO_SLACK: ${{ needs.determine-test-metadata.outputs.BRANCH == 'main' }} # only post to slack on main branch
POST_TO_SLACK: true

run-forge-state-sync-perf-fullnode-apply-test:
if: ${{ github.event_name != 'pull_request' }}
Expand All @@ -295,43 +321,4 @@ jobs:
FORGE_NAMESPACE: forge-state-sync-perf-fullnode-apply-${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_RUNNER_DURATION_SECS: 2400
FORGE_TEST_SUITE: state_sync_perf_fullnodes_apply_outputs
POST_TO_SLACK: ${{ needs.determine-test-metadata.outputs.BRANCH == 'main' }} # only post to slack on main branch

### Additional real-world-network tests. Eventually all consensus-related tests should migrate to real-world-network.

run-forge-realistic-env-max-throughput:
if: ${{ github.event_name != 'pull_request' }}
needs: determine-test-metadata
uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
secrets: inherit
with:
IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_NAMESPACE: forge-realistic-env-max-throughput-${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_RUNNER_DURATION_SECS: 600
FORGE_TEST_SUITE: realistic_env_max_throughput
POST_TO_SLACK: true

run-forge-realistic-env-load-sweep:
if: ${{ github.event_name != 'pull_request' }}
needs: determine-test-metadata
uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
secrets: inherit
with:
IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_NAMESPACE: forge-realistic-env-load-sweep-${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
# 5 tests, each 300s
FORGE_RUNNER_DURATION_SECS: 1500
FORGE_TEST_SUITE: realistic_env_load_sweep
POST_TO_SLACK: true

run-forge-three-region-graceful-overload:
if: ${{ github.event_name != 'pull_request' }}
needs: determine-test-metadata
uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
secrets: inherit
with:
IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_NAMESPACE: forge-three-region-graceful-overload-${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_RUNNER_DURATION_SECS: 1800
FORGE_TEST_SUITE: three_region_simulation_graceful_overload
POST_TO_SLACK: true
11 changes: 8 additions & 3 deletions crates/transaction-emitter-lib/src/emitter/account_minter.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -196,8 +196,10 @@ impl<'t> AccountMinter<'t> {
request_counters.show_simple(),
);
info!(
"Creating additional {} accounts with {} coins each",
num_accounts, coins_per_account
"Creating additional {} accounts with {} coins each (txn {} gas price)",
num_accounts,
coins_per_account,
txn_factory.get_gas_unit_price(),
);

let seed_rngs = gen_rng_for_reusable_account(actual_num_seed_accounts);
Expand Down Expand Up @@ -281,7 +283,10 @@ impl<'t> AccountMinter<'t> {
max_submit_batch_size: usize,
counters: &CounterState,
) -> Result<Vec<LocalAccount>> {
info!("Creating and funding seeds accounts");
info!(
"Creating and funding seeds accounts (txn {} gas price)",
self.txn_factory.get_gas_unit_price()
);
let mut i = 0;
let mut seed_accounts = vec![];
while i < seed_account_num {
Expand Down
47 changes: 30 additions & 17 deletions crates/transaction-emitter-lib/src/emitter/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ use again::RetryPolicy;
use anyhow::{ensure, format_err, Result};
use aptos_config::config::DEFAULT_MAX_SUBMIT_TRANSACTION_BATCH_SIZE;
use aptos_logger::{debug, error, info, sample, sample::SampleRate, warn};
use aptos_rest_client::Client as RestClient;
use aptos_rest_client::{aptos_api_types::AptosErrorCode, error::RestError, Client as RestClient};
use aptos_sdk::{
move_types::account_address::AccountAddress,
transaction_builder::{aptos_stdlib, TransactionFactory},
Expand Down Expand Up @@ -916,34 +916,47 @@ pub async fn query_sequence_numbers<'a, I>(
where
I: Iterator<Item = &'a AccountAddress>,
{
let (addresses, futures): (Vec<_>, Vec<_>) = addresses
.map(|address| {
(
*address,
RETRY_POLICY.retry(move || client.get_account_bcs(*address)),
)
})
.unzip();
let futures = addresses
.map(|address| RETRY_POLICY.retry(move || get_account_if_exists(client, *address)));

let (seq_nums, timestamps): (Vec<_>, Vec<_>) = try_join_all(futures)
.await
.map_err(|e| format_err!("Get accounts failed: {:?}", e))?
.into_iter()
.zip(addresses.iter())
.map(|(resp, address)| {
let (account, state) = resp.into_parts();
(
(*address, account.sequence_number()),
Duration::from_micros(state.timestamp_usecs).as_secs(),
)
})
.unzip();

// return min for the timestamp, to make sure
// all sequence numbers were <= to return values at that timestamp
Ok((seq_nums, timestamps.into_iter().min().unwrap()))
}

async fn get_account_if_exists(
client: &RestClient,
address: AccountAddress,
) -> Result<((AccountAddress, u64), u64)> {
let result = client.get_account_bcs(address).await;
match &result {
Ok(resp) => Ok((
(address, resp.inner().sequence_number()),
Duration::from_micros(resp.state().timestamp_usecs).as_secs(),
)),
Err(e) => {
// if account is not present, that is equivalent to sequence_number = 0
if let RestError::Api(api_error) = e {
if let AptosErrorCode::AccountNotFound = api_error.error.error_code {
return Ok((
(address, 0),
Duration::from_micros(api_error.state.as_ref().unwrap().timestamp_usecs)
.as_secs(),
));
}
}
result?;
unreachable!()
},
}
}

pub fn gen_transfer_txn_request(
sender: &mut LocalAccount,
receiver: &AccountAddress,
Expand Down
Loading

0 comments on commit 81bc224

Please sign in to comment.