Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

significant performance improvement when creating transactions, especially txo spend #3187

Merged
merged 4 commits into from
Feb 8, 2021
Merged

significant performance improvement when creating transactions, especially txo spend #3187

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
8cf88f8
significant performance improvement when creating transactions and wi…
eukreign Feb 7, 2021
5f717ef
tx._reset() the transaction after internal changes
eukreign Feb 7, 2021
967c3cd
tx._reset() the transaction after more internal changes
eukreign Feb 7, 2021
6adda3f
do not tx._reset() in generate_channel_private_key
eukreign Feb 7, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 10 additions & 2 deletions lbry/extras/daemon/daemon.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2617,6 +2617,7 @@ async def jsonrpc_channel_create(
)
txo = tx.outputs[0]
await txo.generate_channel_private_key()
tx._reset()

await tx.sign(funding_accounts)

Expand Down Expand Up @@ -2773,6 +2774,7 @@ async def jsonrpc_channel_update(
new_txo.private_key = old_txo.private_key

new_txo.script.generate()
tx._reset()

await tx.sign(funding_accounts)

Expand Down Expand Up @@ -3343,6 +3345,7 @@ async def jsonrpc_stream_create(
file_stream = await self.file_manager.create_stream(file_path)
claim.stream.source.sd_hash = file_stream.sd_hash
new_txo.script.generate()
tx._reset()

if channel:
new_txo.sign(channel)
Expand Down Expand Up @@ -3562,6 +3565,7 @@ async def jsonrpc_stream_update(
file_stream = await self.file_manager.create_stream(file_path)
new_txo.claim.stream.source.sd_hash = file_stream.sd_hash
new_txo.script.generate()
tx._reset()
stream_hash = file_stream.stream_hash
elif old_stream:
stream_hash = old_stream.stream_hash
Expand Down Expand Up @@ -3957,6 +3961,7 @@ async def jsonrpc_collection_update(
new_txo = tx.outputs[0]

new_txo.script.generate()
tx._reset()

if channel:
new_txo.sign(channel)
Expand Down Expand Up @@ -4461,7 +4466,7 @@ def jsonrpc_txo_list(

@requires(WALLET_COMPONENT)
async def jsonrpc_txo_spend(
self, account_id=None, wallet_id=None, batch_size=500,
self, account_id=None, wallet_id=None, batch_size=100,
include_full_tx=False, preview=False, blocking=False, **kwargs):
"""
Spend transaction outputs, batching into multiple transactions as necessary.
Expand Down Expand Up @@ -4500,7 +4505,10 @@ async def jsonrpc_txo_spend(
accounts = [wallet.get_account_or_error(account_id)] if account_id else wallet.accounts
txos = await self.ledger.get_txos(
wallet=wallet, accounts=accounts, read_only=True,
**self._constrain_txo_from_kwargs({}, is_not_spent=True, is_my_output=True, **kwargs)
no_tx=True, no_channel_info=True,
**self._constrain_txo_from_kwargs(
{}, is_not_spent=True, is_my_output=True, **kwargs
)
)
txs = []
while txos:
Expand Down
55 changes: 29 additions & 26 deletions lbry/wallet/database.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -965,16 +965,18 @@ async def select_txos(
sql.append("LEFT JOIN txi ON (txi.position=0 AND txi.txid=txo.txid)")
return await self.db.execute_fetchall(*query(' '.join(sql), **constraints), read_only=read_only)

async def get_txos(self, wallet=None, no_tx=False, read_only=False, **constraints):
async def get_txos(self, wallet=None, no_tx=False, no_channel_info=False, read_only=False, **constraints):
include_is_spent = constraints.get('include_is_spent', False)
include_is_my_input = constraints.get('include_is_my_input', False)
include_is_my_output = constraints.pop('include_is_my_output', False)
include_received_tips = constraints.pop('include_received_tips', False)

select_columns = [
"tx.txid, raw, tx.height, tx.position as tx_position, tx.is_verified, "
"tx.txid, tx.height, tx.position as tx_position, tx.is_verified, "
"txo_type, txo.position as txo_position, amount, script"
]
if not no_tx:
select_columns.append("raw")

my_accounts = {a.public_key.address for a in wallet.accounts} if wallet else set()
my_accounts_sql = ""
Expand Down Expand Up @@ -1052,32 +1054,33 @@ async def get_txos(self, wallet=None, no_tx=False, read_only=False, **constraint
txo.received_tips = row['received_tips']
txos.append(txo)

channel_ids = set()
for txo in txos:
if txo.is_claim and txo.can_decode_claim:
if txo.claim.is_signed:
channel_ids.add(txo.claim.signing_channel_id)
if txo.claim.is_channel and wallet:
for account in wallet.accounts:
private_key = await account.get_channel_private_key(
txo.claim.channel.public_key_bytes
)
if private_key:
txo.private_key = private_key
break

if channel_ids:
channels = {
txo.claim_id: txo for txo in
(await self.get_channels(
wallet=wallet,
claim_id__in=channel_ids,
read_only=read_only
))
}
if not no_channel_info:
channel_ids = set()
for txo in txos:
if txo.is_claim and txo.can_decode_claim:
txo.channel = channels.get(txo.claim.signing_channel_id, None)
if txo.claim.is_signed:
channel_ids.add(txo.claim.signing_channel_id)
if txo.claim.is_channel and wallet:
for account in wallet.accounts:
private_key = await account.get_channel_private_key(
txo.claim.channel.public_key_bytes
)
if private_key:
txo.private_key = private_key
break

if channel_ids:
channels = {
txo.claim_id: txo for txo in
(await self.get_channels(
wallet=wallet,
claim_id__in=channel_ids,
read_only=read_only
))
}
for txo in txos:
if txo.is_claim and txo.can_decode_claim:
txo.channel = channels.get(txo.claim.signing_channel_id, None)

return txos

Expand Down
2 changes: 1 addition & 1 deletion lbry/wallet/ledger.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,7 @@ async def get_account_for_address(self, wallet, address):
async def get_effective_amount_estimators(self, funding_accounts: Iterable[Account]):
estimators = []
for account in funding_accounts:
utxos = await account.get_utxos()
utxos = await account.get_utxos(no_tx=True, no_channel_info=True)
for utxo in utxos:
estimators.append(utxo.get_estimator(self))
return estimators
Expand Down
18 changes: 12 additions & 6 deletions lbry/wallet/transaction.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -539,6 +539,7 @@ def __init__(self, raw=None, version: int = 1, locktime: int = 0, is_verified: b
height: int = -2, position: int = -1, julian_day: int = None) -> None:
self._raw = raw
self._raw_sans_segwit = None
self._raw_outputs = None
self.is_segwit_flag = 0
self.witnesses: List[bytes] = []
self.ref = TXRefMutable(self)
Expand Down Expand Up @@ -600,6 +601,7 @@ def raw_sans_segwit(self):
def _reset(self):
self._raw = None
self._raw_sans_segwit = None
self._raw_outputs = None
self.ref.reset()

@property
Expand Down Expand Up @@ -693,9 +695,7 @@ def _serialize(self, with_inputs: bool = True, sans_segwit: bool = False) -> byt
stream.write_compact_size(len(self._inputs))
for txin in self._inputs:
txin.serialize_to(stream)
stream.write_compact_size(len(self._outputs))
for txout in self._outputs:
txout.serialize_to(stream)
self._serialize_outputs(stream)
stream.write_uint32(self.locktime)
return stream.get_bytes()

Expand All @@ -709,13 +709,19 @@ def _serialize_for_signature(self, signing_input: int) -> bytes:
txin.serialize_to(stream, txin.txo_ref.txo.script.source)
else:
txin.serialize_to(stream, b'')
stream.write_compact_size(len(self._outputs))
for txout in self._outputs:
txout.serialize_to(stream)
self._serialize_outputs(stream)
stream.write_uint32(self.locktime)
stream.write_uint32(self.signature_hash_type(1)) # signature hash type: SIGHASH_ALL
return stream.get_bytes()

def _serialize_outputs(self, stream):
if self._raw_outputs is None:
self._raw_outputs = BCDataStream()
self._raw_outputs.write_compact_size(len(self._outputs))
for txout in self._outputs:
txout.serialize_to(self._raw_outputs)
stream.write(self._raw_outputs.get_bytes())

def _deserialize(self):
if self._raw is not None:
stream = BCDataStream(self._raw)
Expand Down