diff --git a/aggregates/account/build.gradle b/aggregates/account/build.gradle new file mode 100644 index 00000000..87bd018b --- /dev/null +++ b/aggregates/account/build.gradle @@ -0,0 +1,22 @@ +dependencies { + implementation project(":components:common") + implementation 'org.springframework.boot:spring-boot-starter-data-jpa' + implementation 'org.springframework.boot:spring-boot-starter-webflux' + implementation(libs.cardano.client.lib) + api(libs.yaci.store.starter) + implementation(libs.yaci.store.utxo.starter) + implementation(libs.yaci.store.account.starter) + + testImplementation 'org.springframework.boot:spring-boot-starter-test' +} + +publishing { + publications { + mavenJava(MavenPublication) { + pom { + name = 'Ledger Sync Account' + description = 'Ledger Sync Account Module' + } + } + } +} diff --git a/aggregates/account/src/main/java/org/cardanofoundation/ledgersync/account/DummyInvalidTransactionStorage.java b/aggregates/account/src/main/java/org/cardanofoundation/ledgersync/account/DummyInvalidTransactionStorage.java new file mode 100644 index 00000000..0694b0ad --- /dev/null +++ b/aggregates/account/src/main/java/org/cardanofoundation/ledgersync/account/DummyInvalidTransactionStorage.java @@ -0,0 +1,21 @@ +package org.cardanofoundation.ledgersync.account; + +import com.bloxbean.cardano.yaci.store.utxo.domain.InvalidTransaction; +import com.bloxbean.cardano.yaci.store.utxo.storage.impl.InvalidTransactionStorageImpl; +import com.bloxbean.cardano.yaci.store.utxo.storage.impl.repository.InvalidTransactionRepository; +import org.springframework.stereotype.Component; + +//TODO -- remove this class later +//Overrides the save method to do nothing to fix the NUL char issue in invalid transaction +//https://github.com/bloxbean/yaci-store/issues/209 +@Component +public class DummyInvalidTransactionStorage extends InvalidTransactionStorageImpl { + public DummyInvalidTransactionStorage(InvalidTransactionRepository repository) { + super(repository); + } + + @Override + public InvalidTransaction save(InvalidTransaction invalidTransaction) { + return invalidTransaction; //do nothing + } +} diff --git a/aggregates/account/src/main/java/org/cardanofoundation/ledgersync/account/domain/AddressTxAmount.java b/aggregates/account/src/main/java/org/cardanofoundation/ledgersync/account/domain/AddressTxAmount.java new file mode 100644 index 00000000..00d89a0c --- /dev/null +++ b/aggregates/account/src/main/java/org/cardanofoundation/ledgersync/account/domain/AddressTxAmount.java @@ -0,0 +1,29 @@ +package org.cardanofoundation.ledgersync.account.domain; + +import com.bloxbean.cardano.yaci.store.common.domain.BlockAwareDomain; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.databind.PropertyNamingStrategies; +import com.fasterxml.jackson.databind.annotation.JsonNaming; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.math.BigInteger; + +@Data +@NoArgsConstructor +@AllArgsConstructor +@SuperBuilder(toBuilder = true) +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class) +public class AddressTxAmount extends BlockAwareDomain { + private String address; + private String unit; + private String txHash; + private Long slot; + private BigInteger quantity; + private String stakeAddress; + private Integer epoch; +} + diff --git a/aggregates/account/src/main/java/org/cardanofoundation/ledgersync/account/processor/AddressTxAmountProcessor.java b/aggregates/account/src/main/java/org/cardanofoundation/ledgersync/account/processor/AddressTxAmountProcessor.java new file mode 100644 index 00000000..2d7420ca --- /dev/null +++ b/aggregates/account/src/main/java/org/cardanofoundation/ledgersync/account/processor/AddressTxAmountProcessor.java @@ -0,0 +1,207 @@ +package org.cardanofoundation.ledgersync.account.processor; + +import com.bloxbean.cardano.yaci.store.client.utxo.UtxoClient; +import com.bloxbean.cardano.yaci.store.common.domain.AddressUtxo; +import com.bloxbean.cardano.yaci.store.common.domain.Amt; +import com.bloxbean.cardano.yaci.store.common.domain.UtxoKey; +import com.bloxbean.cardano.yaci.store.events.EventMetadata; +import com.bloxbean.cardano.yaci.store.events.RollbackEvent; +import com.bloxbean.cardano.yaci.store.events.internal.ReadyForBalanceAggregationEvent; +import com.bloxbean.cardano.yaci.store.utxo.domain.AddressUtxoEvent; +import com.bloxbean.cardano.yaci.store.utxo.domain.TxInputOutput; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.cardanofoundation.ledgersync.account.domain.AddressTxAmount; +import org.cardanofoundation.ledgersync.account.storage.AddressTxAmountStorage; +import org.springframework.context.event.EventListener; +import org.springframework.data.util.Pair; +import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; + +import java.math.BigInteger; +import java.util.*; + +import static org.cardanofoundation.ledgersync.account.util.AddressUtil.getAddress; + +@Component +@RequiredArgsConstructor +@Slf4j +public class AddressTxAmountProcessor { + public static final int BLOCK_ADDRESS_TX_AMT_THRESHOLD = 100; //Threshold to save address_tx_amounts records for block + + private final AddressTxAmountStorage addressTxAmountStorage; + private final UtxoClient utxoClient; + + private List> pendingTxInputOutputListCache = Collections.synchronizedList(new ArrayList<>()); + private List addressTxAmountListCache = Collections.synchronizedList(new ArrayList<>()); + + @EventListener + @Transactional + public void processAddressUtxoEvent(AddressUtxoEvent addressUtxoEvent) { + //Ignore Genesis Txs as it's handled by GEnesisBlockAddressTxAmtProcessor + if (addressUtxoEvent.getEventMetadata().getSlot() == -1) + return; + + var txInputOutputList = addressUtxoEvent.getTxInputOutputs(); + if (txInputOutputList == null || txInputOutputList.isEmpty()) + return; + + List addressTxAmountList = new ArrayList<>(); + + for (var txInputOutput : txInputOutputList) { + var txAddressTxAmountEntities = processAddressAmountForTx(addressUtxoEvent.getEventMetadata(), txInputOutput, false); + if (txAddressTxAmountEntities == null || txAddressTxAmountEntities.isEmpty()) continue; + + addressTxAmountList.addAll(txAddressTxAmountEntities); + } + + if (addressTxAmountList.size() > BLOCK_ADDRESS_TX_AMT_THRESHOLD) { + if (log.isDebugEnabled()) + log.debug("Saving address_tx_amounts records : {} -- {}", addressTxAmountList.size(), addressUtxoEvent.getEventMetadata().getBlock()); + addressTxAmountStorage.save(addressTxAmountList); //Save + } else if (addressTxAmountList.size() > 0) { + addressTxAmountListCache.addAll(addressTxAmountList); + } + } + + private List processAddressAmountForTx(EventMetadata metadata, TxInputOutput txInputOutput, + boolean throwExceptionOnFailure) { + var txHash = txInputOutput.getTxHash(); + var inputs = txInputOutput.getInputs(); + var outputs = txInputOutput.getOutputs(); + if (inputs == null || inputs.isEmpty() || outputs == null || outputs.isEmpty()) + return null; + + var inputUtxoKeys = inputs.stream() + .map(input -> new UtxoKey(input.getTxHash(), input.getOutputIndex())) + .toList(); + + var inputAddressUtxos = utxoClient.getUtxosByIds(inputUtxoKeys) + .stream() + .filter(Objects::nonNull) + .toList(); + + if (inputAddressUtxos.size() != inputUtxoKeys.size()) { + log.debug("Unable to get inputs for all input keys for account balance calculation. Add this Tx to cache to process later : " + txHash); + if (throwExceptionOnFailure) + throw new IllegalStateException("Unable to get inputs for all input keys for account balance calculation : " + inputUtxoKeys); + else + pendingTxInputOutputListCache.add(Pair.of(metadata, txInputOutput)); + + return Collections.emptyList(); + } + + var txAddressTxAmount = + processTxAmount(txHash, metadata, inputAddressUtxos, outputs); + return txAddressTxAmount; + } + + private List processTxAmount(String txHash, EventMetadata metadata, List inputs, List outputs) { + Map, BigInteger> addressTxAmountMap = new HashMap<>(); + Map addressToAddressDetailsMap = new HashMap<>(); + Map unitToAssetDetailsMap = new HashMap<>(); + + //Subtract input amounts + for (var input : inputs) { + for (Amt amt : input.getAmounts()) { + var key = Pair.of(input.getOwnerAddr(), amt.getUnit()); + var amount = addressTxAmountMap.getOrDefault(key, BigInteger.ZERO); + amount = amount.subtract(amt.getQuantity()); + addressTxAmountMap.put(key, amount); + + var addressDetails = new AddressDetails(input.getOwnerStakeAddr(), input.getOwnerPaymentCredential(), input.getOwnerStakeCredential()); + addressToAddressDetailsMap.put(input.getOwnerAddr(), addressDetails); + + var assetDetails = new AssetDetails(amt.getPolicyId(), amt.getAssetName()); + unitToAssetDetailsMap.put(amt.getUnit(), assetDetails); + } + } + + //Add output amounts + for (var output : outputs) { + for (Amt amt : output.getAmounts()) { + var key = Pair.of(output.getOwnerAddr(), amt.getUnit()); + var amount = addressTxAmountMap.getOrDefault(key, BigInteger.ZERO); + amount = amount.add(amt.getQuantity()); + addressTxAmountMap.put(key, amount); + + var addressDetails = new AddressDetails(output.getOwnerStakeAddr(), output.getOwnerPaymentCredential(), output.getOwnerStakeCredential()); + addressToAddressDetailsMap.put(output.getOwnerAddr(), addressDetails); + + var assetDetails = new AssetDetails(amt.getPolicyId(), amt.getAssetName()); + unitToAssetDetailsMap.put(amt.getUnit(), assetDetails); + } + } + + return (List) addressTxAmountMap.entrySet() + .stream() + .filter(entry -> entry.getValue().compareTo(BigInteger.ZERO) != 0) + .map(entry -> { + var addressDetails = addressToAddressDetailsMap.get(entry.getKey().getFirst()); + var assetDetails = unitToAssetDetailsMap.get(entry.getKey().getSecond()); + + //address and full address if the address is too long + var addressTuple = getAddress(entry.getKey().getFirst()); + + return AddressTxAmount.builder() + .address(addressTuple._1) + .unit(entry.getKey().getSecond()) + .txHash(txHash) + .slot(metadata.getSlot()) + .quantity(entry.getValue()) + .stakeAddress(addressDetails.ownerStakeAddress) + .epoch(metadata.getEpochNumber()) + .blockNumber(metadata.getBlock()) + .blockTime(metadata.getBlockTime()) + .build(); + }).toList(); + } + + @EventListener + @Transactional //We can also listen to CommitEvent here + public void handleRemainingTxInputOuputs(ReadyForBalanceAggregationEvent readyForBalanceAggregationEvent) { + try { + List addressTxAmountList = new ArrayList<>(); + for (var pair : pendingTxInputOutputListCache) { + EventMetadata metadata = pair.getFirst(); + TxInputOutput txInputOutput = pair.getSecond(); + + var addrAmountEntitiesForTx = processAddressAmountForTx(metadata, txInputOutput, true); + + if (addrAmountEntitiesForTx != null) { + addressTxAmountList.addAll(addrAmountEntitiesForTx); + } + } + + if (addressTxAmountList.size() > 0) { + addressTxAmountListCache.addAll(addressTxAmountList); + } + + long t1 = System.currentTimeMillis(); + if (addressTxAmountListCache.size() > 0) { + addressTxAmountStorage.save(addressTxAmountListCache); + } + + long t2 = System.currentTimeMillis(); + log.info("Time taken to save additional address_tx_amounts records : {}, time: {} ms", addressTxAmountListCache.size(), (t2 - t1)); + + } finally { + pendingTxInputOutputListCache.clear(); + addressTxAmountListCache.clear(); + } + } + + @EventListener + @Transactional + public void handleRollback(RollbackEvent rollbackEvent) { + int addressTxAmountDeleted = addressTxAmountStorage.deleteAddressBalanceBySlotGreaterThan(rollbackEvent.getRollbackTo().getSlot()); + + log.info("Rollback -- {} address_tx_amounts records", addressTxAmountDeleted); + } + + record AssetDetails(String policy, String assetName) { + } + + record AddressDetails(String ownerStakeAddress, String ownerPaymentCredential, String ownerStakeCredential) { + } +} diff --git a/aggregates/account/src/main/java/org/cardanofoundation/ledgersync/account/processor/GensisBlockAddressTxAmtProcessor.java b/aggregates/account/src/main/java/org/cardanofoundation/ledgersync/account/processor/GensisBlockAddressTxAmtProcessor.java new file mode 100644 index 00000000..e9cf050d --- /dev/null +++ b/aggregates/account/src/main/java/org/cardanofoundation/ledgersync/account/processor/GensisBlockAddressTxAmtProcessor.java @@ -0,0 +1,85 @@ +package org.cardanofoundation.ledgersync.account.processor; + +import com.bloxbean.cardano.client.address.Address; +import com.bloxbean.cardano.client.address.AddressProvider; +import com.bloxbean.cardano.yaci.core.util.HexUtil; +import com.bloxbean.cardano.yaci.store.events.GenesisBlockEvent; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.cardanofoundation.ledgersync.account.domain.AddressTxAmount; +import org.cardanofoundation.ledgersync.account.storage.AddressTxAmountStorage; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; + +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.List; + +import static com.bloxbean.cardano.yaci.core.util.Constants.LOVELACE; +import static org.cardanofoundation.ledgersync.account.util.AddressUtil.getAddress; + +@Component +@RequiredArgsConstructor +@Slf4j +public class GensisBlockAddressTxAmtProcessor { + private final AddressTxAmountStorage addressTxAmountStorage; + + @EventListener + @Transactional + public void handleAddressTxAmtForGenesisBlock(GenesisBlockEvent genesisBlockEvent) { + var genesisBalances = genesisBlockEvent.getGenesisBalances(); + if (genesisBalances == null || genesisBalances.isEmpty()) { + log.info("No genesis balances found"); + return; + } + + List genesisAddressTxAmtList = new ArrayList<>(); + for (var genesisBalance : genesisBalances) { + var receiverAddress = genesisBalance.getAddress(); + var txnHash = genesisBalance.getTxnHash(); + var balance = genesisBalance.getBalance(); + + if (balance == null || balance.compareTo(BigInteger.ZERO) == 0) { + continue; + } + + //address and full address if the address is too long + var addressTuple = getAddress(receiverAddress); + + String ownerPaymentCredential = null; + String stakeAddress = null; + if (genesisBalance.getAddress() != null && + genesisBalance.getAddress().startsWith("addr")) { //If shelley address + try { + Address address = new Address(genesisBalance.getAddress()); + ownerPaymentCredential = address.getPaymentCredential().map(paymentKeyHash -> HexUtil.encodeHexString(paymentKeyHash.getBytes())) + .orElse(null); + stakeAddress = address.getDelegationCredential().map(delegationHash -> AddressProvider.getStakeAddress(address).toBech32()) + .orElse(null); + } catch (Exception e) { + log.warn("Error getting address details: " + genesisBalance.getAddress()); + } + } + + var addressTxAmount = AddressTxAmount.builder() + .address(addressTuple._1) + .unit(LOVELACE) + .txHash(txnHash) + .slot(genesisBlockEvent.getSlot()) + .quantity(balance) + .stakeAddress(stakeAddress) + .epoch(0) + .blockNumber(genesisBlockEvent.getBlock()) + .blockTime(genesisBlockEvent.getBlockTime()) + .build(); + + genesisAddressTxAmtList.add(addressTxAmount); + } + + if (genesisAddressTxAmtList.size() > 0) { + addressTxAmountStorage.save(genesisAddressTxAmtList); + } + } + +} diff --git a/aggregates/account/src/main/java/org/cardanofoundation/ledgersync/account/storage/AddressTxAmountStorage.java b/aggregates/account/src/main/java/org/cardanofoundation/ledgersync/account/storage/AddressTxAmountStorage.java new file mode 100644 index 00000000..a18ae2f8 --- /dev/null +++ b/aggregates/account/src/main/java/org/cardanofoundation/ledgersync/account/storage/AddressTxAmountStorage.java @@ -0,0 +1,10 @@ +package org.cardanofoundation.ledgersync.account.storage; + +import org.cardanofoundation.ledgersync.account.domain.AddressTxAmount; + +import java.util.List; + +public interface AddressTxAmountStorage { + void save(List addressTxAmount); + int deleteAddressBalanceBySlotGreaterThan(Long slot); +} diff --git a/aggregates/account/src/main/java/org/cardanofoundation/ledgersync/account/storage/impl/AddressTxAmountStorageImpl.java b/aggregates/account/src/main/java/org/cardanofoundation/ledgersync/account/storage/impl/AddressTxAmountStorageImpl.java new file mode 100644 index 00000000..61fa6b2c --- /dev/null +++ b/aggregates/account/src/main/java/org/cardanofoundation/ledgersync/account/storage/impl/AddressTxAmountStorageImpl.java @@ -0,0 +1,106 @@ +package org.cardanofoundation.ledgersync.account.storage.impl; + +import com.bloxbean.cardano.yaci.store.account.AccountStoreProperties; +import com.bloxbean.cardano.yaci.store.common.util.ListUtil; +import jakarta.annotation.PostConstruct; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.cardanofoundation.ledgersync.account.domain.AddressTxAmount; +import org.cardanofoundation.ledgersync.account.storage.AddressTxAmountStorage; +import org.cardanofoundation.ledgersync.account.storage.impl.mapper.AggrMapper; +import org.cardanofoundation.ledgersync.account.storage.impl.model.AddressTxAmountEntity; +import org.cardanofoundation.ledgersync.account.storage.impl.repository.AddressTxAmountRepository; +import org.jooq.DSLContext; +import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; + +import java.util.List; + +import static org.cardanofoundation.ledgersync.account.jooq.Tables.ADDRESS_TX_AMOUNT; + +@Component +@RequiredArgsConstructor +@Slf4j +public class AddressTxAmountStorageImpl implements AddressTxAmountStorage { + private final AddressTxAmountRepository addressTxAmountRepository; + private final DSLContext dsl; + private final AccountStoreProperties accountStoreProperties; + + private final AggrMapper aggrMapper = AggrMapper.INSTANCE; + + @PostConstruct + public void postConstruct() { + this.dsl.settings().setBatchSize(accountStoreProperties.getJooqWriteBatchSize()); + } + + @Override + @Transactional + public void save(List addressTxAmount) { + var addressTxAmtEntities = addressTxAmount.stream() + .map(addressTxAmount1 -> aggrMapper.toAddressTxAmountEntity(addressTxAmount1)) + .toList(); + + if (accountStoreProperties.isParallelWrite() + && addressTxAmtEntities.size() > accountStoreProperties.getPerThreadBatchSize()) { + ListUtil.partitionAndApplyInParallel(addressTxAmtEntities, accountStoreProperties.getPerThreadBatchSize(), this::saveBatch); + } else { + saveBatch(addressTxAmtEntities); + } + } + + private void saveBatch(List addressTxAmountEntities) { + var inserts = addressTxAmountEntities.stream() + .map(addressTxAmount -> dsl.insertInto(ADDRESS_TX_AMOUNT) + .set(ADDRESS_TX_AMOUNT.ADDRESS, addressTxAmount.getAddress()) + .set(ADDRESS_TX_AMOUNT.UNIT, addressTxAmount.getUnit()) + .set(ADDRESS_TX_AMOUNT.TX_HASH, addressTxAmount.getTxHash()) + .set(ADDRESS_TX_AMOUNT.SLOT, addressTxAmount.getSlot()) + .set(ADDRESS_TX_AMOUNT.QUANTITY, addressTxAmount.getQuantity()) + .set(ADDRESS_TX_AMOUNT.ADDR_FULL, addressTxAmount.getAddrFull()) + .set(ADDRESS_TX_AMOUNT.STAKE_ADDRESS, addressTxAmount.getStakeAddress()) + .set(ADDRESS_TX_AMOUNT.BLOCK, addressTxAmount.getBlockNumber()) + .set(ADDRESS_TX_AMOUNT.BLOCK_TIME, addressTxAmount.getBlockTime()) + .set(ADDRESS_TX_AMOUNT.EPOCH, addressTxAmount.getEpoch()) + .onDuplicateKeyUpdate() + .set(ADDRESS_TX_AMOUNT.SLOT, addressTxAmount.getSlot()) + .set(ADDRESS_TX_AMOUNT.QUANTITY, addressTxAmount.getQuantity()) + .set(ADDRESS_TX_AMOUNT.ADDR_FULL, addressTxAmount.getAddrFull()) + .set(ADDRESS_TX_AMOUNT.STAKE_ADDRESS, addressTxAmount.getStakeAddress()) + .set(ADDRESS_TX_AMOUNT.BLOCK, addressTxAmount.getBlockNumber()) + .set(ADDRESS_TX_AMOUNT.BLOCK_TIME, addressTxAmount.getBlockTime()) + .set(ADDRESS_TX_AMOUNT.EPOCH, addressTxAmount.getEpoch())).toList(); + dsl.batch(inserts).execute(); + + /** + dsl.batched(c -> { + for (var addressTxAmount : addressTxAmountEntities) { + c.dsl().insertInto(ADDRESS_TX_AMOUNT) + .set(ADDRESS_TX_AMOUNT.ADDRESS, addressTxAmount.getAddress()) + .set(ADDRESS_TX_AMOUNT.UNIT, addressTxAmount.getUnit()) + .set(ADDRESS_TX_AMOUNT.TX_HASH, addressTxAmount.getTxHash()) + .set(ADDRESS_TX_AMOUNT.SLOT, addressTxAmount.getSlot()) + .set(ADDRESS_TX_AMOUNT.QUANTITY, addressTxAmount.getQuantity()) + .set(ADDRESS_TX_AMOUNT.ADDR_FULL, addressTxAmount.getAddrFull()) + .set(ADDRESS_TX_AMOUNT.STAKE_ADDRESS, addressTxAmount.getStakeAddress()) + .set(ADDRESS_TX_AMOUNT.BLOCK, addressTxAmount.getBlockNumber()) + .set(ADDRESS_TX_AMOUNT.BLOCK_TIME, addressTxAmount.getBlockTime()) + .set(ADDRESS_TX_AMOUNT.EPOCH, addressTxAmount.getEpoch()) + .onDuplicateKeyUpdate() + .set(ADDRESS_TX_AMOUNT.SLOT, addressTxAmount.getSlot()) + .set(ADDRESS_TX_AMOUNT.QUANTITY, addressTxAmount.getQuantity()) + .set(ADDRESS_TX_AMOUNT.ADDR_FULL, addressTxAmount.getAddrFull()) + .set(ADDRESS_TX_AMOUNT.STAKE_ADDRESS, addressTxAmount.getStakeAddress()) + .set(ADDRESS_TX_AMOUNT.BLOCK, addressTxAmount.getBlockNumber()) + .set(ADDRESS_TX_AMOUNT.BLOCK_TIME, addressTxAmount.getBlockTime()) + .set(ADDRESS_TX_AMOUNT.EPOCH, addressTxAmount.getEpoch()) + .execute(); + } + }); + **/ + } + + @Override + public int deleteAddressBalanceBySlotGreaterThan(Long slot) { + return addressTxAmountRepository.deleteAddressBalanceBySlotGreaterThan(slot); + } +} diff --git a/aggregates/account/src/main/java/org/cardanofoundation/ledgersync/account/storage/impl/mapper/AggrMapper.java b/aggregates/account/src/main/java/org/cardanofoundation/ledgersync/account/storage/impl/mapper/AggrMapper.java new file mode 100644 index 00000000..cc9ec167 --- /dev/null +++ b/aggregates/account/src/main/java/org/cardanofoundation/ledgersync/account/storage/impl/mapper/AggrMapper.java @@ -0,0 +1,17 @@ +package org.cardanofoundation.ledgersync.account.storage.impl.mapper; + +import org.cardanofoundation.ledgersync.account.domain.AddressTxAmount; +import org.cardanofoundation.ledgersync.account.storage.impl.model.AddressTxAmountEntity; +import org.mapstruct.DecoratedWith; +import org.mapstruct.Mapper; +import org.mapstruct.factory.Mappers; + +@Mapper(componentModel = "default") +@DecoratedWith(AggrMapperDecorator.class) +public interface AggrMapper { + AggrMapper INSTANCE = Mappers.getMapper(AggrMapper.class); + + AddressTxAmount toAddressTxAmount(AddressTxAmountEntity entity); + AddressTxAmountEntity toAddressTxAmountEntity(AddressTxAmount addressTxAmount); +} + diff --git a/aggregates/account/src/main/java/org/cardanofoundation/ledgersync/account/storage/impl/mapper/AggrMapperDecorator.java b/aggregates/account/src/main/java/org/cardanofoundation/ledgersync/account/storage/impl/mapper/AggrMapperDecorator.java new file mode 100644 index 00000000..673f019c --- /dev/null +++ b/aggregates/account/src/main/java/org/cardanofoundation/ledgersync/account/storage/impl/mapper/AggrMapperDecorator.java @@ -0,0 +1,36 @@ +package org.cardanofoundation.ledgersync.account.storage.impl.mapper; + +import org.cardanofoundation.ledgersync.account.domain.AddressTxAmount; +import org.cardanofoundation.ledgersync.account.storage.impl.model.AddressTxAmountEntity; + +public class AggrMapperDecorator implements AggrMapper { + public static final int MAX_ADDR_SIZE = 500; + private final AggrMapper delegate; + + public AggrMapperDecorator(AggrMapper delegate) { + this.delegate = delegate; + } + + @Override + public AddressTxAmount toAddressTxAmount(AddressTxAmountEntity entity) { + AddressTxAmount addrTxAmount = delegate.toAddressTxAmount(entity); + + if (entity.getAddrFull() != null && entity.getAddrFull().length() > 0) + addrTxAmount.setAddress(entity.getAddrFull()); + + return addrTxAmount; + } + + @Override + public AddressTxAmountEntity toAddressTxAmountEntity(AddressTxAmount addressTxAmount) { + AddressTxAmountEntity entity = delegate.toAddressTxAmountEntity(addressTxAmount); + + if (addressTxAmount.getAddress() != null && addressTxAmount.getAddress().length() > MAX_ADDR_SIZE) { + entity.setAddress(addressTxAmount.getAddress().substring(0, MAX_ADDR_SIZE)); + entity.setAddrFull(addressTxAmount.getAddress()); + } + + return entity; + } + +} diff --git a/aggregates/account/src/main/java/org/cardanofoundation/ledgersync/account/storage/impl/model/AddressTxAmountEntity.java b/aggregates/account/src/main/java/org/cardanofoundation/ledgersync/account/storage/impl/model/AddressTxAmountEntity.java new file mode 100644 index 00000000..bff2d895 --- /dev/null +++ b/aggregates/account/src/main/java/org/cardanofoundation/ledgersync/account/storage/impl/model/AddressTxAmountEntity.java @@ -0,0 +1,54 @@ +package org.cardanofoundation.ledgersync.account.storage.impl.model; + +import jakarta.persistence.*; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.hibernate.annotations.DynamicUpdate; + +import java.math.BigInteger; + +@Data +@NoArgsConstructor +@AllArgsConstructor +@Builder +@Entity +@Table(name = "address_tx_amount") +@IdClass(AddressTxAmountId.class) +@DynamicUpdate +public class AddressTxAmountEntity { + @Id + @Column(name = "address") + private String address; + + @Id + @Column(name = "unit") + private String unit; + + @Id + @Column(name = "tx_hash") + private String txHash; + + @Column(name = "slot") + private Long slot; + + @Column(name = "quantity") + private BigInteger quantity; + + //Only set if address doesn't fit in ownerAddr field. Required for few Byron Era addr + @Column(name = "addr_full") + private String addrFull; + + @Column(name = "stake_address") + private String stakeAddress; + + @Column(name = "epoch") + private Integer epoch; + + @Column(name = "block") + private Long blockNumber; + + @Column(name = "block_time") + private Long blockTime; +} diff --git a/aggregates/account/src/main/java/org/cardanofoundation/ledgersync/account/storage/impl/model/AddressTxAmountId.java b/aggregates/account/src/main/java/org/cardanofoundation/ledgersync/account/storage/impl/model/AddressTxAmountId.java new file mode 100644 index 00000000..0fc63378 --- /dev/null +++ b/aggregates/account/src/main/java/org/cardanofoundation/ledgersync/account/storage/impl/model/AddressTxAmountId.java @@ -0,0 +1,20 @@ +package org.cardanofoundation.ledgersync.account.storage.impl.model; + +import jakarta.persistence.Column; +import lombok.*; + +import java.io.Serializable; + +@Getter +@NoArgsConstructor +@AllArgsConstructor +@EqualsAndHashCode +@Builder +public class AddressTxAmountId implements Serializable { + @Column(name = "address") + private String address; + @Column(name = "unit") + private String unit; + @Column(name = "tx_hash") + private String txHash; +} diff --git a/aggregates/account/src/main/java/org/cardanofoundation/ledgersync/account/storage/impl/repository/AddressTxAmountRepository.java b/aggregates/account/src/main/java/org/cardanofoundation/ledgersync/account/storage/impl/repository/AddressTxAmountRepository.java new file mode 100644 index 00000000..bb3ade6d --- /dev/null +++ b/aggregates/account/src/main/java/org/cardanofoundation/ledgersync/account/storage/impl/repository/AddressTxAmountRepository.java @@ -0,0 +1,11 @@ +package org.cardanofoundation.ledgersync.account.storage.impl.repository; + +import org.cardanofoundation.ledgersync.account.storage.impl.model.AddressTxAmountEntity; +import org.cardanofoundation.ledgersync.account.storage.impl.model.AddressTxAmountId; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +@Repository +public interface AddressTxAmountRepository extends JpaRepository { + int deleteAddressBalanceBySlotGreaterThan(long slot); +} diff --git a/aggregates/account/src/main/java/org/cardanofoundation/ledgersync/account/util/AddressUtil.java b/aggregates/account/src/main/java/org/cardanofoundation/ledgersync/account/util/AddressUtil.java new file mode 100644 index 00000000..dec22558 --- /dev/null +++ b/aggregates/account/src/main/java/org/cardanofoundation/ledgersync/account/util/AddressUtil.java @@ -0,0 +1,19 @@ +package org.cardanofoundation.ledgersync.account.util; + +import com.bloxbean.cardano.yaci.store.common.util.Tuple; + +public class AddressUtil { + private static final int MAX_ADDR_SIZE = 500; //Required for Byron Addresses + + //Return the address and full address if the address is too long + //Using Tuple as Pair doesn't allow null values + public static Tuple getAddress(String address) { + if (address != null && address.length() > MAX_ADDR_SIZE) { + String addr = address.substring(0, MAX_ADDR_SIZE); + String fullAddr = address; + return new Tuple<>(addr, fullAddr); + } else { + return new Tuple<>(address, null); + } + } +} diff --git a/aggregates/account/src/main/resources/db/account/V2_1_1__init.sql b/aggregates/account/src/main/resources/db/account/V2_1_1__init.sql new file mode 100644 index 00000000..bc42aea3 --- /dev/null +++ b/aggregates/account/src/main/resources/db/account/V2_1_1__init.sql @@ -0,0 +1,18 @@ +drop table if exists address_tx_amount; +create table address_tx_amount +( + address varchar(500), + unit varchar(255), + tx_hash varchar(64), + slot bigint, + quantity numeric(38) null, + addr_full text, + stake_address varchar(255), + block bigint, + block_time bigint, + epoch integer, + primary key (address, unit, tx_hash) +); + +CREATE INDEX idx_address_tx_amount_slot + ON address_tx_amount(slot); diff --git a/aggregates/build.gradle b/aggregates/build.gradle new file mode 100644 index 00000000..8e6ee257 --- /dev/null +++ b/aggregates/build.gradle @@ -0,0 +1,3 @@ +subprojects { + apply from: "../../publish-common.gradle" +} diff --git a/aggregation-app/build.gradle b/aggregation-app/build.gradle new file mode 100644 index 00000000..7116b316 --- /dev/null +++ b/aggregation-app/build.gradle @@ -0,0 +1,37 @@ +plugins { + id 'org.springframework.boot' version '3.2.2' +} + +dependencies { + implementation 'org.springframework.boot:spring-boot-starter' + implementation 'org.springframework.boot:spring-boot-starter-validation' + implementation 'org.springframework.boot:spring-boot-starter-data-jpa' + implementation 'org.springframework.boot:spring-boot-starter-webflux' + implementation 'org.springframework.boot:spring-boot-configuration-processor' + implementation 'org.springframework.boot:spring-boot-starter-actuator' + + implementation(libs.yaci.store.starter) + implementation(libs.yaci.store.utxo.starter) + implementation(libs.yaci.store.account.starter) + + implementation project(':aggregates:account') + runtimeOnly 'org.postgresql:postgresql' + + compileOnly(libs.lombok) + annotationProcessor(libs.lombok) + annotationProcessor(libs.mapstruct.processor) + + testImplementation 'org.springframework.boot:spring-boot-starter-test' + testImplementation('org.hsqldb:hsqldb') + testImplementation(libs.pitest) + testCompileOnly(libs.lombok) + testAnnotationProcessor(libs.lombok) +} + +compileJava { + options.compilerArgs += ['-Amapstruct.defaultComponentModel=spring'] +} + +jar { + enabled = false +} diff --git a/aggregation-app/src/main/java/org/cardanofoundation/ledgersync/aggregation/app/AggregationApplication.java b/aggregation-app/src/main/java/org/cardanofoundation/ledgersync/aggregation/app/AggregationApplication.java new file mode 100644 index 00000000..25e4dc05 --- /dev/null +++ b/aggregation-app/src/main/java/org/cardanofoundation/ledgersync/aggregation/app/AggregationApplication.java @@ -0,0 +1,20 @@ +package org.cardanofoundation.ledgersync.aggregation.app; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.domain.EntityScan; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.data.jpa.repository.config.EnableJpaRepositories; + +@SpringBootApplication +@ComponentScan(basePackages = "org.cardanofoundation.*") +@ComponentScan(basePackages = "org.cardanofoundation.*") +@EntityScan("org.cardanofoundation.*") +@EnableJpaRepositories("org.cardanofoundation.*") +public class AggregationApplication { + + public static void main(String[] args) { + SpringApplication.run(AggregationApplication.class, args); + } + +} diff --git a/aggregation-app/src/main/java/org/cardanofoundation/ledgersync/aggregation/app/DBIndexService.java b/aggregation-app/src/main/java/org/cardanofoundation/ledgersync/aggregation/app/DBIndexService.java new file mode 100644 index 00000000..bd1175e9 --- /dev/null +++ b/aggregation-app/src/main/java/org/cardanofoundation/ledgersync/aggregation/app/DBIndexService.java @@ -0,0 +1,106 @@ +package org.cardanofoundation.ledgersync.aggregation.app; + +import com.bloxbean.cardano.yaci.store.events.BlockHeaderEvent; +import com.bloxbean.cardano.yaci.store.events.ByronMainBlockEvent; +import jakarta.annotation.PostConstruct; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.event.EventListener; +import org.springframework.core.io.ClassPathResource; +import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; +import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Propagation; +import org.springframework.transaction.annotation.Transactional; + +import javax.sql.DataSource; +import java.sql.SQLException; +import java.util.concurrent.atomic.AtomicBoolean; + +@Component +@RequiredArgsConstructor +@Slf4j +@ConditionalOnProperty(name = "store.auto-index-management", havingValue = "true", matchIfMissing = false) +public class DBIndexService { + private final DataSource dataSource; + + private AtomicBoolean indexRemoved = new AtomicBoolean(false); + private AtomicBoolean indexApplied = new AtomicBoolean(false); + + @PostConstruct + public void init() { + log.info("<< Enable DBIndexService >>"); + } + + @EventListener + @Transactional(propagation = Propagation.REQUIRES_NEW) + public void handleFirstBlockEvent(BlockHeaderEvent blockHeaderEvent) { + if (indexRemoved.get() || blockHeaderEvent.getMetadata().getBlock() > 1 + || blockHeaderEvent.getMetadata().isSyncMode()) + return; + + runDeleteIndexes(); + } + + @EventListener + @Transactional(propagation = Propagation.REQUIRES_NEW) + public void handleFirstBlockEventToCreateIndex(BlockHeaderEvent blockHeaderEvent) { + if (blockHeaderEvent.getMetadata().isSyncMode() && !indexApplied.get()) { + if (blockHeaderEvent.getMetadata().getBlock() < 50000) { + reApplyIndexes(); + } else { + log.info("<< I can't manage the creation of automatic indexes because the number of actual blocks in database exceeds the # of blocks threshold for automatic index application >>"); + log.info("Please manually reapply the required indexes if not done yet. For more details, refer to the 'create-index.sql' file !!!"); + indexApplied.set(true); + } + } + } + + @EventListener + @Transactional(propagation = Propagation.REQUIRES_NEW) + @SneakyThrows + public void handleFirstBlockEvent(ByronMainBlockEvent byronMainBlockEvent) { + if (indexRemoved.get() || byronMainBlockEvent.getMetadata().getBlock() > 1 + || byronMainBlockEvent.getMetadata().isSyncMode()) + return; + + runDeleteIndexes(); + } + + private void runDeleteIndexes() { + try { + String scriptPath = "sql/drop-index.sql"; + + log.info("Deleting optional indexes to speed-up the sync process ..... " + scriptPath); + indexRemoved.set(true); + ResourceDatabasePopulator populator = new ResourceDatabasePopulator(); + populator.addScripts( + new ClassPathResource(scriptPath)); + populator.execute(this.dataSource); + + log.info("Optional indexes have been removed successfully."); + } catch (Exception e) { + log.error("Index deletion failed.", e); + } + } + + private void reApplyIndexes() { + try { + String scriptPath = "sql/create-index.sql"; + + log.info("Re-applying optional indexes after sync process ..... " + scriptPath); + indexApplied.set(true); + + ResourceDatabasePopulator populator = new ResourceDatabasePopulator(); + populator.addScripts( + new ClassPathResource(scriptPath)); + populator.execute(this.dataSource); + + log.info("Optional indexes have been re-applied successfully."); + } catch (Exception e) { + log.error("Filed to re-apply indexes.", e); + } + } + +} diff --git a/aggregation-app/src/main/resources/application.yml b/aggregation-app/src/main/resources/application.yml new file mode 100644 index 00000000..cad62292 --- /dev/null +++ b/aggregation-app/src/main/resources/application.yml @@ -0,0 +1,51 @@ +spring: + banner: + location: classpath:/banner.txt + application: + name: Ledger Sync Aggregation App + + flyway: + locations: + - classpath:db/store/{vendor} + - classpath:db/account + out-of-order: true + datasource: + hikari: + maximum-pool-size: 30 + minimum-idle: 5 + jpa: + properties: + hibernate: + jdbc: + batch_size: 100 + order_inserts: true + +apiPrefix: /api/v1 + +logging: + file: + name: ./logs/ls-aggregation.log + +store: + event-publisher-id: 1000 + auto-index-management: true + cardano: + keep-alive-interval: 3000 + account: + enabled: true + balance-aggregation-enabled: true + history-cleanup-enabled: true + # 3 months + balance-cleanup-slot-count: 7889238 + api-enabled: true + parallel-write: true + per-thread-batch-size: 6000 + jooq-write-batch-size: 3000 + executor: + enable-parallel-processing: true + block-processing-threads: 15 + event-processing-threads: 30 + blocks-batch-size: 100 + blocks-partition-size: 10 + use-virtual-thread-for-batch-processing: false + use-virtual-thread-for-event-processing: false diff --git a/aggregation-app/src/main/resources/banner.txt b/aggregation-app/src/main/resources/banner.txt new file mode 100644 index 00000000..d201825b --- /dev/null +++ b/aggregation-app/src/main/resources/banner.txt @@ -0,0 +1,12 @@ + █████╗ ██████╗ ██████╗ ██████╗ ███████╗ ██████╗ █████╗ ████████╗██╗ ██████╗ ███╗ ██╗ █████╗ ██████╗ ██████╗ +██╔══██╗██╔════╝ ██╔════╝ ██╔══██╗██╔════╝██╔════╝ ██╔══██╗╚══██╔══╝██║██╔═══██╗████╗ ██║ ██╔══██╗██╔══██╗██╔══██╗ +███████║██║ ███╗██║ ███╗██████╔╝█████╗ ██║ ███╗███████║ ██║ ██║██║ ██║██╔██╗ ██║ ███████║██████╔╝██████╔╝ +██╔══██║██║ ██║██║ ██║██╔══██╗██╔══╝ ██║ ██║██╔══██║ ██║ ██║██║ ██║██║╚██╗██║ ██╔══██║██╔═══╝ ██╔═══╝ +██║ ██║╚██████╔╝╚██████╔╝██║ ██║███████╗╚██████╔╝██║ ██║ ██║ ██║╚██████╔╝██║ ╚████║ ██║ ██║██║ ██║ +╚═╝ ╚═╝ ╚═════╝ ╚═════╝ ╚═╝ ╚═╝╚══════╝ ╚═════╝ ╚═╝ ╚═╝ ╚═╝ ╚═╝ ╚═════╝ ╚═╝ ╚═══╝ ╚═╝ ╚═╝╚═╝ ╚═╝ + +██╗ ███████╗██████╗ ██████╗ ███████╗██████╗ ███████╗██╗ ██╗███╗ ██╗ ██████╗ +██║ ██╔════╝██╔══██╗██╔════╝ ██╔════╝██╔══██╗ ██╔════╝╚██╗ ██╔╝████╗ ██║██╔════╝ +██║ █████╗ ██║ ██║██║ ███╗█████╗ ██████╔╝ ███████╗ ╚████╔╝ ██╔██╗ ██║██║ +██║ ██╔══╝ ██║ ██║██║ ██║██╔══╝ ██╔══██╗ ╚════██║ ╚██╔╝ ██║╚██╗██║██║ +███████╗███████╗██████╔╝╚██████╔╝███████╗██║ ██║ ███████║ ██║ ██║ ╚████║╚██████╗ diff --git a/aggregation-app/src/main/resources/sql/create-index.sql b/aggregation-app/src/main/resources/sql/create-index.sql new file mode 100644 index 00000000..73b40679 --- /dev/null +++ b/aggregation-app/src/main/resources/sql/create-index.sql @@ -0,0 +1,52 @@ +-- set search_path to mainnet; + +-- utxo store + +CREATE INDEX if not exists idx_address_utxo_owner_addr + ON address_utxo(owner_addr); + +CREATE INDEX if not exists idx_address_utxo_owner_stake_addr + ON address_utxo(owner_stake_addr); + +CREATE INDEX if not exists idx_address_utxo_owner_paykey_hash + ON address_utxo(owner_payment_credential); + +CREATE INDEX if not exists idx_address_utxo_owner_stakekey_hash + ON address_utxo(owner_stake_credential); + +CREATE INDEX if not exists idx_address_utxo_epoch + ON address_utxo(epoch); + +-- account balance + +CREATE INDEX if not exists idx_address_balance_address + ON address_balance (address); + +CREATE INDEX if not exists idx_address_balance_block_time + ON address_balance (block_time); + +CREATE INDEX if not exists idx_address_balance_epoch + ON address_balance (epoch); + +CREATE INDEX if not exists idx_address_balance_unit + ON address_balance (unit); + +CREATE INDEX if not exists idx_address_balance_policy + ON address_balance (policy); + +CREATE INDEX if not exists idx_address_stake_address + ON address_balance (stake_address); + +CREATE INDEX if not exists idx_address_balance_policy_asset + ON address_balance (policy, asset_name); + +-- stake address balance + +CREATE INDEX if not exists idx_stake_addr_balance_stake_addr + ON stake_address_balance (address); + +CREATE INDEX if not exists idx_stake_addr_balance_block_time + ON stake_address_balance (block_time); + +CREATE INDEX if not exists idx_stake_addr_balance_epoch + ON stake_address_balance (epoch); diff --git a/aggregation-app/src/main/resources/sql/drop-index.sql b/aggregation-app/src/main/resources/sql/drop-index.sql new file mode 100644 index 00000000..1d8903ed --- /dev/null +++ b/aggregation-app/src/main/resources/sql/drop-index.sql @@ -0,0 +1,23 @@ +-- set search_path to mainnet; + +-- utxo store +drop index idx_address_utxo_owner_addr; +drop index idx_address_utxo_owner_stake_addr; +drop index idx_address_utxo_owner_paykey_hash; +drop index idx_address_utxo_owner_stakekey_hash; +drop index idx_address_utxo_epoch; + +-- account balance +drop index idx_address_balance_address; +drop index idx_address_balance_block_time; +drop index idx_address_balance_epoch; +drop index idx_address_balance_unit; +drop index idx_address_balance_policy; +drop index idx_address_stake_address; +drop index idx_address_balance_policy_asset; + +-- stake address balance + +drop index idx_stake_addr_balance_stake_addr; +drop index idx_stake_addr_balance_block_time; +drop index idx_stake_addr_balance_epoch; diff --git a/build.gradle b/build.gradle index 4c15f01e..1bcd3fc6 100644 --- a/build.gradle +++ b/build.gradle @@ -37,7 +37,12 @@ subprojects { } dependencies { + implementation(libs.map.struct) + compileOnly(libs.lombok) + annotationProcessor(libs.lombok) + annotationProcessor(libs.lombok.mapstruct.binding) + annotationProcessor(libs.mapstruct.processor) } compileJava { diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index bce986b3..66e490a7 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -1,8 +1,10 @@ [libraries] -yaci-store-starter="com.bloxbean.cardano:yaci-store-spring-boot-starter:0.1.0-rc2-891f739-SNAPSHOT" -yaci-store-remote-starter="com.bloxbean.cardano:yaci-store-remote-spring-boot-starter:0.1.0-rc2-891f739-SNAPSHOT" +yaci-store-starter="com.bloxbean.cardano:yaci-store-spring-boot-starter:0.1.0-rc2-94c9d24-SNAPSHOT" +yaci-store-utxo-starter="com.bloxbean.cardano:yaci-store-utxo-spring-boot-starter:0.1.0-rc2-94c9d24-SNAPSHOT" +yaci-store-account-starter="com.bloxbean.cardano:yaci-store-account-spring-boot-starter:0.1.0-rc2-94c9d24-SNAPSHOT" +yaci-store-remote-starter="com.bloxbean.cardano:yaci-store-remote-spring-boot-starter:0.1.0-rc2-94c9d24-SNAPSHOT" -cardano-client-lib="com.bloxbean.cardano:cardano-client-lib:0.5.0" +cardano-client-lib="com.bloxbean.cardano:cardano-client-lib:0.5.1" snakeyaml="org.yaml:snakeyaml:2.0" guava="com.google.guava:guava:33.0.0-jre" log4j-api="org.apache.logging.log4j:log4j-api:2.22.1" @@ -26,4 +28,4 @@ junit-engine="org.junit.jupiter:junit-jupiter-engine:5.9.0" junit-api="org.junit.jupiter:junit-jupiter-api:5.9.0" assertj-core="org.assertj:assertj-core:3.25.1" -hypersistence-utils-hibernate="io.hypersistence:hypersistence-utils-hibernate-63:3.7.1" \ No newline at end of file +hypersistence-utils-hibernate="io.hypersistence:hypersistence-utils-hibernate-63:3.7.1" diff --git a/publish-common.gradle b/publish-common.gradle index 8ccd6ad7..eb375f2d 100644 --- a/publish-common.gradle +++ b/publish-common.gradle @@ -1,7 +1,7 @@ apply plugin: 'maven-publish' apply plugin: 'signing' apply plugin: 'org.flywaydb.flyway' -//apply plugin: 'nu.studer.jooq' +apply plugin: 'nu.studer.jooq' if (!name.equalsIgnoreCase("components")) { publishing { @@ -57,6 +57,95 @@ if (!name.equalsIgnoreCase("components")) { } } +//JOOQ generator +//Add store modules to this array +def jooq_modules = ['account'] +if (jooq_modules.contains(name)) { + configurations { + flywayMigration + } + + dependencies { + flywayMigration 'com.h2database:h2' + jooqGenerator 'com.h2database:h2' + } + + flyway { + configurations = ['flywayMigration'] + url = 'jdbc:h2:' + project.buildDir.absolutePath + File.separator + 'testdb;DATABASE_TO_LOWER=TRUE;CASE_INSENSITIVE_IDENTIFIERS=TRUE;INIT=CREATE SCHEMA IF NOT EXISTS \"PUBLIC\"\\;' + user = 'sa' + password = '' + locations = ['filesystem:src/main/resources/db/'] + createSchemas = true + schemas = ['PUBLIC'] +// cleanOnValidationError = true +// cleanDisabled = false + } + + jooq { + version = '3.18.9' + configurations { + main { + generationTool { +// logging = org.jooq.meta.jaxb.Logging.WARN + jdbc { + driver = 'org.h2.Driver' + url = flyway.url + user = flyway.user + password = flyway.password + } + generator { + name = 'org.jooq.codegen.DefaultGenerator' + database { + name = 'org.jooq.meta.h2.H2Database' + includes = 'PUBLIC.*' + excludes = 'FLYWAY_SCHEMA_HISTORY | UNUSED_TABLE | PREFIX_.* | SECRET_SCHEMA.SECRET_TABLE | SECRET_ROUTINE' + + schemata { + schema { + inputSchema = 'PUBLIC' + outputSchemaToDefault = true + } + } + } + target { + packageName = 'org.cardanofoundation.ledgersync.' + project.name + ".jooq"; + } + + generate { + // Generate the DAO classes + daos = true + // Annotate DAOs (and other types) with spring annotations, such as @Repository and @Autowired + // for auto-wiring the Configuration instance, e.g. from Spring Boot's jOOQ starter + springAnnotations = true + // Generate Spring-specific DAOs containing @Transactional annotations + springDao = true + } + } + } + } + } + } + + // configure jOOQ task such that it only executes when something has changed that potentially affects the generated JOOQ sources + // - the jOOQ configuration has changed (Jdbc, Generator, Strategy, etc.) + // - the classpath used to execute the jOOQ generation tool has changed (jOOQ library, database driver, strategy classes, etc.) + // - the schema files from which the schema is generated and which is used by jOOQ to generate the sources have changed (scripts added, modified, etc.) + tasks.named('generateJooq').configure { + // ensure database schema has been prepared by Flyway before generating the jOOQ sources + dependsOn tasks.named('flywayMigrate') + + // declare Flyway migration scripts as inputs on the jOOQ task + inputs.files(fileTree('src/main/resources/db/')) + .withPropertyName('migrations') + .withPathSensitivity(PathSensitivity.RELATIVE) + + // make jOOQ task participate in incremental builds (and build caching) + allInputsDeclared = true + } +} + + ext.isReleaseVersion = !version.endsWith("SNAPSHOT") if (isReleaseVersion && !project.hasProperty("skipSigning")) { diff --git a/settings.gradle b/settings.gradle index 99927d67..37a4f1aa 100644 --- a/settings.gradle +++ b/settings.gradle @@ -10,7 +10,9 @@ rootProject.name = 'ledger-sync' include 'components:common' include 'components:consumer-common' include 'components:scheduler' +include 'aggregates:account' include 'application' include 'scheduler-app' include 'streamer-app' +include 'aggregation-app'