From d7f125d6a1a28ededfd433c3b4d47653da9160b6 Mon Sep 17 00:00:00 2001 From: Anatol Sialitski Date: Mon, 9 Dec 2024 09:01:14 +0100 Subject: [PATCH] Vacuum Task for Transient repositories #10690 --- .../VersionTableVacuumCommand.java | 35 ++++++-- .../versiontable/VersionTableVacuumTask.java | 40 ++++++---- .../VersionTableVacuumTaskTest.java | 80 +++++++++++++++++++ 3 files changed, 134 insertions(+), 21 deletions(-) diff --git a/modules/core/core-repo/src/main/java/com/enonic/xp/repo/impl/vacuum/versiontable/VersionTableVacuumCommand.java b/modules/core/core-repo/src/main/java/com/enonic/xp/repo/impl/vacuum/versiontable/VersionTableVacuumCommand.java index c05bfd8ab40..1a0b0779ca1 100644 --- a/modules/core/core-repo/src/main/java/com/enonic/xp/repo/impl/vacuum/versiontable/VersionTableVacuumCommand.java +++ b/modules/core/core-repo/src/main/java/com/enonic/xp/repo/impl/vacuum/versiontable/VersionTableVacuumCommand.java @@ -1,6 +1,8 @@ package com.enonic.xp.repo.impl.vacuum.versiontable; +import java.time.Clock; import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.HashSet; import java.util.Set; @@ -58,6 +60,8 @@ public class VersionTableVacuumCommand private final Instant until; + private final Instant untilForTransientRepository; + private final VacuumListener listener; private final int batchSize; @@ -76,9 +80,12 @@ private VersionTableVacuumCommand( final Builder builder ) versionService = builder.versionService; blobStore = builder.blobStore; branchService = builder.branchService; - until = Instant.now().minusMillis( builder.params.getAgeThreshold() ); listener = builder.params.getListener(); batchSize = builder.params.getVersionsBatchSize(); + + final Instant now = builder.clock != null ? Instant.now( builder.clock ) : Instant.now(); + until = now.minusMillis( builder.params.getAgeThreshold() ); + untilForTransientRepository = now.minus( 1, ChronoUnit.MINUTES ); } public static Builder create() @@ -109,7 +116,11 @@ private void doProcessRepository( final Repository repository ) NodeVersionId lastVersionId = null; - NodeVersionQuery query = createQuery( lastVersionId ); + final Instant ageThreshold = repository.isTransient() ? untilForTransientRepository : until; + + LOG.debug( "Repo is transient: {}, ageThreshold: {}", repository.isTransient(), ageThreshold ); + + NodeVersionQuery query = createQuery( lastVersionId, ageThreshold ); NodeVersionQueryResult versionsResult = nodeService.findVersions( query ); long hits = versionsResult.getHits(); @@ -134,6 +145,8 @@ private void doProcessRepository( final Repository repository ) final boolean toDelete = processVersion( repository, version ); if ( toDelete ) { + LOG.debug( "Version's timestamp = '{}', nodeId = '{}', versionId = '{}'", version.getTimestamp(), version.getNodeId(), + version.getNodeVersionId() ); result.deleted(); versionService.delete( version.getNodeVersionId(), context ); nodeBlobToCheckSet.add( version.getNodeVersionKey().getNodeBlobKey() ); @@ -155,7 +168,7 @@ private void doProcessRepository( final Repository repository ) .filter( blobKey -> !isBlobKeyUsed( blobKey, VersionIndexPath.BINARY_BLOB_KEYS ) ) .forEach( blobKey -> removeNodeBlobRecord( repository.getId(), NodeConstants.BINARY_SEGMENT_LEVEL, blobKey ) ); - query = createQuery( lastVersionId ); + query = createQuery( lastVersionId, ageThreshold ); versionsResult = nodeService.findVersions( query ); hits = versionsResult.getHits(); } @@ -185,10 +198,10 @@ private boolean processVersion( final Repository repository, final NodeVersionMe switch ( findVersionsInBranches( repository, version ) ) { case NO_VERSION_FOUND: - LOG.debug( "No version found in branch for [{}/ {}]", version.getNodeId(), version.getNodeVersionId() ); + LOG.debug( "No version found in branch for [{}/ {}]", version.getNodeId(), version.getNodeVersionId() ); return true; case OTHER_VERSION_FOUND: - LOG.debug( "Other version found in branch for [{}/ {}]", version.getNodeId(), version.getNodeVersionId() ); + LOG.debug( "Other version found in branch for [{}/ {}]", version.getNodeId(), version.getNodeVersionId() ); return version.getNodeCommitId() == null; default: return false; @@ -222,7 +235,7 @@ private BRANCH_CHECK_RESULT findVersionsInBranches( final Repository repository, return nodeFound ? BRANCH_CHECK_RESULT.OTHER_VERSION_FOUND : BRANCH_CHECK_RESULT.NO_VERSION_FOUND; } - private NodeVersionQuery createQuery( NodeVersionId lastVersionId ) + private NodeVersionQuery createQuery( NodeVersionId lastVersionId, Instant ageThreshold ) { final NodeVersionQuery.Builder builder = NodeVersionQuery.create(); @@ -236,7 +249,7 @@ private NodeVersionQuery createQuery( NodeVersionId lastVersionId ) } final RangeFilter mustBeOlderThanFilter = - RangeFilter.create().fieldName( VersionIndexPath.TIMESTAMP.getPath() ).to( ValueFactory.newDateTime( until ) ).build(); + RangeFilter.create().fieldName( VersionIndexPath.TIMESTAMP.getPath() ).to( ValueFactory.newDateTime( ageThreshold ) ).build(); return builder.addQueryFilter( mustBeOlderThanFilter ) .addOrderBy( FieldOrderExpr.create( VersionIndexPath.VERSION_ID, OrderExpr.Direction.ASC ) ) @@ -246,6 +259,8 @@ private NodeVersionQuery createQuery( NodeVersionId lastVersionId ) public static final class Builder { + private Clock clock; + private NodeService nodeService; private RepositoryService repositoryService; @@ -298,6 +313,12 @@ public Builder branchService( final BranchService branchService ) return this; } + public Builder clock( final Clock clock ) + { + this.clock = clock; + return this; + } + public VersionTableVacuumCommand build() { return new VersionTableVacuumCommand( this ); diff --git a/modules/core/core-repo/src/main/java/com/enonic/xp/repo/impl/vacuum/versiontable/VersionTableVacuumTask.java b/modules/core/core-repo/src/main/java/com/enonic/xp/repo/impl/vacuum/versiontable/VersionTableVacuumTask.java index 366f39b196a..2109faa98c2 100644 --- a/modules/core/core-repo/src/main/java/com/enonic/xp/repo/impl/vacuum/versiontable/VersionTableVacuumTask.java +++ b/modules/core/core-repo/src/main/java/com/enonic/xp/repo/impl/vacuum/versiontable/VersionTableVacuumTask.java @@ -1,5 +1,7 @@ package com.enonic.xp.repo.impl.vacuum.versiontable; +import java.time.Clock; + import org.osgi.service.component.annotations.Activate; import org.osgi.service.component.annotations.Component; import org.osgi.service.component.annotations.Reference; @@ -31,35 +33,45 @@ public class VersionTableVacuumTask private final BlobStore blobStore; + private Clock clock; + @Activate public VersionTableVacuumTask( @Reference final NodeService nodeService, @Reference final RepositoryService repositoryService, - @Reference final VersionService versionService, @Reference final - BranchService branchService, @Reference final BlobStore blobStore ) + @Reference final VersionService versionService, @Reference final BranchService branchService, + @Reference final BlobStore blobStore ) { this.nodeService = nodeService; this.repositoryService = repositoryService; this.versionService = versionService; this.blobStore = blobStore; this.branchService = branchService; + this.clock = Clock.systemUTC(); + } + + public void setClock( final Clock clock ) + { + this.clock = clock; } @Override public VacuumTaskResult execute( final VacuumTaskParams params ) { - if (params.hasListener()) { + if ( params.hasListener() ) + { params.getListener().taskBegin( NAME, null ); } - return VersionTableVacuumCommand.create(). - repositoryService( repositoryService ). - nodeService( nodeService ). - versionService( versionService ). - branchService( branchService ). - blobStore( blobStore ). - params( params ). - build(). - execute(). - taskName( NAME ). - build(); + return VersionTableVacuumCommand.create() + .repositoryService( repositoryService ) + .nodeService( nodeService ) + .versionService( versionService ) + .branchService( branchService ) + .blobStore( blobStore ) + .params( params ) + .clock( clock ) + .build() + .execute() + .taskName( NAME ) + .build(); } @Override diff --git a/modules/itest/itest-core/src/test/java/com/enonic/xp/core/repo/vacuum/versiontable/VersionTableVacuumTaskTest.java b/modules/itest/itest-core/src/test/java/com/enonic/xp/core/repo/vacuum/versiontable/VersionTableVacuumTaskTest.java index e60ca6cd00c..223d55b28d3 100644 --- a/modules/itest/itest-core/src/test/java/com/enonic/xp/core/repo/vacuum/versiontable/VersionTableVacuumTaskTest.java +++ b/modules/itest/itest-core/src/test/java/com/enonic/xp/core/repo/vacuum/versiontable/VersionTableVacuumTaskTest.java @@ -1,11 +1,18 @@ package com.enonic.xp.core.repo.vacuum.versiontable; +import java.time.Clock; import java.time.Duration; +import java.time.Instant; +import java.time.ZoneId; import java.time.temporal.ChronoUnit; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import com.enonic.xp.content.ContentConstants; +import com.enonic.xp.context.Context; +import com.enonic.xp.context.ContextAccessor; +import com.enonic.xp.context.ContextBuilder; import com.enonic.xp.core.AbstractNodeTest; import com.enonic.xp.node.Node; import com.enonic.xp.node.NodeId; @@ -16,6 +23,14 @@ import com.enonic.xp.repo.impl.node.NodeHelper; import com.enonic.xp.repo.impl.vacuum.VacuumTaskParams; import com.enonic.xp.repo.impl.vacuum.versiontable.VersionTableVacuumTask; +import com.enonic.xp.repository.CreateRepositoryParams; +import com.enonic.xp.repository.RepositoryId; +import com.enonic.xp.security.PrincipalKey; +import com.enonic.xp.security.RoleKeys; +import com.enonic.xp.security.User; +import com.enonic.xp.security.acl.AccessControlEntry; +import com.enonic.xp.security.acl.AccessControlList; +import com.enonic.xp.security.auth.AuthenticationInfo; import com.enonic.xp.vacuum.VacuumTaskResult; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -139,6 +154,71 @@ void age_threshold() assertVersions( node1.id(), 3 ); } + @Test + void testDeleteOnTransientRepository() + { + final Context oldContext = ContextAccessor.current(); + + try + { + final RepositoryId repositoryId = RepositoryId.from( "com.test.transient" + System.currentTimeMillis() ); + final Context context = ContextBuilder.create() + .branch( ContentConstants.BRANCH_MASTER ) + .repositoryId( repositoryId ) + .authInfo( AuthenticationInfo.create() + .principals( RoleKeys.ADMIN ) + .user( User.create().key( PrincipalKey.ofSuperUser() ).login( PrincipalKey.ofSuperUser().getId() ).build() ) + .build() ) + .build(); + + ContextAccessor.INSTANCE.set( context ); + + context.callWith( () -> { + createTransientRepo( repositoryId ); + + Instant initTime = Instant.now(); + Clock clock = Clock.fixed( initTime, ZoneId.systemDefault() ); + + this.task.setClock( clock ); + + final Node node1 = createNode( NodePath.ROOT, "node1" ); + updateNode( node1.id(), 1 ); + + refresh(); + assertVersions( node1.id(), 2 ); + + Instant newTime = initTime.plus( 3, ChronoUnit.MINUTES ); + this.task.setClock( Clock.fixed( newTime, ZoneId.systemDefault() ) ); + + final VacuumTaskResult result = NodeHelper.runAsAdmin( () -> this.task.execute( VacuumTaskParams.create().build() ) ); + + refresh(); + + assertEquals( 3, result.getProcessed() ); + assertEquals( 1, result.getDeleted() ); + + assertVersions( node1.id(), 1 ); + + return null; + } ); + } + finally + { + ContextAccessor.INSTANCE.set( oldContext ); + } + } + + private void createTransientRepo( final RepositoryId repositoryId ) + { + final AccessControlList rootPermissions = + AccessControlList.of( AccessControlEntry.create().principal( TEST_DEFAULT_USER.getKey() ).allowAll().build() ); + + this.repositoryService.createRepository( + CreateRepositoryParams.create().repositoryId( repositoryId ).rootPermissions( rootPermissions ).transientFlag( true ).build() ); + + refresh(); + } + private void assertVersions( final NodeId nodeId, final int versions ) { final NodeVersionQueryResult result = this.nodeService.findVersions( NodeVersionQuery.create().