Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Don't remove node and blob versions for non-transient repositories #10794 #10817

Merged
merged 2 commits into from
Dec 18, 2024
Merged

Don't remove node and blob versions for non-transient repositories #10794 #10817

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1d99b32
Do not remove node and blob versions for non-transient repositories #…
anatol-sialitski Dec 17, 2024
af3f230
Do not remove node and blob versions for non-transient repositories #…
anatol-sialitski Dec 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 37 additions & 5 deletions modules/core/core-repo/src/main/java/com/enonic/xp/repo/impl/vacuum/VacuumServiceImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package com.enonic.xp.repo.impl.vacuum;

import java.time.Duration;
import java.time.Instant;
import java.util.Set;

import org.osgi.service.component.annotations.Activate;
Expand All @@ -11,7 +12,9 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.enonic.xp.node.DeleteSnapshotParams;
import com.enonic.xp.repo.impl.SecurityHelper;
import com.enonic.xp.snapshot.SnapshotService;
import com.enonic.xp.vacuum.VacuumParameters;
import com.enonic.xp.vacuum.VacuumResult;
import com.enonic.xp.vacuum.VacuumService;
Expand All @@ -25,8 +28,16 @@ public class VacuumServiceImpl

private final VacuumTasks tasks = new VacuumTasks();

private final SnapshotService snapshotService;

private VacuumConfig config;

@Activate
public VacuumServiceImpl( @Reference final SnapshotService snapshotService )
{
this.snapshotService = snapshotService;
}

@Activate
public void activate( final VacuumConfig config )
{
Expand All @@ -46,6 +57,8 @@ public VacuumResult vacuum( final VacuumParameters params )

private VacuumResult doVacuum( final VacuumParameters params )
{
final Instant vacuumStartedAt = Instant.now();

//Retrieves the tasks to execute
VacuumTasks tasks = getTasks( params );
LOG.info( "Starting vacuum. Running " + tasks.size() + " tasks..." );
Expand All @@ -61,22 +74,41 @@ private VacuumResult doVacuum( final VacuumParameters params )
{
LOG.info( "Running vacuum task [" + task.name() + "]..." );

final VacuumTaskParams taskParams = VacuumTaskParams.create().
listener( params.getVacuumListener() ).
ageThreshold( ageThreshold ).
versionsBatchSize( config.versionsBatchSize() ).
build();
final VacuumTaskParams taskParams = VacuumTaskParams.create()
.listener( params.getVacuumListener() )
.ageThreshold( ageThreshold )
.versionsBatchSize( config.versionsBatchSize() )
.vacuumStartedAt( vacuumStartedAt )
.build();
final VacuumTaskResult taskResult = task.execute( taskParams );

LOG.info( task.name() + " : " + taskResult.toString() );
taskResults.add( taskResult );
LOG.info( "Vacuum task [" + task.name() + "] done" );
}

deleteSnapshotsIfNecessary( vacuumStartedAt );

LOG.info( "Vacuum done" );

return taskResults.build();
}

private void deleteSnapshotsIfNecessary( final Instant startedAt )
{
if ( tasks.hasTaskByName( "BinaryBlobVacuumTask" ) || tasks.hasTaskByName( "NodeBlobVacuumTask" ) )
{
try
{
snapshotService.delete( DeleteSnapshotParams.create().before( startedAt ).build() );
}
catch ( Exception e )
{
LOG.error( "Failed to delete snapshots", e );
}
}
}

private VacuumTasks getTasks( final VacuumParameters params )
{
final Set<String> taskNames = params.getTaskNames();
Expand Down
21 changes: 20 additions & 1 deletion modules/core/core-repo/src/main/java/com/enonic/xp/repo/impl/vacuum/VacuumTaskParams.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package com.enonic.xp.repo.impl.vacuum;

import java.time.Duration;
import java.time.Instant;
import java.util.Objects;

import com.enonic.xp.vacuum.VacuumListener;

Expand All @@ -14,11 +16,14 @@ public final class VacuumTaskParams

private final int versionsBatchSize;

private final Instant vacuumStartedAt;

private VacuumTaskParams( final Builder builder )
{
ageThreshold = builder.ageThreshold;
listener = builder.listener;
versionsBatchSize = builder.versionsBatchSize;
vacuumStartedAt = Objects.requireNonNull( builder.vacuumStartedAt );
}

public long getAgeThreshold()
Expand All @@ -31,7 +36,8 @@ public VacuumListener getListener()
return listener;
}

public boolean hasListener() {
public boolean hasListener()
{
return listener != null;
}

Expand All @@ -40,6 +46,11 @@ public int getVersionsBatchSize()
return versionsBatchSize;
}

public Instant getVacuumStartedAt()
{
return vacuumStartedAt;
}

public static Builder create()
{
return new Builder();
Expand All @@ -53,6 +64,8 @@ public static final class Builder

private int versionsBatchSize = DEFAULT_VERSIONS_BATCH_SIZE;

private Instant vacuumStartedAt;

private Builder()
{
}
Expand All @@ -75,6 +88,12 @@ public Builder versionsBatchSize( final int versionsBatchSize )
return this;
}

public Builder vacuumStartedAt( final Instant vacuumStartedAt )
{
this.vacuumStartedAt = vacuumStartedAt;
return this;
}

public VacuumTaskParams build()
{
return new VacuumTaskParams( this );
Expand Down
5 changes: 5 additions & 0 deletions modules/core/core-repo/src/main/java/com/enonic/xp/repo/impl/vacuum/VacuumTasks.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,4 +30,9 @@ public boolean remove( final VacuumTask task )
{
return this.tasks.remove( task );
}

public boolean hasTaskByName( final String taskName )
{
return this.tasks.stream().anyMatch( task -> task.name().equals( taskName ) );
}
}
2 changes: 1 addition & 1 deletion ...ore-repo/src/main/java/com/enonic/xp/repo/impl/vacuum/blob/AbstractBlobVacuumCommand.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ private boolean shouldDelete( final Segment segment, final BlobRecord blobRecord

private boolean isOldBlobRecord( final BlobRecord blobRecord )
{
return System.currentTimeMillis() - blobRecord.lastModified() >= params.getAgeThreshold();
return params.getVacuumStartedAt().toEpochMilli() - blobRecord.lastModified() >= params.getAgeThreshold();
}

private boolean isUsedByVersion( final Segment segment, final BlobKey blobKey )
Expand Down
39 changes: 18 additions & 21 deletions ...ore/core-repo/src/main/java/com/enonic/xp/repo/impl/vacuum/blob/BinaryBlobVacuumTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.enonic.xp.repo.impl.vacuum.blob;

import org.osgi.service.component.annotations.Activate;
import org.osgi.service.component.annotations.Component;
import org.osgi.service.component.annotations.Reference;

Expand All @@ -17,9 +18,16 @@ public class BinaryBlobVacuumTask

private static final String NAME = "BinaryBlobVacuumTask";

private NodeService nodeService;
private final NodeService nodeService;

private BlobStore blobStore;
private final BlobStore blobStore;

@Activate
public BinaryBlobVacuumTask( @Reference final NodeService nodeService, @Reference final BlobStore blobStore )
{
this.nodeService = nodeService;
this.blobStore = blobStore;
}

@Override
public VacuumTaskResult execute( final VacuumTaskParams params )
Expand All @@ -28,14 +36,14 @@ public VacuumTaskResult execute( final VacuumTaskParams params )
{
params.getListener().taskBegin( NAME, null );
}
return BinaryBlobVacuumCommand.create().
blobStore( blobStore ).
nodeService( nodeService ).
params( params ).
build().
execute().
taskName( NAME ).
build();
return BinaryBlobVacuumCommand.create()
.blobStore( blobStore )
.nodeService( nodeService )
.params( params )
.build()
.execute()
.taskName( NAME )
.build();
}

@Override
Expand All @@ -50,15 +58,4 @@ public String name()
return NAME;
}

@Reference
public void setBlobStore( final BlobStore blobStore )
{
this.blobStore = blobStore;
}

@Reference
public void setNodeService( final NodeService nodeService )
{
this.nodeService = nodeService;
}
}
43 changes: 20 additions & 23 deletions .../core/core-repo/src/main/java/com/enonic/xp/repo/impl/vacuum/blob/NodeBlobVacuumTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.enonic.xp.repo.impl.vacuum.blob;

import org.osgi.service.component.annotations.Activate;
import org.osgi.service.component.annotations.Component;
import org.osgi.service.component.annotations.Reference;

Expand All @@ -17,24 +18,32 @@ public class NodeBlobVacuumTask

private static final String NAME = "NodeBlobVacuumTask";

private NodeService nodeService;
private final NodeService nodeService;

private BlobStore blobStore;
private final BlobStore blobStore;

@Activate
public NodeBlobVacuumTask( @Reference final NodeService nodeService, @Reference final BlobStore blobStore )
{
this.nodeService = nodeService;
this.blobStore = blobStore;
}

@Override
public VacuumTaskResult execute( final VacuumTaskParams params )
{
if (params.hasListener()) {
if ( params.hasListener() )
{
params.getListener().taskBegin( NAME, null );
}
return NodeBlobVacuumCommand.create().
blobStore( blobStore ).
nodeService( nodeService ).
params( params ).
build().
execute().
taskName( NAME ).
build();
return NodeBlobVacuumCommand.create()
.blobStore( blobStore )
.nodeService( nodeService )
.params( params )
.build()
.execute()
.taskName( NAME )
.build();
}

@Override
Expand All @@ -48,16 +57,4 @@ public String name()
{
return NAME;
}

@Reference
public void setBlobStore( final BlobStore blobStore )
{
this.blobStore = blobStore;
}

@Reference
public void setNodeService( final NodeService nodeService )
{
this.nodeService = nodeService;
}
}
3 changes: 1 addition & 2 deletions .../core-repo/src/main/java/com/enonic/xp/repo/impl/vacuum/segment/SegmentVacuumCommand.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@

import com.enonic.xp.blob.BlobStore;
import com.enonic.xp.blob.Segment;
import com.enonic.xp.content.ContentConstants;
import com.enonic.xp.context.Context;
import com.enonic.xp.context.ContextAccessor;
import com.enonic.xp.context.ContextBuilder;
Expand Down Expand Up @@ -109,7 +108,7 @@ private boolean isRepositoryToKeep( final RepositoryId repositoryId )
.repositoryId( SystemConstants.SYSTEM_REPO_ID )
.branch( SystemConstants.BRANCH_SYSTEM )
.build();
final Instant since = Instant.now().minusMillis( params.getAgeThreshold() );
final Instant since = params.getVacuumStartedAt().minusMillis( params.getAgeThreshold() );
final NodeVersionQuery findRecentVersionsQuery = NodeVersionQuery.create()
.nodeId( NodeId.from( repositoryId ) )
.addQueryFilter( RangeFilter.create()
Expand Down
4 changes: 1 addition & 3 deletions ...core-repo/src/main/java/com/enonic/xp/repo/impl/vacuum/snapshots/SnapshotsVacuumTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
package com.enonic.xp.repo.impl.vacuum.snapshots;

import java.time.Instant;

import org.osgi.service.component.annotations.Activate;
import org.osgi.service.component.annotations.Component;
import org.osgi.service.component.annotations.Reference;
Expand Down Expand Up @@ -40,7 +38,7 @@ public VacuumTaskResult execute( final VacuumTaskParams params )
final VacuumTaskResult.Builder builder = VacuumTaskResult.create().taskName( NAME );

final DeleteSnapshotsResult deleteSnapshotsResult =
snapshotService.delete( DeleteSnapshotParams.create().before( Instant.now().minusMillis( params.getAgeThreshold() ) ).build() );
snapshotService.delete( DeleteSnapshotParams.create().before( params.getVacuumStartedAt().minusMillis( params.getAgeThreshold() ) ).build() );

deleteSnapshotsResult.getDeletedSnapshots().forEach( snapshot -> builder.processed() );
deleteSnapshotsResult.getFailedSnapshots().forEach( snapshot -> builder.failed() );
Expand Down
9 changes: 6 additions & 3 deletions .../src/main/java/com/enonic/xp/repo/impl/vacuum/versiontable/VersionTableVacuumCommand.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ private VersionTableVacuumCommand( final Builder builder )
listener = builder.params.getListener();
batchSize = builder.params.getVersionsBatchSize();

final Instant now = builder.clock != null ? Instant.now( builder.clock ) : Instant.now();
final Instant now = builder.clock != null ? Instant.now( builder.clock ) : builder.params.getVacuumStartedAt();
until = now.minusMillis( builder.params.getAgeThreshold() );
untilForTransientRepository = now.minus( 1, ChronoUnit.MINUTES );
}
Expand Down Expand Up @@ -149,8 +149,11 @@ private void doProcessRepository( final Repository repository )
version.getNodeVersionId() );
result.deleted();
versionService.delete( version.getNodeVersionId(), context );
nodeBlobToCheckSet.add( version.getNodeVersionKey().getNodeBlobKey() );
binaryBlobToCheckSet.addAll( version.getBinaryBlobKeys().getSet() );
if ( repository.isTransient() )
{
nodeBlobToCheckSet.add( version.getNodeVersionKey().getNodeBlobKey() );
binaryBlobToCheckSet.addAll( version.getBinaryBlobKeys().getSet() );
}
}
else
{
Expand Down
Loading
Loading