Skip to content

Commit

Permalink
branchless programming in hot path
Browse files Browse the repository at this point in the history
  • Loading branch information
@hamdaankhalidmsft
hamdaankhalidmsft committed Dec 23, 2024
1 parent 6d76b5d commit 9cb6983
Show file tree
Hide file tree
Showing 4 changed files with 64 additions and 56 deletions.
2 changes: 2 additions & 0 deletions libs/server/Storage/Functions/MainStore/DeleteMethods.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ namespace Garnet.server
public bool SingleDeleter(ref SpanByte key, ref SpanByte value, ref DeleteInfo deleteInfo, ref RecordInfo recordInfo)
{
functionsState.watchVersionMap.IncrementVersion(deleteInfo.KeyHash);
recordInfo.ClearHasETag();
return true;
}

Expand All @@ -31,6 +32,7 @@ public bool ConcurrentDeleter(ref SpanByte key, ref SpanByte value, ref DeleteIn
functionsState.watchVersionMap.IncrementVersion(deleteInfo.KeyHash);
if (functionsState.appendOnlyFile != null)
WriteLogDelete(ref key, deleteInfo.Version, deleteInfo.SessionID);
recordInfo.ClearHasETag();
return true;
}
}
Expand Down
82 changes: 46 additions & 36 deletions libs/server/Storage/Functions/MainStore/RMWMethods.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
using System;
using System.Buffers;
using System.Diagnostics;
using System.Runtime.CompilerServices;
using Garnet.common;
using Tsavorite.core;

Expand Down Expand Up @@ -54,7 +55,6 @@ public bool NeedInitialUpdate(ref SpanByte key, ref RawStringInput input, ref Sp
/// <inheritdoc />
public bool InitialUpdater(ref SpanByte key, ref RawStringInput input, ref SpanByte value, ref SpanByteAndMemory output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo)
{
recordInfo.ClearHasETag();
rmwInfo.ClearExtraValueLength(ref recordInfo, ref value, value.TotalSize);

RespCommand cmd = input.header.cmd;
Expand Down Expand Up @@ -286,19 +286,23 @@ private bool InPlaceUpdaterWorker(ref SpanByte key, ref RawStringInput input, re
if (value.MetadataSize > 0 && input.header.CheckExpiry(value.ExtraMetadata))
{
rmwInfo.Action = RMWAction.ExpireAndResume;
recordInfo.ClearHasETag();
return false;
}

var cmd = input.header.cmd;
int etagIgnoredOffset = 0;
int etagIgnoredEnd = -1;
long oldEtag = Constants.BaseEtag;
if (recordInfo.ETag)
{
etagIgnoredOffset = Constants.EtagSize;
etagIgnoredEnd = value.LengthWithoutMetadata;
oldEtag = *(long*)value.ToPointer();
}
RespCommand cmd = input.header.cmd;

bool hasETag = recordInfo.ETag;
int etagMultiplier = Unsafe.As<bool, byte>(ref hasETag);

This comment has been minimized.

Sign in to view

Copy link
@TedHartMS

TedHartMS Dec 23, 2024 •
edited
Loading

Contributor

Add a property to RecordInfo:
int HasETagMultiplier => (int)((word & kETagBitMask) >> kEtagBitOffset);

// 0 if no etag else EtagSize
int etagIgnoredOffset = etagMultiplier * Constants.EtagSize;
// -1 if no etag or oldValue.LengthWithoutMEtada if etag
int etagIgnoredEnd = etagMultiplier * (value.LengthWithoutMetadata + 1);
etagIgnoredEnd--;

// 0 if no Etag exists else the first 8 bytes of value
long oldEtag = etagMultiplier * *(long*)value.ToPointer();

switch (cmd)
{
Expand Down Expand Up @@ -747,24 +751,25 @@ private bool InPlaceUpdaterWorker(ref SpanByte key, ref RawStringInput input, re
}

// increment the Etag transparently if in place update happened
if (recordInfo.ETag && rmwInfo.Action == RMWAction.Default)
{
*(long*)value.ToPointer() = oldEtag + 1;
}
// if etag needs to be updated set frist 8 bytes to oldEtag + 1 or leave it unchanged
long existingDataAtPtr = *(long*)value.ToPointer();
*(long*)value.ToPointer() = (etagMultiplier * (oldEtag + 1)) +
((1 - etagMultiplier) * existingDataAtPtr);

return true;
}

/// <inheritdoc />
public bool NeedCopyUpdate(ref SpanByte key, ref RawStringInput input, ref SpanByte oldValue, ref SpanByteAndMemory output, ref RMWInfo rmwInfo)
{
int etagIgnoredOffset = 0;
int etagIgnoredEnd = -1;
if (rmwInfo.RecordInfo.ETag)
{
etagIgnoredOffset = sizeof(long);
etagIgnoredEnd = oldValue.LengthWithoutMetadata;
}
bool hasETag = rmwInfo.RecordInfo.ETag;
int etagMultiplier = Unsafe.As<bool, byte>(ref hasETag);

// 0 if no etag else EtagSize
int etagIgnoredOffset = etagMultiplier * Constants.EtagSize;
// -1 if no etag or oldValue.LengthWithoutMEtada if etag
int etagIgnoredEnd = etagMultiplier * (oldValue.LengthWithoutMetadata + 1);
etagIgnoredEnd--;

switch (input.header.cmd)
{
Expand Down Expand Up @@ -795,6 +800,7 @@ public bool NeedCopyUpdate(ref SpanByte key, ref RawStringInput input, ref SpanB
if (oldValue.MetadataSize > 0 && input.header.CheckExpiry(oldValue.ExtraMetadata))
{
rmwInfo.Action = RMWAction.ExpireAndResume;
rmwInfo.RecordInfo.ClearHasETag();
return false;
}

Expand Down Expand Up @@ -852,15 +858,17 @@ public bool CopyUpdater(ref SpanByte key, ref RawStringInput input, ref SpanByte

RespCommand cmd = input.header.cmd;
bool shouldUpdateEtag = true;
int etagIgnoredOffset = 0;
int etagIgnoredEnd = -1;
long oldEtag = Constants.BaseEtag;
if (recordInfo.ETag)
{
etagIgnoredEnd = oldValue.LengthWithoutMetadata;
etagIgnoredOffset = Constants.EtagSize;
oldEtag = *(long*)oldValue.ToPointer();
}

bool hasETag = recordInfo.ETag;
int etagMultiplier = Unsafe.As<bool, byte>(ref hasETag);

// 0 if no etag else EtagSize
int etagIgnoredOffset = etagMultiplier * Constants.EtagSize;
// -1 if no etag else oldValue.LenghWithoutMetadata
int etagIgnoredEnd = etagMultiplier * (oldValue.LengthWithoutMetadata + 1);
etagIgnoredEnd--;

long oldEtag = etagMultiplier * *(long*)oldValue.ToPointer();

switch (cmd)
{
Expand Down Expand Up @@ -1199,11 +1207,13 @@ public bool CopyUpdater(ref SpanByte key, ref RawStringInput input, ref SpanByte

rmwInfo.SetUsedValueLength(ref recordInfo, ref newValue, newValue.TotalSize);

// increment the Etag transparently if in place update happened
if (recordInfo.ETag && shouldUpdateEtag)
{
*(long*)newValue.ToPointer() = oldEtag + 1;
}
long existingDataAtPtr = *(long*)newValue.ToPointer();
bool hasEtagAndShouldUpdate = recordInfo.ETag && shouldUpdateEtag;
long hasEtagAndShouldUpdateMultiplier = Unsafe.As<bool, byte>(ref hasEtagAndShouldUpdate);

// if etag needs to be updated set frist 8 bytes to oldEtag + 1 or leave it unchanged
*(long*)newValue.ToPointer() = (hasEtagAndShouldUpdateMultiplier * (oldEtag + 1)) +
((1 - hasEtagAndShouldUpdateMultiplier) * existingDataAtPtr);

return true;
}
Expand Down
35 changes: 16 additions & 19 deletions libs/server/Storage/Functions/MainStore/ReadMethods.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

using System.Buffers;
using System.Diagnostics;
using System.Runtime.CompilerServices;
using Garnet.common;
using Tsavorite.core;

Expand All @@ -18,19 +19,17 @@ public bool SingleReader(
ref SpanByte key, ref RawStringInput input,
ref SpanByte value, ref SpanByteAndMemory dst, ref ReadInfo readInfo)
{
var hasEtag = readInfo.RecordInfo.ETag;
bool hasEtag = readInfo.RecordInfo.ETag;
if (value.MetadataSize != 0 && CheckExpiry(ref value))
return false;

var cmd = input.header.cmd;

var isEtagCmd = cmd is RespCommand.GETWITHETAG or RespCommand.GETIFNOTMATCH;

if (cmd == RespCommand.GETIFNOTMATCH)
{
long etagToMatchAgainst = input.parseState.GetLong(0);
// Any value without an etag is treated the same as a value with an etag
long existingEtag = hasEtag ? *(long*)value.ToPointer() : 0;
long existingEtag = Unsafe.As<bool, byte>(ref hasEtag) * *(long*)value.ToPointer();
if (existingEtag == etagToMatchAgainst)
{
// write back array of the format [etag, nil]
Expand All @@ -54,13 +53,12 @@ public bool SingleReader(
}

// Unless the command explicitly asks for the ETag in response, we do not write back the ETag
var start = 0;
var end = -1;
if (!isEtagCmd && hasEtag)
{
start = Constants.EtagSize;
end = value.LengthWithoutMetadata;
}
bool isNotEtagCmdAndRecordHasEtag = cmd is not (RespCommand.GETWITHETAG or RespCommand.GETIFNOTMATCH) && hasEtag;
int isNotEtagCmdAndRecordHasEtagMultiplier = Unsafe.As<bool, byte>(ref isNotEtagCmdAndRecordHasEtag);

int start = isNotEtagCmdAndRecordHasEtagMultiplier * Constants.EtagSize;
int end = isNotEtagCmdAndRecordHasEtagMultiplier * (value.LengthWithoutMetadata + 1);
end--;

if (cmd == RespCommand.NONE)
CopyRespTo(ref value, ref dst, start, end);
Expand Down Expand Up @@ -89,7 +87,7 @@ public bool ConcurrentReader(
{
long etagToMatchAgainst = input.parseState.GetLong(0);
// Any value without an etag is treated the same as a value with an etag
long existingEtag = hasEtag ? *(long*)value.ToPointer() : 0;
long existingEtag = Unsafe.As<bool, byte>(ref hasEtag) * *(long*)value.ToPointer();
if (existingEtag == etagToMatchAgainst)
{
// write back array of the format [etag, nil]
Expand All @@ -113,13 +111,12 @@ public bool ConcurrentReader(
}

// Unless the command explicitly asks for the ETag in response, we do not write back the ETag
var start = 0;
var end = -1;
if (!isEtagCmd && hasEtag)
{
start = Constants.EtagSize;
end = value.LengthWithoutMetadata;
}
bool isNotEtagCmdAndRecordHasEtag = cmd is not (RespCommand.GETWITHETAG or RespCommand.GETIFNOTMATCH) && hasEtag;
int isNotEtagCmdAndRecordHasEtagMultiplier = Unsafe.As<bool, byte>(ref isNotEtagCmdAndRecordHasEtag);

int start = isNotEtagCmdAndRecordHasEtagMultiplier * Constants.EtagSize;
int end = isNotEtagCmdAndRecordHasEtagMultiplier * (value.LengthWithoutMetadata + 1);
end--;

if (cmd == RespCommand.NONE)
CopyRespTo(ref value, ref dst, start, end);
Expand Down
1 change: 0 additions & 1 deletion test/Garnet.test/RespEtagTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -653,7 +653,6 @@ public void SetExpiryIncrForEtagSetData()
ClassicAssert.AreEqual(1, n);

nRetVal = Convert.ToInt64(db.StringGet(strKey));
ClassicAssert.AreEqual(n, nRetVal);
ClassicAssert.AreEqual(1, nRetVal);

var etagGet = (RedisResult[])db.Execute("GETWITHETAG", [strKey]);
Expand Down

0 comments on commit 9cb6983

Please sign in to comment.