diff --git a/libs/common/RespReadUtils.cs b/libs/common/RespReadUtils.cs index 4554f97de3..39c802cfe0 100644 --- a/libs/common/RespReadUtils.cs +++ b/libs/common/RespReadUtils.cs @@ -3,6 +3,7 @@ using System; using System.Buffers.Text; +using System.Diagnostics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Text; @@ -458,6 +459,47 @@ public static bool TryRead64Int(out long number, ref byte* ptr, byte* end, out b return true; } + /// + /// Given a buffer check if the value is nil ($-1\r\n) + /// If the value is nil it advances the buffer forward + /// + /// The starting position in the RESP string. Will be advanced if parsing is successful. + /// The current end of the RESP string. + /// + /// True if value is nil on the buffer, false if the value on buffer is not nil + public static bool ReadNil(ref byte* ptr, byte* end, out byte? unexpectedToken) + { + unexpectedToken = null; + if (end - ptr < 5) + { + return false; + } + + ReadOnlySpan expectedNilRepr = "$-1\r\n"u8; + + if (*(uint*)ptr != MemoryMarshal.Read(expectedNilRepr.Slice(0, 4)) || *(ptr + 4) != expectedNilRepr[4]) + { + ReadOnlySpan ptrNext5Bytes = new ReadOnlySpan(ptr, 5); + for (int i = 0; i < 5; i++) + { + // first place where the sequence differs we have found the unexpected token + if (expectedNilRepr[i] != ptrNext5Bytes[i]) + { + // move the pointer to the unexpected token + ptr += i; + unexpectedToken = ptrNext5Bytes[i]; + return false; + } + } + // If the sequence is not equal we shouldn't even reach this because atleast one byte should have mismatched + Debug.Assert(false); + return false; + } + + ptr += 5; + return true; + } + /// /// Tries to read a RESP array length header from the given ASCII-encoded RESP string /// and, if successful, moves the given ptr to the end of the length header. diff --git a/libs/resources/RespCommandsInfo.json b/libs/resources/RespCommandsInfo.json index 9ce6742527..faf4b7d70d 100644 --- a/libs/resources/RespCommandsInfo.json +++ b/libs/resources/RespCommandsInfo.json @@ -1418,6 +1418,20 @@ ], "SubCommands": null }, + { + "Command": "GETIFNOTMATCH", + "Name": "GETIFNOTMATCH", + "IsInternal": false, + "Arity": 3, + "Flags": "NONE", + "FirstKey": 1, + "LastKey": 1, + "Step": 1, + "AclCategories": "Fast, String, Read", + "Tips": null, + "KeySpecifications": null, + "SubCommands": null + }, { "Command": "GETRANGE", "Name": "GETRANGE", @@ -1472,6 +1486,20 @@ ], "SubCommands": null }, + { + "Command": "GETWITHETAG", + "Name": "GETWITHETAG", + "IsInternal": false, + "Arity": 2, + "Flags": "NONE", + "FirstKey": 1, + "LastKey": 1, + "Step": 1, + "AclCategories": "Fast, String, Read", + "Tips": null, + "KeySpecifications": null, + "SubCommands": null + }, { "Command": "HDEL", "Name": "HDEL", @@ -3395,6 +3423,20 @@ ], "SubCommands": null }, + { + "Command": "SETIFMATCH", + "Name": "SETIFMATCH", + "IsInternal": false, + "Arity": 4, + "Flags": "NONE", + "FirstKey": 1, + "LastKey": 1, + "Step": 1, + "AclCategories": "Fast, String, Write", + "Tips": null, + "KeySpecifications": null, + "SubCommands": null + }, { "Command": "SETRANGE", "Name": "SETRANGE", @@ -3420,6 +3462,20 @@ } ] }, + { + "Command": "SETWITHETAG", + "Name": "SETWITHETAG", + "IsInternal": false, + "Arity": -3, + "Flags": "NONE", + "FirstKey": 1, + "LastKey": 1, + "Step": 1, + "AclCategories": "Fast, String, Write", + "Tips": null, + "KeySpecifications": null, + "SubCommands": null + }, { "Command": "SINTER", "Name": "SINTER", diff --git a/libs/server/API/GarnetApi.cs b/libs/server/API/GarnetApi.cs index 34c91d00a5..23120c5ed6 100644 --- a/libs/server/API/GarnetApi.cs +++ b/libs/server/API/GarnetApi.cs @@ -127,8 +127,8 @@ public GarnetStatus SET_Conditional(ref SpanByte key, ref RawStringInput input) => storageSession.SET_Conditional(ref key, ref input, ref context); /// - public GarnetStatus SET_Conditional(ref SpanByte key, ref RawStringInput input, ref SpanByteAndMemory output) - => storageSession.SET_Conditional(ref key, ref input, ref output, ref context); + public GarnetStatus SET_Conditional(ref SpanByte key, ref RawStringInput input, ref SpanByteAndMemory output, RespCommand cmd) + => storageSession.SET_Conditional(ref key, ref input, ref output, ref context, cmd); /// public GarnetStatus SET(ArgSlice key, Memory value) diff --git a/libs/server/API/GarnetStatus.cs b/libs/server/API/GarnetStatus.cs index 2277461ad4..3e9c26eb02 100644 --- a/libs/server/API/GarnetStatus.cs +++ b/libs/server/API/GarnetStatus.cs @@ -23,6 +23,10 @@ public enum GarnetStatus : byte /// /// Wrong type /// - WRONGTYPE + WRONGTYPE, + /// + /// ETAG mismatch result for an etag based command + /// + ETAGMISMATCH, } } \ No newline at end of file diff --git a/libs/server/API/IGarnetApi.cs b/libs/server/API/IGarnetApi.cs index 3c4e0d6c27..778279ae29 100644 --- a/libs/server/API/IGarnetApi.cs +++ b/libs/server/API/IGarnetApi.cs @@ -41,7 +41,7 @@ public interface IGarnetApi : IGarnetReadApi, IGarnetAdvancedApi /// /// SET Conditional /// - GarnetStatus SET_Conditional(ref SpanByte key, ref RawStringInput input, ref SpanByteAndMemory output); + GarnetStatus SET_Conditional(ref SpanByte key, ref RawStringInput input, ref SpanByteAndMemory output, RespCommand cmd); /// /// SET diff --git a/libs/server/Constants.cs b/libs/server/Constants.cs new file mode 100644 index 0000000000..f6e01df9af --- /dev/null +++ b/libs/server/Constants.cs @@ -0,0 +1,11 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + + +namespace Garnet.server +{ + internal static class Constants + { + public const int EtagSize = sizeof(long); + } +} \ No newline at end of file diff --git a/libs/server/InputHeader.cs b/libs/server/InputHeader.cs index 3a9ed6c7ee..4e477f9e21 100644 --- a/libs/server/InputHeader.cs +++ b/libs/server/InputHeader.cs @@ -27,6 +27,13 @@ public enum RespInputFlags : byte /// Expired /// Expired = 128, + + /// + /// Flag indicating if a SET operation should retain the etag of the previous value if it exists. + /// This is used for conditional setting. + /// + RetainEtag = 129, + } /// @@ -123,6 +130,17 @@ internal ListOperation ListOp /// internal unsafe void SetSetGetFlag() => flags |= RespInputFlags.SetGet; + /// + /// Set "RetainEtag" flag, used to update the old etag of a key after conditionally setting it + /// + internal unsafe void SetRetainEtagFlag() => flags |= RespInputFlags.RetainEtag; + + /// + /// Check if the RetainEtagFlag is set + /// + /// + internal unsafe bool CheckRetainEtagFlag() => (flags & RespInputFlags.RetainEtag) != 0; + /// /// Check if record is expired, either deterministically during log replay, /// or based on current time in normal operation. diff --git a/libs/server/Resp/BasicCommands.cs b/libs/server/Resp/BasicCommands.cs index 0e5406c731..c09cb6311d 100644 --- a/libs/server/Resp/BasicCommands.cs +++ b/libs/server/Resp/BasicCommands.cs @@ -311,6 +311,167 @@ private bool NetworkGETSET(ref TGarnetApi storageApi) return NetworkSETEXNX(ref storageApi); } + /// + /// GETWITHETAG key + /// Given a key get the value and ETag + /// + private bool NetworkGETWITHETAG(ref TGarnetApi storageApi) + where TGarnetApi : IGarnetApi + { + Debug.Assert(parseState.Count == 1); + + var key = parseState.GetArgSliceByRef(0).SpanByte; + var output = new SpanByteAndMemory(dcurr, (int)(dend - dcurr)); + + // Setup input buffer + // (len of spanbyte + input header size) is our input buffer size + var inputSize = sizeof(int) + RespInputHeader.Size; + SpanByte input = SpanByte.Reinterpret(stackalloc byte[inputSize]); + byte* inputPtr = input.ToPointer(); + ((RespInputHeader*)inputPtr)->cmd = RespCommand.GETWITHETAG; + ((RespInputHeader*)inputPtr)->flags = 0; + + var status = storageApi.GET(ref key, ref input, ref output); + + switch (status) + { + case GarnetStatus.NOTFOUND: + Debug.Assert(output.IsSpanByte); + while (!RespWriteUtils.WriteDirect(CmdStrings.RESP_ERRNOTFOUND, ref dcurr, dend)) + SendAndReset(); + break; + default: + if (!output.IsSpanByte) + SendAndReset(output.Memory, output.Length); + else + dcurr += output.Length; + break; + } + + return true; + } + + /// + /// GETIFNOTMATCH key etag + /// Given a key and an etag, return the value and it's etag only if the sent ETag does not match the existing ETag + /// If the ETag matches then we just send back a string indicating the value has not changed. + /// + private bool NetworkGETIFNOTMATCH(ref TGarnetApi storageApi) + where TGarnetApi : IGarnetApi + { + Debug.Assert(parseState.Count == 2); + + var key = parseState.GetArgSliceByRef(0).SpanByte; + var etagToCheckWith = parseState.GetLong(1); + + var output = new SpanByteAndMemory(dcurr, (int)(dend - dcurr)); + + // Setup input buffer to pass command info, and the ETag to check with. + // len + header + etag's data type size + var inputSize = RespInputHeader.Size + Constants.EtagSize + sizeof(int); + SpanByte input = SpanByte.Reinterpret(stackalloc byte[inputSize]); + byte* inputPtr = input.ToPointer(); + ((RespInputHeader*)inputPtr)->cmd = RespCommand.GETIFNOTMATCH; + ((RespInputHeader*)inputPtr)->flags = 0; + *(long*)(inputPtr + RespInputHeader.Size) = etagToCheckWith; + + var status = storageApi.GET(ref key, ref input, ref output); + + switch (status) + { + case GarnetStatus.NOTFOUND: + Debug.Assert(output.IsSpanByte); + while (!RespWriteUtils.WriteDirect(CmdStrings.RESP_ERRNOTFOUND, ref dcurr, dend)) + SendAndReset(); + break; + default: + if (!output.IsSpanByte) + SendAndReset(output.Memory, output.Length); + else + dcurr += output.Length; + break; + } + + return true; + } + + /// + /// SETIFNOTMATCH key val etag + /// Sets a key value pair only if an already existing etag does not match the etag sent as a part of the request + /// + /// + /// + /// + private bool NetworkSETIFMATCH(ref TGarnetApi storageApi) + where TGarnetApi : IGarnetApi + { + Debug.Assert(parseState.Count == 3); + + var key = parseState.GetArgSliceByRef(0).SpanByte; + var value = parseState.GetArgSliceByRef(1).SpanByte; + long etagToCheckWith = parseState.GetLong(2); + + /* + The network buffer holds key, value, and etag in a contiguous chunk of memory, in order, along with padding for separators in RESP. + Shift the etag down over the post-value padding to immediately follow the value: + [] -> [] + */ + + int initialSizeOfValueSpan = value.Length; + value.Length = initialSizeOfValueSpan + Constants.EtagSize; + *(long*)(value.ToPointer() + initialSizeOfValueSpan) = etagToCheckWith; + + // Here Etag retain argument does not really matter because setifmatch may or may not update etag based on the "if match" condition + NetworkSET_Conditional(RespCommand.SETIFMATCH, 0, ref key, value.ToPointer() - sizeof(int), value.Length, getValue: true, highPrecision: false, retainEtag: true, ref storageApi); + + return true; + } + + /// + /// SETWITHETAG key val [RETAINETAG] + /// Sets a key value pair with an ETAG associated with the value internally + /// Calling this on a key that already exists is an error case + /// + private bool NetworkSETWITHETAG(ref TGarnetApi storageApi) + where TGarnetApi : IGarnetApi + { + Debug.Assert(parseState.Count == 2 || parseState.Count == 3); + + var key = parseState.GetArgSliceByRef(0).SpanByte; + var value = parseState.GetArgSliceByRef(1).SpanByte; + + bool retainEtag = false; + if (parseState.Count == 3) + { + Span opt = parseState.GetArgSliceByRef(2).Span; + if (opt.SequenceEqual(CmdStrings.RETAINETAG)) + { + retainEtag = true; + } + else + { + AsciiUtils.ToUpperInPlace(opt); + if (opt.SequenceEqual(CmdStrings.RETAINETAG)) + { + retainEtag = true; + } + else + { + // Unknown option + while (!RespWriteUtils.WriteError(CmdStrings.RESP_ERR_GENERIC_UNK_CMD, ref dcurr, dend)) + SendAndReset(); + return true; + } + + } + } + + // calling set with etag on an exisitng key will update the etag of the existing key + NetworkSET_Conditional(RespCommand.SETWITHETAG, 0, ref key, value.ToPointer() - sizeof(int), value.Length, getValue: true, highPrecision: false, retainEtag, ref storageApi); + + return true; + } + /// /// SETRANGE /// @@ -440,6 +601,12 @@ private bool NetworkSETNX(bool highPrecision, ref TGarnetApi storage return NetworkSETEXNX(ref storageApi); } + enum EtagRetentionOption : byte + { + None, + RETAIN, + } + enum ExpirationOption : byte { None, @@ -473,6 +640,7 @@ private bool NetworkSETEXNX(ref TGarnetApi storageApi) ReadOnlySpan errorMessage = default; var existOptions = ExistOptions.None; var expOption = ExpirationOption.None; + var retainEtagOption = EtagRetentionOption.None; var getValue = false; var tokenIdx = 2; @@ -561,9 +729,17 @@ private bool NetworkSETEXNX(ref TGarnetApi storageApi) } else if (nextOpt.SequenceEqual(CmdStrings.GET)) { - tokenIdx++; getValue = true; } + else if (nextOpt.SequenceEqual(CmdStrings.RETAINETAG)) + { + if (retainEtagOption != EtagRetentionOption.None) + { + errorMessage = CmdStrings.RESP_ERR_GENERIC_SYNTAX_ERROR; + break; + } + retainEtagOption = EtagRetentionOption.RETAIN; + } else { if (!optUpperCased) @@ -587,6 +763,12 @@ private bool NetworkSETEXNX(ref TGarnetApi storageApi) return true; } + // Make space for value header + var valPtr = sbVal.ToPointer() - sizeof(int); + var vSize = sbVal.Length; + + bool isEtagRetained = retainEtagOption == EtagRetentionOption.RETAIN; + switch (expOption) { case ExpirationOption.None: @@ -594,16 +776,26 @@ private bool NetworkSETEXNX(ref TGarnetApi storageApi) switch (existOptions) { case ExistOptions.None: - return getValue - ? NetworkSET_Conditional(RespCommand.SET, expiry, ref sbKey, true, - false, ref storageApi) - : NetworkSET_EX(RespCommand.SET, expOption, expiry, ref sbKey, ref sbVal, ref storageApi); // Can perform a blind update + if (isEtagRetained) + { + // cannot do blind upsert if isEtagRetained + return NetworkSET_Conditional(RespCommand.SET, expiry, ref sbKey, valPtr, vSize, getValue, + highPrecision: false, retainEtag: true, ref storageApi); + } + else + { + return getValue + ? NetworkSET_Conditional(RespCommand.SET, expiry, ref sbKey, valPtr, vSize, getValue: true, + highPrecision: false, retainEtag: false, ref storageApi) + : NetworkSET_EX(RespCommand.SET, expiry, ref sbKey, valPtr, vSize, highPrecision: false, + ref storageApi); // Can perform a blind update + } case ExistOptions.XX: - return NetworkSET_Conditional(RespCommand.SETEXXX, expiry, ref sbKey, getValue, false, - ref storageApi); + return NetworkSET_Conditional(RespCommand.SETEXXX, expiry, ref sbKey, valPtr, vSize, + getValue, highPrecision: false, isEtagRetained, ref storageApi); case ExistOptions.NX: - return NetworkSET_Conditional(RespCommand.SETEXNX, expiry, ref sbKey, getValue, false, - ref storageApi); + return NetworkSET_Conditional(RespCommand.SETEXNX, expiry, ref sbKey, valPtr, vSize, + getValue, highPrecision: false, isEtagRetained, ref storageApi); } break; @@ -611,16 +803,26 @@ private bool NetworkSETEXNX(ref TGarnetApi storageApi) switch (existOptions) { case ExistOptions.None: - return getValue - ? NetworkSET_Conditional(RespCommand.SET, expiry, ref sbKey, true, - true, ref storageApi) - : NetworkSET_EX(RespCommand.SET, expOption, expiry, ref sbKey, ref sbVal, ref storageApi); // Can perform a blind update + if (isEtagRetained) + { + // cannot do a blind update + return NetworkSET_Conditional(RespCommand.SET, expiry, ref sbKey, valPtr, vSize, getValue, + highPrecision: true, retainEtag: true, ref storageApi); + } + else + { + return getValue + ? NetworkSET_Conditional(RespCommand.SET, expiry, ref sbKey, valPtr, vSize, getValue: true, + highPrecision: true, retainEtag: false, ref storageApi) + : NetworkSET_EX(RespCommand.SET, expiry, ref sbKey, valPtr, vSize, highPrecision: true, + ref storageApi); // Can perform a blind update + } case ExistOptions.XX: - return NetworkSET_Conditional(RespCommand.SETEXXX, expiry, ref sbKey, getValue, true, - ref storageApi); + return NetworkSET_Conditional(RespCommand.SETEXXX, expiry, ref sbKey, valPtr, vSize, + getValue, highPrecision: true, isEtagRetained, ref storageApi); case ExistOptions.NX: - return NetworkSET_Conditional(RespCommand.SETEXNX, expiry, ref sbKey, getValue, true, - ref storageApi); + return NetworkSET_Conditional(RespCommand.SETEXNX, expiry, ref sbKey, valPtr, vSize, + getValue, highPrecision: true, isEtagRetained, ref storageApi); } break; @@ -631,14 +833,14 @@ private bool NetworkSETEXNX(ref TGarnetApi storageApi) { case ExistOptions.None: // We can never perform a blind update due to KEEPTTL - return NetworkSET_Conditional(RespCommand.SETKEEPTTL, expiry, ref sbKey, getValue, false, - ref storageApi); + return NetworkSET_Conditional(RespCommand.SETKEEPTTL, expiry, ref sbKey, valPtr, vSize, + getValue, highPrecision: false, isEtagRetained, ref storageApi); case ExistOptions.XX: - return NetworkSET_Conditional(RespCommand.SETKEEPTTLXX, expiry, ref sbKey, getValue, false, - ref storageApi); + return NetworkSET_Conditional(RespCommand.SETKEEPTTLXX, expiry, ref sbKey, valPtr, vSize, + getValue, highPrecision: false, isEtagRetained, ref storageApi); case ExistOptions.NX: - return NetworkSET_Conditional(RespCommand.SETEXNX, expiry, ref sbKey, getValue, false, - ref storageApi); + return NetworkSET_Conditional(RespCommand.SETEXNX, expiry, ref sbKey, valPtr, vSize, + getValue, highPrecision: false, isEtagRetained, ref storageApi); } break; @@ -670,7 +872,7 @@ private unsafe bool NetworkSET_EX(RespCommand cmd, ExpirationOption return true; } - private bool NetworkSET_Conditional(RespCommand cmd, int expiry, ref SpanByte key, bool getValue, bool highPrecision, ref TGarnetApi storageApi) + private bool NetworkSET_Conditional(RespCommand cmd, int expiry, ref SpanByte key, bool getValue, bool highPrecision, bool retainEtag, ref TGarnetApi storageApi) where TGarnetApi : IGarnetApi { var inputArg = expiry == 0 @@ -682,6 +884,9 @@ private bool NetworkSET_Conditional(RespCommand cmd, int expiry, ref var input = new RawStringInput(cmd, ref parseState, startIdx: 1, arg1: inputArg); + if (retainEtag) + input.header.SetRetainEtagFlag(); + if (getValue) input.header.SetSetGetFlag(); @@ -689,21 +894,33 @@ private bool NetworkSET_Conditional(RespCommand cmd, int expiry, ref { var o = new SpanByteAndMemory(dcurr, (int)(dend - dcurr)); var status = storageApi.SET_Conditional(ref key, - ref input, ref o); + ref input, ref o, cmd); - // Status tells us whether an old image was found during RMW or not - if (status == GarnetStatus.NOTFOUND) + // not found for a setwithetag is okay, and so we invert it + if (cmd == RespCommand.SETWITHETAG && status == GarnetStatus.NOTFOUND) { - Debug.Assert(o.IsSpanByte); - while (!RespWriteUtils.WriteNull(ref dcurr, dend)) - SendAndReset(); + status = GarnetStatus.OK; } - else + + // Status tells us whether an old image was found during RMW or not + switch (status) { - if (!o.IsSpanByte) - SendAndReset(o.Memory, o.Length); - else - dcurr += o.Length; + case GarnetStatus.NOTFOUND: + Debug.Assert(o.IsSpanByte); + while (!RespWriteUtils.WriteNull(ref dcurr, dend)) + SendAndReset(); + break; + // SETIFNOTMATCH is the only one who can return this and is always called with getvalue so we only handle this here + case GarnetStatus.ETAGMISMATCH: + while (!RespWriteUtils.WriteDirect(CmdStrings.RESP_ETAGMISMTACH, ref dcurr, dend)) + SendAndReset(); + break; + default: + if (!o.IsSpanByte) + SendAndReset(o.Memory, o.Length); + else + dcurr += o.Length; + break; } } else @@ -713,7 +930,7 @@ private bool NetworkSET_Conditional(RespCommand cmd, int expiry, ref var ok = status != GarnetStatus.NOTFOUND; // Status tells us whether an old image was found during RMW or not - // For a "set if not exists", NOTFOUND means the operation succeeded + // For a "set if not exists" NOTFOUND means the operation succeeded // So we invert the ok flag if (cmd == RespCommand.SETEXNX) ok = !ok; diff --git a/libs/server/Resp/Bitmap/BitmapManagerBitPos.cs b/libs/server/Resp/Bitmap/BitmapManagerBitPos.cs index 2273a27c0f..c02bc0feda 100644 --- a/libs/server/Resp/Bitmap/BitmapManagerBitPos.cs +++ b/libs/server/Resp/Bitmap/BitmapManagerBitPos.cs @@ -130,7 +130,7 @@ public static long BitPosDriver(byte setVal, long startOffset, long endOffset, b /// Find pos of set/clear bit in a sequence of bytes. /// /// Pointer to start of bitmap. - /// + /// The bit value to search for (0 for cleared bit or 1 for set bit). /// Starting offset into bitmap. /// End offset into bitmap. /// @@ -138,27 +138,28 @@ private static long BitPosByte(byte* value, byte bSetVal, long startOffset, long { // Mask set to look for 0 or 1 depending on clear/set flag bool bflag = (bSetVal == 0); - long mask = bflag ? -1 : 0; + long mask = bflag ? -1 : 0; // Mask for all 1's (-1 for 0 search) or all 0's (0 for 1 search) long len = (endOffset - startOffset) + 1; - long remainder = len & 7; + long remainder = len & 7; // Check if length is divisible by 8 byte* curr = value + startOffset; - byte* end = curr + (len - remainder); + byte* end = curr + (len - remainder); // Process up to the aligned part of the bitmap - // Search for first word not matching mask. + // Search for first word not matching the mask. while (curr < end) { long v = *(long*)(curr); if (v != mask) break; - curr += 8; + curr += 8; // Move by 64-bit chunks } - long pos = (((long)(curr - value)) << 3); + // Calculate bit position from start of bitmap + long pos = (((long)(curr - value)) << 3); // Convert byte position to bit position long payload = 0; - // Adjust end so we can retrieve word + // Adjust end to account for remainder end = end + remainder; - // Build payload at least one byte to examine + // Build payload from remaining bytes if (curr < end) payload |= (long)curr[0] << 56; if (curr + 1 < end) payload |= (long)curr[1] << 48; if (curr + 2 < end) payload |= (long)curr[2] << 40; @@ -168,14 +169,20 @@ private static long BitPosByte(byte* value, byte bSetVal, long startOffset, long if (curr + 6 < end) payload |= (long)curr[6] << 8; if (curr + 7 < end) payload |= (long)curr[7]; - // Transform to count leading zeros - payload = (bSetVal == 0) ? ~payload : payload; + // Transform payload for bit search + payload = (bflag) ? ~payload : payload; + // Handle edge cases where the bitmap is all 0's or all 1's if (payload == mask) return pos + 0; + // Otherwise, count leading zeros to find the position of the first 1 or 0 pos += (long)Lzcnt.X64.LeadingZeroCount((ulong)payload); + // if we are exceeding it, return -1 + if (pos >= len * 8) + return -1; + return pos; } } diff --git a/libs/server/Resp/CmdStrings.cs b/libs/server/Resp/CmdStrings.cs index e52eb2cbbb..a075b9a8b4 100644 --- a/libs/server/Resp/CmdStrings.cs +++ b/libs/server/Resp/CmdStrings.cs @@ -98,6 +98,7 @@ static partial class CmdStrings public static ReadOnlySpan KEEPTTL => "KEEPTTL"u8; public static ReadOnlySpan NX => "NX"u8; public static ReadOnlySpan XX => "XX"u8; + public static ReadOnlySpan RETAINETAG => "RETAINETAG"u8; public static ReadOnlySpan UNSAFETRUNCATELOG => "UNSAFETRUNCATELOG"u8; public static ReadOnlySpan SAMPLES => "SAMPLES"u8; public static ReadOnlySpan RANK => "RANK"u8; @@ -126,6 +127,8 @@ static partial class CmdStrings public static ReadOnlySpan RESP_PONG => "+PONG\r\n"u8; public static ReadOnlySpan RESP_EMPTY => "$0\r\n\r\n"u8; public static ReadOnlySpan RESP_QUEUED => "+QUEUED\r\n"u8; + public static ReadOnlySpan RESP_VALNOTCHANGED => "+NOTCHANGED\r\n"u8; + public static ReadOnlySpan RESP_ETAGMISMTACH => "+ETAGMISMATCH\r\n"u8; /// /// Simple error response strings, i.e. these are of the form "-errorString\r\n" diff --git a/libs/server/Resp/Parser/RespCommand.cs b/libs/server/Resp/Parser/RespCommand.cs index f5dff8dd55..2c8ab4bf8d 100644 --- a/libs/server/Resp/Parser/RespCommand.cs +++ b/libs/server/Resp/Parser/RespCommand.cs @@ -33,7 +33,9 @@ public enum RespCommand : ushort GEOSEARCH, GET, GETBIT, + GETIFNOTMATCH, GETRANGE, + GETWITHETAG, HEXISTS, HGET, HGETALL, @@ -145,9 +147,11 @@ public enum RespCommand : ushort SETEXNX, SETEXXX, SETNX, + SETIFMATCH, SETKEEPTTL, SETKEEPTTLXX, SETRANGE, + SETWITHETAG, SINTERSTORE, SMOVE, SPOP, @@ -1354,6 +1358,10 @@ private RespCommand FastParseArrayCommand(ref int count, ref ReadOnlySpan { return RespCommand.SDIFFSTORE; } + else if (*(ulong*)(ptr + 1) == MemoryMarshal.Read("10\r\nSETI"u8) && *(ulong*)(ptr + 9) == MemoryMarshal.Read("FMATCH\r\n"u8)) + { + return RespCommand.SETIFMATCH; + } else if (*(ulong*)(ptr + 1) == MemoryMarshal.Read("10\r\nEXPI"u8) && *(ulong*)(ptr + 9) == MemoryMarshal.Read("RETIME\r\n"u8)) { return RespCommand.EXPIRETIME; @@ -1367,7 +1375,6 @@ private RespCommand FastParseArrayCommand(ref int count, ref ReadOnlySpan return RespCommand.ZDIFFSTORE; } break; - case 11: if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("1\r\nUNSUB"u8) && *(ulong*)(ptr + 10) == MemoryMarshal.Read("SCRIBE\r\n"u8)) { @@ -1393,6 +1400,14 @@ private RespCommand FastParseArrayCommand(ref int count, ref ReadOnlySpan { return RespCommand.SINTERSTORE; } + else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("1\r\nGETWI"u8) && *(ulong*)(ptr + 10) == MemoryMarshal.Read("THETAG\r\n"u8)) + { + return RespCommand.GETWITHETAG; + } + else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("1\r\nSETWI"u8) && *(ulong*)(ptr + 10) == MemoryMarshal.Read("THETAG\r\n"u8)) + { + return RespCommand.SETWITHETAG; + } else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("1\r\nPEXPI"u8) && *(ulong*)(ptr + 10) == MemoryMarshal.Read("RETIME\r\n"u8)) { return RespCommand.PEXPIRETIME; @@ -1419,6 +1434,10 @@ private RespCommand FastParseArrayCommand(ref int count, ref ReadOnlySpan { return RespCommand.ZRANGEBYSCORE; } + else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("\nGETIFNO"u8) && *(ulong*)(ptr + 12) == MemoryMarshal.Read("TMATCH\r\n"u8)) + { + return RespCommand.GETIFNOTMATCH; + } break; case 14: diff --git a/libs/server/Resp/RespServerSession.cs b/libs/server/Resp/RespServerSession.cs index 52a80df8f1..dd36ad8a1c 100644 --- a/libs/server/Resp/RespServerSession.cs +++ b/libs/server/Resp/RespServerSession.cs @@ -506,11 +506,15 @@ private bool ProcessBasicCommands(RespCommand cmd, ref TGarnetApi st { RespCommand.GET => NetworkGET(ref storageApi), RespCommand.GETEX => NetworkGETEX(ref storageApi), + RespCommand.GETWITHETAG => NetworkGETWITHETAG(ref storageApi), + RespCommand.GETIFNOTMATCH => NetworkGETIFNOTMATCH(ref storageApi), RespCommand.SET => NetworkSET(ref storageApi), RespCommand.SETEX => NetworkSETEX(false, ref storageApi), RespCommand.SETNX => NetworkSETNX(false, ref storageApi), RespCommand.PSETEX => NetworkSETEX(true, ref storageApi), RespCommand.SETEXNX => NetworkSETEXNX(ref storageApi), + RespCommand.SETWITHETAG => NetworkSETWITHETAG(ref storageApi), + RespCommand.SETIFMATCH => NetworkSETIFMATCH(ref storageApi), RespCommand.DEL => NetworkDEL(ref storageApi), RespCommand.RENAME => NetworkRENAME(ref storageApi), RespCommand.RENAMENX => NetworkRENAMENX(ref storageApi), diff --git a/libs/server/Storage/Functions/MainStore/PrivateMethods.cs b/libs/server/Storage/Functions/MainStore/PrivateMethods.cs index 3feaa63b2a..064eb92dc2 100644 --- a/libs/server/Storage/Functions/MainStore/PrivateMethods.cs +++ b/libs/server/Storage/Functions/MainStore/PrivateMethods.cs @@ -36,6 +36,7 @@ static void CopyTo(ref SpanByte src, ref SpanByteAndMemory dst, MemoryPool void CopyRespTo(ref SpanByte src, ref SpanByteAndMemory dst, int start = 0, int end = -1) { + // src length of the value indicating no end is supplied defaults to lengthWithoutMetadata, else it chooses the bigger of 0 or (end - start) int srcLength = end == -1 ? src.LengthWithoutMetadata : ((start < end) ? (end - start) : 0); if (srcLength == 0) { @@ -82,7 +83,7 @@ void CopyRespTo(ref SpanByte src, ref SpanByteAndMemory dst, int start = 0, int } } - void CopyRespToWithInput(ref RawStringInput input, ref SpanByte value, ref SpanByteAndMemory dst, bool isFromPending) + void CopyRespToWithInput(ref RawStringInput input, ref SpanByte value, ref SpanByteAndMemory dst, bool isFromPending, int payloadEtagAccountedEndOffset, int etagAccountedEnd, bool hasEtagInVal) { switch (input.header.cmd) { @@ -92,7 +93,7 @@ void CopyRespToWithInput(ref RawStringInput input, ref SpanByte value, ref SpanB // This is accomplished by calling ConvertToHeap on the destination SpanByteAndMemory if (isFromPending) dst.ConvertToHeap(); - CopyRespTo(ref value, ref dst); + CopyRespTo(ref value, ref dst, payloadEtagAccountedEndOffset, etagAccountedEnd); break; case RespCommand.MIGRATE: @@ -122,20 +123,20 @@ void CopyRespToWithInput(ref RawStringInput input, ref SpanByte value, ref SpanB // Get value without RESP header; exclude expiration if (value.LengthWithoutMetadata <= dst.Length) { - dst.Length = value.LengthWithoutMetadata; - value.AsReadOnlySpan().CopyTo(dst.SpanByte.AsSpan()); + dst.Length = value.LengthWithoutMetadata - payloadEtagAccountedEndOffset; + value.AsReadOnlySpan(payloadEtagAccountedEndOffset).CopyTo(dst.SpanByte.AsSpan()); return; } dst.ConvertToHeap(); - dst.Length = value.LengthWithoutMetadata; + dst.Length = value.LengthWithoutMetadata - payloadEtagAccountedEndOffset; dst.Memory = functionsState.memoryPool.Rent(value.LengthWithoutMetadata); - value.AsReadOnlySpan().CopyTo(dst.Memory.Memory.Span); + value.AsReadOnlySpan(payloadEtagAccountedEndOffset).CopyTo(dst.Memory.Memory.Span); break; case RespCommand.GETBIT: var offset = input.parseState.GetLong(0); - var oldValSet = BitmapManager.GetBit(offset, value.ToPointer(), value.Length); + var oldValSet = BitmapManager.GetBit(offset, value.ToPointer() + payloadEtagAccountedEndOffset, value.Length - payloadEtagAccountedEndOffset); if (oldValSet == 0) CopyDefaultResp(CmdStrings.RESP_RETURN_VAL_0, ref dst); else @@ -159,7 +160,7 @@ void CopyRespToWithInput(ref RawStringInput input, ref SpanByte value, ref SpanB } } - var count = BitmapManager.BitCountDriver(bcStartOffset, bcEndOffset, bcOffsetType, value.ToPointer(), value.Length); + var count = BitmapManager.BitCountDriver(bcStartOffset, bcEndOffset, bcOffsetType, value.ToPointer() + payloadEtagAccountedEndOffset, value.Length - payloadEtagAccountedEndOffset); CopyRespNumber(count, ref dst); break; @@ -185,13 +186,13 @@ void CopyRespToWithInput(ref RawStringInput input, ref SpanByte value, ref SpanB } var pos = BitmapManager.BitPosDriver(bpSetVal, bpStartOffset, bpEndOffset, bpOffsetType, - value.ToPointer(), value.Length); + value.ToPointer() + payloadEtagAccountedEndOffset, value.Length - payloadEtagAccountedEndOffset); *(long*)dst.SpanByte.ToPointer() = pos; CopyRespNumber(pos, ref dst); break; case RespCommand.BITOP: - var bitmap = (IntPtr)value.ToPointer(); + var bitmap = (IntPtr)value.ToPointer() + payloadEtagAccountedEndOffset; var output = dst.SpanByte.ToPointer(); *(long*)output = bitmap.ToInt64(); @@ -201,7 +202,7 @@ void CopyRespToWithInput(ref RawStringInput input, ref SpanByte value, ref SpanB case RespCommand.BITFIELD: var bitFieldArgs = GetBitFieldArguments(ref input); - var (retValue, overflow) = BitmapManager.BitFieldExecute(bitFieldArgs, value.ToPointer(), value.Length); + var (retValue, overflow) = BitmapManager.BitFieldExecute(bitFieldArgs, value.ToPointer() + payloadEtagAccountedEndOffset, value.Length - payloadEtagAccountedEndOffset); if (!overflow) CopyRespNumber(retValue, ref dst); else @@ -210,16 +211,16 @@ void CopyRespToWithInput(ref RawStringInput input, ref SpanByte value, ref SpanB case RespCommand.PFCOUNT: case RespCommand.PFMERGE: - if (!HyperLogLog.DefaultHLL.IsValidHYLL(value.ToPointer(), value.Length)) + if (!HyperLogLog.DefaultHLL.IsValidHYLL(value.ToPointer() + payloadEtagAccountedEndOffset, value.Length - payloadEtagAccountedEndOffset)) { *(long*)dst.SpanByte.ToPointer() = -1; return; } - if (value.Length <= dst.Length) + if (value.Length - payloadEtagAccountedEndOffset <= dst.Length) { - Buffer.MemoryCopy(value.ToPointer(), dst.SpanByte.ToPointer(), value.Length, value.Length); - dst.SpanByte.Length = value.Length; + Buffer.MemoryCopy(value.ToPointer() + payloadEtagAccountedEndOffset, dst.SpanByte.ToPointer(), value.Length - payloadEtagAccountedEndOffset, value.Length - payloadEtagAccountedEndOffset); + dst.SpanByte.Length = value.Length - payloadEtagAccountedEndOffset; return; } @@ -236,12 +237,52 @@ void CopyRespToWithInput(ref RawStringInput input, ref SpanByte value, ref SpanB return; case RespCommand.GETRANGE: - var len = value.LengthWithoutMetadata; + var len = value.LengthWithoutMetadata - payloadEtagAccountedEndOffset; var start = input.parseState.GetInt(0); var end = input.parseState.GetInt(1); (start, end) = NormalizeRange(start, end, len); - CopyRespTo(ref value, ref dst, start, end); + CopyRespTo(ref value, ref dst, start + payloadEtagAccountedEndOffset, end + payloadEtagAccountedEndOffset); + return; + case RespCommand.SETIFMATCH: + // extract ETAG, write as long into dst, and then value + long etag = *(long*)value.ToPointer(); + // remove the length of the ETAG + int valueLength = value.LengthWithoutMetadata - sizeof(long); + // here we know the value span has first bytes set to etag so we hardcode skipping past the bytes for the etag below + ReadOnlySpan etagTruncatedVal = value.AsReadOnlySpan(sizeof(long)); + // *2\r\n :(etag digits)\r\n $(val Len digits)\r\n (value len)\r\n + int desiredLength = 4 + 1 + NumUtils.NumDigitsInLong(etag) + 2 + 1 + NumUtils.NumDigits(valueLength) + 2 + valueLength + 2; + WriteValAndEtagToDst(desiredLength, ref etagTruncatedVal, etag, ref dst); + return; + + case RespCommand.GETIFNOTMATCH: + case RespCommand.GETWITHETAG: + // If this has an etag then we want to use it other wise null + // we know somethgin doesnt have an etag if + etag = -1; + valueLength = value.LengthWithoutMetadata; + + if (hasEtagInVal) + { + // Get value without RESP header; exclude expiration + // extract ETAG, write as long into dst, and then value + etag = *(long*)value.ToPointer(); + // remove the length of the ETAG + valueLength -= sizeof(long); + // here we know the value span has first bytes set to etag so we hardcode skipping past the bytes for the etag below + etagTruncatedVal = value.AsReadOnlySpan(sizeof(long)); + // *2\r\n :(etag digits)\r\n $(val Len digits)\r\n (value len)\r\n + desiredLength = 4 + 1 + NumUtils.NumDigitsInLong(etag) + 2 + 1 + NumUtils.NumDigits(valueLength) + 2 + valueLength + 2; + } + else + { + etagTruncatedVal = value.AsReadOnlySpan(); + // instead of :(etagdigits) we will have nil after array len + desiredLength = 4 + 3 + 2 + 1 + NumUtils.NumDigits(valueLength) + 2 + valueLength + 2; + } + + WriteValAndEtagToDst(desiredLength, ref etagTruncatedVal, etag, ref dst); return; case RespCommand.EXPIRETIME: @@ -259,7 +300,44 @@ void CopyRespToWithInput(ref RawStringInput input, ref SpanByte value, ref SpanB } } - bool EvaluateExpireInPlace(ExpireOption optionType, bool expiryExists, long newExpiry, ref SpanByte value, ref SpanByteAndMemory output) + void WriteValAndEtagToDst(int desiredLength, ref ReadOnlySpan value, long etag, ref SpanByteAndMemory dst) + { + if (desiredLength <= dst.Length) + { + dst.Length = desiredLength; + byte* curr = dst.SpanByte.ToPointer(); + byte* end = curr + dst.SpanByte.Length; + RespWriteEtagValArray(etag, ref value, ref curr, end); + return; + } + + dst.ConvertToHeap(); + dst.Length = desiredLength; + dst.Memory = functionsState.memoryPool.Rent(desiredLength); + fixed (byte* ptr = dst.Memory.Memory.Span) + { + byte* curr = ptr; + byte* end = ptr + desiredLength; + RespWriteEtagValArray(etag, ref value, ref curr, end); + } + } + + static void RespWriteEtagValArray(long etag, ref ReadOnlySpan value, ref byte* curr, byte* end) + { + // Writes a Resp encoded Array of Integer for ETAG as first element, and bulk string for value as second element + RespWriteUtils.WriteArrayLength(2, ref curr, end); + if (etag == -1) + { + RespWriteUtils.WriteNull(ref curr, end); + } + else + { + RespWriteUtils.WriteInteger(etag, ref curr, end); + } + RespWriteUtils.WriteBulkString(value, ref curr, end); + } + + bool EvaluateExpireInPlace(ExpireOption optionType, bool expiryExists, ref SpanByte input, ref SpanByte value, ref SpanByteAndMemory output) { ObjectOutputHeader* o = (ObjectOutputHeader*)output.SpanByte.ToPointer(); if (expiryExists) @@ -402,48 +480,50 @@ void EvaluateExpireCopyUpdate(ExpireOption optionType, bool expiryExists, long n internal static bool CheckExpiry(ref SpanByte src) => src.ExtraMetadata < DateTimeOffset.UtcNow.Ticks; - static bool InPlaceUpdateNumber(long val, ref SpanByte value, ref SpanByteAndMemory output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + static bool InPlaceUpdateNumber(long val, ref SpanByte value, ref SpanByteAndMemory output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo, int valueOffset) { var fNeg = false; var ndigits = NumUtils.NumDigitsInLong(val, ref fNeg); ndigits += fNeg ? 1 : 0; - if (ndigits > value.LengthWithoutMetadata) + if (ndigits > value.LengthWithoutMetadata - valueOffset) return false; rmwInfo.ClearExtraValueLength(ref recordInfo, ref value, value.TotalSize); - value.ShrinkSerializedLength(ndigits + value.MetadataSize); - _ = NumUtils.LongToSpanByte(val, value.AsSpan()); + value.ShrinkSerializedLength(ndigits + value.MetadataSize + valueOffset); + _ = NumUtils.LongToSpanByte(val, value.AsSpan(valueOffset)); rmwInfo.SetUsedValueLength(ref recordInfo, ref value, value.TotalSize); Debug.Assert(output.IsSpanByte, "This code assumes it is called in-place and did not go pending"); - value.AsReadOnlySpan().CopyTo(output.SpanByte.AsSpan()); - output.SpanByte.Length = value.LengthWithoutMetadata; + value.AsReadOnlySpan(valueOffset).CopyTo(output.SpanByte.AsSpan()); + output.SpanByte.Length = value.LengthWithoutMetadata - valueOffset; return true; } - static bool InPlaceUpdateNumber(double val, ref SpanByte value, ref SpanByteAndMemory output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + static bool InPlaceUpdateNumber(double val, ref SpanByte value, ref SpanByteAndMemory output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo, int valueOffset) { var ndigits = NumUtils.NumOfCharInDouble(val, out var _, out var _, out var _); - if (ndigits > value.LengthWithoutMetadata) + if (ndigits > value.LengthWithoutMetadata - valueOffset) return false; rmwInfo.ClearExtraValueLength(ref recordInfo, ref value, value.TotalSize); - value.ShrinkSerializedLength(ndigits + value.MetadataSize); - _ = NumUtils.DoubleToSpanByte(val, value.AsSpan()); + value.ShrinkSerializedLength(ndigits + value.MetadataSize + valueOffset); + _ = NumUtils.DoubleToSpanByte(val, value.AsSpan(valueOffset)); rmwInfo.SetUsedValueLength(ref recordInfo, ref value, value.TotalSize); Debug.Assert(output.IsSpanByte, "This code assumes it is called in-place and did not go pending"); - value.AsReadOnlySpan().CopyTo(output.SpanByte.AsSpan()); - output.SpanByte.Length = value.LengthWithoutMetadata; + value.AsReadOnlySpan(valueOffset).CopyTo(output.SpanByte.AsSpan()); + output.SpanByte.Length = value.LengthWithoutMetadata - valueOffset; return true; } - static bool TryInPlaceUpdateNumber(ref SpanByte value, ref SpanByteAndMemory output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo, long input) + static bool TryInPlaceUpdateNumber(ref SpanByte value, ref SpanByteAndMemory output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo, long input, int valueOffset) { // Check if value contains a valid number - if (!IsValidNumber(value.LengthWithoutMetadata, value.ToPointer(), output.SpanByte.AsSpan(), out var val)) + int valLen = value.LengthWithoutMetadata - valueOffset; + byte* valPtr = value.ToPointer() + valueOffset; + if (!IsValidNumber(valLen, valPtr, output.SpanByte.AsSpan(), out var val)) return true; try @@ -456,13 +536,15 @@ static bool TryInPlaceUpdateNumber(ref SpanByte value, ref SpanByteAndMemory out return true; } - return InPlaceUpdateNumber(val, ref value, ref output, ref rmwInfo, ref recordInfo); + return InPlaceUpdateNumber(val, ref value, ref output, ref rmwInfo, ref recordInfo, valueOffset); } - static bool TryInPlaceUpdateNumber(ref SpanByte value, ref SpanByteAndMemory output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo, double input) + static bool TryInPlaceUpdateNumber(ref SpanByte value, ref SpanByteAndMemory output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo, double input, int valueOffset) { // Check if value contains a valid number - if (!IsValidDouble(value.LengthWithoutMetadata, value.ToPointer(), output.SpanByte.AsSpan(), out var val)) + int valLen = value.LengthWithoutMetadata - valueOffset; + byte* valPtr = value.ToPointer() + valueOffset; + if (!IsValidDouble(valLen, valPtr, output.SpanByte.AsSpan(), out var val)) return true; val += input; @@ -473,14 +555,21 @@ static bool TryInPlaceUpdateNumber(ref SpanByte value, ref SpanByteAndMemory out return true; } - return InPlaceUpdateNumber(val, ref value, ref output, ref rmwInfo, ref recordInfo); + return InPlaceUpdateNumber(val, ref value, ref output, ref rmwInfo, ref recordInfo, valueOffset); } - static void CopyUpdateNumber(long next, ref SpanByte newValue, ref SpanByteAndMemory output) + static void CopyUpdateNumber(long next, ref SpanByte newValue, ref SpanByteAndMemory output, int etagIgnoredOffset) { - NumUtils.LongToSpanByte(next, newValue.AsSpan()); - newValue.AsReadOnlySpan().CopyTo(output.SpanByte.AsSpan()); - output.SpanByte.Length = newValue.LengthWithoutMetadata; + NumUtils.LongToSpanByte(next, newValue.AsSpan(etagIgnoredOffset)); + newValue.AsReadOnlySpan(etagIgnoredOffset).CopyTo(output.SpanByte.AsSpan()); + output.SpanByte.Length = newValue.LengthWithoutMetadata - etagIgnoredOffset; + } + + static void CopyUpdateNumber(double next, ref SpanByte newValue, ref SpanByteAndMemory output, int etagIgnoredOffset) + { + NumUtils.DoubleToSpanByte(next, newValue.AsSpan(etagIgnoredOffset)); + newValue.AsReadOnlySpan(etagIgnoredOffset).CopyTo(output.SpanByte.AsSpan()); + output.SpanByte.Length = newValue.LengthWithoutMetadata - etagIgnoredOffset; } static void CopyUpdateNumber(double next, ref SpanByte newValue, ref SpanByteAndMemory output) @@ -497,12 +586,12 @@ static void CopyUpdateNumber(double next, ref SpanByte newValue, ref SpanByteAnd /// New value copying to /// Output value /// Parsed input value - static void TryCopyUpdateNumber(ref SpanByte oldValue, ref SpanByte newValue, ref SpanByteAndMemory output, long input) + static void TryCopyUpdateNumber(ref SpanByte oldValue, ref SpanByte newValue, ref SpanByteAndMemory output, long input, int etagIgnoredOffset) { newValue.ExtraMetadata = oldValue.ExtraMetadata; // Check if value contains a valid number - if (!IsValidNumber(oldValue.LengthWithoutMetadata, oldValue.ToPointer(), output.SpanByte.AsSpan(), out var val)) + if (!IsValidNumber(oldValue.LengthWithoutMetadata - etagIgnoredOffset, oldValue.ToPointer() + etagIgnoredOffset, output.SpanByte.AsSpan(), out var val)) { // Move to tail of the log even when oldValue is alphanumeric // We have already paid the cost of bringing from disk so we are treating as a regular access and bring it into memory @@ -522,7 +611,7 @@ static void TryCopyUpdateNumber(ref SpanByte oldValue, ref SpanByte newValue, re } // Move to tail of the log and update - CopyUpdateNumber(val, ref newValue, ref output); + CopyUpdateNumber(val, ref newValue, ref output, etagIgnoredOffset); } /// @@ -532,12 +621,13 @@ static void TryCopyUpdateNumber(ref SpanByte oldValue, ref SpanByte newValue, re /// New value copying to /// Output value /// Parsed input value - static void TryCopyUpdateNumber(ref SpanByte oldValue, ref SpanByte newValue, ref SpanByteAndMemory output, double input) + /// Number of bytes to skip for ignoring etag in value payload + static void TryCopyUpdateNumber(ref SpanByte oldValue, ref SpanByte newValue, ref SpanByteAndMemory output, double input, int etagIgnoredOffset) { newValue.ExtraMetadata = oldValue.ExtraMetadata; // Check if value contains a valid number - if (!IsValidDouble(oldValue.LengthWithoutMetadata, oldValue.ToPointer(), output.SpanByte.AsSpan(), out var val)) + if (!IsValidDouble(oldValue.LengthWithoutMetadata - etagIgnoredOffset, oldValue.ToPointer() + etagIgnoredOffset, output.SpanByte.AsSpan(), out var val)) { // Move to tail of the log even when oldValue is alphanumeric // We have already paid the cost of bringing from disk so we are treating as a regular access and bring it into memory @@ -553,7 +643,7 @@ static void TryCopyUpdateNumber(ref SpanByte oldValue, ref SpanByte newValue, re } // Move to tail of the log and update - CopyUpdateNumber(val, ref newValue, ref output); + CopyUpdateNumber(val, ref newValue, ref output, etagIgnoredOffset); } /// @@ -650,13 +740,13 @@ void CopyRespNumber(long number, ref SpanByteAndMemory dst) /// /// Copy length of value to output (as ASCII bytes) /// - static void CopyValueLengthToOutput(ref SpanByte value, ref SpanByteAndMemory output) + static void CopyValueLengthToOutput(ref SpanByte value, ref SpanByteAndMemory output, int eTagIgnoredOffset) { - int numDigits = NumUtils.NumDigits(value.LengthWithoutMetadata); + int numDigits = NumUtils.NumDigits(value.LengthWithoutMetadata - eTagIgnoredOffset); Debug.Assert(output.IsSpanByte, "This code assumes it is called in a non-pending context or in a pending context where dst.SpanByte's pointer remains valid"); var outputPtr = output.SpanByte.ToPointer(); - NumUtils.IntToBytes(value.LengthWithoutMetadata, numDigits, ref outputPtr); + NumUtils.IntToBytes(value.LengthWithoutMetadata - eTagIgnoredOffset, numDigits, ref outputPtr); output.SpanByte.Length = numDigits; } diff --git a/libs/server/Storage/Functions/MainStore/RMWMethods.cs b/libs/server/Storage/Functions/MainStore/RMWMethods.cs index ec5b8b1462..404348192c 100644 --- a/libs/server/Storage/Functions/MainStore/RMWMethods.cs +++ b/libs/server/Storage/Functions/MainStore/RMWMethods.cs @@ -19,6 +19,7 @@ public bool NeedInitialUpdate(ref SpanByte key, ref RawStringInput input, ref Sp { switch (input.header.cmd) { + case RespCommand.SETIFMATCH: case RespCommand.SETKEEPTTLXX: case RespCommand.SETEXXX: case RespCommand.PERSIST: @@ -128,7 +129,7 @@ public bool InitialUpdater(ref SpanByte key, ref RawStringInput input, ref SpanB var newValue = input.parseState.GetArgSliceByRef(1).ReadOnlySpan; newValue.CopyTo(value.AsSpan().Slice(offset)); - CopyValueLengthToOutput(ref value, ref output); + CopyValueLengthToOutput(ref value, ref output, 0); break; case RespCommand.APPEND: @@ -136,8 +137,7 @@ public bool InitialUpdater(ref SpanByte key, ref RawStringInput input, ref SpanB // Copy value to be appended to the newly allocated value buffer appendValue.ReadOnlySpan.CopyTo(value.AsSpan()); - - CopyValueLengthToOutput(ref value, ref output); + CopyValueLengthToOutput(ref value, ref output, 0); break; case RespCommand.INCR: value.UnmarkExtraMetadata(); @@ -150,7 +150,8 @@ public bool InitialUpdater(ref SpanByte key, ref RawStringInput input, ref SpanB var incrBy = input.arg1; var ndigits = NumUtils.NumDigitsInLong(incrBy, ref fNeg); value.ShrinkSerializedLength(ndigits + (fNeg ? 1 : 0)); - CopyUpdateNumber(incrBy, ref value, ref output); + // If incrby is being made for initial update then it was not made with etag so the offset is sent as 0 + CopyUpdateNumber(incrBy, ref value, ref output, 0); break; case RespCommand.DECR: value.UnmarkExtraMetadata(); @@ -163,7 +164,22 @@ public bool InitialUpdater(ref SpanByte key, ref RawStringInput input, ref SpanB var decrBy = -input.arg1; ndigits = NumUtils.NumDigitsInLong(decrBy, ref fNeg); value.ShrinkSerializedLength(ndigits + (fNeg ? 1 : 0)); - CopyUpdateNumber(decrBy, ref value, ref output); + // If incrby is being made for initial update then it was not made with etag so the offset is sent as 0 + CopyUpdateNumber(decrBy, ref value, ref output, 0); + break; + case RespCommand.SETWITHETAG: + recordInfo.SetHasETag(); + + // Copy input to value + value.ShrinkSerializedLength(input.Length - RespInputHeader.Size + Constants.EtagSize); + value.ExtraMetadata = input.ExtraMetadata; + + // initial etag set to 0, this is a counter based etag that is incremented on change + *(long*)value.ToPointer() = 0; + input.AsReadOnlySpan()[RespInputHeader.Size..].CopyTo(value.AsSpan(Constants.EtagSize)); + + // Copy initial etag to output + CopyRespNumber(0, ref output); break; case RespCommand.INCRBYFLOAT: value.UnmarkExtraMetadata(); @@ -177,7 +193,7 @@ public bool InitialUpdater(ref SpanByte key, ref RawStringInput input, ref SpanB break; default: value.UnmarkExtraMetadata(); - + recordInfo.ClearHasETag(); if ((ushort)input.header.cmd >= CustomCommandManager.StartOffset) { var functions = functionsState.customCommands[(ushort)input.header.cmd - CustomCommandManager.StartOffset].functions; @@ -252,68 +268,145 @@ private bool InPlaceUpdaterWorker(ref SpanByte key, ref RawStringInput input, re } // First byte of input payload identifies command - switch (input.header.cmd) + var cmd = input.header.cmd; + + int etagIgnoredOffset = 0; + int etagIgnoredEnd = -1; + long oldEtag = -1; + if (recordInfo.ETag) + { + etagIgnoredOffset = Constants.EtagSize; + etagIgnoredEnd = value.LengthWithoutMetadata; + oldEtag = *(long*)value.ToPointer(); + } + + switch (cmd) { case RespCommand.SETEXNX: // Check if SetGet flag is set if (input.header.CheckSetGetFlag()) { // Copy value to output for the GET part of the command. - CopyRespTo(ref value, ref output); + CopyRespTo(ref value, ref output, etagIgnoredOffset, etagIgnoredEnd); } return true; + case RespCommand.SETIFMATCH: + // Cancelling the operation and returning false is used to indicate no RMW because of ETAGMISMATCH + // In this case no etag will match the "nil" etag on a record without an etag + if (!recordInfo.ETag) + { + rmwInfo.Action = RMWAction.CancelOperation; + return false; + } + + long prevEtag = *(long*)value.ToPointer(); + + byte* locationOfEtagInInputPtr = inputPtr + input.LengthWithoutMetadata - Constants.EtagSize; + long etagFromClient = *(long*)locationOfEtagInInputPtr; + + if (prevEtag != etagFromClient) + { + // Cancelling the operation and returning false is used to indicate no RMW because of ETAGMISMATCH + rmwInfo.Action = RMWAction.CancelOperation; + return false; + } + + // Need CU if no space for new value + if (input.Length - RespInputHeader.Size > value.Length) + return false; + + // Increment the ETag + long newEtag = prevEtag + 1; + + // Adjust value length + rmwInfo.ClearExtraValueLength(ref recordInfo, ref value, value.TotalSize); + value.UnmarkExtraMetadata(); + value.ShrinkSerializedLength(input.Length - RespInputHeader.Size); + + // Copy input to value + value.ExtraMetadata = input.ExtraMetadata; + + *(long*)value.ToPointer() = newEtag; + input.AsReadOnlySpan().Slice(0, input.LengthWithoutMetadata - Constants.EtagSize)[RespInputHeader.Size..].CopyTo(value.AsSpan(Constants.EtagSize)); + + rmwInfo.SetUsedValueLength(ref recordInfo, ref value, value.TotalSize); + + CopyRespToWithInput(ref input, ref value, ref output, false, 0, -1, true); + // early return since we already updated the ETag + return true; + case RespCommand.SET: case RespCommand.SETEXXX: var setValue = input.parseState.GetArgSliceByRef(0); + var nextUpdateEtagOffset = etagIgnoredOffset; + var nextUpdateEtagIgnoredEnd = etagIgnoredEnd; + if (!((RespInputHeader*)inputPtr)->CheckRetainEtagFlag()) + { + // if the user did not explictly asked for retaining the etag we need to ignore the etag even if it existed on the previous record + nextUpdateEtagOffset = 0; + nextUpdateEtagIgnoredEnd = -1; + recordInfo.ClearHasETag(); + } + // Need CU if no space for new value var metadataSize = input.arg1 == 0 ? 0 : sizeof(long); - if (setValue.Length + metadataSize > value.Length) return false; + if (setValue.Length + metadataSize > value.Length - etagIgnoredOffset) return false; // Check if SetGet flag is set if (input.header.CheckSetGetFlag()) { // Copy value to output for the GET part of the command. - CopyRespTo(ref value, ref output); + CopyRespTo(ref value, ref output, etagIgnoredOffset, etagIgnoredEnd); } // Adjust value length rmwInfo.ClearExtraValueLength(ref recordInfo, ref value, value.TotalSize); value.UnmarkExtraMetadata(); - value.ShrinkSerializedLength(setValue.Length + metadataSize); + value.ShrinkSerializedLength(setValue.Length + metadataSize + nextUpdateEtagOffset); // Copy input to value value.ExtraMetadata = input.arg1; - setValue.ReadOnlySpan.CopyTo(value.AsSpan()); + setValue.ReadOnlySpan.CopyTo(value.AsSpan(nextUpdateEtagOffset)); rmwInfo.SetUsedValueLength(ref recordInfo, ref value, value.TotalSize); - return true; + break; case RespCommand.SETKEEPTTLXX: case RespCommand.SETKEEPTTL: setValue = input.parseState.GetArgSliceByRef(0); + // respect etag retention only if input header tells you to explicitly + if (!((RespInputHeader*)inputPtr)->CheckRetainEtagFlag()) + { + // if the user did not explictly asked for retaining the etag we need to ignore the etag even if it existed on the previous record + etagIgnoredOffset = 0; + etagIgnoredEnd = -1; + recordInfo.ClearHasETag(); + } + // Need CU if no space for new value - if (setValue.Length + value.MetadataSize > value.Length) return false; + if (setValue.Length + value.MetadataSize > value.Length - etagIgnoredOffset) return false; // Check if SetGet flag is set if (input.header.CheckSetGetFlag()) { // Copy value to output for the GET part of the command. - CopyRespTo(ref value, ref output); + CopyRespTo(ref value, ref output, etagIgnoredOffset, etagIgnoredEnd); } // Adjust value length rmwInfo.ClearExtraValueLength(ref recordInfo, ref value, value.TotalSize); - value.ShrinkSerializedLength(setValue.Length + value.MetadataSize); + value.ShrinkSerializedLength(setValue.Length + value.MetadataSize + etagIgnoredOffset); // Copy input to value - setValue.ReadOnlySpan.CopyTo(value.AsSpan()); + setValue.ReadOnlySpan.CopyTo(value.AsSpan(etagIgnoredOffset)); rmwInfo.SetUsedValueLength(ref recordInfo, ref value, value.TotalSize); return true; case RespCommand.PEXPIRE: case RespCommand.EXPIRE: + // doesn't update etag var expiryExists = value.MetadataSize > 0; var expiryValue = input.parseState.GetLong(0); @@ -347,22 +440,30 @@ private bool InPlaceUpdaterWorker(ref SpanByte key, ref RawStringInput input, re rmwInfo.SetUsedValueLength(ref recordInfo, ref value, value.TotalSize); output.SpanByte.AsSpan()[0] = 1; } + // does not update etag return true; case RespCommand.INCR: - return TryInPlaceUpdateNumber(ref value, ref output, ref rmwInfo, ref recordInfo, input: 1); - + if (!TryInPlaceUpdateNumber(ref value, ref output, ref rmwInfo, ref recordInfo, input: 1, etagIgnoredOffset)) + return false; + break; case RespCommand.DECR: - return TryInPlaceUpdateNumber(ref value, ref output, ref rmwInfo, ref recordInfo, input: -1); + if (!TryInPlaceUpdateNumber(ref value, ref output, ref rmwInfo, ref recordInfo, input: -1, etagIgnoredOffset)) + return false; + break; case RespCommand.INCRBY: // Check if input contains a valid number var incrBy = input.arg1; - return TryInPlaceUpdateNumber(ref value, ref output, ref rmwInfo, ref recordInfo, input: incrBy); + if (!TryInPlaceUpdateNumber(ref value, ref output, ref rmwInfo, ref recordInfo, input: incrBy, etagIgnoredOffset)) + return false; + break; case RespCommand.DECRBY: var decrBy = input.arg1; - return TryInPlaceUpdateNumber(ref value, ref output, ref rmwInfo, ref recordInfo, input: -decrBy); + if (!TryInPlaceUpdateNumber(ref value, ref output, ref rmwInfo, ref recordInfo, input: -decrBy, etagIgnoredOffset)) + return false; + break; case RespCommand.INCRBYFLOAT: // Check if input contains a valid number @@ -371,14 +472,17 @@ private bool InPlaceUpdaterWorker(ref SpanByte key, ref RawStringInput input, re output.SpanByte.AsSpan()[0] = (byte)OperationError.INVALID_TYPE; return true; } - return TryInPlaceUpdateNumber(ref value, ref output, ref rmwInfo, ref recordInfo, incrByFloat); + if (!TryInPlaceUpdateNumber(ref value, ref output, ref rmwInfo, ref recordInfo, incrByFloat, etagIgnoredOffset)) + return false; + break; case RespCommand.SETBIT: var v = value.ToPointer(); var bOffset = input.parseState.GetLong(0); - var bSetVal = (byte)(input.parseState.GetArgSliceByRef(1).ReadOnlySpan[0] - '0'); + var bSetVal = (byte)(input.parseState.GetArgSliceByRef(1).ReadOnlySpan[0] - '0') + etagIgnoredOffset; - if (!BitmapManager.IsLargeEnough(value.Length, bOffset)) return false; + // the "- etagIgnoredOffset" accounts for subtracting the space for the etag in the payload if it exists in the Value + if (!BitmapManager.IsLargeEnough(value.Length, bOffset - etagIgnoredOffset)) return false; rmwInfo.ClearExtraValueLength(ref recordInfo, ref value, value.TotalSize); value.UnmarkExtraMetadata(); @@ -390,11 +494,13 @@ private bool InPlaceUpdaterWorker(ref SpanByte key, ref RawStringInput input, re CopyDefaultResp(CmdStrings.RESP_RETURN_VAL_0, ref output); else CopyDefaultResp(CmdStrings.RESP_RETURN_VAL_1, ref output); - return true; + break; case RespCommand.BITFIELD: var bitFieldArgs = GetBitFieldArguments(ref input); - v = value.ToPointer(); - if (!BitmapManager.IsLargeEnoughForType(bitFieldArgs, value.Length)) return false; + v = value.ToPointer() + etagIgnoredOffset; + + // the "- etagIgnoredOffset" accounts for subtracting the space for the etag in the payload if it exists in the Value + if (!BitmapManager.IsLargeEnoughForType(bitFieldArgs, value.Length - etagIgnoredOffset)) return false; rmwInfo.ClearExtraValueLength(ref recordInfo, ref value, value.TotalSize); value.UnmarkExtraMetadata(); @@ -407,7 +513,8 @@ private bool InPlaceUpdaterWorker(ref SpanByte key, ref RawStringInput input, re CopyRespNumber(bitfieldReturnValue, ref output); else CopyDefaultResp(CmdStrings.RESP_ERRNOTFOUND, ref output); - return true; + + break; case RespCommand.PFADD: v = value.ToPointer(); @@ -426,6 +533,8 @@ private bool InPlaceUpdaterWorker(ref SpanByte key, ref RawStringInput input, re if (result) *output.SpanByte.ToPointer() = updated ? (byte)1 : (byte)0; + + // doesnt update etag because this doesnt work with etag data return result; case RespCommand.PFMERGE: @@ -442,23 +551,25 @@ private bool InPlaceUpdaterWorker(ref SpanByte key, ref RawStringInput input, re rmwInfo.ClearExtraValueLength(ref recordInfo, ref value, value.TotalSize); value.ShrinkSerializedLength(value.Length + value.MetadataSize); rmwInfo.SetUsedValueLength(ref recordInfo, ref value, value.TotalSize); + // doesnt update etag because this doesnt work with etag data return HyperLogLog.DefaultHLL.TryMerge(srcHLL, dstHLL, value.Length); + case RespCommand.SETRANGE: var offset = input.parseState.GetInt(0); var newValue = input.parseState.GetArgSliceByRef(1).ReadOnlySpan; - if (newValue.Length + offset > value.LengthWithoutMetadata) + if (newValue.Length + offset > value.LengthWithoutMetadata - etagIgnoredOffset) return false; - newValue.CopyTo(value.AsSpan().Slice(offset)); + newValue.CopyTo(value.AsSpan(etagIgnoredOffset).Slice(offset)); - CopyValueLengthToOutput(ref value, ref output); - return true; + CopyValueLengthToOutput(ref value, ref output, etagIgnoredOffset); + break; case RespCommand.GETDEL: // Copy value to output for the GET part of the command. // Then, set ExpireAndStop action to delete the record. - CopyRespTo(ref value, ref output); + CopyRespTo(ref value, ref output, etagIgnoredOffset, etagIgnoredEnd); rmwInfo.Action = RMWAction.ExpireAndStop; return false; @@ -491,6 +602,26 @@ private bool InPlaceUpdaterWorker(ref SpanByte key, ref RawStringInput input, re } return true; + case RespCommand.SETWITHETAG: + if (input.Length - RespInputHeader.Size + Constants.EtagSize > value.Length) + return false; + + // retain the older etag (and increment it to account for this update) if requested and if it also exists otherwise set etag to initial etag of 0 + long etagVal = ((RespInputHeader*)inputPtr)->CheckRetainEtagFlag() && recordInfo.ETag ? (oldEtag + 1) : 0; + + recordInfo.SetHasETag(); + + // Copy input to value + value.ShrinkSerializedLength(input.Length - RespInputHeader.Size + Constants.EtagSize); + value.ExtraMetadata = input.ExtraMetadata; + + *(long*)value.ToPointer() = etagVal; + input.AsReadOnlySpan()[RespInputHeader.Size..].CopyTo(value.AsSpan(Constants.EtagSize)); + + // Copy initial etag to output + CopyRespNumber(etagVal, ref output); + // early return since initial etag setting does not need to be incremented + return true; case RespCommand.APPEND: // If nothing to append, can avoid copy update. @@ -498,7 +629,7 @@ private bool InPlaceUpdaterWorker(ref SpanByte key, ref RawStringInput input, re if (appendSize == 0) { - CopyValueLengthToOutput(ref value, ref output); + CopyValueLengthToOutput(ref value, ref output, etagIgnoredOffset); return true; } @@ -532,13 +663,13 @@ private bool InPlaceUpdaterWorker(ref SpanByte key, ref RawStringInput input, re value.ExtraMetadata = expiration; } - var valueLength = value.LengthWithoutMetadata; + var valueLength = value.LengthWithoutMetadata - etagIgnoredOffset; (IMemoryOwner Memory, int Length) outp = (output.Memory, 0); - var ret = functions.InPlaceUpdater(key.AsReadOnlySpan(), ref input, value.AsSpan(), ref valueLength, ref outp, ref rmwInfo); + var ret = functions.InPlaceUpdater(key.AsReadOnlySpan(), ref input, value.AsSpan(etagIgnoredOffset), ref valueLength, ref outp, ref rmwInfo); Debug.Assert(valueLength <= value.LengthWithoutMetadata); // Adjust value length if user shrinks it - if (valueLength < value.LengthWithoutMetadata) + if (valueLength < value.LengthWithoutMetadata - etagIgnoredOffset) { rmwInfo.ClearExtraValueLength(ref recordInfo, ref value, value.TotalSize); value.ShrinkSerializedLength(valueLength + value.MetadataSize); @@ -547,17 +678,52 @@ private bool InPlaceUpdaterWorker(ref SpanByte key, ref RawStringInput input, re output.Memory = outp.Memory; output.Length = outp.Length; - return ret; + if (!ret) + return false; + + break; } throw new GarnetException("Unsupported operation on input"); } + + // increment the Etag transparently if in place update happened + if (recordInfo.ETag && rmwInfo.Action == RMWAction.Default) + { + *(long*)value.ToPointer() = oldEtag + 1; + } + return true; } /// public bool NeedCopyUpdate(ref SpanByte key, ref RawStringInput input, ref SpanByte oldValue, ref SpanByteAndMemory output, ref RMWInfo rmwInfo) { + + int etagIgnoredOffset = 0; + int etagIgnoredEnd = -1; + if (rmwInfo.RecordInfo.ETag) + { + etagIgnoredOffset = Constants.EtagSize; + etagIgnoredEnd = oldValue.LengthWithoutMetadata; + } + switch (input.header.cmd) { + case RespCommand.SETIFMATCH: + if (!rmwInfo.RecordInfo.ETag) + return false; + + long existingEtag = *(long*)oldValue.ToPointer(); + + byte* locationOfEtagInInputPtr = inputPtr + input.LengthWithoutMetadata - Constants.EtagSize; + long etagToCheckWith = *(long*)locationOfEtagInInputPtr; + + if (existingEtag != etagToCheckWith) + { + // cancellation and return false indicates ETag mismatch + rmwInfo.Action = RMWAction.CancelOperation; + return false; + } + return true; case RespCommand.SETEXNX: // Expired data, return false immediately // ExpireAndResume ensures that we set as new value, since it does not exist @@ -570,7 +736,7 @@ public bool NeedCopyUpdate(ref SpanByte key, ref RawStringInput input, ref SpanB if (input.header.CheckSetGetFlag()) { // Copy value to output for the GET part of the command. - CopyRespTo(ref oldValue, ref output); + CopyRespTo(ref oldValue, ref output, etagIgnoredOffset, etagIgnoredEnd); } return false; case RespCommand.SETEXXX: @@ -587,7 +753,7 @@ public bool NeedCopyUpdate(ref SpanByte key, ref RawStringInput input, ref SpanB { (IMemoryOwner Memory, int Length) outp = (output.Memory, 0); var ret = functionsState.customCommands[(ushort)input.header.cmd - CustomCommandManager.StartOffset].functions - .NeedCopyUpdate(key.AsReadOnlySpan(), ref input, oldValue.AsReadOnlySpan(), ref outp); + .NeedCopyUpdate(key.AsReadOnlySpan(), ref input, oldValue.AsReadOnlySpan(etagIgnoredOffset), ref outp); output.Memory = outp.Memory; output.Length = outp.Length; return ret; @@ -609,15 +775,77 @@ public bool CopyUpdater(ref SpanByte key, ref RawStringInput input, ref SpanByte rmwInfo.ClearExtraValueLength(ref recordInfo, ref newValue, newValue.TotalSize); - switch (input.header.cmd) + var cmd = (RespCommand)(*inputPtr); + + bool shouldUpdateEtag = true; + int etagIgnoredOffset = 0; + int etagIgnoredEnd = -1; + long oldEtag = -1; + if (recordInfo.ETag) + { + etagIgnoredOffset = Constants.EtagSize; + etagIgnoredEnd = oldValue.LengthWithoutMetadata; + oldEtag = *(long*)oldValue.ToPointer(); + } + + switch (cmd) { + case RespCommand.SETWITHETAG: + Debug.Assert(input.Length - RespInputHeader.Size + Constants.EtagSize == newValue.Length); + + // etag setting will be done here so does not need to be incremented outside switch + shouldUpdateEtag = false; + // retain the older etag (and increment it to account for this update) if requested and if it also exists otherwise set etag to initial etag of 0 + long etagVal = ((RespInputHeader*)inputPtr)->CheckRetainEtagFlag() && recordInfo.ETag ? (oldEtag + 1) : 0; + recordInfo.SetHasETag(); + // Copy input to value + newValue.ExtraMetadata = input.ExtraMetadata; + input.AsReadOnlySpan()[RespInputHeader.Size..].CopyTo(newValue.AsSpan(Constants.EtagSize)); + // set the etag + *(long*)newValue.ToPointer() = etagVal; + // Copy initial etag to output + CopyRespNumber(etagVal, ref output); + break; + + case RespCommand.SETIFMATCH: + Debug.Assert(recordInfo.ETag, "We should never be able to CU for ETag command on non-etag data. Inplace update should have returned mismatch."); + + // this update is so the early call to send the resp command works, outside of the switch + // we are doing a double op of setting the etag to normalize etag update for other operations + byte* locationOfEtagInInputPtr = inputPtr + input.LengthWithoutMetadata - Constants.EtagSize; + long etagToCheckWith = *(long*)locationOfEtagInInputPtr; + + // Copy input to value + newValue.ExtraMetadata = input.ExtraMetadata; + + *(long*)newValue.ToPointer() = etagToCheckWith + 1; + input.AsReadOnlySpan().Slice(0, input.LengthWithoutMetadata - Constants.EtagSize)[RespInputHeader.Size..].CopyTo(newValue.AsSpan(Constants.EtagSize)); + + + // Write Etag and Val back to Client + CopyRespToWithInput(ref input, ref newValue, ref output, false, 0, -1, true); + break; + case RespCommand.SET: case RespCommand.SETEXXX: + var nextUpdateEtagOffset = etagIgnoredOffset; + var nextUpdateEtagIgnoredEnd = etagIgnoredEnd; + if (!((RespInputHeader*)inputPtr)->CheckRetainEtagFlag()) + { + // if the user did not explictly asked for retaining the etag we need to ignore the etag if it existed on the previous record + nextUpdateEtagOffset = 0; + nextUpdateEtagIgnoredEnd = -1; + recordInfo.ClearHasETag(); + } + + // new value when allocated should have 8 bytes more if the previous record had etag and the cmd was not SETEXXX + Debug.Assert(input.Length - RespInputHeader.Size == newValue.Length - etagIgnoredOffset); + // Check if SetGet flag is set if (input.header.CheckSetGetFlag()) { // Copy value to output for the GET part of the command. - CopyRespTo(ref oldValue, ref output); + CopyRespTo(ref oldValue, ref output, etagIgnoredOffset, etagIgnoredEnd); } // Copy input to value @@ -633,22 +861,32 @@ public bool CopyUpdater(ref SpanByte key, ref RawStringInput input, ref SpanByte case RespCommand.SETKEEPTTLXX: case RespCommand.SETKEEPTTL: var setValue = input.parseState.GetArgSliceByRef(0).ReadOnlySpan; + nextUpdateEtagOffset = etagIgnoredOffset; + nextUpdateEtagIgnoredEnd = etagIgnoredEnd; + if (!((RespInputHeader*)inputPtr)->CheckRetainEtagFlag()) + { + // if the user did not explictly asked for retaining the etag we need to ignore the etag if it existed on the previous record + nextUpdateEtagOffset = 0; + nextUpdateEtagIgnoredEnd = -1; + } + Debug.Assert(oldValue.MetadataSize + setValue.Length == newValue.Length); // Check if SetGet flag is set if (input.header.CheckSetGetFlag()) { // Copy value to output for the GET part of the command. - CopyRespTo(ref oldValue, ref output); + CopyRespTo(ref oldValue, ref output, etagIgnoredOffset, etagIgnoredEnd); } // Copy input to value, retain metadata of oldValue newValue.ExtraMetadata = oldValue.ExtraMetadata; - setValue.CopyTo(newValue.AsSpan()); + setValue.CopyTo(newValue.AsSpan(nextUpdateEtagOffset)); break; case RespCommand.EXPIRE: case RespCommand.PEXPIRE: + shouldUpdateEtag = false; var expiryExists = oldValue.MetadataSize > 0; var expiryValue = input.parseState.GetLong(0); @@ -663,6 +901,7 @@ public bool CopyUpdater(ref SpanByte key, ref RawStringInput input, ref SpanByte case RespCommand.PEXPIREAT: case RespCommand.EXPIREAT: + shouldUpdateEtag = false; expiryExists = oldValue.MetadataSize > 0; var expiryTimestamp = input.parseState.GetLong(0); @@ -675,6 +914,7 @@ public bool CopyUpdater(ref SpanByte key, ref RawStringInput input, ref SpanByte break; case RespCommand.PERSIST: + shouldUpdateEtag = false; oldValue.AsReadOnlySpan().CopyTo(newValue.AsSpan()); if (oldValue.MetadataSize != 0) { @@ -686,21 +926,21 @@ public bool CopyUpdater(ref SpanByte key, ref RawStringInput input, ref SpanByte break; case RespCommand.INCR: - TryCopyUpdateNumber(ref oldValue, ref newValue, ref output, input: 1); + TryCopyUpdateNumber(ref oldValue, ref newValue, ref output, input: 1, etagIgnoredOffset); break; case RespCommand.DECR: - TryCopyUpdateNumber(ref oldValue, ref newValue, ref output, input: -1); + TryCopyUpdateNumber(ref oldValue, ref newValue, ref output, input: -1, etagIgnoredOffset); break; case RespCommand.INCRBY: var incrBy = input.arg1; - TryCopyUpdateNumber(ref oldValue, ref newValue, ref output, input: incrBy); + TryCopyUpdateNumber(ref oldValue, ref newValue, ref output, input: incrBy, etagIgnoredOffset); break; case RespCommand.DECRBY: var decrBy = input.arg1; - TryCopyUpdateNumber(ref oldValue, ref newValue, ref output, input: -decrBy); + TryCopyUpdateNumber(ref oldValue, ref newValue, ref output, input: -decrBy, etagIgnoredOffset); break; case RespCommand.INCRBYFLOAT: @@ -711,13 +951,13 @@ public bool CopyUpdater(ref SpanByte key, ref RawStringInput input, ref SpanByte oldValue.CopyTo(ref newValue); break; } - TryCopyUpdateNumber(ref oldValue, ref newValue, ref output, input: incrByFloat); + TryCopyUpdateNumber(ref oldValue, ref newValue, ref output, input: incrByFloat, etagIgnoredOffset); break; case RespCommand.SETBIT: var bOffset = input.parseState.GetLong(0); var bSetVal = (byte)(input.parseState.GetArgSliceByRef(1).ReadOnlySpan[0] - '0'); - Buffer.MemoryCopy(oldValue.ToPointer(), newValue.ToPointer(), newValue.Length, oldValue.Length); + Buffer.MemoryCopy(oldValue.ToPointer() + etagIgnoredOffset, newValue.ToPointer() + etagIgnoredOffset, newValue.Length - etagIgnoredOffset, oldValue.Length - etagIgnoredOffset); var oldValSet = BitmapManager.UpdateBitmap(newValue.ToPointer(), bOffset, bSetVal); if (oldValSet == 0) CopyDefaultResp(CmdStrings.RESP_RETURN_VAL_0, ref output); @@ -727,8 +967,8 @@ public bool CopyUpdater(ref SpanByte key, ref RawStringInput input, ref SpanByte case RespCommand.BITFIELD: var bitFieldArgs = GetBitFieldArguments(ref input); - Buffer.MemoryCopy(oldValue.ToPointer(), newValue.ToPointer(), newValue.Length, oldValue.Length); - var (bitfieldReturnValue, overflow) = BitmapManager.BitFieldExecute(bitFieldArgs, newValue.ToPointer(), newValue.Length); + Buffer.MemoryCopy(oldValue.ToPointer() + etagIgnoredOffset, newValue.ToPointer() + etagIgnoredOffset, newValue.Length - etagIgnoredOffset, oldValue.Length - etagIgnoredOffset); + var (bitfieldReturnValue, overflow) = BitmapManager.BitFieldExecute(bitFieldArgs, newValue.ToPointer() + etagIgnoredOffset, newValue.Length - etagIgnoredOffset); if (!overflow) CopyRespNumber(bitfieldReturnValue, ref output); @@ -737,6 +977,7 @@ public bool CopyUpdater(ref SpanByte key, ref RawStringInput input, ref SpanByte break; case RespCommand.PFADD: + // HYPERLOG doesnt work with non hyperlog key values var updated = false; var newValPtr = newValue.ToPointer(); var oldValPtr = oldValue.ToPointer(); @@ -745,13 +986,14 @@ public bool CopyUpdater(ref SpanByte key, ref RawStringInput input, ref SpanByte updated = HyperLogLog.DefaultHLL.CopyUpdate(ref input, oldValPtr, newValPtr, newValue.Length); else { - Buffer.MemoryCopy(oldValPtr, newValPtr, newValue.Length, oldValue.Length); + Buffer.MemoryCopy(oldValPtr, newValPtr, newValue.Length - etagIgnoredOffset, oldValue.Length); HyperLogLog.DefaultHLL.Update(ref input, newValPtr, newValue.Length, ref updated); } *output.SpanByte.ToPointer() = updated ? (byte)1 : (byte)0; break; case RespCommand.PFMERGE: + // HYPERLOG doesnt work with non hyperlog key values //srcA offset: [hll allocated size = 4 byte] + [hll data structure] //memcpy +4 (skip len size) var srcHLLPtr = input.parseState.GetArgSliceByRef(0).SpanByte.ToPointer(); // HLL merging from var oldDstHLLPtr = oldValue.ToPointer(); // original HLL merging to (too small to hold its data plus srcA) @@ -765,15 +1007,15 @@ public bool CopyUpdater(ref SpanByte key, ref RawStringInput input, ref SpanByte oldValue.CopyTo(ref newValue); newInputValue = input.parseState.GetArgSliceByRef(1).ReadOnlySpan; - newInputValue.CopyTo(newValue.AsSpan().Slice(offset)); + newInputValue.CopyTo(newValue.AsSpan(etagIgnoredOffset).Slice(offset)); - CopyValueLengthToOutput(ref newValue, ref output); + CopyValueLengthToOutput(ref newValue, ref output, etagIgnoredOffset); break; case RespCommand.GETDEL: // Copy value to output for the GET part of the command. // Then, set ExpireAndStop action to delete the record. - CopyRespTo(ref oldValue, ref output); + CopyRespTo(ref oldValue, ref output, etagIgnoredOffset, etagIgnoredEnd); rmwInfo.Action = RMWAction.ExpireAndStop; return false; @@ -812,9 +1054,10 @@ public bool CopyUpdater(ref SpanByte key, ref RawStringInput input, ref SpanByte var appendValue = input.parseState.GetArgSliceByRef(0); // Append the new value with the client input at the end of the old data + // the oldValue.LengthWithoutMetadata already contains the etag offset here appendValue.ReadOnlySpan.CopyTo(newValue.AsSpan().Slice(oldValue.LengthWithoutMetadata)); - CopyValueLengthToOutput(ref newValue, ref output); + CopyValueLengthToOutput(ref newValue, ref output, etagIgnoredOffset); break; default: @@ -836,7 +1079,7 @@ public bool CopyUpdater(ref SpanByte key, ref RawStringInput input, ref SpanByte (IMemoryOwner Memory, int Length) outp = (output.Memory, 0); var ret = functions - .CopyUpdater(key.AsReadOnlySpan(), ref input, oldValue.AsReadOnlySpan(), newValue.AsSpan(), ref outp, ref rmwInfo); + .CopyUpdater(key.AsReadOnlySpan(), ref input, oldValue.AsReadOnlySpan(etagIgnoredOffset), newValue.AsSpan(etagIgnoredOffset), ref outp, ref rmwInfo); output.Memory = outp.Memory; output.Length = outp.Length; return ret; @@ -845,6 +1088,13 @@ public bool CopyUpdater(ref SpanByte key, ref RawStringInput input, ref SpanByte } rmwInfo.SetUsedValueLength(ref recordInfo, ref newValue, newValue.TotalSize); + + // increment the Etag transparently if in place update happened + if (recordInfo.ETag && shouldUpdateEtag) + { + *(long*)newValue.ToPointer() = oldEtag + 1; + } + return true; } diff --git a/libs/server/Storage/Functions/MainStore/ReadMethods.cs b/libs/server/Storage/Functions/MainStore/ReadMethods.cs index cd0a0be785..45c24204d0 100644 --- a/libs/server/Storage/Functions/MainStore/ReadMethods.cs +++ b/libs/server/Storage/Functions/MainStore/ReadMethods.cs @@ -19,7 +19,21 @@ public bool SingleReader(ref SpanByte key, ref RawStringInput input, ref SpanByt return false; var cmd = input.header.cmd; - if ((ushort)cmd >= CustomCommandManager.StartOffset) + + var isEtagCmd = cmd is RespCommand.GETWITHETAG or RespCommand.GETIFNOTMATCH; + + if (isEtagCmd && cmd == RespCommand.GETIFNOTMATCH) + { + long existingEtag = *(long*)value.ToPointer(); + long etagToMatchAgainst = *(long*)(input.ToPointer() + RespInputHeader.Size); + if (existingEtag == etagToMatchAgainst) + { + // write the value not changed message to dst, and early return + CopyDefaultResp(CmdStrings.RESP_VALNOTCHANGED, ref dst); + return true; + } + } + else if ((ushort)cmd >= CustomCommandManager.StartOffset) { var valueLength = value.LengthWithoutMetadata; (IMemoryOwner Memory, int Length) output = (dst.Memory, 0); @@ -31,10 +45,19 @@ public bool SingleReader(ref SpanByte key, ref RawStringInput input, ref SpanByt return ret; } + // Unless the command explicitly asks for the ETag in response, we do not write back the ETag + var start = 0; + var end = -1; + if (!isEtagCmd && readInfo.RecordInfo.ETag) + { + start = Constants.EtagSize; + end = value.LengthWithoutMetadata; + } + if (cmd == RespCommand.NONE) - CopyRespTo(ref value, ref dst); + CopyRespTo(ref value, ref dst, start, end); else - CopyRespToWithInput(ref input, ref value, ref dst, readInfo.IsFromPending); + CopyRespToWithInput(ref input, ref value, ref dst, readInfo.IsFromPending, start, end, readInfo.RecordInfo.ETag); return true; } @@ -50,7 +73,21 @@ public bool ConcurrentReader(ref SpanByte key, ref RawStringInput input, ref Spa } var cmd = input.header.cmd; - if ((ushort)cmd >= CustomCommandManager.StartOffset) + + var isEtagCmd = cmd is RespCommand.GETWITHETAG or RespCommand.GETIFNOTMATCH; + + if (isEtagCmd && cmd == RespCommand.GETIFNOTMATCH) + { + long existingEtag = *(long*)value.ToPointer(); + long etagToMatchAgainst = *(long*)(input.ToPointer() + RespInputHeader.Size); + if (existingEtag == etagToMatchAgainst) + { + // write the value not changed message to dst, and early return + CopyDefaultResp(CmdStrings.RESP_VALNOTCHANGED, ref dst); + return true; + } + } + else if ((ushort)cmd >= CustomCommandManager.StartOffset) { var valueLength = value.LengthWithoutMetadata; (IMemoryOwner Memory, int Length) output = (dst.Memory, 0); @@ -62,11 +99,20 @@ public bool ConcurrentReader(ref SpanByte key, ref RawStringInput input, ref Spa return ret; } + // Unless the command explicitly asks for the ETag in response, we do not write back the ETag + var start = 0; + var end = -1; + if (!isEtagCmd && recordInfo.ETag) + { + start = Constants.EtagSize; + end = value.LengthWithoutMetadata; + } + if (cmd == RespCommand.NONE) - CopyRespTo(ref value, ref dst); + CopyRespTo(ref value, ref dst, start, end); else { - CopyRespToWithInput(ref input, ref value, ref dst, readInfo.IsFromPending); + CopyRespToWithInput(ref input, ref value, ref dst, readInfo.IsFromPending, start, end, recordInfo.ETag); } return true; diff --git a/libs/server/Storage/Functions/MainStore/UpsertMethods.cs b/libs/server/Storage/Functions/MainStore/UpsertMethods.cs index 80f5cb6127..15fb47ecb2 100644 --- a/libs/server/Storage/Functions/MainStore/UpsertMethods.cs +++ b/libs/server/Storage/Functions/MainStore/UpsertMethods.cs @@ -12,7 +12,10 @@ namespace Garnet.server { /// public bool SingleWriter(ref SpanByte key, ref RawStringInput input, ref SpanByte src, ref SpanByte dst, ref SpanByteAndMemory output, ref UpsertInfo upsertInfo, WriteReason reason, ref RecordInfo recordInfo) - => SpanByteFunctions.DoSafeCopy(ref src, ref dst, ref upsertInfo, ref recordInfo, input.arg1); + { + recordInfo.ClearHasETag(); + return SpanByteFunctions.DoSafeCopy(ref src, ref dst, ref upsertInfo, ref recordInfo, input.arg1); + } /// public void PostSingleWriter(ref SpanByte key, ref RawStringInput input, ref SpanByte src, ref SpanByte dst, ref SpanByteAndMemory output, ref UpsertInfo upsertInfo, WriteReason reason) @@ -25,6 +28,7 @@ public void PostSingleWriter(ref SpanByte key, ref RawStringInput input, ref Spa /// public bool ConcurrentWriter(ref SpanByte key, ref RawStringInput input, ref SpanByte src, ref SpanByte dst, ref SpanByteAndMemory output, ref UpsertInfo upsertInfo, ref RecordInfo recordInfo) { + recordInfo.ClearHasETag(); if (ConcurrentWriterWorker(ref src, ref dst, ref input, ref upsertInfo, ref recordInfo)) { if (!upsertInfo.RecordInfo.Modified) diff --git a/libs/server/Storage/Functions/MainStore/VarLenInputMethods.cs b/libs/server/Storage/Functions/MainStore/VarLenInputMethods.cs index b0b3803465..170ff078a0 100644 --- a/libs/server/Storage/Functions/MainStore/VarLenInputMethods.cs +++ b/libs/server/Storage/Functions/MainStore/VarLenInputMethods.cs @@ -109,7 +109,9 @@ public int GetRMWInitialValueLength(ref RawStringInput input) ndigits = NumUtils.NumDigitsInLong(-input.arg1, ref fNeg); return sizeof(int) + ndigits + (fNeg ? 1 : 0); - + case RespCommand.SETWITHETAG: + // same space as SET but with 8 additional bytes for etag at the front of the payload + return sizeof(int) + input.Length - RespInputHeader.Size + Constants.EtagSize; case RespCommand.INCRBYFLOAT: if (!input.parseState.TryGetDouble(0, out var incrByFloat)) return sizeof(int); @@ -138,48 +140,51 @@ public int GetRMWInitialValueLength(ref RawStringInput input) } /// - public int GetRMWModifiedValueLength(ref SpanByte t, ref RawStringInput input) + public int GetRMWModifiedValueLength(ref SpanByte t, ref RawStringInput input, bool hasEtag) { if (input.header.cmd != RespCommand.NONE) { var cmd = input.header.cmd; + int etagOffset = hasEtag ? Constants.EtagSize : 0; + bool retainEtag = ((RespInputHeader*)inputPtr)->CheckRetainEtagFlag(); + switch (cmd) { case RespCommand.INCR: case RespCommand.INCRBY: var incrByValue = input.header.cmd == RespCommand.INCRBY ? input.arg1 : 1; - var curr = NumUtils.BytesToLong(t.AsSpan()); + var curr = NumUtils.BytesToLong(t.AsSpan(etagOffset)); var next = curr + incrByValue; var fNeg = false; var ndigits = NumUtils.NumDigitsInLong(next, ref fNeg); ndigits += fNeg ? 1 : 0; - return sizeof(int) + ndigits + t.MetadataSize; + return sizeof(int) + ndigits + t.MetadataSize + etagOffset; case RespCommand.DECR: case RespCommand.DECRBY: var decrByValue = input.header.cmd == RespCommand.DECRBY ? input.arg1 : 1; - curr = NumUtils.BytesToLong(t.AsSpan()); + curr = NumUtils.BytesToLong(t.AsSpan(etagOffset)); next = curr - decrByValue; fNeg = false; ndigits = NumUtils.NumDigitsInLong(next, ref fNeg); ndigits += fNeg ? 1 : 0; - return sizeof(int) + ndigits + t.MetadataSize; + return sizeof(int) + ndigits + t.MetadataSize + etagOffset; case RespCommand.INCRBYFLOAT: // We don't need to TryGetDouble here because InPlaceUpdater will raise an error before we reach this point var incrByFloat = input.parseState.GetDouble(0); - NumUtils.TryBytesToDouble(t.AsSpan(), out var currVal); + NumUtils.TryBytesToDouble(t.AsSpan(etagOffset), out var currVal); var nextVal = currVal + incrByFloat; ndigits = NumUtils.NumOfCharInDouble(nextVal, out _, out _, out _); - return sizeof(int) + ndigits + t.MetadataSize; + return sizeof(int) + ndigits + t.MetadataSize + etagOffset; case RespCommand.SETBIT: var bOffset = input.parseState.GetLong(0); return sizeof(int) + BitmapManager.NewBlockAllocLength(t.Length, bOffset); @@ -202,14 +207,21 @@ public int GetRMWModifiedValueLength(ref SpanByte t, ref RawStringInput input) case RespCommand.SETKEEPTTLXX: case RespCommand.SETKEEPTTL: var setValue = input.parseState.GetArgSliceByRef(0); - return sizeof(int) + t.MetadataSize + setValue.Length; + if (!retainEtag) + etagOffset = 0; + return sizeof(int) + t.MetadataSize + setValue.Length + etagOffset; case RespCommand.SET: case RespCommand.SETEXXX: - break; + if (!retainEtag) + etagOffset = 0; + return sizeof(int) + input.Length - RespInputHeader.Size + etagOffset; + case RespCommand.SETIFMATCH: case RespCommand.PERSIST: return sizeof(int) + t.LengthWithoutMetadata; - + case RespCommand.SETWITHETAG: + // same space as SET but with 8 additional bytes for etag at the front of the payload + return sizeof(int) + input.Length - RespInputHeader.Size + Constants.EtagSize; case RespCommand.EXPIRE: case RespCommand.PEXPIRE: case RespCommand.EXPIREAT: @@ -220,8 +232,8 @@ public int GetRMWModifiedValueLength(ref SpanByte t, ref RawStringInput input) var offset = input.parseState.GetInt(0); var newValue = input.parseState.GetArgSliceByRef(1).ReadOnlySpan; - if (newValue.Length + offset > t.LengthWithoutMetadata) - return sizeof(int) + newValue.Length + offset + t.MetadataSize; + if (newValue.Length + offset > t.LengthWithoutMetadata - etagOffset) + return sizeof(int) + newValue.Length + offset + t.MetadataSize + etagOffset; return sizeof(int) + t.Length; case RespCommand.GETDEL: diff --git a/libs/server/Storage/Functions/ObjectStore/VarLenInputMethods.cs b/libs/server/Storage/Functions/ObjectStore/VarLenInputMethods.cs index 63af04149b..6227bb272c 100644 --- a/libs/server/Storage/Functions/ObjectStore/VarLenInputMethods.cs +++ b/libs/server/Storage/Functions/ObjectStore/VarLenInputMethods.cs @@ -12,7 +12,7 @@ namespace Garnet.server public readonly unsafe partial struct ObjectSessionFunctions : ISessionFunctions { /// - public int GetRMWModifiedValueLength(ref IGarnetObject value, ref ObjectInput input) + public int GetRMWModifiedValueLength(ref IGarnetObject value, ref ObjectInput input, bool hasEtag) { throw new GarnetException("GetRMWModifiedValueLength is not available on the object store"); } diff --git a/libs/server/Storage/Session/MainStore/MainStoreOps.cs b/libs/server/Storage/Session/MainStore/MainStoreOps.cs index f4f333963c..1d9b6f8f77 100644 --- a/libs/server/Storage/Session/MainStore/MainStoreOps.cs +++ b/libs/server/Storage/Session/MainStore/MainStoreOps.cs @@ -2,6 +2,7 @@ // Licensed under the MIT license. using System; +using System.Buffers; using System.Diagnostics; using System.Runtime.CompilerServices; using Garnet.common; @@ -375,8 +376,37 @@ public unsafe GarnetStatus SET_Conditional(ref SpanByte key, ref RawSt } } - public unsafe GarnetStatus SET_Conditional(ref SpanByte key, ref RawStringInput input, ref SpanByteAndMemory output, ref TContext context) - where TContext : ITsavoriteContext + public unsafe GarnetStatus SET_Conditional(ref SpanByte key, ref SpanByte input, ref SpanByteAndMemory output, ref TContext context) + where TContext : ITsavoriteContext + { + var status = context.RMW(ref key, ref input, ref output); + + if (status.IsPending) + { + StartPendingMetrics(); + CompletePendingForSession(ref status, ref output, ref context); + StopPendingMetrics(); + } + + if (status.NotFound) + { + incr_session_notfound(); + return GarnetStatus.NOTFOUND; + } + else if (cmd == RespCommand.SETIFMATCH && status.IsCanceled) + { + // The RMW operation for SETIFMATCH upon not finding the etags match between the existing record and sent etag returns Cancelled Operation + return GarnetStatus.ETAGMISMATCH; + } + else + { + incr_session_found(); + return GarnetStatus.OK; + } + } + + public unsafe GarnetStatus SET_Conditional(ref SpanByte key, ref SpanByte input, ref SpanByteAndMemory output, ref TContext context, RespCommand cmd) + where TContext : ITsavoriteContext { var status = context.RMW(ref key, ref input, ref output); @@ -392,6 +422,11 @@ public unsafe GarnetStatus SET_Conditional(ref SpanByte key, ref RawSt incr_session_notfound(); return GarnetStatus.NOTFOUND; } + else if (cmd == RespCommand.SETIFMATCH && status.IsCanceled) + { + // The RMW operation for SETIFMATCH upon not finding the etags match between the existing record and sent etag returns Cancelled Operation + return GarnetStatus.ETAGMISMATCH; + } else { incr_session_found(); @@ -595,18 +630,50 @@ private unsafe GarnetStatus RENAME(ArgSlice oldKeySlice, ArgSlice newKeySlice, S { try { - var newKey = newKeySlice.SpanByte; + // GET with etag to find if the key alrady exists, and if it exists then we can check if it also has an etag + var inputSize = sizeof(int) + RespInputHeader.Size; + SpanByte getWithEtagInput = SpanByte.Reinterpret(stackalloc byte[inputSize]); + byte* inputPtr = getWithEtagInput.ToPointer(); + ((RespInputHeader*)inputPtr)->cmd = RespCommand.GETWITHETAG; + ((RespInputHeader*)inputPtr)->flags = 0; - var o = new SpanByteAndMemory(); - var status = GET(ref oldKey, ref input, ref o, ref context); + SpanByteAndMemory etagAndDataOutput = new SpanByteAndMemory(); + GarnetStatus status = GET(ref oldKey, ref getWithEtagInput, ref etagAndDataOutput, ref context); if (status == GarnetStatus.OK) { - Debug.Assert(!o.IsSpanByte); - var memoryHandle = o.Memory.Memory.Pin(); - var ptrVal = (byte*)memoryHandle.Pointer; - RespReadUtils.ReadUnsignedLengthHeader(out var headerLength, ref ptrVal, ptrVal + o.Length); + bool hasEtag = false; + + // since we didn't give the etagAndDataOutput span any memory when creating it, the backend would necessarily have had to allocate heap memory if item is not NOTFOUND + Debug.Assert(!etagAndDataOutput.IsSpanByte); + using MemoryHandle outputMemHandle = etagAndDataOutput.Memory.Memory.Pin(); + byte* outputBufCurr = (byte*)outputMemHandle.Pointer; + byte* end = outputBufCurr + etagAndDataOutput.Length; + + // GETWITHETAG returns an array of two items, etag, and value. + // we need to read past RESP metadata and control sequences to get to etag, and value + RespReadUtils.ReadUnsignedArrayLength(out int numItemInArr, ref outputBufCurr, end); + + Debug.Assert(numItemInArr == 2, "GETWITHETAG output RESP array should be of 2 elements only."); + + // we know a key-val pair does not have an etag if the first element is not null + // if read nil is successful it will point to the ptrVal otherwise we need to re-read past the etag as RESP int64 + byte* startOfEtagPtr = outputBufCurr; + hasEtag = !RespReadUtils.ReadNil(ref outputBufCurr, end, out _); + if (hasEtag) + { + // read past the etag so we can then get to the ptrVal, we don't need specific val of etag, just need to move past it if it exists + bool etagReadingSuccessful = RespReadUtils.Read64Int(out _, ref startOfEtagPtr, end); + Debug.Assert(etagReadingSuccessful, "Etag should have been read succesffuly"); + // now startOfEtagPtr is past the etag and points to value + outputBufCurr = startOfEtagPtr; + } + + // get length of value from the header + RespReadUtils.ReadUnsignedLengthHeader(out int headerLength, ref outputBufCurr, end); + // outputBuf now points to start of value + byte* ptrVal = outputBufCurr; // Find expiration time of the old key var expireSpan = new SpanByteAndMemory(); @@ -623,50 +690,130 @@ private unsafe GarnetStatus RENAME(ArgSlice oldKeySlice, ArgSlice newKeySlice, S input = new RawStringInput(RespCommand.SETEXNX); // If the key has an expiration, set the new key with the expiration - if (expireTimeMs > 0) + if (!hasEtag) { - if (isNX) + if (expireTimeMs > 0) { - // Move payload forward to make space for RespInputHeader and Metadata - parseState.InitializeWithArgument(newValSlice); - input.parseState = parseState; - input.arg1 = DateTimeOffset.UtcNow.Ticks + TimeSpan.FromMilliseconds(expireTimeMs).Ticks; - - var setStatus = SET_Conditional(ref newKey, ref input, ref context); - - // For SET NX `NOTFOUND` means the operation succeeded - result = setStatus == GarnetStatus.NOTFOUND ? 1 : 0; - returnStatus = GarnetStatus.OK; + if (isNX) + { + // Move payload forward to make space for RespInputHeader and Metadata + var setValue = scratchBufferManager.FormatScratch(RespInputHeader.Size + sizeof(long), new ArgSlice(ptrVal, headerLength)); + var setValueSpan = setValue.SpanByte; + var setValuePtr = setValueSpan.ToPointerWithMetadata(); + setValueSpan.ExtraMetadata = DateTimeOffset.UtcNow.Ticks + TimeSpan.FromMilliseconds(expireTimeMs).Ticks; + ((RespInputHeader*)(setValuePtr + sizeof(long)))->cmd = RespCommand.SETEXNX; + ((RespInputHeader*)(setValuePtr + sizeof(long)))->flags = 0; + var newKey = newKeySlice.SpanByte; + var setStatus = SET_Conditional(ref newKey, ref setValueSpan, ref context); + + // For SET NX `NOTFOUND` means the operation succeeded + result = setStatus == GarnetStatus.NOTFOUND ? 1 : 0; + returnStatus = GarnetStatus.OK; + } + else + { + SETEX(newKeySlice, new ArgSlice(ptrVal, headerLength), TimeSpan.FromMilliseconds(expireTimeMs), ref context); + } } - else + else if (expireTimeMs == -1) // Its possible to have expireTimeMs as 0 (Key expired or will be expired now) or -2 (Key does not exist), in those cases we don't SET the new key { - SETEX(newKeySlice, newValSlice, TimeSpan.FromMilliseconds(expireTimeMs), ref context); + if (isNX) + { + // Move payload forward to make space for RespInputHeader + var setValue = scratchBufferManager.FormatScratch(RespInputHeader.Size, new ArgSlice(ptrVal, headerLength)); + var setValueSpan = setValue.SpanByte; + var setValuePtr = setValueSpan.ToPointerWithMetadata(); + ((RespInputHeader*)setValuePtr)->cmd = RespCommand.SETEXNX; + ((RespInputHeader*)setValuePtr)->flags = 0; + var newKey = newKeySlice.SpanByte; + var setStatus = SET_Conditional(ref newKey, ref setValueSpan, ref context); + + // For SET NX `NOTFOUND` means the operation succeeded + result = setStatus == GarnetStatus.NOTFOUND ? 1 : 0; + returnStatus = GarnetStatus.OK; + } + else + { + var value = SpanByte.FromPinnedPointer(ptrVal, headerLength); + SET(ref newKey, ref value, ref context); + } } } - else if (expireTimeMs == -1) // Its possible to have expireTimeMs as 0 (Key expired or will be expired now) or -2 (Key does not exist), in those cases we don't SET the new key + else if ( + (expireTimeMs == -1 || expireTimeMs > 0) && + hasEtag) { - if (isNX) - { - // Build parse state - parseState.InitializeWithArgument(newValSlice); - input.parseState = parseState; + // IsNX means if the newKey Exists do not set it + // We can't use SET with not exist here so instead we will do an Exists and skip the seeting if exists + bool newKeyAlreadyExists = EXISTS(newKeySlice, storeType, ref context, ref objectContext) == GarnetStatus.OK; - var setStatus = SET_Conditional(ref newKey, ref input, ref context); - - // For SET NX `NOTFOUND` means the operation succeeded - result = setStatus == GarnetStatus.NOTFOUND ? 1 : 0; + if (isNX && newKeyAlreadyExists) + { + // Skip setting the new key and go to calling the part after that + result = 0; returnStatus = GarnetStatus.OK; } else { var value = SpanByte.FromPinnedPointer(ptrVal, headerLength); - SET(ref newKey, ref value, ref context); + var initialValueSize = value.Length; + + var valPtr = value.ToPointer(); + + SpanByte key = newKeySlice.SpanByte; + + GarnetStatus setStatus; + if (expireTimeMs == -1) // no expiration provided + { + /* + * Make Space for Resp Input Header Behind The valPtr: + * This will not underflow because SpanByte is being created from pinnned pointer on output buffer that had etag and control sequences behind ptrVal. + * we need 6 bytes behind valPtr that we know exists because even in worst case where we only an etag of 0 (1 byte in resp output buffer), the output + * buffer will be of structure: + * *2\r\n + * :0\r\n + * <_VALUE_>\r\n + * this gives us 6 bytes behind it in pinned memory that we can borrow, and 2 bytes infront of value + */ + valPtr -= RespInputHeader.Size + sizeof(int); + // set the length + *(int*)valPtr = RespInputHeader.Size + value.Length; + ((RespInputHeader*)(valPtr + sizeof(int)))->cmd = RespCommand.SETWITHETAG; + ((RespInputHeader*)(valPtr + sizeof(int)))->flags = 0; + // This handles the edge case where we are renaming to a key that already exists and has an etag we want to retain its existing etag + // if there wasn't already an existing key same as the "rename to" key or without an etag, this will initialize the etag to 0 on the renamed key + ((RespInputHeader*)(valPtr + sizeof(int)))->SetRetainEtagFlag(); + + setStatus = SET_Conditional(ref key, ref Unsafe.AsRef(valPtr), ref context); + } + else + { + // make space for metadata to be added to valPtr + var setValue = scratchBufferManager.FormatScratch(RespInputHeader.Size + sizeof(long), new ArgSlice(ptrVal, headerLength)); + var setValueSpan = setValue.SpanByte; + var setValuePtr = setValueSpan.ToPointerWithMetadata(); + ((RespInputHeader*)setValuePtr + sizeof(long))->cmd = RespCommand.SETWITHETAG; + ((RespInputHeader*)setValuePtr + sizeof(long))->flags = 0; + // This handles the edge case where we are renaming to a key that already exists and has an etag we want to retain its existing etag + // if there wasn't already an existing key same as the "rename to" key or without an etag, this will initialize the etag to 0 on the renamed key + ((RespInputHeader*)setValuePtr)->SetRetainEtagFlag(); + + setValueSpan.ExtraMetadata = DateTimeOffset.UtcNow.Ticks + TimeSpan.FromMilliseconds(expireTimeMs).Ticks; + + setStatus = SET_Conditional(ref key, ref setValueSpan, ref context); + } + + + // isNx or/and new key does not exist, either way we can set result to 1 result var will only get used if !isNx, and otherwise is ignored. + // Setting result regardless will avoid the need to add branching here + // For SET NX `NOTFOUND` means the operation succeeded + result = setStatus == GarnetStatus.NOTFOUND ? 1 : 0; + + returnStatus = GarnetStatus.OK; } } - + etagAndDataOutput.Memory.Dispose(); expireSpan.Memory.Dispose(); - memoryHandle.Dispose(); - o.Memory.Dispose(); // Delete the old key only when SET NX succeeded if (isNX && result == 1) diff --git a/libs/server/Transaction/TxnKeyManager.cs b/libs/server/Transaction/TxnKeyManager.cs index 54f6cff259..bee1b57606 100644 --- a/libs/server/Transaction/TxnKeyManager.cs +++ b/libs/server/Transaction/TxnKeyManager.cs @@ -139,7 +139,11 @@ internal int GetKeys(RespCommand command, int inputCount, out ReadOnlySpan RespCommand.HSTRLEN => HashObjectKeys((byte)HashOperation.HSTRLEN), RespCommand.HVALS => HashObjectKeys((byte)HashOperation.HVALS), RespCommand.GET => SingleKey(1, false, LockType.Shared), + RespCommand.GETIFNOTMATCH => SingleKey(1, false, LockType.Shared), + RespCommand.GETWITHETAG => SingleKey(1, false, LockType.Shared), RespCommand.SET => SingleKey(1, false, LockType.Exclusive), + RespCommand.SETWITHETAG => SingleKey(1, false, LockType.Exclusive), + RespCommand.SETIFMATCH => SingleKey(1, false, LockType.Exclusive), RespCommand.GETRANGE => SingleKey(1, false, LockType.Shared), RespCommand.SETRANGE => SingleKey(1, false, LockType.Exclusive), RespCommand.PFADD => SingleKey(1, false, LockType.Exclusive), diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/AllocatorScan.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/AllocatorScan.cs index 213b67956c..6249b1db13 100644 --- a/libs/storage/Tsavorite/cs/src/core/Allocator/AllocatorScan.cs +++ b/libs/storage/Tsavorite/cs/src/core/Allocator/AllocatorScan.cs @@ -334,7 +334,7 @@ public void PostInitialUpdater(ref TKey key, ref TInput input, ref TValue value, public void RMWCompletionCallback(ref TKey key, ref TInput input, ref TOutput output, Empty ctx, Status status, RecordMetadata recordMetadata) { } - public int GetRMWModifiedValueLength(ref TValue value, ref TInput input) => 0; + public int GetRMWModifiedValueLength(ref TValue value, ref TInput input, bool hasEtag) => 0; public int GetRMWInitialValueLength(ref TInput input) => 0; public int GetUpsertValueLength(ref TValue value, ref TInput input) => 0; diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/SpanByteAllocatorImpl.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/SpanByteAllocatorImpl.cs index e25ac476a7..a4f141a23e 100644 --- a/libs/storage/Tsavorite/cs/src/core/Allocator/SpanByteAllocatorImpl.cs +++ b/libs/storage/Tsavorite/cs/src/core/Allocator/SpanByteAllocatorImpl.cs @@ -127,7 +127,7 @@ public ref SpanByte GetAndInitializeValue(long physicalAddress, long endAddress) { // Used by RMW to determine the length of copy destination (taking Input into account), so does not need to get filler length. var keySize = key.TotalSize; - var size = RecordInfo.GetLength() + RoundUp(keySize, Constants.kRecordAlignment) + varlenInput.GetRMWModifiedValueLength(ref value, ref input); + var size = RecordInfo.GetLength() + RoundUp(keySize, Constants.kRecordAlignment) + varlenInput.GetRMWModifiedValueLength(ref value, ref input, recordInfo.ETag); return (size, RoundUp(size, Constants.kRecordAlignment), keySize); } diff --git a/libs/storage/Tsavorite/cs/src/core/ClientSession/SessionFunctionsWrapper.cs b/libs/storage/Tsavorite/cs/src/core/ClientSession/SessionFunctionsWrapper.cs index 9f3eb13825..bd8f2a38bb 100644 --- a/libs/storage/Tsavorite/cs/src/core/ClientSession/SessionFunctionsWrapper.cs +++ b/libs/storage/Tsavorite/cs/src/core/ClientSession/SessionFunctionsWrapper.cs @@ -195,7 +195,7 @@ public void UnlockTransientShared(ref TKey key, ref OperationStackContext _clientSession.functions.GetRMWInitialValueLength(ref input); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public int GetRMWModifiedValueLength(ref TValue t, ref TInput input) => _clientSession.functions.GetRMWModifiedValueLength(ref t, ref input); + public int GetRMWModifiedValueLength(ref TValue t, ref TInput input, bool hasEtag) => _clientSession.functions.GetRMWModifiedValueLength(ref t, ref input, hasEtag); [MethodImpl(MethodImplOptions.AggressiveInlining)] public int GetUpsertValueLength(ref TValue t, ref TInput input) => _clientSession.functions.GetUpsertValueLength(ref t, ref input); diff --git a/libs/storage/Tsavorite/cs/src/core/Compaction/LogCompactionFunctions.cs b/libs/storage/Tsavorite/cs/src/core/Compaction/LogCompactionFunctions.cs index 4c7e79f8cf..2d7ddee19a 100644 --- a/libs/storage/Tsavorite/cs/src/core/Compaction/LogCompactionFunctions.cs +++ b/libs/storage/Tsavorite/cs/src/core/Compaction/LogCompactionFunctions.cs @@ -50,7 +50,7 @@ public void ReadCompletionCallback(ref TKey key, ref TInput input, ref TOutput o public void RMWCompletionCallback(ref TKey key, ref TInput input, ref TOutput output, TContext ctx, Status status, RecordMetadata recordMetadata) { } - public int GetRMWModifiedValueLength(ref TValue value, ref TInput input) => 0; + public int GetRMWModifiedValueLength(ref TValue value, ref TInput input, bool hasEtag) => 0; public int GetRMWInitialValueLength(ref TInput input) => 0; public int GetUpsertValueLength(ref TValue value, ref TInput input) => _functions.GetUpsertValueLength(ref value, ref input); diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Common/RecordInfo.cs b/libs/storage/Tsavorite/cs/src/core/Index/Common/RecordInfo.cs index e2176dcbc7..5d82c473f5 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Common/RecordInfo.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Common/RecordInfo.cs @@ -11,7 +11,7 @@ namespace Tsavorite.core { // RecordInfo layout (64 bits total): - // [Unused1][Modified][InNewVersion][Filler][Dirty][Unused2][Sealed][Valid][Tombstone][LLLLLLL] [RAAAAAAA] [AAAAAAAA] [AAAAAAAA] [AAAAAAAA] [AAAAAAAA] [AAAAAAAA] + // [Unused1][Modified][InNewVersion][Filler][Dirty][ETag][Sealed][Valid][Tombstone][LLLLLLL] [RAAAAAAA] [AAAAAAAA] [AAAAAAAA] [AAAAAAAA] [AAAAAAAA] [AAAAAAAA] // where L = leftover, R = readcache, A = address [StructLayout(LayoutKind.Explicit, Size = 8)] public struct RecordInfo @@ -30,8 +30,8 @@ public struct RecordInfo const int kTombstoneBitOffset = kPreviousAddressBits + kLeftoverBitCount; const int kValidBitOffset = kTombstoneBitOffset + 1; const int kSealedBitOffset = kValidBitOffset + 1; - const int kUnused2BitOffset = kSealedBitOffset + 1; - const int kDirtyBitOffset = kUnused2BitOffset + 1; + const int kEtagBitOffset = kSealedBitOffset + 1; + const int kDirtyBitOffset = kEtagBitOffset + 1; const int kFillerBitOffset = kDirtyBitOffset + 1; const int kInNewVersionBitOffset = kFillerBitOffset + 1; const int kModifiedBitOffset = kInNewVersionBitOffset + 1; @@ -40,7 +40,7 @@ public struct RecordInfo const long kTombstoneBitMask = 1L << kTombstoneBitOffset; const long kValidBitMask = 1L << kValidBitOffset; const long kSealedBitMask = 1L << kSealedBitOffset; - const long kUnused2BitMask = 1L << kUnused2BitOffset; + const long kETagBitMask = 1L << kEtagBitOffset; const long kDirtyBitMask = 1L << kDirtyBitOffset; const long kFillerBitMask = 1L << kFillerBitOffset; const long kInNewVersionBitMask = 1L << kInNewVersionBitOffset; @@ -275,18 +275,21 @@ internal bool Unused1 set => word = value ? word | kUnused1BitMask : word & ~kUnused1BitMask; } - internal bool Unused2 + public bool ETag { - readonly get => (word & kUnused2BitMask) != 0; - set => word = value ? word | kUnused2BitMask : word & ~kUnused2BitMask; + readonly get => (word & kETagBitMask) != 0; + set => word = value ? word | kETagBitMask : word & ~kETagBitMask; } + public void SetHasETag() => word |= kETagBitMask; + public void ClearHasETag() => word &= ~kETagBitMask; + public override readonly string ToString() { var paRC = IsReadCache(PreviousAddress) ? "(rc)" : string.Empty; static string bstr(bool value) => value ? "T" : "F"; return $"prev {AbsoluteAddress(PreviousAddress)}{paRC}, valid {bstr(Valid)}, tomb {bstr(Tombstone)}, seal {bstr(IsSealed)}," - + $" mod {bstr(Modified)}, dirty {bstr(Dirty)}, fill {bstr(HasFiller)}, Un1 {bstr(Unused1)}, Un2 {bstr(Unused2)}"; + + $" mod {bstr(Modified)}, dirty {bstr(Dirty)}, fill {bstr(HasFiller)}, etag {bstr(ETag)}, Un1 {bstr(Unused1)}"; } } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Interfaces/ISessionFunctions.cs b/libs/storage/Tsavorite/cs/src/core/Index/Interfaces/ISessionFunctions.cs index f9c3990984..18c01c4c17 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Interfaces/ISessionFunctions.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Interfaces/ISessionFunctions.cs @@ -183,7 +183,7 @@ public interface ISessionFunctions /// /// Length of resulting value object when performing RMW modification of value using given input /// - int GetRMWModifiedValueLength(ref TValue value, ref TInput input); + int GetRMWModifiedValueLength(ref TValue value, ref TInput input, bool hasEtag); /// /// Initial expected length of value object when populated by RMW using given input diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Interfaces/SessionFunctionsBase.cs b/libs/storage/Tsavorite/cs/src/core/Index/Interfaces/SessionFunctionsBase.cs index 2d222e4b33..77ee2dbf83 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Interfaces/SessionFunctionsBase.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Interfaces/SessionFunctionsBase.cs @@ -54,7 +54,7 @@ public virtual void ReadCompletionCallback(ref TKey key, ref TInput input, ref T public virtual void RMWCompletionCallback(ref TKey key, ref TInput input, ref TOutput output, TContext ctx, Status status, RecordMetadata recordMetadata) { } /// - public virtual int GetRMWModifiedValueLength(ref TValue value, ref TInput input) => throw new TsavoriteException("GetRMWModifiedValueLength is only available for SpanByte Functions"); + public virtual int GetRMWModifiedValueLength(ref TValue value, ref TInput input, bool hasEtag) => throw new TsavoriteException("GetRMWModifiedValueLength is only available for SpanByte Functions"); /// public virtual int GetRMWInitialValueLength(ref TInput input) => throw new TsavoriteException("GetRMWInitialValueLength is only available for SpanByte Functions"); /// diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/InternalRMW.cs b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/InternalRMW.cs index 0652f03c90..77f01fb663 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/InternalRMW.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/InternalRMW.cs @@ -443,6 +443,9 @@ private OperationStatus CreateNewRecordRMW= hlogBase.ReadOnlyAddress) { srcRecordInfo = ref stackCtx.recSrc.GetInfo(); + srcRecordInfo.ClearHasETag(); // Mutable Region: Update the record in-place. We perform mutable updates only if we are in normal processing phase of checkpointing UpsertInfo upsertInfo = new() diff --git a/libs/storage/Tsavorite/cs/src/core/Utilities/Status.cs b/libs/storage/Tsavorite/cs/src/core/Utilities/Status.cs index c5acc553fc..3ecf39eca4 100644 --- a/libs/storage/Tsavorite/cs/src/core/Utilities/Status.cs +++ b/libs/storage/Tsavorite/cs/src/core/Utilities/Status.cs @@ -133,6 +133,11 @@ public bool IsCompletedSuccessfully /// public byte Value => (byte)statusCode; + /// + /// Whther the operation performed an update on the record or not + /// + public bool IsUpdated => Record.InPlaceUpdated || Record.CopyUpdated; + /// /// "Found" is zero, so does not appear in the output by default; this handles that explicitly public override string ToString() => (Found ? "Found, " : string.Empty) + statusCode.ToString(); diff --git a/libs/storage/Tsavorite/cs/src/core/VarLen/IVariableLengthInput.cs b/libs/storage/Tsavorite/cs/src/core/VarLen/IVariableLengthInput.cs index 121a051d53..d67f5f5d48 100644 --- a/libs/storage/Tsavorite/cs/src/core/VarLen/IVariableLengthInput.cs +++ b/libs/storage/Tsavorite/cs/src/core/VarLen/IVariableLengthInput.cs @@ -11,7 +11,7 @@ public interface IVariableLengthInput /// /// Length of resulting value object when performing RMW modification of value using given input /// - int GetRMWModifiedValueLength(ref TValue value, ref TInput input); + int GetRMWModifiedValueLength(ref TValue value, ref TInput input, bool hasEtag); /// /// Initial expected length of value object when populated by RMW using given input diff --git a/libs/storage/Tsavorite/cs/src/core/VarLen/SpanByte.cs b/libs/storage/Tsavorite/cs/src/core/VarLen/SpanByte.cs index adfd7eda32..051d59630c 100644 --- a/libs/storage/Tsavorite/cs/src/core/VarLen/SpanByte.cs +++ b/libs/storage/Tsavorite/cs/src/core/VarLen/SpanByte.cs @@ -175,26 +175,32 @@ public bool Invalid /// /// Get Span<byte> for this 's payload (excluding metadata if any) + /// + /// Optional Parameter to avoid having to call slice when wanting to interact directly with payload skipping ETag at the front of the payload + /// /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Span AsSpan() + public Span AsSpan(int offset = 0) { if (Serialized) - return new Span(MetadataSize + (byte*)Unsafe.AsPointer(ref payload), Length - MetadataSize); + return new Span(MetadataSize + (byte*)Unsafe.AsPointer(ref payload) + offset, Length - MetadataSize - offset); else - return new Span(MetadataSize + (byte*)payload, Length - MetadataSize); + return new Span(MetadataSize + (byte*)payload + offset, Length - MetadataSize - offset); } /// /// Get ReadOnlySpan<byte> for this 's payload (excluding metadata if any) + /// + /// Optional Parameter to avoid having to call slice when wanting to interact directly with payload skipping ETag at the front of the payload + /// /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public ReadOnlySpan AsReadOnlySpan() + public ReadOnlySpan AsReadOnlySpan(int offset = 0) { if (Serialized) - return new ReadOnlySpan(MetadataSize + (byte*)Unsafe.AsPointer(ref payload), Length - MetadataSize); + return new ReadOnlySpan(MetadataSize + (byte*)Unsafe.AsPointer(ref payload) + offset, Length - MetadataSize - offset); else - return new ReadOnlySpan(MetadataSize + (byte*)payload, Length - MetadataSize); + return new ReadOnlySpan(MetadataSize + (byte*)payload + offset, Length - MetadataSize - offset); } /// diff --git a/libs/storage/Tsavorite/cs/src/core/VarLen/SpanByteFunctions.cs b/libs/storage/Tsavorite/cs/src/core/VarLen/SpanByteFunctions.cs index d9a8625b3e..dec7b6ae12 100644 --- a/libs/storage/Tsavorite/cs/src/core/VarLen/SpanByteFunctions.cs +++ b/libs/storage/Tsavorite/cs/src/core/VarLen/SpanByteFunctions.cs @@ -119,7 +119,7 @@ public override bool InPlaceUpdater(ref SpanByte key, ref SpanByte input, ref Sp /// Length of resulting object when doing RMW with given value and input. Here we set the length /// to the max of input and old value lengths. You can provide a custom implementation for other cases. /// - public override int GetRMWModifiedValueLength(ref SpanByte t, ref SpanByte input) + public override int GetRMWModifiedValueLength(ref SpanByte t, ref SpanByte input, bool hasEtag) => sizeof(int) + (t.Length > input.Length ? t.Length : input.Length); /// diff --git a/libs/storage/Tsavorite/cs/test/ExpirationTests.cs b/libs/storage/Tsavorite/cs/test/ExpirationTests.cs index 04adfc9bae..0df92a2656 100644 --- a/libs/storage/Tsavorite/cs/test/ExpirationTests.cs +++ b/libs/storage/Tsavorite/cs/test/ExpirationTests.cs @@ -470,7 +470,7 @@ public override void ReadCompletionCallback(ref SpanByte key, ref ExpirationInpu } /// - public override int GetRMWModifiedValueLength(ref SpanByte value, ref ExpirationInput input) => value.TotalSize; + public override int GetRMWModifiedValueLength(ref SpanByte value, ref ExpirationInput input, bool hasEtag) => value.TotalSize; /// public override int GetRMWInitialValueLength(ref ExpirationInput input) => MinValueLen; diff --git a/libs/storage/Tsavorite/cs/test/RevivificationTests.cs b/libs/storage/Tsavorite/cs/test/RevivificationTests.cs index 4ec87cdb82..d10cd125bc 100644 --- a/libs/storage/Tsavorite/cs/test/RevivificationTests.cs +++ b/libs/storage/Tsavorite/cs/test/RevivificationTests.cs @@ -576,7 +576,7 @@ public override bool InPlaceUpdater(ref SpanByte key, ref SpanByte input, ref Sp } // Override the default SpanByteFunctions impelementation; for these tests, we always want the input length. - public override int GetRMWModifiedValueLength(ref SpanByte value, ref SpanByte input) => input.TotalSize; + public override int GetRMWModifiedValueLength(ref SpanByte value, ref SpanByte input, bool hasEtag) => input.TotalSize; public override bool SingleDeleter(ref SpanByte key, ref SpanByte value, ref DeleteInfo deleteInfo, ref RecordInfo recordInfo) { diff --git a/playground/CommandInfoUpdater/SupportedCommand.cs b/playground/CommandInfoUpdater/SupportedCommand.cs index 75acf2464b..24afca592d 100644 --- a/playground/CommandInfoUpdater/SupportedCommand.cs +++ b/playground/CommandInfoUpdater/SupportedCommand.cs @@ -129,7 +129,9 @@ public class SupportedCommand new("GETEX", RespCommand.GETEX), new("GETBIT", RespCommand.GETBIT), new("GETDEL", RespCommand.GETDEL), + new("GETIFNOTMATCH", RespCommand.GETIFNOTMATCH), new("GETRANGE", RespCommand.GETRANGE), + new("GETWITHETAG", RespCommand.GETWITHETAG), new("GETSET", RespCommand.GETSET), new("HDEL", RespCommand.HDEL), new("HELLO", RespCommand.HELLO), @@ -230,8 +232,10 @@ public class SupportedCommand new("SET", RespCommand.SET), new("SETBIT", RespCommand.SETBIT), new("SETEX", RespCommand.SETEX), + new("SETIFMATCH", RespCommand.SETIFMATCH), new("SETNX", RespCommand.SETNX), new("SETRANGE", RespCommand.SETRANGE), + new("SETWITHETAG", RespCommand.SETWITHETAG), new("SISMEMBER", RespCommand.SISMEMBER), new("SLAVEOF", RespCommand.SECONDARYOF), new("SMEMBERS", RespCommand.SMEMBERS), diff --git a/test/Garnet.test/GarnetBitmapTests.cs b/test/Garnet.test/GarnetBitmapTests.cs index c1281a26d4..3e24274a1e 100644 --- a/test/Garnet.test/GarnetBitmapTests.cs +++ b/test/Garnet.test/GarnetBitmapTests.cs @@ -165,11 +165,6 @@ public void BitmapSimpleSetGet_PCT(int bytesPerSend) public void BitmapSetGetBitTest_LTM(bool preSet) { int bitmapBytes = 512; - server.Dispose(); - server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, - lowMemory: true, - MemorySize: (bitmapBytes << 2).ToString(), - PageSize: (bitmapBytes << 1).ToString()); server.Start(); using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); var db = redis.GetDatabase(0); @@ -677,6 +672,28 @@ public void BitmapBitPosTest_LTM() } } + [Test] + [Category("BITPOS")] + public void BitmapBitPosTest_BoundaryConditions() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + const int bitmapSize = 24; + byte[] bitmap = new byte[bitmapSize]; + + string key = "mybitmap"; + ClassicAssert.IsTrue(db.StringSet(key, bitmap)); + + // first unset bit, should increment + for (int i = 0; i < bitmapSize; i ++) + { + // first unset bit + ClassicAssert.AreEqual(i, db.StringBitPosition(key, false)); + ClassicAssert.IsFalse(db.StringSetBit(key, i, true)); + } + } + [Test, Order(15)] [TestCase(10)] [TestCase(20)] diff --git a/test/Garnet.test/Resp/ACL/RespCommandTests.cs b/test/Garnet.test/Resp/ACL/RespCommandTests.cs index 25d8193a82..f4942444db 100644 --- a/test/Garnet.test/Resp/ACL/RespCommandTests.cs +++ b/test/Garnet.test/Resp/ACL/RespCommandTests.cs @@ -4902,6 +4902,66 @@ static async Task DoSetKeepTtlXxAsync(GarnetClient client) } } + [Test] + public async Task SetWithEtagACLsAsync() + { + await CheckCommandsAsync( + "SETWITHETAG", + [DoSetWithEtagAsync] + ); + + static async Task DoSetWithEtagAsync(GarnetClient client) + { + long val = await client.ExecuteForLongResultAsync("SETWITHETAG", ["foo", "bar"]); + ClassicAssert.AreEqual(0, val); + } + } + + [Test] + public async Task SetIfMatchACLsAsync() + { + await CheckCommandsAsync( + "SETIFMATCH", + [DoSetIfMatchAsync] + ); + + static async Task DoSetIfMatchAsync(GarnetClient client) + { + var res = await client.ExecuteForStringResultAsync("SETIFMATCH", ["foo", "rizz", "0"]); + ClassicAssert.IsNull(res); + } + } + + [Test] + public async Task GetIfNotMatchACLsAsync() + { + await CheckCommandsAsync( + "GETIFNOTMATCH", + [DoGetIfNotMatchAsync] + ); + + static async Task DoGetIfNotMatchAsync(GarnetClient client) + { + var res = await client.ExecuteForStringResultAsync("GETIFNOTMATCH", ["foo", "0"]); + ClassicAssert.IsNull(res); + } + } + + [Test] + public async Task GetWithEtagACLsAsync() + { + await CheckCommandsAsync( + "GETWITHETAG", + [DoGetWithEtagAsync] + ); + + static async Task DoGetWithEtagAsync(GarnetClient client) + { + var res = await client.ExecuteForStringResultAsync("GETWITHETAG", ["foo"]); + ClassicAssert.IsNull(res); + } + } + [Test] public async Task SetBitACLsAsync() { diff --git a/test/Garnet.test/Resp/RespReadUtilsTests.cs b/test/Garnet.test/Resp/RespReadUtilsTests.cs index 7860dd7bc2..c0a576d7c3 100644 --- a/test/Garnet.test/Resp/RespReadUtilsTests.cs +++ b/test/Garnet.test/Resp/RespReadUtilsTests.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +using System; using System.Text; using Garnet.common; using Garnet.common.Parsing; @@ -288,5 +289,28 @@ public static unsafe void ReadBoolWithLengthHeaderTest(string text, bool expecte ClassicAssert.IsTrue(start == end); } } + + /// + /// Tests that Readnil successfully parses valid inputs. + /// + [TestCase("", false, null)] // Too short + [TestCase("S$-1\r\n", false, "S")] // Long enough but not nil leading + [TestCase("$-1\n1738\r\n", false, "\n")] // Long enough but not nil + [TestCase("$-1\r\n", true, null)] // exact nil + [TestCase("$-1\r\nxyzextra", true, null)] // leading nil but with extra bytes after + public static unsafe void ReadNilTest(string testSequence, bool expected, string firstMismatch) + { + ReadOnlySpan testSeq = new ReadOnlySpan(Encoding.ASCII.GetBytes(testSequence)); + + fixed (byte* ptr = testSeq) + { + byte* start = ptr; + byte* end = ptr + testSeq.Length; + var isNil = RespReadUtils.ReadNil(ref start, end, out byte? unexpectedToken); + + ClassicAssert.AreEqual(expected, isNil); + ClassicAssert.AreEqual((byte?)firstMismatch?[0], unexpectedToken); + } + } } } \ No newline at end of file diff --git a/test/Garnet.test/RespAofTests.cs b/test/Garnet.test/RespAofTests.cs index c32e78bd81..92d056c627 100644 --- a/test/Garnet.test/RespAofTests.cs +++ b/test/Garnet.test/RespAofTests.cs @@ -229,6 +229,8 @@ public void AofRMWStoreRecoverTest() var db = redis.GetDatabase(0); db.StringSet("SeAofUpsertRecoverTestKey1", "SeAofUpsertRecoverTestValue1", expiry: TimeSpan.FromDays(1), when: When.NotExists); db.StringSet("SeAofUpsertRecoverTestKey2", "SeAofUpsertRecoverTestValue2", expiry: TimeSpan.FromDays(1), when: When.NotExists); + db.Execute("SETWITHETAG", "SeAofUpsertRecoverTestKey3", "SeAofUpsertRecoverTestValue3"); + db.Execute("SETIFMATCH", "SeAofUpsertRecoverTestKey3", "UpdatedSeAofUpsertRecoverTestValue3", "0"); } server.Store.CommitAOF(true); @@ -243,6 +245,7 @@ public void AofRMWStoreRecoverTest() ClassicAssert.AreEqual("SeAofUpsertRecoverTestValue1", recoveredValue.ToString()); recoveredValue = db.StringGet("SeAofUpsertRecoverTestKey2"); ClassicAssert.AreEqual("SeAofUpsertRecoverTestValue2", recoveredValue.ToString()); + ExpectedEtagTest(db, "SeAofUpsertRecoverTestKey3", "UpdatedSeAofUpsertRecoverTestValue3", 1); } } @@ -689,5 +692,26 @@ public void AofCustomTxnRecoverTest() ClassicAssert.AreEqual(readVal, writeKeysVal2); } } + + private static void ExpectedEtagTest(IDatabase db, string key, string expectedValue, long expected) + { + RedisResult res = db.Execute("GETWITHETAG", key); + if (expectedValue == null) + { + ClassicAssert.IsTrue(res.IsNull); + return; + } + + RedisResult[] etagAndVal = (RedisResult[])res; + RedisResult etag = etagAndVal[0]; + RedisResult val = etagAndVal[1]; + + if (expected == -1) + { + ClassicAssert.IsTrue(etag.IsNull); + } + + ClassicAssert.AreEqual(expectedValue, val.ToString()); + } } } \ No newline at end of file diff --git a/test/Garnet.test/RespEtagTests.cs b/test/Garnet.test/RespEtagTests.cs new file mode 100644 index 0000000000..6eaa82f1d2 --- /dev/null +++ b/test/Garnet.test/RespEtagTests.cs @@ -0,0 +1,2122 @@ + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Garnet.server; +using NUnit.Framework; +using NUnit.Framework.Legacy; +using StackExchange.Redis; + +namespace Garnet.test +{ + [TestFixture] + public class RespEtagTests + { + private GarnetServer server; + private Random r; + + [SetUp] + public void Setup() + { + r = new Random(674386); + TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, disablePubSub: false); + server.Start(); + } + + [TearDown] + public void TearDown() + { + server.Dispose(); + TestUtils.DeleteDirectory(TestUtils.MethodTestDir); + } + + #region ETAG SET Happy Paths + + [Test] + public void SetWithEtagReturnsEtagForNewData() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + IDatabase db = redis.GetDatabase(0); + RedisResult res = db.Execute("SETWITHETAG", ["rizz", "buzz"]); + long etag = long.Parse(res.ToString()); + ClassicAssert.AreEqual(0, etag); + } + + [Test] + public void SetIfMatchReturnsNewValueAndEtagWhenEtagMatches() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + IDatabase db = redis.GetDatabase(0); + + var key = "florida"; + RedisResult res = (RedisResult)db.Execute("SETWITHETAG", [key, "one"]); + long initalEtag = long.Parse(res.ToString()); + ClassicAssert.AreEqual(0, initalEtag); + + // ETAGMISMATCH test + var incorrectEtag = 1738; + RedisResult etagMismatchMsg = db.Execute("SETIFMATCH", [key, "nextone", incorrectEtag]); + ClassicAssert.AreEqual("ETAGMISMATCH", etagMismatchMsg.ToString()); + + // set a bigger val + RedisResult[] setIfMatchRes = (RedisResult[])db.Execute("SETIFMATCH", [key, "nextone", initalEtag]); + + long nextEtag = long.Parse(setIfMatchRes[0].ToString()); + string value = setIfMatchRes[1].ToString(); + + ClassicAssert.AreEqual(1, nextEtag); + ClassicAssert.AreEqual(value, "nextone"); + + // set a bigger val + setIfMatchRes = (RedisResult[])db.Execute("SETIFMATCH", [key, "nextnextone", nextEtag]); + nextEtag = long.Parse(setIfMatchRes[0].ToString()); + value = setIfMatchRes[1].ToString(); + + ClassicAssert.AreEqual(2, nextEtag); + ClassicAssert.AreEqual(value, "nextnextone"); + + // ETAGMISMATCH again + etagMismatchMsg = db.Execute("SETIFMATCH", [key, "lastOne", incorrectEtag]); + ClassicAssert.AreEqual("ETAGMISMATCH", etagMismatchMsg.ToString()); + + // set a smaller val + setIfMatchRes = (RedisResult[])db.Execute("SETIFMATCH", [key, "lastOne", nextEtag]); + nextEtag = long.Parse(setIfMatchRes[0].ToString()); + value = setIfMatchRes[1].ToString(); + + ClassicAssert.AreEqual(3, nextEtag); + ClassicAssert.AreEqual(value, "lastOne"); + } + + #endregion + + #region ETAG GET Happy Paths + + [Test] + public void GetWithEtagReturnsValAndEtagForKey() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + IDatabase db = redis.GetDatabase(0); + + var key = "florida"; + // Data that does not exist returns nil + RedisResult nonExistingData = db.Execute("GETWITHETAG", [key]); + ClassicAssert.IsTrue(nonExistingData.IsNull); + + // insert data + var initEtag = db.Execute("SETWITHETAG", [key, "hkhalid"]); + ClassicAssert.AreEqual(0, long.Parse(initEtag.ToString())); + + RedisResult[] res = (RedisResult[])db.Execute("GETWITHETAG", [key]); + long etag = long.Parse(res[0].ToString()); + string value = res[1].ToString(); + + ClassicAssert.AreEqual(0, etag); + ClassicAssert.AreEqual("hkhalid", value); + } + + [Test] + public void GetIfNotMatchReturnsDataWhenEtagDoesNotMatch() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + IDatabase db = redis.GetDatabase(0); + + var key = "florida"; + // GetIfNotMatch on non-existing data will return null + RedisResult nonExistingData = db.Execute("GETIFNOTMATCH", [key, 0]); + ClassicAssert.IsTrue(nonExistingData.IsNull); + + // insert data + var _ = db.Execute("SETWITHETAG", [key, "maximus"]); + + RedisResult noDataOnMatch = db.Execute("GETIFNOTMATCH", [key, 0]); + + ClassicAssert.AreEqual("NOTCHANGED", noDataOnMatch.ToString()); + + RedisResult[] res = (RedisResult[])db.Execute("GETIFNOTMATCH", [key, 1]); + long etag = long.Parse(res[0].ToString()); + string value = res[1].ToString(); + + ClassicAssert.AreEqual(0, etag); + ClassicAssert.AreEqual("maximus", value); + } + + #endregion + + # region Edgecases + + [Test] + public void SetWithEtagOnAlreadyExistingSetWithEtagDataOverridesItWithInitialEtag() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + IDatabase db = redis.GetDatabase(0); + + RedisResult res = db.Execute("SETWITHETAG", ["rizz", "buzz"]); + long etag = (long)res; + ClassicAssert.AreEqual(0, etag); + + // update to value to update the etag + RedisResult[] updateRes = (RedisResult[])db.Execute("SETIFMATCH", ["rizz", "fixx", etag.ToString()]); + etag = (long)updateRes[0]; + ClassicAssert.AreEqual(1, etag); + ClassicAssert.AreEqual("fixx", updateRes[1].ToString()); + + // inplace update + res = db.Execute("SETWITHETAG", ["rizz", "meow"]); + etag = (long)res; + ClassicAssert.AreEqual(0, etag); + + // update to value to update the etag + updateRes = (RedisResult[])db.Execute("SETIFMATCH", ["rizz", "fooo", etag.ToString()]); + etag = (long)updateRes[0]; + ClassicAssert.AreEqual(1, etag); + ClassicAssert.AreEqual("fooo", updateRes[1].ToString()); + + // Copy update + res = db.Execute("SETWITHETAG", ["rizz", "oneofus"]); + etag = (long)res; + ClassicAssert.AreEqual(0, etag); + } + + [Test] + public void SetWithEtagWithRetainEtagOnAlreadyExistingSetWithEtagDataOverridesItButUpdatesEtag() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + IDatabase db = redis.GetDatabase(0); + + RedisResult res = db.Execute("SETWITHETAG", ["rizz", "buzz", "RETAINETAG"]); + long etag = (long)res; + ClassicAssert.AreEqual(0, etag); + + // update to value to update the etag + RedisResult[] updateRes = (RedisResult[])db.Execute("SETIFMATCH", ["rizz", "fixx", etag.ToString()]); + etag = (long)updateRes[0]; + ClassicAssert.AreEqual(1, etag); + ClassicAssert.AreEqual("fixx", updateRes[1].ToString()); + + // inplace update + res = db.Execute("SETWITHETAG", ["rizz", "meow", "RETAINETAG"]); + etag = (long)res; + ClassicAssert.AreEqual(2, etag); + + // update to value to update the etag + updateRes = (RedisResult[])db.Execute("SETIFMATCH", ["rizz", "fooo", etag.ToString()]); + etag = (long)updateRes[0]; + ClassicAssert.AreEqual(3, etag); + ClassicAssert.AreEqual("fooo", updateRes[1].ToString()); + + // Copy update + res = db.Execute("SETWITHETAG", ["rizz", "oneofus", "RETAINETAG"]); + etag = (long)res; + ClassicAssert.AreEqual(4, etag); + } + + [Test] + public void SetWithEtagWithRetainEtagOnAlreadyExistingNonEtagDataOverridesItToInitialEtag() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + IDatabase db = redis.GetDatabase(0); + + ClassicAssert.IsTrue(db.StringSet("rizz", "used")); + + // inplace update + RedisResult res = db.Execute("SETWITHETAG", ["rizz", "buzz", "RETAINETAG"]); + long etag = long.Parse(res.ToString()); + ClassicAssert.AreEqual(0, etag); + + db.KeyDelete("rizz"); + + ClassicAssert.IsTrue(db.StringSet("rizz", "my")); + + // Copy update + res = db.Execute("SETWITHETAG", ["rizz", "some", "RETAINETAG"]); + etag = long.Parse(res.ToString()); + ClassicAssert.AreEqual(0, etag); + } + + #endregion + + #region ETAG Apis with non-etag data + + [Test] + public void SetWithEtagOnAlreadyExistingNonEtagDataOverridesIt() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + IDatabase db = redis.GetDatabase(0); + + ClassicAssert.IsTrue(db.StringSet("rizz", "used")); + + // inplace update + RedisResult res = db.Execute("SETWITHETAG", ["rizz", "buzz"]); + long etag = long.Parse(res.ToString()); + ClassicAssert.AreEqual(0, etag); + + db.KeyDelete("rizz"); + + ClassicAssert.IsTrue(db.StringSet("rizz", "my")); + + // Copy update + res = db.Execute("SETWITHETAG", ["rizz", "some"]); + etag = long.Parse(res.ToString()); + ClassicAssert.AreEqual(0, etag); + } + + [Test] + public void SetWithEtagWithRetainEtagOnAlreadyExistingNonEtagDataOverridesIt() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + IDatabase db = redis.GetDatabase(0); + + ClassicAssert.IsTrue(db.StringSet("rizz", "used")); + + // inplace update + RedisResult res = db.Execute("SETWITHETAG", ["rizz", "buzz"]); + long etag = long.Parse(res.ToString()); + ClassicAssert.AreEqual(0, etag); + + db.KeyDelete("rizz"); + + ClassicAssert.IsTrue(db.StringSet("rizz", "my")); + + // Copy update + res = db.Execute("SETWITHETAG", ["rizz", "some"]); + etag = long.Parse(res.ToString()); + ClassicAssert.AreEqual(0, etag); + } + + + [Test] + public void SetIfMatchOnNonEtagDataReturnsEtagMismatch() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + IDatabase db = redis.GetDatabase(0); + + var _ = db.StringSet("h", "k"); + + var res = db.Execute("SETIFMATCH", ["h", "t", "0"]); + ClassicAssert.AreEqual("ETAGMISMATCH", res.ToString()); + } + + [Test] + public void GetIfNotMatchOnNonEtagDataReturnsNilForEtagAndCorrectData() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + IDatabase db = redis.GetDatabase(0); + + var _ = db.StringSet("h", "k"); + + var res = (RedisResult[])db.Execute("GETIFNOTMATCH", ["h", "1"]); + + ClassicAssert.IsTrue(res[0].IsNull); + ClassicAssert.AreEqual("k", res[1].ToString()); + } + + [Test] + public void GetWithEtagOnNonEtagDataReturnsNilForEtagAndCorrectData() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + IDatabase db = redis.GetDatabase(0); + + var _ = db.StringSet("h", "k"); + + var res = (RedisResult[])db.Execute("GETWITHETAG", ["h"]); + ClassicAssert.IsTrue(res[0].IsNull); + ClassicAssert.AreEqual("k", res[1].ToString()); + } + + #endregion + + #region Backwards Compatability Testing + + [Test] + public void SingleEtagSetGet() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + string origValue = "abcdefg"; + db.Execute("SETWITHETAG", ["mykey", origValue]); + + string retValue = db.StringGet("mykey"); + + ClassicAssert.AreEqual(origValue, retValue); + } + + [Test] + public async Task SingleUnicodeEtagSetGetGarnetClient() + { + using var db = TestUtils.GetGarnetClient(); + db.Connect(); + + string origValue = "笑い男"; + await db.ExecuteForLongResultAsync("SETWITHETAG", ["mykey", origValue]); + + string retValue = await db.StringGetAsync("mykey"); + + ClassicAssert.AreEqual(origValue, retValue); + } + + [Test] + public async Task LargeEtagSetGet() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + const int length = (1 << 19) + 100; + var value = new byte[length]; + + for (int i = 0; i < length; i++) + value[i] = (byte)((byte)'a' + ((byte)i % 26)); + + RedisResult res = await db.ExecuteAsync("SETWITHETAG", ["mykey", value]); + long initalEtag = long.Parse(res.ToString()); + ClassicAssert.AreEqual(0, initalEtag); + + // Backwards compatability of data set with etag and plain GET call + var retvalue = (byte[])await db.StringGetAsync("mykey"); + + ClassicAssert.IsTrue(new ReadOnlySpan(value).SequenceEqual(new ReadOnlySpan(retvalue))); + } + + [Test] + public void SetExpiryForEtagSetData() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + string origValue = "abcdefghij"; + + // set with etag + long initalEtag = long.Parse(db.Execute("SETWITHETAG", ["mykey", origValue]).ToString()); + ClassicAssert.AreEqual(0, initalEtag); + + // Expire the key in few seconds from now + ClassicAssert.IsTrue( + db.KeyExpire("mykey", TimeSpan.FromSeconds(2)) + ); + + string retValue = db.StringGet("mykey"); + ClassicAssert.AreEqual(origValue, retValue, "Get() before expiration"); + + var actualDbSize = db.Execute("DBSIZE"); + ClassicAssert.AreEqual(1, (ulong)actualDbSize, "DBSIZE before expiration"); + + var actualKeys = db.Execute("KEYS", ["*"]); + ClassicAssert.AreEqual(1, ((RedisResult[])actualKeys).Length, "KEYS before expiration"); + + var actualScan = db.Execute("SCAN", "0"); + ClassicAssert.AreEqual(1, ((RedisValue[])((RedisResult[])actualScan!)[1]).Length, "SCAN before expiration"); + + Thread.Sleep(2500); + + retValue = db.StringGet("mykey"); + ClassicAssert.AreEqual(null, retValue, "Get() after expiration"); + + actualDbSize = db.Execute("DBSIZE"); + ClassicAssert.AreEqual(0, (ulong)actualDbSize, "DBSIZE after expiration"); + + actualKeys = db.Execute("KEYS", ["*"]); + ClassicAssert.AreEqual(0, ((RedisResult[])actualKeys).Length, "KEYS after expiration"); + + actualScan = db.Execute("SCAN", "0"); + ClassicAssert.AreEqual(0, ((RedisValue[])((RedisResult[])actualScan!)[1]).Length, "SCAN after expiration"); + } + + [Test] + public void SetExpiryHighPrecisionForEtagSetDatat() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + var origValue = "abcdeghijklmno"; + // set with etag + long initalEtag = long.Parse(db.Execute("SETWITHETAG", ["mykey", origValue]).ToString()); + ClassicAssert.AreEqual(0, initalEtag); + + // Expire the key in few seconds from now + db.KeyExpire("mykey", TimeSpan.FromSeconds(1.9)); + + string retValue = db.StringGet("mykey"); + ClassicAssert.AreEqual(origValue, retValue); + + Thread.Sleep(1000); + retValue = db.StringGet("mykey"); + ClassicAssert.AreEqual(origValue, retValue); + + Thread.Sleep(2000); + retValue = db.StringGet("mykey"); + ClassicAssert.AreEqual(null, retValue); + } + + [Test] + public void SetGetWithRetainEtagForEtagSetData() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + string key = "mykey"; + string origValue = "abcdefghijklmnopqrst"; + + // Initial set + var _ = db.Execute("SETWITHETAG", [key, origValue]); + string retValue = db.StringGet(key); + ClassicAssert.AreEqual(origValue, retValue); + + // Smaller new value without expiration + string newValue1 = "abcdefghijklmnopqrs"; + + retValue = db.Execute("SET", [key, newValue1, "GET", "RETAINETAG"]).ToString(); + + ClassicAssert.AreEqual(origValue, retValue); + + retValue = db.StringGet(key); + ClassicAssert.AreEqual(newValue1, retValue); + + // This should increase the ETAG internally so we have a check for that here + long checkEtag = long.Parse(db.Execute("GETWITHETAG", [key])[0].ToString()); + ClassicAssert.AreEqual(1, checkEtag); + + // Smaller new value with KeepTtl + string newValue2 = "abcdefghijklmnopqr"; + retValue = db.Execute("SET", [key, newValue2, "GET", "RETAINETAG"]).ToString(); + + // This should increase the ETAG internally so we have a check for that here + checkEtag = long.Parse(db.Execute("GETWITHETAG", [key])[0].ToString()); + ClassicAssert.AreEqual(2, checkEtag); + + ClassicAssert.AreEqual(newValue1, retValue); + retValue = db.StringGet(key); + ClassicAssert.AreEqual(newValue2, retValue); + var expiry = db.KeyTimeToLive(key); + ClassicAssert.IsNull(expiry); + + // Smaller new value with expiration + string newValue3 = "01234"; + retValue = db.Execute("SET", [key, newValue3, "EX", "10", "GET", "RETAINETAG"]).ToString(); + ClassicAssert.AreEqual(newValue2, retValue); + + // This should increase the ETAG internally so we have a check for that here + checkEtag = long.Parse(db.Execute("GETWITHETAG", [key])[0].ToString()); + ClassicAssert.AreEqual(3, checkEtag); + + retValue = db.StringGet(key); + ClassicAssert.AreEqual(newValue3, retValue); + expiry = db.KeyTimeToLive(key); + ClassicAssert.IsTrue(expiry.Value.TotalSeconds > 0); + + // Larger new value with expiration + string newValue4 = "abcdefghijklmnopqrstabcdefghijklmnopqrst"; + retValue = db.Execute("SET", [key, newValue4, "EX", "100", "GET", "RETAINETAG"]).ToString(); + ClassicAssert.AreEqual(newValue3, retValue); + + // This should increase the ETAG internally so we have a check for that here + checkEtag = long.Parse(db.Execute("GETWITHETAG", [key])[0].ToString()); + ClassicAssert.AreEqual(4, checkEtag); + + retValue = db.StringGet(key); + ClassicAssert.AreEqual(newValue4, retValue); + expiry = db.KeyTimeToLive(key); + ClassicAssert.IsTrue(expiry.Value.TotalSeconds > 0); + + // Smaller new value without expiration + string newValue5 = "0123401234"; + retValue = db.Execute("SET", [key, newValue5, "GET", "RETAINETAG"]).ToString(); + ClassicAssert.AreEqual(newValue4, retValue); + retValue = db.StringGet(key); + ClassicAssert.AreEqual(newValue5, retValue); + expiry = db.KeyTimeToLive(key); + ClassicAssert.IsNull(expiry); + + // This should increase the ETAG internally so we have a check for that here + checkEtag = long.Parse(db.Execute("GETWITHETAG", [key])[0].ToString()); + ClassicAssert.AreEqual(5, checkEtag); + + // Larger new value without expiration + string newValue6 = "abcdefghijklmnopqrstabcdefghijklmnopqrst"; + retValue = db.Execute("SET", [key, newValue6, "GET", "RETAINETAG"]).ToString(); + ClassicAssert.AreEqual(newValue5, retValue); + retValue = db.StringGet(key); + ClassicAssert.AreEqual(newValue6, retValue); + expiry = db.KeyTimeToLive(key); + ClassicAssert.IsNull(expiry); + + // This should increase the ETAG internally so we have a check for that here + checkEtag = long.Parse(db.Execute("GETWITHETAG", [key])[0].ToString()); + ClassicAssert.AreEqual(6, checkEtag); + } + + [Test] + public void SetExpiryIncrForEtagSetData() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + // Key storing integer + var nVal = -100000; + var strKey = "key1"; + db.Execute("SETWITHETAG", [strKey, nVal]); + db.KeyExpire(strKey, TimeSpan.FromSeconds(5)); + + string res1 = db.StringGet(strKey); + + long n = db.StringIncrement(strKey); + + // This should increase the ETAG internally so we have a check for that here + var checkEtag = long.Parse(db.Execute("GETWITHETAG", [strKey])[0].ToString()); + ClassicAssert.AreEqual(1, checkEtag); + + string res = db.StringGet(strKey); + long nRetVal = Convert.ToInt64(res); + ClassicAssert.AreEqual(n, nRetVal); + ClassicAssert.AreEqual(-99999, nRetVal); + + n = db.StringIncrement(strKey); + + // This should increase the ETAG internally so we have a check for that here + checkEtag = long.Parse(db.Execute("GETWITHETAG", [strKey])[0].ToString()); + ClassicAssert.AreEqual(2, checkEtag); + + nRetVal = Convert.ToInt64(db.StringGet(strKey)); + ClassicAssert.AreEqual(n, nRetVal); + ClassicAssert.AreEqual(-99998, nRetVal); + + var res69 = db.KeyTimeToLive(strKey); + + Thread.Sleep(5000); + + // Expired key, restart increment,after exp this is treated as new record + // without etag + n = db.StringIncrement(strKey); + ClassicAssert.AreEqual(1, n); + + nRetVal = Convert.ToInt64(db.StringGet(strKey)); + ClassicAssert.AreEqual(n, nRetVal); + ClassicAssert.AreEqual(1, nRetVal); + + var etagGet = (RedisResult[])db.Execute("GETWITHETAG", [strKey]); + ClassicAssert.IsTrue(etagGet[0].IsNull); + ClassicAssert.AreEqual(1, Convert.ToInt64(etagGet[1])); + } + + [Test] + public void IncrDecrChangeDigitsWithExpiry() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + var strKey = "key1"; + + db.Execute("SETWITHETAG", [strKey, 9]); + + long checkEtag = long.Parse(db.Execute("GETWITHETAG", [strKey])[0].ToString()); + ClassicAssert.AreEqual(0, checkEtag); + + db.KeyExpire(strKey, TimeSpan.FromSeconds(5)); + + long n = db.StringIncrement(strKey); + long nRetVal = Convert.ToInt64(db.StringGet(strKey)); + ClassicAssert.AreEqual(n, nRetVal); + ClassicAssert.AreEqual(10, nRetVal); + + checkEtag = long.Parse(db.Execute("GETWITHETAG", [strKey])[0].ToString()); + ClassicAssert.AreEqual(1, checkEtag); + + n = db.StringDecrement(strKey); + nRetVal = Convert.ToInt64(db.StringGet(strKey)); + ClassicAssert.AreEqual(n, nRetVal); + ClassicAssert.AreEqual(9, nRetVal); + + checkEtag = long.Parse(db.Execute("GETWITHETAG", [strKey])[0].ToString()); + ClassicAssert.AreEqual(2, checkEtag); + + Thread.Sleep(TimeSpan.FromSeconds(5)); + + var res = (string)db.StringGet(strKey); + ClassicAssert.IsNull(res); + } + + [Test] + public void StringSetOnAnExistingEtagDataOverrides() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + var strKey = "mykey"; + db.Execute("SETWITHETAG", [strKey, 9]); + + long checkEtag = long.Parse(db.Execute("GETWITHETAG", [strKey])[0].ToString()); + ClassicAssert.AreEqual(0, checkEtag); + + // Unless the SET was called with RETAINETAG a call to set will override the setwithetag to a new + // value altogether, this will make it lose it's etag capability. This is a limitation for Etags + // because plain sets are upserts (blind updates), and currently we cannot increase the latency in + // the common path for set to check beyong Readonly address for the existence of a record with ETag. + // This means that sets are complete upserts and clients need to use setifmatch, or set with RETAINETAG + // if they want each consequent set to maintain the key value pair's etag property. + ClassicAssert.IsTrue(db.StringSet(strKey, "ciaociao")); + + string retVal = db.StringGet(strKey).ToString(); + ClassicAssert.AreEqual("ciaociao", retVal); + + var res = (RedisResult[])db.Execute("GETWITHETAG", [strKey]); + ClassicAssert.IsTrue(res[0].IsNull); + ClassicAssert.AreEqual("ciaociao", res[1].ToString()); + } + + [Test] + public void StringSetOnAnExistingEtagDataUpdatesEtagIfEtagRetain() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + var strKey = "mykey"; + db.Execute("SETWITHETAG", [strKey, 9]); + + long checkEtag = (long)db.Execute("GETWITHETAG", [strKey])[0]; + ClassicAssert.AreEqual(0, checkEtag); + + // Unless you explicitly call SET with RETAINETAG option you will lose the etag on the previous key-value pair + db.Execute("SET", [strKey, "ciaociao", "RETAINETAG"]); + + string retVal = db.StringGet(strKey).ToString(); + ClassicAssert.AreEqual("ciaociao", retVal); + + var res = (RedisResult[])db.Execute("GETWITHETAG", strKey); + ClassicAssert.AreEqual(1, (long)res[0]); + + // on subsequent upserts we are still increasing the etag transparently + db.Execute("SET", [strKey, "ciaociaociao", "RETAINETAG"]); + + retVal = db.StringGet(strKey).ToString(); + ClassicAssert.AreEqual("ciaociaociao", retVal); + + res = (RedisResult[])db.Execute("GETWITHETAG", strKey); + ClassicAssert.AreEqual(2, (long)res[0]); + ClassicAssert.AreEqual("ciaociaociao", res[1].ToString()); + } + + [Test] + public void LockTakeReleaseOnAValueInitiallySetWithEtag() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + string key = "lock-key"; + string value = "lock-value"; + + var initalEtag = long.Parse(db.Execute("SETWITHETAG", [key, value]).ToString()); + ClassicAssert.AreEqual(0, initalEtag); + + var success = db.LockTake(key, value, TimeSpan.FromSeconds(100)); + ClassicAssert.IsFalse(success); + + success = db.LockRelease(key, value); + ClassicAssert.IsTrue(success); + + success = db.LockRelease(key, value); + ClassicAssert.IsFalse(success); + + success = db.LockTake(key, value, TimeSpan.FromSeconds(100)); + ClassicAssert.IsTrue(success); + + success = db.LockRelease(key, value); + ClassicAssert.IsTrue(success); + + // Test auto-lock-release + success = db.LockTake(key, value, TimeSpan.FromSeconds(1)); + ClassicAssert.IsTrue(success); + + Thread.Sleep(2000); + success = db.LockTake(key, value, TimeSpan.FromSeconds(1)); + ClassicAssert.IsTrue(success); + + success = db.LockRelease(key, value); + ClassicAssert.IsTrue(success); + } + + [Test] + [TestCase("key1", 1000)] + [TestCase("key1", 0)] + public void SingleDecrForEtagSetData(string strKey, int nVal) + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + // Key storing integer + var initalEtag = long.Parse(db.Execute("SETWITHETAG", [strKey, nVal]).ToString()); + ClassicAssert.AreEqual(0, initalEtag); + + long n = db.StringDecrement(strKey); + ClassicAssert.AreEqual(nVal - 1, n); + long nRetVal = Convert.ToInt64(db.StringGet(strKey)); + ClassicAssert.AreEqual(n, nRetVal); + + long checkEtag = long.Parse(db.Execute("GETWITHETAG", [strKey])[0].ToString()); + ClassicAssert.AreEqual(1, checkEtag); + } + + [Test] + [TestCase(-1000, 100)] + [TestCase(-1000, -9000)] + [TestCase(-10000, 9000)] + [TestCase(9000, 10000)] + public void SingleDecrByForEtagSetData(long nVal, long nDecr) + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + // Key storing integer val + var strKey = "key1"; + var initalEtag = long.Parse(db.Execute("SETWITHETAG", [strKey, nVal]).ToString()); + ClassicAssert.AreEqual(0, initalEtag); + + long n = db.StringDecrement(strKey, nDecr); + + int nRetVal = Convert.ToInt32(db.StringGet(strKey)); + ClassicAssert.AreEqual(n, nRetVal); + + long checkEtag = long.Parse(db.Execute("GETWITHETAG", [strKey])[0].ToString()); + ClassicAssert.AreEqual(1, checkEtag); + } + + [Test] + [TestCase(RespCommand.INCR)] + [TestCase(RespCommand.DECR)] + [TestCase(RespCommand.INCRBY)] + [TestCase(RespCommand.DECRBY)] + public void SimpleIncrementInvalidValueForEtagSetdata(RespCommand cmd) + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + string[] values = ["", "7 3", "02+(34", "笑い男", "01", "-01", "7ab"]; + + for (var i = 0; i < values.Length; i++) + { + var key = $"key{i}"; + var exception = false; + var initalEtag = long.Parse(db.Execute("SETWITHETAG", [key, values[i]]).ToString()); + ClassicAssert.AreEqual(0, initalEtag); + + try + { + _ = cmd switch + { + RespCommand.INCR => db.StringIncrement(key), + RespCommand.DECR => db.StringDecrement(key), + RespCommand.INCRBY => db.StringIncrement(key, 10L), + RespCommand.DECRBY => db.StringDecrement(key, 10L), + _ => throw new Exception($"Command {cmd} not supported!"), + }; + } + catch (Exception ex) + { + exception = true; + var msg = ex.Message; + ClassicAssert.AreEqual("ERR value is not an integer or out of range.", msg); + } + ClassicAssert.IsTrue(exception); + } + } + + [Test] + [TestCase(RespCommand.INCR)] + [TestCase(RespCommand.DECR)] + [TestCase(RespCommand.INCRBY)] + [TestCase(RespCommand.DECRBY)] + public void SimpleIncrementOverflowForEtagSetData(RespCommand cmd) + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + var exception = false; + + var key = "test"; + + try + { + switch (cmd) + { + case RespCommand.INCR: + _ = db.Execute("SETWITHETAG", [key, long.MaxValue.ToString()]); + _ = db.StringIncrement(key); + break; + case RespCommand.DECR: + _ = db.Execute("SETWITHETAG", [key, long.MinValue.ToString()]); + _ = db.StringDecrement(key); + break; + case RespCommand.INCRBY: + _ = db.Execute("SETWITHETAG", [key, 0]); + _ = db.Execute("INCRBY", [key, ulong.MaxValue.ToString()]); + break; + case RespCommand.DECRBY: + _ = db.Execute("SETWITHETAG", [key, 0]); + _ = db.Execute("DECRBY", [key, ulong.MaxValue.ToString()]); + break; + } + } + catch (Exception ex) + { + exception = true; + var msg = ex.Message; + ClassicAssert.AreEqual("ERR value is not an integer or out of range.", msg); + } + ClassicAssert.IsTrue(exception); + } + + [Test] + [TestCase(0, 12.6)] + [TestCase(12.6, 0)] + [TestCase(10, 10)] + [TestCase(910151, 0.23659)] + [TestCase(663.12336412, 12342.3)] + [TestCase(10, -110)] + [TestCase(110, -110.234)] + [TestCase(-2110.95255555, -110.234)] + [TestCase(-2110.95255555, 100000.526654512219412)] + [TestCase(double.MaxValue, double.MinValue)] + public void SimpleIncrementByFloatForEtagSetData(double initialValue, double incrByValue) + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + var key = "key1"; + db.Execute("SETWITHETAG", key, initialValue); + + var expectedResult = initialValue + incrByValue; + + var actualResultStr = (string)db.Execute("INCRBYFLOAT", [key, incrByValue]); + var actualResultRawStr = db.StringGet(key); + + var actualResult = double.Parse(actualResultStr, CultureInfo.InvariantCulture); + var actualResultRaw = double.Parse(actualResultRawStr, CultureInfo.InvariantCulture); + + Assert.That(actualResult, Is.EqualTo(expectedResult).Within(1.0 / Math.Pow(10, 15))); + Assert.That(actualResult, Is.EqualTo(actualResultRaw).Within(1.0 / Math.Pow(10, 15))); + + RedisResult[] res = (RedisResult[])db.Execute("GETWITHETAG", key); + long etag = (long)res[0]; + double value = double.Parse(res[1].ToString(), CultureInfo.InvariantCulture); + Assert.That(value, Is.EqualTo(actualResultRaw).Within(1.0 / Math.Pow(10, 15))); + ClassicAssert.AreEqual(1, etag); + } + + [Test] + public void SingleDeleteForEtagSetData() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + // Key storing integer + var nVal = 100; + var strKey = "key1"; + db.Execute("SETWITHETAG", [strKey, nVal]); + db.KeyDelete(strKey); + var retVal = Convert.ToBoolean(db.StringGet(strKey)); + ClassicAssert.AreEqual(retVal, false); + } + + [Test] + public void SingleDeleteWithObjectStoreDisabledForEtagSetData() + { + TearDown(); + + TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, DisableObjects: true); + server.Start(); + + var key = "delKey"; + var value = "1234"; + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + db.Execute("SETWITHETAG", [key, value]); + + var resp = (string)db.StringGet(key); + ClassicAssert.AreEqual(resp, value); + + var respDel = db.KeyDelete(key); + ClassicAssert.IsTrue(respDel); + + respDel = db.KeyDelete(key); + ClassicAssert.IsFalse(respDel); + } + + private string GetRandomString(int len) + { + const string chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; + return new string(Enumerable.Repeat(chars, len) + .Select(s => s[r.Next(s.Length)]).ToArray()); + } + + [Test] + public void SingleDeleteWithObjectStoreDisable_LTMForEtagSetData() + { + TearDown(); + + TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, lowMemory: true, DisableObjects: true); + server.Start(); + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + int keyCount = 5; + int valLen = 256; + int keyLen = 8; + + List> data = []; + for (int i = 0; i < keyCount; i++) + { + data.Add(new Tuple(GetRandomString(keyLen), GetRandomString(valLen))); + var pair = data.Last(); + db.Execute("SETWITHETAG", [pair.Item1, pair.Item2]); + } + + + for (int i = 0; i < keyCount; i++) + { + var pair = data[i]; + + var resp = (string)db.StringGet(pair.Item1); + ClassicAssert.AreEqual(resp, pair.Item2); + + var respDel = db.KeyDelete(pair.Item1); + resp = (string)db.StringGet(pair.Item1); + ClassicAssert.IsNull(resp); + + respDel = db.KeyDelete(pair.Item2); + ClassicAssert.IsFalse(respDel); + } + } + + [Test] + public void MultiKeyDeleteForEtagSetData([Values] bool withoutObjectStore) + { + if (withoutObjectStore) + { + TearDown(); + TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, DisableObjects: true); + server.Start(); + } + + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + int keyCount = 10; + int valLen = 16; + int keyLen = 8; + + List> data = []; + for (int i = 0; i < keyCount; i++) + { + data.Add(new Tuple(GetRandomString(keyLen), GetRandomString(valLen))); + var pair = data.Last(); + db.Execute("SETWITHETAG", [pair.Item1, pair.Item2]); + } + + var keys = data.Select(x => (RedisKey)x.Item1).ToArray(); + var keysDeleted = db.KeyDeleteAsync(keys); + keysDeleted.Wait(); + ClassicAssert.AreEqual(keysDeleted.Result, 10); + + var keysDel = db.KeyDelete(keys); + ClassicAssert.AreEqual(keysDel, 0); + } + + [Test] + public void MultiKeyUnlinkForEtagSetData([Values] bool withoutObjectStore) + { + if (withoutObjectStore) + { + TearDown(); + TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, DisableObjects: true); + server.Start(); + } + + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + int keyCount = 10; + int valLen = 16; + int keyLen = 8; + + List> data = []; + for (int i = 0; i < keyCount; i++) + { + data.Add(new Tuple(GetRandomString(keyLen), GetRandomString(valLen))); + var pair = data.Last(); + db.Execute("SETWITHETAG", [pair.Item1, pair.Item2]); + } + + var keys = data.Select(x => (object)x.Item1).ToArray(); + var keysDeleted = (string)db.Execute("unlink", keys); + ClassicAssert.AreEqual(10, int.Parse(keysDeleted)); + + keysDeleted = (string)db.Execute("unlink", keys); + ClassicAssert.AreEqual(0, int.Parse(keysDeleted)); + } + + [Test] + public void SingleExistsForEtagSetData([Values] bool withoutObjectStore) + { + if (withoutObjectStore) + { + TearDown(); + TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, DisableObjects: true); + server.Start(); + } + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + // Key storing integer + var nVal = 100; + var strKey = "key1"; + ClassicAssert.IsFalse(db.KeyExists(strKey)); + + db.Execute("SETWITHETAG", [strKey, nVal]); + + bool fExists = db.KeyExists("key1", CommandFlags.None); + ClassicAssert.AreEqual(fExists, true); + + fExists = db.KeyExists("key2", CommandFlags.None); + ClassicAssert.AreEqual(fExists, false); + } + + + [Test] + public void MultipleExistsKeysAndObjectsAndEtagData() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + var count = db.ListLeftPush("listKey", ["a", "b", "c", "d"]); + ClassicAssert.AreEqual(4, count); + + var zaddItems = db.SortedSetAdd("zset:test", [new SortedSetEntry("a", 1), new SortedSetEntry("b", 2)]); + ClassicAssert.AreEqual(2, zaddItems); + + db.StringSet("foo", "bar"); + + db.Execute("SETWITHETAG", ["rizz", "bar"]); + + var exists = db.KeyExists(["key", "listKey", "zset:test", "foo", "rizz"]); + ClassicAssert.AreEqual(4, exists); + } + + [Test] + public void SingleRenameEtagSetData() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + string origValue = "test1"; + long etag = long.Parse(db.Execute("SETWITHETAG", ["key1", origValue]).ToString()); + ClassicAssert.AreEqual(0, etag); + + db.KeyRename("key1", "key2"); + string retValue = db.StringGet("key2"); + + ClassicAssert.AreEqual(origValue, retValue); + + // other key now gives no result + ClassicAssert.AreEqual("", db.Execute("GETWITHETAG", ["key1"]).ToString()); + + // new Key value pair created with older value, the etag is reset here back to 0 + var res = (RedisResult[])db.Execute("GETWITHETAG", ["key2"]); + ClassicAssert.AreEqual("0", res[0].ToString()); + ClassicAssert.AreEqual(origValue, res[1].ToString()); + + origValue = db.StringGet("key1"); + ClassicAssert.AreEqual(null, origValue); + } + + [Test] + public void SingleRenameEtagShouldRetainEtagOfNewKeyIfExistsWithEtag() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + string existingNewKey = "key2"; + string existingVal = "foo"; + long etag = (long)db.Execute("SETWITHETAG", [existingNewKey, existingVal]); + ClassicAssert.AreEqual(0, etag); + RedisResult[] updateRes = (RedisResult[])db.Execute("SETIFMATCH", [existingNewKey, "updated", etag.ToString()]); + long updatedEtag = (long)updateRes[0]; + + string origValue = "test1"; + etag = long.Parse(db.Execute("SETWITHETAG", ["key1", origValue]).ToString()); + ClassicAssert.AreEqual(0, etag); + + db.KeyRename("key1", existingNewKey); + string retValue = db.StringGet(existingNewKey); + ClassicAssert.AreEqual(origValue, retValue); + + // new Key value pair created with older value, the etag is reusing the existingnewkey etag + var res = (RedisResult[])db.Execute("GETWITHETAG", [existingNewKey]); + ClassicAssert.AreEqual(updatedEtag + 1, (long)res[0]); + ClassicAssert.AreEqual(origValue, res[1].ToString()); + + origValue = db.StringGet("key1"); + ClassicAssert.AreEqual(null, origValue); + } + + [Test] + public void SingleRenameKeyEdgeCaseEtagSetData([Values] bool withoutObjectStore) + { + if (withoutObjectStore) + { + TearDown(); + TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, DisableObjects: true); + server.Start(); + } + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + //1. Key rename does not exist + try + { + var res = db.KeyRename("key1", "key2"); + } + catch (Exception ex) + { + ClassicAssert.AreEqual("ERR no such key", ex.Message); + } + + //2. Key rename oldKey.Equals(newKey) + string origValue = "test1"; + db.Execute("SETWITHETAG", ["key1", origValue]); + + bool renameRes = db.KeyRename("key1", "key1"); + ClassicAssert.IsTrue(renameRes); + string retValue = db.StringGet("key1"); + ClassicAssert.AreEqual(origValue, retValue); + } + + [Test] + public void SingleRenameShouldNotAddEtagEvenIfExistingKeyHadEtagButNotTheOriginal() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + string existingNewKey = "key2"; + string existingVal = "foo"; + long etag = (long)db.Execute("SETWITHETAG", [existingNewKey, existingVal]); + ClassicAssert.AreEqual(0, etag); + RedisResult[] updateRes = (RedisResult[])db.Execute("SETIFMATCH", [existingNewKey, "updated", etag.ToString()]); + long updatedEtag = (long)updateRes[0]; + + string origValue = "test1"; + ClassicAssert.IsTrue(db.StringSet("key1", origValue)); + + db.KeyRename("key1", existingNewKey); + string retValue = db.StringGet(existingNewKey); + ClassicAssert.AreEqual(origValue, retValue); + + // new Key value pair created with older value, the etag is reusing the existingnewkey etag + var res = (RedisResult[])db.Execute("GETWITHETAG", [existingNewKey]); + ClassicAssert.IsTrue(res[0].IsNull); + ClassicAssert.AreEqual(origValue, res[1].ToString()); + + origValue = db.StringGet("key1"); + ClassicAssert.AreEqual(null, origValue); + } + + [Test] + public void SingleRenameShouldAddEtagIfOldKeyHadEtagButNotExistingNewkey() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + string existingNewKey = "key2"; + string existingVal = "foo"; + + ClassicAssert.IsTrue(db.StringSet(existingNewKey, existingVal)); + + string origValue = "test1"; + long etag = (long)db.Execute("SETWITHETAG", ["key1", origValue]); + ClassicAssert.AreEqual(0, etag); + + db.KeyRename("key1", existingNewKey); + string retValue = db.StringGet(existingNewKey); + ClassicAssert.AreEqual(origValue, retValue); + + // new Key value pair created with older value + var res = (RedisResult[])db.Execute("GETWITHETAG", [existingNewKey]); + ClassicAssert.AreEqual(0, (long)res[0]); + ClassicAssert.AreEqual(origValue, res[1].ToString()); + + origValue = db.StringGet("key1"); + ClassicAssert.AreEqual(null, origValue); + } + + [Test] + public void SingleRenameShouldAddEtagAndMetadataIfOldKeyHadEtagAndMetadata() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + string origKey = "key1"; + string origValue = "test1"; + long etag = (long)db.Execute("SETWITHETAG", ["key1", origValue]); + ClassicAssert.AreEqual(0, etag); + + ClassicAssert.IsTrue(db.KeyExpire(origKey, TimeSpan.FromSeconds(10))); + + string newKey = "key2"; + db.KeyRename(origKey, newKey); + + string retValue = db.StringGet(newKey); + ClassicAssert.AreEqual(origValue, retValue); + + // new Key value pair created with older value + var res = (RedisResult[])db.Execute("GETWITHETAG", [newKey]); + ClassicAssert.AreEqual(0, (long)res[0]); + ClassicAssert.AreEqual(origValue, res[1].ToString()); + + // check that the ttl is not empty on new key because it inherited it from prev key + TimeSpan? ttl = db.KeyTimeToLive(newKey); + ClassicAssert.IsNotNull(ttl); + + origValue = db.StringGet(origKey); + ClassicAssert.AreEqual(null, origValue); + } + + + [Test] + public void PersistTTLTestForEtagSetData() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + var key = "expireKey"; + var val = "expireValue"; + var expire = 2; + + var ttl = db.Execute("TTL", key); + ClassicAssert.AreEqual(-2, (int)ttl); + + db.Execute("SETWITHETAG", [key, val]); + ttl = db.Execute("TTL", key); + ClassicAssert.AreEqual(-1, (int)ttl); + + db.KeyExpire(key, TimeSpan.FromSeconds(expire)); + + var res = (RedisResult[])db.Execute("GETWITHETAG", [key]); + ClassicAssert.AreEqual(0, long.Parse(res[0].ToString())); + ClassicAssert.AreEqual(val, res[1].ToString()); + + var time = db.KeyTimeToLive(key); + ClassicAssert.IsTrue(time.Value.TotalSeconds > 0); + + db.KeyExpire(key, TimeSpan.FromSeconds(expire)); + res = (RedisResult[])db.Execute("GETWITHETAG", [key]); + ClassicAssert.AreEqual(0, long.Parse(res[0].ToString())); + ClassicAssert.AreEqual(val, res[1].ToString()); + + db.KeyPersist(key); + res = (RedisResult[])db.Execute("GETWITHETAG", [key]); + ClassicAssert.AreEqual(0, long.Parse(res[0].ToString())); + ClassicAssert.AreEqual(val, res[1].ToString()); + + Thread.Sleep((expire + 1) * 1000); + + var _val = db.StringGet(key); + ClassicAssert.AreEqual(val, _val.ToString()); + + time = db.KeyTimeToLive(key); + ClassicAssert.IsNull(time); + + res = (RedisResult[])db.Execute("GETWITHETAG", [key]); + ClassicAssert.AreEqual(0, long.Parse(res[0].ToString())); + ClassicAssert.AreEqual(val, res[1].ToString()); + } + + [Test] + public void PersistTestForEtagSetData() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + int expire = 100; + var keyA = "keyA"; + db.Execute("SETWITHETAG", [keyA, keyA]); + + var response = db.KeyPersist(keyA); + ClassicAssert.IsFalse(response); + + db.KeyExpire(keyA, TimeSpan.FromSeconds(expire)); + var time = db.KeyTimeToLive(keyA); + ClassicAssert.IsTrue(time.Value.TotalSeconds > 0); + + response = db.KeyPersist(keyA); + ClassicAssert.IsTrue(response); + + time = db.KeyTimeToLive(keyA); + ClassicAssert.IsTrue(time == null); + + var value = db.StringGet(keyA); + ClassicAssert.AreEqual(value, keyA); + + var res = (RedisResult[])db.Execute("GETWITHETAG", [keyA]); + ClassicAssert.AreEqual(0, long.Parse(res[0].ToString())); + ClassicAssert.AreEqual(keyA, res[1].ToString()); + + var noKey = "noKey"; + response = db.KeyPersist(noKey); + ClassicAssert.IsFalse(response); + } + + [Test] + [TestCase("EXPIRE")] + [TestCase("PEXPIRE")] + public void KeyExpireStringTestForEtagSetData(string command) + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + var key = "keyA"; + db.Execute("SETWITHETAG", [key, key]); + + var value = db.StringGet(key); + ClassicAssert.AreEqual(key, (string)value); + + if (command.Equals("EXPIRE")) + db.KeyExpire(key, TimeSpan.FromSeconds(1)); + else + db.Execute(command, [key, 1000]); + + Thread.Sleep(1500); + + value = db.StringGet(key); + ClassicAssert.AreEqual(null, (string)value); + } + + [Test] + [TestCase("EXPIRE")] + [TestCase("PEXPIRE")] + public void KeyExpireOptionsTestForEtagSetData(string command) + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + var key = "keyA"; + object[] args = [key, 1000, ""]; + db.Execute("SETWITHETAG", [key, key]); + + args[2] = "XX";// XX -- Set expiry only when the key has an existing expiry + bool resp = (bool)db.Execute($"{command}", args); + ClassicAssert.IsFalse(resp);//XX return false no existing expiry + + args[2] = "NX";// NX -- Set expiry only when the key has no expiry + resp = (bool)db.Execute($"{command}", args); + ClassicAssert.IsTrue(resp);// NX return true no existing expiry + + args[2] = "NX";// NX -- Set expiry only when the key has no expiry + resp = (bool)db.Execute($"{command}", args); + ClassicAssert.IsFalse(resp);// NX return false existing expiry + + args[1] = 50; + args[2] = "XX";// XX -- Set expiry only when the key has an existing expiry + resp = (bool)db.Execute($"{command}", args); + ClassicAssert.IsTrue(resp);// XX return true existing expiry + var time = db.KeyTimeToLive(key); + ClassicAssert.IsTrue(time.Value.TotalSeconds <= (double)((int)args[1]) && time.Value.TotalSeconds > 0); + + args[1] = 1; + args[2] = "GT";// GT -- Set expiry only when the new expiry is greater than current one + resp = (bool)db.Execute($"{command}", args); + ClassicAssert.IsFalse(resp); // GT return false new expiry < current expiry + + args[1] = 1000; + args[2] = "GT";// GT -- Set expiry only when the new expiry is greater than current one + resp = (bool)db.Execute($"{command}", args); + ClassicAssert.IsTrue(resp); // GT return true new expiry > current expiry + time = db.KeyTimeToLive(key); + + if (command.Equals("EXPIRE")) + ClassicAssert.IsTrue(time.Value.TotalSeconds > 500); + else + ClassicAssert.IsTrue(time.Value.TotalMilliseconds > 500); + + args[1] = 2000; + args[2] = "LT";// LT -- Set expiry only when the new expiry is less than current one + resp = (bool)db.Execute($"{command}", args); + ClassicAssert.IsFalse(resp); // LT return false new expiry > current expiry + + args[1] = 15; + args[2] = "LT";// LT -- Set expiry only when the new expiry is less than current one + resp = (bool)db.Execute($"{command}", args); + ClassicAssert.IsTrue(resp); // LT return true new expiry < current expiry + time = db.KeyTimeToLive(key); + + if (command.Equals("EXPIRE")) + ClassicAssert.IsTrue(time.Value.TotalSeconds <= (double)((int)args[1]) && time.Value.TotalSeconds > 0); + else + ClassicAssert.IsTrue(time.Value.TotalMilliseconds <= (double)((int)args[1]) && time.Value.TotalMilliseconds > 0); + } + + [Test] + public void MainObjectKeyForEtagSetData() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var server = redis.GetServers()[0]; + var db = redis.GetDatabase(0); + + const string key = "test:1"; + + // Do SETIWTHETAG + ClassicAssert.AreEqual(0, long.Parse(db.Execute("SETWITHETAG", [key, "v1"]).ToString())); + + // Do SetAdd using the same key + ClassicAssert.IsTrue(db.SetAdd(key, "v2")); + + // Two keys "test:1" - this is expected as of now + // because Garnet has a separate main and object store + var keys = server.Keys(db.Database, key).ToList(); + ClassicAssert.AreEqual(2, keys.Count); + ClassicAssert.AreEqual(key, (string)keys[0]); + ClassicAssert.AreEqual(key, (string)keys[1]); + + // do ListRightPush using the same key, expected error + var ex = Assert.Throws(() => db.ListRightPush(key, "v3")); + var expectedError = Encoding.ASCII.GetString(CmdStrings.RESP_ERR_WRONG_TYPE); + ClassicAssert.IsNotNull(ex); + ClassicAssert.AreEqual(expectedError, ex.Message); + } + + [Test] + public void GetSliceTestForEtagSetData() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + string key = "rangeKey"; + string value = "0123456789"; + + var resp = (string)db.StringGetRange(key, 2, 10); + ClassicAssert.AreEqual(string.Empty, resp); + + ClassicAssert.AreEqual(0, long.Parse(db.Execute("SETWITHETAG", [key, value]).ToString())); + + //0,0 + resp = (string)db.StringGetRange(key, 0, 0); + ClassicAssert.AreEqual("0", resp); + + //actual value + resp = (string)db.StringGetRange(key, 0, -1); + ClassicAssert.AreEqual(value, resp); + + #region testA + //s[2,len] s < e & e = len + resp = (string)db.StringGetRange(key, 2, 10); + ClassicAssert.AreEqual(value.Substring(2), resp); + + //s[2,len] s < e & e = len - 1 + resp = (string)db.StringGetRange(key, 2, 9); + ClassicAssert.AreEqual(value.Substring(2), resp); + + //s[2,len] s < e < len + resp = (string)db.StringGetRange(key, 2, 5); + ClassicAssert.AreEqual(value.Substring(2, 4), resp); + + //s[2,len] s < len < e + resp = (string)db.StringGetRange(key, 2, 15); + ClassicAssert.AreEqual(value.Substring(2), resp); + + //s[4,len] e < s < len + resp = (string)db.StringGetRange(key, 4, 2); + ClassicAssert.AreEqual("", resp); + + //s[4,len] e < 0 < s < len + resp = (string)db.StringGetRange(key, 4, -2); + ClassicAssert.AreEqual(value.Substring(4, 5), resp); + + //s[4,len] e < -len < 0 < s < len + resp = (string)db.StringGetRange(key, 4, -12); + ClassicAssert.AreEqual("", resp); + #endregion + + #region testB + //-len < s < 0 < len < e + resp = (string)db.StringGetRange(key, -4, 15); + ClassicAssert.AreEqual(value.Substring(6, 4), resp); + + //-len < s < 0 < e < len where len + s > e + resp = (string)db.StringGetRange(key, -4, 5); + ClassicAssert.AreEqual("", resp); + + //-len < s < 0 < e < len where len + s < e + resp = (string)db.StringGetRange(key, -4, 8); + ClassicAssert.AreEqual(value.Substring(value.Length - 4, 2), resp); + + //-len < s < e < 0 + resp = (string)db.StringGetRange(key, -4, -1); + ClassicAssert.AreEqual(value.Substring(value.Length - 4, 4), resp); + + //-len < e < s < 0 + resp = (string)db.StringGetRange(key, -4, -7); + ClassicAssert.AreEqual("", resp); + #endregion + + //range start > end > len + resp = (string)db.StringGetRange(key, 17, 13); + ClassicAssert.AreEqual("", resp); + + //range 0 > start > end + resp = (string)db.StringGetRange(key, -1, -4); + ClassicAssert.AreEqual("", resp); + + //equal offsets + resp = db.StringGetRange(key, 4, 4); + ClassicAssert.AreEqual("4", resp); + + //equal offsets + resp = db.StringGetRange(key, -4, -4); + ClassicAssert.AreEqual("6", resp); + + //equal offsets + resp = db.StringGetRange(key, -100, -100); + ClassicAssert.AreEqual("0", resp); + + //equal offsets + resp = db.StringGetRange(key, -101, -101); + ClassicAssert.AreEqual("9", resp); + + //start larger than end + resp = db.StringGetRange(key, -1, -3); + ClassicAssert.AreEqual("", resp); + + //2,-1 -> 2 9 + var negend = -1; + resp = db.StringGetRange(key, 2, negend); + ClassicAssert.AreEqual(value.Substring(2, 8), resp); + + //2,-3 -> 2 7 + negend = -3; + resp = db.StringGetRange(key, 2, negend); + ClassicAssert.AreEqual(value.Substring(2, 6), resp); + + //-5,-3 -> 5,7 + var negstart = -5; + resp = db.StringGetRange(key, negstart, negend); + ClassicAssert.AreEqual(value.Substring(5, 3), resp); + } + + [Test] + public void SetRangeTestForEtagSetData() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + string key = "setRangeKey"; + string value = "0123456789"; + string newValue = "ABCDE"; + + db.Execute("SETWITHETAG", [key, value]); + + var resp = db.StringGet(key); + ClassicAssert.AreEqual("0123456789", resp.ToString()); + + // new key, length 10, offset 5 -> 15 ("\0\0\0\0\00123456789") + resp = db.StringSetRange(key, 5, value); + ClassicAssert.AreEqual("15", resp.ToString()); + resp = db.StringGet(key); + ClassicAssert.AreEqual("012340123456789", resp.ToString()); + + // should update the etag internally + var updatedEtagRes = db.Execute("GETWITHETAG", key); + ClassicAssert.AreEqual(1, long.Parse(updatedEtagRes[0].ToString())); + + ClassicAssert.IsTrue(db.KeyDelete(key)); + + // new key, length 10, offset -1 -> RedisServerException ("ERR offset is out of range") + try + { + db.StringSetRange(key, -1, value); + Assert.Fail(); + } + catch (RedisServerException ex) + { + ClassicAssert.AreEqual(Encoding.ASCII.GetString(CmdStrings.RESP_ERR_GENERIC_OFFSETOUTOFRANGE), ex.Message); + } + + // existing key, length 10, offset 0, value length 5 -> 10 ("ABCDE56789") + db.Execute("SETWITHETAG", [key, value]); + + resp = db.StringSetRange(key, 0, newValue); + ClassicAssert.AreEqual("10", resp.ToString()); + resp = db.StringGet(key); + ClassicAssert.AreEqual("ABCDE56789", resp.ToString()); + + // should update the etag internally + updatedEtagRes = db.Execute("GETWITHETAG", key); + ClassicAssert.AreEqual(1, long.Parse(updatedEtagRes[0].ToString())); + + ClassicAssert.IsTrue(db.KeyDelete(key)); + + // key, length 10, offset 5, value length 5 -> 10 ("01234ABCDE") + db.Execute("SETWITHETAG", [key, value]); + + resp = db.StringSetRange(key, 5, newValue); + ClassicAssert.AreEqual("10", resp.ToString()); + + updatedEtagRes = db.Execute("GETWITHETAG", key); + ClassicAssert.AreEqual(1, long.Parse(updatedEtagRes[0].ToString())); + + resp = db.StringGet(key); + ClassicAssert.AreEqual("01234ABCDE", resp.ToString()); + ClassicAssert.IsTrue(db.KeyDelete(key)); + + // existing key, length 10, offset 10, value length 5 -> 15 ("0123456789ABCDE") + db.Execute("SETWITHETAG", [key, value]); + resp = db.StringSetRange(key, 10, newValue); + ClassicAssert.AreEqual("15", resp.ToString()); + resp = db.StringGet(key); + ClassicAssert.AreEqual("0123456789ABCDE", resp.ToString()); + ClassicAssert.IsTrue(db.KeyDelete(key)); + + // existing key, length 10, offset 15, value length 5 -> 20 ("0123456789\0\0\0\0\0ABCDE") + db.Execute("SETWITHETAG", [key, value]); + + resp = db.StringSetRange(key, 15, newValue); + ClassicAssert.AreEqual("20", resp.ToString()); + resp = db.StringGet(key); + ClassicAssert.AreEqual("0123456789\0\0\0\0\0ABCDE", resp.ToString()); + ClassicAssert.IsTrue(db.KeyDelete(key)); + + // existing key, length 10, offset -1, value length 5 -> RedisServerException ("ERR offset is out of range") + db.Execute("SETWITHETAG", [key, value]); + try + { + db.StringSetRange(key, -1, newValue); + Assert.Fail(); + } + catch (RedisServerException ex) + { + ClassicAssert.AreEqual(Encoding.ASCII.GetString(CmdStrings.RESP_ERR_GENERIC_OFFSETOUTOFRANGE), ex.Message); + } + } + + [Test] + public void KeepTtlTestForDataInitiallySetWithEtag() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + int expire = 3; + var keyA = "keyA"; + var keyB = "keyB"; + db.Execute("SETWITHETAG", [keyA, keyA]); + db.Execute("SETWITHETAG", [keyB, keyB]); + + db.KeyExpire(keyA, TimeSpan.FromSeconds(expire)); + db.KeyExpire(keyB, TimeSpan.FromSeconds(expire)); + + db.StringSet(keyA, keyA, keepTtl: true); + var time = db.KeyTimeToLive(keyA); + ClassicAssert.IsTrue(time.Value.Ticks > 0); + + db.StringSet(keyB, keyB, keepTtl: false); + time = db.KeyTimeToLive(keyB); + ClassicAssert.IsTrue(time == null); + + Thread.Sleep(expire * 1000 + 100); + + string value = db.StringGet(keyB); + ClassicAssert.AreEqual(keyB, value); + + value = db.StringGet(keyA); + ClassicAssert.AreEqual(null, value); + } + + [Test] + public void StrlenTestOnEtagSetData() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + db.Execute("SETWITHETAG", ["mykey", "foo bar"]); + + ClassicAssert.AreEqual(7, db.StringLength("mykey")); + ClassicAssert.AreEqual(0, db.StringLength("nokey")); + + var etagToCheck = db.Execute("GETWITHETAG", "mykey"); + ClassicAssert.AreEqual(0, long.Parse(etagToCheck[0].ToString())); + } + + [Test] + public void TTLTestMillisecondsForEtagSetData() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + var key = "myKey"; + var val = "myKeyValue"; + var expireTimeInMilliseconds = 3000; + + var pttl = db.Execute("PTTL", key); + ClassicAssert.AreEqual(-2, (int)pttl); + + db.Execute("SETWITHETAG", [key, val]); + + pttl = db.Execute("PTTL", key); + ClassicAssert.AreEqual(-1, (int)pttl); + + db.KeyExpire(key, TimeSpan.FromMilliseconds(expireTimeInMilliseconds)); + + //check TTL of the key in milliseconds + pttl = db.Execute("PTTL", key); + + ClassicAssert.IsTrue(long.TryParse(pttl.ToString(), out var pttlInMs)); + ClassicAssert.IsTrue(pttlInMs > 0); + + db.KeyPersist(key); + Thread.Sleep(expireTimeInMilliseconds); + + var _val = db.StringGet(key); + ClassicAssert.AreEqual(val, _val.ToString()); + + var ttl = db.KeyTimeToLive(key); + ClassicAssert.IsNull(ttl); + + // nothing should have affected the etag in the above commands + long etagToCheck = long.Parse(((RedisResult[])db.Execute("GETWITHETAG", [key]))[0].ToString()); + ClassicAssert.AreEqual(0, etagToCheck); + } + + [Test] + public void GetDelTestForEtagSetData() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + var key = "myKey"; + var val = "myKeyValue"; + + // Key Setup + db.Execute("SETWITHETAG", [key, val]); + + var retval = db.StringGet(key); + ClassicAssert.AreEqual(val, retval.ToString()); + + retval = db.StringGetDelete(key); + ClassicAssert.AreEqual(val, retval.ToString()); + + // Try retrieving already deleted key + retval = db.StringGetDelete(key); + ClassicAssert.AreEqual(string.Empty, retval.ToString()); + + // Try retrieving & deleting non-existent key + retval = db.StringGetDelete("nonExistentKey"); + ClassicAssert.AreEqual(string.Empty, retval.ToString()); + + // Key setup with metadata + key = "myKeyWithMetadata"; + val = "myValueWithMetadata"; + + db.Execute("SETWITHETAG", [key, val]); + db.KeyExpire(key, TimeSpan.FromSeconds(10000)); + + retval = db.StringGet(key); + ClassicAssert.AreEqual(val, retval.ToString()); + + retval = db.StringGetDelete(key); + ClassicAssert.AreEqual(val, retval.ToString()); + + // Try retrieving already deleted key with metadata + retval = db.StringGetDelete(key); + ClassicAssert.AreEqual(string.Empty, retval.ToString()); + } + + [Test] + public void AppendTestForEtagSetData() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + var key = "myKey"; + var val = "myKeyValue"; + var val2 = "myKeyValue2"; + + db.Execute("SETWITHETAG", [key, val]); + + var len = db.StringAppend(key, val2); + ClassicAssert.AreEqual(val.Length + val2.Length, len); + + var _val = db.StringGet(key); + ClassicAssert.AreEqual(val + val2, _val.ToString()); + + long etagToCheck = long.Parse(((RedisResult[])db.Execute("GETWITHETAG", [key]))[0].ToString()); + ClassicAssert.AreEqual(1, etagToCheck); + + db.KeyDelete(key); + + // Test appending an empty string + db.Execute("SETWITHETAG", [key, val]); + + var len1 = db.StringAppend(key, ""); + ClassicAssert.AreEqual(val.Length, len1); + + _val = db.StringGet(key); + ClassicAssert.AreEqual(val, _val.ToString()); + + etagToCheck = long.Parse(((RedisResult[])db.Execute("GETWITHETAG", [key]))[0].ToString()); + // we appended nothing so this remains 0 + ClassicAssert.AreEqual(0, etagToCheck); + + // Test appending to a non-existent key + var nonExistentKey = "nonExistentKey"; + var len2 = db.StringAppend(nonExistentKey, val2); + ClassicAssert.AreEqual(val2.Length, len2); + + _val = db.StringGet(nonExistentKey); + ClassicAssert.AreEqual(val2, _val.ToString()); + + db.KeyDelete(key); + + // Test appending to a key with a large value + var largeVal = new string('a', 1000000); + db.Execute("SETWITHETAG", [key, largeVal]); + var len3 = db.StringAppend(key, val2); + ClassicAssert.AreEqual(largeVal.Length + val2.Length, len3); + + etagToCheck = long.Parse(((RedisResult[])db.Execute("GETWITHETAG", [key]))[0].ToString()); + ClassicAssert.AreEqual(1, etagToCheck); + + // Test appending to a key with metadata + var keyWithMetadata = "keyWithMetadata"; + db.Execute("SETWITHETAG", [keyWithMetadata, val]); + db.KeyExpire(keyWithMetadata, TimeSpan.FromSeconds(10000)); + etagToCheck = long.Parse(((RedisResult[])db.Execute("GETWITHETAG", [keyWithMetadata]))[0].ToString()); + ClassicAssert.AreEqual(0, etagToCheck); + + var len4 = db.StringAppend(keyWithMetadata, val2); + ClassicAssert.AreEqual(val.Length + val2.Length, len4); + + _val = db.StringGet(keyWithMetadata); + ClassicAssert.AreEqual(val + val2, _val.ToString()); + + var time = db.KeyTimeToLive(keyWithMetadata); + ClassicAssert.IsTrue(time.Value.TotalSeconds > 0); + + etagToCheck = long.Parse(((RedisResult[])db.Execute("GETWITHETAG", [keyWithMetadata]))[0].ToString()); + ClassicAssert.AreEqual(1, etagToCheck); + } + + [Test] + public void SetBitOperationsOnEtagSetData() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + string key = "miki"; + // 64 BIT BITMAP + Byte[] initialBitmap = new byte[8]; + string bitMapAsStr = Encoding.UTF8.GetString(initialBitmap); ; + + db.Execute("SETWITHETAG", [key, bitMapAsStr]); + + long setbits = db.StringBitCount(key); + ClassicAssert.AreEqual(0, setbits); + + long etagToCheck = long.Parse(((RedisResult[])db.Execute("GETWITHETAG", [key]))[0].ToString()); + ClassicAssert.AreEqual(0, etagToCheck); + + // set all 64 bits one by one + var expectedBitCount = 0; + for (int i = 0; i < 64; i++) + { + // SET the ith bit in the bitmap + bool originalValAtBit = db.StringSetBit(key, i, true); + ClassicAssert.IsFalse(originalValAtBit); + + expectedBitCount++; + + bool currentBitVal = db.StringGetBit(key, i); + ClassicAssert.IsTrue(currentBitVal); + + setbits = db.StringBitCount(key); + ClassicAssert.AreEqual(expectedBitCount, setbits); + + // Use BitPosition to find the first set bit + long firstSetBitPosition = db.StringBitPosition(key, true); + ClassicAssert.AreEqual(0, firstSetBitPosition); // As we are setting bits in order, first set bit should be 0 + + // find the first unset bit + long firstUnsetBitPos = db.StringBitPosition(key, false); + long firstUnsetBitPosExpected = i == 63 ? -1 : i + 1; + ClassicAssert.AreEqual(firstUnsetBitPosExpected, firstUnsetBitPos); // As we are setting bits in order, first unset bit should be 1 ahead + + + // with each bit set that we do, we are increasing the etag as well by 1 + etagToCheck = long.Parse(((RedisResult[])db.Execute("GETWITHETAG", [key]))[0].ToString()); + ClassicAssert.AreEqual(expectedBitCount, etagToCheck); + } + + var expectedEtag = expectedBitCount; + // unset all 64 bits one by one in reverse order + for (int i = 63; i > -1; i--) + { + bool originalValAtBit = db.StringSetBit(key, i, false); + ClassicAssert.IsTrue(originalValAtBit); + + expectedEtag++; + expectedBitCount--; + + bool currentBitVal = db.StringGetBit(key, i); + ClassicAssert.IsFalse(currentBitVal); + + setbits = db.StringBitCount(key); + ClassicAssert.AreEqual(expectedBitCount, setbits); + + // find the first set bit + long firstSetBit = db.StringBitPosition(key, true); + long expectedSetBit = i == 0 ? -1 : 0; + ClassicAssert.AreEqual(expectedSetBit, firstSetBit); + + // Use BitPosition to find the first unset bit + long firstUnsetBitPosition = db.StringBitPosition(key, false); + ClassicAssert.AreEqual(i, firstUnsetBitPosition); // After unsetting, the first unset bit should be i + + etagToCheck = long.Parse(((RedisResult[])db.Execute("GETWITHETAG", [key]))[0].ToString()); + ClassicAssert.AreEqual(expectedEtag, etagToCheck); + } + } + + [Test] + public void BitFieldSetGetOnEtagSetData() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + var key = "mewo"; + + // Arrange - Set an 8-bit unsigned value at offset 0 + db.Execute("SETWITHETAG", [key, Encoding.UTF8.GetString(new byte[1])]); // Initialize key with an empty byte + + // Act - Set value to 127 (binary: 01111111) + db.Execute("BITFIELD", key, "SET", "u8", "0", "127"); + + long etagToCheck = long.Parse(((RedisResult[])db.Execute("GETWITHETAG", [key]))[0].ToString()); + ClassicAssert.AreEqual(1, etagToCheck); + + // Get value back + var getResult = (RedisResult[])db.Execute("BITFIELD", key, "GET", "u8", "0"); + + // Assert + ClassicAssert.AreEqual(127, (long)getResult[0]); // Ensure the value set was retrieved correctly + } + + [Test] + public void BitFieldIncrementWithWrapOverflowOnEtagSetData() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + var key = "mewo"; + + // Arrange - Set an 8-bit unsigned value at offset 0 + db.Execute("SETWITHETAG", [key, Encoding.UTF8.GetString(new byte[1])]); // Initialize key with an empty byte + + // Act - Set initial value to 255 and try to increment by 1 + db.Execute("BITFIELD", key, "SET", "u8", "0", "255"); + long etagToCheck = long.Parse(((RedisResult[])db.Execute("GETWITHETAG", [key]))[0].ToString()); + ClassicAssert.AreEqual(1, etagToCheck); + + var incrResult = db.Execute("BITFIELD", key, "INCRBY", "u8", "0", "1"); + + etagToCheck = long.Parse(((RedisResult[])db.Execute("GETWITHETAG", [key]))[0].ToString()); + ClassicAssert.AreEqual(2, etagToCheck); + + // Assert + ClassicAssert.AreEqual(0, (long)incrResult); // Should wrap around and return 0 + } + + [Test] + public void BitFieldIncrementWithSaturateOverflowOnEtagSetData() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + var key = "mewo"; + + // Arrange - Set an 8-bit unsigned value at offset 0 + db.Execute("SETWITHETAG", [key, Encoding.UTF8.GetString(new byte[1])]); // Initialize key with an empty byte + + // Act - Set initial value to 250 and try to increment by 10 with saturate overflow + db.Execute("BITFIELD", key, "SET", "u8", "0", "250"); + + long etagToCheck = long.Parse(((RedisResult[])db.Execute("GETWITHETAG", [key]))[0].ToString()); + ClassicAssert.AreEqual(1, etagToCheck); + + var incrResult = db.Execute("BITFIELD", key, "OVERFLOW", "SAT", "INCRBY", "u8", "0", "10"); + + etagToCheck = long.Parse(((RedisResult[])db.Execute("GETWITHETAG", [key]))[0].ToString()); + ClassicAssert.AreEqual(2, etagToCheck); + + // Assert + ClassicAssert.AreEqual(255, (long)incrResult); // Should saturate at the max value of 255 for u8 + } + + [Test] + public void HyperLogLogCommandsShouldReturnWrongTypeErrorForEtagSetData() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + var key = "mewo"; + var key2 = "dude"; + + db.Execute("SETWITHETAG", [key, "mars"]); + db.Execute("SETWITHETAG", [key2, "marsrover"]); + + RedisServerException ex = Assert.Throws(() => db.Execute("PFADD", [key, "woohoo"])); + + ClassicAssert.IsNotNull(ex); + ClassicAssert.AreEqual(Encoding.ASCII.GetString(CmdStrings.RESP_ERR_WRONG_TYPE_HLL), ex.Message); + + ex = Assert.Throws(() => db.Execute("PFMERGE", [key, key2])); + + ClassicAssert.IsNotNull(ex); + ClassicAssert.AreEqual(Encoding.ASCII.GetString(CmdStrings.RESP_ERR_WRONG_TYPE_HLL), ex.Message); + } + + [Test] + public void SetWithRetainEtagOnANewUpsertWillCreateKeyValueWithoutEtag() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + string key = "mickey"; + string val = "mouse"; + + // a new upsert on a non-existing key will retain the "nil" etag + db.Execute("SET", [key, val, "RETAINETAG"]).ToString(); + + RedisResult[] res = (RedisResult[])db.Execute("GETWITHETAG", [key]); + RedisResult etag = res[0]; + string value = res[1].ToString(); + + ClassicAssert.IsTrue(etag.IsNull); + ClassicAssert.AreEqual(val, value); + + string newval = "clubhouse"; + + // a new upsert on an existing key will retain the "nil" etag from the prev + db.Execute("SET", [key, newval, "RETAINETAG"]).ToString(); + res = (RedisResult[])db.Execute("GETWITHETAG", [key]); + etag = res[0]; + value = res[1].ToString(); + + ClassicAssert.IsTrue(etag.IsNull); + ClassicAssert.AreEqual(newval, value); + } + + #endregion + } +} \ No newline at end of file diff --git a/test/Garnet.test/RespTests.cs b/test/Garnet.test/RespTests.cs index d49c8c48e1..d85f7cd511 100644 --- a/test/Garnet.test/RespTests.cs +++ b/test/Garnet.test/RespTests.cs @@ -1848,6 +1848,48 @@ public void SingleRenameNxWithOldKeyAndNewKeyAsSame() ClassicAssert.AreEqual(origValue, retValue); } + [Test] + public void SingleRenameNXWithEtagSetOldAndNewKey() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + var origValue = "test1"; + var key = "key1"; + var newKey = "key2"; + + db.Execute("SETWITHETAG", key, origValue); + db.Execute("SETWITHETAG", newKey, "foo"); + + var result = db.KeyRename(key, newKey, When.NotExists); + ClassicAssert.IsFalse(result); + } + + [Test] + public void SingleRenameNXWithEtagSetOldKey() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + var origValue = "test1"; + var key = "key1"; + var newKey = "key2"; + + db.Execute("SETWITHETAG", key, origValue); + + var result = db.KeyRename(key, newKey, When.NotExists); + ClassicAssert.IsTrue(result); + + string retValue = db.StringGet(newKey); + ClassicAssert.AreEqual(origValue, retValue); + + var oldKeyRes = db.StringGet(key); + ClassicAssert.IsTrue(oldKeyRes.IsNull); + + // Since the original key was set with etag, the new key should have an etag attached to it + var etagRes = (RedisResult[])db.Execute("GETWITHETAG", newKey); + ClassicAssert.AreEqual(0, (long)etagRes[0]); + ClassicAssert.AreEqual(origValue, etagRes[1].ToString()); + } + #endregion [Test] diff --git a/test/Garnet.test/TransactionTests.cs b/test/Garnet.test/TransactionTests.cs index 76e91799b1..0fde102b8f 100644 --- a/test/Garnet.test/TransactionTests.cs +++ b/test/Garnet.test/TransactionTests.cs @@ -2,6 +2,7 @@ // Licensed under the MIT license. using System; +using System.Text; using System.Threading.Tasks; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -212,10 +213,66 @@ public async Task SimpleWatchTest() res = lightClientRequest.SendCommand("EXEC"); expectedResponse = "*2\r\n$14\r\nvalue1_updated\r\n+OK\r\n"; - ClassicAssert.AreEqual(res.AsSpan().Slice(0, expectedResponse.Length).ToArray(), expectedResponse); + ClassicAssert.AreEqual( + expectedResponse, + Encoding.ASCII.GetString(res.AsSpan().Slice(0, expectedResponse.Length))); } + [Test] + public async Task WatchTestWithSetWithEtag() + { + var lightClientRequest = TestUtils.CreateRequest(); + byte[] res; + + string expectedResponse = ":0\r\n"; + res = lightClientRequest.SendCommand("SETWITHETAG key1 value1"); + ClassicAssert.AreEqual(res.AsSpan().Slice(0, expectedResponse.Length).ToArray(), expectedResponse); + + expectedResponse = "+OK\r\n"; + res = lightClientRequest.SendCommand("WATCH key1"); + ClassicAssert.AreEqual(res.AsSpan().Slice(0, expectedResponse.Length).ToArray(), expectedResponse); + + res = lightClientRequest.SendCommand("MULTI"); + ClassicAssert.AreEqual(res.AsSpan().Slice(0, expectedResponse.Length).ToArray(), expectedResponse); + + res = lightClientRequest.SendCommand("GET key1"); + expectedResponse = "+QUEUED\r\n"; + ClassicAssert.AreEqual(res.AsSpan().Slice(0, expectedResponse.Length).ToArray(), expectedResponse); + res = lightClientRequest.SendCommand("SET key2 value2"); + ClassicAssert.AreEqual(res.AsSpan().Slice(0, expectedResponse.Length).ToArray(), expectedResponse); + + await Task.Run(() => updateKey("key1", "value1_updated", retainEtag: true)); + + res = lightClientRequest.SendCommand("EXEC"); + expectedResponse = "*-1"; + ClassicAssert.AreEqual(res.AsSpan().Slice(0, expectedResponse.Length).ToArray(), expectedResponse); + + // This one should Commit + lightClientRequest.SendCommand("MULTI"); + + lightClientRequest.SendCommand("GET key1"); + lightClientRequest.SendCommand("SET key2 value2"); + // check that all the etag commands can be called inside a transaction + lightClientRequest.SendCommand("SETWITHETAG key3 value2"); + lightClientRequest.SendCommand("GETWITHETAG key3"); + lightClientRequest.SendCommand("GETIFNOTMATCH key3 0"); + lightClientRequest.SendCommand("SETIFMATCH key3 anotherVal 0"); + lightClientRequest.SendCommand("SETWITHETAG key3 arandomval RETAINETAG"); + + res = lightClientRequest.SendCommand("EXEC"); + + expectedResponse = "*7\r\n$14\r\nvalue1_updated\r\n+OK\r\n:0\r\n*2\r\n:0\r\n$6\r\nvalue2\r\n+NOTCHANGED\r\n*2\r\n:1\r\n$10\r\nanotherVal\r\n:2\r\n"; + string response = Encoding.ASCII.GetString(res.AsSpan().Slice(0, expectedResponse.Length)); + ClassicAssert.AreEqual(response, expectedResponse); + + // check if we still have the appropriate etag on the key we had set + var otherLighClientRequest = TestUtils.CreateRequest(); + res = otherLighClientRequest.SendCommand("GETWITHETAG key1"); + expectedResponse = "*2\r\n:1\r\n$14\r\nvalue1_updated\r\n"; + response = Encoding.ASCII.GetString(res.AsSpan().Slice(0, expectedResponse.Length)); + ClassicAssert.AreEqual(response, expectedResponse); + } [Test] public async Task WatchNonExistentKey() @@ -304,10 +361,12 @@ public async Task WatchKeyFromDisk() ClassicAssert.AreEqual(res.AsSpan().Slice(0, expectedResponse.Length).ToArray(), expectedResponse); } - private static void updateKey(string key, string value) + private static void updateKey(string key, string value, bool retainEtag = false) { using var lightClientRequest = TestUtils.CreateRequest(); - byte[] res = lightClientRequest.SendCommand("SET " + key + " " + value); + string command = $"SET {key} {value}"; + command += retainEtag ? " RETAINETAG" : ""; + byte[] res = lightClientRequest.SendCommand(command); string expectedResponse = "+OK\r\n"; ClassicAssert.AreEqual(res.AsSpan().Slice(0, expectedResponse.Length).ToArray(), expectedResponse); } diff --git a/website/docs/commands/etag-commands.md b/website/docs/commands/etag-commands.md new file mode 100644 index 0000000000..7d1cef53be --- /dev/null +++ b/website/docs/commands/etag-commands.md @@ -0,0 +1,109 @@ +--- +id: etag-commands +sidebar_label: ETags +title: ETAG +slug: etag +--- + +--- + +## ETag Support + +Garnet provides support for ETags on raw strings. By using the ETag-related commands outlined below, you can associate any string-based key-value pair inserted into Garnet with an automatically updated ETag. + +Compatibility with non-ETag commands and the behavior of data inserted with ETags are detailed at the end of this document. + +--- + +### **SETWITHETAG** + +#### **Syntax** + +```bash +SETWITHETAG key value [RETAINETAG] +``` + +Inserts a key-value string pair into Garnet, associating an ETag that will be updated upon changes to the value. + +**Options:** + +* RETAINETAG -- Update the Etag associated with the previous key-value pair, while setting the new value for the key. If not etag existed for the previous key this will initialize one. + +#### **Response** + +- **Integer reply**: A response integer indicating the initial ETag value on success. + +--- + +### **GETWITHETAG** + +#### **Syntax** + +```bash +GETWITHETAG key +``` + +Retrieves the value and the ETag associated with the given key. + +#### **Response** + +One of the following: + +- **Array reply**: An array of two items returned on success. The first item is an integer representing the ETag, and the second is the bulk string value of the key. If called on a key-value pair without ETag, the first item will be nil. +- **Nil reply**: If the key does not exist. + +--- + +### **SETIFMATCH** + +#### **Syntax** + +```bash +SETIFMATCH key value etag +``` + +Updates the value of a key if the provided ETag matches the current ETag of the key. + +#### **Response** + +One of the following: + +- **Integer reply**: The updated ETag if the value was successfully updated. +- **Nil reply**: If the key does not exist. +- **Simple string reply**: If the provided ETag does not match the current ETag or If the command is called on a record without an ETag a simple string indicating ETag mismatch is returned. + +--- + +### **GETIFNOTMATCH** + +#### **Syntax** + +```bash +GETIFNOTMATCH key etag +``` + +Retrieves the value if the ETag associated with the key has changed; otherwise, returns a response indicating no change. + +#### **Response** + +One of the following: + +- **Array reply**: If the ETag does not match, an array of two items is returned. The first item is the updated ETag, and the second item is the value associated with the key. If called on a record without an ETag the first item in the array will be nil. +- **Nil reply**: If the key does not exist. +- **Simple string reply**: if the provided ETag matches the current ETag, returns a simple string indicating the value is unchanged. + +--- + +## Compatibility and Behavior with Non-ETag Commands + +Below is the expected behavior of ETag-associated key-value pairs when non-ETag commands are used. + +- **MSET, BITOP**: These commands will replace an existing ETag-associated key-value pair with a non-ETag key-value pair, effectively removing the ETag. + +- **SET**: Only if used with additional option "RETAINETAG" will calling SET update the etag while inserting the new key-value pair over the existing key-value pair. + +- **RENAME**: Renaming an ETag-associated key-value pair will reset the ETag to 0 for the renamed key. Unless the key being renamed to already existed before hand, in that case it will retain the etag of the existing key that was the target of the rename. + +All other commands will update the etag internally if they modify the underlying data, and any responses from them will not expose the etag to the client. To the users the etag and it's updates remain hidden in non-etag commands. + +--- \ No newline at end of file diff --git a/website/docs/commands/raw-string.md b/website/docs/commands/raw-string.md index c673da5b1b..3dc85182c8 100644 --- a/website/docs/commands/raw-string.md +++ b/website/docs/commands/raw-string.md @@ -289,7 +289,7 @@ Simple string reply: OK. #### Syntax ```bash - SET key value [NX | XX] [GET] [EX seconds | PX milliseconds | + SET key value [NX | XX] [GET] [EX seconds | PX milliseconds] [KEEPTTL] [RETAINETAG] ``` Set **key** to hold the string value. If key already holds a value, it is overwritten, regardless of its type. Any previous time to live associated with the **key** is discarded on successful SET operation. @@ -301,6 +301,7 @@ Set **key** to hold the string value. If key already holds a value, it is overwr * NX -- Only set the key if it does not already exist. * XX -- Only set the key if it already exists. * KEEPTTL -- Retain the time to live associated with the key. +* RETAINETAG -- Update the Etag associated with the previous key-value pair, while setting the new value for the key. If no etag existed on the previous key-value pair this will create the new key-value pair without any etag as well. #### Resp Reply diff --git a/website/sidebars.js b/website/sidebars.js index 6e019e3563..b83b69815f 100644 --- a/website/sidebars.js +++ b/website/sidebars.js @@ -20,7 +20,7 @@ const sidebars = { {type: 'category', label: 'Welcome', collapsed: false, items: ["welcome/intro", "welcome/news", "welcome/features", "welcome/releases", "welcome/compatibility", "welcome/roadmap", "welcome/faq", "welcome/about-us"]}, {type: 'category', label: 'Getting Started', items: ["getting-started/build", "getting-started/configuration", "getting-started/memory", "getting-started/security", "getting-started/compaction"]}, {type: 'category', label: 'Benchmarking', items: ["benchmarking/overview", "benchmarking/results-resp-bench", "benchmarking/resp-bench"]}, - {type: 'category', label: 'Commands', items: ["commands/overview", "commands/api-compatibility", "commands/raw-string", "commands/generic-commands", "commands/analytics-commands", "commands/data-structures", "commands/server-commands", "commands/client-commands", "commands/checkpoint-commands", "commands/transactions-commands", "commands/cluster", "commands/acl-commands", "commands/scripting-commands", "commands/garnet-specific-commands"]}, + {type: 'category', label: 'Commands', items: ["commands/overview", "commands/api-compatibility", "commands/raw-string", "commands/etag-commands", "commands/generic-commands", "commands/analytics-commands", "commands/data-structures", "commands/server-commands", "commands/client-commands", "commands/checkpoint-commands", "commands/transactions-commands", "commands/cluster", "commands/acl-commands", "commands/scripting-commands", "commands/garnet-specific-commands"]}, {type: 'category', label: 'Server Extensions', items: ["extensions/overview", "extensions/raw-strings", "extensions/objects", "extensions/transactions", "extensions/procedure", "extensions/module"]}, {type: 'category', label: 'Cluster Mode', items: ["cluster/overview", "cluster/replication", "cluster/key-migration"]}, {type: 'category', label: 'Developer Guide', items: ["dev/onboarding", "dev/code-structure", "dev/configuration", "dev/network", "dev/processing", "dev/garnet-api",