Skip to content

Commit

Permalink
Merged latest
Browse files Browse the repository at this point in the history
  • Loading branch information
Vijay-Nirmal committed Feb 14, 2025
2 parents 02b6077 + d7d2bc6 commit a32bd7e
Show file tree
Hide file tree
Showing 152 changed files with 8,266 additions and 4,941 deletions.
28 changes: 16 additions & 12 deletions .azure/pipelines/azure-pipelines-external-release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -58,12 +58,14 @@ jobs:
- task: EsrpCodeSigning@5
displayName: Sign the binaries for nuget and zipped files
inputs:
ConnectedServiceName: 'GarnetCodeSigningOneCert'
AppRegistrationClientId: '19bbb452-ce7a-4b3d-bdc4-42a0090e797b'
AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47'
AuthAKVName: 'GarnetCodeSignKV'
AuthCertName: 'Garnet-CodeSign-AuthCert'
AuthSignCertName: 'Garnet-CodeSigning-SigningCertificate'
ConnectedServiceName: 'GarnetCodeSignConn'
UseMSIAuthentication: false
AppRegistrationClientId: 'ff81d1e2-748b-49e1-a771-b57246668c2b'
AppRegistrationTenantId: '975f013f-7f24-47e8-a7d3-abc4752bf346'
AuthAKVName: 'GarnetCodeSignKeyVault'
AuthSignCertName: 'garnet-codesign-signing-cert'
AuthCertName: 'garnet-codesign-auth-cert'
ServiceEndpointUrl: 'https://api.esrp.microsoft.com/api/v2'
FolderPath: .
Pattern: Garnet*.dll,Tsavorite*.dll,Garnet*.exe,HdrHistogram.dll,native_device.dll,*Lua.dll
signConfigType: inlineSignParams
Expand Down Expand Up @@ -108,12 +110,14 @@ jobs:
- task: EsrpCodeSigning@5
displayName: Sign the NuGet Packages
inputs:
ConnectedServiceName: 'GarnetCodeSigningOneCert'
AppRegistrationClientId: '19bbb452-ce7a-4b3d-bdc4-42a0090e797b'
AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47'
AuthAKVName: 'GarnetCodeSignKV'
AuthCertName: 'Garnet-CodeSign-AuthCert'
AuthSignCertName: 'Garnet-CodeSigning-SigningCertificate'
ConnectedServiceName: 'GarnetCodeSignConn'
UseMSIAuthentication: false
AppRegistrationClientId: 'ff81d1e2-748b-49e1-a771-b57246668c2b'
AppRegistrationTenantId: '975f013f-7f24-47e8-a7d3-abc4752bf346'
AuthAKVName: 'GarnetCodeSignKeyVault'
AuthSignCertName: 'garnet-codesign-signing-cert'
AuthCertName: 'garnet-codesign-auth-cert'
ServiceEndpointUrl: 'https://api.esrp.microsoft.com/api/v2'
FolderPath: $(Build.ArtifactStagingDirectory)
Pattern: Microsoft.Garnet.*.nupkg, garnet-server.*.nupkg
signConfigType: inlineSignParams
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/ci-bdnbenchmark.yml
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ jobs:
os: [ ubuntu-latest, windows-latest ]
framework: [ 'net8.0' ]
configuration: [ 'Release' ]
test: [ 'Operations.BasicOperations', 'Operations.ObjectOperations', 'Operations.HashObjectOperations', 'Cluster.ClusterMigrate', 'Cluster.ClusterOperations', 'Lua.LuaScripts', 'Lua.LuaScriptCacheOperations','Lua.LuaRunnerOperations','Operations.CustomOperations', 'Operations.RawStringOperations', 'Operations.ScriptOperations', 'Operations.JsonOperations', 'Operations.ModuleOperations', 'Operations.PubSubOperations', 'Network.BasicOperations', 'Network.RawStringOperations' ]
test: [ 'Operations.BasicOperations', 'Operations.ObjectOperations', 'Operations.HashObjectOperations', 'Operations.SortedSetOperations', 'Cluster.ClusterMigrate', 'Cluster.ClusterOperations', 'Lua.LuaScripts', 'Lua.LuaScriptCacheOperations','Lua.LuaRunnerOperations','Operations.CustomOperations', 'Operations.RawStringOperations', 'Operations.ScriptOperations', 'Operations.JsonOperations', 'Operations.ModuleOperations', 'Operations.PubSubOperations', 'Network.BasicOperations', 'Network.RawStringOperations' ]
steps:
- name: Check out code
uses: actions/checkout@v4
Expand Down
2 changes: 1 addition & 1 deletion Version.props
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
<Project>
<!-- Versioning property for builds and packages -->
<PropertyGroup>
<VersionPrefix>1.0.54</VersionPrefix>
<VersionPrefix>1.0.55</VersionPrefix>
</PropertyGroup>
</Project>
3 changes: 2 additions & 1 deletion benchmark/BDN.benchmark/Cluster/ClusterContext.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.

using System.Net;
using System.Runtime.InteropServices;
using System.Text;
using BDN.benchmark.CustomProcs;
Expand Down Expand Up @@ -34,7 +35,7 @@ public void SetupSingleInstance(bool disableSlotVerification = false)
{
QuietMode = true,
EnableCluster = !disableSlotVerification,
Port = port,
EndPoint = new IPEndPoint(IPAddress.Loopback, port),
CleanClusterConfig = true,
};
if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
Expand Down
6 changes: 6 additions & 0 deletions benchmark/BDN.benchmark/Embedded/EmbeddedNetworkHandler.cs
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,12 @@ public EmbeddedNetworkHandler(GarnetServerEmbedded serverHook, EmbeddedNetworkSe

public override string RemoteEndpointName => throw new NotImplementedException();
public override string LocalEndpointName => throw new NotImplementedException();

public override bool IsLocalConnection()
{
return false;
}

public override void Dispose()
{
DisposeImpl();
Expand Down
5 changes: 5 additions & 0 deletions benchmark/BDN.benchmark/Embedded/EmbeddedNetworkSender.cs
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,11 @@ public EmbeddedNetworkSender()

public string LocalEndpointName => "";

public bool IsLocalConnection()
{
return true;
}

/// <summary>
/// Cleanup this DummyNetworkSender instance
/// </summary>
Expand Down
5 changes: 2 additions & 3 deletions benchmark/BDN.benchmark/Embedded/EmbeddedRespServer.cs
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ namespace Embedded.server
internal sealed class EmbeddedRespServer : GarnetServer
{
readonly GarnetServerEmbedded garnetServerEmbedded;
readonly SubscribeBroker<SpanByte, SpanByte, IKeySerializer<SpanByte>> subscribeBroker;
readonly SubscribeBroker subscribeBroker;

/// <summary>
/// Creates an EmbeddedRespServer instance
Expand All @@ -27,8 +27,7 @@ public EmbeddedRespServer(GarnetServerOptions opts, ILoggerFactory loggerFactory
{
this.garnetServerEmbedded = server;
this.subscribeBroker = opts.DisablePubSub ? null :
new SubscribeBroker<SpanByte, SpanByte, IKeySerializer<SpanByte>>(
new SpanByteKeySerializer(),
new SubscribeBroker(
null,
opts.PubSubPageSizeBytes(),
opts.SubscriberRefreshFrequencyMs,
Expand Down
7 changes: 4 additions & 3 deletions benchmark/BDN.benchmark/Embedded/GarnetServerEmbedded.cs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
// Licensed under the MIT license.

using System;
using System.Net;
using System.Net.Security;
using System.Threading;
using Garnet.common;
Expand All @@ -13,7 +14,7 @@ namespace Embedded.server
{
internal class GarnetServerEmbedded : GarnetServerBase, IServerHook
{
public GarnetServerEmbedded() : base("0.0.0.0", 0, 1 << 10)
public GarnetServerEmbedded() : base(new IPEndPoint(IPAddress.Loopback, 0), 1 << 10)
{
}

Expand All @@ -36,7 +37,7 @@ public EmbeddedNetworkHandler CreateNetworkHandler(SslClientAuthenticationOption
throw new Exception("Unable to add handler to dictionary");

handler.Start(tlsOptions, remoteEndpointName);
incr_conn_recv();
IncrementConnectionsReceived();
return handler;
}
catch (Exception ex)
Expand All @@ -59,7 +60,7 @@ public void DisposeMessageConsumer(INetworkHandler session)
if (activeHandlers.TryRemove(session, out _))
{
Interlocked.Decrement(ref activeHandlerCount);
incr_conn_disp();
IncrementConnectionsDisposed();
try
{
session.Session?.Dispose();
Expand Down
11 changes: 0 additions & 11 deletions benchmark/BDN.benchmark/Operations/ObjectOperations.cs
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,6 @@ namespace BDN.benchmark.Operations
[MemoryDiagnoser]
public unsafe class ObjectOperations : OperationsBase
{
static ReadOnlySpan<byte> ZADDREM => "*4\r\n$4\r\nZADD\r\n$1\r\nc\r\n$1\r\n1\r\n$1\r\nc\r\n*3\r\n$4\r\nZREM\r\n$1\r\nc\r\n$1\r\nc\r\n"u8;
Request zAddRem;

static ReadOnlySpan<byte> LPUSHPOP => "*3\r\n$5\r\nLPUSH\r\n$1\r\nd\r\n$1\r\ne\r\n*2\r\n$4\r\nLPOP\r\n$1\r\nd\r\n"u8;
Request lPushPop;

Expand All @@ -25,22 +22,14 @@ public override void GlobalSetup()
{
base.GlobalSetup();

SetupOperation(ref zAddRem, ZADDREM);
SetupOperation(ref lPushPop, LPUSHPOP);
SetupOperation(ref sAddRem, SADDREM);

// Pre-populate data
SlowConsumeMessage("*4\r\n$4\r\nZADD\r\n$1\r\nc\r\n$1\r\n1\r\n$1\r\nd\r\n"u8);
SlowConsumeMessage("*3\r\n$5\r\nLPUSH\r\n$1\r\nd\r\n$1\r\nf\r\n"u8);
SlowConsumeMessage("*3\r\n$4\r\nSADD\r\n$1\r\ne\r\n$1\r\nb\r\n"u8);
}

[Benchmark]
public void ZAddRem()
{
Send(zAddRem);
}

[Benchmark]
public void LPushPop()
{
Expand Down
7 changes: 1 addition & 6 deletions benchmark/BDN.benchmark/Operations/PubSubOperations.cs
Original file line number Diff line number Diff line change
Expand Up @@ -32,18 +32,13 @@ public override void GlobalSetup()
subscribeSession = server.GetRespSession();

SetupOperation(ref publish, PUBLISH);
SetupOperation(ref subscribe, SUBSCRIBE);
SetupOperation(ref subscribe, SUBSCRIBE, batchSize: 1);

// Subscribe to secondary session
_ = subscribeSession.TryConsumeMessages(subscribe.bufferPtr, subscribe.buffer.Length);

// Warm up
SlowConsumeMessage(new Span<byte>(publish.bufferPtr, publish.buffer.Length));
SlowConsumeMessage(new Span<byte>(publish.bufferPtr, publish.buffer.Length));
SlowConsumeMessage(new Span<byte>(publish.bufferPtr, publish.buffer.Length));
SlowConsumeMessage(new Span<byte>(publish.bufferPtr, publish.buffer.Length));
SlowConsumeMessage(new Span<byte>(publish.bufferPtr, publish.buffer.Length));
SlowConsumeMessage(new Span<byte>(publish.bufferPtr, publish.buffer.Length));
}

[Benchmark]
Expand Down
166 changes: 166 additions & 0 deletions benchmark/BDN.benchmark/Operations/SortedSetOperations.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,166 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.

using BenchmarkDotNet.Attributes;
using Embedded.server;

namespace BDN.benchmark.Operations
{
/// <summary>
/// Benchmark for SortedSetOperations
/// </summary>
[MemoryDiagnoser]
public unsafe class SortedSetOperations : OperationsBase
{
static ReadOnlySpan<byte> ZADDREM => "*4\r\n$4\r\nZADD\r\n$1\r\nc\r\n$1\r\n1\r\n$1\r\nc\r\n*3\r\n$4\r\nZREM\r\n$1\r\nc\r\n$1\r\nc\r\n"u8;
static ReadOnlySpan<byte> ZCARD => "*2\r\n$5\r\nZCARD\r\n$1\r\nc\r\n"u8;
static ReadOnlySpan<byte> ZCOUNT => "*4\r\n$6\r\nZCOUNT\r\n$1\r\nc\r\n$1\r\n0\r\n$1\r\n2\r\n"u8;
static ReadOnlySpan<byte> ZDIFF => "*4\r\n$5\r\nZDIFF\r\n$1\r\n2\r\n$1\r\nc\r\n$1\r\nh\r\n"u8;
static ReadOnlySpan<byte> ZDIFFSTORE => "*5\r\n$10\r\nZDIFFSTORE\r\n$4\r\ndest\r\n$1\r\n2\r\n$1\r\nc\r\n$1\r\nh\r\n"u8;
static ReadOnlySpan<byte> ZINCRBY => "*4\r\n$7\r\nZINCRBY\r\n$1\r\nc\r\n$1\r\n1\r\n$1\r\nd\r\n"u8;
static ReadOnlySpan<byte> ZINTER => "*4\r\n$6\r\nZINTER\r\n$1\r\n2\r\n$1\r\nc\r\n$1\r\nh\r\n"u8;
static ReadOnlySpan<byte> ZINTERCARD => "*4\r\n$10\r\nZINTERCARD\r\n$1\r\n2\r\n$1\r\nc\r\n$1\r\nh\r\n"u8;
static ReadOnlySpan<byte> ZINTERSTORE => "*5\r\n$11\r\nZINTERSTORE\r\n$4\r\ndest\r\n$1\r\n2\r\n$1\r\nc\r\n$1\r\nh\r\n"u8;
static ReadOnlySpan<byte> ZLEXCOUNT => "*4\r\n$9\r\nZLEXCOUNT\r\n$1\r\nc\r\n$1\r\n-\r\n$1\r\n+\r\n"u8;
static ReadOnlySpan<byte> ZMPOP => "*4\r\n$5\r\nZMPOP\r\n$1\r\n1\r\n$1\r\nc\r\n$3\r\nMIN\r\n*4\r\n$4\r\nZADD\r\n$1\r\nc\r\n$1\r\n1\r\n$1\r\nd\r\n"u8;
static ReadOnlySpan<byte> ZMSCORE => "*3\r\n$7\r\nZMSCORE\r\n$1\r\nc\r\n$1\r\nd\r\n"u8;
static ReadOnlySpan<byte> ZPOPMAX => "*2\r\n$7\r\nZPOPMAX\r\n$1\r\nc\r\n*4\r\n$4\r\nZADD\r\n$1\r\nc\r\n$1\r\n1\r\n$1\r\nd\r\n"u8;
static ReadOnlySpan<byte> ZPOPMIN => "*2\r\n$7\r\nZPOPMIN\r\n$1\r\nc\r\n*4\r\n$4\r\nZADD\r\n$1\r\nc\r\n$1\r\n1\r\n$1\r\nd\r\n"u8;
static ReadOnlySpan<byte> ZRANDMEMBER => "*2\r\n$10\r\nZRANDMEMBER\r\n$1\r\nc\r\n"u8;
static ReadOnlySpan<byte> ZRANGE => "*4\r\n$6\r\nZRANGE\r\n$1\r\nc\r\n$1\r\n0\r\n$1\r\n1\r\n"u8;
static ReadOnlySpan<byte> ZRANGESTORE => "*5\r\n$11\r\nZRANGESTORE\r\n$4\r\ndest\r\n$1\r\nc\r\n$1\r\n0\r\n$1\r\n1\r\n"u8;
static ReadOnlySpan<byte> ZRANK => "*3\r\n$5\r\nZRANK\r\n$1\r\nc\r\n$1\r\nd\r\n"u8;
static ReadOnlySpan<byte> ZREMRANGEBYLEX => "*4\r\n$14\r\nZREMRANGEBYLEX\r\n$1\r\nc\r\n$1\r\n-\r\n$1\r\n+\r\n*4\r\n$4\r\nZADD\r\n$1\r\nc\r\n$1\r\n1\r\n$1\r\nd\r\n"u8;
static ReadOnlySpan<byte> ZREMRANGEBYRANK => "*4\r\n$15\r\nZREMRANGEBYRANK\r\n$1\r\nc\r\n$1\r\n0\r\n$1\r\n1\r\n*4\r\n$4\r\nZADD\r\n$1\r\nc\r\n$1\r\n1\r\n$1\r\nd\r\n"u8;
static ReadOnlySpan<byte> ZREMRANGEBYSCORE => "*4\r\n$16\r\nZREMRANGEBYSCORE\r\n$1\r\nc\r\n$1\r\n0\r\n$1\r\n2\r\n*4\r\n$4\r\nZADD\r\n$1\r\nc\r\n$1\r\n1\r\n$1\r\nd\r\n"u8;
static ReadOnlySpan<byte> ZREVRANK => "*3\r\n$8\r\nZREVRANK\r\n$1\r\nc\r\n$1\r\nd\r\n"u8;
static ReadOnlySpan<byte> ZSCAN => "*6\r\n$5\r\nZSCAN\r\n$1\r\nc\r\n$1\r\n0\r\n$5\r\nCOUNT\r\n$1\r\n5\r\n"u8;
static ReadOnlySpan<byte> ZSCORE => "*3\r\n$6\r\nZSCORE\r\n$1\r\nc\r\n$1\r\nd\r\n"u8;
static ReadOnlySpan<byte> ZUNION => "*4\r\n$6\r\nZUNION\r\n$1\r\n2\r\n$1\r\nc\r\n$1\r\nh\r\n"u8;
static ReadOnlySpan<byte> ZUNIONSTORE => "*5\r\n$11\r\nZUNIONSTORE\r\n$4\r\ndest\r\n$1\r\n2\r\n$1\r\nc\r\n$1\r\nh\r\n"u8;

Request zAddRem, zCard, zCount, zDiff, zDiffStore, zIncrby, zInter, zInterCard,
zInterStore, zLexCount, zMPop, zMScore, zPopMax, zPopMin,
zRandMember, zRange, zRangeStore, zRank, zRemRangeByLex,
zRemRangeByRank, zRemRangeByScore, zRevRank, zScan,
zScore, zUnion, zUnionStore;

public override void GlobalSetup()
{
base.GlobalSetup();

SetupOperation(ref zAddRem, ZADDREM);
SetupOperation(ref zCard, ZCARD);
SetupOperation(ref zCount, ZCOUNT);
SetupOperation(ref zDiff, ZDIFF);
SetupOperation(ref zDiffStore, ZDIFFSTORE);
SetupOperation(ref zIncrby, ZINCRBY);
SetupOperation(ref zInter, ZINTER);
SetupOperation(ref zInterCard, ZINTERCARD);
SetupOperation(ref zInterStore, ZINTERSTORE);
SetupOperation(ref zLexCount, ZLEXCOUNT);
SetupOperation(ref zMPop, ZMPOP);
SetupOperation(ref zMScore, ZMSCORE);
SetupOperation(ref zPopMax, ZPOPMAX);
SetupOperation(ref zPopMin, ZPOPMIN);
SetupOperation(ref zRandMember, ZRANDMEMBER);
SetupOperation(ref zRange, ZRANGE);
SetupOperation(ref zRangeStore, ZRANGESTORE);
SetupOperation(ref zRank, ZRANK);
SetupOperation(ref zRemRangeByLex, ZREMRANGEBYLEX);
SetupOperation(ref zRemRangeByRank, ZREMRANGEBYRANK);
SetupOperation(ref zRemRangeByScore, ZREMRANGEBYSCORE);
SetupOperation(ref zRevRank, ZREVRANK);
SetupOperation(ref zScan, ZSCAN);
SetupOperation(ref zScore, ZSCORE);
SetupOperation(ref zUnion, ZUNION);
SetupOperation(ref zUnionStore, ZUNIONSTORE);

// Pre-populate data for two sorted sets
SlowConsumeMessage("*4\r\n$4\r\nZADD\r\n$1\r\nc\r\n$1\r\n1\r\n$1\r\nd\r\n"u8);
SlowConsumeMessage("*6\r\n$4\r\nZADD\r\n$1\r\nh\r\n$1\r\n1\r\n$1\r\nd\r\n$1\r\n2\r\n$1\r\ne\r\n"u8);
SlowConsumeMessage("*4\r\n$4\r\nZADD\r\n$4\r\ndest\r\n$1\r\n1\r\n$1\r\nd\r\n"u8);
}

[Benchmark]
public void ZAddRem()
{
Send(zAddRem);
}

[Benchmark]
public void ZCard() => Send(zCard);

[Benchmark]
public void ZCount() => Send(zCount);

[Benchmark]
public void ZDiff() => Send(zDiff);

[Benchmark]
public void ZDiffStore() => Send(zDiffStore);

[Benchmark]
public void ZIncrby() => Send(zIncrby);

[Benchmark]
public void ZInter() => Send(zInter);

[Benchmark]
public void ZInterCard() => Send(zInterCard);

[Benchmark]
public void ZInterStore() => Send(zInterStore);

[Benchmark]
public void ZLexCount() => Send(zLexCount);

[Benchmark]
public void ZMPop() => Send(zMPop);

[Benchmark]
public void ZMScore() => Send(zMScore);

[Benchmark]
public void ZPopMax() => Send(zPopMax);

[Benchmark]
public void ZPopMin() => Send(zPopMin);

[Benchmark]
public void ZRandMember() => Send(zRandMember);

[Benchmark]
public void ZRange() => Send(zRange);

[Benchmark]
public void ZRangeStore() => Send(zRangeStore);

[Benchmark]
public void ZRank() => Send(zRank);

[Benchmark]
public void ZRemRangeByLex() => Send(zRemRangeByLex);

[Benchmark]
public void ZRemRangeByRank() => Send(zRemRangeByRank);

[Benchmark]
public void ZRemRangeByScore() => Send(zRemRangeByScore);

[Benchmark]
public void ZRevRank() => Send(zRevRank);

[Benchmark]
public void ZScan() => Send(zScan);

[Benchmark]
public void ZScore() => Send(zScore);

[Benchmark]
public void ZUnion() => Send(zUnion);

[Benchmark]
public void ZUnionStore() => Send(zUnionStore);
}
}
Loading

0 comments on commit a32bd7e

Please sign in to comment.