From af1a61218f646ea293e649c5016607375195bfe8 Mon Sep 17 00:00:00 2001 From: Adam Larsen Date: Tue, 6 Feb 2024 01:16:05 -0600 Subject: [PATCH] Add support for an Advertised Host other than PodIP when using Kubernetes cluster provider (#2097) * Add Host to k8s pod labels * Removed logging of k8s workload kind * Added new Public API to get K8s Pod FQDN hardened GetKubeNamespace() so it will not read in more data then kuberneties allows for a namespace, it will also now throw a with a descriptive error exception message vs FileNotFound. --- ProtoActor.sln | 70 ++++++++++++++ .../ClusterK8sGrains/Clusterk8sGrains.sln | 28 ++++++ .../ClusterK8sGrains/Messages/Messages.csproj | 22 +++++ .../ClusterK8sGrains/Messages/Protos.proto | 12 +++ examples/ClusterK8sGrains/Node1/Dockerfile | 27 ++++++ examples/ClusterK8sGrains/Node1/Node1.csproj | 22 +++++ examples/ClusterK8sGrains/Node1/Program.cs | 88 ++++++++++++++++++ examples/ClusterK8sGrains/Node2/Dockerfile | 27 ++++++ examples/ClusterK8sGrains/Node2/Node2.csproj | 22 +++++ examples/ClusterK8sGrains/Node2/Program.cs | 93 +++++++++++++++++++ examples/ClusterK8sGrains/chart/.helmignore | 23 +++++ examples/ClusterK8sGrains/chart/Chart.yaml | 5 + .../chart/templates/namespace.yaml | 4 + .../chart/templates/node1-deployment.yaml | 26 ++++++ .../chart/templates/node2-deployment.yaml | 25 +++++ .../protoactor-k8s-cluster-port-service.yaml | 16 ++++ .../templates/protoactor-k8s-grains-role.yaml | 16 ++++ .../protoactor-k8s-grains-rolebinding.yaml | 14 +++ .../protoactor-k8s-grains-serviceaccount.yaml | 7 ++ examples/ClusterK8sGrains/chart/values.yaml | 13 +++ examples/ClusterK8sGrains/readme.md | 37 ++++++++ .../KubernetesClusterMonitor.cs | 2 +- .../KubernetesExtensions.cs | 92 +++++++++++++++--- .../KubernetesHelper.cs | 59 ++++++++++++ .../KubernetesProvider.cs | 50 +++++++++- .../KubernetesProviderConfig.cs | 13 ++- src/Proto.Cluster.Kubernetes/ProtoLabels.cs | 2 + 27 files changed, 796 insertions(+), 19 deletions(-) create mode 100644 examples/ClusterK8sGrains/Clusterk8sGrains.sln create mode 100644 examples/ClusterK8sGrains/Messages/Messages.csproj create mode 100644 examples/ClusterK8sGrains/Messages/Protos.proto create mode 100644 examples/ClusterK8sGrains/Node1/Dockerfile create mode 100644 examples/ClusterK8sGrains/Node1/Node1.csproj create mode 100644 examples/ClusterK8sGrains/Node1/Program.cs create mode 100644 examples/ClusterK8sGrains/Node2/Dockerfile create mode 100644 examples/ClusterK8sGrains/Node2/Node2.csproj create mode 100644 examples/ClusterK8sGrains/Node2/Program.cs create mode 100644 examples/ClusterK8sGrains/chart/.helmignore create mode 100644 examples/ClusterK8sGrains/chart/Chart.yaml create mode 100644 examples/ClusterK8sGrains/chart/templates/namespace.yaml create mode 100644 examples/ClusterK8sGrains/chart/templates/node1-deployment.yaml create mode 100644 examples/ClusterK8sGrains/chart/templates/node2-deployment.yaml create mode 100644 examples/ClusterK8sGrains/chart/templates/protoactor-k8s-cluster-port-service.yaml create mode 100644 examples/ClusterK8sGrains/chart/templates/protoactor-k8s-grains-role.yaml create mode 100644 examples/ClusterK8sGrains/chart/templates/protoactor-k8s-grains-rolebinding.yaml create mode 100644 examples/ClusterK8sGrains/chart/templates/protoactor-k8s-grains-serviceaccount.yaml create mode 100644 examples/ClusterK8sGrains/chart/values.yaml create mode 100644 examples/ClusterK8sGrains/readme.md create mode 100644 src/Proto.Cluster.Kubernetes/KubernetesHelper.cs diff --git a/ProtoActor.sln b/ProtoActor.sln index 77a26d372d..9d5f389977 100644 --- a/ProtoActor.sln +++ b/ProtoActor.sln @@ -278,6 +278,34 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Proto.Cluster.SeedNode.Mong EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GossipDecoder", "benchmarks\GossipDecoder\GossipDecoder.csproj", "{FC144547-78F5-4C0B-B886-B7BC1563893B}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "ClusterK8sGrains", "ClusterK8sGrains", "{ADE7A14E-FFE9-4137-AC25-E2F2A82B0A8C}" + ProjectSection(SolutionItems) = preProject + examples\ClusterK8sGrains\readme.md = examples\ClusterK8sGrains\readme.md + EndProjectSection +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Messages", "examples\ClusterK8sGrains\Messages\Messages.csproj", "{44B8EBA7-5A47-4D20-B62F-48413B676473}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Node1", "examples\ClusterK8sGrains\Node1\Node1.csproj", "{80DC28A1-D361-4A25-AC5D-8F5B6DCE276A}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Node2", "examples\ClusterK8sGrains\Node2\Node2.csproj", "{B196FBFE-0DAA-4533-9A56-BB5826A57923}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "chart", "chart", "{CDCE3D4C-1BDD-460F-93B8-75123A258183}" + ProjectSection(SolutionItems) = preProject + examples\ClusterK8sGrains\chart\Chart.yaml = examples\ClusterK8sGrains\chart\Chart.yaml + examples\ClusterK8sGrains\chart\values.yaml = examples\ClusterK8sGrains\chart\values.yaml + EndProjectSection +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "template", "template", "{087E5441-1582-4D55-8233-014C0FB06FF0}" + ProjectSection(SolutionItems) = preProject + examples\ClusterK8sGrains\chart\templates\namespace.yaml = examples\ClusterK8sGrains\chart\templates\namespace.yaml + examples\ClusterK8sGrains\chart\templates\node1-deployment.yaml = examples\ClusterK8sGrains\chart\templates\node1-deployment.yaml + examples\ClusterK8sGrains\chart\templates\node2-deployment.yaml = examples\ClusterK8sGrains\chart\templates\node2-deployment.yaml + examples\ClusterK8sGrains\chart\templates\protoactor-k8s-cluster-port-service.yaml = examples\ClusterK8sGrains\chart\templates\protoactor-k8s-cluster-port-service.yaml + examples\ClusterK8sGrains\chart\templates\protoactor-k8s-grains-role.yaml = examples\ClusterK8sGrains\chart\templates\protoactor-k8s-grains-role.yaml + examples\ClusterK8sGrains\chart\templates\protoactor-k8s-grains-rolebinding.yaml = examples\ClusterK8sGrains\chart\templates\protoactor-k8s-grains-rolebinding.yaml + examples\ClusterK8sGrains\chart\templates\protoactor-k8s-grains-serviceaccount.yaml = examples\ClusterK8sGrains\chart\templates\protoactor-k8s-grains-serviceaccount.yaml + EndProjectSection +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -1416,6 +1444,42 @@ Global {FC144547-78F5-4C0B-B886-B7BC1563893B}.Release|x64.Build.0 = Release|Any CPU {FC144547-78F5-4C0B-B886-B7BC1563893B}.Release|x86.ActiveCfg = Release|Any CPU {FC144547-78F5-4C0B-B886-B7BC1563893B}.Release|x86.Build.0 = Release|Any CPU + {44B8EBA7-5A47-4D20-B62F-48413B676473}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {44B8EBA7-5A47-4D20-B62F-48413B676473}.Debug|Any CPU.Build.0 = Debug|Any CPU + {44B8EBA7-5A47-4D20-B62F-48413B676473}.Debug|x64.ActiveCfg = Debug|Any CPU + {44B8EBA7-5A47-4D20-B62F-48413B676473}.Debug|x64.Build.0 = Debug|Any CPU + {44B8EBA7-5A47-4D20-B62F-48413B676473}.Debug|x86.ActiveCfg = Debug|Any CPU + {44B8EBA7-5A47-4D20-B62F-48413B676473}.Debug|x86.Build.0 = Debug|Any CPU + {44B8EBA7-5A47-4D20-B62F-48413B676473}.Release|Any CPU.ActiveCfg = Release|Any CPU + {44B8EBA7-5A47-4D20-B62F-48413B676473}.Release|Any CPU.Build.0 = Release|Any CPU + {44B8EBA7-5A47-4D20-B62F-48413B676473}.Release|x64.ActiveCfg = Release|Any CPU + {44B8EBA7-5A47-4D20-B62F-48413B676473}.Release|x64.Build.0 = Release|Any CPU + {44B8EBA7-5A47-4D20-B62F-48413B676473}.Release|x86.ActiveCfg = Release|Any CPU + {44B8EBA7-5A47-4D20-B62F-48413B676473}.Release|x86.Build.0 = Release|Any CPU + {80DC28A1-D361-4A25-AC5D-8F5B6DCE276A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {80DC28A1-D361-4A25-AC5D-8F5B6DCE276A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {80DC28A1-D361-4A25-AC5D-8F5B6DCE276A}.Debug|x64.ActiveCfg = Debug|Any CPU + {80DC28A1-D361-4A25-AC5D-8F5B6DCE276A}.Debug|x64.Build.0 = Debug|Any CPU + {80DC28A1-D361-4A25-AC5D-8F5B6DCE276A}.Debug|x86.ActiveCfg = Debug|Any CPU + {80DC28A1-D361-4A25-AC5D-8F5B6DCE276A}.Debug|x86.Build.0 = Debug|Any CPU + {80DC28A1-D361-4A25-AC5D-8F5B6DCE276A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {80DC28A1-D361-4A25-AC5D-8F5B6DCE276A}.Release|Any CPU.Build.0 = Release|Any CPU + {80DC28A1-D361-4A25-AC5D-8F5B6DCE276A}.Release|x64.ActiveCfg = Release|Any CPU + {80DC28A1-D361-4A25-AC5D-8F5B6DCE276A}.Release|x64.Build.0 = Release|Any CPU + {80DC28A1-D361-4A25-AC5D-8F5B6DCE276A}.Release|x86.ActiveCfg = Release|Any CPU + {80DC28A1-D361-4A25-AC5D-8F5B6DCE276A}.Release|x86.Build.0 = Release|Any CPU + {B196FBFE-0DAA-4533-9A56-BB5826A57923}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B196FBFE-0DAA-4533-9A56-BB5826A57923}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B196FBFE-0DAA-4533-9A56-BB5826A57923}.Debug|x64.ActiveCfg = Debug|Any CPU + {B196FBFE-0DAA-4533-9A56-BB5826A57923}.Debug|x64.Build.0 = Debug|Any CPU + {B196FBFE-0DAA-4533-9A56-BB5826A57923}.Debug|x86.ActiveCfg = Debug|Any CPU + {B196FBFE-0DAA-4533-9A56-BB5826A57923}.Debug|x86.Build.0 = Debug|Any CPU + {B196FBFE-0DAA-4533-9A56-BB5826A57923}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B196FBFE-0DAA-4533-9A56-BB5826A57923}.Release|Any CPU.Build.0 = Release|Any CPU + {B196FBFE-0DAA-4533-9A56-BB5826A57923}.Release|x64.ActiveCfg = Release|Any CPU + {B196FBFE-0DAA-4533-9A56-BB5826A57923}.Release|x64.Build.0 = Release|Any CPU + {B196FBFE-0DAA-4533-9A56-BB5826A57923}.Release|x86.ActiveCfg = Release|Any CPU + {B196FBFE-0DAA-4533-9A56-BB5826A57923}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -1545,6 +1609,12 @@ Global {5FECD1A8-A873-4927-81C3-E5C5A37D80C5} = {0F3AB331-C042-4371-A2F0-0AFDFA13DC9F} {6611DA4A-6471-45CE-A288-45BC7BF00B52} = {3D12F5E5-9774-4D7E-8A5B-B1F64544925B} {FC144547-78F5-4C0B-B886-B7BC1563893B} = {0F3AB331-C042-4371-A2F0-0AFDFA13DC9F} + {ADE7A14E-FFE9-4137-AC25-E2F2A82B0A8C} = {59DCCC96-DDAF-469F-9E8E-9BC733285082} + {44B8EBA7-5A47-4D20-B62F-48413B676473} = {ADE7A14E-FFE9-4137-AC25-E2F2A82B0A8C} + {80DC28A1-D361-4A25-AC5D-8F5B6DCE276A} = {ADE7A14E-FFE9-4137-AC25-E2F2A82B0A8C} + {B196FBFE-0DAA-4533-9A56-BB5826A57923} = {ADE7A14E-FFE9-4137-AC25-E2F2A82B0A8C} + {CDCE3D4C-1BDD-460F-93B8-75123A258183} = {ADE7A14E-FFE9-4137-AC25-E2F2A82B0A8C} + {087E5441-1582-4D55-8233-014C0FB06FF0} = {CDCE3D4C-1BDD-460F-93B8-75123A258183} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {CD0D1E44-8118-4682-8793-6B20ABFA824C} diff --git a/examples/ClusterK8sGrains/Clusterk8sGrains.sln b/examples/ClusterK8sGrains/Clusterk8sGrains.sln new file mode 100644 index 0000000000..fce2766600 --- /dev/null +++ b/examples/ClusterK8sGrains/Clusterk8sGrains.sln @@ -0,0 +1,28 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Messages", "Messages\Messages.csproj", "{A5E3B3E1-5517-4E08-9E50-50EF17215AB5}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Node2", "Node2\Node2.csproj", "{95685292-A5C9-48DA-871A-EB00A78A39A5}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Node1", "Node1\Node1.csproj", "{B8D93ECE-0203-41C2-B845-7BBDDBA222DE}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {A5E3B3E1-5517-4E08-9E50-50EF17215AB5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A5E3B3E1-5517-4E08-9E50-50EF17215AB5}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A5E3B3E1-5517-4E08-9E50-50EF17215AB5}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A5E3B3E1-5517-4E08-9E50-50EF17215AB5}.Release|Any CPU.Build.0 = Release|Any CPU + {95685292-A5C9-48DA-871A-EB00A78A39A5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {95685292-A5C9-48DA-871A-EB00A78A39A5}.Debug|Any CPU.Build.0 = Debug|Any CPU + {95685292-A5C9-48DA-871A-EB00A78A39A5}.Release|Any CPU.ActiveCfg = Release|Any CPU + {95685292-A5C9-48DA-871A-EB00A78A39A5}.Release|Any CPU.Build.0 = Release|Any CPU + {B8D93ECE-0203-41C2-B845-7BBDDBA222DE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B8D93ECE-0203-41C2-B845-7BBDDBA222DE}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B8D93ECE-0203-41C2-B845-7BBDDBA222DE}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B8D93ECE-0203-41C2-B845-7BBDDBA222DE}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection +EndGlobal diff --git a/examples/ClusterK8sGrains/Messages/Messages.csproj b/examples/ClusterK8sGrains/Messages/Messages.csproj new file mode 100644 index 0000000000..9d58765594 --- /dev/null +++ b/examples/ClusterK8sGrains/Messages/Messages.csproj @@ -0,0 +1,22 @@ + + + + net7.0 + 11 + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/examples/ClusterK8sGrains/Messages/Protos.proto b/examples/ClusterK8sGrains/Messages/Protos.proto new file mode 100644 index 0000000000..9d2f115490 --- /dev/null +++ b/examples/ClusterK8sGrains/Messages/Protos.proto @@ -0,0 +1,12 @@ +syntax = "proto3"; +package HelloHelloWorld; +option csharp_namespace = "ClusterHelloWorld.Messages"; + +message HelloRequest {} +message HelloResponse { + string Message=1; +} + +service HelloGrain { + rpc SayHello(HelloRequest) returns (HelloResponse) {} +} \ No newline at end of file diff --git a/examples/ClusterK8sGrains/Node1/Dockerfile b/examples/ClusterK8sGrains/Node1/Dockerfile new file mode 100644 index 0000000000..61b5064803 --- /dev/null +++ b/examples/ClusterK8sGrains/Node1/Dockerfile @@ -0,0 +1,27 @@ +FROM mcr.microsoft.com/dotnet/aspnet:7.0 AS base +WORKDIR /app + +FROM mcr.microsoft.com/dotnet/sdk:7.0 AS build +ARG BUILD_CONFIGURATION=Release +WORKDIR /src +COPY ["examples/ClusterK8sGrains/Node1/Node1.csproj", "examples/ClusterK8sGrains/Node1/"] +COPY ["examples/ClusterK8sGrains/Node2/Node2.csproj", "examples/ClusterK8sGrains/Node2/"] +COPY ["src/Proto.Cluster.Kubernetes/Proto.Cluster.Kubernetes.csproj", "src/Proto.Cluster.Kubernetes/"] +COPY ["src/Proto.Cluster/Proto.Cluster.csproj", "src/Proto.Cluster/"] +COPY ["src/Proto.Remote/Proto.Remote.csproj", "src/Proto.Remote/"] +COPY ["src/Proto.Actor/Proto.Actor.csproj", "src/Proto.Actor/"] +COPY ["examples/ClusterK8sGrains/Messages/Messages.csproj", "examples/ClusterK8sGrains/Messages/"] +RUN dotnet restore "examples/ClusterK8sGrains/Node1/Node1.csproj" +RUN dotnet restore "examples/ClusterK8sGrains/Node2/Node2.csproj" +COPY . . +WORKDIR "/src/examples/ClusterK8sGrains/Node1" +RUN dotnet build "Node1.csproj" -c $BUILD_CONFIGURATION -o /app/build + +FROM build AS publish +ARG BUILD_CONFIGURATION=Release +RUN dotnet publish "Node1.csproj" -c $BUILD_CONFIGURATION -o /app/publish /p:UseAppHost=false --no-restore + +FROM base AS final +WORKDIR /app +COPY --from=publish /app/publish . +ENTRYPOINT ["dotnet", "Node1.dll"] diff --git a/examples/ClusterK8sGrains/Node1/Node1.csproj b/examples/ClusterK8sGrains/Node1/Node1.csproj new file mode 100644 index 0000000000..0f5ad3a86f --- /dev/null +++ b/examples/ClusterK8sGrains/Node1/Node1.csproj @@ -0,0 +1,22 @@ + + + Exe + net7.0 + true + true + 10 + Linux + proto.actor example + + + + + + + + + + .dockerignore + + + \ No newline at end of file diff --git a/examples/ClusterK8sGrains/Node1/Program.cs b/examples/ClusterK8sGrains/Node1/Program.cs new file mode 100644 index 0000000000..5fbef14230 --- /dev/null +++ b/examples/ClusterK8sGrains/Node1/Program.cs @@ -0,0 +1,88 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2015-2022 Asynkron AB All rights reserved +// +// ----------------------------------------------------------------------- + +using System; +using System.Threading; +using System.Threading.Tasks; +using ClusterHelloWorld.Messages; +using Microsoft.Extensions.Logging; +using Proto; +using Proto.Cluster; +using Proto.Cluster.Kubernetes; +using Proto.Cluster.PartitionActivator; +using Proto.Remote; +using Proto.Remote.GrpcNet; +using static Proto.CancellationTokens; +using ProtosReflection = ClusterHelloWorld.Messages.ProtosReflection; +using System.Runtime.Loader; +using Microsoft.Extensions.Configuration; +using Extensions = Proto.Remote.GrpcNet.Extensions; + +// Hook SIGTERM to a cancel token to know when k8s is shutting us down +// hostBuilder should be used in production +var cts = new CancellationTokenSource(); +AssemblyLoadContext.Default.Unloading += ctx => cts.Cancel(); + +Log.SetLoggerFactory( + LoggerFactory.Create(l => l.AddConsole(options => + { + //options.FormatterName = "json"; // Use the JSON formatter + }).SetMinimumLevel(LogLevel.Debug) + .AddFilter("Proto.Cluster.Gossip", LogLevel.Information) + .AddFilter("Proto.Context.ActorContext", LogLevel.Information))); + +// Required to allow unencrypted GrpcNet connections +AppContext.SetSwitch("System.Net.Http.SocketsHttpHandler.Http2UnencryptedSupport", true); + +var kubernetesProvider = new KubernetesProvider(); +var advertisedHost = await kubernetesProvider.GetPodFqdn(); + +var system = new ActorSystem() + .WithRemote(GrpcNetRemoteConfig + .BindToAllInterfaces(advertisedHost: advertisedHost, port: 4020) + .WithProtoMessages(ProtosReflection.Descriptor)) + .WithCluster(ClusterConfig + .Setup("MyCluster", + kubernetesProvider, + new PartitionActivatorLookup()) + ); + +system.EventStream.Subscribe( + e => { Console.WriteLine($"{DateTime.Now:O} My members {e.TopologyHash}"); } +); + +await system + .Cluster() + .StartMemberAsync(); + +Console.WriteLine("Started"); + +try +{ + var helloGrain = system.Cluster().GetHelloGrain("MyGrain"); + + var res = await helloGrain.SayHello(new HelloRequest(), FromSeconds(15)); + Console.WriteLine(res?.Message ?? "RES IS NULL"); + + res = await helloGrain.SayHello(new HelloRequest(), FromSeconds(5)); + Console.WriteLine(res?.Message ?? "RES IS NULL"); +} +catch (Exception e) +{ + Log.CreateLogger("Program").LogError(e, "Error sending messages"); +} + +Console.WriteLine("Press CTRL-C to exit"); +Console.CancelKeyPress += (_, e) => +{ + e.Cancel = true; // prevent the process from terminating. + cts.Cancel(); +}; + +await Task.Delay(Timeout.Infinite, cts.Token); + +Console.WriteLine("Shutting Down..."); +await system.Cluster().ShutdownAsync(); \ No newline at end of file diff --git a/examples/ClusterK8sGrains/Node2/Dockerfile b/examples/ClusterK8sGrains/Node2/Dockerfile new file mode 100644 index 0000000000..2d701fecf4 --- /dev/null +++ b/examples/ClusterK8sGrains/Node2/Dockerfile @@ -0,0 +1,27 @@ +FROM mcr.microsoft.com/dotnet/aspnet:7.0 AS base +WORKDIR /app + +FROM mcr.microsoft.com/dotnet/sdk:7.0 AS build +ARG BUILD_CONFIGURATION=Release +WORKDIR /src +COPY ["examples/ClusterK8sGrains/Node1/Node1.csproj", "examples/ClusterK8sGrains/Node1/"] +COPY ["examples/ClusterK8sGrains/Node2/Node2.csproj", "examples/ClusterK8sGrains/Node2/"] +COPY ["src/Proto.Cluster.Kubernetes/Proto.Cluster.Kubernetes.csproj", "src/Proto.Cluster.Kubernetes/"] +COPY ["src/Proto.Cluster/Proto.Cluster.csproj", "src/Proto.Cluster/"] +COPY ["src/Proto.Remote/Proto.Remote.csproj", "src/Proto.Remote/"] +COPY ["src/Proto.Actor/Proto.Actor.csproj", "src/Proto.Actor/"] +COPY ["examples/ClusterK8sGrains/Messages/Messages.csproj", "examples/ClusterK8sGrains/Messages/"] +RUN dotnet restore "examples/ClusterK8sGrains/Node1/Node1.csproj" +RUN dotnet restore "examples/ClusterK8sGrains/Node2/Node2.csproj" +COPY . . +WORKDIR "/src/examples/ClusterK8sGrains/Node2" +RUN dotnet build "Node2.csproj" -c $BUILD_CONFIGURATION -o /app/build + +FROM build AS publish +ARG BUILD_CONFIGURATION=Release +RUN dotnet publish "Node2.csproj" -c $BUILD_CONFIGURATION -o /app/publish /p:UseAppHost=false --no-restore + +FROM base AS final +WORKDIR /app +COPY --from=publish /app/publish . +ENTRYPOINT ["dotnet", "Node2.dll"] diff --git a/examples/ClusterK8sGrains/Node2/Node2.csproj b/examples/ClusterK8sGrains/Node2/Node2.csproj new file mode 100644 index 0000000000..059a3f9d3c --- /dev/null +++ b/examples/ClusterK8sGrains/Node2/Node2.csproj @@ -0,0 +1,22 @@ + + + Exe + net7.0 + true + true + 11 + Linux + proto.actor example + + + + + + + + + + .dockerignore + + + \ No newline at end of file diff --git a/examples/ClusterK8sGrains/Node2/Program.cs b/examples/ClusterK8sGrains/Node2/Program.cs new file mode 100644 index 0000000000..caca7d070a --- /dev/null +++ b/examples/ClusterK8sGrains/Node2/Program.cs @@ -0,0 +1,93 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2015-2022 Asynkron AB All rights reserved +// +// ----------------------------------------------------------------------- + +using System; +using System.Runtime.Loader; +using System.Threading.Tasks; +using System.Threading; +using ClusterHelloWorld.Messages; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using Proto; +using Proto.Cluster; +using Proto.Cluster.Kubernetes; +using Proto.Cluster.PartitionActivator; +using Proto.Remote; +using Proto.Remote.GrpcNet; +using static System.Threading.Tasks.Task; +using ProtosReflection = ClusterHelloWorld.Messages.ProtosReflection; + +// Hook SIGTERM to a cancel token to know when k8s is shutting us down +// hostBuilder should be used in production +var cts = new CancellationTokenSource(); +AssemblyLoadContext.Default.Unloading += ctx => cts.Cancel(); + +Log.SetLoggerFactory( + LoggerFactory.Create(l => l.AddConsole(options => + { + //options.FormatterName = "json"; // Use the JSON formatter + }).SetMinimumLevel(LogLevel.Debug) + .AddFilter("Proto.Cluster.Gossip", LogLevel.Information) + .AddFilter("Proto.Context.ActorContext", LogLevel.Information))); + +// Required to allow unencrypted GrpcNet connections +AppContext.SetSwitch("System.Net.Http.SocketsHttpHandler.Http2UnencryptedSupport", true); + +var kubernetesProvider = new KubernetesProvider(); +var advertisedHost = await kubernetesProvider.GetPodFqdn(); + +var system = new ActorSystem(new ActorSystemConfig().WithDeveloperSupervisionLogging(true)) + .WithRemote(GrpcNetRemoteConfig + .BindToAllInterfaces(advertisedHost: advertisedHost, port: 4020) + .WithProtoMessages(ProtosReflection.Descriptor)) + .WithCluster(ClusterConfig + .Setup("MyCluster", + kubernetesProvider, + new PartitionActivatorLookup()) + .WithClusterKind( + HelloGrainActor.GetClusterKind((ctx, identity) => new HelloGrain(ctx, identity.Identity))) + ); + +system.EventStream.Subscribe( + e => { Console.WriteLine($"{DateTime.Now:O} My members {e.TopologyHash}"); } +); + +await system + .Cluster() + .StartMemberAsync(); + +Console.WriteLine("Started..."); + +Console.CancelKeyPress += (_, e) => +{ + e.Cancel = true; // prevent the process from terminating. + cts.Cancel(); +}; + +await Delay(Timeout.Infinite, cts.Token); +Console.WriteLine("Shutting Down..."); + +public class HelloGrain : HelloGrainBase +{ + private readonly string _identity; + + public HelloGrain(IContext ctx, string identity) : base(ctx) + { + _identity = identity; + } + + public override Task SayHello(HelloRequest request) + { + Console.WriteLine("Got request!!"); + + var res = new HelloResponse + { + Message = $"Hello from typed grain {_identity} | {DateTime.UtcNow:O}" + }; + + return FromResult(res); + } +} \ No newline at end of file diff --git a/examples/ClusterK8sGrains/chart/.helmignore b/examples/ClusterK8sGrains/chart/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/examples/ClusterK8sGrains/chart/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/examples/ClusterK8sGrains/chart/Chart.yaml b/examples/ClusterK8sGrains/chart/Chart.yaml new file mode 100644 index 0000000000..9720986a52 --- /dev/null +++ b/examples/ClusterK8sGrains/chart/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v2 +name: protoactor-example-k8s-grains +description: A hello world Proto.Actor Grains cluster in k8s +type: application +version: 0.1.0 \ No newline at end of file diff --git a/examples/ClusterK8sGrains/chart/templates/namespace.yaml b/examples/ClusterK8sGrains/chart/templates/namespace.yaml new file mode 100644 index 0000000000..77db5f9f65 --- /dev/null +++ b/examples/ClusterK8sGrains/chart/templates/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: {{ .Values.namespace }} diff --git a/examples/ClusterK8sGrains/chart/templates/node1-deployment.yaml b/examples/ClusterK8sGrains/chart/templates/node1-deployment.yaml new file mode 100644 index 0000000000..68e3bcdde7 --- /dev/null +++ b/examples/ClusterK8sGrains/chart/templates/node1-deployment.yaml @@ -0,0 +1,26 @@ +# protoactor-widget-actor-deployment.yaml + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: protoactor-k8s-grains-node1 + namespace: {{ .Values.namespace }} +spec: + serviceName: {{ .Values.protoActorCluster.subdomain }} + replicas: 1 + selector: + matchLabels: + app: protoactor-k8s-grains-node1 + template: + metadata: + labels: + app: protoactor-k8s-grains-node1 + protoActorMember: "true" + spec: + serviceAccountName: protoactor-k8s-grains-serviceaccount + containers: + - name: protoactor-k8s-grains-node1 + image: {{ .Values.node1.image }} + ports: + - containerPort: 4020 + name: protoactor \ No newline at end of file diff --git a/examples/ClusterK8sGrains/chart/templates/node2-deployment.yaml b/examples/ClusterK8sGrains/chart/templates/node2-deployment.yaml new file mode 100644 index 0000000000..46c1a7f070 --- /dev/null +++ b/examples/ClusterK8sGrains/chart/templates/node2-deployment.yaml @@ -0,0 +1,25 @@ +# protoactor-widget-actor-deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: protoactor-k8s-grains-node2 + namespace: {{ .Values.namespace }} +spec: + replicas: 1 + selector: + matchLabels: + app: protoactor-k8s-grains-node2 + template: + metadata: + labels: + app: protoactor-k8s-grains-node2 + protoActorMember: "true" + spec: + serviceAccountName: protoactor-k8s-grains-serviceaccount + subdomain: {{ .Values.protoActorCluster.subdomain }} + containers: + - name: protoactor-k8s-grains-node2 + image: {{ .Values.node2.image }} + ports: + - containerPort: 4020 + name: protoactor \ No newline at end of file diff --git a/examples/ClusterK8sGrains/chart/templates/protoactor-k8s-cluster-port-service.yaml b/examples/ClusterK8sGrains/chart/templates/protoactor-k8s-cluster-port-service.yaml new file mode 100644 index 0000000000..acc1f27f83 --- /dev/null +++ b/examples/ClusterK8sGrains/chart/templates/protoactor-k8s-cluster-port-service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.protoActorCluster.subdomain }} + namespace: {{ .Values.namespace }} +spec: + selector: + protoActorMember: "true" + ports: + - name: protoactor-k8s-cluster-port-service + protocol: TCP + port: 4020 + targetPort: 4020 + type: ClusterIP + publishNotReadyAddresses: true + clusterIP: None \ No newline at end of file diff --git a/examples/ClusterK8sGrains/chart/templates/protoactor-k8s-grains-role.yaml b/examples/ClusterK8sGrains/chart/templates/protoactor-k8s-grains-role.yaml new file mode 100644 index 0000000000..3ae02a612b --- /dev/null +++ b/examples/ClusterK8sGrains/chart/templates/protoactor-k8s-grains-role.yaml @@ -0,0 +1,16 @@ +# protoactor-widget-actor-role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: protoactor-k8s-grains-role + namespace: {{ .Values.namespace }} +rules: + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch + - patch \ No newline at end of file diff --git a/examples/ClusterK8sGrains/chart/templates/protoactor-k8s-grains-rolebinding.yaml b/examples/ClusterK8sGrains/chart/templates/protoactor-k8s-grains-rolebinding.yaml new file mode 100644 index 0000000000..a5c995eb48 --- /dev/null +++ b/examples/ClusterK8sGrains/chart/templates/protoactor-k8s-grains-rolebinding.yaml @@ -0,0 +1,14 @@ +# protoactor-widget-actor-rolebinding.yaml + +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: protoactor-k8s-grains-rolebinding + namespace: {{ .Values.namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: protoactor-k8s-grains-role +subjects: + - kind: ServiceAccount + name: protoactor-k8s-grains-serviceaccount \ No newline at end of file diff --git a/examples/ClusterK8sGrains/chart/templates/protoactor-k8s-grains-serviceaccount.yaml b/examples/ClusterK8sGrains/chart/templates/protoactor-k8s-grains-serviceaccount.yaml new file mode 100644 index 0000000000..23e84580d7 --- /dev/null +++ b/examples/ClusterK8sGrains/chart/templates/protoactor-k8s-grains-serviceaccount.yaml @@ -0,0 +1,7 @@ +# protoactor-widget-actor-serviceaccount.yaml + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: protoactor-k8s-grains-serviceaccount + namespace: {{ .Values.namespace }} \ No newline at end of file diff --git a/examples/ClusterK8sGrains/chart/values.yaml b/examples/ClusterK8sGrains/chart/values.yaml new file mode 100644 index 0000000000..65198079e8 --- /dev/null +++ b/examples/ClusterK8sGrains/chart/values.yaml @@ -0,0 +1,13 @@ +# values.yaml +namespace: protoactor-example-k8s-grains + +protoActorCluster: + subdomain: protoactor-cluster + name: protoactor-example-cluster + +node1: + image: localhost:5000/protoactor.example.k8s.grains.node1:latest + +node2: + image: localhost:5000/protoactor.example.k8s.grains.node2:latest + diff --git a/examples/ClusterK8sGrains/readme.md b/examples/ClusterK8sGrains/readme.md new file mode 100644 index 0000000000..520e9e3fd1 --- /dev/null +++ b/examples/ClusterK8sGrains/readme.md @@ -0,0 +1,37 @@ +Example of a Proto.Actor cluster running in k8s with the kuberneties service locator. +Is also using k8s DNS name, to support the use of TLS between the cluster members. + +# Setup +Was build and testing using k3s and Rancher Desktop and registry for local development. + +## Local registry (optional) +If you don't already have container repository, you can setup a local registry for loading and testing this example. +```shell +docker volume create registry_data +docker run -d -p 5000:5000 --name registry --restart=always -v registry_data:/var/lib/registry registry:2 +``` + +## Build and push images + +Run from the root of the repository to build and push the images to the local registry. +```shell +docker build -t protoactor.example.k8s.grains.node1:latest -f .\examples\ClusterK8sGrains\Node1\Dockerfile . +docker tag protoactor.example.k8s.grains.node1:latest localhost:5000/protoactor.example.k8s.grains.node1:latest +docker push localhost:5000/protoactor.example.k8s.grains.node1:latest + +docker build -t protoactor.example.k8s.grains.node2:latest -f .\examples\ClusterK8sGrains\Node2\Dockerfile . +docker tag protoactor.example.k8s.grains.node2:latest localhost:5000/protoactor.example.k8s.grains.node2:latest +docker push localhost:5000/protoactor.example.k8s.grains.node2:latest +``` + +## Deploy to k8s + +From the ClusterK8sGrains folder run the following commands to deploy the application to k8s. +```shell +helm install protoactor-example-k8s-grains .\examples\ClusterK8sGrains\chart +``` + +To remove the application from k8s run the following command. +```shell +helm uninstall protoactor-example-k8s-grains +``` \ No newline at end of file diff --git a/src/Proto.Cluster.Kubernetes/KubernetesClusterMonitor.cs b/src/Proto.Cluster.Kubernetes/KubernetesClusterMonitor.cs index ce5adbdcad..d8c2f68463 100644 --- a/src/Proto.Cluster.Kubernetes/KubernetesClusterMonitor.cs +++ b/src/Proto.Cluster.Kubernetes/KubernetesClusterMonitor.cs @@ -302,7 +302,7 @@ private void UpdateTopology() } var memberStatuses = _clusterPods.Values - .Select(x => x.GetMemberStatus()) + .Select(x => x.GetMemberStatus(_config)) .Where(x => x is not null) .Where(x => x.IsRunning && (x.IsReady || x.Member.Id == _cluster.System.Id)) .Select(x => x.Member) diff --git a/src/Proto.Cluster.Kubernetes/KubernetesExtensions.cs b/src/Proto.Cluster.Kubernetes/KubernetesExtensions.cs index 9337035d94..2231f301dd 100644 --- a/src/Proto.Cluster.Kubernetes/KubernetesExtensions.cs +++ b/src/Proto.Cluster.Kubernetes/KubernetesExtensions.cs @@ -8,6 +8,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Text; using System.Text.Json; using System.Threading.Tasks; using JetBrains.Annotations; @@ -57,15 +58,16 @@ IDictionary annotations /// Get the pod status. The pod must be running in order to be considered as a candidate. /// /// Kubernetes Pod object + /// Kubernetes provider configuration /// [CanBeNull] - internal static MemberStatus GetMemberStatus(this V1Pod pod) + internal static MemberStatus GetMemberStatus(this V1Pod pod, KubernetesProviderConfig config) { var isRunning = pod.Status is { Phase: "Running", PodIP: not null }; if (pod.Status?.ContainerStatuses is null) return null; - + if (pod.Metadata?.Labels is null) return null; @@ -77,6 +79,21 @@ internal static MemberStatus GetMemberStatus(this V1Pod pod) .ToArray(); var host = pod.Status.PodIP ?? ""; + if (pod.Metadata.Labels.TryGetValue(LabelHost, out var hostOverride)) + host = hostOverride; + else if (pod.Metadata.Labels.TryGetValue(LabelHostPrefix, out var hostPrefix)) + { + var dnsPostfix = $".{pod.Namespace()}.svc.{config.ClusterDomain}"; + + // If we have a subdomain, then we can add that to the dnsPostfix, as it will be known to the cluster + if (!string.IsNullOrEmpty(pod.Spec.Subdomain)) + { + dnsPostfix = $".{pod.Spec.Subdomain}{dnsPostfix}"; + } + + host = hostPrefix + dnsPostfix; + } + var port = Convert.ToInt32(pod.Metadata.Labels[LabelPort]); var mid = pod.Metadata.Labels[LabelMemberId]; var alive = pod.Status.ContainerStatuses.All(x => x.Ready); @@ -95,23 +112,70 @@ internal static MemberStatus GetMemberStatus(this V1Pod pod) /// Get the namespace of the current pod /// /// The pod namespace + /// Failed to get the Kubernetes namespace, not running in a k8s cluster internal static string GetKubeNamespace() { - if (cachedNamespace is null) + if (TryGetKubeNamespace(out var kubeNamespace)) + { + return kubeNamespace; + } + + throw new InvalidOperationException("The application doesn't seem to be running in Kubernetes"); + } + + internal static bool TryGetKubeNamespace(out string kubeNamespace) + { + if (cachedNamespace is not null) + { + if (cachedNamespace.Length == 0) + { + // We have already tried to get the namespace, and it was not found. + kubeNamespace = null; + return false; + } + + kubeNamespace = cachedNamespace; + return true; + } + + var namespaceFile = Path.Combine( + $"{Path.DirectorySeparatorChar}var", + "run", + "secrets", + "kubernetes.io", + "serviceaccount", + "namespace" + ); + + if (!File.Exists(namespaceFile)) + { + // Note setting it to empty, so we don't try again. + kubeNamespace = null; + return false; + } + + // k8s has a limit of 63 characters for namespace names + // Limit to reading 63 characters, in case a larger files is there, we will just ignore it. + using var reader = new StreamReader(namespaceFile, Encoding.UTF8); + var buffer = new char[65]; + var read = reader.Read(buffer, 0, 64); + if (read == 0 || read > 63) + { + cachedNamespace = string.Empty; + kubeNamespace = null; + return false; + } + + kubeNamespace = cachedNamespace = new string(buffer, 0, read).Trim(); + if (string.IsNullOrWhiteSpace(kubeNamespace)) { - var namespaceFile = Path.Combine( - $"{Path.DirectorySeparatorChar}var", - "run", - "secrets", - "kubernetes.io", - "serviceaccount", - "namespace" - ); - - cachedNamespace = File.ReadAllText(namespaceFile); + // Set it to something, so we know we read it, and we don't try again. + cachedNamespace = string.Empty; + kubeNamespace = null; + return false; } - return cachedNamespace; + return true; } /// diff --git a/src/Proto.Cluster.Kubernetes/KubernetesHelper.cs b/src/Proto.Cluster.Kubernetes/KubernetesHelper.cs new file mode 100644 index 0000000000..fa57207e95 --- /dev/null +++ b/src/Proto.Cluster.Kubernetes/KubernetesHelper.cs @@ -0,0 +1,59 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using JetBrains.Annotations; +using k8s; +using k8s.Models; + +namespace Proto.Cluster.Kubernetes; + +public static class KubernetesHelper +{ + /// + /// Checks if the Kubernetes namespace file exists. + /// + [PublicAPI] + public static bool HasKubeNamespace() => KubernetesExtensions.TryGetKubeNamespace(out _); + + /// + /// Attempts to get the FQDN for the Pod by querying the Kubernetes API. + /// + /// Unable to detected k8s or if Subdomain is missing from Pod Definition. + [PublicAPI] + public static async ValueTask GetPodFqdn(this KubernetesProvider kubernetesProvider) + { + return await GetPodFqdn(kubernetesProvider.Config); + } + + /// + /// Attempts to get the FQDN for the Pod by querying the Kubernetes API. + /// + /// Unable to detected k8s or if Subdomain is missing from Pod Definition. + [PublicAPI] + public static async ValueTask GetPodFqdn(this KubernetesProviderConfig config) + { + var kubeNamespace = KubernetesExtensions.GetKubeNamespace(); + + var k8SClient = config.ClientFactory(); + var pod = await k8SClient.CoreV1.ReadNamespacedPodAsync(Environment.MachineName, kubeNamespace) + .ConfigureAwait(false); + + if (string.IsNullOrEmpty(pod.Spec.Subdomain)) + { + throw new ApplicationException("Failed to get the FQDN for the Pod, the spec.subdomain or spec.serviceName is not set"); + } + + // Look up our workload kind, and then determine if we are a pod with a stable name and so can use our host name + // of if we are a workload like a Deployment, and need to use our IP address for the FQDN. + + var ownerReferences = pod.Metadata.OwnerReferences; + var isStableHostnameWorkload = ownerReferences.Any( + // Add other workload kinds with stable hostnames as needed + owner => owner.Kind is "StatefulSet" or "DaemonSet" + ); + + var host = isStableHostnameWorkload ? pod.Metadata.Name : pod.Status.PodIP.Replace('.', '-'); + var hostFqdn = $"{host}.{pod.Spec.Subdomain}.{pod.Namespace()}.svc.{config.ClusterDomain}"; + return hostFqdn; + } +} \ No newline at end of file diff --git a/src/Proto.Cluster.Kubernetes/KubernetesProvider.cs b/src/Proto.Cluster.Kubernetes/KubernetesProvider.cs index 8279fd3983..59e73b5de0 100644 --- a/src/Proto.Cluster.Kubernetes/KubernetesProvider.cs +++ b/src/Proto.Cluster.Kubernetes/KubernetesProvider.cs @@ -39,6 +39,8 @@ public class KubernetesProvider : IClusterProvider private string _podName; private int _port; + internal KubernetesProviderConfig Config => _config; + public async Task GetDiagnostics() { try @@ -67,7 +69,7 @@ public KubernetesProvider() : this(new KubernetesProviderConfig()) public KubernetesProvider(KubernetesProviderConfig config) { - if (KubernetesExtensions.GetKubeNamespace() is null) + if (!KubernetesExtensions.TryGetKubeNamespace(out _)) { throw new InvalidOperationException("The application doesn't seem to be running in Kubernetes"); } @@ -149,14 +151,16 @@ public async Task RegisterMemberInner() Logger.LogInformation("[Cluster][KubernetesProvider] Using Kubernetes namespace: {Namespace}", pod.Namespace()); Logger.LogInformation("[Cluster][KubernetesProvider] Using Kubernetes port: {Port}", _port); - + var labels = new Dictionary { [LabelCluster] = _clusterName, [LabelPort] = _port.ToString(), - [LabelMemberId] = _cluster.System.Id + [LabelMemberId] = _cluster.System.Id, }; + AppendHostToPodLabels(pod, labels); + foreach (var existing in pod.Metadata.Labels) { labels.TryAdd(existing.Key, existing.Value); @@ -189,6 +193,46 @@ public async Task RegisterMemberInner() } } + /// + /// k8s labels have a limit of 63 characters, so we need to be careful about what we add to the labels. + /// + protected void AppendHostToPodLabels(V1Pod pod, Dictionary labels) + { + // If our advertised host is something other then our pod IP, we need to add a label to the pod + // So others can find us by our advertised host. + var podId = pod.Status.PodIP; + if (!_host.Equals(podId, StringComparison.OrdinalIgnoreCase)) + { + if (_host.Length <= 63) + { + labels[LabelHost] = _host; + } + else + { + var dnsPostfix = $".{pod.Namespace()}.svc.{_config.ClusterDomain}"; + + // If we have a subdomain, then we can add that to the dnsPostfix, as it will be known to the cluster + if (!string.IsNullOrEmpty(pod.Spec.Subdomain)) + { + dnsPostfix = $".{pod.Spec.Subdomain}{dnsPostfix}"; + } + + if (_host.EndsWith(dnsPostfix)) + { + labels[LabelHostPrefix] = _host[..^dnsPostfix.Length]; + } + else + { + // TODO: what else could we do here? + // Going to fall back to the IP address, and hope that works... + Logger.LogWarning( + "[Cluster][KubernetesProvider] AdvertisedHost is too long to be used as a label, falling back to PodID, host: {Host}", + _host); + } + } + } + } + private void StartClusterMonitor() { var props = Props diff --git a/src/Proto.Cluster.Kubernetes/KubernetesProviderConfig.cs b/src/Proto.Cluster.Kubernetes/KubernetesProviderConfig.cs index 76a1643226..12fe0e25c8 100644 --- a/src/Proto.Cluster.Kubernetes/KubernetesProviderConfig.cs +++ b/src/Proto.Cluster.Kubernetes/KubernetesProviderConfig.cs @@ -35,6 +35,11 @@ public KubernetesProviderConfig(int watchTimeoutSeconds = 30, bool developerLogg /// Enables more detailed logging /// private bool DeveloperLogging { get; } + + /// + /// The k8s Cluster Domain (TLD), defaults to "cluster.local" + /// + public string ClusterDomain { get; init; } = "cluster.local"; /// /// Override the default implementation to configure the kubernetes client @@ -43,5 +48,11 @@ public KubernetesProviderConfig(int watchTimeoutSeconds = 30, bool developerLogg internal LogLevel DebugLogLevel => DeveloperLogging ? LogLevel.Information : LogLevel.Debug; - private static IKubernetes DefaultFactory() => new k8s.Kubernetes(KubernetesClientConfiguration.InClusterConfig()); + internal static IKubernetes DefaultFactory() => new k8s.Kubernetes(KubernetesClientConfiguration.InClusterConfig()); + + /// + /// The k8s Cluster Domain (TLD), defaults to "cluster.local" + /// + public KubernetesProviderConfig WithClusterDomain(string clusterDomain = "cluster.local") + => this with { ClusterDomain = clusterDomain }; } \ No newline at end of file diff --git a/src/Proto.Cluster.Kubernetes/ProtoLabels.cs b/src/Proto.Cluster.Kubernetes/ProtoLabels.cs index e8c38a899c..62a819bdde 100644 --- a/src/Proto.Cluster.Kubernetes/ProtoLabels.cs +++ b/src/Proto.Cluster.Kubernetes/ProtoLabels.cs @@ -13,4 +13,6 @@ public static class ProtoLabels public const string LabelCluster = ProtoClusterPrefix + "cluster"; public const string LabelMemberId = ProtoClusterPrefix + "member-id"; public const string AnnotationKinds = ProtoClusterPrefix + "kinds"; + public const string LabelHost = ProtoClusterPrefix + "host"; + public const string LabelHostPrefix = ProtoClusterPrefix + "host-prefix"; } \ No newline at end of file