Skip to content

Commit

Permalink
[WIP] Add support for dedicated chaincode nodes
Browse files Browse the repository at this point in the history
See #228

Signed-off-by: James Taylor <[email protected]>
  • Loading branch information
jt-nti committed Jan 17, 2025
1 parent 0dff17f commit 0f62a30
Show file tree
Hide file tree
Showing 12 changed files with 187 additions and 46 deletions.
129 changes: 98 additions & 31 deletions cmd/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,75 +5,141 @@ package cmd
import (
"context"
"os"
"strconv"

"github.com/hyperledger-labs/fabric-builder-k8s/internal/builder"
"github.com/hyperledger-labs/fabric-builder-k8s/internal/log"
"github.com/hyperledger-labs/fabric-builder-k8s/internal/util"
"k8s.io/apimachinery/pkg/api/validation"
apivalidation "k8s.io/apimachinery/pkg/api/validation"
"k8s.io/apimachinery/pkg/util/validation"
)

func Run() int {
const (
expectedArgsLength = 3
buildOutputDirectoryArg = 1
runMetadataDirectoryArg = 2
maximumKubeNamePrefixLength = 30
)

debug, _ := strconv.ParseBool(util.GetOptionalEnv(util.DebugVariable, "false"))
ctx := log.NewCmdContext(context.Background(), debug)
logger := log.New(ctx)

if len(os.Args) != expectedArgsLength {
logger.Println("Expected BUILD_OUTPUT_DIR and RUN_METADATA_DIR arguments")

return 1
}

buildOutputDirectory := os.Args[buildOutputDirectoryArg]
runMetadataDirectory := os.Args[runMetadataDirectoryArg]

logger.Debugf("Build output directory: %s", buildOutputDirectory)
logger.Debugf("Run metadata directory: %s", runMetadataDirectory)

//nolint:nonamedreturns // using the ok bool convention to indicate errors
func getPeerID(logger *log.CmdLogger) (peerID string, ok bool) {
peerID, err := util.GetRequiredEnv(util.PeerIDVariable)
if err != nil {
logger.Printf("Expected %s environment variable\n", util.PeerIDVariable)

return 1
return peerID, false
}

logger.Debugf("%s=%s", util.PeerIDVariable, peerID)

return peerID, true
}

func getKubeconfigPath(logger *log.CmdLogger) string {
kubeconfigPath := util.GetOptionalEnv(util.KubeconfigPathVariable, "")
logger.Debugf("%s=%s", util.KubeconfigPathVariable, kubeconfigPath)

return kubeconfigPath
}

func getKubeNamespace(logger *log.CmdLogger) string {
kubeNamespace := util.GetOptionalEnv(util.ChaincodeNamespaceVariable, "")
logger.Debugf("%s=%s", util.ChaincodeNamespaceVariable, kubeNamespace)

if kubeNamespace == "" {
var err error

kubeNamespace, err = util.GetKubeNamespace()
if err != nil {
logger.Debugf("Error getting namespace: %+v\n", util.DefaultNamespace, err)
kubeNamespace = util.DefaultNamespace
}

logger.Debugf("Using default namespace: %s\n", util.DefaultNamespace)
}

return kubeNamespace
}

//nolint:nonamedreturns // using the ok bool convention to indicate errors
func getKubeNodeRole(logger *log.CmdLogger) (kubeNodeRole string, ok bool) {
kubeNodeRole = util.GetOptionalEnv(util.ChaincodeNodeRoleVariable, "")
logger.Debugf("%s=%s", util.ChaincodeNodeRoleVariable, kubeNodeRole)

// TODO: are valid taint values the same?!
if msgs := validation.IsValidLabelValue(kubeNodeRole); len(msgs) > 0 {
logger.Printf("The %s environment variable must be a valid Kubernetes label value: %s", util.ChaincodeNodeRoleVariable, msgs[0])

return kubeNodeRole, false
}

return kubeNodeRole, true
}

func getKubeServiceAccount(logger *log.CmdLogger) string {
kubeServiceAccount := util.GetOptionalEnv(util.ChaincodeServiceAccountVariable, util.DefaultServiceAccountName)
logger.Debugf("%s=%s", util.ChaincodeServiceAccountVariable, kubeServiceAccount)

kubeNamePrefix := util.GetOptionalEnv(util.ObjectNamePrefixVariable, util.DefaultObjectNamePrefix)
return kubeServiceAccount
}

//nolint:nonamedreturns // using the ok bool convention to indicate errors
func getKubeNamePrefix(logger *log.CmdLogger) (kubeNamePrefix string, ok bool) {
const maximumKubeNamePrefixLength = 30

kubeNamePrefix = util.GetOptionalEnv(util.ObjectNamePrefixVariable, util.DefaultObjectNamePrefix)
logger.Debugf("%s=%s", util.ObjectNamePrefixVariable, kubeNamePrefix)

if len(kubeNamePrefix) > maximumKubeNamePrefixLength {
logger.Printf("The FABRIC_K8S_BUILDER_OBJECT_NAME_PREFIX environment variable must be a maximum of 30 characters")
logger.Printf("The %s environment variable must be a maximum of 30 characters", util.ObjectNamePrefixVariable)

return kubeNamePrefix, false
}

if msgs := apivalidation.NameIsDNS1035Label(kubeNamePrefix, true); len(msgs) > 0 {
logger.Printf("The %s environment variable must be a valid DNS-1035 label: %s", util.ObjectNamePrefixVariable, msgs[0])

return kubeNamePrefix, false
}

return kubeNamePrefix, true
}

func Run() int {
const (
expectedArgsLength = 3
buildOutputDirectoryArg = 1
runMetadataDirectoryArg = 2
)

debug := util.GetOptionalEnv(util.DebugVariable, "false")
ctx := log.NewCmdContext(context.Background(), debug == "true")
logger := log.New(ctx)

if len(os.Args) != expectedArgsLength {
logger.Println("Expected BUILD_OUTPUT_DIR and RUN_METADATA_DIR arguments")

return 1
}

buildOutputDirectory := os.Args[buildOutputDirectoryArg]
runMetadataDirectory := os.Args[runMetadataDirectoryArg]

logger.Debugf("Build output directory: %s", buildOutputDirectory)
logger.Debugf("Run metadata directory: %s", runMetadataDirectory)

//nolint:varnamelen // using the ok bool convention to indicate errors
var ok bool

peerID, ok := getPeerID(logger)
if !ok {
return 1
}

kubeconfigPath := getKubeconfigPath(logger)
kubeNamespace := getKubeNamespace(logger)

kubeNodeRole, ok := getKubeNodeRole(logger)
if !ok {
return 1
}

if msgs := validation.NameIsDNS1035Label(kubeNamePrefix, true); len(msgs) > 0 {
logger.Printf("The FABRIC_K8S_BUILDER_OBJECT_NAME_PREFIX environment variable must be a valid DNS-1035 label: %s", msgs[0])
kubeServiceAccount := getKubeServiceAccount(logger)

kubeNamePrefix, ok := getKubeNamePrefix(logger)
if !ok {
return 1
}

Expand All @@ -83,6 +149,7 @@ func Run() int {
PeerID: peerID,
KubeconfigPath: kubeconfigPath,
KubeNamespace: kubeNamespace,
KubeNodeRole: kubeNodeRole,
KubeServiceAccount: kubeServiceAccount,
KubeNamePrefix: kubeNamePrefix,
}
Expand Down
22 changes: 22 additions & 0 deletions cmd/run/main_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,28 @@ var _ = Describe("Main", func() {
),
)

DescribeTable("Running the run command produces the correct error for invalid FABRIC_K8S_BUILDER_NODE_ROLE environment variable values",
func(kubeNodeRoleValue, expectedErrorMessage string) {
args := []string{"BUILD_OUTPUT_DIR", "RUN_METADATA_DIR"}
command := exec.Command(runCmdPath, args...)
command.Env = append(os.Environ(),
"CORE_PEER_ID=core-peer-id-abcdefghijklmnopqrstuvwxyz-0123456789",
"FABRIC_K8S_BUILDER_NODE_ROLE="+kubeNodeRoleValue,
)
session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
Expect(err).NotTo(HaveOccurred())

Eventually(session).Should(gexec.Exit(1))
Eventually(
session.Err,
).Should(gbytes.Say(expectedErrorMessage))
},
Entry("When the FABRIC_K8S_BUILDER_NODE_ROLE is too long", "long-node-role-is-looooooooooooooooooooooooooooooooooooooooooong", `run \[\d+\]: The FABRIC_K8S_BUILDER_NODE_ROLE environment variable must be a valid Kubernetes label value: must be no more than 63 characters`),
Entry("When the FABRIC_K8S_BUILDER_NODE_ROLE contains invalid characters", "invalid*value", `run \[\d+\]: The FABRIC_K8S_BUILDER_NODE_ROLE environment variable must be a valid Kubernetes label value: a valid label must be an empty string or consist of alphanumeric characters, '-', '_' or '\.', and must start and end with an alphanumeric character`),
Entry("When the FABRIC_K8S_BUILDER_NODE_ROLE does not start with an alphanumeric character", ".role", `run \[\d+\]: The FABRIC_K8S_BUILDER_NODE_ROLE environment variable must be a valid Kubernetes label value: a valid label must be an empty string or consist of alphanumeric characters, '-', '_' or '\.', and must start and end with an alphanumeric character`),
Entry("When the FABRIC_K8S_BUILDER_NODE_ROLE does not end with an alphanumeric character", "role-", `run \[\d+\]: The FABRIC_K8S_BUILDER_NODE_ROLE environment variable must be a valid Kubernetes label value: a valid label must be an empty string or consist of alphanumeric characters, '-', '_' or '\.', and must start and end with an alphanumeric character`),
)

DescribeTable("Running the run command produces the correct error for invalid FABRIC_K8S_BUILDER_OBJECT_NAME_PREFIX environment variable values",
func(kubeNamePrefixValue, expectedErrorMessage string) {
args := []string{"BUILD_OUTPUT_DIR", "RUN_METADATA_DIR"}
Expand Down
15 changes: 15 additions & 0 deletions docs/configuring/dedicated-nodes.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
# Dedicated nodes

TBC

The `FABRIC_K8S_BUILDER_NODE_ROLE` environment variable can be used to...

For example, if `FABRIC_K8S_BUILDER_NODE_ROLE` is set to `chaincode`, ... using the following command.

```shell
kubectl label nodes node1 fabric-builder-k8s-role=chaincode
kubectl taint nodes node1 fabric-builder-k8s-role=chaincode:NoSchedule
```

More complex requirements should be handled with Dynamic Admission Control using a Mutating Webhook.
For example, it looks like the namespace-node-affinity webhook could be used to assign node affinity and tolerations to all pods in the FABRIC_K8S_BUILDER_NAMESPACE namespace.
2 changes: 2 additions & 0 deletions docs/configuring/overview.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ External builders are configured in the `core.yaml` file, for example:
- CORE_PEER_ID
- FABRIC_K8S_BUILDER_DEBUG
- FABRIC_K8S_BUILDER_NAMESPACE
- FABRIC_K8S_BUILDER_NODE_ROLE
- FABRIC_K8S_BUILDER_OBJECT_NAME_PREFIX
- FABRIC_K8S_BUILDER_SERVICE_ACCOUNT
- KUBERNETES_SERVICE_HOST
Expand All @@ -30,6 +31,7 @@ The k8s builder is configured using the following environment variables.
| ------------------------------------- | -------------------------------- | ---------------------------------------------------- |
| CORE_PEER_ID | | The Fabric peer ID (required) |
| FABRIC_K8S_BUILDER_NAMESPACE | The peer namespace or `default` | The Kubernetes namespace to run chaincode with |
| FABRIC_K8S_BUILDER_NODE_ROLE | | TBC |
| FABRIC_K8S_BUILDER_OBJECT_NAME_PREFIX | `hlfcc` | Eye-catcher prefix for Kubernetes object names |
| FABRIC_K8S_BUILDER_SERVICE_ACCOUNT | `default` | The Kubernetes service account to run chaincode with |
| FABRIC_K8S_BUILDER_DEBUG | `false` | Set to `true` to enable k8s builder debug messages |
Expand Down
4 changes: 0 additions & 4 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,6 @@ require (
github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/gorilla/websocket v1.5.0 // indirect
github.com/itchyny/gojq v0.12.13 // indirect
github.com/itchyny/timefmt-go v0.1.5 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/moby/spdystream v0.4.0 // indirect
Expand All @@ -47,12 +45,10 @@ require (
golang.org/x/sync v0.8.0 // indirect
golang.org/x/tools v0.26.0 // indirect
k8s.io/component-base v0.31.1 // indirect
mvdan.cc/sh/v3 v3.7.0 // indirect
sigs.k8s.io/controller-runtime v0.19.0 // indirect
)

require (
github.com/bitfield/script v0.23.0
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/go-logr/logr v1.4.2 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
Expand Down
10 changes: 0 additions & 10 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,6 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPd
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bitfield/script v0.23.0 h1:N0R5yLEl6wJIS9PR/A6xXwjMsplMubyxdi05N5l0X28=
github.com/bitfield/script v0.23.0/go.mod h1:fv+6x4OzVsRs6qAlc7wiGq8fq1b5orhtQdtW0dwjUHI=
github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
Expand All @@ -16,8 +14,6 @@ github.com/emicklei/go-restful/v3 v3.11.2 h1:1onLa9DcsMYO9P+CXaL0dStDqQ2EHHXLiz+
github.com/emicklei/go-restful/v3 v3.11.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg=
github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ=
github.com/frankban/quicktest v1.14.5 h1:dfYrrRyLtiqT9GyKXgdh+k4inNeTvmGbuSgZ3lx3GhA=
github.com/frankban/quicktest v1.14.5/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E=
github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ=
github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
Expand Down Expand Up @@ -52,10 +48,6 @@ github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWm
github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4=
github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
github.com/itchyny/gojq v0.12.13 h1:IxyYlHYIlspQHHTE0f3cJF0NKDMfajxViuhBLnHd/QU=
github.com/itchyny/gojq v0.12.13/go.mod h1:JzwzAqenfhrPUuwbmEz3nu3JQmFLlQTQMUcOdnu/Sf4=
github.com/itchyny/timefmt-go v0.1.5 h1:G0INE2la8S6ru/ZI5JecgyzbbJNs5lG1RcBqa7Jm6GE=
github.com/itchyny/timefmt-go v0.1.5/go.mod h1:nEP7L+2YmAbT2kZ2HfSs1d8Xtw9LY8D2stDBckWakZ8=
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
Expand Down Expand Up @@ -191,8 +183,6 @@ k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7F
k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98=
k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A=
k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
mvdan.cc/sh/v3 v3.7.0 h1:lSTjdP/1xsddtaKfGg7Myu7DnlHItd3/M2tomOcNNBg=
mvdan.cc/sh/v3 v3.7.0/go.mod h1:K2gwkaesF/D7av7Kxl0HbF5kGOd2ArupNTX3X44+8l8=
sigs.k8s.io/controller-runtime v0.19.0 h1:nWVM7aq+Il2ABxwiCizrVDSlmDcshi9llbaFbC0ji/Q=
sigs.k8s.io/controller-runtime v0.19.0/go.mod h1:iRmWllt8IlaLjvTTDLhRBXIEtkCK6hwVBJJsYS9Ajf4=
sigs.k8s.io/e2e-framework v0.5.0 h1:YLhk8R7EHuTFQAe6Fxy5eBzn5Vb+yamR5u8MH1Rq3cE=
Expand Down
2 changes: 2 additions & 0 deletions internal/builder/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ type Run struct {
PeerID string
KubeconfigPath string
KubeNamespace string
KubeNodeRole string
KubeServiceAccount string
KubeNamePrefix string
}
Expand Down Expand Up @@ -73,6 +74,7 @@ func (r *Run) Run(ctx context.Context) error {
kubeObjectName,
r.KubeNamespace,
r.KubeServiceAccount,
r.KubeNodeRole,
r.PeerID,
chaincodeData,
imageData,
Expand Down
1 change: 1 addition & 0 deletions internal/util/env.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
const (
builderVariablePrefix = "FABRIC_K8S_BUILDER_"
ChaincodeNamespaceVariable = builderVariablePrefix + "NAMESPACE"
ChaincodeNodeRoleVariable = builderVariablePrefix + "NODE_ROLE"
ObjectNamePrefixVariable = builderVariablePrefix + "OBJECT_NAME_PREFIX"
ChaincodeServiceAccountVariable = builderVariablePrefix + "SERVICE_ACCOUNT"
DebugVariable = builderVariablePrefix + "DEBUG"
Expand Down
37 changes: 36 additions & 1 deletion internal/util/k8s.go
Original file line number Diff line number Diff line change
Expand Up @@ -365,7 +365,7 @@ func CreateChaincodeJob(
ctx context.Context,
logger *log.CmdLogger,
jobsClient typedBatchv1.JobInterface,
objectName, namespace, serviceAccount, peerID string,
objectName, namespace, serviceAccount, nodeRole, peerID string,
chaincodeData *ChaincodeJSON,
imageData *ImageJSON,
) (*batchv1.Job, error) {
Expand All @@ -381,6 +381,41 @@ func CreateChaincodeJob(
return nil, fmt.Errorf("error getting chaincode job definition for chaincode ID %s: %w", chaincodeData.ChaincodeID, err)
}

if nodeRole != "" {
logger.Debugf(
"Adding node affinity and toleration to job definition for chaincode ID %s: %s",
chaincodeData.ChaincodeID,
nodeRole,
)

jobDefinition.Spec.Template.Spec.Affinity = &apiv1.Affinity{
NodeAffinity: &apiv1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &apiv1.NodeSelector{
NodeSelectorTerms: []apiv1.NodeSelectorTerm{
{
MatchExpressions: []apiv1.NodeSelectorRequirement{
{
Key: "fabric-builder-k8s-role",
Operator: apiv1.NodeSelectorOpIn,
Values: []string{nodeRole},
},
},
},
},
},
},
}

jobDefinition.Spec.Template.Spec.Tolerations = []apiv1.Toleration{
{
Key: "fabric-builder-k8s-role",
Operator: apiv1.TolerationOpEqual,
Value: nodeRole,
Effect: apiv1.TaintEffectNoSchedule,
},
}
}

jobName := jobDefinition.ObjectMeta.Name

logger.Debugf(
Expand Down
1 change: 1 addition & 0 deletions mkdocs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,7 @@ nav:
- Kubernetes permissions: configuring/kubernetes-permissions.md
- Kubernetes namespace: configuring/kubernetes-namespace.md
- Kubernetes service account: configuring/kubernetes-service-account.md
- Dedicated nodes: configuring/dedicated-nodes.md
- Tutorials:
- Developing and debuging chaincode: tutorials/develop-chaincode.md
- Creating a chaincode package: tutorials/package-chaincode.md
2 changes: 2 additions & 0 deletions test/integration/main_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,11 +30,13 @@ func TestMain(m *testing.M) {

testenv.Setup(
envfuncs.CreateCluster(kind.NewProvider(), clusterName),
// envfuncs.CreateClusterWithConfig(kind.NewProvider(), clusterName, "testdata/kind-config.yaml", kind.WithImage("kindest/node:v1.22.2")),
envfuncs.CreateNamespace(envCfg.Namespace()),
)

testenv.Finish(
envfuncs.DeleteNamespace(envCfg.Namespace()),
// envfuncs.ExportClusterLogs(kindClusterName, "./logs"),
envfuncs.DestroyCluster(clusterName),
)

Expand Down
Loading

0 comments on commit 0f62a30

Please sign in to comment.