Support external network connectivity for the pod via pod VM network

src/cloud-api-adaptor/cmd/cloud-api-adaptor/main.go

@@ -122,6 +122,9 @@ func (cfg *daemonConfig) Setup() (cmd.Starter, error) {
 		flags.StringVar(&cfg.networkConfig.HostInterface, "host-interface", "", "Host Interface")
 		flags.IntVar(&cfg.networkConfig.VXLAN.Port, "vxlan-port", vxlan.DefaultVXLANPort, "VXLAN UDP port number (VXLAN tunnel mode only")
 		flags.IntVar(&cfg.networkConfig.VXLAN.MinID, "vxlan-min-id", vxlan.DefaultVXLANMinID, "Minimum VXLAN ID (VXLAN tunnel mode only")
+		flags.BoolVar(&cfg.networkConfig.ExternalNetViaPodVM, "ext-network-via-podvm", false, "Enable external networking via pod VM")
+		// Local pod subnets. This will be used by APF to create routes for local pod subnets when using external networking via pod VM
+		flags.Var(&cfg.networkConfig.PodSubnetCIDRs, "pod-subnet-cidrs", "Local pod subnets")
 		flags.StringVar(&cfg.serverConfig.Initdata, "initdata", "", "Default initdata for all Pods")
 		flags.BoolVar(&cfg.serverConfig.EnableCloudConfigVerify, "cloud-config-verify", false, "Enable cloud config verify - should use it for production")
 		flags.IntVar(&cfg.serverConfig.PeerPodsLimitPerNode, "peerpods-limit-per-node", 10, "peer pods limit per node (default=10)")

src/cloud-api-adaptor/docs/external-network.md

@@ -0,0 +1,35 @@
+# Introduction
+
+Currently, all pod traffic is routed via the worker node (WN).
+This is by design and is  fine as all existing network policies that were applicable to pods running on the WN also apply to peer pods.
+However, the peer pods approach has enabled some additional use cases that were earlier impossible or impractical. Concurrently, these new use cases have resulted in new requirements.
+
+Consider a scenario where you are running a Kind K8s cluster on your laptop, experimenting with an AI model, and needing additional computing power or GPUs. With peer pods, you can spin up a pod using the appropriate instance type from the Kind K8s cluster running on your laptop.
+
+From a networking standpoint, this means that there should be bidirectional connectivity between the cloud and your development setup.
+
+One solution is to use a VPN, which complicates the setup. A relatively easier alternative is to use a public IP associated with the cloud instance for control-plane traffic and use the cloud instance network for external connectivity requirements (like accessing cloud object storage for data) instead of routing all traffic via your development setup.
+
+The external network connectivity for the pod via pod VM network is enabled via the
+option `ext-network-via-podvm` in cloud-api-adaptor. The equivalent option in the `peer-pods-cm` is `EXTERNAL_NETWORK_VIA_PODVM`
+
+The prerequisite is for the pod VM to have a secondary interface with an IP. This interface will be moved to the pod network namespace and default routes adjusted so that pod network traverses via worker node, and any other traffic uses the secondary interface.
+
+## Specifying Pod subnet CIDRs
+
+When using the `ext-network-via-podvm` feature, the default route for the pod is set to the secondary interface. Traffic on the pod subnet is routed via the pod network specific interface (vxlan). The pod subnet is auto detected based on the pod IP by Linux for the route. This may not be sufficient for all cases and you may want to specify pod subnets explicitly.
+We have an option for the same named `pod-subnet-cidrs` to provide a comma separated list of CIDRs that will be routed via the pod network specific interface.
+
+The pod subnet CIDR, service CIDR depends on Kubernetes CNI configuration. Here are some examples:
+
+* A generic way to detect the cluster service CIDR as mentioned [here](Ref: https://stackoverflow.com/questions/44190607/how-do-you-find-the-cluster-service-cidr-of-a-kubernetes-cluster)
+
+```sh
+SVCRANGE=$(echo '{"apiVersion":"v1","kind":"Service","metadata":{"name":"tst"},"spec":{"clusterIP":"1.1.1.1","ports":[{"port":443}]}}' | kubectl apply -f - 2>&1 | sed 's/.*valid IPs is //')
+echo $SVCRANGE
+```
+
+* The default pod CIDR for Flannel CNI: `10.244.0.0/16`
+* AWS VPC CNI uses VPC CIDR as the pod CIDR
+* Azure CNI uses the subnet CIDR as the pod CIDR
+* The default pod CIDR for OVN-Kubernetes: `10.128.0.0/14`

src/cloud-api-adaptor/entrypoint.sh

@@ -30,6 +30,8 @@ optionals+=""
 [[ "${SECURE_COMMS_PP_OUTBOUNDS}" ]] && optionals+="-secure-comms-pp-outbounds ${SECURE_COMMS_PP_OUTBOUNDS} "
 [[ "${SECURE_COMMS_KBS_ADDR}" ]] && optionals+="-secure-comms-kbs ${SECURE_COMMS_KBS_ADDR} "
 [[ "${PEERPODS_LIMIT_PER_NODE}" ]] && optionals+="-peerpods-limit-per-node ${PEERPODS_LIMIT_PER_NODE} "
+[[ "${EXTERNAL_NETWORK_VIA_PODVM}" ]] && optionals+="-ext-network-via-podvm  "
+[[ "${POD_SUBNET_CIDRS}" ]] && optionals+="-pod-subnet-cidrs ${POD_SUBNET_CIDRS} "
 
 test_vars() {
     for i in "$@"; do

src/cloud-api-adaptor/install/overlays/aws/kustomization.yaml

@@ -34,6 +34,8 @@ configMapGenerator:
   #- AWS_SUBNET_ID="" # if not set retrieved from IMDS
   #- TAGS="" # Uncomment and add key1=value1,key2=value2 etc if you want to use specific tags for podvm
   #- USE_PUBLIC_IP="true" # Uncomment if you want to use public ip for podvm
+  #- EXTERNAL_NETWORK_VIA_PODVM="true" # Uncomment if you want to use podvm as external network
+  #- POD_SUBNET_CIDRS="10.244.0.0/16,10.96.0.0/12" # Uncomment and set if you want to use specific subnet cidrs for podvm. Comma separated. The default is for a kind cluster
   #- ROOT_VOLUME_SIZE="30" # Uncomment and set if you want to use a specific root volume size. Defaults to 30
   #- FORWARDER_PORT="" # Uncomment and set if you want to use a specific port for agent-protocol-forwarder. Defaults to 15150
   #- PEERPODS_LIMIT_PER_NODE="10" # Max number of peer pods that can be created per node. Default is 10

src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go

@@ -226,25 +226,30 @@ func (s *cloudService) CreateVM(ctx context.Context, req *pb.CreateVMRequest) (r
 	// Get Pod VM image from annotations
 	image := util.GetImageFromAnnotation(req.Annotations)
 
+	netNSPath := req.NetworkNamespacePath
+
+	podNetworkConfig, err := s.workerNode.Inspect(netNSPath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to inspect netns %s: %w", netNSPath, err)
+	}
+
+	if podNetworkConfig == nil {
+		return nil, fmt.Errorf("pod network config is nil")
+	}
+
 	// Pod VM spec
 	vmSpec := provider.InstanceTypeSpec{
 		InstanceType: instanceType,
 		VCPUs:        vcpus,
 		Memory:       memory,
 		GPUs:         gpus,
 		Image:        image,
+		MultiNic:     podNetworkConfig.ExternalNetViaPodVM,
 	}
 
 	// TODO: server name is also generated in each cloud provider, and possibly inconsistent
 	serverName := putil.GenerateInstanceName(pod, string(sid), 63)
 
-	netNSPath := req.NetworkNamespacePath
-
-	podNetworkConfig, err := s.workerNode.Inspect(netNSPath)
-	if err != nil {
-		return nil, fmt.Errorf("failed to inspect netns %s: %w", netNSPath, err)
-	}
-
 	podDir := filepath.Join(s.serverConfig.PodsDir, string(sid))
 	if err := os.MkdirAll(podDir, os.ModePerm); err != nil {
 		return nil, fmt.Errorf("creating a pod directory: %s, %w", podDir, err)

src/cloud-api-adaptor/pkg/adaptor/cloud/cloud_test.go

@@ -16,6 +16,7 @@ import (
 
 	"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/adaptor/proxy"
 	"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/forwarder"
+	"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/podnetwork"
 	"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/podnetwork/tunneler"
 	"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/securecomms/kubemgr"
 	"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/securecomms/ppssh"
@@ -97,7 +98,11 @@ func (f *mockProxyFactory) New(serverName, socketPath string) proxy.AgentProxy {
 type mockWorkerNode struct{}
 
 func (n mockWorkerNode) Inspect(nsPath string) (*tunneler.Config, error) {
-	return nil, nil
+	return &tunneler.Config{
+		TunnelType:          podnetwork.DefaultTunnelType,
+		Index:               0,
+		ExternalNetViaPodVM: false,
+	}, nil
 }
 
 func (n *mockWorkerNode) Setup(nsPath string, podNodeIPs []netip.Addr, config *tunneler.Config) error {

src/cloud-api-adaptor/pkg/forwarder/forwarder.go

@@ -58,13 +58,14 @@ type Daemon interface {
 }
 
 type daemon struct {
-	tlsConfig   *tlsutil.TLSConfig
-	interceptor interceptor.Interceptor
-	podNode     podnetwork.PodNode
-	readyCh     chan struct{}
-	stopCh      chan struct{}
-	listenAddr  string
-	stopOnce    sync.Once
+	tlsConfig           *tlsutil.TLSConfig
+	interceptor         interceptor.Interceptor
+	podNode             podnetwork.PodNode
+	readyCh             chan struct{}
+	stopCh              chan struct{}
+	listenAddr          string
+	stopOnce            sync.Once
+	externalNetViaPodVM bool
 }
 
 func NewDaemon(spec *Config, listenAddr string, tlsConfig *tlsutil.TLSConfig, interceptor interceptor.Interceptor, podNode podnetwork.PodNode) Daemon {
@@ -87,6 +88,10 @@ func NewDaemon(spec *Config, listenAddr string, tlsConfig *tlsutil.TLSConfig, in
 		stopCh:      make(chan struct{}),
 	}
 
+	if spec.PodNetwork != nil {
+		daemon.externalNetViaPodVM = spec.PodNetwork.ExternalNetViaPodVM
+	}
+
 	return daemon
 }