Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implement log fetching #243

Merged
merged 4 commits into from
Dec 18, 2023
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
feat: update sessionhandler and command line for logs support
Log access is disabled by default.
smlx committed Dec 18, 2023
commit b781495acee59968bb2d8ee4c3a7ddc7c6eabd10
1 change: 1 addition & 0 deletions cmd/ssh-portal/main.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
// Package main implements the ssh-portal executable.
package main

import (
13 changes: 7 additions & 6 deletions cmd/ssh-portal/serve.go
Original file line number Diff line number Diff line change
@@ -16,11 +16,12 @@ import (

// ServeCmd represents the serve command.
type ServeCmd struct {
NATSServer string `kong:"required,env='NATS_URL',help='NATS server URL (nats://... or tls://...)'"`
SSHServerPort uint `kong:"default='2222',env='SSH_SERVER_PORT',help='Port the SSH server will listen on for SSH client connections'"`
HostKeyECDSA string `kong:"env='HOST_KEY_ECDSA',help='PEM encoded ECDSA host key'"`
HostKeyED25519 string `kong:"env='HOST_KEY_ED25519',help='PEM encoded Ed25519 host key'"`
HostKeyRSA string `kong:"env='HOST_KEY_RSA',help='PEM encoded RSA host key'"`
NATSServer string `kong:"required,env='NATS_URL',help='NATS server URL (nats://... or tls://...)'"`
SSHServerPort uint `kong:"default='2222',env='SSH_SERVER_PORT',help='Port the SSH server will listen on for SSH client connections'"`
HostKeyECDSA string `kong:"env='HOST_KEY_ECDSA',help='PEM encoded ECDSA host key'"`
HostKeyED25519 string `kong:"env='HOST_KEY_ED25519',help='PEM encoded Ed25519 host key'"`
HostKeyRSA string `kong:"env='HOST_KEY_RSA',help='PEM encoded RSA host key'"`
LogAccessEnabled bool `kong:"env='LOG_ACCESS_ENABLED',help='Allow any user who can SSH into a pod to also access its logs.'"`
}

// Run the serve command to handle SSH connection requests.
@@ -72,5 +73,5 @@ func (cmd *ServeCmd) Run(log *zap.Logger) error {
}
}
// start serving SSH connection requests
return sshserver.Serve(ctx, log, nc, l, c, hostkeys)
return sshserver.Serve(ctx, log, nc, l, c, hostkeys, cmd.LogAccessEnabled)
}
6 changes: 3 additions & 3 deletions internal/sshserver/serve.go
Original file line number Diff line number Diff line change
@@ -32,11 +32,11 @@ func disableSHA1Kex(ctx ssh.Context) *gossh.ServerConfig {

// Serve contains the main ssh session logic
func Serve(ctx context.Context, log *zap.Logger, nc *nats.EncodedConn,
l net.Listener, c *k8s.Client, hostKeys [][]byte) error {
l net.Listener, c *k8s.Client, hostKeys [][]byte, logAccessEnabled bool) error {
srv := ssh.Server{
Handler: sessionHandler(log, c, false),
Handler: sessionHandler(log, c, false, logAccessEnabled),
SubsystemHandlers: map[string]ssh.SubsystemHandler{
"sftp": ssh.SubsystemHandler(sessionHandler(log, c, true)),
"sftp": ssh.SubsystemHandler(sessionHandler(log, c, true, logAccessEnabled)),
},
PublicKeyHandler: pubKeyAuth(log, nc, c),
ServerConfigCallback: disableSHA1Kex,
201 changes: 167 additions & 34 deletions internal/sshserver/sessionhandler.go
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
package sshserver

import (
"context"
"fmt"
"strings"
"time"

"github.com/gliderlabs/ssh"
"github.com/prometheus/client_golang/prometheus"
@@ -44,20 +46,19 @@ func getSSHIntent(sftp bool, cmd []string) []string {
// handler is that the command is set to sftp-server. This implies that the
// target container must have a sftp-server binary installed for sftp to work.
// There is no support for a built-in sftp server.
func sessionHandler(log *zap.Logger, c *k8s.Client, sftp bool) ssh.Handler {
func sessionHandler(log *zap.Logger, c *k8s.Client,
sftp, logAccessEnabled bool) ssh.Handler {
return func(s ssh.Session) {
sessionTotal.Inc()
ctx := s.Context()
sid := ctx.SessionID()
// start the command
log.Debug("starting command exec",
log.Debug("starting session",
zap.String("sessionID", sid),
zap.Strings("rawCommand", s.Command()),
zap.String("subsystem", s.Subsystem()),
)
// parse the command line arguments to extract any service or container args
service, container, rawCmd := parseConnectionParams(s.Command())
cmd := getSSHIntent(sftp, rawCmd)
service, container, logs, rawCmd := parseConnectionParams(s.Command())
// validate the service and container
if err := k8s.ValidateLabelValue(service); err != nil {
log.Debug("invalid service name",
@@ -103,8 +104,6 @@ func sessionHandler(log *zap.Logger, c *k8s.Client, sftp bool) ssh.Handler {
}
return
}
// check if a pty was requested, and get the window size channel
_, winch, pty := s.Pty()
// extract info passed through the context by the authhandler
eid, ok := ctx.Value(environmentIDKey).(int)
if !ok {
@@ -126,6 +125,71 @@ func sessionHandler(log *zap.Logger, c *k8s.Client, sftp bool) ssh.Handler {
if !ok {
log.Warn("couldn't extract SSH key fingerprint from session context")
}
if len(logs) != 0 {
if !logAccessEnabled {
log.Debug("logs access is not enabled",
zap.String("logsArgument", logs),
zap.String("sessionID", sid))
_, err = fmt.Fprintf(s.Stderr(), "error executing command. SID: %s\r\n",
sid)
if err != nil {
log.Warn("couldn't send error to client",
zap.String("sessionID", sid),
zap.Error(err))
}
// Send a non-zero exit code to the client on internal logs error.
// OpenSSH uses 255 for this, 254 is an exec failure, so use 253 to
// differentiate this error.
if err = s.Exit(253); err != nil {
log.Warn("couldn't send exit code to client",
zap.String("sessionID", sid),
zap.Error(err))
}
return
}
follow, tailLines, err := parseLogsArg(service, logs, rawCmd)
if err != nil {
log.Debug("couldn't parse logs argument",
zap.String("logsArgument", logs),
zap.String("sessionID", sid),
zap.Error(err))
_, err = fmt.Fprintf(s.Stderr(), "error executing command. SID: %s\r\n",
sid)
if err != nil {
log.Warn("couldn't send error to client",
zap.String("sessionID", sid),
zap.Error(err))
}
// Send a non-zero exit code to the client on internal logs error.
// OpenSSH uses 255 for this, 254 is an exec failure, so use 253 to
// differentiate this error.
if err = s.Exit(253); err != nil {
log.Warn("couldn't send exit code to client",
zap.String("sessionID", sid),
zap.Error(err))
}
return
}
log.Info("sending logs to SSH client",
zap.Int("environmentID", eid),
zap.Int("projectID", pid),
zap.String("SSHFingerprint", fingerprint),
zap.String("container", container),
zap.String("deployment", deployment),
zap.String("environmentName", ename),
zap.String("namespace", s.User()),
zap.String("projectName", pname),
zap.String("sessionID", sid),
zap.Bool("follow", follow),
zap.Int64("tailLines", tailLines),
)
doLogs(ctx, log, s, deployment, container, follow, tailLines, c, sid)
return
}
// handle sftp and sh fallback
cmd := getSSHIntent(sftp, rawCmd)
// check if a pty was requested, and get the window size channel
_, winch, pty := s.Pty()
log.Info("executing SSH command",
zap.Bool("pty", pty),
zap.Int("environmentID", eid),
@@ -139,39 +203,108 @@ func sessionHandler(log *zap.Logger, c *k8s.Client, sftp bool) ssh.Handler {
zap.String("sessionID", sid),
zap.Strings("command", cmd),
)
err = c.Exec(ctx, s.User(), deployment, container, cmd, s,
s.Stderr(), pty, winch)
doExec(ctx, log, s, deployment, container, cmd, c, pty, winch, sid)
}
}

// startClientKeepalive sends a keepalive request to the client via the channel
// embedded in ssh.Session at a regular interval. If the client fails to
// respond, the channel is closed, and cancel is called.
func startClientKeepalive(ctx context.Context, cancel context.CancelFunc,
log *zap.Logger, s ssh.Session) {
ticker := time.NewTicker(2 * time.Second)
defer ticker.Stop()
for {
select {
case <-ticker.C:
// https://github.com/openssh/openssh-portable/blob/
// edc2ef4e418e514c99701451fae4428ec04ce538/serverloop.c#L127-L158
_, err := s.SendRequest("keepalive@openssh.com", true, nil)
if err != nil {
log.Debug("client closed connection", zap.Error(err))
_ = s.Close()
cancel()
return
}
case <-ctx.Done():
return
}
}
}

func doLogs(ctx ssh.Context, log *zap.Logger, s ssh.Session, deployment,
container string, follow bool, tailLines int64, c *k8s.Client, sid string) {
// Wrap the ssh.Context so we can cancel goroutines started from this
// function without affecting the SSH session.
childCtx, cancel := context.WithCancel(ctx)
defer cancel()
// In a multiplexed connection (multiple SSH channels to the single TCP
// connection), if the client disconnects from the channel the session
// context will not be cancelled (because the TCP connection is still up),
// and k8s.Logs() will hang.
//
// To work around this problem, start a goroutine to send a regular keepalive
// ping to the client. If the keepalive fails, close the channel and cancel
// the childCtx.
go startClientKeepalive(childCtx, cancel, log, s)
err := c.Logs(childCtx, s.User(), deployment, container, follow, tailLines, s)
if err != nil {
log.Warn("couldn't send logs",
zap.String("sessionID", sid),
zap.Error(err))
_, err = fmt.Fprintf(s.Stderr(), "error executing command. SID: %s\r\n",
sid)
if err != nil {
if exitErr, ok := err.(exec.ExitError); ok {
log.Debug("couldn't execute command",
log.Warn("couldn't send error to client",
zap.String("sessionID", sid),
zap.Error(err))
}
// Send a non-zero exit code to the client on internal logs error.
// OpenSSH uses 255 for this, 254 is an exec failure, so use 253 to
// differentiate this error.
if err = s.Exit(253); err != nil {
log.Warn("couldn't send exit code to client",
zap.String("sessionID", sid),
zap.Error(err))
}
}
log.Debug("finished command logs", zap.String("sessionID", sid))
}

func doExec(ctx ssh.Context, log *zap.Logger, s ssh.Session, deployment,
container string, cmd []string, c *k8s.Client, pty bool,
winch <-chan ssh.Window, sid string) {
err := c.Exec(ctx, s.User(), deployment, container, cmd, s,
s.Stderr(), pty, winch)
if err != nil {
if exitErr, ok := err.(exec.ExitError); ok {
log.Debug("couldn't execute command",
zap.String("sessionID", sid),
zap.Error(err))
if err = s.Exit(exitErr.ExitStatus()); err != nil {
log.Warn("couldn't send exit code to client",
zap.String("sessionID", sid),
zap.Error(err))
if err = s.Exit(exitErr.ExitStatus()); err != nil {
log.Warn("couldn't send exit code to client",
zap.String("sessionID", sid),
zap.Error(err))
}
} else {
log.Warn("couldn't execute command",
}
} else {
log.Warn("couldn't execute command",
zap.String("sessionID", sid),
zap.Error(err))
_, err = fmt.Fprintf(s.Stderr(), "error executing command. SID: %s\r\n",
sid)
if err != nil {
log.Warn("couldn't send error to client",
zap.String("sessionID", sid),
zap.Error(err))
}
// Send a non-zero exit code to the client on internal exec error.
// OpenSSH uses 255 for this, so use 254 to differentiate the error.
if err = s.Exit(254); err != nil {
log.Warn("couldn't send exit code to client",
zap.String("sessionID", sid),
zap.Error(err))
_, err = fmt.Fprintf(s.Stderr(), "error executing command. SID: %s\r\n",
sid)
if err != nil {
log.Warn("couldn't send error to client",
zap.String("sessionID", sid),
zap.Error(err))
}
// Send a non-zero exit code to the client on internal exec error.
// OpenSSH uses 255 for this, so use 254 to differentiate the error.
if err = s.Exit(254); err != nil {
log.Warn("couldn't send exit code to client",
zap.String("sessionID", sid),
zap.Error(err))
}
}
}
log.Debug("finished command exec",
zap.String("sessionID", sid))
}
log.Debug("finished command exec", zap.String("sessionID", sid))
}