diff options
author | Ben Kochie <superq@gmail.com> | 2021-02-16 11:51:45 +0100 |
---|---|---|
committer | Ben Kochie <superq@gmail.com> | 2021-02-16 20:38:43 +0100 |
commit | 18b8454681056907a67a29abc5f457143e8b27af (patch) | |
tree | d2e3cb59a1f40ae7df2b588453a9611bc9b91150 | |
parent | a300cf87a12644b49b260287f0b26d2800049b34 (diff) | |
download | gitlab-shell-18b8454681056907a67a29abc5f457143e8b27af.tar.gz |
Add basic metrics to sshdbjk/monitoring
* Counter for how many times the max concurrent sessions limit was hit.
* Histogram for duration of each SSH connection.
https://gitlab.com/gitlab-org/gitlab-shell/-/issues/121
Signed-off-by: Ben Kochie <superq@gmail.com>
-rw-r--r-- | internal/sshd/sshd.go | 48 |
1 files changed, 48 insertions, 0 deletions
diff --git a/internal/sshd/sshd.go b/internal/sshd/sshd.go index 648e29b..74b4bf4 100644 --- a/internal/sshd/sshd.go +++ b/internal/sshd/sshd.go @@ -10,6 +10,8 @@ import ( "strconv" "time" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" log "github.com/sirupsen/logrus" "gitlab.com/gitlab-org/gitlab-shell/internal/command" "gitlab.com/gitlab-org/gitlab-shell/internal/command/commandargs" @@ -20,6 +22,46 @@ import ( "golang.org/x/sync/semaphore" ) +const ( + namespace = "gitlab_shell" + sshdSubsystem = "sshd" +) + +func secondsDurationBuckets() []float64 { + return []float64{ + 0.005, /* 5ms */ + 0.025, /* 25ms */ + 0.1, /* 100ms */ + 0.5, /* 500ms */ + 1.0, /* 1s */ + 10.0, /* 10s */ + 30.0, /* 30s */ + 60.0, /* 1m */ + 300.0, /* 10m */ + } +} + +var ( + sshdConnectionDuration = promauto.NewHistogram( + prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: sshdSubsystem, + Name: "connection_duration_seconds", + Help: "A histogram of latencies for connections to gitlab-shell sshd.", + Buckets: secondsDurationBuckets(), + }, + ) + + sshdHitMaxSessions = promauto.NewCounter( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: sshdSubsystem, + Name: "concurrent_limited_sessions_total", + Help: "The number of times the concurrent sessions limit was hit in gitlab-shell sshd.", + }, + ) +) + func Run(cfg *config.Config) error { authorizedKeysClient, err := authorizedkeys.NewClient(cfg) if err != nil { @@ -108,6 +150,11 @@ func exitSession(ch ssh.Channel, exitStatus uint32) { } func handleConn(nconn net.Conn, sshCfg *ssh.ServerConfig, cfg *config.Config) { + begin := time.Now() + defer func() { + sshdConnectionDuration.Observe(time.Since(begin).Seconds()) + }() + ctx, cancel := context.WithCancel(context.Background()) defer cancel() defer nconn.Close() @@ -127,6 +174,7 @@ func handleConn(nconn net.Conn, sshCfg *ssh.ServerConfig, cfg *config.Config) { } if !concurrentSessions.TryAcquire(1) { newChannel.Reject(ssh.ResourceShortage, "too many concurrent sessions") + sshdHitMaxSessions.Inc() continue } ch, requests, err := newChannel.Accept() |