Exclude Gitaly unavailable error from error rate

When a user hits repository rate limit, Gitaly returns an error
that the request can't be handled (Gitaly unavailable)

We should avoid this error to avoid exceeding the error rate

4 files changed, 26 insertions, 24 deletions

diff --git a/internal/config/config_test.go b/internal/config/config_test.go
index 9d9e20a..740ce77 100644
--- a/internal/config/config_test.go
+++ b/internal/config/config_test.go
@@ -41,7 +41,7 @@ func TestCustomPrometheusMetrics(t *testing.T) {
 	require.NoError(t, err)
 
 	var actualNames []string
-	for _, m := range ms[0:10] {
+	for _, m := range ms[0:9] {
 		actualNames = append(actualNames, m.GetName())
 	}
 
@@ -49,7 +49,6 @@ func TestCustomPrometheusMetrics(t *testing.T) {
 		"gitlab_shell_http_in_flight_requests",
 		"gitlab_shell_http_request_duration_seconds",
 		"gitlab_shell_http_requests_total",
-		"gitlab_shell_sshd_canceled_sessions",
 		"gitlab_shell_sshd_concurrent_limited_sessions_total",
 		"gitlab_shell_sshd_in_flight_connections",
 		"gitlab_shell_sshd_session_duration_seconds",
diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go
index e3f335d..713a99a 100644
--- a/internal/metrics/metrics.go
+++ b/internal/metrics/metrics.go
@@ -77,15 +77,6 @@ var (
 		},
 	)
 
-	SshdCanceledSessions = promauto.NewCounter(
-		prometheus.CounterOpts{
-			Namespace: namespace,
-			Subsystem: sshdSubsystem,
-			Name:      sshdCanceledSessionsName,
-			Help:      "The number of canceled gitlab-sshd sessions.",
-		},
-	)
-
 	SliSshdSessionsTotal = promauto.NewCounter(
 		prometheus.CounterOpts{
 			Name: sliSshdSessionsTotalName,
diff --git a/internal/sshd/connection.go b/internal/sshd/connection.go
index 7e56244..61234a3 100644
--- a/internal/sshd/connection.go
+++ b/internal/sshd/connection.go
@@ -89,14 +89,7 @@ func (c *connection) handle(ctx context.Context, chans <-chan ssh.NewChannel, ha
 			metrics.SliSshdSessionsTotal.Inc()
 			err := handler(ctx, channel, requests)
 			if err != nil {
-				if grpcstatus.Convert(err).Code() == grpccodes.Canceled {
-					metrics.SshdCanceledSessions.Inc()
-				} else {
-					var apiError *client.ApiError
-					if !errors.As(err, &apiError) {
-						metrics.SliSshdSessionsErrorsTotal.Inc()
-					}
-				}
+				c.trackError(err)
 			}
 
 			ctxlog.Info("connection: handle: done")
@@ -126,3 +119,17 @@ func (c *connection) sendKeepAliveMsg(ctx context.Context, ticker *time.Ticker)
 		}
 	}
 }
+
+func (c *connection) trackError(err error) {
+	var apiError *client.ApiError
+	if errors.As(err, &apiError) {
+		return
+	}
+
+	grpcCode := grpcstatus.Code(err)
+	if grpcCode == grpccodes.Canceled || grpcCode == grpccodes.Unavailable {
+		return
+	}
+
+	metrics.SliSshdSessionsErrorsTotal.Inc()
+}
diff --git a/internal/sshd/connection_test.go b/internal/sshd/connection_test.go
index 942e9cd..a6dad8d 100644
--- a/internal/sshd/connection_test.go
+++ b/internal/sshd/connection_test.go
@@ -200,7 +200,6 @@ func TestSessionsMetrics(t *testing.T) {
 	// https://pkg.go.dev/github.com/prometheus/client_golang/prometheus#pkg-index
 	initialSessionsTotal := testutil.ToFloat64(metrics.SliSshdSessionsTotal)
 	initialSessionsErrorTotal := testutil.ToFloat64(metrics.SliSshdSessionsErrorsTotal)
-	initialCanceledSessions := testutil.ToFloat64(metrics.SshdCanceledSessions)
 
 	newChannel := &fakeNewChannel{channelType: "session"}
 
@@ -212,17 +211,15 @@ func TestSessionsMetrics(t *testing.T) {
 
 	require.InDelta(t, initialSessionsTotal+1, testutil.ToFloat64(metrics.SliSshdSessionsTotal), 0.1)
 	require.InDelta(t, initialSessionsErrorTotal+1, testutil.ToFloat64(metrics.SliSshdSessionsErrorsTotal), 0.1)
-	require.InDelta(t, initialCanceledSessions, testutil.ToFloat64(metrics.SshdCanceledSessions), 0.1)
 
 	conn, chans = setup(1, newChannel)
 	conn.handle(context.Background(), chans, func(context.Context, ssh.Channel, <-chan *ssh.Request) error {
 		close(chans)
-		return grpcstatus.Error(grpccodes.Canceled, "error")
+		return grpcstatus.Error(grpccodes.Canceled, "canceled")
 	})
 
 	require.InDelta(t, initialSessionsTotal+2, testutil.ToFloat64(metrics.SliSshdSessionsTotal), 0.1)
 	require.InDelta(t, initialSessionsErrorTotal+1, testutil.ToFloat64(metrics.SliSshdSessionsErrorsTotal), 0.1)
-	require.InDelta(t, initialCanceledSessions+1, testutil.ToFloat64(metrics.SshdCanceledSessions), 0.1)
 
 	conn, chans = setup(1, newChannel)
 	conn.handle(context.Background(), chans, func(context.Context, ssh.Channel, <-chan *ssh.Request) error {
@@ -232,5 +229,13 @@ func TestSessionsMetrics(t *testing.T) {
 
 	require.InDelta(t, initialSessionsTotal+3, testutil.ToFloat64(metrics.SliSshdSessionsTotal), 0.1)
 	require.InDelta(t, initialSessionsErrorTotal+1, testutil.ToFloat64(metrics.SliSshdSessionsErrorsTotal), 0.1)
-	require.InDelta(t, initialCanceledSessions+1, testutil.ToFloat64(metrics.SshdCanceledSessions), 0.1)
+
+	conn, chans = setup(1, newChannel)
+	conn.handle(context.Background(), chans, func(context.Context, ssh.Channel, <-chan *ssh.Request) error {
+		close(chans)
+		return grpcstatus.Error(grpccodes.Unavailable, "unavailable")
+	})
+
+	require.InDelta(t, initialSessionsTotal+4, testutil.ToFloat64(metrics.SliSshdSessionsTotal), 0.1)
+	require.InDelta(t, initialSessionsErrorTotal+1, testutil.ToFloat64(metrics.SliSshdSessionsErrorsTotal), 0.1)
 }