diff options
| author | Igor Drozdov <idrozdov@gitlab.com> | 2022-05-16 01:32:07 +0400 |
|---|---|---|
| committer | Igor Drozdov <idrozdov@gitlab.com> | 2022-05-17 08:17:47 +0400 |
| commit | c05ed91ea824ce015f51417196021b7e8a5837d0 (patch) | |
| tree | f931815591e9cbaca48e97dcef6a2a493b54e4e0 | |
| parent | a77babe96fac9c880061fa63fffabfc8406f11bf (diff) | |
| download | gitlab-shell-c05ed91ea824ce015f51417196021b7e8a5837d0.tar.gz | |
Wait until all Gitaly sessions are executed
If they haven't been executed within a timeout, we unblock the
execution.
Issue: https://gitlab.com/gitlab-org/gitlab-shell/-/issues/563
We have an issue when a lot of git clones operations get canceled.
It was assumed that some clients close the connection just after
all the data has been received from Git server. If there was a
network delay and gitlab-sshd hadn't managed to gracefully close
the connection, context get canceled and Gitaly cancels the
execution and returns the error.
Let's wait for a perion to allow Gitaly to gracefully complete the
operation
| -rw-r--r-- | internal/sshd/connection.go | 16 | ||||
| -rw-r--r-- | internal/sshd/connection_test.go | 5 |
2 files changed, 15 insertions, 6 deletions
diff --git a/internal/sshd/connection.go b/internal/sshd/connection.go index 060156d..ebb4fe0 100644 --- a/internal/sshd/connection.go +++ b/internal/sshd/connection.go @@ -15,19 +15,25 @@ import ( const KeepAliveMsg = "keepalive@openssh.com" +var EOFTimeout = 10 * time.Second + type connection struct { cfg *config.Config concurrentSessions *semaphore.Weighted remoteAddr string sconn *ssh.ServerConn + maxSessions int64 } type channelHandler func(context.Context, ssh.Channel, <-chan *ssh.Request) error func newConnection(cfg *config.Config, remoteAddr string, sconn *ssh.ServerConn) *connection { + maxSessions := cfg.Server.ConcurrentSessionsLimit + return &connection{ cfg: cfg, - concurrentSessions: semaphore.NewWeighted(cfg.Server.ConcurrentSessionsLimit), + maxSessions: maxSessions, + concurrentSessions: semaphore.NewWeighted(maxSessions), remoteAddr: remoteAddr, sconn: sconn, } @@ -85,6 +91,14 @@ func (c *connection) handle(ctx context.Context, chans <-chan ssh.NewChannel, ha ctxlog.Info("connection: handle: done") }() } + + // When a connection has been prematurely closed we block execution until all concurrent sessions are released + // in order to allow Gitaly complete the operations and close all the channels gracefully. + // If it didn't happen within timeout, we unblock the execution + // Related issue: https://gitlab.com/gitlab-org/gitlab-shell/-/issues/563 + ctx, cancel := context.WithTimeout(ctx, EOFTimeout) + defer cancel() + c.concurrentSessions.Acquire(ctx, c.maxSessions) } func (c *connection) sendKeepAliveMsg(ctx context.Context, ticker *time.Ticker) { diff --git a/internal/sshd/connection_test.go b/internal/sshd/connection_test.go index 0a06255..d3a2287 100644 --- a/internal/sshd/connection_test.go +++ b/internal/sshd/connection_test.go @@ -55,14 +55,9 @@ type fakeConn struct { ssh.Conn sentRequestName string - waitErr error mu sync.Mutex } -func (f *fakeConn) Wait() error { - return f.waitErr -} - func (f *fakeConn) SentRequestName() string { f.mu.Lock() defer f.mu.Unlock() |