From 67a58bebd117870ccb2e9e12bbdab84bc89698ea Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Wed, 2 Jul 2014 13:40:42 +0000 Subject: Change backup-gitlab to use database dump instead of Rake script The Gitlab Rake script for backing up doesn't backup the database that Gitlab CI uses, and additonally it does a git bundle for each git repository. Our GitLab instance has hundreds of git repositories, making the bundling be a very expensive things. Instead, we use the Postgres tool to dump all databases to a file, and arrange to backup the dump files, the git repositories, and all other relevant files directly. --- gitlab-server/usr/share/gitlab-install/backup-gitlab | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/gitlab-server/usr/share/gitlab-install/backup-gitlab b/gitlab-server/usr/share/gitlab-install/backup-gitlab index a6c3e3f5..1320df81 100644 --- a/gitlab-server/usr/share/gitlab-install/backup-gitlab +++ b/gitlab-server/usr/share/gitlab-install/backup-gitlab @@ -4,19 +4,13 @@ # /home/git/gitlab-backup.tar . A separate backup host will need to # back that up. # -# This script be run as the git user on a GitLab host. +# This script be run as the postgres user on a Baserock GitLab host. set -e export PATH=/usr/local/bin:/usr/bin:/bin -cd /home/git/gitlab -mkdir -p /home/git/gitlab/tmp/backups -bundle exec rake gitlab:backup:create RAILS_ENV=production - -ls /home/git/gitlab/tmp/backups/[0-9]*.tar | -tail -n1 | -while read filename -do - mv "$filename" /home/git/gitlab-backup.tar -done +mkdir -p /home/postgres/dumps +cd /home/postgres/dumps +pg_dumpall > new.dump +mv new.dump gitlab.pg_dumpall -- cgit v1.2.1 From e9f76f4d8cc75ecd7e51c97b3af621dbfa8c6dd0 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Wed, 2 Jul 2014 13:42:50 +0000 Subject: Add systemd units to run backup-gitlab periodically --- .../usr/share/gitlab-install/systemd-units/gitlab-backup.service | 4 ++-- .../usr/share/gitlab-install/systemd-units/gitlab-backup.timer | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/gitlab-server/usr/share/gitlab-install/systemd-units/gitlab-backup.service b/gitlab-server/usr/share/gitlab-install/systemd-units/gitlab-backup.service index 967a65e3..8672c36d 100644 --- a/gitlab-server/usr/share/gitlab-install/systemd-units/gitlab-backup.service +++ b/gitlab-server/usr/share/gitlab-install/systemd-units/gitlab-backup.service @@ -7,5 +7,5 @@ WantedBy=gitlab.target [Service] ExecStart=/usr/share/gitlab-install/backup-gitlab -User=git -Group=git +User=postgres +Group=postgres diff --git a/gitlab-server/usr/share/gitlab-install/systemd-units/gitlab-backup.timer b/gitlab-server/usr/share/gitlab-install/systemd-units/gitlab-backup.timer index 1c2f57ba..1686843f 100644 --- a/gitlab-server/usr/share/gitlab-install/systemd-units/gitlab-backup.timer +++ b/gitlab-server/usr/share/gitlab-install/systemd-units/gitlab-backup.timer @@ -5,4 +5,4 @@ Description=GitLab Backup Timer WantedBy=gitlab.target [Timer] -OnCalendar=*-*-* 00/4:00:00 +OnCalendar=*-*-* 00:*:* -- cgit v1.2.1 From 616922c6125a1954ef51aac47cfce55186138699 Mon Sep 17 00:00:00 2001 From: Michael Drake Date: Tue, 1 Jul 2014 16:20:43 +0000 Subject: Add fstab configuration extension to gitlab-server system. This is useful so that we can deploy a Gitlab system with a separate /home disk. --- gitlab-server.morph | 1 + 1 file changed, 1 insertion(+) diff --git a/gitlab-server.morph b/gitlab-server.morph index 6142b8c6..8dc9f3cf 100644 --- a/gitlab-server.morph +++ b/gitlab-server.morph @@ -6,6 +6,7 @@ configuration-extensions: - nfsboot - install-files - gitlab +- fstab description: system which hosts a gitlab server kind: system name: gitlab-server -- cgit v1.2.1 From c661f1349f44120b3cfcfc718ebb44b72f299cb8 Mon Sep 17 00:00:00 2001 From: Michael Drake Date: Wed, 2 Jul 2014 09:57:32 +0100 Subject: Add remote backup and restoration scripts. - Scripts written by Lars Wirzenius. - Reviewed by Michael Drake. These scripts are to be copied to the system on which the backups are made. They are stored here to keep the related logic in one source location. --- .../share/gitlab-install/gitlab-remote-backup.sh | 22 +++++++++ .../share/gitlab-install/gitlab-remote-restore.sh | 57 ++++++++++++++++++++++ 2 files changed, 79 insertions(+) create mode 100644 gitlab-server/usr/share/gitlab-install/gitlab-remote-backup.sh create mode 100644 gitlab-server/usr/share/gitlab-install/gitlab-remote-restore.sh diff --git a/gitlab-server/usr/share/gitlab-install/gitlab-remote-backup.sh b/gitlab-server/usr/share/gitlab-install/gitlab-remote-backup.sh new file mode 100644 index 00000000..85618811 --- /dev/null +++ b/gitlab-server/usr/share/gitlab-install/gitlab-remote-backup.sh @@ -0,0 +1,22 @@ +#!/bin/sh +# +# Copy relevant files of a Baserock Gitlab instance out of the instance. +# +# Usage: backup.sh ADDR +# where ADDR is the address (domain name, IP address) of the instance. +# The files are copied to the current directory. + +set -eux + +ADDR="$1" + +backup() +{ + rsync -ahHS --delete "root@$ADDR:$1" "$2" +} + +mkdir -p dumps repositories uploads +backup /home/postgres/dumps/. dumps/. +backup /home/git/repositories/. repositories/. +backup /home/git/gitlab/public/uploads/. uploads/. + diff --git a/gitlab-server/usr/share/gitlab-install/gitlab-remote-restore.sh b/gitlab-server/usr/share/gitlab-install/gitlab-remote-restore.sh new file mode 100644 index 00000000..78ff691a --- /dev/null +++ b/gitlab-server/usr/share/gitlab-install/gitlab-remote-restore.sh @@ -0,0 +1,57 @@ +#!/bin/sh +# +# Restore a Baserock Gitlab system backup to a fresh instance. +# +# Usage: restore.sh ADDR +# where ADDR is the address (domain name, IP address) of the instance. +# +# What this does is a) stop services b) copy files over c) reset the Postgres +# databases. + +set -eux + +ADDR="$1" + +restore() +{ + rsync -ahHS --delete "$2" "root@$ADDR:$1" +} + +# Stop services so we don't modify files and databases from underneath +# them, and also so they don't modify things while restore is happening. + +ssh "root@$ADDR" systemctl stop \ + crond gitlab-backup.service \ + gitlab-ci-sidekiq.service \ + gitlab-ci-unicorn.service \ + gitlab-sidekiq.service \ + gitlab-unicorn.service \ + gitlab.target \ + gitlab-backup.timer \ + nginx.service \ + redis.service + +# Create the directory where postgres dump files go. + +ssh "root@$ADDR" install -d -o postgres -g postgres /home/postgres/dumps + +# Restore the various files. + +restore /home/postgres/dumps/. dumps/. +restore /home/git/repositories/. repositories/. +restore /home/git/gitlab/public/uploads/. uploads/. + +# And thier uid/gid +ssh "root@$ADDR" chown -R git:git /home/git/repositories /home/git/gitlab/public/uploads + +# Delete tables and roles from Postgres so that the restore can happen. + +ssh "root@$ADDR" sudo -u postgres psql <