From 046b28312704f3131e72dcd2dbdacc5264d4aa62 Mon Sep 17 00:00:00 2001
From: Douwe Maan <douwe@gitlab.com>
Date: Tue, 25 Aug 2015 18:42:46 -0700
Subject: Groundwork for merging CI into CE

---
 lib/tasks/ci/.gitkeep             |  0
 lib/tasks/ci/backup.rake          | 62 +++++++++++++++++++++++++++++++++++++++
 lib/tasks/ci/cleanup.rake         |  8 +++++
 lib/tasks/ci/schedule_builds.rake |  6 ++++
 lib/tasks/ci/setup.rake           |  7 +++++
 lib/tasks/ci/sidekiq.rake         | 13 ++++++++
 6 files changed, 96 insertions(+)
 create mode 100644 lib/tasks/ci/.gitkeep
 create mode 100644 lib/tasks/ci/backup.rake
 create mode 100644 lib/tasks/ci/cleanup.rake
 create mode 100644 lib/tasks/ci/schedule_builds.rake
 create mode 100644 lib/tasks/ci/setup.rake
 create mode 100644 lib/tasks/ci/sidekiq.rake

(limited to 'lib/tasks')

diff --git a/lib/tasks/ci/.gitkeep b/lib/tasks/ci/.gitkeep
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/lib/tasks/ci/backup.rake b/lib/tasks/ci/backup.rake
new file mode 100644
index 00000000000..1cb2e43f875
--- /dev/null
+++ b/lib/tasks/ci/backup.rake
@@ -0,0 +1,62 @@
+namespace :ci do
+  namespace :backup do
+    
+    desc "GITLAB | Create a backup of the GitLab CI database"
+    task create: :environment do
+      configure_cron_mode
+
+      $progress.puts "Dumping database ... ".blue
+      Ci::Backup::Database.new.dump
+      $progress.puts "done".green
+
+      $progress.puts "Dumping builds ... ".blue
+      Ci::Backup::Builds.new.dump
+      $progress.puts "done".green
+
+      backup = Ci::Backup::Manager.new
+      backup.pack
+      backup.cleanup
+      backup.remove_old
+    end
+
+    desc "GITLAB | Restore a previously created backup"
+    task restore: :environment do
+      configure_cron_mode
+
+      backup = Ci::Backup::Manager.new
+      backup.unpack
+
+      $progress.puts "Restoring database ... ".blue
+      Ci::Backup::Database.new.restore
+      $progress.puts "done".green
+
+      $progress.puts "Restoring builds ... ".blue
+      Ci::Backup::Builds.new.restore
+      $progress.puts "done".green
+
+      backup.cleanup
+    end
+
+    def configure_cron_mode
+      if ENV['CRON']
+        # We need an object we can say 'puts' and 'print' to; let's use a
+        # StringIO.
+        require 'stringio'
+        $progress = StringIO.new
+      else
+        $progress = $stdout
+      end
+    end
+  end
+
+  # Disable colors for CRON
+  unless STDOUT.isatty
+    module Colored
+      extend self
+
+      def colorize(string, options={})
+        string
+      end
+    end
+  end
+end
diff --git a/lib/tasks/ci/cleanup.rake b/lib/tasks/ci/cleanup.rake
new file mode 100644
index 00000000000..2f4d11bd942
--- /dev/null
+++ b/lib/tasks/ci/cleanup.rake
@@ -0,0 +1,8 @@
+namespace :ci do
+  namespace :cleanup do
+    desc "GitLab CI | Clean running builds"
+    task builds: :environment do
+      Ci::Build.running.update_all(status: 'canceled')
+    end
+  end
+end
diff --git a/lib/tasks/ci/schedule_builds.rake b/lib/tasks/ci/schedule_builds.rake
new file mode 100644
index 00000000000..49435504c67
--- /dev/null
+++ b/lib/tasks/ci/schedule_builds.rake
@@ -0,0 +1,6 @@
+namespace :ci do
+  desc "GitLab CI | Clean running builds"
+  task schedule_builds: :environment do
+    Ci::Scheduler.new.perform
+  end
+end
diff --git a/lib/tasks/ci/setup.rake b/lib/tasks/ci/setup.rake
new file mode 100644
index 00000000000..ab83581ec1b
--- /dev/null
+++ b/lib/tasks/ci/setup.rake
@@ -0,0 +1,7 @@
+namespace :ci do
+  desc "GitLab CI | Setup gitlab db"
+  task :setup do
+    Rake::Task["db:setup"].invoke
+    Rake::Task["ci:add_limits_mysql"].invoke
+  end
+end
diff --git a/lib/tasks/ci/sidekiq.rake b/lib/tasks/ci/sidekiq.rake
new file mode 100644
index 00000000000..12fd3635933
--- /dev/null
+++ b/lib/tasks/ci/sidekiq.rake
@@ -0,0 +1,13 @@
+namespace :ci do
+  namespace :sidekiq do
+    desc "GitLab CI | Stop sidekiq"
+    task :stop do
+      exec({'RAILS_ENV' => Rails.env}, 'script/background_jobs stop')
+    end
+
+    desc "GitLab CI | Start sidekiq"
+    task :start do
+      exec({'RAILS_ENV' => Rails.env}, 'script/background_jobs start')
+    end
+  end
+end
-- 
cgit v1.2.1


From 9a3d0f1d92ee99792b40c401f83121adb8fd3387 Mon Sep 17 00:00:00 2001
From: Kamil Trzcinski <ayufan@ayufan.eu>
Date: Tue, 15 Sep 2015 00:28:11 +0200
Subject: Add rake task to migrate CI tags

---
 lib/tasks/ci/migrate.rake | 40 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 40 insertions(+)
 create mode 100644 lib/tasks/ci/migrate.rake

(limited to 'lib/tasks')

diff --git a/lib/tasks/ci/migrate.rake b/lib/tasks/ci/migrate.rake
new file mode 100644
index 00000000000..c00b17f7a2d
--- /dev/null
+++ b/lib/tasks/ci/migrate.rake
@@ -0,0 +1,40 @@
+namespace :ci do
+  namespace :migrate do
+    def list_objects(type)
+      ids = ActiveRecord::Base.connection.select_all(
+        'select distinct taggable_id from ci_taggings where taggable_type = $1',
+        nil, [[nil, type]]
+      )
+      ids.map { |id| id['taggable_id'] }
+    end
+
+    def list_tags(type, id)
+      tags = ActiveRecord::Base.connection.select_all(
+        'select ci_tags.name from ci_tags ' +
+        'join ci_taggings on ci_tags.id = ci_taggings.tag_id ' +
+        'where taggable_type = $1 and taggable_id = $2 and context = $3',
+        nil, [[nil, type], [nil, id], [nil, 'tags']]
+      )
+      tags.map { |tag| tag['name'] }
+    end
+
+    desc 'GITLAB | Migrate CI tags'
+    task tags: :environment do
+      list_objects('Runner').each do |id|
+        runner = Ci::Runner.find_by_id(id)
+        if runner
+          tags = list_tags('Runner', id)
+          runner.update_attributes(tag_list: tags)
+        end
+      end
+
+      list_objects('Build').each do |id|
+        build = Ci::Build.find_by_id(id)
+        if build
+          tags = list_tags('Build', id)
+          build.update_attributes(tag_list: tags)
+        end
+      end
+    end
+  end
+end
-- 
cgit v1.2.1


From ed18e04bb3d921d14f1bfb9adf7e0a4ad4dfc0b2 Mon Sep 17 00:00:00 2001
From: Kamil Trzcinski <ayufan@ayufan.eu>
Date: Tue, 15 Sep 2015 10:24:30 +0200
Subject: Cleanup CI backup => migrate with GitLab

---
 lib/tasks/ci/backup.rake     | 62 --------------------------------------------
 lib/tasks/gitlab/backup.rake | 21 +++++++++++++++
 2 files changed, 21 insertions(+), 62 deletions(-)
 delete mode 100644 lib/tasks/ci/backup.rake

(limited to 'lib/tasks')

diff --git a/lib/tasks/ci/backup.rake b/lib/tasks/ci/backup.rake
deleted file mode 100644
index 1cb2e43f875..00000000000
--- a/lib/tasks/ci/backup.rake
+++ /dev/null
@@ -1,62 +0,0 @@
-namespace :ci do
-  namespace :backup do
-    
-    desc "GITLAB | Create a backup of the GitLab CI database"
-    task create: :environment do
-      configure_cron_mode
-
-      $progress.puts "Dumping database ... ".blue
-      Ci::Backup::Database.new.dump
-      $progress.puts "done".green
-
-      $progress.puts "Dumping builds ... ".blue
-      Ci::Backup::Builds.new.dump
-      $progress.puts "done".green
-
-      backup = Ci::Backup::Manager.new
-      backup.pack
-      backup.cleanup
-      backup.remove_old
-    end
-
-    desc "GITLAB | Restore a previously created backup"
-    task restore: :environment do
-      configure_cron_mode
-
-      backup = Ci::Backup::Manager.new
-      backup.unpack
-
-      $progress.puts "Restoring database ... ".blue
-      Ci::Backup::Database.new.restore
-      $progress.puts "done".green
-
-      $progress.puts "Restoring builds ... ".blue
-      Ci::Backup::Builds.new.restore
-      $progress.puts "done".green
-
-      backup.cleanup
-    end
-
-    def configure_cron_mode
-      if ENV['CRON']
-        # We need an object we can say 'puts' and 'print' to; let's use a
-        # StringIO.
-        require 'stringio'
-        $progress = StringIO.new
-      else
-        $progress = $stdout
-      end
-    end
-  end
-
-  # Disable colors for CRON
-  unless STDOUT.isatty
-    module Colored
-      extend self
-
-      def colorize(string, options={})
-        string
-      end
-    end
-  end
-end
diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake
index 4c73f90bbf2..f20c7f71ba5 100644
--- a/lib/tasks/gitlab/backup.rake
+++ b/lib/tasks/gitlab/backup.rake
@@ -11,6 +11,7 @@ namespace :gitlab do
       Rake::Task["gitlab:backup:db:create"].invoke
       Rake::Task["gitlab:backup:repo:create"].invoke
       Rake::Task["gitlab:backup:uploads:create"].invoke
+      Rake::Task["gitlab:backup:builds:create"].invoke
 
       backup = Backup::Manager.new
       backup.pack
@@ -30,6 +31,7 @@ namespace :gitlab do
       Rake::Task["gitlab:backup:db:restore"].invoke unless backup.skipped?("db")
       Rake::Task["gitlab:backup:repo:restore"].invoke unless backup.skipped?("repositories")
       Rake::Task["gitlab:backup:uploads:restore"].invoke unless backup.skipped?("uploads")
+      Rake::Task["gitlab:backup:builds:restore"].invoke unless backup.skipped?("builds")
       Rake::Task["gitlab:shell:setup"].invoke
 
       backup.cleanup
@@ -73,6 +75,25 @@ namespace :gitlab do
       end
     end
 
+    namespace :builds do
+      task create: :environment do
+        $progress.puts "Dumping builds ... ".blue
+
+        if ENV["SKIP"] && ENV["SKIP"].include?("builds")
+          $progress.puts "[SKIPPED]".cyan
+        else
+          Backup::Builds.new.dump
+          $progress.puts "done".green
+        end
+      end
+
+      task restore: :environment do
+        $progress.puts "Restoring builds ... ".blue
+        Backup::Builds.new.restore
+        $progress.puts "done".green
+      end
+    end
+
     namespace :uploads do
       task create: :environment do
         $progress.puts "Dumping uploads ... ".blue
-- 
cgit v1.2.1


From 7e07bc06980e56cabb90538c4c1ebd3dd027ed50 Mon Sep 17 00:00:00 2001
From: Kamil Trzcinski <ayufan@ayufan.eu>
Date: Tue, 15 Sep 2015 10:25:49 +0200
Subject: Remove unused tasks

---
 lib/tasks/ci/setup.rake   |  7 -------
 lib/tasks/ci/sidekiq.rake | 13 -------------
 2 files changed, 20 deletions(-)
 delete mode 100644 lib/tasks/ci/setup.rake
 delete mode 100644 lib/tasks/ci/sidekiq.rake

(limited to 'lib/tasks')

diff --git a/lib/tasks/ci/setup.rake b/lib/tasks/ci/setup.rake
deleted file mode 100644
index ab83581ec1b..00000000000
--- a/lib/tasks/ci/setup.rake
+++ /dev/null
@@ -1,7 +0,0 @@
-namespace :ci do
-  desc "GitLab CI | Setup gitlab db"
-  task :setup do
-    Rake::Task["db:setup"].invoke
-    Rake::Task["ci:add_limits_mysql"].invoke
-  end
-end
diff --git a/lib/tasks/ci/sidekiq.rake b/lib/tasks/ci/sidekiq.rake
deleted file mode 100644
index 12fd3635933..00000000000
--- a/lib/tasks/ci/sidekiq.rake
+++ /dev/null
@@ -1,13 +0,0 @@
-namespace :ci do
-  namespace :sidekiq do
-    desc "GitLab CI | Stop sidekiq"
-    task :stop do
-      exec({'RAILS_ENV' => Rails.env}, 'script/background_jobs stop')
-    end
-
-    desc "GitLab CI | Start sidekiq"
-    task :start do
-      exec({'RAILS_ENV' => Rails.env}, 'script/background_jobs start')
-    end
-  end
-end
-- 
cgit v1.2.1


From 269b9224cf6a6c00c163ecff0b503db2d1f88ec8 Mon Sep 17 00:00:00 2001
From: Kamil Trzcinski <ayufan@ayufan.eu>
Date: Tue, 15 Sep 2015 10:48:52 +0200
Subject: Use GitLab instead of GITLAB for rake task

---
 lib/tasks/ci/migrate.rake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib/tasks')

diff --git a/lib/tasks/ci/migrate.rake b/lib/tasks/ci/migrate.rake
index c00b17f7a2d..7d99664dcf3 100644
--- a/lib/tasks/ci/migrate.rake
+++ b/lib/tasks/ci/migrate.rake
@@ -18,7 +18,7 @@ namespace :ci do
       tags.map { |tag| tag['name'] }
     end
 
-    desc 'GITLAB | Migrate CI tags'
+    desc 'GitLab | Migrate CI tags'
     task tags: :environment do
       list_objects('Runner').each do |id|
         runner = Ci::Runner.find_by_id(id)
-- 
cgit v1.2.1


From 9c5833d5acd6efecf54e4c910dd7c4e89d4ddca6 Mon Sep 17 00:00:00 2001
From: Kamil Trzcinski <ayufan@ayufan.eu>
Date: Tue, 15 Sep 2015 16:35:27 +0200
Subject: Add rake task for easy migration of SQL dumps

---
 lib/tasks/ci/migrate.rake | 70 +++++++++++++++++++++++++++++------------------
 1 file changed, 43 insertions(+), 27 deletions(-)

(limited to 'lib/tasks')

diff --git a/lib/tasks/ci/migrate.rake b/lib/tasks/ci/migrate.rake
index 7d99664dcf3..2760c503e22 100644
--- a/lib/tasks/ci/migrate.rake
+++ b/lib/tasks/ci/migrate.rake
@@ -1,38 +1,54 @@
 namespace :ci do
-  namespace :migrate do
-    def list_objects(type)
-      ids = ActiveRecord::Base.connection.select_all(
-        'select distinct taggable_id from ci_taggings where taggable_type = $1',
-        nil, [[nil, type]]
-      )
-      ids.map { |id| id['taggable_id'] }
+  desc 'GitLab | Import and migrate CI database'
+  task migrate: :environment do
+    unless ENV['force'] == 'yes'
+      puts "This will truncate all CI tables and restore it from provided backup."
+      puts "You will lose any previous CI data stored in the database."
+      ask_to_continue
+      puts ""
     end
 
-    def list_tags(type, id)
-      tags = ActiveRecord::Base.connection.select_all(
-        'select ci_tags.name from ci_tags ' +
-        'join ci_taggings on ci_tags.id = ci_taggings.tag_id ' +
-        'where taggable_type = $1 and taggable_id = $2 and context = $3',
-        nil, [[nil, type], [nil, id], [nil, 'tags']]
-      )
-      tags.map { |tag| tag['name'] }
+    Rake::Task["ci:migrate:db"].invoke
+    Rake::Task["ci:migrate:autoincrements"].invoke
+    Rake::Task["ci:migrate:tags"].invoke
+  end
+
+  namespace :migrate do
+    desc 'GitLab | Import CI database'
+    task db: :environment do
+      if ENV["CI_DUMP"].nil?
+        puts "No CI SQL dump specified:"
+        puts "rake gitlab:backup:restore CI_DUMP=ci_dump.sql"
+        exit 1
+      end
+
+      ci_dump = ENV["CI_DUMP"]
+      unless File.exists?(ci_dump)
+        puts "The specified sql dump doesn't exist!"
+        exit 1
+      end
+
+      ::Ci::Migrate::Database.new.restore(ci_dump)
     end
 
     desc 'GitLab | Migrate CI tags'
     task tags: :environment do
-      list_objects('Runner').each do |id|
-        runner = Ci::Runner.find_by_id(id)
-        if runner
-          tags = list_tags('Runner', id)
-          runner.update_attributes(tag_list: tags)
-        end
-      end
+      ::Ci::Migrate::Tags.new.restore
+    end
 
-      list_objects('Build').each do |id|
-        build = Ci::Build.find_by_id(id)
-        if build
-          tags = list_tags('Build', id)
-          build.update_attributes(tag_list: tags)
+    desc 'GitLab | Migrate CI auto-increments'
+    task autoincrements: :environment do
+      c = ActiveRecord::Base.connection
+      c.tables.select { |t| t.start_with?('ci_') }.each do |table|
+        result = c.select_one("SELECT id FROM #{table} ORDER BY id DESC LIMIT 1")
+        if result
+          ai_val = result['id'].to_i + 1
+          puts "Resetting auto increment ID for #{table} to #{ai_val}"
+          if c.adapter_name == 'PostgreSQL'
+            c.execute("ALTER SEQUENCE #{table}_id_seq RESTART WITH #{ai_val}")
+          else
+            c.execute("ALTER TABLE #{table} AUTO_INCREMENT = #{ai_val}")
+          end
         end
       end
     end
-- 
cgit v1.2.1


From df7d807d5a4e26efc78516459c04be286d6624c9 Mon Sep 17 00:00:00 2001
From: Kamil Trzcinski <ayufan@ayufan.eu>
Date: Wed, 16 Sep 2015 00:43:04 +0200
Subject: Migrate CI services

---
 lib/tasks/ci/migrate.rake | 7 +++++++
 1 file changed, 7 insertions(+)

(limited to 'lib/tasks')

diff --git a/lib/tasks/ci/migrate.rake b/lib/tasks/ci/migrate.rake
index 2760c503e22..2bbcd0f578c 100644
--- a/lib/tasks/ci/migrate.rake
+++ b/lib/tasks/ci/migrate.rake
@@ -11,6 +11,7 @@ namespace :ci do
     Rake::Task["ci:migrate:db"].invoke
     Rake::Task["ci:migrate:autoincrements"].invoke
     Rake::Task["ci:migrate:tags"].invoke
+    Rake::Task["ci:migrate:services"].invoke
   end
 
   namespace :migrate do
@@ -52,5 +53,11 @@ namespace :ci do
         end
       end
     end
+
+    desc 'GitLab | Migrate CI services'
+    task services: :environment do
+      c = ActiveRecord::Base.connection
+      c.execute("UPDATE ci_services SET type=CONCAT('Ci::'', type) WHERE type NOT LIKE 'Ci::%'")
+    end
   end
 end
-- 
cgit v1.2.1


From 0859ba75a873ce78f77587369b9e761a2cc782db Mon Sep 17 00:00:00 2001
From: Kamil Trzcinski <ayufan@ayufan.eu>
Date: Wed, 16 Sep 2015 13:30:03 +0200
Subject: Fix migrate task

---
 lib/tasks/ci/migrate.rake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib/tasks')

diff --git a/lib/tasks/ci/migrate.rake b/lib/tasks/ci/migrate.rake
index 2bbcd0f578c..e7d41874a11 100644
--- a/lib/tasks/ci/migrate.rake
+++ b/lib/tasks/ci/migrate.rake
@@ -57,7 +57,7 @@ namespace :ci do
     desc 'GitLab | Migrate CI services'
     task services: :environment do
       c = ActiveRecord::Base.connection
-      c.execute("UPDATE ci_services SET type=CONCAT('Ci::'', type) WHERE type NOT LIKE 'Ci::%'")
+      c.execute("UPDATE ci_services SET type=CONCAT('Ci::', type) WHERE type NOT LIKE 'Ci::%'")
     end
   end
 end
-- 
cgit v1.2.1