baserock_database: Update on top of Fedora 23, redeploy

Change-Id: I9a25d9aad540c291aaea45f00e38065981ff3f50

5 files changed, 33 insertions, 29 deletions

diff --git a/README.mdwn b/README.mdwn
index b47a54d3..3c0dce1b 100644
--- a/README.mdwn
+++ b/README.mdwn
@@ -185,7 +185,7 @@ To deploy this system to production:
         --key-name=$keyname \
         --flavor dc1.1x1 \
         --image=$fedora_image_id \
-        --nic="net-id=$network_id,v4-fixed-ip=192.168.222.30" \
+        --nic="net-id=$network_id,v4-fixed-ip=192.168.222.146" \
         --security-groups default,database-mysql \
         --user-data ./baserock-ops-team.cloud-config
     nova volume-create \
@@ -197,6 +197,7 @@ To deploy this system to production:
 
     ansible-playbook -i hosts baserock_database/image-config.yml
     ansible-playbook -i hosts baserock_database/instance-config.yml
+    ansible-playbook -i hosts baserock_database/instance-backup-config.yml
 
 At this point, if you are restoring from a backup, rsync the data across
 from your backup server on the instance, then start the mariadb service and you
@@ -225,6 +226,11 @@ you want to check the database configuration matches what you expect.
     ansible -i hosts -m service -a 'name=mariadb enabled=true state=started'
     ansible-playbook -i hosts baserock_database/instance-mariadb-config.yml
 
+The internal IP address of this machine is hardcoded in some places (beyond the
+usual haproxy.cfg file), use 'git grep' to find all of them. You'll need to
+update all the relevant config files. We really need some internal DNS system
+to avoid this hassle.
+
 [MariaDB]: https://www.mariadb.org
 
 ### Mail relay
diff --git a/baserock_backup/backup.sh b/baserock_backup/backup.sh
index e971aaa6..a65b1445 100755
--- a/baserock_backup/backup.sh
+++ b/baserock_backup/backup.sh
@@ -13,7 +13,7 @@
 # Database
 /usr/bin/rsync --archive --delete-before --delete-excluded \
     --hard-links --human-readable --progress --sparse \
-    root@192.168.222.30: /srv/backup/database \
+    root@192.168.222.146: /srv/backup/database \
   && date > /srv/backup/database.timestamp
 
 # Gerrit
@@ -23,14 +23,5 @@
     root@192.168.222.69: /srv/backup/gerrit \
   && date > /srv/backup/gerrit.timestamp
 
-# Storyboard Database
-/usr/bin/rsync --archive --delete-before --delete-excluded \
-    --hard-links --human-readable --progress --sparse \
-    root@192.168.222.30: /srv/backup/storyboard-database \
-  && date > /srv/backup/storyboard-database.timestamp
-
-# Storyboard Database
-/usr/bin/rsync --archive --delete-before --delete-excluded \
-    --hard-links --human-readable --progress --sparse \
-    root@192.168.222.30: /srv/backup/storyboard-database
-date > /srv/backup/storyboard-database.timestamp
+# FIXME: Storyboard database is not currently backed up, see:
+# <https://storyboard.baserock.org/#!/story/74>.
diff --git a/baserock_database/image-config.yml b/baserock_database/image-config.yml
index 32795e28..7b89e700 100644
--- a/baserock_database/image-config.yml
+++ b/baserock_database/image-config.yml
@@ -1,28 +1,43 @@
 # System configuration for Baserock database server.
 #
-# This Ansible playbook expects to be run on a Fedora 21 Cloud image.
+# This Ansible playbook expects to be run on a Fedora 23 Cloud image.
 ---
 - hosts: database-mariadb
   gather_facts: False
   sudo: True
   tasks:
+  # See: https://fedoramagazine.org/getting-ansible-working-fedora-23/
+  - name: install Python2 and required deps for Ansible modules
+    raw: dnf install -y python2 python2-dnf libselinux-python
+
+  - name: ensure system up to date
+    dnf: name=* state=latest
+
   - name: enable persistant journal
     shell: mkdir /var/log/journal
     args:
       creates: /var/log/journal
 
   - name: install lvm2 tools
-    yum: name=lvm2 state=latest
+    dnf: name=lvm2 state=latest
 
   - name: install MariaDB
-    yum: name={{ item }} state=latest
+    dnf: name={{ item }} state=latest
     with_items:
         - mariadb
         - mariadb-server
         - MySQL-python
 
-  - name: install libselinux-python, so Ansible can control selinux
-    yum: name=libselinux-python state=latest
+  # By default this is set to /var/lib/mysql, but this causes a hidden
+  # directory to be created in /var/lib/mysql (.local/share/systemd) which
+  # breaks MariaDB because it expects each directory in there to represent a
+  # database, and you see this when upgrading:
+  #
+  #   Phase 2/6: Fixing views
+  #   mysqlcheck: Got error: 1102: Incorrect database name '#mysql50#.local' when selecting the database
+  #
+  - name: fix home directory of MySQL user
+    user: name=mysql home=/
 
   - name: disable SELinux on subsequent boots
     selinux: state=disabled
diff --git a/baserock_database/instance-config.yml b/baserock_database/instance-config.yml
index 91a29d1b..b3f6a8c6 100644
--- a/baserock_database/instance-config.yml
+++ b/baserock_database/instance-config.yml
@@ -6,18 +6,10 @@
   gather_facts: False
   sudo: yes
   tasks:
-    - name: ensure system up to date
-      yum: name=* state=latest
-
-    # FIXME: the create-data-volume.yml role should handle this... the gotcha
-    # is that this won't work in Baserock systems right now. Once there's an
-    # lvm2-lvmetad.service in Baserock we can move this entry to
-    # create-data-volume.yml.
-    - name: ensure LVM metadata service is running
-      service: name=lvm2-lvmetad enabled=yes state=started
-
     - include: ../tasks/create-data-volume.yml lv_name=database lv_size=25g mountpoint=/var/lib/mysql
 
     - name: ensure mysql user owns /var/lib/mysql
       file: path=/var/lib/mysql owner=mysql group=mysql mode=600 state=directory
 
+    - name: start MariaDB service
+      service: name=mariadb state=started
diff --git a/baserock_hosts b/baserock_hosts
index f72c964b..5d03d255 100644
--- a/baserock_hosts
+++ b/baserock_hosts
@@ -18,7 +18,7 @@ mason-x86-64 ansible_ssh_host=192.168.222.80
 
 [fedora]
 frontend-haproxy ansible_ssh_host=185.43.218.170
-database-mariadb ansible_ssh_host=192.168.222.30
+database-mariadb ansible_ssh_host=192.168.222.146
 mail ansible_ssh_host=192.168.222.145
 openid ansible_ssh_host=192.168.222.144
 webserver ansible_ssh_host=192.168.222.127