summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMark Doffman <mark.doffman@codethink.co.uk>2013-10-24 20:05:53 +0000
committerMark Doffman <mark.doffman@codethink.co.uk>2013-10-31 06:13:31 +0000
commit1e190584e7fd18bdfce81c833163862b83758978 (patch)
tree9fdf20c50445a179786c8349032c2e1d8d18873d
parent5e44e4d31afb21f51adfe97e68f840e64841064f (diff)
downloaddefinitions-1e190584e7fd18bdfce81c833163862b83758978.tar.gz
Add a configuration extension for ceph daemons.
Add a configuration extension that can set up ceph Monitor, Object storage and metadata daemons. This can be used to deploy baserock images ready to add in to an exising ceph storage cluster or to create a new cluster.
-rw-r--r--ceph-service-x86_64-generic.morph1
-rw-r--r--ceph.configure216
2 files changed, 217 insertions, 0 deletions
diff --git a/ceph-service-x86_64-generic.morph b/ceph-service-x86_64-generic.morph
index f1d65b12..bdc200cf 100644
--- a/ceph-service-x86_64-generic.morph
+++ b/ceph-service-x86_64-generic.morph
@@ -26,3 +26,4 @@ configuration-extensions:
- simple-network
- nfsboot
- install-files
+- ceph
diff --git a/ceph.configure b/ceph.configure
new file mode 100644
index 00000000..14fa6f6b
--- /dev/null
+++ b/ceph.configure
@@ -0,0 +1,216 @@
+#!/usr/bin/python
+# Copyright (C) 2013 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.5
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import cliapp
+import sys
+import os
+import subprocess
+import shutil
+import re
+
+systemd_monitor_template = """
+[Unit]
+Description=Ceph Monitor firstboot setup
+
+[Service]
+ExecStart=/usr/bin/ceph-mon --cluster {cluster} --mkfs -c {conf} -i {hostname} {keyring}
+ExecStartPost=/bin/rm /etc/systemd/system/multi-user.target.wants/ceph-{cluster}-mon-fboot.service
+
+[Install]
+Wanted-By=multi-user.target
+"""
+systemd_monitor_fname_template = "ceph-{cluster}-mon-fboot.service"
+
+systemd_osd_template = """
+[Unit]
+Description=Ceph osd firstboot setup
+
+[Service]
+ExecStart=/usr/sbin/ceph-disk prepare --cluster {cluster} --fs-type btrfs {data}
+ExecStartPost=/bin/rm /etc/systemd/system/multi-user.target.wants/ceph-{cluster}-osd-{osd_id}-fboot.service
+
+[Install]
+Wanted-By=multi-user.target
+"""
+systemd_osd_fname_template = "ceph-{cluster}-osd-{osd_id}-fboot.service"
+
+class CephConfigurationExtension(cliapp.Application):
+ """
+ Set up ceph server daemons.
+
+ Must include the following environment variables:
+
+ HOSTNAME - Must be defined it is used as the ID for
+ the monitor and metadata daemons.
+ CEPH_CONF - Provide a ceph configuration file.
+
+ Optional environment variables:
+
+ CEPH_CLUSTER - Cluster name, if not provided defaults to 'ceph'.
+
+ CEPH_BOOTSTRAP_OSD - Registered key capable of generating OSD
+ keys.
+ CEPH_BOOTSTRAP_MDS - Registered key capable of generating MDS
+ keys.
+
+ Bootstrap keys are required for creating OSD daemons on servers
+ that do not have a running monitor daemon. They are gathered
+ by 'ceph-deploy gatherkeys' but can be generated and registered
+ separately.
+
+ CEPH_MON - (Blank) Create a ceph monitor daemon on the image.
+ CEPH_MON_KEYRING - Location of monitor keyring. Required by the
+ monitor if using cephx authentication.
+
+
+ CEPH_OSD_X_DATA_DIR - Location of data directory for OSD.
+ Create an OSD daemon on image. 'X' is an integer
+ id, many osd daemons may be run on same server.
+
+ CEPH_MDS - (Blank) Create a metadata server daemon on server.
+ """
+
+ def process_args(self, args):
+
+ if "HOSTNAME" not in os.environ:
+ print "ERROR: Need a hostname defined by 'HOSTNAME'"
+ sys.exit(1)
+ if "CEPH_CLUSTER" not in os.environ:
+ print "ERROR: Need a cluster name defined by 'CEPH_CLUSTER'"
+ sys.exit(1)
+ if "CEPH_CONF" not in os.environ:
+ print "ERROR: Need a ceph conf file defined by 'CEPH_CONF'"
+ sys.exit(1)
+
+ self.dest_dir = args[0]
+
+ self.cluster_name = os.environ["CEPH_CLUSTER"]
+ self.hostname = os.environ["HOSTNAME"]
+
+ self.conf_file = "/etc/ceph/{}.conf".format(self.cluster_name)
+ self.mon_dir = "/var/lib/ceph/mon/"
+ self.osd_dir = "/var/lib/ceph/osd/"
+ self.mds_dir = "/var/lib/ceph/mds/"
+ self.tmp_dir = "/var/lib/ceph/tmp/"
+ self.bootstrap_mds_dir = "/var/lib/ceph/bootstrap-mds/"
+ self.bootstrap_osd_dir = "/var/lib/ceph/bootstrap-osd/"
+ self.systemd_dir = "/etc/systemd/system/"
+ self.systemd_multiuser_dir = "/etc/systemd/system/multi-user.target.wants/"
+
+ self.copy_to_img(os.environ["CEPH_CONF"], self.conf_file)
+
+ # Copy over bootstrap keyrings
+ if "CEPH_BOOTSTRAP_OSD" in os.environ:
+ self.copy_bootstrap_osd(os.environ["CEPH_BOOTSTRAP_OSD"]);
+ if "CEPH_BOOTSTRAP_MDS" in os.environ:
+ self.copy_bootstrap_mds(os.environ["CEPH_BOOTSTRAP_MDS"]);
+
+ # Configure any monitor daemons
+ if "CEPH_MON" in os.environ:
+ self.create_mon_data_dir(os.environ.get("CEPH_MON_KEYRING"))
+
+ # Configure any object storage daemons
+ osd_re = r"CEPH_OSD_(\d+)_DATA_DIR$"
+
+ for env in os.environ.keys():
+ match = re.match(osd_re, env)
+ if match:
+ osd_data_dir_env = match.group(0)
+ osd_id = match.group(1)
+
+ self.create_osd_data_dir(osd_id, os.environ.get(osd_data_dir_env))
+
+ # Configure any mds daemons
+ if "CEPH_MDS" in os.environ:
+ self.create_mds_data_dir()
+
+ def copy_to_img(self, src_file, dest_file):
+ shutil.copy(src_file, self.dest_dir + dest_file)
+
+ def copy_bootstrap_osd(self, src_file):
+ self.copy_to_img(src_file,
+ os.path.join(self.bootstrap_osd_dir, "{}.keyring".format(self.cluster_name)))
+
+ def copy_bootstrap_mds(self, src_file):
+ self.copy_to_img(src_file,
+ os.path.join(self.bootstrap_mds_dir, "{}.keyring".format(self.cluster_name)))
+
+ def symlink_to_multiuser(self, fname):
+ print >> sys.stderr, os.path.join("../", fname)
+ print >> sys.stderr, self.dest_dir + os.path.join(self.systemd_multiuser_dir, fname)
+ os.symlink(os.path.join("../", fname),
+ self.dest_dir + os.path.join(self.systemd_multiuser_dir, fname))
+
+ def create_mon_data_dir(self, src_keyring):
+
+ #Create the monitor data directory
+ mon_data_dir = os.path.join(self.mon_dir, "{}-{}".format(self.cluster_name, self.hostname))
+ os.makedirs(self.dest_dir + mon_data_dir)
+
+ #Create sysvinit file to start via sysvinit
+ sysvinit_file = os.path.join(mon_data_dir, "sysvinit")
+ open(self.dest_dir + sysvinit_file, 'a').close()
+
+ #Create systemd file to initialize the monitor data directory
+ keyring = ""
+ if src_keyring:
+ #Copy the keyring from local to the image
+ dest_keyring = os.path.join(self.tmp_dir,
+ "{}-{}.mon.keyring".format(self.cluster_name, self.hostname))
+ self.copy_to_img(src_keyring, dest_keyring)
+
+ keyring = "--keyring " + dest_keyring
+
+ mon_systemd_fname = systemd_monitor_fname_template.format(cluster=self.cluster_name)
+ mon_systemd = open(self.dest_dir + os.path.join(self.systemd_dir, mon_systemd_fname), "w")
+ mon_systemd.write(systemd_monitor_template.format(cluster=self.cluster_name,
+ conf=self.conf_file,
+ hostname=self.hostname,
+ keyring=keyring))
+ mon_systemd.close()
+
+ #Create a symlink to the multi user target
+ self.symlink_to_multiuser(mon_systemd_fname)
+
+ def create_osd_data_dir(self, osd_id, data_dir):
+ if not data_dir:
+ data_dir = '/srv/osd' + osd_id
+
+ #Create the osd data dir
+ os.makedirs(self.dest_dir + data_dir)
+
+ osd_systemd_fname = systemd_osd_fname_template.format(cluster=self.cluster_name, osd_id=osd_id)
+ osd_systemd = open(self.dest_dir + os.path.join(self.systemd_dir, osd_systemd_fname), "w")
+ osd_systemd.write(systemd_osd_template.format(cluster=self.cluster_name,
+ data=data_dir,
+ osd_id=osd_id))
+ osd_systemd.close()
+
+ #Create a symlink to the multi user target
+ self.symlink_to_multiuser(osd_systemd_fname)
+
+ def create_mds_data_dir(self):
+
+ #Create the monitor data directory
+ mds_data_dir = os.path.join(self.mds_dir, "{}-{}".format(self.cluster_name, self.hostname))
+ os.makedirs(self.dest_dir + mds_data_dir)
+
+ #Create sysvinit file to start via sysvinit
+ sysvinit_file = os.path.join(mds_data_dir, "sysvinit")
+ open(self.dest_dir + sysvinit_file, 'a').close()
+
+
+CephConfigurationExtension().run()