summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMark Doffman <mark.doffman@codethink.co.uk>2013-11-04 15:39:13 +0000
committerMark Doffman <mark.doffman@codethink.co.uk>2013-11-04 15:39:13 +0000
commitcefbbe80f79d75c3881b8047fe3d472cdb68cee3 (patch)
tree9fdf20c50445a179786c8349032c2e1d8d18873d
parentdd3127272239487e29652f109476cfff41f7eaeb (diff)
parent1e190584e7fd18bdfce81c833163862b83758978 (diff)
downloaddefinitions-cefbbe80f79d75c3881b8047fe3d472cdb68cee3.tar.gz
Merge branch 'baserock/markdoffman/ceph-services'
Reviewed-by: Richard Maw <richard.maw@codethink.co.uk>
-rw-r--r--ceph-service-x86_64-generic.morph29
-rw-r--r--ceph-service.morph67
-rw-r--r--ceph.configure216
3 files changed, 312 insertions, 0 deletions
diff --git a/ceph-service-x86_64-generic.morph b/ceph-service-x86_64-generic.morph
new file mode 100644
index 00000000..bdc200cf
--- /dev/null
+++ b/ceph-service-x86_64-generic.morph
@@ -0,0 +1,29 @@
+name: ceph-service-x86_64-generic
+description: The set of strata required to have a minimal system for a 64-bit x86
+ system + the service daemons for ceph object, block and file storage.
+kind: system
+arch: x86_64
+strata:
+- morph: build-essential
+ repo: baserock:baserock/morphs
+ ref: master
+- morph: core
+ repo: baserock:baserock/morphs
+ ref: master
+- morph: foundation
+ repo: baserock:baserock/morphs
+ ref: master
+- morph: bsp-x86_64-generic
+ repo: baserock:baserock/morphs
+ ref: master
+- morph: ceph-service
+ repo: baserock:baserock/morphs
+ ref: master
+configuration-extensions:
+- set-hostname
+- ssh
+- add-config-files
+- simple-network
+- nfsboot
+- install-files
+- ceph
diff --git a/ceph-service.morph b/ceph-service.morph
new file mode 100644
index 00000000..734b6ad7
--- /dev/null
+++ b/ceph-service.morph
@@ -0,0 +1,67 @@
+name: ceph-service
+kind: stratum
+build-depends:
+- morph: core
+ repo: baserock:baserock/morphs
+ ref: master
+chunks:
+- name: sqlite3
+ repo: upstream:sqlite3
+ ref: baserock/morph
+ build-depends: []
+- name: nspr
+ repo: upstream:nspr-hg
+ ref: baserock/morph
+ build-depends: []
+- name: nss
+ repo: upstream:nss
+ ref: baserock/morph
+ build-depends:
+ - nspr
+ - sqlite3
+- name: libaio
+ repo: upstream:libaio
+ ref: baserock/morph
+ build-depends: []
+- name: keyutils
+ repo: upstream:keyutils
+ ref: baserock/morph
+ build-depends: []
+- name: libunwind
+ repo: upstream:libunwind
+ ref: v0.99
+ build-depends: []
+- name: gperftools
+ repo: upstream:gperftools
+ ref: baserock/morph
+ build-depends:
+ - libunwind
+- name: snappy
+ repo: upstream:snappy
+ ref: baserock/morph
+ build-depends: []
+- name: leveldb
+ repo: upstream:leveldb
+ ref: baserock/morph
+ build-depends:
+ - snappy
+ - gperftools
+- name: libeditline
+ repo: upstream:libeditline-tarball
+ ref: baserock/morph
+ build-depends: []
+- name: boost
+ repo: upstream:boost-tarball
+ ref: baserock/markdoffman/morph
+ build-depends: []
+- name: ceph
+ repo: upstream:ceph
+ ref: baserock/morph
+ build-depends:
+ - libaio
+ - gperftools
+ - nss
+ - leveldb
+ - libeditline
+ - keyutils
+ - boost
diff --git a/ceph.configure b/ceph.configure
new file mode 100644
index 00000000..14fa6f6b
--- /dev/null
+++ b/ceph.configure
@@ -0,0 +1,216 @@
+#!/usr/bin/python
+# Copyright (C) 2013 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.5
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import cliapp
+import sys
+import os
+import subprocess
+import shutil
+import re
+
+systemd_monitor_template = """
+[Unit]
+Description=Ceph Monitor firstboot setup
+
+[Service]
+ExecStart=/usr/bin/ceph-mon --cluster {cluster} --mkfs -c {conf} -i {hostname} {keyring}
+ExecStartPost=/bin/rm /etc/systemd/system/multi-user.target.wants/ceph-{cluster}-mon-fboot.service
+
+[Install]
+Wanted-By=multi-user.target
+"""
+systemd_monitor_fname_template = "ceph-{cluster}-mon-fboot.service"
+
+systemd_osd_template = """
+[Unit]
+Description=Ceph osd firstboot setup
+
+[Service]
+ExecStart=/usr/sbin/ceph-disk prepare --cluster {cluster} --fs-type btrfs {data}
+ExecStartPost=/bin/rm /etc/systemd/system/multi-user.target.wants/ceph-{cluster}-osd-{osd_id}-fboot.service
+
+[Install]
+Wanted-By=multi-user.target
+"""
+systemd_osd_fname_template = "ceph-{cluster}-osd-{osd_id}-fboot.service"
+
+class CephConfigurationExtension(cliapp.Application):
+ """
+ Set up ceph server daemons.
+
+ Must include the following environment variables:
+
+ HOSTNAME - Must be defined it is used as the ID for
+ the monitor and metadata daemons.
+ CEPH_CONF - Provide a ceph configuration file.
+
+ Optional environment variables:
+
+ CEPH_CLUSTER - Cluster name, if not provided defaults to 'ceph'.
+
+ CEPH_BOOTSTRAP_OSD - Registered key capable of generating OSD
+ keys.
+ CEPH_BOOTSTRAP_MDS - Registered key capable of generating MDS
+ keys.
+
+ Bootstrap keys are required for creating OSD daemons on servers
+ that do not have a running monitor daemon. They are gathered
+ by 'ceph-deploy gatherkeys' but can be generated and registered
+ separately.
+
+ CEPH_MON - (Blank) Create a ceph monitor daemon on the image.
+ CEPH_MON_KEYRING - Location of monitor keyring. Required by the
+ monitor if using cephx authentication.
+
+
+ CEPH_OSD_X_DATA_DIR - Location of data directory for OSD.
+ Create an OSD daemon on image. 'X' is an integer
+ id, many osd daemons may be run on same server.
+
+ CEPH_MDS - (Blank) Create a metadata server daemon on server.
+ """
+
+ def process_args(self, args):
+
+ if "HOSTNAME" not in os.environ:
+ print "ERROR: Need a hostname defined by 'HOSTNAME'"
+ sys.exit(1)
+ if "CEPH_CLUSTER" not in os.environ:
+ print "ERROR: Need a cluster name defined by 'CEPH_CLUSTER'"
+ sys.exit(1)
+ if "CEPH_CONF" not in os.environ:
+ print "ERROR: Need a ceph conf file defined by 'CEPH_CONF'"
+ sys.exit(1)
+
+ self.dest_dir = args[0]
+
+ self.cluster_name = os.environ["CEPH_CLUSTER"]
+ self.hostname = os.environ["HOSTNAME"]
+
+ self.conf_file = "/etc/ceph/{}.conf".format(self.cluster_name)
+ self.mon_dir = "/var/lib/ceph/mon/"
+ self.osd_dir = "/var/lib/ceph/osd/"
+ self.mds_dir = "/var/lib/ceph/mds/"
+ self.tmp_dir = "/var/lib/ceph/tmp/"
+ self.bootstrap_mds_dir = "/var/lib/ceph/bootstrap-mds/"
+ self.bootstrap_osd_dir = "/var/lib/ceph/bootstrap-osd/"
+ self.systemd_dir = "/etc/systemd/system/"
+ self.systemd_multiuser_dir = "/etc/systemd/system/multi-user.target.wants/"
+
+ self.copy_to_img(os.environ["CEPH_CONF"], self.conf_file)
+
+ # Copy over bootstrap keyrings
+ if "CEPH_BOOTSTRAP_OSD" in os.environ:
+ self.copy_bootstrap_osd(os.environ["CEPH_BOOTSTRAP_OSD"]);
+ if "CEPH_BOOTSTRAP_MDS" in os.environ:
+ self.copy_bootstrap_mds(os.environ["CEPH_BOOTSTRAP_MDS"]);
+
+ # Configure any monitor daemons
+ if "CEPH_MON" in os.environ:
+ self.create_mon_data_dir(os.environ.get("CEPH_MON_KEYRING"))
+
+ # Configure any object storage daemons
+ osd_re = r"CEPH_OSD_(\d+)_DATA_DIR$"
+
+ for env in os.environ.keys():
+ match = re.match(osd_re, env)
+ if match:
+ osd_data_dir_env = match.group(0)
+ osd_id = match.group(1)
+
+ self.create_osd_data_dir(osd_id, os.environ.get(osd_data_dir_env))
+
+ # Configure any mds daemons
+ if "CEPH_MDS" in os.environ:
+ self.create_mds_data_dir()
+
+ def copy_to_img(self, src_file, dest_file):
+ shutil.copy(src_file, self.dest_dir + dest_file)
+
+ def copy_bootstrap_osd(self, src_file):
+ self.copy_to_img(src_file,
+ os.path.join(self.bootstrap_osd_dir, "{}.keyring".format(self.cluster_name)))
+
+ def copy_bootstrap_mds(self, src_file):
+ self.copy_to_img(src_file,
+ os.path.join(self.bootstrap_mds_dir, "{}.keyring".format(self.cluster_name)))
+
+ def symlink_to_multiuser(self, fname):
+ print >> sys.stderr, os.path.join("../", fname)
+ print >> sys.stderr, self.dest_dir + os.path.join(self.systemd_multiuser_dir, fname)
+ os.symlink(os.path.join("../", fname),
+ self.dest_dir + os.path.join(self.systemd_multiuser_dir, fname))
+
+ def create_mon_data_dir(self, src_keyring):
+
+ #Create the monitor data directory
+ mon_data_dir = os.path.join(self.mon_dir, "{}-{}".format(self.cluster_name, self.hostname))
+ os.makedirs(self.dest_dir + mon_data_dir)
+
+ #Create sysvinit file to start via sysvinit
+ sysvinit_file = os.path.join(mon_data_dir, "sysvinit")
+ open(self.dest_dir + sysvinit_file, 'a').close()
+
+ #Create systemd file to initialize the monitor data directory
+ keyring = ""
+ if src_keyring:
+ #Copy the keyring from local to the image
+ dest_keyring = os.path.join(self.tmp_dir,
+ "{}-{}.mon.keyring".format(self.cluster_name, self.hostname))
+ self.copy_to_img(src_keyring, dest_keyring)
+
+ keyring = "--keyring " + dest_keyring
+
+ mon_systemd_fname = systemd_monitor_fname_template.format(cluster=self.cluster_name)
+ mon_systemd = open(self.dest_dir + os.path.join(self.systemd_dir, mon_systemd_fname), "w")
+ mon_systemd.write(systemd_monitor_template.format(cluster=self.cluster_name,
+ conf=self.conf_file,
+ hostname=self.hostname,
+ keyring=keyring))
+ mon_systemd.close()
+
+ #Create a symlink to the multi user target
+ self.symlink_to_multiuser(mon_systemd_fname)
+
+ def create_osd_data_dir(self, osd_id, data_dir):
+ if not data_dir:
+ data_dir = '/srv/osd' + osd_id
+
+ #Create the osd data dir
+ os.makedirs(self.dest_dir + data_dir)
+
+ osd_systemd_fname = systemd_osd_fname_template.format(cluster=self.cluster_name, osd_id=osd_id)
+ osd_systemd = open(self.dest_dir + os.path.join(self.systemd_dir, osd_systemd_fname), "w")
+ osd_systemd.write(systemd_osd_template.format(cluster=self.cluster_name,
+ data=data_dir,
+ osd_id=osd_id))
+ osd_systemd.close()
+
+ #Create a symlink to the multi user target
+ self.symlink_to_multiuser(osd_systemd_fname)
+
+ def create_mds_data_dir(self):
+
+ #Create the monitor data directory
+ mds_data_dir = os.path.join(self.mds_dir, "{}-{}".format(self.cluster_name, self.hostname))
+ os.makedirs(self.dest_dir + mds_data_dir)
+
+ #Create sysvinit file to start via sysvinit
+ sysvinit_file = os.path.join(mds_data_dir, "sysvinit")
+ open(self.dest_dir + sysvinit_file, 'a').close()
+
+
+CephConfigurationExtension().run()