diff options
author | Joshua Hesketh <josh@nitrotech.org> | 2015-07-13 23:49:18 +1000 |
---|---|---|
committer | Joshua Hesketh <josh@nitrotech.org> | 2015-07-13 23:49:18 +1000 |
commit | 648f74b0068793f52dd3296c051ef60cbb5e82ed (patch) | |
tree | 9f325f53ffbdf58e56387e4fa9bbd42c16602a8c | |
parent | e4c3c390d9e93633d40faab9ffa0001fc62435b6 (diff) | |
download | turbo-hipster-648f74b0068793f52dd3296c051ef60cbb5e82ed.tar.gz |
Add tool for updating datasets
Crude copy of migration script for updating datasets for testing.
Change-Id: Ic92809c2b9f963af94ca43872835a7e43a9c6d35
-rw-r--r-- | tools/update_datasets.sh | 278 |
1 files changed, 278 insertions, 0 deletions
diff --git a/tools/update_datasets.sh b/tools/update_datasets.sh new file mode 100644 index 0000000..b635b84 --- /dev/null +++ b/tools/update_datasets.sh @@ -0,0 +1,278 @@ +#!/bin/bash +# +# Copyright 2014 Rackspace Australia +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# A tool to update a given dataset to a given version. Used to keep datasets +# somewhat fresh rather than over-exercising old migrations. + + +# Usage: ./update_datasets.sh VENV_NAME WORKING_DIR GIT_PATH DB_USER DB_PASS DB_NAME SEED_DATA OUTPUT_DATA +# apt-get install git virtualenvwrapper python-pip mysql-server python-lxml build-essential libffi-dev + +# $1 is the unique job id +# $2 is the working dir path +# $3 is the path to the git repo path +# $4 is the nova db user +# $5 is the nova db password +# $6 is the nova db name +# $7 is the path to the seed dataset to test against +# $8 is the logging.conf for openstack +# $9 is the pip cache dir + +UNIQUE_ID=$1 +WORKING_DIR_PATH=`realpath $2` +GIT_REPO_PATH=`realpath $3` +DB_USER=$4 +DB_PASS=$5 +DB_NAME=$6 +DATASET_SEED_SQL=`realpath $7` +DATASET_OUTPUT_SQL=$8 + +SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) + +# We also support the following environment variables to tweak our behavour: +# NOCLEANUP: if set to anything, don't cleanup at the end of the run + +pip_requires() { + pip install -q mysql-python + pip install -q eventlet + requires="tools/pip-requires" + if [ ! -e $requires ] + then + requires="requirements.txt" + fi + echo "Install pip requirements from $requires" + pip install -q -r $requires + echo "Requirements installed" +} + +db_sync() { + # $1 is the test target (ie branch name) + # $2 is an (optional) destination version number + + # Create a nova.conf file + cat - > $WORKING_DIR_PATH/nova-$1.conf <<EOF +[DEFAULT] +sql_connection = mysql://$DB_USER:$DB_PASS@localhost/$DB_NAME?charset=utf8 +#log_config = $LOG_CONF_FILE +EOF + + # Silently return git to a known good state (delete untracked files) + git clean -xfdq + + echo "***** Start DB upgrade to state of $1 *****" + echo "HEAD of branch under test is:" + git log -n 1 + + echo "Setting up the nova-manage entry point" + python setup.py -q clean + python setup.py -q develop + python setup.py -q install + + # Log the migrations present + echo "Migrations present:" + ls $GIT_REPO_PATH/nova/db/sqlalchemy/migrate_repo/versions/*.py | sed 's/.*\///' | egrep "^[0-9]+_" + + # Flush innodb's caches + echo "Restarting mysql" + sudo service mysql stop + sudo service mysql start + + start_version=`mysql -u $DB_USER --password=$DB_PASS $DB_NAME -e "select * from migrate_version \G" | grep version | sed 's/.*: //'` + + if [ "%$2%" == "%%" ] + then + end_version=`ls $GIT_REPO_PATH/nova/db/sqlalchemy/migrate_repo/versions/*.py | sed 's/.*\///' | egrep "^[0-9]+_" | tail -1 | cut -f 1 -d "_"` + else + end_version=$2 + fi + + echo "Test will migrate from $start_version to $end_version" + if [ $end_version -lt $start_version ] + then + increment=-1 + end_version=$(( $end_version + 1 )) + else + increment=1 + start_version=$(( $start_version + 1)) + fi + + for i in `seq $start_version $increment $end_version` + do + set -x + $SCRIPT_DIR/nova-manage-wrapper.sh $VENV_PATH --config-file $WORKING_DIR_PATH/nova-$1.conf --verbose db sync --version $i + manage_exit=$? + set +x + + echo "nova-manage returned exit code $manage_exit" + if [ $manage_exit -gt 0 ] + then + echo "Aborting early" + exit $manage_exit + fi + done + + echo "***** Finished DB upgrade to state of $1 *****" +} + +stable_release_db_sync() { + version=`mysql -u $DB_USER --password=$DB_PASS $DB_NAME -e "select * from migrate_version \G" | grep version | sed 's/.*: //'` + + # Some databases are from Folsom + echo "Schema version is $version" + if [ $version -lt "161" ] + then + echo "Database is from Folsom! Upgrade via Grizzly" + git branch -D eol/grizzly || true + git remote update + git checkout -b eol/grizzly + # Use tag + git reset --hard grizzly-eol + pip_requires + db_sync "grizzly" + fi + + version=`mysql -u $DB_USER --password=$DB_PASS $DB_NAME -e "select * from migrate_version \G" | grep version | sed 's/.*: //'` + # Some databases are from Grizzly + echo "Schema version is $version" + if [ $version -lt "216" ] + then + echo "Database is from Grizzly! Upgrade via Havana" + git branch -D eol/havana || true + git remote update + git checkout -b eol/havana + # Use tag + git reset --hard havana-eol + pip_requires + db_sync "havana" + fi + + version=`mysql -u $DB_USER --password=$DB_PASS $DB_NAME -e "select * from migrate_version \G" | grep version | sed 's/.*: //'` + # Some databases are from Havana + echo "Schema version is $version" + if [ $version -lt "234" ] + then + echo "Database is from Havana! Upgrade via Icehouse" + git branch -D eol/icehouse || true + git remote update + git checkout -b eol/icehouse + # Use tag + git reset --hard icehouse-eol + pip_requires + db_sync "icehouse" + fi + + version=`mysql -u $DB_USER --password=$DB_PASS $DB_NAME -e "select * from migrate_version \G" | grep version | sed 's/.*: //'` + # Some databases are from Icehouse + echo "Schema version is $version" + if [ $version -lt "254" ] + then + echo "Database is from Icehouse! Upgrade via Juno" + git branch -D stable/juno || true + git remote update + git checkout -b stable/juno + git reset --hard remotes/origin/stable/juno + pip_requires + db_sync "juno" + fi + + version=`mysql -u $DB_USER --password=$DB_PASS $DB_NAME -e "select * from migrate_version \G" | grep version | sed 's/.*: //'` + # Some databases are from Juno + echo "Schema version is $version" + if [ $version -lt "280" ] + then + echo "Database is from Juno! Upgrade via Kilo" + git branch -D stable/kilo || true + git remote update + git checkout -b stable/kilo + git reset --hard remotes/origin/stable/kilo + pip_requires + db_sync "kilo" + + # TODO(jhesketh): This is a bit of a hack until we update our datasets to + # have the flavour data migrated. We have to do this before upgrading from + set -x + $SCRIPT_DIR/nova-manage-wrapper.sh $VENV_PATH --config-file $WORKING_DIR_PATH/nova-kilo.conf --verbose db migrate_flavor_data --force + set +x + fi + + # TODO(jhesketh): Add in Liberty here once released + + # TODO(jhesketh): Make this more DRY and/or automatically match migration + # numbers to releases. +} + +echo "Test running on "`hostname`" as "`whoami`" ("`echo ~`", $HOME)" +echo "To execute this script manually, run this:" +echo "$0 $@" + +# Setup the environment +set -x +export PATH=/usr/lib/ccache:$PATH +#export PIP_INDEX_URL="http://www.rcbops.com/pypi/mirror" +export PIP_INDEX_URL="http://pypi.openstack.org/simple/" +export PIP_EXTRA_INDEX_URL="https://pypi.python.org/simple/" +which pip +pip --version +which virtualenv +virtualenv --version +which mkvirtualenv +set +x + +# Restore database to known good state +echo "Restoring test database $DB_NAME" +set -x +mysql -u $DB_USER --password=$DB_PASS -e "drop database $DB_NAME" +mysql -u $DB_USER --password=$DB_PASS -e "create database $DB_NAME" +mysql -u $DB_USER --password=$DB_PASS $DB_NAME < $DATASET_SEED_SQL +set +x + +echo "Build test environment" +cd $GIT_REPO_PATH + +echo "Setting up virtual env" +source ~/.bashrc +export WORKON_HOME=`pwd`/envs +mkdir -p $WORKON_HOME +VENV_PATH=$WORKON_HOME/$UNIQUE_ID +rm -rf $VENV_PATH +source /usr/local/bin/virtualenvwrapper.sh +source /etc/bash_completion.d/virtualenvwrapper +mkvirtualenv --no-site-packages $UNIQUE_ID +#toggleglobalsitepackages +export PYTHONPATH=$PYTHONPATH:$GIT_REPO_PATH + +if [ ! -e $VENV_PATH ] +then + echo "Error: making the virtual env failed" + exit 1 +fi + +stable_release_db_sync + +# Determine the final schema version +version=`mysql -u $DB_USER --password=$DB_PASS $DB_NAME -e "select * from migrate_version \G" | grep version | sed 's/.*: //'` +echo "Final schema version is $version" + +if [ "%$NOCLEANUP%" == "%%" ] +then + # Cleanup virtual env + echo "Cleaning up virtual env" + deactivate + rmvirtualenv $UNIQUE_ID +fi + +cd $SCRIPT_DIR +mysqldump -u $DB_USER --password=$DB_PASS $DB_NAME > $DATASET_OUTPUT_SQL |