mirror of
https://github.com/gryf/openstack.git
synced 2025-12-17 11:30:24 +01:00
Open-source the OpenStack DB migration scripts and workshop
This commit is contained in:
PTC
237
migration-scripts/scripts/do_migration.sh
Executable file
237
migration-scripts/scripts/do_migration.sh
Executable file
@@ -0,0 +1,237 @@
|
||||
#!/usr/bin/env sh
|
||||
# Copyright 2018, Oath Inc.
|
||||
# Licensed under the terms of the MIT license. See LICENSE file for terms.
|
||||
|
||||
set -Eexo pipefail
|
||||
|
||||
#########################################################
|
||||
########## RUN THIS SCRIPT IN A SCREEN SESSION ##########
|
||||
#########################################################
|
||||
|
||||
# Requirements:
|
||||
# 1. This script must be run from an openstack db host
|
||||
# 2. Must have root privs on the machine
|
||||
# 3. Must be run alongside a build directory that is generated by clone.sh in
|
||||
# OpenStack/migration-scripts git repo
|
||||
# 4. It is highly recommended to run this script in a screen session.
|
||||
|
||||
# Usage:
|
||||
# ./do_migration.sh <db_host> <db_port> <db_host_user> <db_host_password> <transport_url> <nova_connection_url> [option]
|
||||
#
|
||||
# ./do_migration.sh <RW_HOSTNAME> 3306 root pw "rabbit://ostk_rabbit_user:<RABBIT_PASSWORD>@<MQ1HOST>:<RABBIT_PORT>/ostk_rabbit_vhost" mysql+pymysql://nova:<NOVA_PASSWORD>@<RW_HOSTNAME>:3306 --baremetal
|
||||
#
|
||||
# - <transport_url>:
|
||||
# - if only using 1 MQ node, then the format for transport URL will be the same as above.
|
||||
# - if we are using more than one MQ, then the transport URL should look like this:
|
||||
# "rabbit://ostk_rabbit_user:<RABBIT_PASSWORD>@<MQ1_HOST>:<RABBIT_PORT>,ostk_rabbit_user:<RABBIT_PASSWORD>@<MQ2_HOST>:<RABBIT_PORT>/ostk_rabbit_vhost"
|
||||
# Notice that rabbit:// comes only once before the list of hosts, and /ostk_rabbit_vhost
|
||||
# comes only once at the very end.
|
||||
#
|
||||
# - <RW_HOSTNAME> is the Brooklyn read/write host for the DB.
|
||||
# - <RABBIT_PASSWORD> is from nova.conf.
|
||||
# - <MQ1HOST> is the hostname of the Rabbit MQ 1 node.
|
||||
# - It can be found in the dashboard. Click on clusters on the left. Find the target
|
||||
# - cluster and click on it. Expand "Hosts" on the resulting page and find "Queue Hosts".
|
||||
# - <RABBIT_PORT> is 5671 if you are using SSL, and 5672 if you are not using SSL
|
||||
# - <NOVA_PASSWORD> is the nova user db password.
|
||||
# The keyname is db_nova_password.
|
||||
# - [option]
|
||||
# -b, --baremetal: also do DB migration for ironic
|
||||
# -h, --help: show brief help
|
||||
|
||||
usage() {
|
||||
echo "Usage:"
|
||||
echo "$0 <db_host> <db_port> <db_host_user> <db_host_password> <transport_url> <nova_connection_url> [option]"
|
||||
echo " option:"
|
||||
echo " -h, --help: Display this help message"
|
||||
echo " -b, --baremetal: also do DB migration for ironic"
|
||||
echo "for more information, see comments in the script file"
|
||||
}
|
||||
|
||||
set -e
|
||||
set -x
|
||||
|
||||
baremetal=0
|
||||
|
||||
error() {
|
||||
echo error_msg="FAIL! $1 directory was not found. Run this script https://git.ouroath.com/OpenStack/ocata-migration-scripts/blob/master/clone.sh from a RHEL jumphost to generate the build directory remotely. Then copy the build directory to this machine."
|
||||
exit 1
|
||||
}
|
||||
|
||||
PARAMS=""
|
||||
while (( "$#" )); do
|
||||
case "$1" in
|
||||
-b|--baremetal)
|
||||
baremetal=1
|
||||
shift
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
-*|--*=) # unsupported flags
|
||||
echo "Error: Unsupported flag '$1'" >&2
|
||||
exit 1
|
||||
;;
|
||||
*) # preserve positional arguments
|
||||
PARAMS="$PARAMS $1"
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# set positional arguments in their proper place
|
||||
eval set -- "$PARAMS"
|
||||
|
||||
if [ $# -ne 6 ] ; then
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
# Make sure git is installed
|
||||
sudo yum install -y git mysql
|
||||
|
||||
db_host="$1"
|
||||
db_port="$2"
|
||||
db_user="$3"
|
||||
db_pass="$4"
|
||||
transport_url="$5"
|
||||
nova_db_url="$6"
|
||||
|
||||
# Each release to migrate through (Juno, Ocata]
|
||||
releases="kilo liberty mitaka newton ocata"
|
||||
# Which components to migrate
|
||||
if [ "$baremetal" == 1 ]; then
|
||||
components="keystone nova glance neutron ironic"
|
||||
else
|
||||
components="keystone nova glance neutron"
|
||||
fi
|
||||
|
||||
# Let's make sure we have everything that's required installed
|
||||
sudo yum install -y python python-pip
|
||||
sudo pip install virtualenv
|
||||
|
||||
sudo yum install -y git libxml2-devel libxslt-devel libffi-devel openssl-devel libvirt-devel
|
||||
|
||||
# Back up the db first :-)
|
||||
sql_dump_file="mysqldump.$db_host.`date +"%s"`"
|
||||
mysqldump -u $db_user --password=$db_pass -h $db_host --all-databases --result-file=$sql_dump_file
|
||||
|
||||
# Some necessary preparations to the DB
|
||||
mysql -u $db_user --password=$db_pass -h $db_host -e "CREATE DATABASE IF NOT EXISTS nova_api;"
|
||||
mysql -u $db_user --password=$db_pass -h $db_host -e "USE mysql ; GRANT ALL ON nova_api.* TO 'nova'@'%' with GRANT option; FLUSH PRIVILEGES;"
|
||||
mysql -u $db_user --password=$db_pass -h $db_host -e "CREATE DATABASE IF NOT EXISTS nova_cell0;"
|
||||
mysql -u $db_user --password=$db_pass -h $db_host -e "USE mysql ; GRANT ALL ON nova_cell0.* TO 'nova'@'%' with GRANT option; FLUSH PRIVILEGES;"
|
||||
#mysql -u $db_user --password=$db_pass -h $db_host -e "SET GLOBAL FOREIGN_KEY_CHECKS=0;"
|
||||
mysql -u $db_user --password=$db_pass -h $db_host -e 'USE glance ; CREATE INDEX ix_images_is_public ON images (is_public);' || true
|
||||
mysql -u $db_user --password=$db_pass -h $db_host -e 'USE glance ; ALTER TABLE image_properties DROP FOREIGN KEY image_properties_ibfk_1;' || true
|
||||
|
||||
# Front load some checks to make sure that the necessary directories are in place
|
||||
if [ ! -e build ]; then
|
||||
error "build"
|
||||
fi
|
||||
cd build
|
||||
|
||||
for comp in $components ; do
|
||||
for release in $releases ; do
|
||||
if [ $comp == 'ironic' ] && [ $release != 'ocata' ] ; then
|
||||
continue
|
||||
else
|
||||
dir_name="${comp}-$release"
|
||||
if [ ! -e $dir_name ]; then
|
||||
error ${dir_name}
|
||||
fi
|
||||
venv_name=venv-$dir_name
|
||||
if [ ! -e $venv_name ]; then
|
||||
error ${venv_name}
|
||||
fi
|
||||
fi
|
||||
done
|
||||
done
|
||||
|
||||
# use this script to clean up deleted instances
|
||||
#if [ ! -e cleaner-venv ]; then
|
||||
# /opt/python/bin/virtualenv -p /opt/python/bin/python2.7 cleaner-venv
|
||||
#fi
|
||||
#source cleaner-venv/bin/activate
|
||||
#pip install mysql-connector-python-rf
|
||||
#echo "Removing deleted instances...."
|
||||
#../remove_deleted_instances.py $db_host $db_user $db_pass
|
||||
#echo "Done."
|
||||
#deactivate
|
||||
|
||||
for comp in $components ; do
|
||||
echo "Migrating $comp";
|
||||
# Component will use this config file to talk to the db
|
||||
db_conf_file=$comp-db.conf
|
||||
rm -f $db_conf_file
|
||||
echo "[database]" >> $db_conf_file
|
||||
if [ $comp == 'nova' ]; then
|
||||
echo "connection = $nova_db_url/nova" >> $db_conf_file
|
||||
else
|
||||
echo "connection = mysql+pymysql://$db_user:$db_pass@$db_host:$db_port/$comp" >> $db_conf_file
|
||||
fi
|
||||
echo "[api_database]" >> $db_conf_file
|
||||
echo "connection = $nova_db_url/nova_api" >> $db_conf_file
|
||||
|
||||
echo "[DEFAULT]" >> $db_conf_file
|
||||
echo "transport_url = $transport_url" >> $db_conf_file
|
||||
echo "log_dir = /tmp/logs/$comp" >> $db_conf_file
|
||||
echo "debug = true" >> $db_conf_file
|
||||
mkdir -p "/tmp/logs/$comp"
|
||||
|
||||
for release in $releases ; do
|
||||
if [ $comp == 'ironic' ] && [ $release != 'ocata' ] ; then
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "Migrating to $release";
|
||||
# A mapping from component -> migration command
|
||||
declare -A migration_command=(["keystone"]="keystone-manage --config-file $db_conf_file db_sync" \
|
||||
["glance"]="glance-manage --config-file $db_conf_file db_sync" \
|
||||
["nova"]="nova-manage --config-file $db_conf_file api_db sync ; nova-manage --config-file $db_conf_file db sync" \
|
||||
["ironic"]="ironic-dbsync --config-file $db_conf_file upgrade" \
|
||||
["neutron"]="neutron-db-manage --config-file $db_conf_file upgrade heads" \
|
||||
["horizon"]=":" )
|
||||
|
||||
dir_name="${comp}-$release"
|
||||
venv_name=venv-$dir_name
|
||||
source $venv_name/bin/activate
|
||||
# Create cells for ocata
|
||||
if [ $release == 'newton' ] && [ $comp == 'nova' ] ; then
|
||||
nova-manage --verbose --config-file $db_conf_file cell_v2 simple_cell_setup
|
||||
fi
|
||||
|
||||
# One time clean-up of null UUIDs
|
||||
if [ $release == 'kilo' ] && [ $comp == 'nova' ] ; then
|
||||
nova-manage --config-file $db_conf_file db null_instance_uuid_scan --delete
|
||||
fi
|
||||
|
||||
# Run migration command in the venv and then exit the venv
|
||||
eval ${migration_command[$comp]}
|
||||
|
||||
# Online data migrations
|
||||
if [ $release != 'kilo' ] && [ $release != 'liberty' ] && [ $comp == 'nova' ] ; then
|
||||
nova-manage --config-file $db_conf_file db online_data_migrations
|
||||
fi
|
||||
|
||||
# The great Kilo flavor migration of 1934
|
||||
if [ $release == 'kilo' ] && [ $comp == 'nova' ] ; then
|
||||
nova-manage --config-file $db_conf_file db migrate_flavor_data --force
|
||||
mysql -u $db_user --password=$db_pass -h $db_host -e 'USE nova ; UPDATE compute_nodes SET host=hypervisor_hostname;'
|
||||
fi
|
||||
|
||||
deactivate
|
||||
done
|
||||
done
|
||||
|
||||
# Remove old endpoints (chef deploy will re-populate these with correct values)
|
||||
mysql -u $db_user --password=$db_pass -h $db_host -e 'USE keystone; DELETE FROM endpoint;'
|
||||
mysql -u $db_user --password=$db_pass -h $db_host -e 'USE keystone; DELETE FROM service;'
|
||||
# Remove invalid role assignments (missing user)
|
||||
|
||||
# Change legacy "config_drive" key to what is used upstream "configdrive"
|
||||
# NOTE(jaypipes): This may take a multiple dozens of seconds on a table with
|
||||
# tens of thousands of records
|
||||
if [ "$baremetal" == 1 ]; then
|
||||
mysql -u $db_user --password=$db_pass -h $db_host -e 'USE ironic; UPDATE nodes SET instance_info = REPLACE(instance_info, "\"config_drive\":", "\"configdrive\":");'
|
||||
fi
|
||||
Reference in New Issue
Block a user