path: root/devstack/lib/ironic

#!/bin/bash
#
# lib/ironic
# Functions to control the configuration and operation of the **Ironic** service

# Dependencies:
#
# - ``functions`` file
# - ``DEST``, ``DATA_DIR``, ``STACK_USER`` must be defined
# - ``SERVICE_{TENANT_NAME|PASSWORD}`` must be defined
# - ``SERVICE_HOST``
# - ``KEYSTONE_TOKEN_FORMAT`` must be defined

# ``stack.sh`` calls the entry points in this order:
#
# - install_ironic
# - install_ironicclient
# - init_ironic
# - start_ironic
# - stop_ironic
# - cleanup_ironic

# Save trace and pipefail settings
_XTRACE_IRONIC=$(set +o | grep xtrace)
_PIPEFAIL_IRONIC=$(set +o | grep pipefail)
set +o xtrace
set +o pipefail

# Defaults
# --------

# Set up default directories
GITDIR["python-ironicclient"]=$DEST/python-ironicclient
GITDIR["ironic-lib"]=$DEST/ironic-lib

GITREPO["pyghmi"]=${PYGHMI_REPO:-${GIT_BASE}/openstack/pyghmi.git}
GITBRANCH["pyghmi"]=${PYGHMI_BRANCH:-master}
GITDIR["pyghmi"]=$DEST/pyghmi

GITREPO["virtualbmc"]=${VIRTUALBMC_REPO:-${GIT_BASE}/openstack/virtualbmc.git}
GITBRANCH["virtualbmc"]=${VIRTUALBMC_BRANCH:-master}
GITDIR["virtualbmc"]=$DEST/virtualbmc

IRONIC_DIR=$DEST/ironic
IRONIC_DEVSTACK_DIR=$IRONIC_DIR/devstack
IRONIC_DEVSTACK_FILES_DIR=$IRONIC_DEVSTACK_DIR/files
IRONIC_PYTHON_AGENT_DIR=$DEST/ironic-python-agent
IRONIC_DATA_DIR=$DATA_DIR/ironic
IRONIC_STATE_PATH=/var/lib/ironic
IRONIC_AUTH_CACHE_DIR=${IRONIC_AUTH_CACHE_DIR:-/var/cache/ironic}
IRONIC_CONF_DIR=${IRONIC_CONF_DIR:-/etc/ironic}
IRONIC_CONF_FILE=$IRONIC_CONF_DIR/ironic.conf
IRONIC_ROOTWRAP_CONF=$IRONIC_CONF_DIR/rootwrap.conf
IRONIC_POLICY_JSON=$IRONIC_CONF_DIR/policy.json

# Deploy callback timeout can be changed from its default (1800), if required.
IRONIC_CALLBACK_TIMEOUT=${IRONIC_CALLBACK_TIMEOUT:-}

# Deploy to hardware platform
IRONIC_HW_NODE_CPU=${IRONIC_HW_NODE_CPU:-1}
IRONIC_HW_NODE_RAM=${IRONIC_HW_NODE_RAM:-512}
IRONIC_HW_NODE_DISK=${IRONIC_HW_NODE_DISK:-10}
IRONIC_HW_EPHEMERAL_DISK=${IRONIC_HW_EPHEMERAL_DISK:-0}
IRONIC_HW_ARCH=${IRONIC_HW_ARCH:-x86_64}

# The file is composed of multiple lines, each line includes fields
# separated by white space, in the format:
#
# <BMC address> <MAC address> <BMC username> <BMC password> [<driver specfic fields>]
#
# For example:
#
# 192.168.110.107 00:1e:67:57:50:4c root otc123
#
# Supported IRONIC_DEPLOY_DRIVERs:
# *_ipmitool:
#   <BMC address> <MAC address> <BMC username> <BMC password>
#
# *_cimc:
#   <BMC address> <MAC address> <BMC username> <BMC password>
#
# *_ucs:
#   <BMC address> <MAC address> <BMC username> <BMC password> <UCS service profile>
#
# *_oneview:
#   <Server Hardware URI> <Server Hardware Type URI> <Enclosure Group URI> <Server Profile Template URI> <MAC of primary connection>
#
# IRONIC_IPMIINFO_FILE is deprecated, please use IRONIC_HWINFO_FILE. IRONIC_IPMIINFO_FILE will be removed in Ocata.
IRONIC_IPMIINFO_FILE=${IRONIC_IPMIINFO_FILE:-""}
if [ ! -z "$IRONIC_IPMIINFO_FILE" ]; then
    deprecated "IRONIC_IPMIINFO_FILE is deprecated, please switch to IRONIC_HWINFO_FILE. IRONIC_IPMIINFO_FILE will be removed in Ocata."
fi
IRONIC_HWINFO_FILE=${IRONIC_HWINFO_FILE:-${IRONIC_IPMIINFO_FILE:-$IRONIC_DATA_DIR/hardware_info}}

# Set up defaults for functional / integration testing
IRONIC_NODE_UUID=${IRONIC_NODE_UUID:-`uuidgen`}
IRONIC_SCRIPTS_DIR=${IRONIC_SCRIPTS_DIR:-$IRONIC_DEVSTACK_DIR/tools/ironic/scripts}
IRONIC_TEMPLATES_DIR=${IRONIC_TEMPLATES_DIR:-$IRONIC_DEVSTACK_DIR/tools/ironic/templates}
IRONIC_BAREMETAL_BASIC_OPS=$(trueorfalse False IRONIC_BAREMETAL_BASIC_OPS)
IRONIC_ENABLED_DRIVERS=${IRONIC_ENABLED_DRIVERS:-fake,pxe_ssh,pxe_ipmitool}
IRONIC_SSH_USERNAME=${IRONIC_SSH_USERNAME:-`whoami`}
IRONIC_SSH_TIMEOUT=${IRONIC_SSH_TIMEOUT:-15}
IRONIC_SSH_ATTEMPTS=${IRONIC_SSH_ATTEMPTS:-5}
IRONIC_SSH_KEY_DIR=${IRONIC_SSH_KEY_DIR:-$IRONIC_DATA_DIR/ssh_keys}
IRONIC_SSH_KEY_FILENAME=${IRONIC_SSH_KEY_FILENAME:-ironic_key}
IRONIC_KEY_FILE=${IRONIC_KEY_FILE:-$IRONIC_SSH_KEY_DIR/$IRONIC_SSH_KEY_FILENAME}
IRONIC_SSH_VIRT_TYPE=${IRONIC_SSH_VIRT_TYPE:-virsh}
IRONIC_TFTPBOOT_DIR=${IRONIC_TFTPBOOT_DIR:-$IRONIC_DATA_DIR/tftpboot}
IRONIC_TFTPSERVER_IP=${IRONIC_TFTPSERVER_IP:-$HOST_IP}
IRONIC_VM_SSH_PORT=${IRONIC_VM_SSH_PORT:-22}
IRONIC_VM_SSH_ADDRESS=${IRONIC_VM_SSH_ADDRESS:-$HOST_IP}
IRONIC_VM_COUNT=${IRONIC_VM_COUNT:-1}
IRONIC_VM_SPECS_CPU=${IRONIC_VM_SPECS_CPU:-1}
IRONIC_VM_SPECS_RAM=${IRONIC_VM_SPECS_RAM:-1280}
IRONIC_VM_SPECS_CPU_ARCH=${IRONIC_VM_SPECS_CPU_ARCH:-'x86_64'}
IRONIC_VM_SPECS_DISK=${IRONIC_VM_SPECS_DISK:-10}
IRONIC_VM_SPECS_DISK_FORMAT=${IRONIC_VM_SPECS_DISK_FORMAT:-qcow2}
IRONIC_VM_EPHEMERAL_DISK=${IRONIC_VM_EPHEMERAL_DISK:-0}
IRONIC_VM_EMULATOR=${IRONIC_VM_EMULATOR:-/usr/bin/qemu-system-x86_64}
IRONIC_VM_NETWORK_BRIDGE=${IRONIC_VM_NETWORK_BRIDGE:-brbm}
IRONIC_VM_NETWORK_RANGE=${IRONIC_VM_NETWORK_RANGE:-192.0.2.0/24}
IRONIC_VM_MACS_CSV_FILE=${IRONIC_VM_MACS_CSV_FILE:-$IRONIC_DATA_DIR/ironic_macs.csv}
IRONIC_AUTHORIZED_KEYS_FILE=${IRONIC_AUTHORIZED_KEYS_FILE:-$HOME/.ssh/authorized_keys}
IRONIC_CLEAN_NET_NAME=${IRONIC_CLEAN_NET_NAME:-$PRIVATE_NETWORK_NAME}
IRONIC_EXTRA_PXE_PARAMS=${IRONIC_EXTRA_PXE_PARAMS:-}
IRONIC_TTY_DEV=${IRONIC_TTY_DEV:-ttyS0}

# By default, baremetal VMs will console output to file.
IRONIC_VM_LOG_CONSOLE=$(trueorfalse True IRONIC_VM_LOG_CONSOLE)
IRONIC_VM_LOG_DIR=${IRONIC_VM_LOG_DIR:-$IRONIC_DATA_DIR/logs/}
IRONIC_VM_LOG_ROTATE=$(trueorfalse True IRONIC_VM_LOG_ROTATE)

# Whether to build the ramdisk or download a prebuilt one.
IRONIC_BUILD_DEPLOY_RAMDISK=$(trueorfalse True IRONIC_BUILD_DEPLOY_RAMDISK)

# Ironic IPA ramdisk type, supported types are:
IRONIC_SUPPORTED_RAMDISK_TYPES_RE="^(coreos|tinyipa|dib)$"
IRONIC_RAMDISK_TYPE=${IRONIC_RAMDISK_TYPE:-tinyipa}

# Confirm we have a supported ramdisk type or fail early.
if [[ ! "$IRONIC_RAMDISK_TYPE" =~ $IRONIC_SUPPORTED_RAMDISK_TYPES_RE ]]; then
    die $LINENO "Unrecognized IRONIC_RAMDISK_TYPE: $IRONIC_RAMDISK_TYPE. Expected 'coreos', 'tinyipa' or 'dib'"
fi

# If present, these files are used as deploy ramdisk/kernel.
# (The value must be an absolute path)
IRONIC_DEPLOY_RAMDISK=${IRONIC_DEPLOY_RAMDISK:-}
IRONIC_DEPLOY_KERNEL=${IRONIC_DEPLOY_KERNEL:-}
IRONIC_DEPLOY_ISO=${IRONIC_DEPLOY_ISO:-}

# NOTE(jroll) this needs to be updated when stable branches are cut
IPA_DOWNLOAD_BRANCH=${IPA_DOWNLOAD_BRANCH:-master}
IPA_DOWNLOAD_BRANCH=$(echo $IPA_DOWNLOAD_BRANCH | tr / -)

# Configure URLs required to download ramdisk if we're not building it, and
# IRONIC_DEPLOY_RAMDISK/KERNEL or the RAMDISK/KERNEL_URLs have not been
# preconfigured.
if [[ "$IRONIC_BUILD_DEPLOY_RAMDISK" == "False" && \
        ! (-e "$IRONIC_DEPLOY_RAMDISK" && -e "$IRONIC_DEPLOY_KERNEL") && \
        (-z "$IRONIC_AGENT_KERNEL_URL" || -z "$IRONIC_AGENT_RAMDISK_URL") ]]; then
    case $IRONIC_RAMDISK_TYPE in
        coreos)
            IRONIC_AGENT_KERNEL_URL=${IRONIC_AGENT_KERNEL_URL:-https://tarballs.openstack.org/ironic-python-agent/coreos/files/coreos_production_pxe-${IPA_DOWNLOAD_BRANCH}.vmlinuz}
            IRONIC_AGENT_RAMDISK_URL=${IRONIC_AGENT_RAMDISK_URL:-https://tarballs.openstack.org/ironic-python-agent/coreos/files/coreos_production_pxe_image-oem-${IPA_DOWNLOAD_BRANCH}.cpio.gz}
        ;;
        tinyipa)
            IRONIC_AGENT_KERNEL_URL=${IRONIC_AGENT_KERNEL_URL:-https://tarballs.openstack.org/ironic-python-agent/tinyipa/files/tinyipa-${IPA_DOWNLOAD_BRANCH}.vmlinuz}
            IRONIC_AGENT_RAMDISK_URL=${IRONIC_AGENT_RAMDISK_URL:-https://tarballs.openstack.org/ironic-python-agent/tinyipa/files/tinyipa-${IPA_DOWNLOAD_BRANCH}.gz}
        ;;
        dib)
            die "IRONIC_RAMDISK_TYPE 'dib' has no official pre-built "\
                "images. To fix this select a different ramdisk type, set "\
                "IRONIC_BUILD_DEPLOY_RAMDISK=True, or manually configure "\
                "IRONIC_DEPLOY_RAMDISK(_URL) and IRONIC_DEPLOY_KERNEL(_URL) "\
                "to use your own pre-built ramdisk."
        ;;
    esac
fi

# This refers the options for disk-image-create and the platform on which
# to build the dib based ironic-python-agent ramdisk.
# "ubuntu" is set as the default value.
IRONIC_DIB_RAMDISK_OPTIONS=${IRONIC_DIB_RAMDISK_OPTIONS:-'ubuntu'}

# Some drivers in Ironic require deploy ramdisk in bootable ISO format.
# Set this variable to "true" to build an ISO for deploy ramdisk and
# upload to Glance.
IRONIC_DEPLOY_ISO_REQUIRED=$(trueorfalse False IRONIC_DEPLOY_ISO_REQUIRED)
if $IRONIC_DEPLOY_ISO_REQUIRED = 'True' && $IRONIC_BUILD_DEPLOY_RAMDISK = 'False'\
    && [ -n $IRONIC_DEPLOY_ISO ];  then
    die "Prebuilt ISOs are not available, provide an ISO via IRONIC_DEPLOY_ISO \
        or set IRONIC_BUILD_DEPLOY_RAMDISK=True to use ISOs"
fi
# Which deploy driver to use - valid choices right now
# are ``pxe_ssh``, ``pxe_ipmitool``, ``agent_ssh`` and ``agent_ipmitool``.
#
# Additional valid choices if IRONIC_IS_HARDWARE == true are:
# ``pxe_iscsi_cimc``, ``pxe_agent_cimc``, ``pxe_ucs``, ``pxe_cimc`` and ``*_pxe_oneview``
IRONIC_DEPLOY_DRIVER=${IRONIC_DEPLOY_DRIVER:-pxe_ssh}

# Support entry points installation of console scripts
IRONIC_BIN_DIR=$(get_python_exec_prefix)

# Ironic connection info.  Note the port must be specified.
IRONIC_SERVICE_PROTOCOL=${IRONIC_SERVICE_PROTOCOL:-$SERVICE_PROTOCOL}
IRONIC_SERVICE_PORT=${IRONIC_SERVICE_PORT:-6385}
IRONIC_HOSTPORT=${IRONIC_HOSTPORT:-$SERVICE_HOST:$IRONIC_SERVICE_PORT}

# Enable iPXE
IRONIC_IPXE_ENABLED=$(trueorfalse True IRONIC_IPXE_ENABLED)
IRONIC_HTTP_DIR=${IRONIC_HTTP_DIR:-$IRONIC_DATA_DIR/httpboot}
IRONIC_HTTP_SERVER=${IRONIC_HTTP_SERVER:-$IRONIC_TFTPSERVER_IP}
IRONIC_HTTP_PORT=${IRONIC_HTTP_PORT:-3928}

# Whether DevStack will be setup for bare metal or VMs
IRONIC_IS_HARDWARE=$(trueorfalse False IRONIC_IS_HARDWARE)

# The first port in the range to bind the Virtual BMCs. The number of
# ports that will be used depends on $IRONIC_VM_COUNT variable, e.g if
# $IRONIC_VM_COUNT=3 the ports 6230, 6231 and 6232 will be used for the
# Virtual BMCs, one for each VM.
IRONIC_VBMC_PORT_RANGE_START=${IRONIC_VBMC_PORT_RANGE_START:-6230}
IRONIC_VBMC_CONFIG_FILE=${IRONIC_VBMC_CONFIG_FILE:-$HOME/.vbmc/virtualbmc.conf}
IRONIC_VBMC_LOGFILE=${IRONIC_VBMC_LOGFILE:-$IRONIC_VM_LOG_DIR/virtualbmc.log}

# To explicitly enable configuration of Glance with Swift
# (which is required by some vendor drivers), set this
# variable to true.
IRONIC_CONFIGURE_GLANCE_WITH_SWIFT=$(trueorfalse False IRONIC_CONFIGURE_GLANCE_WITH_SWIFT)

# The path to the libvirt hooks directory, used if IRONIC_VM_LOG_ROTATE is True
IRONIC_LIBVIRT_HOOKS_PATH=${IRONIC_LIBVIRT_HOOKS_PATH:-/etc/libvirt/hooks/}

# The authentication strategy used by ironic-api. Valid values are:
# keystone and noauth.
IRONIC_AUTH_STRATEGY=${IRONIC_AUTH_STRATEGY:-keystone}

# By default, terminal SSL certificate is disabled.
IRONIC_TERMINAL_SSL=$(trueorfalse False IRONIC_TERMINAL_SSL)
IRONIC_TERMINAL_CERT_DIR=${IRONIC_TERMINAL_CERT_DIR:-$IRONIC_DATA_DIR/terminal_cert/}

# This flag is used to allow adding Link-Local-Connection info
# to ironic port-create command. LLC info is obtained from
# IRONIC_{VM,HW}_NODES_FILE
IRONIC_USE_LINK_LOCAL=$(trueorfalse False IRONIC_USE_LINK_LOCAL)

# This flag is used to specify enabled network drivers
IRONIC_ENABLED_NETWORK_INTERFACES=${IRONIC_ENABLED_NETWORK_INTERFACES:-}

# This is the network interface to use for a node
IRONIC_NETWORK_INTERFACE=${IRONIC_NETWORK_INTERFACE:-}

# Ironic provision network name
IRONIC_PROVISION_NETWORK_NAME=${IRONIC_PROVISION_NETWORK_NAME:-}

# Provision network provider type. Can be flat or vlan.
IRONIC_PROVISION_PROVIDER_NETWORK_TYPE=${IRONIC_PROVISION_PROVIDER_NETWORK_TYPE:-'vlan'}

# If IRONIC_PROVISION_PROVIDER_NETWORK_TYPE is vlan. VLAN_ID may be specified. If it is not set,
# vlan will be allocated dynamically.
IRONIC_PROVISION_SEGMENTATION_ID=${IRONIC_PROVISION_SEGMENTATION_ID:-}

# Allocation network pool for provision network
# Example: IRONIC_PROVISION_ALLOCATION_POOL=start=10.0.5.10,end=10.0.5.100
IRONIC_PROVISION_ALLOCATION_POOL=${IRONIC_PROVISION_ALLOCATION_POOL:-}

# Ironic provision subnet name.
IRONIC_PROVISION_PROVIDER_SUBNET_NAME=${IRONIC_PROVISION_PROVIDER_SUBNET_NAME:-${IRONIC_PROVISION_NETWORK_NAME}-subnet}

# Ironic provision subnet gateway.
IRONIC_PROVISION_SUBNET_GATEWAY=${IRONIC_PROVISION_SUBNET_GATEWAY:-}

# Ironic provision subnet prefix
# Example: IRONIC_PROVISION_SUBNET_PREFIX=10.0.5.0/24
IRONIC_PROVISION_SUBNET_PREFIX=${IRONIC_PROVISION_SUBNET_PREFIX:-}

# get_pxe_boot_file() - Get the PXE/iPXE boot file path
function get_pxe_boot_file {
    local relpath=syslinux/pxelinux.0
    if [[ "$IRONIC_IPXE_ENABLED" == "True" ]] ; then
        relpath=ipxe/undionly.kpxe
    fi

    local pxe_boot_file
    if is_ubuntu; then
        pxe_boot_file=/usr/lib/$relpath
    elif is_fedora || is_suse; then
        pxe_boot_file=/usr/share/$relpath
    fi

    echo $pxe_boot_file
}

# PXE boot image
IRONIC_PXE_BOOT_IMAGE=${IRONIC_PXE_BOOT_IMAGE:-$(get_pxe_boot_file)}

IRONIC_AUTOMATED_CLEAN_ENABLED=$(trueorfalse True IRONIC_AUTOMATED_CLEAN_ENABLED)


# Functions
# ---------

# Test if any Ironic services are enabled
# is_ironic_enabled
function is_ironic_enabled {
    [[ ,${ENABLED_SERVICES} =~ ,"ir-" ]] && return 0
    return 1
}

function is_deployed_by_agent {
    [[ -z "${IRONIC_DEPLOY_DRIVER%%agent*}" ]] && return 0
    return 1
}

function is_deployed_by_ipmitool {
    [[ -z "${IRONIC_DEPLOY_DRIVER##*_ipmitool}" ]] && return 0
    return 1
}

function is_deployed_by_cimc {
    [[ -z "${IRONIC_DEPLOY_DRIVER##*_cimc}" ]] && return 0
    return 1
}

function is_deployed_by_ucs {
    [[ -z "${IRONIC_DEPLOY_DRIVER##*_ucs}" ]] && return 0
    return 1
}

function is_deployed_by_oneview {
    [[ -z "${IRONIC_DEPLOY_DRIVER##*_oneview}" ]] && return 0
}

function is_deployed_by_ilo {
    [[ -z "${IRONIC_DEPLOY_DRIVER##*_ilo}" ]] && return 0
    return 1
}

function is_glance_configuration_required {
    is_deployed_by_agent || [[ "$IRONIC_CONFIGURE_GLANCE_WITH_SWIFT" == "True" ]] && return 0
    return 1
}

function is_deploy_iso_required {
    [[ "$IRONIC_IS_HARDWARE" == "True" && "$IRONIC_DEPLOY_ISO_REQUIRED" == "True" ]] && return 0
    return 1
}

function setup_virtualbmc {
    # Install pyghmi from source, if requested, otherwise it will be
    # downloaded as part of the virtualbmc installation
    if use_library_from_git "pyghmi"; then
        git_clone_by_name "pyghmi"
        setup_dev_lib "pyghmi"
    fi

    if use_library_from_git "virtualbmc"; then
        git_clone_by_name "virtualbmc"
        setup_dev_lib "virtualbmc"
    else
        pip_install "virtualbmc"
    fi

    if [[ ! -d $(dirname $IRONIC_VBMC_CONFIG_FILE) ]]; then
        mkdir -p $(dirname $IRONIC_VBMC_CONFIG_FILE)
    fi

    iniset $IRONIC_VBMC_CONFIG_FILE log debug True
    iniset $IRONIC_VBMC_CONFIG_FILE log logfile $IRONIC_VBMC_LOGFILE
}

# install_ironic() - Install the things!
function install_ironic {
    # make sure all needed service were enabled
    local req_services="key"
    if [[ "$VIRT_DRIVER" == "ironic" ]]; then
        req_services+=" nova glance neutron"
    fi
    for srv in $req_services; do
        if ! is_service_enabled "$srv"; then
            die $LINENO "$srv should be enabled for Ironic."
        fi
    done

    if use_library_from_git "ironic-lib"; then
        git_clone_by_name "ironic-lib"
        setup_dev_lib "ironic-lib"
    fi

    setup_develop $IRONIC_DIR

    if [[ "$IRONIC_IPXE_ENABLED" == "True" ]] ; then
        install_apache_wsgi
    fi

    if is_deployed_by_ipmitool && [[ "$IRONIC_IS_HARDWARE" == "False" ]]; then
        setup_virtualbmc
    fi
}

# install_ironicclient() - Collect sources and prepare
function install_ironicclient {
    if use_library_from_git "python-ironicclient"; then
        git_clone_by_name "python-ironicclient"
        setup_dev_lib "python-ironicclient"
        sudo install -D -m 0644 -o $STACK_USER {${GITDIR["python-ironicclient"]}/tools/,/etc/bash_completion.d/}ironic.bash_completion
    else
        # nothing actually "requires" ironicclient, so force instally from pypi
        pip_install_gr python-ironicclient
    fi
}

# _cleanup_ironic_apache_wsgi() - Remove wsgi files, disable and remove apache vhost file
function _cleanup_ironic_apache_wsgi {
    sudo rm -rf $IRONIC_HTTP_DIR
    disable_apache_site ironic
    sudo rm -f $(apache_site_config_for ironic)
    restart_apache_server
}

# _config_ironic_apache_wsgi() - Set WSGI config files of Ironic
function _config_ironic_apache_wsgi {
    local ironic_apache_conf
    ironic_apache_conf=$(apache_site_config_for ironic)
    sudo cp $IRONIC_DEVSTACK_FILES_DIR/apache-ironic.template $ironic_apache_conf
    sudo sed -e "
        s|%PUBLICPORT%|$IRONIC_HTTP_PORT|g;
        s|%HTTPROOT%|$IRONIC_HTTP_DIR|g;
    " -i $ironic_apache_conf
    enable_apache_site ironic
}

# cleanup_ironic_config_files() - Remove residual cache/config/log files,
# left over from previous runs that would need to clean up.
function cleanup_ironic_config_files {
    sudo rm -rf $IRONIC_AUTH_CACHE_DIR $IRONIC_CONF_DIR
    sudo rm -rf $IRONIC_VM_LOG_DIR/*
}

# cleanup_ironic() - Clean everything left from Ironic
function cleanup_ironic {
    cleanup_ironic_config_files

    # Cleanup the WSGI files
    if [[ "$IRONIC_IPXE_ENABLED" == "True" ]] ; then
        _cleanup_ironic_apache_wsgi
    fi

    # Remove the hook to disable log rotate
    sudo rm -rf $IRONIC_LIBVIRT_HOOKS_PATH/qemu
}

# configure_ironic_dirs() - Create all directories required by Ironic and
# associated services.
function configure_ironic_dirs {
    sudo install -d -o $STACK_USER $IRONIC_CONF_DIR $STACK_USER $IRONIC_DATA_DIR \
        $IRONIC_STATE_PATH $IRONIC_TFTPBOOT_DIR $IRONIC_TFTPBOOT_DIR/pxelinux.cfg
    sudo chown -R $STACK_USER:$LIBVIRT_GROUP $IRONIC_TFTPBOOT_DIR

    if [[ "$IRONIC_IPXE_ENABLED" == "True" ]] ; then
        sudo install -d -o $STACK_USER -g $LIBVIRT_GROUP $IRONIC_HTTP_DIR
    fi

    if [ ! -f $IRONIC_PXE_BOOT_IMAGE ]; then
        die $LINENO "PXE boot file $IRONIC_PXE_BOOT_IMAGE not found."
    fi

    # Copy PXE binary
    if [[ "$IRONIC_IPXE_ENABLED" == "True" ]] ; then
        cp $IRONIC_PXE_BOOT_IMAGE $IRONIC_TFTPBOOT_DIR
    else
        # Syslinux >= 5.00 pxelinux.0 binary is not "stand-alone" anymore,
        # it depends on some c32 modules to work correctly.
        # More info: http://www.syslinux.org/wiki/index.php/Library_modules
        cp -aR $(dirname $IRONIC_PXE_BOOT_IMAGE)/*.{c32,0} $IRONIC_TFTPBOOT_DIR
    fi
}

function configure_ironic_provision_network {

    die_if_not_set $LINENO IRONIC_PROVISION_SUBNET_PREFIX "You must specify the IRONIC_PROVISION_SUBNET_PREFIX"
    die_if_not_set $LINENO PHYSICAL_NETWORK "You must specify the PHYSICAL_NETWORK"
    die_if_not_set $LINENO IRONIC_PROVISION_SUBNET_GATEWAY "You must specify the IRONIC_PROVISION_SUBNET_GATEWAY"

    local net_id
    net_id=$(neutron net-create --provider:network_type $IRONIC_PROVISION_PROVIDER_NETWORK_TYPE \
        --provider:physical_network "$PHYSICAL_NETWORK" \
        ${IRONIC_PROVISION_SEGMENTATION_ID:+--provider:segmentation_id $IRONIC_PROVISION_SEGMENTATION_ID} \
        ${IRONIC_PROVISION_NETWORK_NAME} | grep ' id ' | get_field 2)

    die_if_not_set $LINENO net_id "Failure creating net_id for $IRONIC_PROVISION_NETWORK_NAME"
    local subnet_id
    subnet_id="$(neutron subnet-create --ip_version 4 \
        ${IRONIC_PROVISION_ALLOCATION_POOL:+--allocation-pool $IRONIC_PROVISION_ALLOCATION_POOL} \
        --name $IRONIC_PROVISION_PROVIDER_SUBNET_NAME \
        --gateway $IRONIC_PROVISION_SUBNET_GATEWAY $net_id \
        $IRONIC_PROVISION_SUBNET_PREFIX | grep ' id ' | get_field 2)"

    die_if_not_set $LINENO subnet_id "Failure creating SUBNET_ID for $IRONIC_PROVISION_NETWORK_NAME"

    iniset $IRONIC_CONF_FILE neutron provisioning_network_uuid $net_id

    IRONIC_PROVISION_SEGMENTATION_ID=${IRONIC_PROVISION_SEGMENTATION_ID:-`neutron net-show ${net_id} | grep -w 'provider:segmentation_id'| get_field 2`}
    provision_net_prefix=${IRONIC_PROVISION_SUBNET_PREFIX##*/}

    # Set provision network GW on physical interface
    # Add vlan on br interface in case of IRONIC_PROVISION_PROVIDER_NETWORK_TYPE==vlan
    # othervise assign ip to br interface directly.
    if [[ "$IRONIC_PROVISION_PROVIDER_NETWORK_TYPE" == "vlan" ]]; then
        sudo vconfig add $OVS_PHYSICAL_BRIDGE $IRONIC_PROVISION_SEGMENTATION_ID
        sudo ip link set dev $OVS_PHYSICAL_BRIDGE.$IRONIC_PROVISION_SEGMENTATION_ID up
        sudo ip addr add dev $OVS_PHYSICAL_BRIDGE.$IRONIC_PROVISION_SEGMENTATION_ID $IRONIC_PROVISION_SUBNET_GATEWAY/$provision_net_prefix
    else
        sudo ip link set dev $OVS_PHYSICAL_BRIDGE up
        sudo ip addr add dev $OVS_PHYSICAL_BRIDGE $IRONIC_PROVISION_SUBNET_GATEWAY/$provision_net_prefix
    fi
}

function cleanup_ironic_provision_network {
    if [[ -z "${IRONIC_PROVISION_NETWORK_NAME}" ]]; then
        return 0
    fi
    # Cleanup OVS_PHYSICAL_BRIDGE subinterfaces
    local bridge_subint
    bridge_subint=$(cat /proc/net/dev | sed -n "s/^\(${OVS_PHYSICAL_BRIDGE}\.[0-9]*\).*/\1/p")
    for sub_int in $bridge_subint; do
        sudo ip link set dev $sub_int down
        sudo ip link del dev $sub_int
    done
}

# configure_ironic() - Set config files, create data dirs, etc
function configure_ironic {
    configure_ironic_dirs

    # Copy over ironic configuration file and configure common parameters.
    cp $IRONIC_DIR/etc/ironic/ironic.conf.sample $IRONIC_CONF_FILE
    iniset $IRONIC_CONF_FILE DEFAULT debug True
    inicomment $IRONIC_CONF_FILE DEFAULT log_file
    iniset $IRONIC_CONF_FILE database connection `database_connection_url ironic`
    iniset $IRONIC_CONF_FILE DEFAULT state_path $IRONIC_STATE_PATH
    iniset $IRONIC_CONF_FILE DEFAULT use_syslog $SYSLOG
    # Configure Ironic conductor, if it was enabled.
    if is_service_enabled ir-cond; then
        configure_ironic_conductor
    fi

    # Configure Ironic API, if it was enabled.
    if is_service_enabled ir-api; then
        configure_ironic_api
    fi

    # Format logging
    if [ "$LOG_COLOR" == "True" ] && [ "$SYSLOG" == "False" ]; then
        setup_colorized_logging $IRONIC_CONF_FILE DEFAULT tenant user
    fi

    if [[ "$IRONIC_IPXE_ENABLED" == "True" ]]; then
        _config_ironic_apache_wsgi
    fi
}

# configure_ironic_api() - Is used by configure_ironic(). Performs
# API specific configuration.
function configure_ironic_api {
    iniset $IRONIC_CONF_FILE DEFAULT auth_strategy $IRONIC_AUTH_STRATEGY
    iniset $IRONIC_CONF_FILE oslo_policy policy_file $IRONIC_POLICY_JSON

    # TODO(Yuki Nishiwaki): This is a temporary work-around until Ironic is fixed(bug#1422632).
    # These codes need to be changed to use the function of configure_auth_token_middleware
    # after Ironic conforms to the new auth plugin.
    iniset $IRONIC_CONF_FILE keystone_authtoken identity_uri $KEYSTONE_AUTH_URI
    iniset $IRONIC_CONF_FILE keystone_authtoken auth_uri $KEYSTONE_SERVICE_URI/v2.0
    iniset $IRONIC_CONF_FILE keystone_authtoken admin_user ironic
    iniset $IRONIC_CONF_FILE keystone_authtoken admin_password $SERVICE_PASSWORD
    iniset $IRONIC_CONF_FILE keystone_authtoken admin_tenant_name $SERVICE_PROJECT_NAME
    iniset $IRONIC_CONF_FILE keystone_authtoken cafile $SSL_BUNDLE_FILE
    iniset $IRONIC_CONF_FILE keystone_authtoken signing_dir $IRONIC_AUTH_CACHE_DIR/api

    iniset_rpc_backend ironic $IRONIC_CONF_FILE
    iniset $IRONIC_CONF_FILE api port $IRONIC_SERVICE_PORT

    iniset $IRONIC_CONF_FILE conductor automated_clean $IRONIC_AUTOMATED_CLEAN_ENABLED

    cp -p $IRONIC_DIR/etc/ironic/policy.json $IRONIC_POLICY_JSON
}

# configure_ironic_conductor() - Is used by configure_ironic().
# Sets conductor specific settings.
function configure_ironic_conductor {
    cp $IRONIC_DIR/etc/ironic/rootwrap.conf $IRONIC_ROOTWRAP_CONF
    cp -r $IRONIC_DIR/etc/ironic/rootwrap.d $IRONIC_CONF_DIR
    local ironic_rootwrap
    ironic_rootwrap=$(get_rootwrap_location ironic)
    local rootwrap_isudoer_cmd="$ironic_rootwrap $IRONIC_CONF_DIR/rootwrap.conf *"

    # Set up the rootwrap sudoers for ironic
    local tempfile
    tempfile=`mktemp`
    echo "$STACK_USER ALL=(root) NOPASSWD: $rootwrap_isudoer_cmd" >$tempfile
    chmod 0440 $tempfile
    sudo chown root:root $tempfile
    sudo mv $tempfile /etc/sudoers.d/ironic-rootwrap

    iniset $IRONIC_CONF_FILE DEFAULT rootwrap_config $IRONIC_ROOTWRAP_CONF
    iniset $IRONIC_CONF_FILE DEFAULT enabled_drivers $IRONIC_ENABLED_DRIVERS
    iniset $IRONIC_CONF_FILE conductor api_url $IRONIC_SERVICE_PROTOCOL://$HOST_IP:$IRONIC_SERVICE_PORT
    if [[ -n "$IRONIC_CALLBACK_TIMEOUT" ]]; then
        iniset $IRONIC_CONF_FILE conductor deploy_callback_timeout $IRONIC_CALLBACK_TIMEOUT
    fi
    iniset $IRONIC_CONF_FILE pxe tftp_server $IRONIC_TFTPSERVER_IP
    iniset $IRONIC_CONF_FILE pxe tftp_root $IRONIC_TFTPBOOT_DIR
    iniset $IRONIC_CONF_FILE pxe tftp_master_path $IRONIC_TFTPBOOT_DIR/master_images
    if [[ "$IRONIC_TERMINAL_SSL" == "True" ]]; then
        # Make sure the cert directory exist
        sudo mkdir -p $IRONIC_TERMINAL_CERT_DIR
        sudo chown $STACK_USER $IRONIC_TERMINAL_CERT_DIR
        iniset $IRONIC_CONF_FILE console terminal_cert_dir $IRONIC_TERMINAL_CERT_DIR
        # Generate the SSL certificate
        openssl req \
            -x509 \
            -days 365 \
            -newkey rsa:1024 \
            -nodes \
            -keyout $IRONIC_TERMINAL_CERT_DIR/certificate.pem.key \
            -out $IRONIC_TERMINAL_CERT_DIR/certificate.pem \
            -subj '/O=OpenStack/OU=DevStack Servers'
    fi

    local pxe_params="nofb nomodeset vga=normal console=${IRONIC_TTY_DEV}"
    pxe_params+=" systemd.journald.forward_to_console=yes ipa-debug=1"

    pxe_params+=" $IRONIC_EXTRA_PXE_PARAMS"

    # When booting with less than 1GB, we need to switch from default tmpfs
    # to ramfs for ramdisks to decompress successfully.
    if ([[ "$IRONIC_IS_HARDWARE" == "True" ]] &&
        [[ "$IRONIC_HW_NODE_RAM" -lt 1024 ]]) ||
        ([[ "$IRONIC_IS_HARDWARE" == "False" ]] &&
        [[ "$IRONIC_VM_SPECS_RAM" -lt 1024 ]]); then

        pxe_params+=" rootfstype=ramfs"
    fi

    if [[ -n "$pxe_params" ]]; then
        iniset $IRONIC_CONF_FILE pxe pxe_append_params "$pxe_params"
    fi

    # Set these options for scenarios in which the agent fetches the image
    # directly from glance, and don't set them where the image is pushed
    # over iSCSI.
    if is_glance_configuration_required; then
        if [[ "$SWIFT_ENABLE_TEMPURLS" == "True" ]] ; then
            iniset $IRONIC_CONF_FILE glance swift_temp_url_key $SWIFT_TEMPURL_KEY
        else
            die $LINENO "SWIFT_ENABLE_TEMPURLS must be True. This is " \
                        "required either because IRONIC_DEPLOY_DRIVER was " \
                        "set to some agent_* driver OR configuration of " \
                        "Glance with Swift was explicitly requested with " \
                        "IRONIC_CONFIGURE_GLANCE_WITH_SWIFT=True"
        fi
        iniset $IRONIC_CONF_FILE glance swift_endpoint_url http://${HOST_IP}:${SWIFT_DEFAULT_BIND_PORT:-8080}
        iniset $IRONIC_CONF_FILE glance swift_api_version v1
        local tenant_id
        tenant_id=$(get_or_create_project $SERVICE_PROJECT_NAME default)
        iniset $IRONIC_CONF_FILE glance swift_account AUTH_${tenant_id}
        iniset $IRONIC_CONF_FILE glance swift_container glance
        iniset $IRONIC_CONF_FILE glance swift_temp_url_duration 3600
    fi

    if is_deployed_by_agent; then
        iniset $IRONIC_CONF_FILE agent heartbeat_timeout 30
    fi

    # FIXME: this really needs to be tested in the gate.  For now, any
    # test using the agent ramdisk should skip the erase_devices clean
    # step  because it is too slow to run in the gate.
    iniset $IRONIC_CONF_FILE deploy erase_devices_priority 0

    if [[ "$IRONIC_IPXE_ENABLED" == "True" ]] ; then
        local pxebin
        pxebin=`basename $IRONIC_PXE_BOOT_IMAGE`
        iniset $IRONIC_CONF_FILE pxe ipxe_enabled True
        iniset $IRONIC_CONF_FILE pxe pxe_config_template '\$pybasedir/drivers/modules/ipxe_config.template'
        iniset $IRONIC_CONF_FILE pxe pxe_bootfile_name $pxebin
        iniset $IRONIC_CONF_FILE deploy http_root $IRONIC_HTTP_DIR
        iniset $IRONIC_CONF_FILE deploy http_url "http://$IRONIC_HTTP_SERVER:$IRONIC_HTTP_PORT"
    fi

    if [[ "$IRONIC_IS_HARDWARE" == "False" ]]; then
        iniset $IRONIC_CONF_FILE neutron port_setup_delay 15
    fi

    if [[ -n "$IRONIC_ENABLED_NETWORK_INTERFACES" ]]; then
        iniset $IRONIC_CONF_FILE DEFAULT enabled_network_interfaces $IRONIC_ENABLED_NETWORK_INTERFACES
    fi
}

# create_ironic_cache_dir() - Part of the init_ironic() process
function create_ironic_cache_dir {
    # Create cache dir
    sudo mkdir -p $IRONIC_AUTH_CACHE_DIR/api
    sudo chown $STACK_USER $IRONIC_AUTH_CACHE_DIR/api
    rm -f $IRONIC_AUTH_CACHE_DIR/api/*
    sudo mkdir -p $IRONIC_AUTH_CACHE_DIR/registry
    sudo chown $STACK_USER $IRONIC_AUTH_CACHE_DIR/registry
    rm -f $IRONIC_AUTH_CACHE_DIR/registry/*
}

# create_ironic_accounts() - Set up common required ironic accounts

# Tenant               User       Roles
# ------------------------------------------------------------------
# service              ironic     admin        # if enabled
function create_ironic_accounts {

    # Ironic
    if [[ "$ENABLED_SERVICES" =~ "ir-api" ]]; then
        # Get ironic user if exists

        # NOTE(Shrews): This user MUST have admin level privileges!
        create_service_user "ironic" "admin"

        get_or_create_service "ironic" "baremetal" "Ironic baremetal provisioning service"
        get_or_create_endpoint "baremetal" \
            "$REGION_NAME" \
            "$IRONIC_SERVICE_PROTOCOL://$IRONIC_HOSTPORT" \
            "$IRONIC_SERVICE_PROTOCOL://$IRONIC_HOSTPORT" \
            "$IRONIC_SERVICE_PROTOCOL://$IRONIC_HOSTPORT"
    fi
}


# init_ironic() - Initialize databases, etc.
function init_ironic {
    if is_service_enabled neutron; then
        # Save private network as cleaning network
        local cleaning_network_uuid
        cleaning_network_uuid=$(openstack network show  "$IRONIC_CLEAN_NET_NAME" -c id -f value)
        die_if_not_set $LINENO cleaning_network_uuid "Failed to get ironic cleaning network id"
        iniset $IRONIC_CONF_FILE neutron cleaning_network_uuid ${cleaning_network_uuid}
    fi

    # (Re)create  ironic database
    recreate_database ironic

    # Migrate ironic database
    $IRONIC_BIN_DIR/ironic-dbsync --config-file=$IRONIC_CONF_FILE

    create_ironic_cache_dir
}

# _ironic_bm_vm_names() - Generates list of names for baremetal VMs.
function _ironic_bm_vm_names {
    local idx
    local num_vms
    num_vms=$(($IRONIC_VM_COUNT - 1))
    for idx in $(seq 0 $num_vms); do
        echo "node-${idx}"
    done
}

# start_ironic() - Start running processes, including screen
function start_ironic {
    # Start Ironic API server, if enabled.
    if is_service_enabled ir-api; then
        start_ironic_api
    fi

    # Start Ironic conductor, if enabled.
    if is_service_enabled ir-cond; then
        start_ironic_conductor
    fi

    # Start Apache if iPXE is enabled
    if [[ "$IRONIC_IPXE_ENABLED" == "True" ]] ; then
        restart_apache_server
    fi
}

# start_ironic_api() - Used by start_ironic().
# Starts Ironic API server.
function start_ironic_api {
    run_process ir-api "$IRONIC_BIN_DIR/ironic-api --config-file=$IRONIC_CONF_FILE"
    echo "Waiting for ir-api ($IRONIC_HOSTPORT) to start..."
    if ! timeout $SERVICE_TIMEOUT sh -c "while ! wget --no-proxy -q -O- $IRONIC_SERVICE_PROTOCOL://$IRONIC_HOSTPORT; do sleep 1; done"; then
        die $LINENO "ir-api did not start"
    fi
}

# start_ironic_conductor() - Used by start_ironic().
# Starts Ironic conductor.
function start_ironic_conductor {
    run_process ir-cond "$IRONIC_BIN_DIR/ironic-conductor --config-file=$IRONIC_CONF_FILE"
    # TODO(romcheg): Find a way to check whether the conductor has started.
}

# stop_ironic() - Stop running processes
function stop_ironic {
    stop_process ir-api
    stop_process ir-cond
}

# create_ovs_taps is also called by the devstack/upgrade/resources.sh script
#
# create_ovs_taps ironic_network_id
# NOTE(vsaienko) Ironic supports only Flat Neutron network.
# create_ovs_taps are needed in order to provide connectivity from ironic-conductor
# to VM. With Neutron Flat network it will be not needed.
function create_ovs_taps {
    local ironic_net_id
    ironic_net_id=$1
    die_if_not_set $LINENO ironic_net_id "Failed to get ironic network id"

    # Work around: No netns exists on host until a Neutron port is created.  We
    # need to create one in Neutron to know what netns to tap into prior to the
    # first node booting.
    local port_id
    port_id=$(neutron port-create ${ironic_net_id} | grep " id " | get_field 2)
    # NOTE(vsaienko) openstack port create was introduced in Newton.
    # Since the function is used during upgrade, it should be backward compatible with Mitaka.
    # Switch to openstack port create in Ocata release.
    #port_id=$(openstack port create --network ${ironic_net_id} temp_port -c id -f value)
    die_if_not_set $LINENO port_id "Failed to create neutron port"

    # intentional sleep to make sure the tag has been set to port
    sleep 10

    local tapdev
    tapdev=$(sudo ip netns exec qdhcp-${ironic_net_id} ip link list | grep " tap" | cut -d':' -f2 | cut -d'@' -f1 | cut -b2-)
    die_if_not_set $LINENO tapdev "Failed to get tap device id"
    local tag_id
    tag_id=$(sudo ovs-vsctl get port ${tapdev} tag)
    die_if_not_set $LINENO tag_id "Failed to get tag id"

    local ovs_tap=ovs-${tag_id}-tap1
    local brbm_tap=brbm-${tag_id}-tap1
    # make sure veth pair is not existing, otherwise delete its links
    sudo ip link show $ovs_tap && sudo ip link delete $ovs_tap
    sudo ip link show $brbm_tap && sudo ip link delete $brbm_tap
    # create veth pair for future interconnection between br-int and brbm
    sudo ip link add $brbm_tap type veth peer name $ovs_tap
    sudo ip link set dev $brbm_tap up
    sudo ip link set dev $ovs_tap up

    sudo ovs-vsctl -- --if-exists del-port $ovs_tap -- add-port br-int $ovs_tap tag=$tag_id
    sudo ovs-vsctl -- --if-exists del-port $brbm_tap -- add-port $IRONIC_VM_NETWORK_BRIDGE $brbm_tap

    # Remove the port needed only for workaround.
    openstack port delete $port_id

    # Finally, share the fixed tenant network across all tenants.  This allows the host
    # to serve TFTP to a single network namespace via the tap device created above.
    openstack network set $ironic_net_id --share
}

function setup_qemu_log_hook {
    local libvirt_service_name

    # Make sure the libvirt hooks directory exist
    sudo mkdir -p $IRONIC_LIBVIRT_HOOKS_PATH

    # Copy the qemu hook to the right directory
    sudo cp $IRONIC_DEVSTACK_FILES_DIR/hooks/qemu $IRONIC_LIBVIRT_HOOKS_PATH/qemu
    sudo chmod -v +x $IRONIC_LIBVIRT_HOOKS_PATH/qemu
    sudo sed -e "
        s|%LOG_DIR%|$IRONIC_VM_LOG_DIR|g;
    " -i $IRONIC_LIBVIRT_HOOKS_PATH/qemu

    # Restart the libvirt daemon
    libvirt_service_name="libvirt-bin"
    if is_fedora; then
        libvirt_service_name="libvirtd"
    fi

    restart_service $libvirt_service_name
}

function create_bridge_and_vms {
    # Call libvirt setup scripts in a new shell to ensure any new group membership
    sudo su $STACK_USER -c "$IRONIC_SCRIPTS_DIR/setup-network.sh"
    if [[ "$IRONIC_VM_LOG_CONSOLE" == "True" ]] ; then
        local log_arg="-l $IRONIC_VM_LOG_DIR"

        if [[ "$IRONIC_VM_LOG_ROTATE" == "True" ]] ; then
            setup_qemu_log_hook
        fi
    else
        local log_arg=""
    fi

    local vbmc_port=$IRONIC_VBMC_PORT_RANGE_START
    local vm_name
    for vm_name in $(_ironic_bm_vm_names); do
        sudo -E su $STACK_USER -c "$IRONIC_SCRIPTS_DIR/create-node.sh -n $vm_name \
            -c $IRONIC_VM_SPECS_CPU -m $IRONIC_VM_SPECS_RAM -d $IRONIC_VM_SPECS_DISK \
            -a $IRONIC_VM_SPECS_CPU_ARCH -b $IRONIC_VM_NETWORK_BRIDGE -e $IRONIC_VM_EMULATOR \
            -p $vbmc_port -f $IRONIC_VM_SPECS_DISK_FORMAT $log_arg" >> $IRONIC_VM_MACS_CSV_FILE
        vbmc_port=$((vbmc_port+1))
    done
    local ironic_net_id
    ironic_net_id=$(openstack network show "$PRIVATE_NETWORK_NAME" -c id -f value)
    create_ovs_taps $ironic_net_id
}

function wait_for_nova_resources {
    # After nodes have been enrolled, we need to wait for both ironic and
    # nova's periodic tasks to populate the resource tracker with available
    # nodes and resources. Wait up to 2 minutes for a given resource before
    # timing out.
    local resource=$1
    local expected_count=$2
    local i
    echo_summary "Waiting 2 minutes for Nova resource tracker to pick up $resource >= $expected_count"
    for i in $(seq 1 120); do
        if [ $(openstack hypervisor stats show -f value -c $resource) -ge $expected_count ]; then
            return 0
        fi
        sleep 1
    done
    die $LINENO "Timed out waiting for Nova hypervisor-stats $resource >= $expected_count"
}

function _clean_ncpu_failure {
    SCREEN_NAME=${SCREEN_NAME:-stack}
    SERVICE_DIR=${SERVICE_DIR:-${DEST}/status}
    n_cpu_failure="$SERVICE_DIR/$SCREEN_NAME/n-cpu.failure"
    if [ -f ${n_cpu_failure} ]; then
        mv ${n_cpu_failure} "${n_cpu_failure}.before-restart-by-ironic"
    fi
}

function enroll_nodes {
    local chassis_id
    chassis_id=$(ironic chassis-create -d "ironic test chassis" | grep " uuid " | get_field 2)
    die_if_not_set $LINENO chassis_id "Failed to create chassis"

    if [[ "$IRONIC_IS_HARDWARE" == "False" ]]; then
        local ironic_node_cpu=$IRONIC_VM_SPECS_CPU
        local ironic_node_ram=$IRONIC_VM_SPECS_RAM
        local ironic_node_disk=$IRONIC_VM_SPECS_DISK
        local ironic_ephemeral_disk=$IRONIC_VM_EPHEMERAL_DISK
        local ironic_node_arch=x86_64
        local ironic_hwinfo_file=$IRONIC_VM_MACS_CSV_FILE

        if is_deployed_by_ipmitool; then
            local node_options="\
                -i ipmi_address=127.0.0.1 \
                -i ipmi_username=admin \
                -i ipmi_password=password"
        else
            local node_options="\
                -i ssh_virt_type=$IRONIC_SSH_VIRT_TYPE \
                -i ssh_address=$IRONIC_VM_SSH_ADDRESS \
                -i ssh_port=$IRONIC_VM_SSH_PORT \
                -i ssh_username=$IRONIC_SSH_USERNAME \
                -i ssh_key_filename=$IRONIC_KEY_FILE"
        fi
        node_options="\
            $node_options \
            -i deploy_kernel=$IRONIC_DEPLOY_KERNEL_ID \
            -i deploy_ramdisk=$IRONIC_DEPLOY_RAMDISK_ID"

    else
        local ironic_node_cpu=$IRONIC_HW_NODE_CPU
        local ironic_node_ram=$IRONIC_HW_NODE_RAM
        local ironic_node_disk=$IRONIC_HW_NODE_DISK
        local ironic_ephemeral_disk=$IRONIC_HW_EPHEMERAL_DISK
        local ironic_node_arch=$IRONIC_HW_ARCH
        local ironic_hwinfo_file=$IRONIC_HWINFO_FILE
    fi

    local total_nodes=0
    local total_cpus=0
    while read hardware_info; do
        if [[ "$IRONIC_IS_HARDWARE" == "False" ]]; then
            local mac_address
            mac_address=$(echo $hardware_info | awk '{print $1}')

            if is_deployed_by_ipmitool; then
                local vbmc_port
                vbmc_port=$(echo $hardware_info | awk '{print $2}')
                node_options+=" -i ipmi_port=$vbmc_port"
            fi
            # Local-link-connection options
            if [[ "${IRONIC_USE_LINK_LOCAL}" == "True" ]]; then
                local llc_opts=""
                local switch_info
                local switch_id
                local port_id

                switch_info=$(echo $hardware_info |awk '{print $3}')
                switch_id=$(echo $hardware_info |awk '{print $4}')
                port_id=$(echo $hardware_info |awk '{print $5}')

                llc_opts="-l switch_id=${switch_id} -l switch_info=${switch_info} -l port_id=${port_id}"

                local ironic_api_version='--ironic-api-version latest'
            fi
        else
            # Currently we require all hardware platform have same CPU/RAM/DISK info
            # in future, this can be enhanced to support different type, and then
            # we create the bare metal flavor with minimum value
            local bmc_address
            bmc_address=$(echo $hardware_info |awk  '{print $1}')
            local mac_address
            mac_address=$(echo $hardware_info |awk '{print $2}')
            local bmc_username
            bmc_username=$(echo $hardware_info |awk '{print $3}')
            local bmc_passwd
            bmc_passwd=$(echo $hardware_info |awk '{print $4}')
            local node_options="-i deploy_kernel=$IRONIC_DEPLOY_KERNEL_ID"
            node_options+=" -i deploy_ramdisk=$IRONIC_DEPLOY_RAMDISK_ID"

            if is_deployed_by_ipmitool; then
                node_options+=" -i ipmi_address=$bmc_address -i ipmi_password=$bmc_passwd\
                    -i ipmi_username=$bmc_username"
            elif is_deployed_by_cimc; then
                node_options+=" -i cimc_address=$bmc_address -i cimc_password=$bmc_passwd\
                    -i cimc_username=$bmc_username"
            elif is_deployed_by_ucs; then
                local ucs_service_profile
                ucs_service_profile=$(echo $hardware_info |awk '{print $5}')
                node_options+=" -i ucs_address=$bmc_address -i ucs_password=$bmc_passwd\
                    -i ucs_username=$bmc_username -i ucs_service_profile=$ucs_service_profile"
            elif is_deployed_by_oneview; then
                local server_hardware_uri
                server_hardware_uri=$(echo $hardware_info |awk  '{print $1}')
                local server_hardware_type_uri
                server_hardware_type_uri=$(echo $hardware_info |awk '{print $2}')
                local enclosure_group_uri
                enclosure_group_uri=$(echo $hardware_info |awk '{print $3}')
                local server_profile_template_uri
                server_profile_template_uri=$(echo $hardware_info |awk '{print $4}')
                mac_address=$(echo $hardware_info |awk '{print $5}')

                node_options+=" -i server_hardware_uri=$server_hardware_uri"
                node_options+=" -p capabilities="
                node_options+="server_hardware_type_uri:$server_hardware_type_uri,"
                node_options+="enclosure_group_uri:$enclosure_group_uri,"
                node_options+="server_profile_template_uri:$server_profile_template_uri"
            elif is_deployed_by_ilo; then
                node_options+=" -i ilo_address=$bmc_address -i ilo_password=$bmc_passwd\
                    -i ilo_username=$bmc_username"
                if [[ $IRONIC_DEPLOY_DRIVER -ne "pxe_ilo" ]]; then
                    node_options+=" -i ilo_deploy_iso=$IRONIC_DEPLOY_ISO_ID"
                fi
            fi
        fi

        # First node created will be used for testing in ironic w/o glance
        # scenario, so we need to know its UUID.
        local standalone_node_uuid=""
        if [ $total_nodes -eq 0 ]; then
            standalone_node_uuid="--uuid $IRONIC_NODE_UUID"
        fi

        local node_id
        node_id=$(ironic node-create $standalone_node_uuid\
            --chassis_uuid $chassis_id \
            --driver $IRONIC_DEPLOY_DRIVER \
            --name node-$total_nodes \
            -p cpus=$ironic_node_cpu\
            -p memory_mb=$ironic_node_ram\
            -p local_gb=$ironic_node_disk\
            -p cpu_arch=$ironic_node_arch \
            $node_options \
            | grep " uuid " | get_field 2)

        # NOTE(vsaienko) IPA didn't automatically recognize root devices less than 4Gb.
        # Setting root hint allows to install OS on such devices.
        # 0x1af4 is VirtIO vendor device ID.
        if [[ "$ironic_node_disk" -lt "4" && is_deployed_by_agent ]]; then
            ironic node-update $node_id add properties/root_device='{"vendor": "0x1af4"}'
        fi

        # In case we using portgroups, we should API version that support them.
        # Othervise API will return 406 ERROR
        ironic $ironic_api_version port-create --address $mac_address --node $node_id $llc_opts

        # NOTE(vsaienko) use node-update instead of specifying network_interface
        # during node creation. If node is added with latest version of API it
        # will NOT go to available state automatically.
        if [[ -n "${IRONIC_NETWORK_INTERFACE}" ]]; then
            local n_id
            ironic node-set-maintenance $node_id true
            n_id=$(ironic $ironic_api_version node-update $node_id add network_interface=$IRONIC_NETWORK_INTERFACE)
            die_if_not_set $LINENO n_id "Failed to update network interface for node"
            ironic node-set-maintenance $node_id false
        fi

        total_nodes=$((total_nodes+1))
        total_cpus=$((total_cpus+$ironic_node_cpu))
    done < $ironic_hwinfo_file

    local adjusted_disk
    adjusted_disk=$(($ironic_node_disk - $ironic_ephemeral_disk))
    openstack flavor create --ephemeral $ironic_ephemeral_disk --ram $ironic_node_ram --disk $adjusted_disk --vcpus $ironic_node_cpu baremetal
    openstack flavor set baremetal --property "cpu_arch"="$ironic_node_arch"

    if [ "$VIRT_DRIVER" == "ironic" ]; then
        # NOTE(dtantsur): sometimes nova compute fails to start with ironic due
        # to keystone restarting and not being able to authenticate us.
        # Restart it just to be sure (and avoid gate problems like bug 1537076)
        stop_nova_compute || /bin/true
        # NOTE(pas-ha) if nova compute failed before restart, .failure file
        # that was created will fail the service_check in the end of the deployment
        _clean_ncpu_failure
        start_nova_compute
        wait_for_nova_resources "count" $total_nodes
        wait_for_nova_resources "vcpus" $total_cpus
    fi
}

function configure_iptables {
    # enable tftp natting for allowing connections to HOST_IP's tftp server
    sudo modprobe nf_conntrack_tftp
    sudo modprobe nf_nat_tftp
    # explicitly allow DHCP - packets are occasionally being dropped here
    sudo iptables -I INPUT -p udp --dport 67:68 --sport 67:68 -j ACCEPT || true
    # nodes boot from TFTP and callback to the API server listening on $HOST_IP
    sudo iptables -I INPUT -d $HOST_IP -p udp --dport 69 -j ACCEPT || true
    sudo iptables -I INPUT -d $HOST_IP -p tcp --dport $IRONIC_SERVICE_PORT -j ACCEPT || true
    if is_deployed_by_agent; then
        # agent ramdisk gets instance image from swift
        sudo iptables -I INPUT -d $HOST_IP -p tcp --dport ${SWIFT_DEFAULT_BIND_PORT:-8080} -j ACCEPT || true
    fi

    if [[ "$IRONIC_IPXE_ENABLED" == "True" ]] ; then
        sudo iptables -I INPUT -d $HOST_IP -p tcp --dport $IRONIC_HTTP_PORT -j ACCEPT || true
    fi
}

function configure_tftpd {
    # stop tftpd and setup serving via xinetd
    stop_service tftpd-hpa || true
    [ -f /etc/init/tftpd-hpa.conf ] && echo "manual" | sudo tee /etc/init/tftpd-hpa.override
    sudo cp $IRONIC_TEMPLATES_DIR/tftpd-xinetd.template /etc/xinetd.d/tftp
    sudo sed -e "s|%TFTPBOOT_DIR%|$IRONIC_TFTPBOOT_DIR|g" -i /etc/xinetd.d/tftp

    # setup tftp file mapping to satisfy requests at the root (booting) and
    # /tftpboot/ sub-dir (as per deploy-ironic elements)
    echo "r ^([^/]) $IRONIC_TFTPBOOT_DIR/\1" >$IRONIC_TFTPBOOT_DIR/map-file
    echo "r ^(/tftpboot/) $IRONIC_TFTPBOOT_DIR/\2" >>$IRONIC_TFTPBOOT_DIR/map-file

    chmod -R 0755 $IRONIC_TFTPBOOT_DIR
    restart_service xinetd
}

function configure_ironic_ssh_keypair {
    if [[ ! -d $HOME/.ssh ]]; then
        mkdir -p $HOME/.ssh
        chmod 700 $HOME/.ssh
    fi
    # recreate ssh if any part is missing
    if [[ ! -e $IRONIC_KEY_FILE ]] || [[ ! -e $IRONIC_KEY_FILE.pub ]]; then
        if [[ ! -d $(dirname $IRONIC_KEY_FILE) ]]; then
            mkdir -p $(dirname $IRONIC_KEY_FILE)
        fi
        echo -e 'n\n' | ssh-keygen -q -t rsa -P '' -f $IRONIC_KEY_FILE
    fi
    cat $IRONIC_KEY_FILE.pub | tee -a $IRONIC_AUTHORIZED_KEYS_FILE
    # remove duplicate keys.
    sort -u -o $IRONIC_AUTHORIZED_KEYS_FILE $IRONIC_AUTHORIZED_KEYS_FILE
}

function ironic_ssh_check {
    local key_file=$1
    local floating_ip=$2
    local port=$3
    local default_instance_user=$4
    local attempt=$5
    local status=false
    local ssh_options="-o BatchMode=yes -o ConnectTimeout=$IRONIC_SSH_TIMEOUT -o StrictHostKeyChecking=no"
    while [[ $attempt -gt 0 ]]; do
        ssh -p $port $ssh_options -i $key_file ${default_instance_user}@$floating_ip exit
        if [[ "$?" == "0" ]]; then
            status=true
            break
        fi
        attempt=$((attempt - 1))
        echo "SSH connection failed. $attempt attempts left."
    done
    if ! $status; then
        die $LINENO "server didn't become ssh-able!"
    fi
}

function configure_ironic_auxiliary {
    configure_ironic_ssh_keypair
    ironic_ssh_check $IRONIC_KEY_FILE $IRONIC_VM_SSH_ADDRESS $IRONIC_VM_SSH_PORT $IRONIC_SSH_USERNAME $IRONIC_SSH_ATTEMPTS
}

function build_ipa_ramdisk {
    local kernel_path=$1
    local ramdisk_path=$2
    local iso_path=$3
    case $IRONIC_RAMDISK_TYPE in
        'coreos')
            build_ipa_coreos_ramdisk $kernel_path $ramdisk_path $iso_path
            ;;
        'tinyipa')
            build_tinyipa_ramdisk $kernel_path $ramdisk_path $iso_path
            ;;
        'dib')
            build_ipa_dib_ramdisk $kernel_path $ramdisk_path $iso_path
            ;;
        *)
            die $LINENO "Unrecognised IRONIC_RAMDISK_TYPE: $IRONIC_RAMDISK_TYPE. Expected either of 'dib', 'coreos', or 'tinyipa'."
            ;;
    esac
}

function build_ipa_coreos_ramdisk {
    echo "Building coreos ironic-python-agent deploy ramdisk"
    local kernel_path=$1
    local ramdisk_path=$2
    local iso_path=$3
    # on fedora services do not start by default
    restart_service docker
    git_clone $IRONIC_PYTHON_AGENT_REPO $IRONIC_PYTHON_AGENT_DIR $IRONIC_PYTHON_AGENT_BRANCH
    cd $IRONIC_PYTHON_AGENT_DIR
    imagebuild/coreos/build_coreos_image.sh
    cp imagebuild/coreos/UPLOAD/coreos_production_pxe_image-oem.cpio.gz $ramdisk_path
    cp imagebuild/coreos/UPLOAD/coreos_production_pxe.vmlinuz $kernel_path
    if is_deploy_iso_required; then
        imagebuild/coreos/iso-image-create -k $kernel_path -i $ramdisk_path -o $iso_path
    fi
    sudo rm -rf UPLOAD
    cd -
}

function build_tinyipa_ramdisk {
    echo "Building ironic-python-agent deploy ramdisk"
    local kernel_path=$1
    local ramdisk_path=$2
    local iso_path=$3
    git_clone $IRONIC_PYTHON_AGENT_REPO $IRONIC_PYTHON_AGENT_DIR $IRONIC_PYTHON_AGENT_BRANCH
    cd $IRONIC_PYTHON_AGENT_DIR/imagebuild/tinyipa
    export BUILD_AND_INSTALL_TINYIPA=true
    make
    cp tinyipa.gz $ramdisk_path
    cp tinyipa.vmlinuz $kernel_path
    if is_deploy_iso_required; then
        make iso
        cp tinyipa.iso $iso_path
    fi
    make clean
    cd -
}

# install_diskimage_builder() - Collect source and prepare or install from pip
function install_diskimage_builder {
    if use_library_from_git "diskimage-builder"; then
        git_clone_by_name "diskimage-builder"
        setup_dev_lib "diskimage-builder"
    else
        pip_install_gr "diskimage-builder"
    fi
}

function build_ipa_dib_ramdisk {
    local kernel_path=$1
    local ramdisk_path=$2
    local iso_path=$3
    local tempdir
    tempdir=$(mktemp -d --tmpdir=${DEST})

    # install diskimage-builder if not present
    if ! $(type -P disk-image-create > /dev/null); then
        install_diskimage_builder
    fi

    echo "Building IPA ramdisk with DIB options: $IRONIC_DIB_RAMDISK_OPTIONS"
    if is_deploy_iso_required; then
        IRONIC_DIB_RAMDISK_OPTIONS+=" iso"
    fi
    disk-image-create "$IRONIC_DIB_RAMDISK_OPTIONS" \
                    -o  "$tempdir/ironic-agent" \
                    ironic-agent
    chmod -R +r $tempdir
    mv "$tempdir/ironic-agent.kernel" "$kernel_path"
    mv "$tempdir/ironic-agent.initramfs" "$ramdisk_path"
    if is_deploy_iso_required; then
        mv "$tempdir/ironic-agent.iso" "$iso_path"
    fi
    rm -rf $tempdir
}

# build deploy kernel+ramdisk, then upload them to glance
# this function sets ``IRONIC_DEPLOY_KERNEL_ID``, ``IRONIC_DEPLOY_RAMDISK_ID``
function upload_baremetal_ironic_deploy {
    declare -g IRONIC_DEPLOY_KERNEL_ID IRONIC_DEPLOY_RAMDISK_ID
    echo_summary "Creating and uploading baremetal images for ironic"

    if [ -z "$IRONIC_DEPLOY_KERNEL" -o -z "$IRONIC_DEPLOY_RAMDISK" -o -z "$IRONIC_DEPLOY_ISO" ]; then
        local IRONIC_DEPLOY_KERNEL_PATH=$TOP_DIR/files/ir-deploy-$IRONIC_DEPLOY_DRIVER.kernel
        local IRONIC_DEPLOY_RAMDISK_PATH=$TOP_DIR/files/ir-deploy-$IRONIC_DEPLOY_DRIVER.initramfs
        local IRONIC_DEPLOY_ISO_PATH=$TOP_DIR/files/ir-deploy-$IRONIC_DEPLOY_DRIVER.iso
    else
        local IRONIC_DEPLOY_KERNEL_PATH=$IRONIC_DEPLOY_KERNEL
        local IRONIC_DEPLOY_RAMDISK_PATH=$IRONIC_DEPLOY_RAMDISK
        local IRONIC_DEPLOY_ISO_PATH=$IRONIC_DEPLOY_ISO
    fi

    if [ ! -e "$IRONIC_DEPLOY_RAMDISK_PATH" ] || \
            [ ! -e "$IRONIC_DEPLOY_KERNEL_PATH" ]  || \
            ( is_deploy_iso_required && [ ! -e "$IRONIC_DEPLOY_ISO_PATH" ] ); then
        # files don't exist, need to build them
        if [ "$IRONIC_BUILD_DEPLOY_RAMDISK" = "True" ]; then
            # we can build them only if we're not offline
            if [ "$OFFLINE" != "True" ]; then
                build_ipa_ramdisk $IRONIC_DEPLOY_KERNEL_PATH $IRONIC_DEPLOY_RAMDISK_PATH $IRONIC_DEPLOY_ISO_PATH
            else
                die $LINENO "Deploy kernel+ramdisk or iso files don't exist and cannot be built in OFFLINE mode"
            fi
        else
            # download the agent image tarball
            wget "$IRONIC_AGENT_KERNEL_URL" -O $IRONIC_DEPLOY_KERNEL_PATH
            wget "$IRONIC_AGENT_RAMDISK_URL" -O $IRONIC_DEPLOY_RAMDISK_PATH
        fi
    fi

    # load them into glance
    IRONIC_DEPLOY_KERNEL_ID=$(openstack \
        image create \
        $(basename $IRONIC_DEPLOY_KERNEL_PATH) \
        --public --disk-format=aki \
        --container-format=aki \
        < $IRONIC_DEPLOY_KERNEL_PATH  | grep ' id ' | get_field 2)
    die_if_not_set $LINENO IRONIC_DEPLOY_KERNEL_ID "Failed to load kernel image into glance"

    IRONIC_DEPLOY_RAMDISK_ID=$(openstack \
        image create \
        $(basename $IRONIC_DEPLOY_RAMDISK_PATH) \
        --public --disk-format=ari \
        --container-format=ari \
        < $IRONIC_DEPLOY_RAMDISK_PATH  | grep ' id ' | get_field 2)
    die_if_not_set $LINENO IRONIC_DEPLOY_RAMDISK_ID "Failed to load ramdisk image into glance"

    if is_deploy_iso_required; then
        IRONIC_DEPLOY_ISO_ID=$(openstack \
            image create \
            $(basename $IRONIC_DEPLOY_ISO_PATH) \
            --public --disk-format=iso \
            --container-format=bare \
            < $IRONIC_DEPLOY_ISO_PATH  -f value -c id)
        die_if_not_set $LINENO IRONIC_DEPLOY_ISO_ID "Failed to load deploy iso into glance"
    fi
}

function prepare_baremetal_basic_ops {
    if [[ "$IRONIC_BAREMETAL_BASIC_OPS" != "True" ]]; then
        return 0
    fi

    if [[ "$IRONIC_IS_HARDWARE" == "False" ]]; then
        configure_ironic_auxiliary
    fi
    upload_baremetal_ironic_deploy
    enroll_nodes
    configure_tftpd
    configure_iptables
}

function cleanup_baremetal_basic_ops {
    if [[ "$IRONIC_BAREMETAL_BASIC_OPS" != "True" ]]; then
        return 0
    fi
    rm -f $IRONIC_VM_MACS_CSV_FILE
    if [ -f $IRONIC_KEY_FILE ]; then
        local key
        key=$(cat $IRONIC_KEY_FILE.pub)
        # remove public key from authorized_keys
        grep -v "$key" $IRONIC_AUTHORIZED_KEYS_FILE > temp && mv temp $IRONIC_AUTHORIZED_KEYS_FILE
        chmod 0600 $IRONIC_AUTHORIZED_KEYS_FILE
    fi
    sudo rm -rf $IRONIC_DATA_DIR $IRONIC_STATE_PATH

    local vm_name
    for vm_name in $(_ironic_bm_vm_names); do
        sudo su $STACK_USER -c "$IRONIC_SCRIPTS_DIR/cleanup-node.sh $vm_name"
        # Cleanup node bridge/interfaces
        sudo ip link set ovs-$vm_name down
        sudo ip link set br-$vm_name down
        sudo ovs-vsctl del-port ovs-$vm_name
        sudo ip link del dev ovs-$vm_name
        sudo ip link del dev br-$vm_name
    done

    sudo ovs-vsctl --if-exists del-br $IRONIC_VM_NETWORK_BRIDGE

    sudo rm -rf /etc/xinetd.d/tftp /etc/init/tftpd-hpa.override
    restart_service xinetd
    sudo iptables -D INPUT -d $HOST_IP -p udp --dport 69 -j ACCEPT || true
    sudo iptables -D INPUT -d $HOST_IP -p tcp --dport $IRONIC_SERVICE_PORT -j ACCEPT || true
    if is_deployed_by_agent; then
        # agent ramdisk gets instance image from swift
        sudo iptables -D INPUT -d $HOST_IP -p tcp --dport ${SWIFT_DEFAULT_BIND_PORT:-8080} -j ACCEPT || true
    fi
    sudo rmmod nf_conntrack_tftp || true
    sudo rmmod nf_nat_tftp || true
}

function ironic_configure_tempest {
    local bm_flavor_id
    bm_flavor_id=$(openstack flavor show baremetal -f value -c id)
    die_if_not_set $LINENO bm_flavor_id "Failed to get id of baremetal flavor"
    iniset $TEMPEST_CONFIG compute flavor_ref $bm_flavor_id
    iniset $TEMPEST_CONFIG compute flavor_ref_alt $bm_flavor_id
    iniset $TEMPEST_CONFIG compute-feature-enabled disk_config False
}

# Restore xtrace + pipefail
$_XTRACE_IRONIC
$_PIPEFAIL_IRONIC

# Tell emacs to use shell-script-mode
## Local variables:
## mode: shell-script
## End: