openStackNotes

#!/bin/bash NOT A REAL SHELL SCRIPT!
# This is a cut'n'paste scriptlet
# be sure to read & understand what you're cutting & pasting!
#
# Setting up OpenStack+OpenShift+Docker
#########################################################################################
### First OpenStack ##########################################################################################################################################
### Install all parts "by hand" using official OpenStack install guide
###    NOTES: At this time did not install Object Storage (Swift) module due to resource
###             constraints in the VirtualBox environment.
###           Assuming Public URL, Internal URL, and Admin URL are the same in most cases
###           
##############################################################################################################################################################
### # VM sizing for test environment
volume group ~ 10G
/boot ~ 0.5G
/     ~ 4.0G
/var  ~ 2.0G
/home ~ 1.0G
swap  ~ 2.0G

##############################################################################################################################################################
### # Steps after base OS install (CentOS 7)
yum -y update
reboot
#####
rpm --import http://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-7
yum -y install http://dl.fedoraproject.org/pub/epel/7/x86_64/e/epel-release-7-2.noarch.rpm
yum -y install http://rdo.fedorapeople.org/openstack-juno/rdo-release-juno.rpm
yum -y install etckeeper
git config --global user.email "root@`hostname -s`.os"
git config --global user.name "root"
etckeeper init
etckeeper commit -m "initial"
yum -y install abrt-addon-ccpp abrt-addon-kerneloops abrt-addon-python abrt-cli atop autojump autojump-zsh bmon bzip2 cpupowerutils crudini dkms dstat gcc git git-all gpm htop hwloc ibmonitor ifstat ifstatus iftop iotop iptraf-ng jnettop kernel-devel kernel-headers lsof make mosh mtr multitail ncdu net-snmp nethogs ntp ntop openstack-selinux perl ps_mem psmisc sg3_utils snappy strace sysdig systemtap tcptrack tmux util-linux-ng wget yum-plugin-priorities yum-utils 
KERN_DIR=/usr/src/kernels/`uname -r`
mount -r /dev/cdrom /mnt ; cd /mnt
./VBoxLinuxAdditions.run 
cd ; umount /mnt ; eject
/bin/sed -i 's/makedumpfile -c/makedumpfile -p/g' /etc/kdump.conf
mkdir /var/crash/cores
cat >>/etc/abrt/abrt.conf <<EOF
DumpLocation = /var/crash/cores
EOF
cat >>/etc/sysctl.conf <<EOF
kernel.core_pattern = |/usr/libexec/abrt-hook-ccpp %s %c %p %u %g %t %e %h
kernel.sched_min_granularity_ns = 10000000
kernel.sched_wakeup_granularity_ns = 15000000
vm.swappiness = 0
vm.dirty_ratio = 10
vm.dirty_background_ratio = 5
net.core.wmem_max = 4194304
net.core.rmem_max = 4194304
EOF
sed -i 's/ProcessUnpackaged = no/ProcessUnpackaged = yes/' /etc/abrt/abrt-action-save-package-data.conf
sed -i 's/OpenGPGCheck = yes/OpenGPGCheck = no/' /etc/abrt/abrt-action-save-package-data.conf
cat >>/etc/fstab <<EOF
tmpfs   /tmp    tmpfs   defaults,size=256m  0 0
EOF
systemctl enable tmp.mount && systemctl start tmp.mount
systemctl stop multipathd.service && systemctl disable multipathd.service
systemctl stop mdmonitor.service && systemctl disable mdmonitor.service
systemctl stop avahi-daemon.service && systemctl disable avahi-daemon.service
systemctl stop avahi-daemon.socket && systemctl disable avahi-daemon.socket
systemctl stop firewalld.service && systemctl disable firewalld.service
grubby --update-kernel=DEFAULT --args="elevator=noop"
cat >>/etc/motd <<EOF
Good day to you!
EOF
reboot
#####


##############################################################################################################################################################
### # On Controller
# Set ntp "server" to appropriate stratum in ntp.conf
sed -i 's/nopeer//g' /etc/ntp.conf ; sed -i 's/noquery//g' /etc/ntp.conf
systemctl enable ntpd.service && systemctl start ntpd.service
yum -y install mariadb mariadb-server MySQL-python rabbitmq-server
crudini --set /etc/my.cnf mysqld bind-address           "$CON_IP"
crudini --set /etc/my.cnf mysqld default-storage-engine "innodb"
crudini --set /etc/my.cnf mysqld innodb_file_per_table  
crudini --set /etc/my.cnf mysqld collation-server       "utf8_general_ci"
crudini --set /etc/my.cnf mysqld init-connect           'SET NAMES utf8'
crudini --set /etc/my.cnf mysqld character-set-server   "utf8"

systemctl enable mariadb.service && systemctl start mariadb.service
mysql_secure_installation
systemctl enable rabbitmq-server.service && systemctl start rabbitmq-server.service
# Create new user, or use existing guest user:
rabbitmqctl change_password ${RABBIT_USER} ${RABBIT_PASS}

##############################################################################################################################################################
### # All non-controller nodes
# Set ntp "server" to "controller" in ntp.conf
systemctl enable ntpd.service && systemctl start ntpd.service

##############################################################################################################################################################
### # Keystone (Identity Service) - normally on controller
Q1="CREATE DATABASE keystone;"
Q2="GRANT ALL PRIVILEGES ON keystone.* TO 'keystone'@'localhost' IDENTIFIED BY '${KEYSTONE_DBPASS}';"
Q3="GRANT ALL PRIVILEGES ON keystone.* TO 'keystone'@'%' IDENTIFIED BY '${KEYSTONE_DBPASS}';"
SQL="${Q1}${Q2}${Q3}"
mysql -u root --password=$MYSQL_PASS -e "$SQL"

# generate install token
export OS_SERVICE_TOKEN=`openssl rand -hex 10`
export OS_SERVICE_ENDPOINT="http://${CONTROLLER}:35357/v2.0"
yum -y install openstack-keystone python-keystoneclient
crudini --set /etc/keystone/keystone.conf DEFAULT  admin_token "${OS_SERVICE_TOKEN}"
crudini --set /etc/keystone/keystone.conf database connection  "mysql://keystone:${KEYSTONE_DBPASS}@${CONTROLLER}/keystone"
crudini --set /etc/keystone/keystone.conf token    provider    "keystone.token.providers.uuid.Provider"
crudini --set /etc/keystone/keystone.conf token    driver      "keystone.token.persistence.backends.sql.Token"
keystone-manage pki_setup --keystone-user keystone --keystone-group keystone
#WARNING keystone.cli [-] keystone-manage pki_setup is not recommended for production use.
chown -R keystone:keystone /var/log/keystone
chown -R keystone:keystone /etc/keystone/ssl
chmod -R o-rwx /etc/keystone/ssl
su -s /bin/sh -c "keystone-manage db_sync" keystone
systemctl enable openstack-keystone.service && systemctl start openstack-keystone.service
(crontab -l -u keystone 2>&1 | grep -q token_flush) || echo '@hourly /usr/bin/keystone-manage token_flush >/var/log/keystone/keystone-tokenflush.log 2>&1' >> /var/spool/cron/keystone
# Create admin tenant/user
keystone tenant-create --name admin --description "Admin Tenant"
keystone user-create   --name admin --pass ${ADMIN_PASS} --email "root@`hostname -s`"
keystone role-create   --name admin
keystone user-role-add --tenant admin --user admin --role admin
keystone role-create   --name _member_
keystone user-role-add --tenant admin --user admin --role _member_
keystone tenant-create --name service --description "Service Tenant"
# create user tenant(s)/login(s)
keystone tenant-create --name demo   --description "Demo Tenant"
keystone user-create   --name demo   --pass ${DEMO_PASS} --email "root@`hostname -s`"
keystone user-role-add --tenant demo --user demo --role _member_
# create service entity/api endpoints
keystone service-create --name keystone --type identity --description "OpenStack Identity"
keystone endpoint-create --service-id $(keystone service-list | awk '/ identity / {print $2}') --publicurl http://${CONTROLLER}:5000/v2.0 --internalurl http://${CONTROLLER}:5000/v2.0 --adminurl http://${CONTROLLER}:35357/v2.0 --region regionOne
## ## Verification of Keystone ## ##
unset OS_SERVICE_TOKEN OS_SERVICE_ENDPOINT
keystone --os-tenant-name admin --os-username admin --os-password ${ADMIN_PASS} --os-auth-url http://${CONTROLLER}:35357/v2.0 token-get
keystone --os-tenant-name admin --os-username admin --os-password ${ADMIN_PASS} --os-auth-url http://${CONTROLLER}:35357/v2.0 tenant-list
# In testing, I could not get the next to to work via password, only via token
keystone --os-tenant-name admin --os-username admin --os-password ${ADMIN_PASS} --os-auth-url http://${CONTROLLER}:35357/v2.0 user-list
keystone --os-tenant-name admin --os-username admin --os-password ${ADMIN_PASS} --os-auth-url http://${CONTROLLER}:35357/v2.0 role-list

##############################################################################################################################################################
### # Glance (Image Service) - normally on controller or storage
# these steps are done on controller regardless of where glance server installed
source /root/bin/settings.sh
SERV="glance"
SERV_DBPASS=${GLANCE_DBPASS}
SERV_PASS=${GLANCE_PASS}
SERV_PORT=9292
SERV_TENANT="service"
SERV_TYPE="image"
SERV_DESC="OpenStack Image Service"

Q1="CREATE DATABASE ${SERV};"
Q2="GRANT ALL PRIVILEGES ON ${SERV}.* TO '${SERV}'@'localhost' IDENTIFIED BY '${SERV_DBPASS}';"
Q3="GRANT ALL PRIVILEGES ON ${SERV}.* TO '${SERV}'@'%' IDENTIFIED BY '${SERV_DBPASS}';"
SQL="${Q1}${Q2}${Q3}"
mysql -u root --password=$MYSQL_PASS -e "$SQL"

gettoken
keystone user-create     --name ${SERV} --pass   ${SERV_PASS}
keystone user-role-add   --user ${SERV} --tenant ${SERV_TENANT} --role admin
keystone service-create  --name ${SERV} --type   ${SERV_TYPE} --description "OpenStack Networking"
keystone endpoint-create --service-id $(keystone service-list | awk '/ image / {print $2}') --publicurl http://${CONTROLLER}:${SERV_PORT} --adminurl http://${CONTROLLER}:${SERV_PORT} --internalurl http://${CONTROLLER}:${SERV_PORT} --region regionOne

## ## glance controller steps ## ##
#  ensure image store has space available & is owned by glance; default /var/lib/glance/images
yum -y install openstack-glance python-glanceclient
crudini --set /etc/glance/glance-api.conf database           connection         "mysql://${SERV}:${SERV_DBPASS}@${CONTROLLER}/${SERV}"
crudini --set /etc/glance/glance-api.conf keystone_authtoken admin_password     "${SERV_PASS}"
crudini --set /etc/glance/glance-api.conf keystone_authtoken admin_tenant_name  "${SERV_TENANT}"
crudini --set /etc/glance/glance-api.conf keystone_authtoken admin_user         "${SERV}"
crudini --set /etc/glance/glance-api.conf keystone_authtoken auth_uri           "http://${CONTROLLER}:5000/v2.0"
crudini --set /etc/glance/glance-api.conf keystone_authtoken identity_uri       "http://${CONTROLLER}:35357"
crudini --set /etc/glance/glance-api.conf paste_deploy       flavor            "keystone"
crudini --set /etc/glance/glance-api.conf glance_store       default_store     "file"
crudini --set /etc/glance/glance-api.conf glance_store       filesystem_store_datadir "/var/lib/glance/images/"

crudini --set /etc/glance/glance-registry.conf database           connection        "mysql://glance:${GLANCE_PASS}@${CONTROLLER}/glance"
crudini --set /etc/glance/glance-registry.conf keystone_authtoken auth_uri          "http://${CONTROLLER}:5000/v2.0"
crudini --set /etc/glance/glance-registry.conf keystone_authtoken identity_uri      "http://${CONTROLLER}:35357"
crudini --set /etc/glance/glance-registry.conf keystone_authtoken admin_tenant_name "service"
crudini --set /etc/glance/glance-registry.conf keystone_authtoken admin_user        "glance"
crudini --set /etc/glance/glance-registry.conf keystone_authtoken admin_password    "${GLANCE_PASS}"
crudini --set /etc/glance/glance-registry.conf paste_deploy       flavor            "keystone"

su -s /bin/sh -c "glance-manage db_sync" glance
systemctl enable openstack-glance-api.service openstack-glance-registry.service && systemctl start openstack-glance-api.service openstack-glance-registry.service
## ## Verification of Glance ## ##
mkdir -p /var/tmp/images
# CirrOS 0.3.3
wget -P /var/tmp/images http://cdn.download.cirros-cloud.net/0.3.3/cirros-0.3.3-x86_64-disk.img
glance image-create --name 'CirrOS 0.3.3 x86_64' --disk-format qcow2 --container-format bare --is-public true --progress --file /var/tmp/images/cirros-0.3.3-x86_64-disk.img 
# Fedora 20
wget -P /var/tmp/images http://cloud.fedoraproject.org/fedora-20.x86_64.qcow2
glance image-create --name 'Fedora 20 x86_64'    --disk-format qcow2 --container-format bare --is-public true --progress --file /var/tmp/images/fedora-20.x86_64.qcow2
# CentOS 7
wget -P /var/tmp/images http://cloud.centos.org/centos/7/devel/CentOS-7-x86_64-GenericCloud.qcow2
glance image-create --name 'CentOS  7 x86_64'    --disk-format qcow2 --container-format bare --is-public true --progress --file /var/tmp/images/CentOS-7-x86_64-GenericCloud.qcow2
# CoreOS Beta
wget -P /var/tmp/images http://beta.release.core-os.net/amd64-usr/current/coreos_production_openstack_image.img.bz2 && bunzip2 /var/tmp/imgaes/coreos_production_openstack_image.img.bz2
glance image-create --name 'CoreOS Beta x86_64'  --disk-format qcow2 --container-format bare --is-public true --progress --file /var/tmp/images/coreos_production_openstack_image.img
# Fedora 21
wget -P /var/tmp/images http://download.fedoraproject.org/pub/fedora/linux/releases/21/Cloud/Images/x86_64/Fedora-Cloud-Base-20141203-21.x86_64.qcow2
glance image-create --name 'Fedora 21 x86_64'    --disk-format qcow2 --container-format bare --is-public true --progress --file /var/tmp/images/Fedora-Cloud-Base-20141203-21.x86_64.qcow2
# Fedora 21 for containers
wget -P /var/tmp/images http://download.fedoraproject.org/pub/fedora/linux/releases/21/Cloud/Images/x86_64/Fedora-Cloud-Atomic-20141203-21.x86_64.qcow2
glance image-create --name 'Fedora 21Atomic x86_64'    --disk-format qcow2 --container-format bare --is-public true --progress --file /var/tmp/images/Fedora-Cloud-Atomic-20141203-21.x86_64.qcow2

glance image-list
rm -Rf /var/tmp/images

##############################################################################################################################################################
### # Nova (Compute Service) - normally on compute nodes, with controller pieces
# these steps are done on controller 
SERV="nova"
SERV_DBPASS=${NOVA_DBPASS}
SERV_PASS=${NOVA_PASS}
SERV_PORT='8774/v2/%(tenant_id)s'
SERV_TENANT="service"
SERV_TYPE="compute"
SERV_DESC="OpenStack Compute"

Q1="CREATE DATABASE ${SERV};"
Q2="GRANT ALL PRIVILEGES ON ${SERV}.* TO '${SERV}'@'localhost' IDENTIFIED BY '${SERV_DBPASS}';"
Q3="GRANT ALL PRIVILEGES ON ${SERV}.* TO '${SERV}'@'%' IDENTIFIED BY '${SERV_DBPASS}';"
SQL="${Q1}${Q2}${Q3}"
mysql -u root --password=$MYSQL_PASS -e "$SQL"

gettoken
keystone user-create     --name ${SERV} --pass   ${SERV_PASS}
keystone user-role-add   --user ${SERV} --tenant ${SERV_TENANT} --role admin
keystone service-create  --name ${SERV} --type   ${SERV_TYPE}   --description "${SERV_DESC}"
keystone endpoint-create --service-id $(keystone service-list | awk '/ compute / {print $2}') --publicurl "http://${CONTROLLER}:${SERV_PORT}" --adminurl "http://${CONTROLLER}:${SERV_PORT}" --internalurl "http://${CONTROLLER}:${SERV_PORT}" --region regionOne
## ## compute controller steps ## ##
yum -y install openstack-nova-api openstack-nova-cert openstack-nova-conductor openstack-nova-console openstack-nova-novncproxy openstack-nova-scheduler python-novaclient
crudini --set /etc/nova/nova.conf database connection    "mysql://${SERV}:${SERV_DBPASS}@${CONTROLLER}/${SERV}"
crudini --set /etc/nova/nova.conf DEFAULT  auth_strategy "keystone"
crudini --set /etc/nova/nova.conf DEFAULT keystone_authtoken admin_password     "${SERV_PASS}"
crudini --set /etc/nova/nova.conf DEFAULT keystone_authtoken admin_tenant_name  "${SERV_TENANT}"
crudini --set /etc/nova/nova.conf DEFAULT keystone_authtoken admin_user         "${SERV}"
crudini --set /etc/nova/nova.conf DEFAULT keystone_authtoken auth_uri           "http://${CONTROLLER}:5000/v2.0"
crudini --set /etc/nova/nova.conf DEFAULT keystone_authtoken identity_uri       "http://${CONTROLLER}:35357"
crudini --set /etc/nova/nova.conf DEFAULT my_ip                        "${CON_IP}"
#crudini --set /etc/nova/nova.conf DEFAULT novncproxy_base_url          "http://${CONTROLLER}:6080/vnc_auto.html"
crudini --set /etc/nova/nova.conf DEFAULT rabbit_host                  "${CONTROLLER}"
crudini --set /etc/nova/nova.conf DEFAULT rabbit_password              "${RABBIT_PASS}"
crudini --set /etc/nova/nova.conf DEFAULT rabbit_userid                "${RABBIT_USER}"
crudini --set /etc/nova/nova.conf DEFAULT rpc_backend                  "rabbit"
crudini --set /etc/nova/nova.conf DEFAULT vncserver_listen             "${CON_IP}"
crudini --set /etc/nova/nova.conf DEFAULT vncserver_proxyclient_address "${CON_IP}"
crudini --set /etc/nova/nova.conf glance  host                         "${CONTROLLER}"

su -s /bin/sh -c "nova-manage db sync" nova
systemctl enable openstack-nova-api.service openstack-nova-cert.service openstack-nova-consoleauth.service openstack-nova-scheduler.service openstack-nova-conductor.service openstack-nova-novncproxy.service
systemctl start openstack-nova-api.service openstack-nova-cert.service openstack-nova-consoleauth.service openstack-nova-scheduler.service openstack-nova-conductor.service openstack-nova-novncproxy.service
## ## compute node steps ## ##
# Set to management IP of compute node
yum -y install openstack-nova-compute sysfsutils
crudini --set /etc/nova/nova.conf DEFAULT auth_strategy                "keystone"
crudini --set /etc/nova/nova.conf DEFAULT my_ip                        "${NODE_CON_IP}"
crudini --set /etc/nova/nova.conf DEFAULT novncproxy_base_url          "http://${CONTROLLER}:6080/vnc_auto.html"
crudini --set /etc/nova/nova.conf DEFAULT rabbit_host                  "${CONTROLLER}"
crudini --set /etc/nova/nova.conf DEFAULT rabbit_password              "${RABBIT_PASS}"
crudini --set /etc/nova/nova.conf DEFAULT rabbit_userid                "${RABBIT_USER}"
crudini --set /etc/nova/nova.conf DEFAULT rpc_backend                  "rabbit"
crudini --set /etc/nova/nova.conf DEFAULT vncserver_listen             "0.0.0.0"
crudini --set /etc/nova/nova.conf DEFAULT vncserver_proxyclient_address "${NODE_CON_IP}"
crudini --set /etc/nova/nova.conf glance  host                         "${CONTROLLER}"
crudini --set /etc/nova/nova.conf keystone_authtoken admin_password    "${NOVA_PASS}"
crudini --set /etc/nova/nova.conf keystone_authtoken admin_tenant_name "service"
crudini --set /etc/nova/nova.conf keystone_authtoken admin_user        "nova"
crudini --set /etc/nova/nova.conf keystone_authtoken auth_uri          "http://${CONTROLLER}:5000/v2.0"
crudini --set /etc/nova/nova.conf keystone_authtoken identity_uri      "http://${CONTROLLER}:35357"
# if `egrep -c '(vmx|svm)' /proc/cpuinfo` == 0 >>
crudini --set /etc/nova/nova.conf libvirt virt_type "qemu"
# <<
systemctl enable libvirtd.service openstack-nova-compute.service
systemctl start libvirtd.service ; sleep 5 ; systemctl start openstack-nova-compute.service


##############################################################################################################################################################
### # Neutron (Network Service) - normally on network node(s), with controller pieces
# these steps are done on controller regardless of where glance server installed
SERV="neutron"
SERV_DBPASS=${NEUTRON_DBPASS}
SERV_PASS=${NEUTRON_PASS}
SERV_PORT=9696
SERV_TENANT="service"
SERV_TYPE="network"
SERV_DESC="OpenStack Networking"

Q1="CREATE DATABASE ${SERV};"
Q2="GRANT ALL PRIVILEGES ON ${SERV}.* TO '${SERV}'@'localhost' IDENTIFIED BY '${SERV_DBPASS}';"
Q3="GRANT ALL PRIVILEGES ON ${SERV}.* TO '${SERV}'@'%' IDENTIFIED BY '${SERV_DBPASS}';"
SQL="${Q1}${Q2}${Q3}"
mysql -u root --password=$MYSQL_PASS -e "$SQL"

gettoken
keystone user-create     --name ${SERV} --pass   ${SERV_PASS}
keystone user-role-add   --user ${SERV} --tenant ${SERV_TENANT} --role admin
keystone service-create  --name ${SERV} --type   ${SERV_TYPE}   --description "${SERV_DESC}"
keystone endpoint-create --service-id $(keystone service-list | awk '/ network / {print $2}') --publicurl http://${CONTROLLER}:${SERV_PORT} --adminurl http://${CONTROLLER}:${SERV_PORT} --internalurl http://${CONTROLLER}:${SERV_PORT} --region regionOne
## ## network controller steps ## ##
yum -y install openstack-neutron openstack-neutron-ml2 python-neutronclient which
crudini --set /etc/neutron/neutron.conf database           connection         "mysql://${SERV}:${SERV_DBPASS}@${CONTROLLER}/${SERV}"
crudini --set /etc/neutron/neutron.conf DEFAULT            auth_strategy      "keystone"
crudini --set /etc/neutron/neutron.conf keystone_authtoken admin_password     "${SERV_PASS}"
crudini --set /etc/neutron/neutron.conf keystone_authtoken admin_tenant_name  "${SERV_TENANT}"
crudini --set /etc/neutron/neutron.conf keystone_authtoken admin_user         "${SERV}"
crudini --set /etc/neutron/neutron.conf keystone_authtoken auth_uri           "http://${CONTROLLER}:5000/v2.0"
crudini --set /etc/neutron/neutron.conf keystone_authtoken identity_uri       "http://${CONTROLLER}:35357"
crudini --set /etc/neutron/neutron.conf DEFAULT allow_overlapping_ips         True
crudini --set /etc/neutron/neutron.conf DEFAULT core_plugin                   "ml2"
crudini --set /etc/neutron/neutron.conf DEFAULT notify_nova_on_port_data_changes    True
crudini --set /etc/neutron/neutron.conf DEFAULT notify_nova_on_port_status_changes  True
crudini --set /etc/neutron/neutron.conf DEFAULT nova_admin_auth_url           "http://${CONTROLLER}:35357/v2.0"
crudini --set /etc/neutron/neutron.conf DEFAULT nova_admin_password           "${NOVA_PASS}"
crudini --set /etc/neutron/neutron.conf DEFAULT nova_admin_tenant_id          "`keystone tenant-get service | awk '/ id / {print $4}'`"
crudini --set /etc/neutron/neutron.conf DEFAULT nova_admin_username           "nova"
crudini --set /etc/neutron/neutron.conf DEFAULT nova_region_name              "regionOne"
crudini --set /etc/neutron/neutron.conf DEFAULT nova_url                      "http://${CONTROLLER}:8774/v2"
crudini --set /etc/neutron/neutron.conf DEFAULT rabbit_host                   "${CONTROLLER}"
crudini --set /etc/neutron/neutron.conf DEFAULT rabbit_password               "${RABBIT_PASS}"
crudini --set /etc/neutron/neutron.conf DEFAULT rabbit_userid                 "${RABBIT_USER}"
crudini --set /etc/neutron/neutron.conf DEFAULT rpc_backend                   "rabbit"
crudini --set /etc/neutron/neutron.conf DEFAULT service_plugins               "router"

crudini --set /etc/neutron/plugins/ml2/ml2_conf.ini ml2 mechanism_drivers     "openvswitch"
crudini --set /etc/neutron/plugins/ml2/ml2_conf.ini ml2 tenant_network_types  "gre"
crudini --set /etc/neutron/plugins/ml2/ml2_conf.ini ml2 type_drivers          "flat,gre"
crudini --set /etc/neutron/plugins/ml2/ml2_conf.ini ml2_type_gre  tunnel_id_ranges      "1:1000"
crudini --set /etc/neutron/plugins/ml2/ml2_conf.ini securitygroup enable_ipset          True
crudini --set /etc/neutron/plugins/ml2/ml2_conf.ini securitygroup enable_security_group True
crudini --set /etc/neutron/plugins/ml2/ml2_conf.ini securitygroup firewall_driver       "neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver"

crudini --set /etc/nova/nova.conf DEFAULT network_api_class  "nova.network.neutronv2.api.API"
crudini --set /etc/nova/nova.conf DEFAULT security_group_api "neutron"
crudini --set /etc/nova/nova.conf DEFAULT linuxnet_interface_driver "nova.network.linux_net.LinuxOVSInterfaceDriver"
crudini --set /etc/nova/nova.conf DEFAULT firewall_driver    "nova.virt.firewall.NoopFirewallDriver"
crudini --set /etc/nova/nova.conf neutron admin_password     "${NEUTRON_PASS}"
crudini --set /etc/nova/nova.conf neutron admin_tenant_name  "service"
crudini --set /etc/nova/nova.conf neutron admin_user         "neutron"
crudini --set /etc/nova/nova.conf neutron admin_auth_uri     "http://${CONTROLLER}:5000/v2.0"
crudini --set /etc/nova/nova.conf neutron url                "http://${CONTROLLER}:9696"
crudini --set /etc/nova/nova.conf neutron service_metadata_proxy True
crudini --set /etc/nova/nova.conf neutron metadata_proxy_shared_secret "${METADATA_SECRET}"

ln -s /etc/neutron/plugins/ml2/ml2_conf.ini /etc/neutron/plugin.ini
su -s /bin/sh -c "neutron-db-manage --config-file /etc/neutron/neutron.conf --config-file /etc/neutron/plugins/ml2/ml2_conf.ini upgrade juno" neutron
systemctl restart openstack-nova-api.service openstack-nova-scheduler.service openstack-nova-conductor.service
systemctl enable neutron-server.service && systemctl start neutron-server.service

## ## Verification of Neutron controller portion ## ##
neutron ext-list

## ## network node steps ## ##
cat >> /etc/sysctl.conf <<END
net.ipv4.ip_forward=1
net.ipv4.conf.all.rp_filter=0
net.ipv4.conf.default.rp_filter=0
END
sysctl -p
yum -y install openstack-neutron openstack-neutron-ml2 openstack-neutron-openvswitch
crudini --set /etc/neutron/neutron.conf DEFAULT rabbit_host     "${CONTROLLER}"
crudini --set /etc/neutron/neutron.conf DEFAULT rabbit_password "${RABBIT_PASS}"
crudini --set /etc/neutron/neutron.conf DEFAULT rabbit_userid   "${RABBIT_USER}"
crudini --set /etc/neutron/neutron.conf DEFAULT rpc_backend     "rabbit"
crudini --set /etc/neutron/neutron.conf DEFAULT auth_strategy   "keystone"
crudini --set /etc/neutron/neutron.conf keystone_authtoken admin_password     "${NEUTRON_PASS}"
crudini --set /etc/neutron/neutron.conf keystone_authtoken admin_tenant_name  "service"
crudini --set /etc/neutron/neutron.conf keystone_authtoken admin_user         "neutron"
crudini --set /etc/neutron/neutron.conf keystone_authtoken auth_uri           "http://${CONTROLLER}:5000/v2.0"
crudini --set /etc/neutron/neutron.conf keystone_authtoken identity_uri       "http://${CONTROLLER}:35357"
crudini --set /etc/neutron/neutron.conf DEFAULT allow_overlapping_ips         True
crudini --set /etc/neutron/neutron.conf DEFAULT core_plugin                   "ml2"
crudini --set /etc/neutron/neutron.conf DEFAULT service_plugins               "router"

crudini --set /etc/neutron/plugins/ml2/ml2_conf.ini ml2 mechanism_drivers     "openvswitch"
crudini --set /etc/neutron/plugins/ml2/ml2_conf.ini ml2 tenant_network_types  "gre"
crudini --set /etc/neutron/plugins/ml2/ml2_conf.ini ml2 type_drivers          "flat,gre"
crudini --set /etc/neutron/plugins/ml2/ml2_conf.ini ml2_type_flat flat_networks         "external"
crudini --set /etc/neutron/plugins/ml2/ml2_conf.ini ml2_type_gre  tunnel_id_ranges      "1:1000"
crudini --set /etc/neutron/plugins/ml2/ml2_conf.ini securitygroup enable_ipset          True
crudini --set /etc/neutron/plugins/ml2/ml2_conf.ini securitygroup enable_security_group True
crudini --set /etc/neutron/plugins/ml2/ml2_conf.ini securitygroup firewall_driver       "neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver"
crudini --set /etc/neutron/plugins/ml2/ml2_conf.ini ovs local_ip "INSTANCE_TUNNELS_INTERFACE_IP_ADDRESS"
crudini --set /etc/neutron/plugins/ml2/ml2_conf.ini ovs tunnel_type "gre"
crudini --set /etc/neutron/plugins/ml2/ml2_conf.ini ovs enable_tunneling True
crudini --set /etc/neutron/plugins/ml2/ml2_conf.ini ovs bridge_mappings "external:br-ex"

crudini --set /etc/neutron/l3_agent.ini DEFAULT interface_driver "neutron.agent.linux.interface.OVSInterfaceDriver"
crudini --set /etc/neutron/l3_agent.ini DEFAULT use_namespaces True
crudini --set /etc/neutron/l3_agent.ini DEFAULT external_network_bridge "br-ex"

crudini --set /etc/neutron/dhcp_agent.ini DEFAULT interface_driver "neutron.agent.linux.interface.OVSInterfaceDriver"
crudini --set /etc/neutron/dhcp_agent.ini DEFAULT dhcp_driver "neutron.agent.linux.dhcp.Dnsmasq"
crudini --set /etc/neutron/dhcp_agent.ini DEFAULT use_namespaces True
crudini --set /etc/neutron/dhcp_agent.ini DEFAULT dnsmasq_config_file "/etc/neutron/dnsmasq-neutron.conf"
cat >> /etc/neutron/dnsmasq-neutron.conf <<END
dhcp-option-force=26,1454
END

crudini --set /etc/neutron/metadata_agent.ini DEFAULT auth_region        "regionOne"
crudini --set /etc/neutron/metadata_agent.ini DEFAULT admin_password     "${NEUTRON_PASS}"
crudini --set /etc/neutron/metadata_agent.ini DEFAULT admin_tenant_name  "service"
crudini --set /etc/neutron/metadata_agent.ini DEFAULT admin_user         "neutron"
crudini --set /etc/neutron/metadata_agent.ini DEFAULT auth_url           "http://${CONTROLLER}:5000/v2.0"
crudini --set /etc/neutron/metadata_agent.ini DEFAULT nova_metadata_ip   "${CONTROLLER}"
crudini --set /etc/neutron/metadata_agent.ini DEFAULT metadata_proxy_shared_secret "${METADATA_SECRET}"

systemctl enable openvswitch.service && systemctl start openvswitch.service
ovs-vsctl add-br br-ex
ovs-vsctl add-port br-ex eth1
ln -s /etc/neutron/plugins/ml2/ml2_conf.ini /etc/neutron/plugin.ini
cp /usr/lib/systemd/system/neutron-openvswitch-agent.service /usr/lib/systemd/system/neutron-openvswitch-agent.service.orig
sed -i 's,plugins/openvswitch/ovs_neutron_plugin.ini,plugin.ini,g' /usr/lib/systemd/system/neutron-openvswitch-agent.service

systemctl enable neutron-openvswitch-agent.service neutron-l3-agent.service neutron-dhcp-agent.service neutron-metadata-agent.service neutron-ovs-cleanup.service
systemctl start neutron-openvswitch-agent.service neutron-l3-agent.service neutron-dhcp-agent.service neutron-metadata-agent.service

## ## Verification of Neutron (execute on controller) ## ##
neutron agent-list | grep <NETWORK SERVER NAME>

## ## compute node steps ## ##
cat >> /etc/sysctl.conf <<END
net.ipv4.conf.all.rp_filter=0
net.ipv4.conf.default.rp_filter=0
END
sysctl -p
yum -y install openstack-neutron-ml2 openstack-neutron-openvswitch
crudini --set /etc/neutron/neutron.conf DEFAULT rabbit_host     "${CONTROLLER}"
crudini --set /etc/neutron/neutron.conf DEFAULT rabbit_password "${RABBIT_PASS}"
crudini --set /etc/neutron/neutron.conf DEFAULT rabbit_userid   "${RABBIT_USER}"
crudini --set /etc/neutron/neutron.conf DEFAULT rpc_backend     "rabbit"
crudini --set /etc/neutron/neutron.conf DEFAULT auth_strategy   "keystone"
crudini --set /etc/neutron/neutron.conf keystone_authtoken admin_password     "${NEUTRON_PASS}"
crudini --set /etc/neutron/neutron.conf keystone_authtoken admin_tenant_name  "service"
crudini --set /etc/neutron/neutron.conf keystone_authtoken admin_user         "neutron"
crudini --set /etc/neutron/neutron.conf keystone_authtoken auth_uri           "http://${CONTROLLER}:5000/v2.0"
crudini --set /etc/neutron/neutron.conf keystone_authtoken identity_uri       "http://${CONTROLLER}:35357"
crudini --set /etc/neutron/neutron.conf DEFAULT allow_overlapping_ips         True
crudini --set /etc/neutron/neutron.conf DEFAULT core_plugin                   "ml2"
crudini --set /etc/neutron/neutron.conf DEFAULT service_plugins               "router"

crudini --set /etc/neutron/plugins/ml2/ml2_conf.ini ml2 mechanism_drivers     "openvswitch"
crudini --set /etc/neutron/plugins/ml2/ml2_conf.ini ml2 tenant_network_types  "gre"
crudini --set /etc/neutron/plugins/ml2/ml2_conf.ini ml2 type_drivers          "flat,gre"
crudini --set /etc/neutron/plugins/ml2/ml2_conf.ini ml2_type_gre  tunnel_id_ranges      "1:1000"
crudini --set /etc/neutron/plugins/ml2/ml2_conf.ini securitygroup enable_ipset          True
crudini --set /etc/neutron/plugins/ml2/ml2_conf.ini securitygroup enable_security_group True
crudini --set /etc/neutron/plugins/ml2/ml2_conf.ini securitygroup firewall_driver       "neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver"
crudini --set /etc/neutron/plugins/ml2/ml2_conf.ini ovs local_ip "INSTANCE_TUNNELS_INTERFACE_IP_ADDRESS"
crudini --set /etc/neutron/plugins/ml2/ml2_conf.ini ovs tunnel_type "gre"
crudini --set /etc/neutron/plugins/ml2/ml2_conf.ini ovs enable_tunneling True

crudini --set /etc/nova/nova.conf DEFAULT network_api_class  "nova.network.neutronv2.api.API"
crudini --set /etc/nova/nova.conf DEFAULT security_group_api "neutron"
crudini --set /etc/nova/nova.conf DEFAULT linuxnet_interface_driver "nova.network.linux_net.LinuxOVSInterfaceDriver"
crudini --set /etc/nova/nova.conf DEFAULT firewall_driver    "nova.virt.firewall.NoopFirewallDriver"
crudini --set /etc/nova/nova.conf DEFAULT neutron_url               "http://${CONTROLLER}:9696"
crudini --set /etc/nova/nova.conf DEFAULT neutron_admin_username    "neutron"
crudini --set /etc/nova/nova.conf DEFAULT neutron_admin_password    "${NEUTRON_PASS}"
crudini --set /etc/nova/nova.conf DEFAULT neutron_admin_tenant_name "service"
crudini --set /etc/nova/nova.conf DEFAULT neutron_admin_auth_url    "http://${CONTROLLER}:5000/v2.0"
crudini --set /etc/nova/nova.conf DEFAULT neutron_auth_strategy     "keystone"

systemctl enable openvswitch.service && systemctl start openvswitch.service

ln -s /etc/neutron/plugins/ml2/ml2_conf.ini /etc/neutron/plugin.ini
cp /usr/lib/systemd/system/neutron-openvswitch-agent.service /usr/lib/systemd/system/neutron-openvswitch-agent.service.orig
sed -i 's,plugins/openvswitch/ovs_neutron_plugin.ini,plugin.ini,g' /usr/lib/systemd/system/neutron-openvswitch-agent.service
systemctl restart openstack-nova-compute.service && systemctl enable neutron-openvswitch-agent.service && systemctl start neutron-openvswitch-agent.service

## ## Verification of Neutron (execute on controller) ## ##
neutron agent-list | grep <COMPUTE NODE NAME>

## ## network setup steps ## ##
# on the controller:
neutron net-create ext-net --shared --router:external True --provider:physical_network external --provider:network_type flat
neutron subnet-create ext-net --name ext-subnet --allocation-pool start=${FLOATING_IP_START},end=${FLOATING_IP_END} --disable-dhcp --gateway $EXTERNAL_NETWORK_GATEWAY $EXTERNAL_NETWORK_CIDR
source /root/bin/demo-openrc.sh
neutron net-create demo-net
neutron subnet-create demo-net --name demo-subnet --gateway $TENANT_NETWORK_GATEWAY $TENANT_NETWORK_CIDR
neutron router-create demo-router
neutron router-interface-add demo-router demo-subnet
neutron router-gateway-set demo-router ext-net

##############################################################################################################################################################
### ### At this point, the core OpenStack environment is ready, the rest is "optional"
##############################################################################################################################################################

##############################################################################################################################################################
### # Horizon (Dashboard Service) - Could be on controller or a utility node
yum -y install openstack-dashboard httpd mod_wsgi memcached python-memcached
/etc/openstack-dashboard/local_settings 
sed -i "s/^ALLOWED_HOSTS.*$/ALLOWED_HOSTS = \['\*'\]/g" /etc/openstack-dashboard/local_settings
sed -i "s/^OPENSTACK_HOST.*$/OPENSTACK_HOST = \"${CONTROLLER}\"/g" /etc/openstack-dashboard/local_settings
sed -i "s/^TIME_ZONE.*$/TIME_ZONE = \"EST\"/g" /etc/openstack-dashboard/local_settings
sed -i "s/_member_/admin/g" /etc/openstack-dashboard/local_settings
sed -i "s/django.core.cache.backends.locmem.LocMemCache/django.core.cache.backends.memcached.MemcachedCache/g" /etc/openstack-dashboard/local_settings
# find the above line, and append this >>
,
       'LOCATION': '127.0.0.1:11211',
# <<
setsebool -P httpd_can_network_connect on
chown -R apache:apache /usr/share/openstack-dashboard/static
systemctl enable httpd.service memcached.service && systemctl start httpd.service memcached.service


##############################################################################################################################################################
### # Cinder (Storage Service) - storage node with controller pieces
# these steps are done on controller regardless of where glance server installed
SERV="cinder"
SERV_DBPASS=${CINDER_DBPASS}
SERV_PASS=${CINDER_PASS}
SERV_PORT='8776/v1/%(tenant_id)s'
SERV_PORT2='8776/v2/%(tenant_id)s'
SERV_TENANT="service"
SERV_TYPE="volume"
SERV_DESC="OpenStack Block Storage"

Q1="CREATE DATABASE ${SERV};"
Q2="GRANT ALL PRIVILEGES ON ${SERV}.* TO '${SERV}'@'localhost' IDENTIFIED BY '${SERV_DBPASS}';"
Q3="GRANT ALL PRIVILEGES ON ${SERV}.* TO '${SERV}'@'%' IDENTIFIED BY '${SERV_DBPASS}';"
SQL="${Q1}${Q2}${Q3}"
mysql -u root --password=$MYSQL_PASS -e "$SQL"

gettoken
keystone user-create     --name ${SERV} --pass   ${SERV_PASS}
keystone user-role-add   --user ${SERV} --tenant ${SERV_TENANT} --role admin
keystone service-create  --name ${SERV} --type   ${SERV_TYPE}   --description "${SERV_DESC}"
keystone service-create  --name "${SERV}v2" --type   "${SERV_TYPE}v2"   --description "${SERV_DESC} V2"
keystone endpoint-create --service-id $(keystone service-list | awk '/ volume / {print $2}') --publicurl http://${CONTROLLER}:${SERV_PORT} --adminurl http://${CONTROLLER}:${SERV_PORT} --internalurl http://${CONTROLLER}:${SERV_PORT} --region regionOne
keystone endpoint-create --service-id $(keystone service-list | awk '/ volumev2 / {print $2}') --publicurl http://${CONTROLLER}:${SERV_PORT2} --adminurl http://${CONTROLLER}:${SERV_PORT2} --internalurl http://${CONTROLLER}:${SERV_PORT2} --region regionOne
yum -y install openstack-cinder python-cinderclient python-oslo-db
crudini --set /etc/cinder/cinder.conf database           connection         "mysql://${SERV}:${SERV_DBPASS}@${CONTROLLER}/${SERV}"
crudini --set /etc/cinder/cinder.conf DEFAULT rabbit_host     "${CONTROLLER}"
crudini --set /etc/cinder/cinder.conf DEFAULT rabbit_password "${RABBIT_PASS}"
crudini --set /etc/cinder/cinder.conf DEFAULT rabbit_userid   "${RABBIT_USER}"
crudini --set /etc/cinder/cinder.conf DEFAULT rpc_backend     "rabbit"
crudini --set /etc/cinder/cinder.conf DEFAULT            auth_strategy      "keystone"
crudini --set /etc/cinder/cinder.conf keystone_authtoken admin_password     "${SERV_PASS}"
crudini --set /etc/cinder/cinder.conf keystone_authtoken admin_tenant_name  "${SERV_TENANT}"
crudini --set /etc/cinder/cinder.conf keystone_authtoken admin_user         "${SERV}"
crudini --set /etc/cinder/cinder.conf keystone_authtoken auth_uri           "http://${CONTROLLER}:5000/v2.0"
crudini --set /etc/cinder/cinder.conf keystone_authtoken identity_uri       "http://${CONTROLLER}:35357"
crudini --set  /etc/cinder/cinder.conf DEFAULT my_ip "${CON_IP}" 

su -s /bin/sh -c "cinder-manage db sync" cinder
systemctl enable openstack-cinder-api.service openstack-cinder-scheduler.service && systemctl start openstack-cinder-api.service openstack-cinder-scheduler.service

## ## storage node steps ## ##
# create 'cinder-volume' volume group
DISK='/dev/sda'
VGROUP='cinder-volume'
pvcreate $DISK
vgcreate $VGROUP $DISK

# in /etc/lvm/lvm.conf, modife the devices { filter =[] } value, example:
filter = [ "a/sda/", "a/sdb/", "a/sdc/", "a/sdd/", "a/sde/", "r/.*/" ]
# do this on the compute nodes as well:
filter = [ "a/sda/", "a/sdb/", "r/.*/" ]

yum -y install openstack-cinder targetcli python-oslo-db MySQL-python
crudini --set /etc/cinder/cinder.conf database           connection         "mysql://${SERV}:${SERV_DBPASS}@${CONTROLLER}/${SERV}"
crudini --set /etc/cinder/cinder.conf DEFAULT rabbit_host     "${CONTROLLER}"
crudini --set /etc/cinder/cinder.conf DEFAULT rabbit_password "${RABBIT_PASS}"
crudini --set /etc/cinder/cinder.conf DEFAULT rabbit_userid   "${RABBIT_USER}"
crudini --set /etc/cinder/cinder.conf DEFAULT rpc_backend     "rabbit"
crudini --set /etc/cinder/cinder.conf DEFAULT            auth_strategy      "keystone"
crudini --set /etc/cinder/cinder.conf keystone_authtoken admin_password     "${SERV_PASS}"
crudini --set /etc/cinder/cinder.conf keystone_authtoken admin_tenant_name  "${SERV_TENANT}"
crudini --set /etc/cinder/cinder.conf keystone_authtoken admin_user         "${SERV}"
crudini --set /etc/cinder/cinder.conf keystone_authtoken auth_uri           "http://${CONTROLLER}:5000/v2.0"
crudini --set /etc/cinder/cinder.conf keystone_authtoken identity_uri       "http://${CONTROLLER}:35357"
crudini --set  /etc/cinder/cinder.conf DEFAULT my_ip "${NODE_CON_IP}" 
crudini --set /etc/cinder/cinder.conf DEFAULT glance_host "${CONTROLLER}"
crudini --set /etc/cinder/cinder.conf DEFAULT iscsi_helper "lioadm"
crudini --set /etc/cinder/cinder.conf DEFAULT volume_group "${VGROUP}"

systemctl enable openstack-cinder-volume.service target.service && systemctl start openstack-cinder-volume.service target.service

##############################################################################################################################################################
### # Heat (Orchestration Service) - on the controller node
SERV="heat"
SERV_DBPASS=${HEAT_DBPASS}
SERV_PASS=${HEAT_PASS}
SERV_PORT='8004/v1/%(tenant_id)s'
SERV_PORT_CFM='8000/v1'
SERV_TENANT="service"
SERV_TYPE="orchestration"
SERV_DESC="Orchestration"

Q1="CREATE DATABASE ${SERV};"
Q2="GRANT ALL PRIVILEGES ON ${SERV}.* TO '${SERV}'@'localhost' IDENTIFIED BY '${SERV_DBPASS}';"
Q3="GRANT ALL PRIVILEGES ON ${SERV}.* TO '${SERV}'@'%' IDENTIFIED BY '${SERV_DBPASS}';"
SQL="${Q1}${Q2}${Q3}"
mysql -u root --password=$MYSQL_PASS -e "$SQL"

gettoken
keystone user-create     --name ${SERV}  --pass   ${SERV_PASS}
keystone user-role-add   --user ${SERV}  --tenant ${SERV_TENANT} --role admin
keystone role-create     --name heat_stack_user
keystone role-create     --name heat_stack_owner
keystone service-create  --name ${SERV}  --type   ${SERV_TYPE}   --description "${SERV_DESC}"
keystone service-create  --name heat-cfn --type   cloudformation --description "${SERV_DESC} cfn"
keystone endpoint-create --service-id $(keystone service-list | awk '/ orchestration / {print $2}') --publicurl http://${CONTROLLER}:${SERV_PORT} --adminurl http://${CONTROLLER}:${SERV_PORT} --internalurl http://${CONTROLLER}:${SERV_PORT} --region regionOne
keystone endpoint-create --service-id $(keystone service-list | awk '/ cloudformation / {print $2}') --publicurl http://${CONTROLLER}:${SERV_PORT_CFM} --adminurl http://${CONTROLLER}:${SERV_PORT_CFM} --internalurl http://${CONTROLLER}:${SERV_PORT_CFM} --region regionOne
yum -y install openstack-heat-api openstack-heat-api-cfn openstack-heat-engine python-heatclient
crudini --set /etc/heat/heat.conf database           connection         "mysql://${SERV}:${SERV_DBPASS}@${CONTROLLER}/${SERV}"
crudini --set /etc/heat/heat.conf DEFAULT rabbit_host     "${CONTROLLER}"
crudini --set /etc/heat/heat.conf DEFAULT rabbit_password "${RABBIT_PASS}"
crudini --set /etc/heat/heat.conf DEFAULT rabbit_userid   "${RABBIT_USER}"
crudini --set /etc/heat/heat.conf DEFAULT rpc_backend     "rabbit"
crudini --set /etc/heat/heat.conf DEFAULT            auth_strategy      "keystone"
crudini --set /etc/heat/heat.conf keystone_authtoken admin_password     "${SERV_PASS}"
crudini --set /etc/heat/heat.conf keystone_authtoken admin_tenant_name  "${SERV_TENANT}"
crudini --set /etc/heat/heat.conf keystone_authtoken admin_user         "${SERV}"
crudini --set /etc/heat/heat.conf keystone_authtoken auth_uri           "http://${CONTROLLER}:5000/v2.0"
crudini --set /etc/heat/heat.conf keystone_authtoken identity_uri       "http://${CONTROLLER}:35357"
crudini --set /etc/heat/heat.conf DEFAULT heat_metadata_server_url      "http://${CONTROLLER}:8000"
crudini --set /etc/heat/heat.conf DEFAULT heat_waitcondition_server_url "http://${CONTROLLER}:8000/v1/waitcondition"

su -s /bin/sh -c "heat-manage db_sync" heat
systemctl enable openstack-heat-api.service openstack-heat-api-cfn.service openstack-heat-engine.service
systemctl start openstack-heat-api.service openstack-heat-api-cfn.service openstack-heat-engine.service


##############################################################################################################################################################
### # Celiometer (Orchestration Service) - on each node

yum -y install mongodb-server mongodb
sed -i "s/bind_ip = 127.0.0.1/bind_ip = ${CONTROLLER}/g" /etc/mongodb.conf
# This is optionsl >>
sed -i '5i\smallfiles = true\' /etc/mongodb.conf
# <<

systemctl enable mongod.service && systemctl start mongod.service

mongo --host ${CONTROLLER} --eval "
  db = db.getSiblingDB(\"ceilometer\");
  db.addUser({user: \"ceilometer\",
  pwd: \"${CEILOMETER_DBPASS}\",
  roles: [ \"readWrite\", \"dbAdmin\" ]})"

MSECRET=`openssl rand -hex 10`
SERV="ceilometer"
SERV_DBPASS=${CEILOMETER_DBPASS}
SERV_PASS=${CEILOMETER_PASS}
SERV_PORT=8777
SERV_TENANT="service"
SERV_TYPE="metering"
SERV_DESC="Telemetry"

gettoken
#source /root/bin/admin-openrc.sh
keystone user-create     --name ${SERV} --pass   ${SERV_PASS}
keystone user-role-add   --user ${SERV} --tenant ${SERV_TENANT} --role admin
keystone service-create  --name ${SERV} --type   ${SERV_TYPE}   --description "${SERV_DESC}"
keystone endpoint-create --service-id $(keystone service-list | awk '/ metering / {print $2}') --publicurl http://${CONTROLLER}:${SERV_PORT} --adminurl http://${CONTROLLER}:${SERV_PORT} --internalurl http://${CONTROLLER}:${SERV_PORT} --region regionOne
yum -y install openstack-ceilometer-api openstack-ceilometer-collector openstack-ceilometer-notification openstack-ceilometer-central openstack-ceilometer-alarm python-ceilometerclient
crudini --set /etc/ceilometer/ceilometer.conf database connection     "mongodb://${SERV}:${SERV_DBPASS}@${CONTROLLER}:27017/${SERV}" 
crudini --set /etc/ceilometer/ceilometer.conf DEFAULT rabbit_host     "${CONTROLLER}" 
crudini --set /etc/ceilometer/ceilometer.conf DEFAULT rabbit_password "${RABBIT_PASS}" 
crudini --set /etc/ceilometer/ceilometer.conf DEFAULT rabbit_userid   "${RABBIT_USER}" 
crudini --set /etc/ceilometer/ceilometer.conf DEFAULT rpc_backend     "rabbit" 
crudini --set /etc/ceilometer/ceilometer.conf DEFAULT auth_strategy   "keystone" 
crudini --set /etc/ceilometer/ceilometer.conf keystone_authtoken admin_password     "${SERV_PASS}" 
crudini --set /etc/ceilometer/ceilometer.conf keystone_authtoken admin_tenant_name  "${SERV_TENANT}" 
crudini --set /etc/ceilometer/ceilometer.conf keystone_authtoken admin_user         "${SERV}" 
crudini --set /etc/ceilometer/ceilometer.conf keystone_authtoken auth_uri           "http://${CONTROLLER}:5000/v2.0" 
crudini --set /etc/ceilometer/ceilometer.conf keystone_authtoken identity_uri       "http://${CONTROLLER}:35357" 
crudini --set /etc/ceilometer/ceilometer.conf service_credentials os_auth_url    "http://${CONTROLLER}:5000/v2.0"
crudini --set /etc/ceilometer/ceilometer.conf service_credentials os_username    "ceilometer"
crudini --set /etc/ceilometer/ceilometer.conf service_credentials os_tenant_name "service"
crudini --set /etc/ceilometer/ceilometer.conf service_credentials os_password    "${CEILOMETER_PASS}"
crudini --set /etc/ceilometer/ceilometer.conf publisher           metering_secret "${MSECRET}"
echo "${MSECRET}" >~/bin/ceilometer-secret

systemctl enable openstack-ceilometer-api.service openstack-ceilometer-notification.service openstack-ceilometer-central.service openstack-ceilometer-collector.service openstack-ceilometer-alarm-evaluator.service openstack-ceilometer-alarm-notifier.service
systemctl start openstack-ceilometer-api.service openstack-ceilometer-notification.service openstack-ceilometer-central.service openstack-ceilometer-collector.service openstack-ceilometer-alarm-evaluator.service openstack-ceilometer-alarm-notifier.service

## ## compute node steps ## ##
yum -y install openstack-ceilometer-compute python-ceilometerclient python-pecan
crudini --set /etc/nova/nova.conf DEFAULT instance_usage_audit        "True"
crudini --set /etc/nova/nova.conf DEFAULT notification_driver         "nova.openstack.common.notifier.rpc_notifier"
crudini --set /etc/nova/nova.conf DEFAULT notification_driver         "ceilometer.compute.nova_notifier"
crudini --set /etc/nova/nova.conf DEFAULT instance_usage_audit_period "hour"
crudini --set /etc/nova/nova.conf DEFAULT notify_on_state_change      "vm_and_task_state"
systemctl restart openstack-nova-compute.service
crudini --set /etc/ceilometer/ceilometer.conf DEFAULT rabbit_host     "${CONTROLLER}" 
crudini --set /etc/ceilometer/ceilometer.conf DEFAULT rabbit_password "${RABBIT_PASS}" 
crudini --set /etc/ceilometer/ceilometer.conf DEFAULT rabbit_userid   "${RABBIT_USER}" 
crudini --set /etc/ceilometer/ceilometer.conf DEFAULT rpc_backend     "rabbit" 
crudini --set /etc/ceilometer/ceilometer.conf DEFAULT auth_strategy   "keystone" 
crudini --set /etc/ceilometer/ceilometer.conf keystone_authtoken admin_password     "${SERV_PASS}" 
crudini --set /etc/ceilometer/ceilometer.conf keystone_authtoken admin_tenant_name  "${SERV_TENANT}" 
crudini --set /etc/ceilometer/ceilometer.conf keystone_authtoken admin_user         "${SERV}" 
crudini --set /etc/ceilometer/ceilometer.conf keystone_authtoken auth_uri           "http://${CONTROLLER}:5000/v2.0" 
crudini --set /etc/ceilometer/ceilometer.conf keystone_authtoken identity_uri       "http://${CONTROLLER}:35357" 
crudini --set /etc/ceilometer/ceilometer.conf service_credentials os_auth_url    "http://${CONTROLLER}:5000/v2.0"
crudini --set /etc/ceilometer/ceilometer.conf service_credentials os_username    "ceilometer"
crudini --set /etc/ceilometer/ceilometer.conf service_credentials os_tenant_name "service"
crudini --set /etc/ceilometer/ceilometer.conf service_credentials os_password    "${CEILOMETER_PASS}"
crudini --set /etc/ceilometer/ceilometer.conf service_credentials os_endpoint_type "internalURL"
systemctl enable openstack-ceilometer-compute.service && systemctl start openstack-ceilometer-compute.service

## ## image service steps ## ##
crudini --set /etc/glance/glance-api.conf DEFAULT  notification_driver "messaging"
crudini --set /etc/glance/glance-api.conf DEFAULT  rabbit_host     "${CONTROLLER}"
crudini --set /etc/glance/glance-api.conf DEFAULT  rabbit_password "${RABBIT_PASS}"
crudini --set /etc/glance/glance-api.conf DEFAULT  rabbit_userid   "${RABBIT_USER}"
crudini --set /etc/glance/glance-api.conf DEFAULT  rpc_backend     "rabbit"
systemctl restart openstack-glance-api.service openstack-glance-registry.service

## ## block storage service steps (both controller and storage node) ## ##
crudini --set /etc/cinder/cinder.conf DEFAULT control_exchange    "cinder"
crudini --set /etc/cinder/cinder.conf DEFAULT notification_driver "cinder.openstack.common.notifier.rpc_notifier"
# controller
systemctl restart openstack-cinder-api.service openstack-cinder-scheduler.service
# storage
systemctl restart openstack-cinder-volume.service


##############################################################################################################################################################
### # Trove (Database Service) - on controller node
SERV="trove"
SERV_DBPASS=${TROVE_DBPASS}
SERV_PASS=${TROVE_PASS}
SERV_PORT='8779/v1.0/%(tenant_id)s'
SERV_TENANT="service"
SERV_TYPE="database"
SERV_DESC="OpenStack Database Service"

Q1="CREATE DATABASE ${SERV};"
Q2="GRANT ALL PRIVILEGES ON ${SERV}.* TO '${SERV}'@'localhost' IDENTIFIED BY '${SERV_DBPASS}';"
Q3="GRANT ALL PRIVILEGES ON ${SERV}.* TO '${SERV}'@'%' IDENTIFIED BY '${SERV_DBPASS}';"
SQL="${Q1}${Q2}${Q3}"
mysql -u root --password=$MYSQL_PASS -e "$SQL"

gettoken
#source /root/bin/admin-openrc.sh
keystone user-create     --name ${SERV} --pass   ${SERV_PASS}
keystone user-role-add   --user ${SERV} --tenant ${SERV_TENANT} --role admin
keystone service-create  --name ${SERV} --type   ${SERV_TYPE}   --description "${SERV_DESC}"
keystone endpoint-create --service-id $(keystone service-list | awk '/ database / {print $2}') --publicurl http://${CONTROLLER}:${SERV_PORT} --adminurl http://${CONTROLLER}:${SERV_PORT} --internalurl http://${CONTROLLER}:${SERV_PORT} --region regionOne
yum -y install openstack-trove python-troveclient
crudini --set  database connection      "mysql://${SERV}:${SERV_DBPASS}@${CONTROLLER}/${SERV}"

# /etc/trove/trove.conf
crudini --set /etc/trove/trove.conf DEFAULT  rabbit_host     "${CONTROLLER}"
crudini --set /etc/trove/trove.conf DEFAULT  rabbit_password "${RABBIT_PASS}"
crudini --set /etc/trove/trove.conf DEFAULT  rabbit_userid   "${RABBIT_USER}"
crudini --set /etc/trove/trove.conf DEFAULT  log_dir                 '/var/log/trove'
crudini --set /etc/trove/trove.conf DEFAULT  trove_auth_url          "http://${CONTROLLER}:5000/v2.0"
crudini --set /etc/trove/trove.conf DEFAULT  nova_compute_url        "http://${CONTROLLER}:8774/v2"
crudini --set /etc/trove/trove.conf DEFAULT  cinder_url              "http://${CONTROLLER}:8776/v1"
crudini --set /etc/trove/trove.conf DEFAULT  swift_url               "http://${CONTROLLER}:8080/v1/AUTH_"
crudini --set /etc/trove/trove.conf DEFAULT  sql_connection          "mysql://${SERV}:${SERV_DBPASS}@${CONTROLLER}/trove"
crudini --set /etc/trove/trove.conf DEFAULT  notifier_queue_hostname "${CONTROLLER}"
crudini --set /etc/trove/trove.conf DEFAULT  default_datastore       'mysql'
crudini --set /etc/trove/trove.conf DEFAULT  add_addresses           'True'
crudini --set /etc/trove/trove.conf DEFAULT  network_label_regex     '^NETWORK_LABEL$'
crudini --set /etc/trove/trove.conf DEFAULT  auth_strategy   "keystone"
crudini --set /etc/trove/trove.conf keystone_authtoken admin_password     "${SERV_PASS}"
crudini --set /etc/trove/trove.conf keystone_authtoken admin_tenant_name  "${SERV_TENANT}"
crudini --set /etc/trove/trove.conf keystone_authtoken admin_user         "${SERV}"
crudini --set /etc/trove/trove.conf keystone_authtoken auth_uri           "http://${CONTROLLER}:5000/v2.0"
crudini --set /etc/trove/trove.conf keystone_authtoken identity_uri       "http://${CONTROLLER}:35357"

# /etc/trove/trove-taskmanager.conf
crudini --set /etc/trove/trove-taskmanager.conf DEFAULT  rabbit_host     "${CONTROLLER}"
crudini --set /etc/trove/trove-taskmanager.conf DEFAULT  rabbit_password "${RABBIT_PASS}"
crudini --set /etc/trove/trove-taskmanager.conf DEFAULT  rabbit_userid   "${RABBIT_USER}"
crudini --set /etc/trove/trove-taskmanager.conf DEFAULT  log_dir                 '/var/log/trove'
crudini --set /etc/trove/trove-taskmanager.conf DEFAULT  trove_auth_url          "http://${CONTROLLER}:5000/v2.0"
crudini --set /etc/trove/trove-taskmanager.conf DEFAULT  nova_compute_url        "http://${CONTROLLER}:8774/v2"
crudini --set /etc/trove/trove-taskmanager.conf DEFAULT  cinder_url              "http://${CONTROLLER}:8776/v1"
crudini --set /etc/trove/trove-taskmanager.conf DEFAULT  swift_url               "http://${CONTROLLER}:8080/v1/AUTH_"
crudini --set /etc/trove/trove-taskmanager.conf DEFAULT  sql_connection          "mysql://${SERV}:${SERV_DBPASS}@${CONTROLLER}/trove"
crudini --set /etc/trove/trove-taskmanager.conf DEFAULT  notifier_queue_hostname "${CONTROLLER}"
crudini --set /etc/trove/trove-taskmanager.conf DEFAULT  taskmanager_manager     'trove.taskmanager.manager.Manager'
crudini --set /etc/trove/trove-taskmanager.conf DEFAULT  nova_proxy_admin_user   'admin'
crudini --set /etc/trove/trove-taskmanager.conf DEFAULT  nova_proxy_admin_pass   "${ADMIN_PASS}"
crudini --set /etc/trove/trove-taskmanager.conf DEFAULT  nova_proxy_admin_tenant_name 'service'
crudini --set /etc/trove/trove-taskmanager.conf DEFAULT  auth_strategy   "keystone"
crudini --set /etc/trove/trove-taskmanager.conf keystone_authtoken admin_password     "${SERV_PASS}"
crudini --set /etc/trove/trove-taskmanager.conf keystone_authtoken admin_tenant_name  "${SERV_TENANT}"
crudini --set /etc/trove/trove-taskmanager.conf keystone_authtoken admin_user         "${SERV}"
crudini --set /etc/trove/trove-taskmanager.conf keystone_authtoken auth_uri           "http://${CONTROLLER}:5000/v2.0"
crudini --set /etc/trove/trove-taskmanager.conf keystone_authtoken identity_uri       "http://${CONTROLLER}:35357"

# /etc/trove/trove-conductor.conf
crudini --set /etc/trove/trove-conductor.conf DEFAULT  rabbit_host     "${CONTROLLER}"
crudini --set /etc/trove/trove-conductor.conf DEFAULT  rabbit_password "${RABBIT_PASS}"
crudini --set /etc/trove/trove-conductor.conf DEFAULT  rabbit_userid   "${RABBIT_USER}"
crudini --set /etc/trove/trove-conductor.conf DEFAULT  log_dir                 '/var/log/trove'
crudini --set /etc/trove/trove-conductor.conf DEFAULT  trove_auth_url          "http://${CONTROLLER}:5000/v2.0"
crudini --set /etc/trove/trove-conductor.conf DEFAULT  nova_compute_url        "http://${CONTROLLER}:8774/v2"
crudini --set /etc/trove/trove-conductor.conf DEFAULT  cinder_url              "http://${CONTROLLER}:8776/v1"
crudini --set /etc/trove/trove-conductor.conf DEFAULT  swift_url               "http://${CONTROLLER}:8080/v1/AUTH_"
crudini --set /etc/trove/trove-conductor.conf DEFAULT  sql_connection          "mysql://${SERV}:${SERV_DBPASS}@${CONTROLLER}/trove"
crudini --set /etc/trove/trove-conductor.conf DEFAULT  notifier_queue_hostname "${CONTROLLER}"
crudini --set /etc/trove/trove-conductor.conf DEFAULT  auth_strategy   "keystone"
crudini --set /etc/trove/trove-conductor.conf keystone_authtoken admin_password     "${SERV_PASS}"
crudini --set /etc/trove/trove-conductor.conf keystone_authtoken admin_tenant_name  "${SERV_TENANT}"
crudini --set /etc/trove/trove-conductor.conf keystone_authtoken admin_user         "${SERV}"
crudini --set /etc/trove/trove-conductor.conf keystone_authtoken auth_uri           "http://${CONTROLLER}:5000/v2.0"
crudini --set /etc/trove/trove-conductor.conf keystone_authtoken identity_uri       "http://${CONTROLLER}:35357"

su -s /bin/sh -c "trove-manage db_sync" trove
su -s /bin/sh -c "trove-manage datastore_update mysql ''" trove
systemctl enable openstack-trove-api.service openstack-trove-taskmanager.service openstack-trove-conductor.service
systemctl start openstack-trove-api.service openstack-trove-taskmanager.service openstack-trove-conductor.service




##############################################################################################################################################################
### # Nodes for test environment
NODE        World "External"  Management  Tunnels
controller  A1    --          A3          --
  magsusupil keystone glance horizon heat ceilometer
networking  A1    A2          A3          A4
  lambat neutron
compute     A1    --          A3          A4
  kuwenta01 nova01
storage     A1    ?           A3          ?
  imbakan01 cinder01

NETWORK     adapter network
World       A1      NAT         (connection to the real world)
External    A2      Host-only   192.168.56/24
Management  A3      Internal    192.168.100/24
Tunnels     A4      Internal    192.168.200/24
Tenant      virt    Neutron     192.168.1/24
##############################################################################################################################################################
### # Template for adding services.

SERV=""
SERV_DBPASS=${_DBPASS}
SERV_PASS=${_PASS}
SERV_PORT=
SERV_TENANT="service"
SERV_TYPE=""
SERV_DESC=""

Q1="CREATE DATABASE ${SERV};"
Q2="GRANT ALL PRIVILEGES ON ${SERV}.* TO '${SERV}'@'localhost' IDENTIFIED BY '${SERV_DBPASS}';"
Q3="GRANT ALL PRIVILEGES ON ${SERV}.* TO '${SERV}'@'%' IDENTIFIED BY '${SERV_DBPASS}';"
SQL="${Q1}${Q2}${Q3}"
mysql -u root --password=$MYSQL_PASS -e "$SQL"

gettoken
#source /root/bin/admin-openrc.sh
keystone user-create     --name ${SERV} --pass   ${SERV_PASS}
keystone user-role-add   --user ${SERV} --tenant ${SERV_TENANT} --role admin
keystone service-create  --name ${SERV} --type   ${SERV_TYPE}   --description "${SERV_DESC}"
keystone endpoint-create --service-id $(keystone service-list | awk '/ SERV_TYPE / {print $2}') --publicurl http://${CONTROLLER}:${SERV_PORT} --adminurl http://${CONTROLLER}:${SERV_PORT} --internalurl http://${CONTROLLER}:${SERV_PORT} --region regionOne
crudini --set  database connection      "mysql://${SERV}:${SERV_DBPASS}@${CONTROLLER}/${SERV}"
crudini --set  DEFAULT  rabbit_host     "${CONTROLLER}"
crudini --set  DEFAULT  rabbit_password "${RABBIT_PASS}"
crudini --set  DEFAULT  rabbit_userid   "${RABBIT_USER}"
crudini --set  DEFAULT  rpc_backend     "rabbit"
crudini --set  DEFAULT  auth_strategy   "keystone"
crudini --set  keystone_authtoken admin_password     "${SERV_PASS}"
crudini --set  keystone_authtoken admin_tenant_name  "${SERV_TENANT}"
crudini --set  keystone_authtoken admin_user         "${SERV}"
crudini --set  keystone_authtoken auth_uri           "http://${CONTROLLER}:5000/v2.0"
crudini --set  keystone_authtoken identity_uri       "http://${CONTROLLER}:35357"
crudini --set 

###################################################################
#These are test settings, use better ones for prod !!!!!!!!!!!!!!!!
###################################################################

export CEILOMETER_DBPASS=qwert123
export CINDER_DBPASS=qwert123
export DASH_DBPASS=qwert123
export GLANCE_DBPASS=qwert123
export HEAT_DBPASS=qwert123
export KEYSTONE_DBPASS=qwert123
export NEUTRON_DBPASS=qwert123
export NOVA_DBPASS=qwert123
export TROVE_DBPASS=qwert123

export ADMIN_PASS=qwert123
export CEILOMETER_PASS=qwert123
export CINDER_PASS=qwert123
export DEMO_PASS=qwert123
export GLANCE_PASS=qwert123
export HEAT_PASS=qwert123
export MYSQL_PASS=qwert123
export NEUTRON_PASS=qwert123
export NOVA_PASS=qwert123
export RABBIT_PASS=qwert123
export TROVE_PASS=qwert123

export RABBIT_USER=guest
export METADATA_SECRET=qwert123

export CONTROLLER=os-control
export CON_IP=`getent hosts | grep $CONTROLLER | awk '{print $1}'`
export NODE_CON_IP=`getent hosts | grep $HOSTNAME | awk '{print $1}'`

alias gettoken="/root/bin/get-os-service-token.sh"

###################################################################
#this is /root/bin/get-os-service-token.sh
###################################################################
#!/bin/bash

# clean old value $ get variables
unset OS_SERVICE_TOKEN OS_SERVICE_ENDPOINT
source /root/bin/admin-openrc.sh

# Set variables
export OS_SERVICE_TOKEN=`keystone token-get | awk '/ id / {print $4}'`
export OS_SERVICE_ENDPOINT="http://${CONTROLLER}:35357/v2.0"

# unset non-token variables & print verification
unset OS_TENANT_NAME OS_USERNAME OS_PASSWORD OS_AUTH_URL
keystone user-list