--- /dev/null
+This Lab as a Serice project aims to create on demand OPNFV resources to developers.
+This project will automate the process, to the requested extent, of running an OPNFV
+installer and creating an Openstack environment within OPNFV automatically and on demand.
+
+To run, execute (from the project root):
+ source/deploy.py
+
+To run the Pharos dahsboard listener, which will continualy poll the dashboard and run deployments in the background:
+ source/listen.py --config <conf/pharos.conf>
+
+
+For convenience, there is a bash script source/stop.sh which will stop the dashboard listener and all related scripts.
+
+BEFORE YOU CAN RUN:
+you must first:
+- Integrate FOG into your infrastructure
+- Fill out the needed configuration files
+- Populate the database with your available hosts
+
+
+FOG:
+Our OPNFV infrastructure uses a FOG server to pxe boot, read and write disk images, and otherwise control the hosts we have available for developers.
+FOG is an open source project, and you can view it here: https://fogproject.org/
+FOG provides an easy and scriptable way to completely wipe and write the disks of our hosts.
+ This makes it quick and simple for us to restore our hosts to a known, clean state after a developer has released control of it.
+
+To run the deploy script, you need to:
+ Have a FOG master running
+ Have your hosts registered to the FOG master
+ Have a 'clean' disk image of for each installer / configuration you wish to support.
+ - Fuel, Compass, and JOID all need different distros / versions to run properly
+ - There is a mapping between images and their installers in the installer's config file
+The FOG server must be reachable by whatever machine is running this LaaS software,
+and have network access to PXE boot all of your hosted dev pods.
+
+
+CONFIGURATION:
+INSTALLERS#############################################################################################
+-database Path to the SQLite database for storing host information.
+ Should be the same for all installers in most cases.
+-dhcp_log Path to log file containing DHCP information for dev pods.
+-dhcp_server IP address or hostname of the DHCP server which contains the above log file
+ set to `null` if the same machine will be running dhcp and this project
+-fog
+--api_key The FOG api key. You may instead give the path to a file containing the api key.
+--server The URL of the fog server.
+ ex: http://myServer.com/fog/
+--user_key The FOG api key specific to your user.
+ You may instead give the path to a secrets file containing the key.
+--image_id The id of the image FOG will use when this installer is requested.
+-installer The name of the installer, as seen from the dashboard.
+ `null` will match when no installer is selected, or the `None` installer is..
+-logging_dir The directory to create log files in.
+ Will create the dir if it does not already exist.
+-scenario The default scenario if one is not specified by the user.
+ NOTE: automation of different scenarios are not currently supported.
+ These values are silently ignored.
+-hypervisor_config
+--networks Path to the config file used to define the virtual networks for this installer.
+--vms Path to the config file used to define the virtual machines for this installer.
+-inventory Path to inventory file mapping dashboard host id's to FOG hostnames.
+-vpn_config Path to the vpn config file
+
+
+#########################################################################################################
+
+DOMAINS##################################################################################################
+-jinja-template Path to the jinja xml template used to create libvirt domain xml documents.
+-domains A list of domains. List as many as you want, but be cognizant of hardware limitations
+--disk Path to the qcow2 disk image for this VM
+--interfaces List of interfaces for the vm
+---name The name of the network or bridge that provides this interface
+---type The source of the interface. Either 'bridge' or 'network' is valid, but the bridge
+ must already exist on the host.
+--iso
+---URL Where to fetch the ISO from
+---location Where to save the ISO to
+---used Whether this host will use an iso as a boot drive
+ if `false`, the ISO will not be downloaded
+--memory Memory to allocate to the VM in KiB
+--name libvirt name of VM
+--vcpus How many vcpus to allocate to this host.
+#########################################################################################################
+
+NETWORKS#################################################################################################
+-jinja-template Path to jinja template used to create libvirt XML network documents
+-networks List of networks that will be created
+--brAddr ip address of the bridge on the host
+--brName name of the bridge on the host
+--cidr cidr of the virtual network
+--dhcp dhcp settingg
+---rangeEnd end of DHCP address range
+---rangeStart start of DHCP address range
+---used Whether to enable dhcp for this network. Should probably be false.
+--forward Libvirt network forwarding settings
+---type forwarding type. See libvirt documentation for possible types.
+---used if `false`, the network is isolated.
+--name Name of this network in Libvirt
+--netmask Netmask for this network.
+########################################################################################################
+
+PHAROS##################################################################################################
+-dashboard url of the dashboard. https://labs.opnfv.org is the public OPNFV dashboard
+-database path to database to store booking information.
+ Should be the same db as the host database in most cases
+-default_configs a mappping of installers and their configuration files.
+-inventory path to the inventory file
+-logging_dir Where the pharos dashboard listener should put log files.
+-poling How many times a second the listener will poll the dashboard
+-token Your paros api token. May also be a path to a file containing the token
+#######################################################################################################
+
+VPN####################################################################################################
+NOTE: this all assumes you use LDAP authentication
+-server Domain name of your vpn server
+-authenticaion
+--pass password for your 'admin' user. May also be a path to a secrets file
+--user full dn of your 'admin' user
+-directory
+--root The lowest directory that this program will need to access
+--user The directory where users are stored, relative to the given root dir
+-user
+--objects A list of object classes that vpn users will belong to.
+ Most general class should be on top, and get more specific from there.
+ ex: -top, -inetOrgPerson because `top` is more general
+-database The booking database
+-permanent_users Users that you want to be persistent, even if they have no bookings active
+ ie: your admin users
+ All other users will be deleted when they have no mroe bookings
+#######################################################################################################
+
+INVENTORY##############################################################################################
+This file is used to map the resource id's known by pharos to the hostnames known by FOG.
+for example,
+50: fog-machine-4
+51: fog-machine-5
+52: fog-virtualPod-5.1
+#######################################################################################################
+
+HOW IT WORKS:
+
+0) lab resources are prepared and information is stored in the database
+1) source/listen.py launches a background instance of pharos.py
+ -pharos.py continually polls the dashboard for booking info, and stores it in the database
+2) A known booking begins and pharos.py launches pod_manager.py
+ - pod_manager is launched in a new process, so that the listener continues to poll the dashboard
+ and multiple hosts can be provisioned at once
+3) pod_manager uses FOG to image the host
+4) if requested, pod_manager hands control to deployment_manager to install and deploy OPNFV
+ - deployment_manager instantiates and calls the go() function of the given source/installers/installer subclass
+5) a vpn user is created and random root password is given to the dev pod
+##########The dashboard does not yet support the following actions#############
+6) public ssh key of the user is fetched from the dashboard
+7) user is automatically notified their pod is ready, and given all needed info
+
+
+GENERAL NOTES:
+
+resetDatabase.py relies on FOG to retrieve a list of all hosts available to developers
+
+running:
+ source/resetDatabase.py --both --config <CONFIG_FILE>
+will create a database and populate it.
+WARNING: This will delete existing information if run on a previously initialized database
+
+To aid in visualization and understanding of the resulting topolgy after fully deploying OPNFV and Openstack in
+a development pod, you may review the LaaS_Diagram in this directory.
--- /dev/null
+---
+- disk: /vm/master.qcow2
+ interfaces:
+ - name: admin
+ type: network
+ - name: public
+ type: network
+ - name: storage
+ type: network
+ - name: management
+ type: network
+ iso:
+ URL: http://artifacts.opnfv.org/fuel/danube/opnfv-danube.2.0.iso
+ location: /vm/fuel.iso
+ used: true
+ memory: 8240000
+ name: master
+ vcpus: 4
+
+- disk: /vm/slave1.qcow2
+ interfaces:
+ - name: admin
+ type: network
+ - name: public
+ type: network
+ - name: storage
+ type: network
+ - name: management
+ type: network
+ iso:
+ URL: http://artifacts.opnfv.org/fuel/danube/opnfv-danube.2.0.iso
+ location: /vm/fuel.iso
+ used: false
+ memory: 8240000
+ name: slave1
+ vcpus: 4
+
+- disk: /vm/slave2.qcow2
+ interfaces:
+ - name: admin
+ type: network
+ - name: public
+ type: network
+ - name: storage
+ type: network
+ - name: management
+ type: network
+ iso:
+ URL: http://artifacts.opnfv.org/fuel/danube/opnfv-danube.2.0.iso
+ location: /vm/fuel.iso
+ used: false
+ memory: 8240000
+ name: slave2
+ vcpus: 4
+
+- disk: /vm/slave3.qcow2
+ interfaces:
+ - name: admin
+ type: network
+ - name: public
+ type: network
+ - name: storage
+ type: network
+ - name: management
+ type: network
+ iso:
+ URL: http://artifacts.opnfv.org/fuel/danube/opnfv-danube.2.0.iso
+ location: /vm/fuel.iso
+ used: false
+ memory: 8240000
+ name: slave3
+ vcpus: 4
+
+- disk: /vm/slave4.qcow2
+ interfaces:
+ - name: admin
+ type: network
+ - name: public
+ type: network
+ - name: storage
+ type: network
+ - name: management
+ type: network
+ iso:
+ URL: http://artifacts.opnfv.org/fuel/danube/opnfv-danube.2.0.iso
+ location: /vm/fuel.iso
+ used: false
+ memory: 8240000
+ name: slave4
+ vcpus: 4
+
+- disk: /vm/slave5.qcow2
+ interfaces:
+ - name: admin
+ type: network
+ - name: public
+ type: network
+ - name: storage
+ type: network
+ - name: management
+ type: network
+ iso:
+ URL: http://artifacts.opnfv.org/fuel/danube/opnfv-danube.2.0.iso
+ location: /vm/fuel.iso
+ used: false
+ memory: 8240000
+ name: slave5
+ vcpus: 4
--- /dev/null
+---
+database: /var/OPNFV/hosts.db
+dhcp_log: /var/log/messages
+dhcp_server: null
+fog:
+ api_key: /path/to/fog.key # may also put the key directly here
+ server: http://fogserver.com/fog/
+ user_key: /path/to/fog_user.key
+ image_id: 5
+installer: Fuel
+logging_dir: /var/log/OPNFV/
+scenario: os-nosdn-nofeature-noha
+hypervisor_config:
+ networks: /root/laas/conf/network.yaml
+ vms: /root/laas/conf/domain.yaml
+inventory: /root/laas/conf/inventory.yaml
+vpn_config: /root/laas/conf/vpn.yaml
--- /dev/null
+---
+# pharos id : fog name
+# for example:
+1: fog-host-1
+2: fog-host-2
+3: fog-host-3
--- /dev/null
+---
+database: /var/OPNFV/hosts.db
+dhcp_log: /var/log/messages
+dhcp_server: null
+fog:
+ api_key: /path/to/fog.key # may also put the key directly here
+ server: http://fogserver.com/fog/
+ user_key: /path/to/fog_user.key
+ image_id: 12
+installer: Joid
+logging_dir: /var/log/OPNFV/
+scenario: os-nosdn-nofeature-noha
+hypervisor_config:
+ networks: /root/laas/conf/network.yaml
+ vms: /root/laas/conf/domain.yaml
+inventory: /root/laas/conf/inventory.yaml
+vpn_config: /root/laas/conf/vpn.yaml
--- /dev/null
+---
+database: /var/OPNFV/hosts.db
+dhcp_log: /var/log/messages
+dhcp_server: null
+fog:
+ api_key: /path/to/fog.key # may also put the key directly here
+ server: http://fogserver.com/fog/
+ user_key: /path/to/fog_user.key
+ image_id: 5
+installer: null
+logging_dir: /var/log/OPNFV/
+scenario: os-nosdn-nofeature-noha
+hypervisor_config:
+ networks: /root/laas/conf/network.yaml
+ vms: /root/laas/conf/domain.yaml
+inventory: /root/laas/conf/inventory.yaml
+vpn_config: /root/laas/conf/vpn.yaml
--- /dev/null
+---
+- brAddr: 10.20.0.1
+ brName: admin-br
+ cidr: 10.20.0.0/24
+ dhcp:
+ rangeEnd: 10.20.0.250
+ rangeStart: 10.20.0.15
+ used: false
+ forward:
+ type: nat
+ used: true
+ name: admin
+ netmask: 255.255.255.0
+
+- brAddr: 10.20.1.1
+ brName: public-br
+ cidr: 10.20.1.0/24
+ dhcp:
+ rangeEnd: 10.20.1.250
+ rangeStart: 10.20.1.15
+ used: false
+ forward:
+ type: nat
+ used: true
+ name: public
+ netmask: 255.255.255.0
+
+- brAddr: 10.20.2.1
+ brName: management-br
+ cidr: 10.20.2.0/24
+ dhcp:
+ rangeEnd: 10.20.2.250
+ rangeStart: 10.20.2.15
+ used: false
+ forward:
+ type: nat
+ used: false
+ name: management
+ netmask: 255.255.255.0
+
+- brAddr: 10.20.3.1
+ brName: storage-br
+ cidr: 10.20.3.0/24
+ dhcp:
+ rangeEnd: 10.20.3.250
+ rangeStart: 10.20.3.15
+ used: false
+ forward:
+ type: nat
+ used: false
+ name: storage
+ netmask: 255.255.255.0
--- /dev/null
+---
+dashboard: https://labs.opnfv.org
+database: /var/OPNFV/laas.db
+default_configs:
+ Fuel: /root/laas/conf/fuel.yaml
+ None: /root/laas/conf/laas.yaml
+ Joid: /rooot/laas/conf/joid.yaml
+inventory: /root/laas/conf/inventory.yaml
+logging_dir: /var/log/OPNFV
+polling: 3
+token: /root/laas/conf/pharos.key
--- /dev/null
+---
+server: vpn.domain.com
+authentication:
+ pass: /path/to/keyfile # you may also put the password directly here
+ user: cn=root,o=opnfv,dc=domain,dc=com
+directory:
+ root: o=opnfv,dc=domain,dc=com
+ user: ou=People # relative to the root dir
+user:
+ objects: # listed in ascending order of specificty
+ - top
+ - inetOrgPerson # last object should be a class that only vpn users have
+database: /var/OPNFV/laas.db # same as the pharos api booking db
+permanent_users: # any users you want to be persistent
+ - pberberian
--- /dev/null
+#!/bin/bash
+
+#############################################################################
+#Copyright 2017 Parker Berberian and others #
+# #
+#Licensed under the Apache License, Version 2.0 (the "License"); #
+#you may not use this file except in compliance with the License. #
+#You may obtain a copy of the License at #
+# #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+# #
+#Unless required by applicable law or agreed to in writing, software #
+#distributed under the License is distributed on an "AS IS" BASIS, #
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
+#See the License for the specific language governing permissions and #
+#limitations under the License. #
+#############################################################################
+
+virsh start master
+
+ret=''
+while [ -z "$ret" ]; do
+ echo "Master node is not accepting ssh. Sleeping 15 seconds..."
+ sleep 15
+ ret=$(nmap 10.20.0.2 -PN -p ssh | grep open)
+done
+
+ssh-keygen -f ~/.ssh/id_rsa -t rsa -N ''
+sshpass -p r00tme ssh-copy-id -o stricthostkeychecking=no root@10.20.0.2
+
+ssh root@10.20.0.2 killall fuelmenu
+
+echo "killed fuel menu. Waiting for installation to complete"
+
+ans=''
+while [ -z "$ans" ]; do
+ echo "fuel api unavailable. Sleeping 15 seconds..."
+ sleep 15
+ ans=$(curl http://10.20.0.2:8000 2>/dev/null )
+done
--- /dev/null
+#!/bin/bash
+#############################################################################
+#Copyright 2017 Parker Berberian and others #
+# #
+#Licensed under the Apache License, Version 2.0 (the "License"); #
+#you may not use this file except in compliance with the License. #
+#You may obtain a copy of the License at #
+# #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+# #
+#Unless required by applicable law or agreed to in writing, software #
+#distributed under the License is distributed on an "AS IS" BASIS, #
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
+#See the License for the specific language governing permissions and #
+#limitations under the License. #
+#############################################################################
+
+MYIP=$1
+DESTINATION=$2
+MYBRIDGE=10.20.1.1
+DESTNETWORK=10.20.1.0/24
+PORT=80
+
+iptables -I INPUT 2 -d "$MYIP" -p tcp --dport "$PORT" -j ACCEPT
+iptables -t nat -I INPUT 1 -d "$MYIP" -p tcp --dport "$PORT" -j ACCEPT
+iptables -I FORWARD -p tcp --dport "$PORT" -j ACCEPT
+
+iptables -t nat -I PREROUTING -p tcp -d "$MYIP" --dport "$PORT" -j DNAT --to-destination "$DESTINATION:$PORT"
+iptables -t nat -I POSTROUTING -p tcp -s "$DESTINATION" ! -d "$DESTNETWORK" -j SNAT --to-source "$MYIP"
+
+iptables -t nat -I POSTROUTING 2 -d "$DESTINATION" -j SNAT --to-source "$MYBRIDGE"
--- /dev/null
+#!/bin/bash
+#############################################################################
+#Copyright 2017 Parker Berberian and others #
+# #
+#Licensed under the Apache License, Version 2.0 (the "License"); #
+#you may not use this file except in compliance with the License. #
+#You may obtain a copy of the License at #
+# #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+# #
+#Unless required by applicable law or agreed to in writing, software #
+#distributed under the License is distributed on an "AS IS" BASIS, #
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
+#See the License for the specific language governing permissions and #
+#limitations under the License. #
+#############################################################################
+
+MYIP=$1
+DESTINATION=10.20.0.2
+MYBRIDGE=10.20.0.1
+DESTNETWORK=10.20.0.0/24
+PORTS=(8000 8443)
+
+for PORT in "${PORTS[@]}"; do
+
+ iptables -I INPUT 2 -d "$MYIP" -p tcp --dport "$PORT" -j ACCEPT
+ iptables -t nat -I INPUT 1 -d "$MYIP" -p tcp --dport "$PORT" -j ACCEPT
+ iptables -I FORWARD -p tcp --dport "$PORT" -j ACCEPT
+
+ iptables -t nat -I PREROUTING -p tcp -d "$MYIP" --dport "$PORT" -j DNAT --to-destination "$DESTINATION:$PORT"
+ iptables -t nat -I POSTROUTING -p tcp -s "$DESTINATION" ! -d "$DESTNETWORK" -j SNAT --to-source "$MYIP"
+
+ iptables -t nat -I POSTROUTING 2 -d "$DESTINATION" -j SNAT --to-source "$MYBRIDGE"
+done
--- /dev/null
+#!/bin/bash
+#############################################################################
+#Copyright 2017 Parker Berberian and others #
+# #
+#Licensed under the Apache License, Version 2.0 (the "License"); #
+#you may not use this file except in compliance with the License. #
+#You may obtain a copy of the License at #
+# #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+# #
+#Unless required by applicable law or agreed to in writing, software #
+#distributed under the License is distributed on an "AS IS" BASIS, #
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
+#See the License for the specific language governing permissions and #
+#limitations under the License. #
+#############################################################################
+
+# parses the passed scenario
+args=($(echo "$1" | tr "-" "\n"))
+# args is array: [os, nosdn, nofeature, noha]
+
+# the deploy script expects 'none' rather than 'nofeature'
+if [ "nofeature" == "${args[2]}" ]; then
+ args[2]="none"
+fi
+# grabs the joid repo
+git clone "https://gerrit.opnfv.org/gerrit/joid.git"
+# working directory has to be where 03-maasdeploy is
+cd joid/ci
+# virtualy deploy maas
+./03-maasdeploy.sh virtual
+# deploys OPNFV with the given scenario
+./deploy.sh -o newton -s "${args[1]}" -t "${args[3]}" -l default -d xenial -m openstack -f "${args[2]}"
--- /dev/null
+#!/bin/bash
+#############################################################################
+#Copyright 2017 Parker Berberian and others #
+# #
+#Licensed under the Apache License, Version 2.0 (the "License"); #
+#you may not use this file except in compliance with the License. #
+#You may obtain a copy of the License at #
+# #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+# #
+#Unless required by applicable law or agreed to in writing, software #
+#distributed under the License is distributed on an "AS IS" BASIS, #
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
+#See the License for the specific language governing permissions and #
+#limitations under the License. #
+#############################################################################
+
+for disk in "$@"; do
+ qemu-img create -f qcow2 "$disk" 100G
+done
--- /dev/null
+#!/bin/bash
+#############################################################################
+#Copyright 2017 Parker Berberian and others #
+# #
+#Licensed under the Apache License, Version 2.0 (the "License"); #
+#you may not use this file except in compliance with the License. #
+#You may obtain a copy of the License at #
+# #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+# #
+#Unless required by applicable law or agreed to in writing, software #
+#distributed under the License is distributed on an "AS IS" BASIS, #
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
+#See the License for the specific language governing permissions and #
+#limitations under the License. #
+#############################################################################
+
+MYIP=X.X.X.X
+PORT="5900:5905"
+iptables -I INPUT 2 -d "$MYIP" -p tcp --dport "$PORT" -j ACCEPT
+iptables -t nat -I INPUT 1 -d "$MYIP" -p tcp --dport "$PORT" -j ACCEPT
+iptables -I FORWARD -p tcp --dport "$PORT" -j ACCEPT
+iptables -I OUTPUT -p tcp --dport "$PORT" -j ACCEPT
--- /dev/null
+"""
+#############################################################################
+#Copyright 2017 Parker Berberian and others #
+# #
+#Licensed under the Apache License, Version 2.0 (the "License"); #
+#you may not use this file except in compliance with the License. #
+#You may obtain a copy of the License at #
+# #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+# #
+#Unless required by applicable law or agreed to in writing, software #
+#distributed under the License is distributed on an "AS IS" BASIS, #
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
+#See the License for the specific language governing permissions and #
+#limitations under the License. #
+#############################################################################
+"""
--- /dev/null
+"""
+#############################################################################
+#Copyright 2017 Parker Berberian and others #
+# #
+#Licensed under the Apache License, Version 2.0 (the "License"); #
+#you may not use this file except in compliance with the License. #
+#You may obtain a copy of the License at #
+# #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+# #
+#Unless required by applicable law or agreed to in writing, software #
+#distributed under the License is distributed on an "AS IS" BASIS, #
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
+#See the License for the specific language governing permissions and #
+#limitations under the License. #
+#############################################################################
+"""
--- /dev/null
+"""
+#############################################################################
+#Copyright 2017 Parker Berberian and others #
+# #
+#Licensed under the Apache License, Version 2.0 (the "License"); #
+#you may not use this file except in compliance with the License. #
+#You may obtain a copy of the License at #
+# #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+# #
+#Unless required by applicable law or agreed to in writing, software #
+#distributed under the License is distributed on an "AS IS" BASIS, #
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
+#See the License for the specific language governing permissions and #
+#limitations under the License. #
+#############################################################################
+"""
+
+import requests
+import sys
+import time
+
+
+class FOG_Handler:
+ """
+ This class talks with the REST web api for the FOG server.
+
+ TODO: convert prints to logs and remove uneeded pass's
+ """
+
+ def __init__(self, baseURL, fogKey=None, userKey=None):
+ """
+ init function
+ baseURL should be http://fog.ip.or.hostname/fog/
+ fogKey and userKey can optionally be supplied here or later
+ They can be found in fog and provide authentication.
+ """
+ self.baseURL = baseURL
+ self.fogKey = fogKey
+ self.userKey = userKey
+ self.header = {}
+ self.updateHeader()
+
+ def setLogger(self, logger):
+ """
+ saves the refference to the log object as
+ self.log
+ """
+ self.log = logger
+
+ def getUserKeyFromFile(self, path):
+ """
+ reads the user api key from a file
+ """
+ self.userKey = open(path).read()
+ self.updateHeader()
+
+ def getFogKeyFromFile(self, path):
+ """
+ reads the api key from a file
+ """
+ self.fogKey = open(path).read()
+ self.updateHeader()
+
+ def setUserKey(self, key):
+ """
+ sets the user key
+ """
+ self.userKey = key
+ self.updateHeader()
+
+ def setFogKey(self, key):
+ """
+ sets the fog key
+ """
+ self.fogKey = key
+ self.updateHeader()
+
+ def updateHeader(self):
+ """
+ recreates the http header used to talk to the fog api
+ """
+ self.header = {}
+ self.header['fog-api-token'] = self.fogKey
+ self.header['fog-user-token'] = self.userKey
+
+ def setImage(self, host, imgNum):
+ """
+ Sets the image to be used during ghosting to the image
+ with id imgNum. host can either be a hostname or number.
+ """
+ try:
+ host = int(host)
+ except:
+ host = self.getHostNumber(host)
+ url = self.baseURL+"host/"+str(host)
+ host_conf = requests.get(url, headers=self.header).json()
+ host_conf['imageID'] = str(imgNum)
+ requests.put(url+"/edit", headers=self.header, json=host_conf)
+
+ def delTask(self, hostNum):
+ """
+ Tries to delete an existing task for the host
+ with hostNum as a host number
+ """
+ try:
+ url = self.baseURL+'fog/host/'+str(hostNum)+'/cancel'
+ req = requests.delete(url, headers=self.header)
+ if req.status_code == 200:
+ self.log.info("%s", "successfully deleted image task")
+ except Exception:
+ self.log.exception("Failed to delete the imaging task!")
+
+ def getHostMac(self, hostname):
+ """
+ returns the primary mac address if the given host.
+ """
+ try:
+ hostNum = int(self.getHostNumber(hostname))
+ url = self.baseURL + "host/"+str(hostNum)
+ req = requests.get(url, headers=self.header)
+ macAddr = req.json()['primac']
+ return macAddr
+ except Exception:
+ self.log.exception('%s', "Failed to connect to the FOG server")
+
+ def getHostNumber(self, hostname):
+ """
+ returns the host number of given host
+ """
+ try:
+ req = requests.get(self.baseURL+"host", headers=self.header)
+ hostData = req.json()
+ if hostData is not None:
+ for hostDict in hostData['hosts']:
+ if hostname == hostDict['name']:
+ return hostDict['id']
+ return -1
+ except Exception:
+ self.log.exception('%s', "Failed to connect to the FOG server")
+
+ def imageHost(self, hostName, recurse=False):
+ """
+ Schedules an imaging task for the given host.
+ This automatically uses the "associated" disk image.
+ To support extra installers, I will need to create
+ a way to change what that image is before calling
+ this method.
+ """
+ num = str(self.getHostNumber(hostName))
+ url = self.baseURL+'host/'+num+'/task'
+
+ try:
+ req = requests.post(
+ url,
+ headers=self.header,
+ json={"taskTypeID": 1}
+ )
+ if req.status_code == 200:
+ self.log.info("%s", "Scheduled image task for host")
+ except Exception:
+ if recurse: # prevents infinite loop
+ self.log.exception("%s", "Failed to schedule task. Exiting")
+ sys.exit(1)
+ self.log.warning("%s", "Failed to schedule host imaging")
+ self.log.warning("%s", "Trying to delete existing image task")
+ self.delTask(num)
+ self.imageHost(num, recurse=True)
+
+ def waitForHost(self, host):
+ """
+ tracks the imaging task to completion.
+ """
+ while True:
+ imageTask = self.getImagingTask(host)
+ if imageTask is None:
+ self.log.info("%s", "Imaging complete")
+ return
+ state = int(imageTask['stateID'])
+ if state == 1:
+ self.log.info("%s", "Waiting for host to check in")
+ self.waitForTaskToActive(host)
+ continue
+ if state == 3:
+ self.waitForTaskToStart(host)
+ self.waitForImaging(host)
+ continue
+ time.sleep(8)
+
+ def waitForImaging(self, host):
+ """
+ Once the host begins being imaged, this tracks progress.
+ """
+ # print "Host has begun the imaging process\n"
+ while True:
+ task = self.getImagingTask(host)
+ if task is None:
+ return
+ per = str(task['percent'])
+ self.log.info("%s percent done imaging", per)
+ time.sleep(15)
+
+ def waitForTaskToActive(self, host):
+ """
+ Waits for the host to reboot and pxe boot
+ into FOG
+ """
+ while True:
+ try:
+ task = self.getImagingTask(host)
+ except:
+ pass
+ state = int(task['stateID'])
+ if state == 1:
+ time.sleep(4)
+ else:
+ return
+
+ def waitForTaskToStart(self, host):
+ """
+ waits for the task to start and imaging to begin.
+ """
+ while True:
+ try:
+ per = str(self.getImagingTask(host)['percent'])
+ except:
+ pass
+ if per.strip() == '':
+ time.sleep(1)
+ else:
+ return
+
+ def getImagingTask(self, host):
+ """
+ Sorts through all current tasks to find the image task
+ associated with the given host.
+ """
+ try:
+ taskList = requests.get(
+ self.baseURL+'task/current',
+ headers=self.header)
+ taskList = taskList.json()['tasks']
+ imageTask = None
+ for task in taskList:
+ hostname = str(task['host']['name'])
+ if hostname == host and int(task['typeID']) == 1:
+ imageTask = task
+ return imageTask
+ except Exception:
+ self.log.exception("%s", "Failed to talk to FOG server")
+ sys.exit(1)
+
+ def getHosts(self):
+ """
+ returns a list of all hosts
+ """
+ req = requests.get(self.baseURL+"host", headers=self.header)
+ return req.json()['hosts']
+
+ def getHostsinGroup(self, groupName):
+ """
+ returns a list of all hosts in groupName
+ """
+ groupID = None
+ groups = requests.get(self.baseURL+"group", headers=self.header)
+ groups = groups.json()['groups']
+ for group in groups:
+ if groupName.lower() in group['name'].lower():
+ groupID = group['id']
+ if groupID is None:
+ return
+ hostIDs = []
+ associations = requests.get(
+ self.baseURL+"groupassociation",
+ headers=self.header
+ )
+ associations = associations.json()['groupassociations']
+ for association in associations:
+ if association['groupID'] == groupID:
+ hostIDs.append(association['hostID'])
+
+ hosts = []
+ for hostID in hostIDs:
+ hosts.append(requests.get(
+ self.baseURL+"host/"+str(hostID),
+ headers=self.header
+ ).json())
+ return hosts
--- /dev/null
+"""
+#############################################################################
+#Copyright 2017 Parker Berberian and others #
+# #
+#Licensed under the Apache License, Version 2.0 (the "License"); #
+#you may not use this file except in compliance with the License. #
+#You may obtain a copy of the License at #
+# #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+# #
+#Unless required by applicable law or agreed to in writing, software #
+#distributed under the License is distributed on an "AS IS" BASIS, #
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
+#See the License for the specific language governing permissions and #
+#limitations under the License. #
+#############################################################################
+"""
+
+import requests
+import time
+import sys
+
+
+class Fuel_api:
+
+ def __init__(self, url, logger, user="admin", password="admin"):
+ """
+ url is the url of the fog api in the form
+ http://ip.or.host.name:8000/
+ logger is a reference to the logger
+ the default creds for fuel is admin/admin
+ """
+ self.logger = logger
+ self.base = url
+ self.user = user
+ self.password = password
+ self.header = {"Content-Type": "application/json"}
+
+ def getKey(self):
+ """
+ authenticates with the user and password
+ to get a keystone key, used in the headers
+ from here on to talk to fuel.
+ """
+ url = self.base + 'keystone/v2.0/tokens/'
+ reqData = {"auth": {
+ "tenantName": self.user,
+ "passwordCredentials": {
+ "username": self.user,
+ "password": self.password
+ }
+ }}
+ self.logger.info("Retreiving keystone token from %s", url)
+ token = requests.post(url, headers=self.header, json=reqData)
+ self.logger.info("Received response code %d", token.status_code)
+ self.token = token.json()['access']['token']['id']
+ self.header['X-Auth-Token'] = self.token
+
+ def getNotifications(self):
+ """
+ returns the fuel notifications
+ """
+ url = self.base+'/api/notifications'
+ try:
+ req = requests.get(url, headers=self.header)
+ return req.json()
+
+ except Exception:
+ self.logger.exception('%s', "Failed to talk to the Fuel api!")
+ sys.exit(1)
+
+ def waitForBootstrap(self):
+ """
+ Waits for the bootstrap image to build.
+ """
+ while True:
+ time.sleep(30)
+ notes = self.getNotifications()
+ for note in notes:
+ if "bootstrap image building done" in note['message']:
+ return
+
+ def getNodes(self):
+ """
+ returns a list of all nodes booted into fuel
+ """
+ url = self.base+'api/nodes'
+ try:
+ req = requests.get(url, headers=self.header)
+ return req.json()
+ except Exception:
+ self.logger.exception('%s', "Failed to talk to the Fuel api!")
+ sys.exit(1)
+
+ def getID(self, mac):
+ """
+ gets the fuel id of node with given mac
+ """
+ for node in self.getNodes():
+ if node['mac'] == mac:
+ return node['id']
+
+ def getNetID(self, name, osid):
+ """
+ gets the id of the network with name
+ """
+ url = self.base+'api/clusters/'
+ url += str(osid)+'/network_configuration/neutron'
+ try:
+ req = requests.get(url, headers=self.header)
+ nets = req.json()['networks']
+ for net in nets:
+ if net['name'] == name:
+ return net['id']
+ return -1
+
+ except Exception:
+ self.logger.exception('%s', "Failed to talk to the Fuel api!")
+ sys.exit(1)
+
+ def createOpenstack(self):
+ """
+ defines a new openstack environment in fuel.
+ """
+ url = self.base+'api/clusters'
+ data = {
+ "nodes": [],
+ "tasks": [],
+ "name": "OpenStack",
+ "release_id": 2,
+ "net_segment_type": "vlan"
+ }
+ try:
+ req = requests.post(url, json=data, headers=self.header)
+ return req.json()['id']
+ except Exception:
+ self.logger.exception('%s', "Failed to talk to the Fuel api!")
+ sys.exit(1)
+
+ def simpleNetDict(self, osID):
+ """
+ returns a simple dict of network names and id numbers
+ """
+ nets = self.getNetworks(osID)
+ netDict = {}
+ targetNets = ['admin', 'public', 'storage', 'management']
+ for net in nets['networks']:
+ for tarNet in targetNets:
+ if tarNet in net['name']:
+ netDict[tarNet] = net['id']
+ return netDict
+
+ def getNetworks(self, osID):
+ """
+ Returns the pythonizezd json of the openstack networks
+ """
+ url = self.base + 'api/clusters/'
+ url += str(osID)+'/network_configuration/neutron/'
+ try:
+ req = requests.get(url, headers=self.header)
+ return req.json()
+ except Exception:
+ self.logger.exception('%s', "Failed to talk to the Fuel api!")
+ sys.exit(1)
+
+ def uploadNetworks(self, netJson, osID):
+ """
+ configures the networks of the openstack
+ environment with id osID based on netJson
+ """
+ url = self.base+'api/clusters/'
+ url += str(osID)+'/network_configuration/neutron'
+ try:
+ req = requests.put(url, headers=self.header, json=netJson)
+ return req.json()
+ except Exception:
+ self.logger.exception('%s', "Failed to talk to the Fuel api!")
+ sys.exit(1)
+
+ def addNodes(self, clusterID, nodes):
+ """
+ Adds the nodes into this openstack environment.
+ nodes is valid json
+ """
+ url = self.base + 'api/clusters/'+str(clusterID)+'/assignment'
+ try:
+ req = requests.post(url, headers=self.header, json=nodes)
+ return req.json()
+
+ except Exception:
+ self.logger.exception('%s', "Failed to talk to the Fuel api!")
+ sys.exit(1)
+
+ def getIfaces(self, nodeID):
+ """
+ returns the pythonized json describing the
+ interfaces of given node
+ """
+ url = self.base + 'api/nodes/'+str(nodeID)+'/interfaces'
+ try:
+ req = requests.get(url, headers=self.header)
+ return req.json()
+
+ except Exception:
+ self.logger.exception('%s', "Failed to talk to the Fuel api!")
+ sys.exit(1)
+
+ def setIfaces(self, nodeID, ifaceJson):
+ """
+ configures the interfaces of node with id nodeID
+ with ifaceJson
+ ifaceJson is valid json that fits fuel's schema for ifaces
+ """
+ url = self.base+'/api/nodes/'+str(nodeID)+'/interfaces'
+ try:
+ req = requests.put(url, headers=self.header, json=ifaceJson)
+ return req.json()
+
+ except Exception:
+ self.logger.exception('%s', "Failed to talk to the Fuel api!")
+ sys.exit(1)
+
+ def getTasks(self):
+ """
+ returns a list of all tasks
+ """
+ url = self.base+"/api/tasks/"
+ try:
+ req = requests.get(url, headers=self.header)
+ return req.json()
+ except Exception:
+ self.logger.exception('%s', "Failed to talk to the Fuel api!")
+ sys.exit(1)
+
+ def waitForTask(self, uuid):
+ """
+ Tracks the progress of task with uuid and
+ returns once the task finishes
+ """
+ progress = 0
+ while progress < 100:
+ for task in self.getTasks():
+ if task['uuid'] == uuid:
+ progress = task['progress']
+ self.logger.info("Task is %s percent done", str(progress))
+ time.sleep(20)
+ # Task may hang a minute at 100% without finishing
+ while True:
+ for task in self.getTasks():
+ if task['uuid'] == uuid and not task['status'] == "ready":
+ time.sleep(10)
+ elif task['uuid'] == uuid and task['status'] == "ready":
+ return
+
+ def getHorizonIP(self, osid):
+ """
+ returns the ip address of the horizon dashboard.
+ Horizon always takes the first ip after the public router's
+ """
+ url = self.base+'api/clusters/'
+ url += str(osid)+'/network_configuration/neutron/'
+ try:
+ req = requests.get(url, headers=self.header)
+ routerIP = req.json()['vips']['vrouter_pub']['ipaddr'].split('.')
+ routerIP[-1] = str(int(routerIP[-1])+1)
+ return '.'.join(routerIP)
+ except Exception:
+ self.logger.exception('%s', "Failed to talk to the Fuel api!")
+ sys.exit(1)
+
+ def deployOpenstack(self, clusterID):
+ """
+ Once openstack and the nodes are configured,
+ this method actually deploys openstack.
+ It takes a while.
+ """
+ # First, we need to provision the cluster
+ url = self.base+'/api/clusters/'+str(clusterID)+'/provision'
+ req = requests.put(url, headers=self.header)
+ if req.status_code < 300:
+ self.logger.info('%s', "Sent provisioning task")
+ else:
+ err = "failed to provision Openstack Environment"
+ self.logger.error('%s', err)
+ sys.exit(1)
+
+ taskUID = ''
+ tasks = self.getTasks()
+ for task in tasks:
+ if task['name'] == "provision" and task['cluster'] == clusterID:
+ taskUID = task['uuid']
+
+ self.waitForTask(taskUID)
+
+ # Then, we deploy cluster
+ url = self.base + '/api/clusters/'+str(clusterID)+'/deploy'
+ req = requests.put(url, headers=self.header)
+ if req.status_code < 300:
+ self.logger.info('%s', "Sent deployment task")
+ taskUID = ''
+ tasks = self.getTasks()
+ for task in tasks:
+ if 'deploy' in task['name'] and task['cluster'] == clusterID:
+ taskUID = task['uuid']
+ if len(taskUID) > 0:
+ self.waitForTask(taskUID)
--- /dev/null
+"""
+#############################################################################
+#Copyright 2017 Parker Berberian and others #
+# #
+#Licensed under the Apache License, Version 2.0 (the "License"); #
+#you may not use this file except in compliance with the License. #
+#You may obtain a copy of the License at #
+# #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+# #
+#Unless required by applicable law or agreed to in writing, software #
+#distributed under the License is distributed on an "AS IS" BASIS, #
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
+#See the License for the specific language governing permissions and #
+#limitations under the License. #
+#############################################################################
+"""
+
+import libvirt
+import time
+import xml.dom
+import xml.dom.minidom
+from domain import Domain
+from network import Network
+from utilities import Utilities
+
+
+class Libvirt:
+ """
+ This class talks to the Libvirt api.
+ Given a config file, this class should create all networks and
+ domains.
+
+ TODO: convert prints to logging and remove uneeded pass statements
+ """
+
+ def __init__(self, hostAddr, net_conf=None, dom_conf=None):
+ """
+ init function
+ hostAddr is the ip address of the host
+ net_conf and dom_conf are the paths
+ to the config files
+ """
+ self.host = hostAddr
+ self.URI = "qemu+ssh://root@"+str(hostAddr)+"/system"
+ self.hypervisor = None
+ self.domains = []
+ self.networks = []
+ self.net_conf = net_conf
+ self.dom_conf = dom_conf
+
+ def setLogger(self, log):
+ """
+ Saves the logger in self.log
+ """
+ self.log = log
+
+ def bootMaster(self):
+ """
+ starts the previously defined master node
+ """
+ for dom in self.domains:
+ if 'master' in dom.name():
+ try:
+ dom.create()
+ except Exception:
+ pass
+
+ def bootSlaves(self):
+ """
+ boots every defined vm with 'slave' in its name
+ """
+ for dom in self.domains:
+ if 'slave' in dom.name():
+ try:
+ dom.create()
+ self.log.info("Booting %s", dom.name())
+ except Exception:
+ self.log.exception("%s", "failed to boot domain")
+ time.sleep(5)
+
+ def getMacs(self, domName):
+ """
+ returns a dictionary with a network name
+ mapped to the mac address of the domain on that net
+ """
+ try:
+ dom = self.hypervisor.lookupByName(domName)
+ xmlDesc = dom.XMLDesc(0)
+ parsedXML = xml.dom.minidom.parseString(xmlDesc)
+ interfacesXML = parsedXML.getElementsByTagName('interface')
+ netDict = {}
+ for iface in interfacesXML:
+ src = iface.getElementsByTagName('source')[0]
+ mac = iface.getElementsByTagName('mac')[0]
+ netDict[src] = mac
+ return netDict
+ except Exception:
+ self.log.exception("%s", "Domain not found")
+
+ def defineVM(self, xmlConfig):
+ """
+ Generic method to define a persistent vm with the
+ given config.
+ Assumes that self.hypervisor is already connected.
+ """
+ if self.checkForVM(xmlConfig):
+ vm = self.hypervisor.defineXML(xmlConfig)
+ if vm is None:
+ name = self.getName(xmlConfig)
+ self.log.error("Failed to define vm %s. exiting", name)
+ exit(1)
+ else:
+ self.log.info("Successfully created vm %s", vm.name())
+ pass
+ self.domains.append(vm)
+
+ def checkForVM(self, xmlConfig):
+ """
+ Checks if another vm with the same name exists
+ on the remote host already. If it does, it will
+ delete that vm
+ """
+ allGood = False
+ vms = self.hypervisor.listAllDomains(0)
+ names = []
+ for dom in vms:
+ names.append(dom.name())
+ vmName = Utilities.getName(xmlConfig)
+ if vmName in names:
+ self.log.warning("domain %s already exists", vmName)
+ self.log.warning("%s", "Atempting to delete it")
+ self.deleteVM(vmName)
+ allGood = True
+ else:
+ allGood = True
+ return allGood
+
+ def deleteVM(self, name):
+ """
+ removes the given vm from the remote host
+ """
+ try:
+ vm = self.hypervisor.lookupByName(name)
+ except:
+ return
+ active = vm.isActive()
+ persistent = vm.isPersistent()
+ if active:
+ try:
+ vm.destroy()
+ except:
+ self.log.exception("%s", "Failed to destroy vm")
+
+ if persistent:
+ try:
+ vm.undefine()
+ except:
+ self.log.exception("%s", "Failed to undefine domain")
+ pass
+
+ def openConnection(self):
+ """
+ opens a connection to the remote host
+ and stores it in self.hypervisor
+ """
+ self.log.info("Attempting to connect to libvirt at %s", self.host)
+ try:
+ hostHypervisor = libvirt.open(self.URI)
+ except:
+ self.log.warning(
+ "Failed to connect to %s. Trying again", self.host
+ )
+ time.sleep(5)
+ try:
+ hostHypervisor = libvirt.open(self.URI)
+ except:
+ self.log.exception("Cannot connect to %s. Exiting", self.host)
+ exit(1)
+
+ if hostHypervisor is None:
+ self.log.error("Failed to connect to %s. Exiting", self.host)
+ exit(1)
+ self.hypervisor = hostHypervisor
+
+ def restartVM(self, vm):
+ """
+ causes the given vm to reboot
+ """
+ dom = self.hypervisor.lookupByName(vm)
+ dom.destroy()
+ time.sleep(15)
+ dom.create()
+
+ def close(self):
+ """
+ Closes connection to remote hypervisor
+ """
+ self.log.info("Closing connection to the hypervisor %s", self.host)
+ self.hypervisor.close()
+
+ def defineAllDomains(self, path):
+ """
+ Defines a domain from all the xml files in a directory
+ """
+ files = Utilities.getXMLFiles(path)
+ definitions = []
+ for xml_desc in files:
+ definitions.append(xml_desc.read())
+
+ for definition in definitions:
+ self.defineVM(definition)
+
+ def createAllNetworks(self, path):
+ """
+ Creates a network from all xml files in a directory
+ """
+ files = Utilities.getXMLFiles(path)
+ definitions = []
+ for xml_desc in files:
+ definitions.append(Utilities.fileToString(xml_desc))
+
+ for definition in definitions:
+ self.createNet(definition)
+
+ def createNet(self, config):
+ """
+ creates the network on the remote host
+ config is the xml in string representation
+ that defines the network
+ """
+ if self.checkNet(config):
+ network = self.hypervisor.networkDefineXML(config)
+
+ if network is None:
+ name = self.getName(config)
+ self.log.warning("Failed to define network %s", name)
+ network.create()
+ if network.isActive() == 1:
+ net = network.name()
+ self.log.info("Successfully defined network %s", net)
+ self.networks.append(network)
+
+ def checkNet(self, config):
+ """
+ Checks if another net with the same name exists, and
+ deletes that network if one is found
+ """
+ allGood = False
+ netName = Utilities.getName(config)
+ if netName not in self.hypervisor.listNetworks():
+ return True
+ else: # net name is already used
+ self.log.warning(
+ "Network %s already exists. Trying to delete it", netName
+ )
+ network = self.hypervisor.networkLookupByName(netName)
+ self.deleteNet(network)
+ allGood = True
+ return allGood
+
+ def deleteNet(self, net):
+ """
+ removes the given network from the host
+ """
+ active = net.isActive()
+ persistent = net.isPersistent()
+ if active:
+ try:
+ net.destroy()
+ except:
+ self.log.warning("%s", "Failed to destroy network")
+
+ if persistent:
+ try:
+ net.undefine()
+ except:
+ self.log.warning("%s", "Failed to undefine network")
+
+ def go(self):
+ """
+ This method does all the work of this class,
+ Parsing the net and vm config files and creating
+ all the requested nets/domains
+ returns a list of all networks and a list of all domains
+ as Network and Domain objects
+ """
+ nets = self.makeNetworks(self.net_conf)
+ doms = self.makeDomains(self.dom_conf)
+ return doms, nets
+
+ def makeNetworks(self, conf):
+ """
+ Given a path to a config file, this method
+ parses the config and creates all requested networks,
+ and returns them in a list of Network objects
+ """
+ networks = []
+ definitions = Network.parseConfigFile(conf)
+ for definition in definitions:
+ network = Network(definition)
+ networks.append(network)
+ self.createNet(network.toXML())
+ return networks
+
+ def makeDomains(self, conf):
+ """
+ Given a path to a config file, this method
+ parses the config and creates all requested vm's,
+ and returns them in a list of Domain objects
+ """
+ domains = []
+ definitions = Domain.parseConfigFile(conf)
+ for definition in definitions:
+ domain = Domain(definition)
+ domains.append(domain)
+ self.defineVM(domain.toXML())
+ return domains
+
+ @staticmethod
+ def getName(xmlString):
+ """
+ given xml with a name tag, this returns the value of name
+ eg:
+ <name>Parker</name>
+ returns 'Parker'
+ """
+ xmlDoc = xml.dom.minidom.parseString(xmlString)
+ nameNode = xmlDoc.documentElement.getElementsByTagName('name')
+ name = str(nameNode[0].firstChild.nodeValue)
+ return name
--- /dev/null
+from abc import ABCMeta, abstractmethod
+import ldap
+import os
+import random
+from base64 import b64encode
+from database import BookingDataBase
+
+
+class VPN_BaseClass:
+ """
+ the vpn handler abstract class / interface
+
+ """
+ __metaclass__ = ABCMeta
+
+ @abstractmethod
+ def __init__(self, config):
+ """
+ config is the parsed vpn.yaml file
+ """
+ pass
+
+ @abstractmethod
+ def makeNewUser(self, user=None):
+ """
+ This method is called when a vpn user is needed.
+ This method should create a vpn user in whatever
+ runs the vpn in our infrastructure. returns the
+ credentials for the vpn user and some uid
+ that will be associated with the booking in the
+ database. This uid is used to track the vpn user and
+ to delete the user when there are no bookings associated
+ with that uid.
+ """
+ user = "username"
+ passwd = "password"
+ uid = "some way for you to identify this user in the database"
+ return user, passwd, uid
+
+ @abstractmethod
+ def removeOldUsers(self):
+ """
+ checks the list of all vpn users against a list of
+ vpn users associated with active bookings and removes
+ users who dont have an active booking
+
+ If you want your vpn accounts to be persistent,
+ you can just ignore this
+ """
+ pass
+
+
+names = [
+ 'frodo baggins', 'samwise gamgee', 'peregrin took', 'meriadoc brandybuck',
+ 'bilbo baggins', 'gandalf grey', 'aragorn dunadan', 'arwen evenstar',
+ 'saruman white', 'pippin took', 'merry brandybuck', 'legolas greenleaf',
+ 'gimli gloin', 'anakin skywalker', 'padme amidala', 'han solo',
+ 'jabba hut', 'mace windu', 'sount dooku', 'qui-gon jinn',
+ 'admiral ackbar', 'emperor palpatine'
+]
+
+
+class VPN:
+ """
+ This class communicates with the ldap server to manage vpn users.
+ This class extends the above ABC, and implements the makeNewUser,
+ removeOldUser, and __init__ abstract functions you must override to
+ extend the VPN_BaseClass
+ """
+
+ def __init__(self, config):
+ """
+ init takes the parsed vpn config file as an arguement.
+ automatically connects and authenticates on the ldap server
+ based on the configuration file
+ """
+ self.config = config
+ server = config['server']
+ self.uri = "ldap://"+server
+
+ self.conn = None
+ user = config['authentication']['user']
+ pswd = config['authentication']['pass']
+ if os.path.isfile(pswd):
+ pswd = open(pswd).read()
+ self.connect(user, pswd)
+
+ def connect(self, root_dn, root_pass):
+ """
+ Opens a connection to the server in the config file
+ and authenticates as the given user
+ """
+ self.conn = ldap.initialize(self.uri)
+ self.conn.simple_bind_s(root_dn, root_pass)
+
+ def addUser(self, full_name, passwd):
+ """
+ Adds a user to the ldap server. Creates the new user with the classes
+ and in the directory given in the config file.
+ full_name should be two tokens seperated by a space. The first token
+ will become the username
+ private helper function for the makeNewUser()
+ """
+ first = full_name.split(' ')[0]
+ last = full_name.split(' ')[1]
+ user_dir = self.config['directory']['user']
+ user_dir += ','+self.config['directory']['root']
+ dn = "uid=" + first + ',' + user_dir
+ record = [
+ ('objectclass', ['top', 'inetOrgPerson']),
+ ('uid', first),
+ ('cn', full_name),
+ ('sn', last),
+ ('userpassword', passwd),
+ ('ou', self.config['directory']['user'].split('=')[1])
+ ]
+ self.conn.add_s(dn, record)
+ return dn
+
+ def makeNewUser(self, name=None):
+ """
+ creates a new user in the ldap database, with the given name
+ if supplied. If no name is given, we will try to select from the
+ pre-written list above, and will resort to generating a random string
+ as a username if the preconfigured names are all taken.
+ Returns the username and password the user needs to authenticate, and
+ the dn that we can use to manage the user.
+ """
+ if name is None:
+ i = 0
+ while not self.checkName(name):
+ i += 1
+ if i == 20:
+ name = self.randoString(8)
+ name += ' '+self.randoString(8)
+ break # generates a random name to prevent infinite loop
+ name = self.genUserName()
+ passwd = self.randoString(15)
+ dn = self.addUser(name, passwd)
+ return name, passwd, dn
+
+ def checkName(self, name):
+ """
+ returns true if the name is available
+ """
+ if name is None:
+ return False
+ uid = name.split(' ')[0]
+ base = self.config['directory']['user'] + ','
+ base += self.config['directory']['root']
+ filtr = '(uid=' + uid + ')'
+ timeout = 5
+ ans = self.conn.search_st(
+ base,
+ ldap.SCOPE_SUBTREE,
+ filtr,
+ timeout=timeout
+ )
+ return len(ans) < 1
+
+ @staticmethod
+ def randoString(n):
+ """
+ uses /dev/urandom to generate a random string of length n
+ """
+ n = int(n)
+ # defines valid characters
+ alpha = 'abcdefghijklmnopqrstuvwxyz'
+ alpha_num = alpha
+ alpha_num += alpha.upper()
+ alpha_num += "0123456789"
+
+ # generates random string from /dev/urandom
+ rnd = b64encode(os.urandom(3*n)).decode('utf-8')
+ random_string = ''
+ for char in rnd:
+ if char in alpha_num:
+ random_string += char
+ return str(random_string[:n])
+
+ def genUserName(self):
+ """
+ grabs a random name from the list above
+ """
+ i = random.randint(0, len(names) - 1)
+ return names[i]
+
+ def deleteUser(self, dn):
+ self.conn.delete(dn)
+
+ def getAllUsers(self):
+ """
+ returns all the user dn's in the ldap database in a list
+ """
+ base = self.config['directory']['user'] + ','
+ base += self.config['directory']['root']
+ filtr = '(objectclass='+self.config['user']['objects'][-1]+')'
+ timeout = 10
+ ans = self.conn.search_st(
+ base,
+ ldap.SCOPE_SUBTREE,
+ filtr,
+ timeout=timeout
+ )
+ users = []
+ for user in ans:
+ users.append(user[0]) # adds the dn of each user
+ return users
+
+ def removeOldUsers(self):
+ """
+ removes users from the ldap server who dont have any active bookings.
+ will not delete a user if their uid's are named in the config
+ file as permanent users.
+ """
+ db = self.config['database']
+ # the dn of all users who have an active booking
+ active_users = BookingDataBase(db).getVPN()
+ all_users = self.getAllUsers()
+ for user in all_users:
+ # checks if they are a permanent user
+ if self.is_permanent_user(user):
+ continue
+ # deletes the user if they dont have an active booking
+ if user not in active_users:
+ self.deleteUser(user)
+
+ def is_permanent_user(self, dn):
+ for user in self.config['permanent_users']:
+ if (user in dn) or (dn in user):
+ return True
+ return False
+
+
+VPN_BaseClass.register(VPN)
--- /dev/null
+"""
+#############################################################################
+#Copyright 2017 Parker Berberian and others #
+# #
+#Licensed under the Apache License, Version 2.0 (the "License"); #
+#you may not use this file except in compliance with the License. #
+#You may obtain a copy of the License at #
+# #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+# #
+#Unless required by applicable law or agreed to in writing, software #
+#distributed under the License is distributed on an "AS IS" BASIS, #
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
+#See the License for the specific language governing permissions and #
+#limitations under the License. #
+#############################################################################
+"""
+import sqlite3
+import sys
+import time
+
+
+class HostDataBase:
+ """
+ This class talks with a simple sqlite database and can select a free host
+ when one is needed.
+ The layout of the database is:
+ TABLE host:
+ name <hostname> status <status_code> book_start
+ <Unix timestamp> book_end <Unix timestamp>
+ status_codes:
+ 0 - idle
+ 1 - deploying
+ 2 - deployed, in use
+ 3 - expired, ready to be reset
+ """
+
+ def __init__(self, path):
+ """
+ init function. Will create the file at the end of path
+ if it doesnt already exist
+ """
+ self.database = sqlite3.connect(path)
+ self.cursor = self.database.cursor()
+
+ def resetHosts(self, hosts):
+ """
+ Recreates the host table in the database.
+ WILL ERASE ALL DATA. USE WITH CAUTION.
+ """
+ try:
+ self.cursor.execute("DROP TABLE hosts")
+ self.createTable()
+ except:
+ pass
+
+ for host in hosts:
+ self.addHost(host)
+
+ def createTable(self):
+ """
+ This method creates the table hosts with
+ a name and status field
+ """
+ self.cursor.execute("CREATE TABLE hosts (name text, status integer)")
+ self.database.commit()
+
+ def addHost(self, name):
+ """
+ Adds a host with name to the available hosts.
+ When first added, the host is assumed to be idle.
+ """
+ host = (name, )
+ self.cursor.execute("INSERT INTO hosts VALUES (?, 0) ", host)
+ self.database.commit()
+
+ def getHost(self, requested=None):
+ """
+ Returns the name of an available host.
+ If a host is specifically requested,
+ that host is returned.
+ If the requested host is not available,
+ this method will throw an error.
+ If no host is specificaly requested,
+ the next available host is returned.
+ """
+ self.cursor.execute("SELECT name FROM hosts WHERE status = 0")
+ hostList = self.cursor.fetchall()
+ if len(hostList) < 1:
+ # throw and exception
+ sys.exit(1)
+ host = None
+ if requested is not None:
+ if (requested, ) in hostList and self.hostIsIdle(requested):
+ host = requested # If requested, exists, and idle, return it
+ else:
+ sys.exit(1)
+ else:
+ host = hostList[0][0]
+ self.makeHostBusy(host)
+ return host
+
+ def makeHostBusy(self, name):
+ """
+ makes the status of host 'name' equal 1,
+ making it 'busy'
+ """
+ host = (name, )
+ self.cursor.execute("UPDATE hosts SET status = 1 WHERE name=?", host)
+ self.database.commit()
+
+ def makeHostDeployed(self, name):
+ """
+ makes the status of host 'name' equal 2,
+ making it 'deployed' and/or in use
+ """
+ host = (name, )
+ self.cursor.execute("UPDATE hosts SET status = 2 WHERE name=?", host)
+ self.database.commit()
+
+ def makeHostExpired(self, name):
+ """
+ makes the status of host 'name' equal 3,
+ meaning its booking has ended and needs to be cleaned.
+ """
+ host = (name, )
+ self.cursor.execute("UPDATE hosts SET status = 3 WHERE name=?", host)
+ self.database.commit()
+
+ def getExpiredHosts(self):
+ """
+ returns a list of all hosts with an expired booking that
+ need to be cleaned.
+ """
+ self.cursor.execute("SELECT name FROM hosts where status = 3")
+ host_tuples = self.cursor.fetchall()
+ hosts = []
+ for host in host_tuples:
+ hosts.append(host[0])
+ return hosts # returns list of strings, not tuples
+
+ def hostIsBusy(self, name):
+ """
+ returns True if the host is not idle
+ """
+ host = (name, )
+ self.cursor.execute("SELECT status FROM hosts WHERE name=?", host)
+ stat = self.cursor.fetchone()[0]
+ if stat < 1:
+ return False
+ return True
+
+ def hostIsIdle(self, name):
+ """
+ returns True if the host is idle.
+ """
+ return not self.hostIsBusy(name)
+
+ def getAllHosts(self):
+ """
+ returns the whole host database.
+ """
+ self.cursor.execute("SELECT * FROM hosts")
+ return self.cursor.fetchall()
+
+ def close(self):
+ """
+ commits and closes connection to the database file.
+ """
+ self.database.commit()
+ self.database.close()
+
+
+class BookingDataBase:
+ """
+ Database to hold all active bookings for our servers.
+ Database contains table bookings - can be same or different
+ db file as the host database
+ bookings contains a field for every json key from the pharos dashboard,
+ plus a "status" integer which is either
+ 0 - waiting to start
+ 1 - started
+ 2 - booking over
+
+ As written, the pharos listener will immediately store all bookings that
+ are both for your dev pods and not
+ yet over, regardless of when the booking starts. Once the booking ends
+ and the dev pod is cleaned, the booking is deleted to save space and cpu.
+ """
+
+ def __init__(self, path):
+ """
+ creates a BookingDataBase object with the database located
+ at path. if path does not yet exist, it will be created.
+ """
+ self.database = sqlite3.connect(path)
+ self.cursor = self.database.cursor()
+
+ def createTable(self):
+ """
+ Creates table in the database to store booking information
+ """
+ try:
+ self.cursor.execute("DROP TABLE bookings")
+ except:
+ pass
+ self.cursor.execute("""CREATE TABLE bookings
+ (id integer, resource_id integer, start double, end double,
+ installer_name text, scenario_name text,
+ purpose text, status integer, vpn text)""")
+ self.database.commit()
+
+ def checkAddBooking(self, booking):
+ """
+ This method accepts a JSON booking definition from the dashboard
+ api and adds it to the database if it does not already exist.
+ """
+ # first, check if booking is already expired
+ if time.time() > booking['end']:
+ return
+ # check if booking is in database already
+ b_id = (booking['id'], )
+ self.cursor.execute("SELECT * FROM bookings WHERE id=?", b_id)
+ if len(self.cursor.fetchall()) > 0: # booking already in the db
+ return
+ tup = (
+ booking['id'],
+ booking['resource_id'],
+ booking['start'],
+ booking['end'],
+ booking['installer_name'],
+ booking['scenario_name'],
+ booking['purpose'],
+ 0,
+ ''
+ )
+ self.cursor.execute(
+ "INSERT INTO bookings VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", tup)
+ self.database.commit()
+
+ def removeBooking(self, idNum):
+ """
+ deletes booking with given id from the database.
+ """
+ booking_id = (idNum, )
+ self.cursor.execute("DELETE FROM bookings WHERE id=?", booking_id)
+
+ def getBookings(self):
+ """
+ returns a list of all bookings.
+ """
+ self.cursor.execute("SELECT * FROM bookings")
+ return self.cursor.fetchall()
+
+ def setStatus(self, booking_id, status):
+ """
+ sets the status of the booking with booking id booking_id.
+ as noted above, the status codes are:
+ 0 - not yet started
+ 1 - started, but not yet over
+ 2 - over, expired
+ """
+ data = (status, booking_id)
+ self.cursor.execute("UPDATE bookings SET status=? WHERE id=?", data)
+ self.database.commit()
+
+ def setVPN(self, resource, uid):
+ data = (uid, resource, 1)
+ self.cursor.execute(
+ "UPDATE bookings SET vpn=? WHERE resource_id=? AND status=?",
+ data
+ )
+ self.database.commit()
+
+ def getVPN(self):
+ """
+ returns a list of all vpn users associated with current
+ bookings.
+ """
+ self.cursor.execute("SELECT vpn FROM bookings WHERE status=1")
+ users_messy = self.cursor.fetchall()
+ users = []
+ for user in users_messy:
+ user = user[0] # get string rather than tuple
+ user = user.strip()
+ if len(user) < 1:
+ continue
+ users.append(user) # a list of non-empty strings
+ return users
+
+ def close(self):
+ """
+ commits changes and closes connection to db file.
+ """
+ self.database.commit()
+ self.database.close()
--- /dev/null
+#!/usr/bin/python
+"""
+#############################################################################
+#Copyright 2017 Parker Berberian and others #
+# #
+#Licensed under the Apache License, Version 2.0 (the "License"); #
+#you may not use this file except in compliance with the License. #
+#You may obtain a copy of the License at #
+# #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+# #
+#Unless required by applicable law or agreed to in writing, software #
+#distributed under the License is distributed on an "AS IS" BASIS, #
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
+#See the License for the specific language governing permissions and #
+#limitations under the License. #
+#############################################################################
+"""
+
+import sys
+import yaml
+from pod_manager import Pod_Manager
+
+"""
+This file is the first executed when a booking begins.
+"""
+
+usage = """
+./deploy [--config CONFIG_FILE] [--host HOSTNAME] [--reset]
+"""
+
+
+def main(config_path, host):
+ """
+ starts the deployment with the given configuration.
+ """
+ config = yaml.safe_load(open(config_path))
+
+ manager = Pod_Manager(config, requested_host=host)
+ manager.start_deploy()
+
+
+def reset(config_path, host):
+ """
+ Tells the Pod Manager to clean and reset the given host.
+ """
+ config = yaml.safe_load(open(config_path))
+ Pod_Manager(config, requested_host=host, reset=True)
+
+
+if __name__ == "__main__":
+ # parse command line
+ host = None
+
+ if "--help" in sys.argv:
+ print usage
+ sys.exit(0)
+
+ if "--config" in sys.argv:
+ try:
+ conf = sys.argv[1+sys.argv.index("--config")]
+ open(conf)
+ except Exception:
+ print "bad config file"
+ sys.exit(1)
+ if "--host" in sys.argv:
+ try:
+ host = sys.argv[1+sys.argv.index("--host")]
+ except:
+ "host not provided. Exiting"
+ sys.exit(1)
+
+ try:
+ config_file = yaml.safe_load(open(conf))
+ except:
+ print "Failed to read from config file"
+ sys.exit(1)
+ # reset or deploy host
+ if "--reset" in sys.argv:
+ reset(conf, host)
+ else:
+ main(conf, host)
--- /dev/null
+"""
+#############################################################################
+#Copyright 2017 Parker Berberian and others #
+# #
+#Licensed under the Apache License, Version 2.0 (the "License"); #
+#you may not use this file except in compliance with the License. #
+#You may obtain a copy of the License at #
+# #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+# #
+#Unless required by applicable law or agreed to in writing, software #
+#distributed under the License is distributed on an "AS IS" BASIS, #
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
+#See the License for the specific language governing permissions and #
+#limitations under the License. #
+#############################################################################
+"""
+
+import logging
+from api.libvirt_api import Libvirt
+
+
+class Deployment_Manager:
+ """
+ This class manages the deployment of OPNFV on a booked host
+ if it was requested. If no OPNFV installer was requested, this class will
+ create the virtual machines and networks in the config files and exit.
+ """
+ def __init__(self, installerType, scenario, utility):
+ """
+ init function
+ """
+ # installerType will either be the constructor for an installer or None
+ self.installer = installerType
+ self.virt = Libvirt(
+ utility.host,
+ net_conf=utility.conf['hypervisor_config']['networks'],
+ dom_conf=utility.conf['hypervisor_config']['vms']
+ )
+ self.host = utility.host
+ self.util = utility
+
+ def getIso(self):
+ """
+ checks if any of the domains expect an ISO file to exist
+ and retrieves it.
+ """
+ isoDom = None
+ for dom in self.doms:
+ if dom.iso['used']:
+ isoDom = dom
+ break
+ if isoDom:
+ path = isoDom.iso['location']
+ url = isoDom.iso['URL']
+ self.util.sshExec(['wget', '-q', '-O', path, url])
+
+ def getDomMacs(self):
+ """
+ assigns the 'macs' instance variable to the domains
+ so that they know the mac addresses of their interfaces.
+ """
+ for dom in self.doms:
+ dom.macs = self.virt.getMacs(dom.name)
+
+ def makeDisks(self):
+ """
+ Creates the qcow2 disk files the domains expect on the remote host.
+ """
+ disks = []
+ for dom in self.doms:
+ disks.append(dom.disk)
+ self.util.execRemoteScript("mkDisks.sh", disks)
+
+ def go(self):
+ """
+ 'main' function.
+ creates virtual machines/networks and either passes control to the
+ OPNFV installer, or finishes up if an installer was not requested.
+ """
+ log = logging.getLogger(self.util.hostname)
+ self.virt.setLogger(log)
+ log.info("%s", "Connecting to the host hypervisor")
+ self.virt.openConnection()
+ domains, networks = self.virt.go()
+ log.info("%s", "Created all networks and VM's on host")
+ self.doms = domains
+ self.nets = networks
+ if self.installer is None:
+ log.warning("%s", "No installer requested. Finishing deployment")
+ self.util.finishDeployment()
+ return
+ log.info("%s", "retrieving ISO")
+ self.getIso()
+ self.getDomMacs()
+ self.util.copyScripts()
+ self.makeDisks()
+ log.info("%s", "Beginning installation of OPNFV")
+ try:
+ installer = self.installer(
+ self.doms,
+ self.nets,
+ self.virt,
+ self.util
+ )
+ installer.go()
+ except Exception:
+ log.exception('%s', "failed to install OPNFV")
--- /dev/null
+"""
+#############################################################################
+#Copyright 2017 Parker Berberian and others #
+# #
+#Licensed under the Apache License, Version 2.0 (the "License"); #
+#you may not use this file except in compliance with the License. #
+#You may obtain a copy of the License at #
+# #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+# #
+#Unless required by applicable law or agreed to in writing, software #
+#distributed under the License is distributed on an "AS IS" BASIS, #
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
+#See the License for the specific language governing permissions and #
+#limitations under the License. #
+#############################################################################
+"""
+
+import xml.dom
+import xml.dom.minidom
+import yaml
+
+
+class Domain:
+ """
+ This class defines a libvirt vm abstraction that can parse our simple
+ config file and add all necessary boiler plate and info to write a full xml
+ definition of itself for libvirt.
+ """
+
+ def __init__(self, propertiesDict):
+ """
+ init function.
+ properiesDict should be one of the dictionaries returned by the static
+ method parseConfigFile
+ """
+ self.name = propertiesDict['name']
+ self.memory = propertiesDict['memory']
+ self.vcpus = propertiesDict['vcpus']
+ self.disk = propertiesDict['disk']
+ self.iso = propertiesDict['iso']
+ # the vm will either boot from an iso or pxe
+ self.netBoot = not self.iso['used']
+ self.interfaces = propertiesDict['interfaces']
+
+ def toXML(self):
+ """
+ combines the given configuration with a lot of
+ boiler plate to create a valid libvirt xml
+ definition of a domain.
+ returns a string
+ """
+ definition = xml.dom.minidom.parseString("<domain>\n</domain>")
+ definition.documentElement.setAttribute('type', 'kvm')
+
+ nameElem = definition.createElement('name')
+ nameElem.appendChild(definition.createTextNode(self.name))
+ definition.documentElement.appendChild(nameElem)
+
+ memElem = definition.createElement('memory')
+ memElem.appendChild(definition.createTextNode(str(self.memory)))
+ definition.documentElement.appendChild(memElem)
+
+ curMemElem = definition.createElement('currentMemory')
+ curMemElem.appendChild(definition.createTextNode(str(self.memory)))
+ definition.documentElement.appendChild(curMemElem)
+
+ vcpuElem = definition.createElement('vcpu')
+ vcpuElem.appendChild(definition.createTextNode(str(self.vcpus)))
+ definition.documentElement.appendChild(vcpuElem)
+
+ osElem = definition.createElement('os')
+
+ typeElem = definition.createElement('type')
+ typeElem.setAttribute('arch', 'x86_64')
+ typeElem.appendChild(definition.createTextNode('hvm'))
+ osElem.appendChild(typeElem)
+
+ if self.netBoot:
+ bootElem = definition.createElement('boot')
+ bootElem.setAttribute('dev', 'network')
+ osElem.appendChild(bootElem)
+
+ bootElem = definition.createElement('boot')
+ bootElem.setAttribute('dev', 'hd')
+ osElem.appendChild(bootElem)
+
+ if self.iso['used']:
+ bootElem = definition.createElement('boot')
+ bootElem.setAttribute('dev', 'cdrom')
+ osElem.appendChild(bootElem)
+
+ definition.documentElement.appendChild(osElem)
+
+ featureElem = definition.createElement('feature')
+ featureElem.appendChild(definition.createElement('acpi'))
+ featureElem.appendChild(definition.createElement('apic'))
+
+ definition.documentElement.appendChild(featureElem)
+
+ cpuElem = definition.createElement('cpu')
+ cpuElem.setAttribute('mode', 'custom')
+ cpuElem.setAttribute('match', 'exact')
+ modelElem = definition.createElement('model')
+ modelElem.appendChild(definition.createTextNode('Broadwell'))
+ cpuElem.appendChild(modelElem)
+
+ definition.documentElement.appendChild(cpuElem)
+
+ clockElem = definition.createElement('clock')
+ clockElem.setAttribute('offset', 'utc')
+
+ timeElem = definition.createElement('timer')
+ timeElem.setAttribute('name', 'rtc')
+ timeElem.setAttribute('tickpolicy', 'catchup')
+ clockElem.appendChild(timeElem)
+
+ timeElem = definition.createElement('timer')
+ timeElem.setAttribute('name', 'pit')
+ timeElem.setAttribute('tickpolicy', 'delay')
+ clockElem.appendChild(timeElem)
+
+ timeElem = definition.createElement('timer')
+ timeElem.setAttribute('name', 'hpet')
+ timeElem.setAttribute('present', 'no')
+ clockElem.appendChild(timeElem)
+
+ definition.documentElement.appendChild(clockElem)
+
+ poweroffElem = definition.createElement('on_poweroff')
+ poweroffElem.appendChild(definition.createTextNode('destroy'))
+
+ definition.documentElement.appendChild(poweroffElem)
+
+ rebootElem = definition.createElement('on_reboot')
+ rebootElem.appendChild(definition.createTextNode('restart'))
+
+ definition.documentElement.appendChild(rebootElem)
+
+ crashElem = definition.createElement('on_reboot')
+ crashElem.appendChild(definition.createTextNode('restart'))
+
+ definition.documentElement.appendChild(crashElem)
+
+ pmElem = definition.createElement('pm')
+ memElem = definition.createElement('suspend-to-mem')
+ memElem.setAttribute('enabled', 'no')
+ pmElem.appendChild(memElem)
+ diskElem = definition.createElement('suspend-to-disk')
+ diskElem.setAttribute('enabled', 'no')
+ pmElem.appendChild(diskElem)
+
+ definition.documentElement.appendChild(pmElem)
+
+ deviceElem = definition.createElement('devices')
+
+ emuElem = definition.createElement('emulator')
+ emuElem.appendChild(definition.createTextNode('/usr/libexec/qemu-kvm'))
+ deviceElem.appendChild(emuElem)
+
+ diskElem = definition.createElement('disk')
+ diskElem.setAttribute('type', 'file')
+ diskElem.setAttribute('device', 'disk')
+
+ driverElem = definition.createElement('driver')
+ driverElem.setAttribute('name', 'qemu')
+ driverElem.setAttribute('type', 'qcow2')
+ diskElem.appendChild(driverElem)
+
+ sourceElem = definition.createElement('source')
+ sourceElem.setAttribute('file', self.disk)
+ diskElem.appendChild(sourceElem)
+
+ targetElem = definition.createElement('target')
+ targetElem.setAttribute('dev', 'hda')
+ targetElem.setAttribute('bus', 'ide')
+ diskElem.appendChild(targetElem)
+
+ deviceElem.appendChild(diskElem)
+
+ if self.iso['used']:
+ diskElem = definition.createElement('disk')
+ diskElem.setAttribute('type', 'file')
+ diskElem.setAttribute('device', 'cdrom')
+
+ driverElem = definition.createElement('driver')
+ driverElem.setAttribute('name', 'qemu')
+ driverElem.setAttribute('type', 'raw')
+ diskElem.appendChild(driverElem)
+
+ sourceElem = definition.createElement('source')
+ sourceElem.setAttribute('file', self.iso['location'])
+ diskElem.appendChild(sourceElem)
+
+ targetElem = definition.createElement('target')
+ targetElem.setAttribute('dev', 'hdb')
+ targetElem.setAttribute('bus', 'ide')
+ diskElem.appendChild(targetElem)
+
+ diskElem.appendChild(definition.createElement('readonly'))
+ deviceElem.appendChild(diskElem)
+
+ for iface in self.interfaces:
+ ifaceElem = definition.createElement('interface')
+ ifaceElem.setAttribute('type', iface['type'])
+ sourceElem = definition.createElement('source')
+ sourceElem.setAttribute(iface['type'], iface['name'])
+ modelElem = definition.createElement('model')
+ modelElem.setAttribute('type', 'e1000')
+ ifaceElem.appendChild(sourceElem)
+ ifaceElem.appendChild(modelElem)
+ deviceElem.appendChild(ifaceElem)
+
+ graphicElem = definition.createElement('graphics')
+ graphicElem.setAttribute('type', 'vnc')
+ graphicElem.setAttribute('port', '-1')
+ deviceElem.appendChild(graphicElem)
+
+ consoleElem = definition.createElement('console')
+ consoleElem.setAttribute('type', 'pty')
+ deviceElem.appendChild(consoleElem)
+
+ definition.documentElement.appendChild(deviceElem)
+ return definition.toprettyxml()
+
+ def writeXML(self, filePath):
+ """
+ writes this domain's xml definition to the given file.
+ """
+ f = open(filePath, 'w')
+ f.write(self.toXML())
+ f.close()
+
+ @staticmethod
+ def parseConfigFile(path):
+ """
+ parses the domains config file
+ """
+ configFile = open(path, 'r')
+ try:
+ config = yaml.safe_load(configFile)
+ except Exception:
+ print "Invalid domain configuration. exiting"
+ return config
--- /dev/null
+"""
+#############################################################################
+#Copyright 2017 Parker Berberian and others #
+# #
+#Licensed under the Apache License, Version 2.0 (the "License"); #
+#you may not use this file except in compliance with the License. #
+#You may obtain a copy of the License at #
+# #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+# #
+#Unless required by applicable law or agreed to in writing, software #
+#distributed under the License is distributed on an "AS IS" BASIS, #
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
+#See the License for the specific language governing permissions and #
+#limitations under the License. #
+#############################################################################
+"""
--- /dev/null
+"""
+#############################################################################
+#Copyright 2017 Parker Berberian and others #
+# #
+#Licensed under the Apache License, Version 2.0 (the "License"); #
+#you may not use this file except in compliance with the License. #
+#You may obtain a copy of the License at #
+# #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+# #
+#Unless required by applicable law or agreed to in writing, software #
+#distributed under the License is distributed on an "AS IS" BASIS, #
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
+#See the License for the specific language governing permissions and #
+#limitations under the License. #
+#############################################################################
+"""
+
+import time
+import sys
+from installer import Installer
+from api.fuel_api import Fuel_api
+
+
+class Fuel_Installer(Installer):
+ """
+ This class is the installer for any OPNFV scenarios which use Fuel as the
+ installer. This class uses the libvirt api handler
+ to create all the virtual hosts,
+ then installs fuel and uses the fuel api handler
+ to create and deploy an openstack environment
+
+ This class will get much smarter and have less configuration hardcoded
+ as we grow support for more OPNFV scenarios
+ """
+
+ def __init__(self, doms, nets, libvirt_handler, util):
+ """
+ init function
+ Calls the super constructor
+ """
+ super(Fuel_Installer, self).__init__(doms, nets, libvirt_handler, util)
+ url = 'http://'+self.libvirt.host+':8000/'
+ self.handler = Fuel_api(url, self.log, 'admin', 'admin')
+ self.fuelNets = None
+
+ def bootMaster(self):
+ """
+ Boots the fuel master node and waits
+ for it to come up
+ """
+ self.libvirt.bootMaster()
+ time.sleep(100)
+
+ def bootNodes(self):
+ """
+ Boots all the slave nodes
+ """
+ self.libvirt.bootSlaves()
+
+ def waitForNodes(self, numNodes):
+ """
+ Waits for the nodes to pxe boot and be recognized by Fuel
+ """
+ done = False
+ self.log.info("Waiting for %i nodes to boot into Fuel", numNodes)
+ discoveredNodes = 0
+ while not done:
+ discoveredNodes = len(self.handler.getNodes())
+ nodes = int(discoveredNodes)
+ self.log.info("found %d nodes", nodes)
+
+ done = discoveredNodes == numNodes
+
+ def installMaster(self):
+ """
+ runs the fuelInstall script, which uses the fuel iso to
+ install fuel onto the master node
+ """
+ self.util.execRemoteScript("ipnat.sh", [self.libvirt.host])
+ self.util.execRemoteScript("fuelInstall.sh", [self.util.remoteDir])
+
+ def makeOpenstack(self):
+ """
+ creates an openstack environment and saves
+ the openstack id
+ """
+ self.osid = self.handler.createOpenstack()
+
+ def addNodesToOpenstack(self):
+ """
+ Adds the nodes to the openstack environment with
+ compute / controller + cinder roles
+ """
+ nodesList = [
+ {"id": 1, "roles": ["controller", "cinder"]},
+ {"id": 2, "roles": ["controller", "cinder"]},
+ {"id": 3, "roles": ["controller", "cinder"]},
+ {"id": 4, "roles": ["compute"]},
+ {"id": 5, "roles": ["compute"]}
+ ]
+
+ self.handler.addNodes(self.osid, nodesList)
+
+ def configNetworks(self):
+ """
+ configures the openstack networks by calling the 3 helper
+ methods
+ """
+ self.configPublicNet()
+ self.configStorageNet()
+ self.configManagementNet()
+
+ def configPublicNet(self):
+ """
+ sets the default public network
+ changes the cidr, gateway, and floating ranges
+ """
+ networks = self.handler.getNetworks(self.osid)
+ for net in networks['networks']:
+ if net['name'] == "public":
+ net["ip_ranges"] = [["10.20.1.10", "10.20.1.126"]]
+ net['cidr'] = "10.20.1.0/24"
+ net['gateway'] = "10.20.1.1"
+
+ # updates the floating ranges
+ rng = [["10.20.1.130", "10.20.1.254"]]
+ networks['networking_parameters']['floating_ranges'] = rng
+ self.handler.uploadNetworks(networks, self.osid)
+
+ def configStorageNet(self):
+ """
+ sets the default storage network to have the right
+ cidr and gateway, and no vlan
+ """
+ networks = self.handler.getNetworks(self.osid)
+ for net in networks['networks']:
+ if net['name'] == "storage":
+ net["ip_ranges"] = [["10.20.3.5", "10.20.3.254"]]
+ net["cidr"] = "10.20.3.0/24"
+ net["meta"]["notation"] = "ip_ranges"
+ net["meta"]["use_gateway"] = True
+ net["gateway"] = "10.20.3.1"
+ net["vlan_start"] = None
+ self.handler.uploadNetworks(networks, self.osid)
+
+ def configManagementNet(self):
+ """
+ sets the default management net to have the right
+ cidr and gatewar and no vlan
+ """
+ networks = self.handler.getNetworks(self.osid)
+ for net in networks['networks']:
+ if net['name'] == "management":
+ net["ip_ranges"] = [["10.20.2.5", "10.20.2.254"]]
+ net["cidr"] = "10.20.2.0/24"
+ net["meta"]["notation"] = "ip_ranges"
+ net["meta"]["use_gateway"] = True
+ net["gateway"] = "10.20.2.1"
+ net["vlan_start"] = None
+ self.handler.uploadNetworks(networks, self.osid)
+
+ # TODO: make this method smarter. I am making too many assumptions about
+ # the order of interfaces and networks
+ def configIfaces(self):
+ """
+ assigns the proper networks to each interface of the nodes
+ """
+ for x in range(1, 6):
+ idNum = x
+ ifaceJson = self.handler.getIfaces(idNum)
+
+ ifaceJson[0]['assigned_networks'] = [
+ {"id": 1, "name": "fuelweb_admin"},
+ {"id": 5, "name": "private"}
+ ]
+ ifaceJson[2]['assigned_networks'] = [
+ {"id": 4, "name": "storage"}
+ ]
+ ifaceJson[3]['assigned_networks'] = [
+ {"id": 3, "name": "management"}
+ ]
+ if idNum < 4:
+ ifaceJson[1]['assigned_networks'] = [{
+ "id": 2,
+ "name": "pubic"
+ }]
+
+ self.handler.setIfaces(idNum, ifaceJson)
+
+ def clearAdminIface(self, ifaceJson, node):
+ """
+ makes the admin interface have *only* the admin network
+ assigned to it
+ """
+ for iface in ifaceJson:
+ if iface['mac'] == node.macs['admin']:
+ iface['assigned_networks'] = [{
+ "id": 1,
+ "name": "fuelweb_admin"
+ }]
+
+ def deployOpenstack(self):
+ """
+ Once openstack is properly configured, this method
+ deploy OS and returns when OS is running
+ """
+ self.log.info("%s", "Deploying Openstack environment.")
+ self.log.info("%s", "This may take a while")
+ self.handler.deployOpenstack(self.osid)
+
+ def getKey(self):
+ """
+ Retrieves authentication tokens for the api handler,
+ while allowing the first few attempts to fail to
+ allow Fuel time to "wake up"
+ """
+ i = 0
+ while i < 20:
+ i += 1
+ try:
+ self.handler.getKey()
+ return
+ except Exception:
+ self.log.warning("%s", "Failed to talk to Fuel api")
+ self.log.warning("Exec try %d/20", i)
+ try:
+ self.handler.getKey()
+ except Exception:
+ self.logger.exception("%s", "Fuel api is unavailable")
+ sys.exit(1)
+
+ def go(self):
+ """
+ This method does all the work of this class.
+ It installs the master node, boots the slaves
+ into Fuel, creates and configures OS, and then
+ deploys it and uses NAT to make the horizon dashboard
+ reachable
+ """
+ self.libvirt.openConnection()
+ self.log.info('%s', 'installing the Fuel master node.')
+ self.log.info('%s', 'This will take some time.')
+ self.installMaster()
+ time.sleep(60)
+ self.getKey()
+ self.log.info('%s', 'The master node is installed.')
+ self.log.info('%s', 'Waiting for bootstrap image to build')
+ self.handler.waitForBootstrap()
+ self.bootNodes()
+ self.waitForNodes(5)
+ self.log.info('%s', "Defining an openstack environment")
+ self.makeOpenstack()
+ self.addNodesToOpenstack()
+ self.log.info('%s', "configuring interfaces...")
+ self.configIfaces()
+ self.log.info('%s', "configuring networks...")
+ self.configNetworks()
+ self.deployOpenstack()
+
+ horizon = self.handler.getHorizonIP(self.osid)
+ self.util.execRemoteScript(
+ '/horizonNat.sh', [self.libvirt.host, horizon])
+ notice = "You may access the Openstack dashboard at %s/horizon"
+ self.log.info(notice, self.libvirt.host)
+
+ self.libvirt.close()
+ self.util.finishDeployment()
--- /dev/null
+"""
+#############################################################################
+#Copyright 2017 Parker Berberian and others #
+# #
+#Licensed under the Apache License, Version 2.0 (the "License"); #
+#you may not use this file except in compliance with the License. #
+#You may obtain a copy of the License at #
+# #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+# #
+#Unless required by applicable law or agreed to in writing, software #
+#distributed under the License is distributed on an "AS IS" BASIS, #
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
+#See the License for the specific language governing permissions and #
+#limitations under the License. #
+#############################################################################
+"""
+
+
+class Installer(object):
+ """
+ This is a simple base class to define a single constructor
+ for all the different installer types.
+ I may move more functionality to this class as we add support for more
+ installers and there becomes common fucntions that would be nice to share
+ between installers.
+ """
+
+ def __init__(self, domList, netList, libvirt_handler, util):
+ self.doms = domList
+ self.nets = netList
+ self.libvirt = libvirt_handler
+ self.osid = 0
+ self.util = util
+ self.log = util.createLogger(util.hostname)
--- /dev/null
+"""
+#############################################################################
+#Copyright 2017 Parker Berberian and others #
+# #
+#Licensed under the Apache License, Version 2.0 (the "License"); #
+#you may not use this file except in compliance with the License. #
+#You may obtain a copy of the License at #
+# #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+# #
+#Unless required by applicable law or agreed to in writing, software #
+#distributed under the License is distributed on an "AS IS" BASIS, #
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
+#See the License for the specific language governing permissions and #
+#limitations under the License. #
+#############################################################################
+"""
+
+"""
+This class will install Joid onto the remote host.
+Currently only supports joid's "default" configuration
+"""
+
+
+class Joid_Installer:
+
+ def __init__(self, doms, nets, libvirt_handler, util):
+ """
+ init function calls the super constructor
+ """
+ super(Joid_Installer, self).__init__(doms, nets, libvirt_handler, util)
+
+ def go(self):
+ """
+ does all the work of this class.
+ Currently just runs the joidInstall script, which installs joid
+ onto the remote host
+ """
+ self.logger.info("%s", "Executing joid virtual installation")
+ self.util.execRemoteScript("joidInstall.sh")
--- /dev/null
+#!/usr/bin/python
+"""
+#############################################################################
+#Copyright 2017 Parker Berberian and others #
+# #
+#Licensed under the Apache License, Version 2.0 (the "License"); #
+#you may not use this file except in compliance with the License. #
+#You may obtain a copy of the License at #
+# #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+# #
+#Unless required by applicable law or agreed to in writing, software #
+#distributed under the License is distributed on an "AS IS" BASIS, #
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
+#See the License for the specific language governing permissions and #
+#limitations under the License. #
+#############################################################################
+"""
+import subprocess
+import sys
+import os
+import yaml
+
+"""
+This is the file that the user will execute to start the whole process.
+This file will start the pharos api listener in a new process and then exit.
+"""
+
+
+def checkArgs():
+ """
+ error checks the cmd line args and gets the path
+ of the config file
+ """
+ usage = "./listen.py --config <path_to_pharos_config>"
+ if "--help" in sys.argv:
+ print usage
+ sys.exit(0)
+
+ if "--config" not in sys.argv:
+ print usage
+ sys.exit(1)
+
+ try:
+ i = sys.argv.index("--config")
+ config_file = sys.argv[i+1]
+ # verifies that the file exists, is readable, and formatted correctly
+ yaml.safe_load(open(config_file))
+ return config_file
+ except Exception:
+ print "Bad config file"
+ sys.exit(1)
+
+
+# reads args and starts the pharos listener in the background
+config = checkArgs()
+source_dir = os.path.dirname(os.path.realpath(__file__))
+pharos_path = os.path.join(source_dir, "pharos.py")
+subprocess.Popen(['/usr/bin/python', pharos_path, '--config', config])
--- /dev/null
+"""
+#############################################################################
+#Copyright 2017 Parker Berberian and others #
+# #
+#Licensed under the Apache License, Version 2.0 (the "License"); #
+#you may not use this file except in compliance with the License. #
+#You may obtain a copy of the License at #
+# #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+# #
+#Unless required by applicable law or agreed to in writing, software #
+#distributed under the License is distributed on an "AS IS" BASIS, #
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
+#See the License for the specific language governing permissions and #
+#limitations under the License. #
+#############################################################################
+"""
+
+import sys
+import xml.dom
+import xml.dom.minidom
+import yaml
+
+
+class Network:
+ """
+ This class has a similar role as the Domain class.
+ This class will parse a config file and
+ write the xml definitions of those networks for libvirt.
+ """
+
+ def __init__(self, propertiesDict):
+ """
+ init. propertiesDict should be
+ one of the dictionaries returned by parseConfigFile
+ """
+ self.name = propertiesDict['name']
+ self.brName = propertiesDict['brName']
+ self.brAddr = propertiesDict['brAddr']
+ self.netmask = propertiesDict['netmask']
+ self.forward = propertiesDict['forward']
+ self.dhcp = propertiesDict['dhcp']
+ self.cidr = propertiesDict['cidr']
+
+ def toXML(self):
+ """
+ Takes the config of this network and writes a valid xml definition
+ for libvirt.
+ returns a string
+ """
+ definition = xml.dom.minidom.parseString("<network>\n</network>")
+ nameElem = definition.createElement('name')
+ nameElem.appendChild(definition.createTextNode(self.name))
+ definition.documentElement.appendChild(nameElem)
+
+ if self.forward['used']:
+ forwardElem = definition.createElement('forward')
+ forwardElem.setAttribute('mode', self.forward['type'])
+ definition.documentElement.appendChild(forwardElem)
+
+ bridgeElem = definition.createElement('bridge')
+ bridgeElem.setAttribute('name', self.brName)
+ bridgeElem.setAttribute('stp', 'on')
+ bridgeElem.setAttribute('delay', '5')
+ definition.documentElement.appendChild(bridgeElem)
+
+ ipElem = definition.createElement('ip')
+ ipElem.setAttribute('address', self.brAddr)
+ ipElem.setAttribute('netmask', self.netmask)
+ if self.dhcp['used']:
+ dhcpElem = definition.createElement('dhcp')
+ rangeElem = definition.createElement('range')
+ rangeElem.setAttribute('start', self.dhcp['rangeStart'])
+ rangeElem.setAttribute('end', self.dhcp['rangeEnd'])
+ dhcpElem.appendChild(rangeElem)
+ ipElem.appendChild(dhcpElem)
+
+ definition.documentElement.appendChild(ipElem)
+
+ self.xml = definition.toprettyxml()
+ return self.xml
+
+ def writeXML(self, filePath):
+ """
+ writes xml definition to given file
+ """
+ f = open(filePath, 'w')
+ f.write(self.toXML())
+ f.close()
+
+ @staticmethod
+ def parseConfigFile(path):
+ """
+ parses given config file
+ """
+ configFile = open(path, 'r')
+ try:
+ config = yaml.safe_load(configFile)
+ except Exception:
+ print "Bad network configuration file. exiting"
+ sys.exit(1)
+
+ return config
--- /dev/null
+#!/usr/bin/python
+"""
+#############################################################################
+#Copyright 2017 Parker Berberian and others #
+# #
+#Licensed under the Apache License, Version 2.0 (the "License"); #
+#you may not use this file except in compliance with the License. #
+#You may obtain a copy of the License at #
+# #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+# #
+#Unless required by applicable law or agreed to in writing, software #
+#distributed under the License is distributed on an "AS IS" BASIS, #
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
+#See the License for the specific language governing permissions and #
+#limitations under the License. #
+#############################################################################
+"""
+
+import requests
+import time
+import calendar
+import subprocess
+import sys
+import yaml
+import os
+import logging
+from utilities import Utilities
+from database import BookingDataBase
+
+
+class Pharos_api:
+ """
+ This class listens to the dashboard and starts/stops bookings accordingly.
+ This class should run in the background indefinitely.
+ Do not execute this file directly - run ./listen.py instead
+ """
+ def __init__(self, config):
+ """
+ init function.
+ config is the already-parsed config file
+ """
+ self.conf = config
+ self.servers = yaml.safe_load(open(config['inventory']))
+ self.log = self.createLogger("pharos_api")
+ self.polling = 60 / int(config['polling'])
+ self.log.info(
+ "polling the dashboard once every %d seconds", self.polling)
+ self.dashboard = config['dashboard']
+ self.log.info("connecting to dashboard at %s", self.dashboard)
+ if os.path.isfile(config['token']):
+ self.token = open(config['token']).read()
+ else:
+ self.token = config['token']
+ self.updateHeader()
+ self.database = BookingDataBase(config['database'])
+ self.log.info("using database at %s", self.conf['database'])
+ self.deploy_path = os.path.join(
+ os.path.dirname(os.path.realpath(__file__)), "deploy.py")
+ if not os.path.isfile(self.deploy_path):
+ self.log.error(
+ "Cannot find the deployment script at %s", self.deploy_path)
+
+ def setToken(self, token):
+ """
+ Sets authentication token. Not yet needed.
+ """
+ self.token = token
+ self.updateHeader()
+
+ def setTokenFromFile(self, path):
+ """
+ reads auth token from a file. Not yet needed.
+ """
+ self.setToken(open(path).read())
+
+ def updateHeader(self):
+ """
+ updates the http header used when talking to the dashboard
+ """
+ self.header = {"Authorization": "Token " + self.token}
+
+ def listen(self):
+ """
+ this method will continuously poll the pharos dashboard.
+ If a booking is found on our server,
+ we will start a deployment in the background with the
+ proper config file for the requested
+ installer and scenario.
+ """
+ self.log.info("%s", "Beginning polling of dashboard")
+ try:
+ while True:
+ time.sleep(self.polling)
+ url = self.dashboard+"/api/bookings/"
+ bookings = requests.get(url, headers=self.header).json()
+ for booking in bookings:
+ if booking['resource_id'] in self.servers.keys():
+ self.convertTimes(booking)
+ self.database.checkAddBooking(booking)
+ self.checkBookings()
+ except Exception:
+ self.log.exception('%s', "failed to connect to dashboard")
+
+ self.listen()
+
+ def convertTimes(self, booking):
+ """
+ this method will take the time reported by Pharos in the
+ format yyyy-mm-ddThh:mm:ssZ
+ and convert it into seconds since the epoch,
+ for easier management
+ """
+ booking['start'] = self.pharosToEpoch(booking['start'])
+ booking['end'] = self.pharosToEpoch(booking['end'])
+
+ def pharosToEpoch(self, timeStr):
+ """
+ Converts the dates from the dashboard to epoch time.
+ """
+ time_struct = time.strptime(timeStr, '%Y-%m-%dT%H:%M:%SZ')
+ epoch_time = calendar.timegm(time_struct)
+ return epoch_time
+
+ def checkBookings(self):
+ """
+ This method checks all the bookings in our database to see if any
+ action is required.
+ """
+ # get all active bookings from database into a usable form
+ bookings = self.database.getBookings()
+ for booking in bookings:
+ # first, check if booking is over
+ if time.time() > booking[3]:
+ self.log.info("ending the booking with id %i", booking[0])
+ self.endBooking(booking)
+ # Then check if booking has begun and the host is still idle
+ elif time.time() > booking[2] and booking[7] < 1:
+ self.log.info("starting the booking with id %i", booking[0])
+ self.startBooking(booking)
+
+ def startBooking(self, booking):
+ """
+ Starts the scheduled booking on the requested host with
+ the correct config file.
+ The provisioning process gets spun up in a subproccess,
+ so the api listener is not interupted.
+ """
+ try:
+ host = self.servers[booking[1]]
+ self.log.info("Detected a new booking started for host %s", host)
+ config_file = self.conf['default_configs']["None"]
+ try:
+ config_file = self.conf['default_configs'][booking[4]]
+ except KeyError:
+ self.log.warning(
+ "No installer detected in the booking request.")
+ self.log.info("New booking started for host %s", host)
+ self.database.setStatus(booking[0], 1) # mark booking started
+ if not os.path.isfile(self.deploy_path):
+ error = "Cannot find the deploment script at %s"
+ self.log.error(error, self.deploy_path)
+ subprocess.Popen([
+ '/usr/bin/python',
+ self.deploy_path,
+ '--config', config_file,
+ '--host', host
+ ])
+ except Exception:
+ self.log.exception("Failed to start booking for %s", host)
+
+ def endBooking(self, booking):
+ """
+ Resets a host once its booking has ended.
+ """
+ try:
+ try:
+ config_file = self.conf['default_configs'][booking[4]]
+ except KeyError:
+ warn = "No installer detected in booking request"
+ self.log.warning("%s", warn)
+ config_file = self.conf['default_configs']["None"]
+
+ host = self.servers[booking[1]]
+ log = logging.getLogger(host)
+ log.info('Lease expired. Resetting host %s', host)
+ self.database.setStatus(booking[0], 3)
+ if not os.path.isfile(self.deploy_path):
+ err = "Cannot find deployment script at %s"
+ self.log.error(err, self.deploy_path)
+ subprocess.Popen([
+ '/usr/bin/python',
+ self.deploy_path,
+ '--config', config_file,
+ '--host', host,
+ '--reset'
+ ])
+ self.database.removeBooking(booking[0])
+ except Exception:
+ self.log.exception("Failed to end booking for %s", host)
+
+ def createLogger(self, name):
+ return Utilities.createLogger(name, self.conf['logging_dir'])
+
+
+if __name__ == "__main__":
+ if "--config" not in sys.argv:
+ print "Specify config file with --config option"
+ sys.exit(1)
+ config = None
+ try:
+ config_file = sys.argv[1+sys.argv.index('--config')]
+ config = yaml.safe_load(open(config_file))
+ except Exception:
+ sys.exit(1)
+ api = Pharos_api(config)
+ api.listen()
--- /dev/null
+#!/usr/bin/python
+"""
+#############################################################################
+#Copyright 2017 Parker Berberian and others #
+# #
+#Licensed under the Apache License, Version 2.0 (the "License"); #
+#you may not use this file except in compliance with the License. #
+#You may obtain a copy of the License at #
+# #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+# #
+#Unless required by applicable law or agreed to in writing, software #
+#distributed under the License is distributed on an "AS IS" BASIS, #
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
+#See the License for the specific language governing permissions and #
+#limitations under the License. #
+#############################################################################
+"""
+
+import time
+import sys
+import yaml
+import os
+from api.fog import FOG_Handler
+from utilities import Utilities
+from deployment_manager import Deployment_Manager
+from database import HostDataBase
+from installers import fuel
+from installers import joid
+
+
+class Pod_Manager:
+ """
+ This is the 'main' class that chooses a host and provisions & deploys it.
+ this class can be run directly from the command line,
+ or it can be called from the pharos dashboard listener when
+ a deployment is requested.
+ Either way, this file should be called with:
+ ./pod_manager.py --config <CONFIG_FILE>
+ """
+ # This dictionary allows me to map the supported installers to the
+ # respective installer classes, for easier parsing of the config file
+ INSTALLERS = {
+ "fuel": fuel.Fuel_Installer,
+ "joid": joid.Joid_Installer,
+ "none": None
+ }
+
+ def __init__(self, conf, requested_host=None, reset=False):
+ """
+ init function.
+ conf is the read and parsed config file for this deployment
+ requested_host is the optional hostname of the host you request
+ if reset, we just flash the host to a clean state and return.
+ """
+ self.conf = conf
+ if self.conf['installer'] is not None:
+ inst = Pod_Manager.INSTALLERS[self.conf['installer'].lower()]
+ self.conf['installer'] = inst
+ self.fog = FOG_Handler(self.conf['fog']['server'])
+ # Sets the fog keys, either from the config file
+ # or the secrets file the config points to
+ if os.path.isfile(self.conf['fog']['api_key']):
+ self.fog.getFogKeyFromFile(self.conf['fog']['api_key'])
+ else:
+ self.fog.setFogKey(self.conf['fog']['api_key'])
+
+ if os.path.isfile(self.conf['fog']['user_key']):
+ self.fog.getUserKeyFromFile(self.conf['fog']['user_key'])
+ else:
+ self.fog.setUserKey(self.conf['fog']['user_key'])
+ self.database = HostDataBase(self.conf['database'])
+ self.request = requested_host
+ if reset:
+ mac = self.fog.getHostMac(self.request)
+ log = self.conf['dhcp_log']
+ dhcp_serv = self.conf['dhcp_server']
+ ip = Utilities.getIPfromMAC(mac, log, remote=dhcp_serv)
+ self.flash_host(self.request, ip)
+
+ def start_deploy(self):
+ """
+ Ghosts the machine with the proper disk image and hands off
+ control to the deployment manager.
+ """
+ try:
+ host = self.database.getHost(self.request)
+ hostMac = self.fog.getHostMac(host)
+ dhcp_log = self.conf['dhcp_log']
+ dhcp_server = self.conf['dhcp_server']
+ host_ip = Utilities.getIPfromMAC(
+ hostMac, dhcp_log, remote=dhcp_server
+ )
+ util = Utilities(host_ip, host, self.conf)
+ util.resetKnownHosts()
+ log = Utilities.createLogger(host, self.conf['logging_dir'])
+ self.fog.setLogger(log)
+ log.info("Starting booking on host %s", host)
+ log.info("host is reachable at %s", host_ip)
+ log.info('ghosting host %s with clean image', host)
+ self.flash_host(host, host_ip, util)
+ log.info('Host %s imaging complete', host)
+ inst = self.conf['installer']
+ scenario = self.conf['scenario']
+ Deployment_Manager(inst, scenario, util).go()
+ except Exception:
+ log.exception("Encountered an unexpected error")
+
+ def flash_host(self, host, host_ip, util=None):
+ """
+ We do this using a FOG server, but you can use whatever fits into your
+ lab infrastructure. This method should put the host into a state as if
+ centos was just freshly installed, updated,
+ and needed virtualization software installed.
+ This is the 'clean' starting point we work from
+ """
+ self.fog.setImage(host, self.conf['fog']['image_id'])
+ self.fog.imageHost(host)
+ Utilities.restartRemoteHost(host_ip)
+ self.fog.waitForHost(host)
+ # if util is not given, then we are just
+ # flashing to reset after a booking expires
+ if util is not None:
+ time.sleep(30)
+ util.waitForBoot()
+ util.checkHost()
+ time.sleep(15)
+ util.checkHost()
+
+
+if __name__ == "__main__":
+ configFile = ""
+ host = ""
+ for i in range(len(sys.argv) - 1):
+ if "--config" in sys.argv[i]:
+ configFile = sys.argv[i+1]
+ elif "--host" in sys.argv[i]:
+ host = sys.argv[i+1]
+ if len(configFile) < 1:
+ print "No config file specified"
+ sys.exit(1)
+ configFile = yaml.safe_load(open(configFile))
+ manager = Pod_Manager(configFile, requested_host=host)
+ manager.start_deploy()
--- /dev/null
+#!/usr/bin/python
+"""
+#############################################################################
+#Copyright 2017 Parker Berberian and others #
+# #
+#Licensed under the Apache License, Version 2.0 (the "License"); #
+#you may not use this file except in compliance with the License. #
+#You may obtain a copy of the License at #
+# #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+# #
+#Unless required by applicable law or agreed to in writing, software #
+#distributed under the License is distributed on an "AS IS" BASIS, #
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
+#See the License for the specific language governing permissions and #
+#limitations under the License. #
+#############################################################################
+"""
+
+import sys
+import os
+import yaml
+from api.fog import FOG_Handler
+from database import HostDataBase
+from database import BookingDataBase
+
+"""
+This file just resets the host database with
+all the hosts in fog, with all of them
+showing as available
+
+This file is just provided to make populating the host db easier.
+If you wanted to do this yourself, you could do the following in
+a python command prompt:
+ from database import HostDataBase
+ db = HostDataBase("/path/to/file")
+ db.addHost("host-name")
+ db.addHost("host-name")
+ db.addHost("host-name")
+
+"""
+config = None
+if "--config" in sys.argv:
+ i = sys.argv.index("--config")
+ if len(sys.argv) > i+1 and os.path.isfile(sys.argv[i+1]):
+ try:
+ config = yaml.safe_load(open(sys.argv[i+1]))
+ except Exception:
+ print "failed to read config file. exiting"
+ sys.exit(1)
+ else:
+ print "config file not found. exiting"
+ sys.exit(1)
+else:
+ print "no config file given. Specify file with '--config <FILE_PATH>'"
+ sys.exit(1)
+
+host = False
+if "--host" in sys.argv or "--both" in sys.argv:
+ host = True
+
+booking = False
+if "--booking" in sys.argv or "--both" in sys.argv:
+ booking = True
+
+
+if host:
+
+ fog = FOG_Handler(
+ config['fog']['server']
+ )
+ if os.path.isfile(config['fog']['api_key']):
+ fog.getFogKeyFromFile(config['fog']['api_key'])
+ else:
+ fog.setFogKey(config['fog']['api_key'])
+
+ if os.path.isfile(config['fog']['user_key']):
+ fog.getUserKeyFromFile(config['fog']['user_key'])
+ else:
+ fog.setUserKey(config['fog']['user_key'])
+ hosts = fog.getHostsinGroup("vm")
+ host_names = []
+ for host in hosts:
+ host_names.append(host['name'])
+
+ # creates the directory of the db, if it doesnt yet exist
+ dbDir = os.path.dirname(config['database'])
+ if not os.path.isdir(dbDir):
+ os.makedirs(dbDir)
+
+ db = HostDataBase(config['database'])
+
+ # check if the table already exists or not
+ try:
+ db.cursor.execute("SELECT * FROM hosts")
+ except Exception as err:
+ if "no such table" in str(err):
+ db.createTable()
+
+ db.resetHosts(host_names)
+
+if booking:
+ db = BookingDataBase(config['database'])
+ db.createTable()
+ db.close()
+
+else:
+ print "you must specify the '--host', '--booking', or '--both' option"
+ print "depending on which database you wish to reset"
+ sys.exit(0)
--- /dev/null
+#!/bin/bash
+#############################################################################
+#Copyright 2017 Parker Berberian and others #
+# #
+#Licensed under the Apache License, Version 2.0 (the "License"); #
+#you may not use this file except in compliance with the License. #
+#You may obtain a copy of the License at #
+# #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+# #
+#Unless required by applicable law or agreed to in writing, software #
+#distributed under the License is distributed on an "AS IS" BASIS, #
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
+#See the License for the specific language governing permissions and #
+#limitations under the License. #
+#############################################################################
+
+
+# This just finds all processes from this program and kills them.
+
+
+PIDS=$(ps -ef | grep laas/source/ | grep python | awk '{print $2}')
+
+kill ${PIDS[*]}
--- /dev/null
+"""
+#############################################################################
+#Copyright 2017 Parker Berberian and others #
+# #
+#Licensed under the Apache License, Version 2.0 (the "License"); #
+#you may not use this file except in compliance with the License. #
+#You may obtain a copy of the License at #
+# #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+# #
+#Unless required by applicable law or agreed to in writing, software #
+#distributed under the License is distributed on an "AS IS" BASIS, #
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
+#See the License for the specific language governing permissions and #
+#limitations under the License. #
+#############################################################################
+"""
+
+import os
+import logging
+import string
+import sys
+import subprocess
+import xml.dom
+import xml.dom.minidom
+import re
+import random
+import yaml
+from database import HostDataBase, BookingDataBase
+from api.vpn import VPN
+LOGGING_DIR = ""
+
+
+class Utilities:
+ """
+ This class defines some useful functions that may be needed
+ throughout the provisioning and deployment stage.
+ The utility object is carried through most of the deployment process.
+ """
+ def __init__(self, host_ip, hostname, conf):
+ """
+ init function
+ host_ip is the ip of the target host
+ hostname is the FOG hostname of the host
+ conf is the parsed config file
+ """
+ self.host = host_ip
+ self.hostname = hostname
+ root_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+ self.scripts = os.path.join(root_dir, "hostScripts/")
+ self.remoteDir = "/root/hostScripts/"
+ self.conf = conf
+ self.logger = logging.getLogger(hostname)
+
+ def execRemoteScript(self, script, args=[]):
+ """
+ executes the given script on the
+ remote host with the given args.
+ script must be found in laas/hostScripts
+ """
+ cmd = [self.remoteDir+script]
+ for arg in args:
+ cmd.append(arg)
+ self.sshExec(cmd)
+
+ def waitForBoot(self):
+ """
+ Continually pings the host, waiting for it to boot
+ """
+ i = 0
+ while (not self.pingHost()) and i < 30:
+ i += 1
+ if i == 30:
+ self.logger.error("Host %s has not booted", self.host)
+ sys.exit(1)
+
+ def checkHost(self):
+ """
+ returns true if the host responds to two pings.
+ Sometimes, while a host is pxe booting, a host will
+ respond to one ping but quickly go back offline.
+ """
+ if self.pingHost() and self.pingHost():
+ return True
+ return False
+
+ def pingHost(self):
+ """
+ returns true if the host responds to a ping
+ """
+ i = 0
+ response = 1
+ cmd = "ping -c 1 "+self.host
+ cmd = cmd.split(' ')
+ nul = open(os.devnull, 'w')
+ while i < 10 and response != 0:
+ response = subprocess.call(cmd, stdout=nul, stderr=nul)
+ i = i + 1
+ if response == 0:
+ return True
+ return False
+
+ def copyDir(self, localDir, remoteDir):
+ """
+ uses scp to copy localDir to remoteDir on the
+ remote host
+ """
+ cmd = "mkdir -p "+remoteDir
+ self.sshExec(cmd.split(" "))
+ cmd = "scp -o StrictHostKeyChecking=no -r "
+ cmd += localDir+" root@"+self.host+":/root"
+ cmd = cmd.split()
+ nul = open(os.devnull, 'w')
+ subprocess.call(cmd, stdout=nul, stderr=nul)
+
+ def copyScripts(self):
+ """
+ Copies the hostScrpts dir to the remote host.
+ """
+ self.copyDir(self.scripts, self.remoteDir)
+
+ def sshExec(self, args):
+ """
+ executes args as an ssh
+ command on the remote host.
+ """
+ cmd = ['ssh', 'root@'+self.host]
+ for arg in args:
+ cmd.append(arg)
+ nul = open(os.devnull, 'w')
+ return subprocess.call(cmd, stdout=nul, stderr=nul)
+
+ def resetKnownHosts(self):
+ """
+ edits your known hosts file to remove the previous entry of host
+ Sometimes, the flashing process gives the remote host a new
+ signature, and ssh complains about it.
+ """
+ lines = []
+ sshFile = open('/root/.ssh/known_hosts', 'r')
+ lines = sshFile.read()
+ sshFile.close()
+ lines = lines.split('\n')
+ sshFile = open('/root/.ssh/known_hosts', 'w')
+ for line in lines:
+ if self.host not in line:
+ sshFile.write(line+'\n')
+ sshFile.close()
+
+ def restartHost(self):
+ """
+ restarts the remote host
+ """
+ cmd = ['shutdown', '-r', 'now']
+ self.sshExec(cmd)
+
+ @staticmethod
+ def randoString(length):
+ """
+ this is an adapted version of the code found here:
+ https://stackoverflow.com/questions/2257441/
+ random-string-generation-with-upper-case-letters-and-digits-in-python
+ generates a random alphanumeric string of length length.
+ """
+ randStr = ''
+ chars = string.ascii_uppercase + string.digits
+ for x in range(length):
+ randStr += random.SystemRandom().choice(chars)
+ return randStr
+
+ def changePassword(self):
+ """
+ Sets the root password to a random string and returns it
+ """
+ paswd = self.randoString(15)
+ command = "printf "+paswd+" | passwd --stdin root"
+ self.sshExec(command.split(' '))
+ return paswd
+
+ def markHostDeployed(self):
+ """
+ Tells the database that this host has finished its deployment
+ """
+ db = HostDataBase(self.conf['database'])
+ db.makeHostDeployed(self.hostname)
+ db.close()
+
+ def make_vpn_user(self):
+ """
+ Creates a vpn user and associates it with this booking
+ """
+ config = yaml.safe_load(open(self.conf['vpn_config']))
+ myVpn = VPN(config)
+ # name = dashboard.getUserName()
+ u, p, uid = myVpn.makeNewUser() # may pass name arg if wanted
+ self.logger.info("%s", "created new vpn user")
+ self.logger.info("username: %s", u)
+ self.logger.info("password: %s", p)
+ self.logger.info("vpn user uid: %s", uid)
+ self.add_vpn_user(uid)
+
+ def add_vpn_user(self, uid):
+ """
+ Adds the dn of the vpn user to the database
+ so that we can clean it once the booking ends
+ """
+ db = BookingDataBase(self.conf['database'])
+ # converts from hostname to pharos resource id
+ inventory = yaml.safe_load(open(self.conf['inventory']))
+ host_id = -1
+ for resource_id in inventory.keys():
+ if inventory[resource_id] == self.hostname:
+ host_id = resource_id
+ break
+ db.setVPN(host_id, uid)
+
+ def finishDeployment(self):
+ """
+ Last method call once a host is finished being deployed.
+ It notifies the database and changes the password to
+ a random string
+ """
+ self.markHostDeployed()
+ self.make_vpn_user()
+ passwd = self.changePassword()
+ self.logger.info("host %s provisioning done", self.hostname)
+ self.logger.info("You may access the host at %s", self.host)
+ self.logger.info("The password is %s", passwd)
+ notice = "You should change all passwords for security"
+ self.logger.warning('%s', notice)
+
+ @staticmethod
+ def restartRemoteHost(host_ip):
+ """
+ This method assumes that you already have ssh access to the target
+ """
+ nul = open(os.devnull, 'w')
+ ret_code = subprocess.call([
+ 'ssh', '-o', 'StrictHostKeyChecking=no',
+ 'root@'+host_ip,
+ 'shutdown', '-r', 'now'],
+ stdout=nul, stderr=nul)
+
+ return ret_code
+
+ @staticmethod
+ def getName(xmlString):
+ """
+ Gets the name value from xml. for example:
+ <name>Parker</name> returns Parker
+ """
+ xmlDoc = xml.dom.minidom.parseString(xmlString)
+ nameNode = xmlDoc.documentElement.getElementsByTagName('name')
+ name = str(nameNode[0].firstChild.nodeValue)
+ return name
+
+ @staticmethod
+ def getXMLFiles(directory):
+ """
+ searches directory non-recursively and
+ returns a list of all xml files
+ """
+ contents = os.listdir(directory)
+ fileContents = []
+ for item in contents:
+ if os.path.isfile(os.path.join(directory, item)):
+ fileContents.append(os.path.join(directory, item))
+ xmlFiles = []
+ for item in fileContents:
+ if 'xml' in os.path.basename(item):
+ xmlFiles.append(item)
+ return xmlFiles
+
+ @staticmethod
+ def createLogger(name, log_dir=LOGGING_DIR):
+ """
+ Initializes the logger if it does not yet exist, and returns it.
+ Because of how python logging works, calling logging.getLogger()
+ with the same name always returns a reference to the same log file.
+ So we can call this method from anywhere with the hostname as
+ the name arguement and it will return the log file for that host.
+ The formatting includes the level of importance and the time stamp
+ """
+ global LOGGING_DIR
+ if log_dir != LOGGING_DIR:
+ LOGGING_DIR = log_dir
+ log = logging.getLogger(name)
+ if len(log.handlers) > 0: # if this logger is already initialized
+ return log
+ log.setLevel(10)
+ han = logging.FileHandler(os.path.join(log_dir, name+".log"))
+ han.setLevel(10)
+ log_format = '[%(levelname)s] %(asctime)s [#] %(message)s'
+ formatter = logging.Formatter(fmt=log_format)
+ han.setFormatter(formatter)
+ log.addHandler(han)
+ return log
+
+ @staticmethod
+ def getIPfromMAC(macAddr, logFile, remote=None):
+ """
+ searches through the dhcp logs for the given mac
+ and returns the associated ip. Will retrieve the
+ logFile from a remote host if remote is given.
+ if given, remote should be an ip address or hostname that
+ we can ssh to.
+ """
+ if remote is not None:
+ logFile = Utilities.retrieveFile(remote, logFile)
+ ip = Utilities.getIPfromLog(macAddr, logFile)
+ if remote is not None:
+ os.remove(logFile)
+ return ip
+
+ @staticmethod
+ def retrieveFile(host, remote_loc, local_loc=os.getcwd()):
+ """
+ Retrieves file from host and puts it in the current directory
+ unless local_loc is given.
+ """
+ subprocess.call(['scp', 'root@'+host+':'+remote_loc, local_loc])
+ return os.path.join(local_loc, os.path.basename(remote_loc))
+
+ @staticmethod
+ def getIPfromLog(macAddr, logFile):
+ """
+ Helper method for getIPfromMAC.
+ uses regex to find the ip address in the
+ log
+ """
+ try:
+ messagesFile = open(logFile, "r")
+ allLines = messagesFile.readlines()
+ except Exception:
+ sys.exit(1)
+ importantLines = []
+ for line in allLines:
+ if macAddr in line and "DHCPACK" in line:
+ importantLines.append(line)
+ ipRegex = r'(\d+\.\d+\.\d+\.\d+)'
+ IPs = []
+ for line in importantLines:
+ IPs.append(re.findall(ipRegex, line))
+ if len(IPs) > 0 and len(IPs[-1]) > 0:
+ return IPs[-1][0]
+ return None
Code Review / pharos.git / commitdiff
