1 ##############################################################################
2 # Copyright (c) 2016 Tim Rozet (trozet@redhat.com) and others.
4 # All rights reserved. This program and the accompanying materials
5 # are made available under the terms of the Apache License, Version 2.0
6 # which accompanies this distribution, and is available at
7 # http://www.apache.org/licenses/LICENSE-2.0
8 ##############################################################################
25 from apex.common import exceptions as exc
29 if isinstance(var, bool):
32 return var.lower() in ("true", "yes")
35 def parse_yaml(yaml_file):
36 with open(yaml_file) as f:
37 parsed_dict = yaml.safe_load(f)
41 def dump_yaml(data, file):
43 Dumps data to a file as yaml
44 :param data: yaml to be written to file
45 :param file: filename to write to
48 logging.debug("Writing file {} with "
49 "yaml data:\n{}".format(file, yaml.safe_dump(data)))
50 with open(file, "w") as fh:
51 yaml.safe_dump(data, fh, default_flow_style=False)
54 def dict_objects_to_str(dictionary):
55 if isinstance(dictionary, list):
57 for element in dictionary:
58 if isinstance(element, dict):
59 tmp_list.append(dict_objects_to_str(element))
61 tmp_list.append(str(element))
63 elif not isinstance(dictionary, dict):
64 if not isinstance(dictionary, bool):
65 return str(dictionary)
68 return dict((k, dict_objects_to_str(v)) for
69 k, v in dictionary.items())
72 def run_ansible(ansible_vars, playbook, host='localhost', user='root',
73 tmp_dir=None, dry_run=False):
75 Executes ansible playbook and checks for errors
76 :param ansible_vars: dictionary of variables to inject into ansible run
77 :param playbook: playbook to execute
78 :param tmp_dir: temp directory to store ansible command
79 :param dry_run: Do not actually apply changes
82 logging.info("Executing ansible playbook: {}".format(playbook))
83 inv_host = "{},".format(host)
84 if host == 'localhost':
88 ansible_command = ['ansible-playbook', '--become', '-i', inv_host,
89 '-u', user, '-c', conn_type, '-T', '30',
92 ansible_command.append('--check')
94 if isinstance(ansible_vars, dict) and ansible_vars:
95 logging.debug("Ansible variables to be set:\n{}".format(
96 pprint.pformat(ansible_vars)))
97 ansible_command.append('--extra-vars')
98 ansible_command.append(json.dumps(ansible_vars))
100 ansible_tmp = os.path.join(tmp_dir,
101 os.path.basename(playbook) + '.rerun')
102 # FIXME(trozet): extra vars are printed without single quotes
103 # so a dev has to add them manually to the command to rerun
104 # the playbook. Need to test if we can just add the single quotes
105 # to the json dumps to the ansible command and see if that works
106 with open(ansible_tmp, 'w') as fh:
107 fh.write("ANSIBLE_HOST_KEY_CHECKING=FALSE {}".format(
108 ' '.join(ansible_command)))
110 my_env = os.environ.copy()
111 my_env['ANSIBLE_HOST_KEY_CHECKING'] = 'False'
112 logging.info("Executing playbook...this may take some time")
113 p = subprocess.Popen(ansible_command,
114 stdin=subprocess.PIPE,
115 stdout=subprocess.PIPE,
118 universal_newlines=True)
120 x = p.stdout.readline()
124 # append lines to task
126 # log the line and read another
127 x = p.stdout.readline()
128 # deliver the task to info when we get a blank line
131 logging.info(task.replace('\\n', '\n'))
133 x = p.stdout.readline()
134 # clean up and get return code
139 e = "Ansible playbook failed. See Ansible logs for details."
144 def fetch_upstream_and_unpack(dest, url, targets, fetch=True):
146 Fetches targets from a url destination and downloads them if they are
147 newer. Also unpacks tar files in dest dir.
148 :param dest: Directory to download and unpack files to
149 :param url: URL where target files are located
150 :param targets: List of target files to download
151 :param fetch: Whether or not to fetch latest from internet (boolean)
154 os.makedirs(dest, exist_ok=True)
155 assert isinstance(targets, list)
156 for target in targets:
157 target_url = urllib.parse.urljoin(url, target)
158 target_dest = os.path.join(dest, target)
159 target_exists = os.path.isfile(target_dest)
161 download_target = True
162 elif not target_exists:
163 logging.warning("no-fetch requested but target: {} is not "
164 "cached, will download".format(target_dest))
165 download_target = True
167 logging.info("no-fetch requested and previous cache exists for "
168 "target: {}. Will skip download".format(target_dest))
169 download_target = False
172 logging.debug("Fetching and comparing upstream"
173 " target: \n{}".format(target_url))
175 u = urllib.request.urlopen(target_url)
176 except urllib.error.URLError as e:
177 logging.error("Failed to fetch target url. Error: {}".format(
180 # Check if previous file and fetch we need to compare files to
181 # determine if download is necessary
182 if target_exists and download_target:
183 logging.debug("Previous file found: {}".format(target_dest))
185 headers = metadata.items()
186 target_url_date = None
187 for header in headers:
188 if isinstance(header, tuple) and len(header) == 2:
189 if header[0] == 'Last-Modified':
190 target_url_date = header[1]
192 if target_url_date is not None:
193 target_dest_mtime = os.path.getmtime(target_dest)
194 target_url_mtime = time.mktime(
195 datetime.datetime.strptime(target_url_date,
198 if target_url_mtime > target_dest_mtime:
199 logging.debug('URL target is newer than disk...will '
202 logging.info('URL target does not need to be downloaded')
203 download_target = False
205 logging.debug('Unable to find last modified url date')
208 urllib.request.urlretrieve(target_url, filename=target_dest)
209 logging.info("Target downloaded: {}".format(target))
210 if target.endswith('.tar'):
211 logging.info('Unpacking tar file')
212 tar = tarfile.open(target_dest)
213 tar.extractall(path=dest)
217 def install_ansible():
218 # we only install for CentOS/Fedora for now
222 elif 'fedora' in dist:
227 # yum python module only exists for 2.x, so use subprocess
229 subprocess.check_call([pkg_mgr, '-y', 'install', 'ansible'])
230 except subprocess.CalledProcessError:
231 logging.warning('Unable to install Ansible')
234 def internet_connectivity():
236 urllib.request.urlopen('http://opnfv.org', timeout=3)
238 except (urllib.request.URLError, socket.timeout):
239 logging.debug('No internet connectivity detected')
243 def open_webpage(url, timeout=5):
245 response = urllib.request.urlopen(url, timeout=timeout)
246 return response.read()
247 except (urllib.request.URLError, socket.timeout):
248 logging.error("Unable to open URL: {}".format(url))
252 def edit_tht_env(env_file, section, settings):
253 assert isinstance(settings, dict)
254 with open(env_file) as fh:
255 data = yaml.safe_load(fh)
257 if section not in data.keys():
259 for setting, value in settings.items():
260 data[section][setting] = value
261 with open(env_file, 'w') as fh:
262 yaml.safe_dump(data, fh, default_flow_style=False)
263 logging.debug("Data written to env file {}:\n{}".format(env_file, data))