1 ##############################################################################
2 # Copyright (c) 2016 Tim Rozet (trozet@redhat.com) and others.
4 # All rights reserved. This program and the accompanying materials
5 # are made available under the terms of the Apache License, Version 2.0
6 # which accompanies this distribution, and is available at
7 # http://www.apache.org/licenses/LICENSE-2.0
8 ##############################################################################
25 from apex.common import exceptions as exc
29 if isinstance(var, bool):
32 return var.lower() in ("true", "yes")
35 def parse_yaml(yaml_file):
36 with open(yaml_file) as f:
37 parsed_dict = yaml.safe_load(f)
41 def dump_yaml(data, file):
43 Dumps data to a file as yaml
44 :param data: yaml to be written to file
45 :param file: filename to write to
48 logging.debug("Writing file {} with "
49 "yaml data:\n{}".format(file, yaml.safe_dump(data)))
50 with open(file, "w") as fh:
51 yaml.safe_dump(data, fh, default_flow_style=False)
54 def dict_objects_to_str(dictionary):
55 if isinstance(dictionary, list):
57 for element in dictionary:
58 if isinstance(element, dict):
59 tmp_list.append(dict_objects_to_str(element))
61 tmp_list.append(str(element))
63 elif not isinstance(dictionary, dict):
64 if not isinstance(dictionary, bool):
65 return str(dictionary)
68 return dict((k, dict_objects_to_str(v)) for
69 k, v in dictionary.items())
72 def run_ansible(ansible_vars, playbook, host='localhost', user='root',
73 tmp_dir=None, dry_run=False):
75 Executes ansible playbook and checks for errors
76 :param ansible_vars: dictionary of variables to inject into ansible run
77 :param playbook: playbook to execute
78 :param tmp_dir: temp directory to store ansible command
79 :param dry_run: Do not actually apply changes
82 logging.info("Executing ansible playbook: {}".format(playbook))
83 inv_host = "{},".format(host)
84 if host == 'localhost':
88 ansible_command = ['ansible-playbook', '--become', '-i', inv_host,
89 '-u', user, '-c', conn_type, '-T', '30',
92 ansible_command.append('--check')
94 if isinstance(ansible_vars, dict) and ansible_vars:
95 logging.debug("Ansible variables to be set:\n{}".format(
96 pprint.pformat(ansible_vars)))
97 ansible_command.append('--extra-vars')
98 ansible_command.append(json.dumps(ansible_vars))
100 ansible_tmp = os.path.join(tmp_dir,
101 os.path.basename(playbook) + '.rerun')
102 # FIXME(trozet): extra vars are printed without single quotes
103 # so a dev has to add them manually to the command to rerun
104 # the playbook. Need to test if we can just add the single quotes
105 # to the json dumps to the ansible command and see if that works
106 with open(ansible_tmp, 'w') as fh:
107 fh.write("ANSIBLE_HOST_KEY_CHECKING=FALSE {}".format(
108 ' '.join(ansible_command)))
110 my_env = os.environ.copy()
111 my_env['ANSIBLE_HOST_KEY_CHECKING'] = 'False'
112 logging.info("Executing playbook...this may take some time")
113 p = subprocess.Popen(ansible_command,
114 stdin=subprocess.PIPE,
115 stdout=subprocess.PIPE,
118 universal_newlines=True)
120 x = p.stdout.readline()
124 # append lines to task
126 # log the line and read another
127 x = p.stdout.readline()
128 # deliver the task to info when we get a blank line
131 logging.info(task.replace('\\n', '\n'))
133 x = p.stdout.readline()
134 # clean up and get return code
139 e = "Ansible playbook failed. See Ansible logs for details."
144 def get_url_modified_date(url):
146 Returns the last modified date for an Tripleo image artifact
147 :param url: URL to examine
148 :return: datetime object of when artifact was last modified
151 u = urllib.request.urlopen(url)
152 except urllib.error.URLError as e:
153 logging.error("Failed to fetch target url. Error: {}".format(
158 headers = metadata.items()
159 for header in headers:
160 if isinstance(header, tuple) and len(header) == 2:
161 if header[0] == 'Last-Modified':
162 return datetime.datetime.strptime(header[1],
163 "%a, %d %b %Y %X GMT")
166 def fetch_upstream_and_unpack(dest, url, targets, fetch=True):
168 Fetches targets from a url destination and downloads them if they are
169 newer. Also unpacks tar files in dest dir.
170 :param dest: Directory to download and unpack files to
171 :param url: URL where target files are located
172 :param targets: List of target files to download
173 :param fetch: Whether or not to fetch latest from internet (boolean)
176 os.makedirs(dest, exist_ok=True)
177 assert isinstance(targets, list)
178 for target in targets:
179 target_url = urllib.parse.urljoin(url, target)
180 target_dest = os.path.join(dest, target)
181 target_exists = os.path.isfile(target_dest)
183 download_target = True
184 elif not target_exists:
185 logging.warning("no-fetch requested but target: {} is not "
186 "cached, will download".format(target_dest))
187 download_target = True
189 logging.info("no-fetch requested and previous cache exists for "
190 "target: {}. Will skip download".format(target_dest))
191 download_target = False
194 logging.debug("Fetching and comparing upstream"
195 " target: \n{}".format(target_url))
196 # Check if previous file and fetch we need to compare files to
197 # determine if download is necessary
198 if target_exists and download_target:
199 logging.debug("Previous file found: {}".format(target_dest))
200 target_url_date = get_url_modified_date(target_url)
201 if target_url_date is not None:
202 target_dest_mtime = os.path.getmtime(target_dest)
203 target_url_mtime = time.mktime(target_url_date.timetuple())
204 if target_url_mtime > target_dest_mtime:
205 logging.debug('URL target is newer than disk...will '
208 logging.info('URL target does not need to be downloaded')
209 download_target = False
211 logging.debug('Unable to find last modified url date')
214 urllib.request.urlretrieve(target_url, filename=target_dest)
215 logging.info("Target downloaded: {}".format(target))
216 if target.endswith('.tar'):
217 logging.info('Unpacking tar file')
218 tar = tarfile.open(target_dest)
219 tar.extractall(path=dest)
223 def install_ansible():
224 # we only install for CentOS/Fedora for now
228 elif 'fedora' in dist:
233 # yum python module only exists for 2.x, so use subprocess
235 subprocess.check_call([pkg_mgr, '-y', 'install', 'ansible'])
236 except subprocess.CalledProcessError:
237 logging.warning('Unable to install Ansible')
240 def internet_connectivity():
242 urllib.request.urlopen('http://opnfv.org', timeout=3)
244 except (urllib.request.URLError, socket.timeout):
245 logging.debug('No internet connectivity detected')
249 def open_webpage(url, timeout=5):
251 response = urllib.request.urlopen(url, timeout=timeout)
252 return response.read()
253 except (urllib.request.URLError, socket.timeout):
254 logging.error("Unable to open URL: {}".format(url))
258 def edit_tht_env(env_file, section, settings):
259 assert isinstance(settings, dict)
260 with open(env_file) as fh:
261 data = yaml.safe_load(fh)
263 if section not in data.keys():
265 for setting, value in settings.items():
266 data[section][setting] = value
267 with open(env_file, 'w') as fh:
268 yaml.safe_dump(data, fh, default_flow_style=False)
269 logging.debug("Data written to env file {}:\n{}".format(env_file, data))