1 # Copyright 2013: Mirantis Inc.
4 # Licensed under the Apache License, Version 2.0 (the "License"); you may
5 # not use this file except in compliance with the License. You may obtain
6 # a copy of the License at
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13 # License for the specific language governing permissions and limitations
17 """High level ssh library.
19 Execute command and get output:
20 ssh = sshclient.SSH('root', 'example.com', port=33)
21 status, stdout, stderr = ssh.execute('ps ax')
23 raise Exception('Command failed with non-zero status.')
24 print stdout.splitlines()
25 Execute command with huge output:
26 class PseudoFile(object):
30 ssh = sshclient.SSH('root', 'example.com')
31 ssh.run('tail -f /var/log/syslog', stdout=PseudoFile(), timeout=False)
32 Execute local script on remote side:
33 ssh = sshclient.SSH('user', 'example.com')
34 status, out, err = ssh.execute('/bin/sh -s arg1 arg2',
35 stdin=open('~/myscript.sh', 'r'))
37 ssh = sshclient.SSH('user', 'example.com')
38 ssh.run('cat > ~/upload/file.gz', stdin=open('/store/file.gz', 'rb'))
40 eventlet.monkey_patch(select=True, time=True)
42 eventlet.monkey_patch()
44 sshclient = eventlet.import_patched("opentstack.common.sshclient")
60 # from rally.openstack.common.gettextutils import _
63 class ConnectionError(Exception):
67 class Connection(object):
70 A base connection class. Not intended to be constructed.
75 self.distro_id_like = None
76 self.distro_version = None
82 def execute(self, cmd, stdin=None, timeout=3600):
85 def __extract_property(self, name, input_str):
86 expr = name + r'="?([\w\.]*)"?'
87 match = re.search(expr, input_str)
92 # Get the linux distro
93 def __get_distro(self):
94 '''cat /etc/*-release | grep ID
105 distro_cmd = "grep ID /etc/*-release"
106 (status, distro_out, _) = self.execute(distro_cmd)
109 self.distro_id = self.__extract_property('ID', distro_out)
110 self.distro_id_like = self.__extract_property('ID_LIKE', distro_out)
111 self.distro_version = self.__extract_property('VERSION_ID', distro_out)
113 def pidof(self, proc_name):
115 Return a list containing the pids of all processes of a given name
116 the list is empty if there is no pid
118 # the path update is necessary for RHEL
119 cmd = "PATH=$PATH:/usr/sbin pidof " + proc_name
120 (status, cmd_output, _) = self.execute(cmd)
123 cmd_output = cmd_output.strip()
124 result = cmd_output.split()
127 # kill pids in the given list of pids
128 def kill_proc(self, pid_list):
129 cmd = "kill -9 " + ' '.join(pid_list)
132 # check stats for a given path
133 def stat(self, path):
134 (status, cmd_output, _) = self.execute('stat ' + path)
139 def ping_check(self, target_ip, ping_count=2, pass_threshold=80):
140 '''helper function to ping from one host to an IP address,
141 for a given count and pass_threshold;
143 ssh to the host and then ping to the target IP
144 then match the output and verify that the loss% is
145 less than the pass_threshold%
146 Return 1 if the criteria passes
147 Return 0, if it fails
149 cmd = "ping -c " + str(ping_count) + " " + str(target_ip)
150 (_, cmd_output, _) = self.execute(cmd)
152 match = re.search(r'(\d*)% packet loss', cmd_output)
153 pkt_loss = match.group(1)
154 if int(pkt_loss) < int(pass_threshold):
157 LOG.error('Ping to %s failed: %s', target_ip, cmd_output)
160 def read_remote_file(self, from_path):
162 Read a remote file and save it to a buffer.
164 cmd = "cat " + from_path
165 (status, cmd_output, _) = self.execute(cmd)
170 def get_host_os_version(self):
172 Identify the host distribution/relase.
174 os_release_file = "/etc/os-release"
175 sys_release_file = "/etc/system-release"
179 if self.stat(os_release_file):
180 data = self.read_remote_file(os_release_file)
182 LOG.error("Failed to read file %s", os_release_file)
185 for line in data.splitlines():
186 mobj = re.match(r'NAME=(.*)', line)
188 name = mobj.group(1).strip("\"")
190 mobj = re.match(r'VERSION_ID=(.*)', line)
192 version = mobj.group(1).strip("\"")
194 os_name = name + " " + version
197 if self.stat(sys_release_file):
198 data = self.read_remote_file(sys_release_file)
200 LOG.error("Failed to read file %s", sys_release_file)
203 for line in data.splitlines():
204 mobj = re.match(r'Red Hat.*', line)
210 def check_rpm_package_installed(self, rpm_pkg):
212 Given a host and a package name, check if it is installed on the
215 check_pkg_cmd = "rpm -qa | grep " + rpm_pkg
217 (status, cmd_output, _) = self.execute(check_pkg_cmd)
221 pkg_pattern = ".*" + rpm_pkg + ".*"
222 rpm_pattern = re.compile(pkg_pattern, re.IGNORECASE)
224 for line in cmd_output.splitlines():
225 mobj = rpm_pattern.match(line)
229 LOG.info("%s pkg installed ", rpm_pkg)
233 def get_openstack_release(self, ver_str):
235 Get the release series name from the package version
236 Refer to here for release tables:
237 https://wiki.openstack.org/wiki/Releases
239 ver_table = {"2015.1": "Kilo",
241 "2014.1": "Icehouse",
251 ver_prefix = re.search(r"20\d\d\.\d", ver_str).group(0)
252 if ver_prefix in ver_table:
253 return ver_table[ver_prefix]
257 def check_openstack_version(self):
259 Identify the openstack version running on the controller.
261 nova_cmd = "nova-manage --version"
262 (status, _, err_output) = self.execute(nova_cmd)
267 ver_str = err_output.strip()
268 release_str = self.get_openstack_release(err_output)
269 return release_str + " (" + ver_str + ")"
271 def get_cpu_info(self):
273 Get the CPU info of the controller.
274 Note: Here we are assuming the controller node has the exact
275 hardware as the compute nodes.
278 cmd = 'cat /proc/cpuinfo | grep -m1 "model name"'
279 (status, std_output, _) = self.execute(cmd)
282 model_name = re.search(r":\s(.*)", std_output).group(1)
284 cmd = 'cat /proc/cpuinfo | grep "model name" | wc -l'
285 (status, std_output, _) = self.execute(cmd)
288 cores = std_output.strip()
290 return (cores + " * " + model_name)
292 def get_nic_name(self, agent_type, encap, internal_iface_dict):
294 Get the NIC info of the controller.
295 Note: Here we are assuming the controller node has the exact
296 hardware as the compute nodes.
299 # The internal_ifac_dict is a dictionary contains the mapping between
300 # hostname and the internal interface name like below:
301 # {u'hh23-4': u'eth1', u'hh23-5': u'eth1', u'hh23-6': u'eth1'}
304 (status, std_output, _) = self.execute(cmd)
307 hostname = std_output.strip()
309 if hostname in internal_iface_dict:
310 iface = internal_iface_dict[hostname]
314 # Figure out which interface is for internal traffic
315 if 'Linux bridge' in agent_type:
317 elif 'Open vSwitch' in agent_type:
319 # [root@hh23-10 ~]# ovs-vsctl list-ports br-inst
322 cmd = 'ovs-vsctl list-ports ' + \
323 iface + ' | grep -E "^[^phy].*"'
324 (status, std_output, _) = self.execute(cmd)
327 ifname = std_output.strip()
328 elif encap == 'vxlan' or encap == 'gre':
329 # This is complicated. We need to first get the local IP address on
330 # br-tun, then do a reverse lookup to get the physical interface.
332 # [root@hh23-4 ~]# ip addr show to "23.23.2.14"
333 # 3: eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP qlen 1000
334 # inet 23.23.2.14/24 brd 23.23.2.255 scope global eth1
335 # valid_lft forever preferred_lft forever
336 cmd = "ip addr show to " + iface + " | awk -F: '{print $2}'"
337 (status, std_output, _) = self.execute(cmd)
340 ifname = std_output.strip()
344 cmd = 'ethtool -i ' + ifname + ' | grep bus-info'
345 (status, std_output, _) = self.execute(cmd)
348 bus_info = re.search(r":\s(.*)", std_output).group(1)
350 cmd = 'lspci -s ' + bus_info
351 (status, std_output, _) = self.execute(cmd)
354 nic_name = re.search(
355 r"Ethernet controller:\s(.*)",
360 def get_l2agent_version(self, agent_type):
362 Get the L2 agent version of the controller.
363 Note: Here we are assuming the controller node has the exact
364 hardware as the compute nodes.
366 if 'Linux bridge' in agent_type:
367 cmd = "brctl --version | awk -F',' '{print $2}'"
368 ver_string = "Linux Bridge "
369 elif 'Open vSwitch' in agent_type:
370 cmd = "ovs-vsctl --version | awk -F')' '{print $2}'"
375 (status, std_output, _) = self.execute(cmd)
379 return ver_string + std_output.strip()
382 class SSHError(Exception):
386 class SSHTimeout(SSHError):
389 # Check IPv4 address syntax - not completely fool proof but will catch
390 # some invalid formats
393 def is_ipv4(address):
395 socket.inet_aton(address)
401 class SSHAccess(object):
404 A class to contain all the information needed to access a host
405 (native or virtual) using SSH
408 def __init__(self, arg_value=None):
410 decode user@host[:pwd]
411 'hugo@1.1.1.1:secret' -> ('hugo', '1.1.1.1', 'secret', None)
412 'huggy@2.2.2.2' -> ('huggy', '2.2.2.2', None, None)
413 None ->(None, None, None, None)
414 Examples of fatal errors (will call exit):
415 'hutch@q.1.1.1' (invalid IP)
416 '@3.3.3.3' (missing username)
417 'hiro@' or 'buggy' (missing host IP)
418 The error field will be None in case of success or will
419 contain a string describing the error
424 # name of the file that contains the private key
425 self.private_key_file = None
426 # this is the private key itself (a long string starting with
427 # -----BEGIN RSA PRIVATE KEY-----
428 # used when the private key is not saved in any file
429 self.private_key = None
430 self.public_key_file = None
436 match = re.search(r'^([^@]+)@([0-9\.]+):?(.*)$', arg_value)
438 self.error = 'Invalid argument: ' + arg_value
440 if not is_ipv4(match.group(2)):
441 self.error = 'Invalid IPv4 address ' + match.group(2)
443 (self.username, self.host, self.password) = match.groups()
445 def copy_from(self, ssh_access):
446 self.username = ssh_access.username
447 self.host = ssh_access.host
448 self.port = ssh_access.port
449 self.password = ssh_access.password
450 self.private_key = ssh_access.private_key
451 self.public_key_file = ssh_access.public_key_file
452 self.private_key_file = ssh_access.private_key_file
455 class SSH(Connection):
457 """Represent ssh connection."""
459 def __init__(self, ssh_access,
461 connect_retry_count=30,
462 connect_retry_wait_sec=2):
463 """Initialize SSH client.
464 :param user: ssh username
465 :param host: hostname or ip address of remote ssh server
466 :param port: remote ssh port
467 :param pkey: RSA or DSS private key string or file object
468 :param key_filename: private key filename
469 :param password: password
470 :param connect_timeout: timeout when connecting ssh
471 :param connect_retry_count: how many times to retry connecting
472 :param connect_retry_wait_sec: seconds to wait between retries
475 self.ssh_access = ssh_access
476 if ssh_access.private_key:
477 self.pkey = self._get_pkey(ssh_access.private_key)
481 self.connect_timeout = connect_timeout
482 self.connect_retry_count = connect_retry_count
483 self.connect_retry_wait_sec = connect_retry_wait_sec
484 super(SSH, self).__init__()
486 def _get_pkey(self, key):
487 '''Get the binary form of the private key
490 if isinstance(key, basestring):
491 key = StringIO.StringIO(key)
493 for key_class in (paramiko.rsakey.RSAKey, paramiko.dsskey.DSSKey):
495 return key_class.from_private_key(key)
496 except paramiko.SSHException as exc:
498 raise SSHError('Invalid pkey: %s' % (errors))
500 def _is_active(self):
503 transport = self._client.get_transport()
504 session = transport.open_session()
512 def _get_client(self, force=False):
513 if not force and self._is_active():
516 LOG.info('Re-establishing ssh connection with %s' % (self.ssh_access.host))
518 self._client = paramiko.SSHClient()
519 self._client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
520 for _ in range(self.connect_retry_count):
522 self._client.connect(self.ssh_access.host,
523 username=self.ssh_access.username,
524 port=self.ssh_access.port,
526 key_filename=self.ssh_access.private_key_file,
527 password=self.ssh_access.password,
528 timeout=self.connect_timeout)
529 self._client.get_transport().set_keepalive(5)
531 except (paramiko.AuthenticationException,
532 paramiko.BadHostKeyException,
533 paramiko.SSHException,
536 time.sleep(self.connect_retry_wait_sec)
539 msg = '[%s] SSH Connection failed after %s attempts' % (self.ssh_access.host,
540 self.connect_retry_count)
543 def _get_session(self):
544 client = self._get_client()
545 for _ in range(self.connect_retry_count):
547 transport = client.get_transport()
548 session = transport.open_session()
551 client = self._get_client(force=True)
555 super(SSH, self).close()
560 def run(self, cmd, stdin=None, stdout=None, stderr=None,
561 raise_on_error=True, timeout=3600, sudo=False):
562 """Execute specified command on the server.
563 :param cmd: Command to be executed.
564 :param stdin: Open file or string to pass to stdin.
565 :param stdout: Open file to connect to stdout.
566 :param stderr: Open file to connect to stderr.
567 :param raise_on_error: If False then exit code will be return. If True
568 then exception will be raized if non-zero code.
569 :param timeout: Timeout in seconds for command execution.
570 Default 1 hour. No timeout if set to 0.
571 :param sudo: Executes command as sudo with default password
574 if isinstance(stdin, basestring):
575 stdin = StringIO.StringIO(stdin)
577 return self._run(cmd, stdin=stdin, stdout=stdout,
578 stderr=stderr, raise_on_error=raise_on_error,
579 timeout=timeout, sudo=sudo)
581 def _run(self, cmd, stdin=None, stdout=None, stderr=None,
582 raise_on_error=True, timeout=3600, sudo=False):
584 session = self._get_session()
587 raise SSHError('Unable to open session to ssh connection')
590 cmd = "echo " + self.ssh_access.password + " | sudo -S -p '' " + cmd
593 session.exec_command(cmd)
594 start_time = time.time()
599 # If we have data to be sent to stdin then `select' should also
600 # check for stdin availability.
601 if stdin and not stdin.closed:
607 # Block until data can be read/write.
608 select.select([session], writes, [session], 1)
610 if session.recv_ready():
611 data = session.recv(4096)
612 if stdout is not None:
616 if session.recv_stderr_ready():
617 stderr_data = session.recv_stderr(4096)
618 if stderr is not None:
619 stderr.write(stderr_data)
622 if session.send_ready():
623 if stdin is not None and not stdin.closed:
625 data_to_send = stdin.read(4096)
628 session.shutdown_write()
631 sent_bytes = session.send(data_to_send)
632 data_to_send = data_to_send[sent_bytes:]
634 if session.exit_status_ready():
637 if timeout and (time.time() - timeout) > start_time:
638 args = {'cmd': cmd, 'host': self.ssh_access.host}
639 raise SSHTimeout(('Timeout executing command '
640 '"%(cmd)s" on host %(host)s') % args)
642 # raise SSHError('Socket error.')
644 exit_status = session.recv_exit_status()
645 if 0 != exit_status and raise_on_error:
646 fmt = ('Command "%(cmd)s" failed with exit_status %(status)d.')
647 details = fmt % {'cmd': cmd, 'status': exit_status}
649 details += (' Last stderr data: "%s".') % stderr_data
650 raise SSHError(details)
653 def execute(self, cmd, stdin=None, timeout=3600, sudo=False):
654 """Execute the specified command on the server.
655 :param cmd: Command to be executed.
656 :param stdin: Open file to be sent on process stdin.
657 :param timeout: Timeout for execution of the command.
658 Return tuple (exit_status, stdout, stderr)
660 stdout = StringIO.StringIO()
661 stderr = StringIO.StringIO()
663 exit_status = self.run(cmd, stderr=stderr,
664 stdout=stdout, stdin=stdin,
665 timeout=timeout, raise_on_error=False, sudo=sudo)
668 return (exit_status, stdout.read(), stderr.read())
670 def wait(self, timeout=120, interval=1):
671 """Wait for the host will be available via ssh."""
672 start_time = time.time()
675 return self.execute('uname')
676 except (socket.error, SSHError):
678 if time.time() > (start_time + timeout):
680 ('Timeout waiting for "%s"') %
681 self.ssh_access.host)
684 class SubprocessTimeout(Exception):
688 class Subprocess(Connection):
690 """Represent subprocess connection."""
692 def execute(self, cmd, stdin=None, timeout=3600):
693 process = subprocess.Popen(shlex.split(cmd), stderr=subprocess.PIPE,
694 stdout=subprocess.PIPE,
696 timer = threading.Timer(timeout, process.kill)
697 stdout, stderr = process.communicate(input=stdin)
698 status = process.wait()
701 raise SubprocessTimeout('Timeout executing command "%(cmd)s"')
702 return (status, stdout, stderr)
705 ##################################################
706 # Only invoke the module directly for test purposes. Should be
707 # invoked from pns script.
708 ##################################################
710 # As argument pass the SSH access string, e.g. "localadmin@1.1.1.1:secret"
711 test_ssh = SSH(SSHAccess(sys.argv[1]))
713 print 'ID=' + test_ssh.distro_id
714 print 'ID_LIKE=' + test_ssh.distro_id_like
715 print 'VERSION_ID=' + test_ssh.distro_version
718 # print ssh.pidof('bash')
719 # print ssh.stat('/tmp')
720 print test_ssh.check_openstack_version()
721 print test_ssh.get_cpu_info()
722 print test_ssh.get_l2agent_version("Open vSwitch agent")
724 if __name__ == "__main__":