Coverage for drivers/util.py : 40%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Copyright (C) Citrix Systems Inc.
2#
3# This program is free software; you can redistribute it and/or modify
4# it under the terms of the GNU Lesser General Public License as published
5# by the Free Software Foundation; version 2.1 only.
6#
7# This program is distributed in the hope that it will be useful,
8# but WITHOUT ANY WARRANTY; without even the implied warranty of
9# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10# GNU Lesser General Public License for more details.
11#
12# You should have received a copy of the GNU Lesser General Public License
13# along with this program; if not, write to the Free Software Foundation, Inc.,
14# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
15#
16# Miscellaneous utility functions
17#
19import os
20import re
21import sys
22import subprocess
23import shutil
24import tempfile
25import signal
26import time
27import datetime
28import errno
29import socket
30import xml.dom.minidom
31import scsiutil
32import stat
33import xs_errors
34import XenAPI
35import xmlrpc.client
36import base64
37import syslog
38import resource
39import traceback
40import glob
41import copy
42import tempfile
44from functools import reduce
46NO_LOGGING_STAMPFILE = '/etc/xensource/no_sm_log'
48IORETRY_MAX = 20 # retries
49IORETRY_PERIOD = 1.0 # seconds
51LOGGING = not (os.path.exists(NO_LOGGING_STAMPFILE))
52_SM_SYSLOG_FACILITY = syslog.LOG_LOCAL2
53LOG_EMERG = syslog.LOG_EMERG
54LOG_ALERT = syslog.LOG_ALERT
55LOG_CRIT = syslog.LOG_CRIT
56LOG_ERR = syslog.LOG_ERR
57LOG_WARNING = syslog.LOG_WARNING
58LOG_NOTICE = syslog.LOG_NOTICE
59LOG_INFO = syslog.LOG_INFO
60LOG_DEBUG = syslog.LOG_DEBUG
62ISCSI_REFDIR = '/var/run/sr-ref'
64CMD_DD = "/bin/dd"
66FIST_PAUSE_PERIOD = 30 # seconds
69class SMException(Exception):
70 """Base class for all SM exceptions for easier catching & wrapping in
71 XenError"""
72 pass
75class CommandException(SMException):
76 def error_message(self, code):
77 if code > 0:
78 return os.strerror(code)
79 elif code < 0:
80 return "Signalled %s" % (abs(code))
81 return "Success"
83 def __init__(self, code, cmd="", reason='exec failed'):
84 self.code = code
85 self.cmd = cmd
86 self.reason = reason
87 Exception.__init__(self, self.error_message(code))
90class SRBusyException(SMException):
91 """The SR could not be locked"""
92 pass
95def logException(tag):
96 info = sys.exc_info()
97 if info[0] == SystemExit: 97 ↛ 99line 97 didn't jump to line 99, because the condition on line 97 was never true
98 # this should not be happening when catching "Exception", but it is
99 sys.exit(0)
100 tb = reduce(lambda a, b: "%s%s" % (a, b), traceback.format_tb(info[2]))
101 str = "***** %s: EXCEPTION %s, %s\n%s" % (tag, info[0], info[1], tb)
102 SMlog(str)
105def roundup(divisor, value):
106 """Retruns the rounded up value so it is divisible by divisor."""
108 if value == 0: 108 ↛ 109line 108 didn't jump to line 109, because the condition on line 108 was never true
109 value = 1
110 if value % divisor != 0:
111 return ((int(value) // divisor) + 1) * divisor
112 return value
115def to_plain_string(obj):
116 if obj is None:
117 return None
118 if type(obj) == str:
119 return obj
120 return str(obj)
123def shellquote(arg):
124 return '"%s"' % arg.replace('"', '\\"')
127def make_WWN(name):
128 hex_prefix = name.find("0x")
129 if (hex_prefix >= 0): 129 ↛ 132line 129 didn't jump to line 132, because the condition on line 129 was never false
130 name = name[name.find("0x") + 2:len(name)]
131 # inject dashes for each nibble
132 if (len(name) == 16): # sanity check 132 ↛ 136line 132 didn't jump to line 136, because the condition on line 132 was never false
133 name = name[0:2] + "-" + name[2:4] + "-" + name[4:6] + "-" + \
134 name[6:8] + "-" + name[8:10] + "-" + name[10:12] + "-" + \
135 name[12:14] + "-" + name[14:16]
136 return name
139def _logToSyslog(ident, facility, priority, message):
140 syslog.openlog(ident, 0, facility)
141 syslog.syslog(priority, "[%d] %s" % (os.getpid(), message))
142 syslog.closelog()
145def SMlog(message, ident="SM", priority=LOG_INFO):
146 if LOGGING: 146 ↛ exitline 146 didn't return from function 'SMlog', because the condition on line 146 was never false
147 for message_line in str(message).split('\n'):
148 _logToSyslog(ident, _SM_SYSLOG_FACILITY, priority, message_line)
151def _getDateString():
152 d = datetime.datetime.now()
153 t = d.timetuple()
154 return "%s-%s-%s:%s:%s:%s" % \
155 (t[0], t[1], t[2], t[3], t[4], t[5])
158def doexec(args, inputtext=None, new_env=None, text=True):
159 """Execute a subprocess, then return its return code, stdout and stderr"""
160 env = None
161 if new_env:
162 env = dict(os.environ)
163 env.update(new_env)
164 proc = subprocess.Popen(args, stdin=subprocess.PIPE,
165 stdout=subprocess.PIPE,
166 stderr=subprocess.PIPE,
167 close_fds=True, env=env,
168 universal_newlines=text)
170 if not text and inputtext is not None: 170 ↛ 171line 170 didn't jump to line 171, because the condition on line 170 was never true
171 inputtext = inputtext.encode()
173 (stdout, stderr) = proc.communicate(inputtext)
175 rc = proc.returncode
176 return rc, stdout, stderr
179def is_string(value):
180 return isinstance(value, str)
183# These are partially tested functions that replicate the behaviour of
184# the original pread,pread2 and pread3 functions. Potentially these can
185# replace the original ones at some later date.
186#
187# cmdlist is a list of either single strings or pairs of strings. For
188# each pair, the first component is passed to exec while the second is
189# written to the logs.
190def pread(cmdlist, close_stdin=False, scramble=None, expect_rc=0,
191 quiet=False, new_env=None, text=True):
192 cmdlist_for_exec = []
193 cmdlist_for_log = []
194 for item in cmdlist:
195 if is_string(item): 195 ↛ 205line 195 didn't jump to line 205, because the condition on line 195 was never false
196 cmdlist_for_exec.append(item)
197 if scramble: 197 ↛ 198line 197 didn't jump to line 198, because the condition on line 197 was never true
198 if item.find(scramble) != -1:
199 cmdlist_for_log.append("<filtered out>")
200 else:
201 cmdlist_for_log.append(item)
202 else:
203 cmdlist_for_log.append(item)
204 else:
205 cmdlist_for_exec.append(item[0])
206 cmdlist_for_log.append(item[1])
208 if not quiet: 208 ↛ 210line 208 didn't jump to line 210, because the condition on line 208 was never false
209 SMlog(cmdlist_for_log)
210 (rc, stdout, stderr) = doexec(cmdlist_for_exec, new_env=new_env, text=text)
211 if rc != expect_rc:
212 SMlog("FAILED in util.pread: (rc %d) stdout: '%s', stderr: '%s'" % \
213 (rc, stdout, stderr))
214 if quiet: 214 ↛ 215line 214 didn't jump to line 215, because the condition on line 214 was never true
215 SMlog("Command was: %s" % cmdlist_for_log)
216 if '' == stderr: 216 ↛ 217line 216 didn't jump to line 217, because the condition on line 216 was never true
217 stderr = stdout
218 raise CommandException(rc, str(cmdlist), stderr.strip())
219 if not quiet: 219 ↛ 221line 219 didn't jump to line 221, because the condition on line 219 was never false
220 SMlog(" pread SUCCESS")
221 return stdout
224# POSIX guaranteed atomic within the same file system.
225# Supply directory to ensure tempfile is created
226# in the same directory.
227def atomicFileWrite(targetFile, directory, text):
229 file = None
230 try:
231 # Create file only current pid can write/read to
232 # our responsibility to clean it up.
233 _, tempPath = tempfile.mkstemp(dir=directory)
234 file = open(tempPath, 'w')
235 file.write(text)
237 # Ensure flushed to disk.
238 file.flush()
239 os.fsync(file.fileno())
240 file.close()
242 os.rename(tempPath, targetFile)
243 except OSError:
244 SMlog("FAILED to atomic write to %s" % (targetFile))
246 finally:
247 if (file is not None) and (not file.closed):
248 file.close()
250 if os.path.isfile(tempPath):
251 os.remove(tempPath)
254#Read STDOUT from cmdlist and discard STDERR output
255def pread2(cmdlist, quiet=False, text=True):
256 return pread(cmdlist, quiet=quiet, text=text)
259#Read STDOUT from cmdlist, feeding 'text' to STDIN
260def pread3(cmdlist, text):
261 SMlog(cmdlist)
262 (rc, stdout, stderr) = doexec(cmdlist, text)
263 if rc:
264 SMlog("FAILED in util.pread3: (errno %d) stdout: '%s', stderr: '%s'" % \
265 (rc, stdout, stderr))
266 if '' == stderr:
267 stderr = stdout
268 raise CommandException(rc, str(cmdlist), stderr.strip())
269 SMlog(" pread3 SUCCESS")
270 return stdout
273def listdir(path, quiet=False):
274 cmd = ["ls", path, "-1", "--color=never"]
275 try:
276 text = pread2(cmd, quiet=quiet)[:-1]
277 if len(text) == 0:
278 return []
279 return text.split('\n')
280 except CommandException as inst:
281 if inst.code == errno.ENOENT:
282 raise CommandException(errno.EIO, inst.cmd, inst.reason)
283 else:
284 raise CommandException(inst.code, inst.cmd, inst.reason)
287def gen_uuid():
288 cmd = ["uuidgen", "-r"]
289 return pread(cmd)[:-1]
292def match_uuid(s):
293 regex = re.compile("^[0-9a-f]{8}-(([0-9a-f]{4})-){3}[0-9a-f]{12}")
294 return regex.search(s, 0)
297def findall_uuid(s):
298 regex = re.compile("[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}")
299 return regex.findall(s, 0)
302def exactmatch_uuid(s):
303 regex = re.compile("^[0-9a-f]{8}-(([0-9a-f]{4})-){3}[0-9a-f]{12}$")
304 return regex.search(s, 0)
307def start_log_entry(srpath, path, args):
308 logstring = str(datetime.datetime.now())
309 logstring += " log: "
310 logstring += srpath
311 logstring += " " + path
312 for element in args:
313 logstring += " " + element
314 try:
315 file = open(srpath + "/filelog.txt", "a")
316 file.write(logstring)
317 file.write("\n")
318 file.close()
319 except:
320 pass
322 # failed to write log ...
324def end_log_entry(srpath, path, args):
325 # for teminating, use "error" or "done"
326 logstring = str(datetime.datetime.now())
327 logstring += " end: "
328 logstring += srpath
329 logstring += " " + path
330 for element in args:
331 logstring += " " + element
332 try:
333 file = open(srpath + "/filelog.txt", "a")
334 file.write(logstring)
335 file.write("\n")
336 file.close()
337 except:
338 pass
340 # failed to write log ...
341 # for now print
342 # print "%s" % logstring
344def ioretry(f, errlist=[errno.EIO], maxretry=IORETRY_MAX, period=IORETRY_PERIOD, **ignored):
345 retries = 0
346 while True:
347 try:
348 return f()
349 except OSError as ose:
350 err = int(ose.errno)
351 if not err in errlist:
352 raise CommandException(err, str(f), "OSError")
353 except CommandException as ce:
354 if not int(ce.code) in errlist:
355 raise
357 retries += 1
358 if retries >= maxretry:
359 break
361 time.sleep(period)
363 raise CommandException(errno.ETIMEDOUT, str(f), "Timeout")
366def ioretry_stat(path, maxretry=IORETRY_MAX):
367 # this ioretry is similar to the previous method, but
368 # stat does not raise an error -- so check its return
369 retries = 0
370 while retries < maxretry:
371 stat = os.statvfs(path)
372 if stat.f_blocks != -1:
373 return stat
374 time.sleep(1)
375 retries += 1
376 raise CommandException(errno.EIO, "os.statvfs")
379def sr_get_capability(sr_uuid):
380 result = []
381 session = get_localAPI_session()
382 sr_ref = session.xenapi.SR.get_by_uuid(sr_uuid)
383 sm_type = session.xenapi.SR.get_record(sr_ref)['type']
384 sm_rec = session.xenapi.SM.get_all_records_where(
385 "field \"type\" = \"%s\"" % sm_type)
387 # SM expects at least one entry of any SR type
388 if len(sm_rec) > 0:
389 result = list(sm_rec.values())[0]['capabilities']
391 session.xenapi.logout()
392 return result
395def sr_get_driver_info(driver_info):
396 results = {}
397 # first add in the vanilla stuff
398 for key in ['name', 'description', 'vendor', 'copyright', \
399 'driver_version', 'required_api_version']:
400 results[key] = driver_info[key]
401 # add the capabilities (xmlrpc array)
402 # enforcing activate/deactivate for blktap2
403 caps = driver_info['capabilities']
404 if "ATOMIC_PAUSE" in caps: 404 ↛ 405line 404 didn't jump to line 405, because the condition on line 404 was never true
405 for cap in ("VDI_ACTIVATE", "VDI_DEACTIVATE"):
406 if not cap in caps:
407 caps.append(cap)
408 elif "VDI_ACTIVATE" in caps or "VDI_DEACTIVATE" in caps: 408 ↛ 409line 408 didn't jump to line 409, because the condition on line 408 was never true
409 SMlog("Warning: vdi_[de]activate present for %s" % driver_info["name"])
411 results['capabilities'] = caps
412 # add in the configuration options
413 options = []
414 for option in driver_info['configuration']:
415 options.append({'key': option[0], 'description': option[1]})
416 results['configuration'] = options
417 return xmlrpc.client.dumps((results, ), "", True)
420def return_nil():
421 return xmlrpc.client.dumps((None, ), "", True, allow_none=True)
424def SRtoXML(SRlist):
425 dom = xml.dom.minidom.Document()
426 driver = dom.createElement("SRlist")
427 dom.appendChild(driver)
429 for key in SRlist.keys():
430 dict = SRlist[key]
431 entry = dom.createElement("SR")
432 driver.appendChild(entry)
434 e = dom.createElement("UUID")
435 entry.appendChild(e)
436 textnode = dom.createTextNode(key)
437 e.appendChild(textnode)
439 if 'size' in dict:
440 e = dom.createElement("Size")
441 entry.appendChild(e)
442 textnode = dom.createTextNode(str(dict['size']))
443 e.appendChild(textnode)
445 if 'storagepool' in dict:
446 e = dom.createElement("StoragePool")
447 entry.appendChild(e)
448 textnode = dom.createTextNode(str(dict['storagepool']))
449 e.appendChild(textnode)
451 if 'aggregate' in dict:
452 e = dom.createElement("Aggregate")
453 entry.appendChild(e)
454 textnode = dom.createTextNode(str(dict['aggregate']))
455 e.appendChild(textnode)
457 return dom.toprettyxml()
460def pathexists(path):
461 try:
462 os.lstat(path)
463 return True
464 except OSError as inst:
465 if inst.errno == errno.EIO: 465 ↛ 466line 465 didn't jump to line 466, because the condition on line 465 was never true
466 time.sleep(1)
467 try:
468 listdir(os.path.realpath(os.path.dirname(path)))
469 os.lstat(path)
470 return True
471 except:
472 pass
473 raise CommandException(errno.EIO, "os.lstat(%s)" % path, "failed")
474 return False
477def force_unlink(path):
478 try:
479 os.unlink(path)
480 except OSError as e:
481 if e.errno != errno.ENOENT:
482 raise
485def create_secret(session, secret):
486 ref = session.xenapi.secret.create({'value': secret})
487 return session.xenapi.secret.get_uuid(ref)
490def get_secret(session, uuid):
491 try:
492 ref = session.xenapi.secret.get_by_uuid(uuid)
493 return session.xenapi.secret.get_value(ref)
494 except:
495 raise xs_errors.XenError('InvalidSecret', opterr='Unable to look up secret [%s]' % uuid)
498def get_real_path(path):
499 "Follow symlinks to the actual file"
500 absPath = path
501 directory = ''
502 while os.path.islink(absPath):
503 directory = os.path.dirname(absPath)
504 absPath = os.readlink(absPath)
505 absPath = os.path.join(directory, absPath)
506 return absPath
509def wait_for_path(path, timeout):
510 for i in range(0, timeout): 510 ↛ 514line 510 didn't jump to line 514, because the loop on line 510 didn't complete
511 if len(glob.glob(path)): 511 ↛ 513line 511 didn't jump to line 513, because the condition on line 511 was never false
512 return True
513 time.sleep(1)
514 return False
517def wait_for_nopath(path, timeout):
518 for i in range(0, timeout):
519 if not os.path.exists(path):
520 return True
521 time.sleep(1)
522 return False
525def wait_for_path_multi(path, timeout):
526 for i in range(0, timeout):
527 paths = glob.glob(path)
528 SMlog("_wait_for_paths_multi: paths = %s" % paths)
529 if len(paths):
530 SMlog("_wait_for_paths_multi: return first path: %s" % paths[0])
531 return paths[0]
532 time.sleep(1)
533 return ""
536def isdir(path):
537 try:
538 st = os.stat(path)
539 return stat.S_ISDIR(st.st_mode)
540 except OSError as inst:
541 if inst.errno == errno.EIO: 541 ↛ 542line 541 didn't jump to line 542, because the condition on line 541 was never true
542 raise CommandException(errno.EIO, "os.stat(%s)" % path, "failed")
543 return False
546def get_single_entry(path):
547 f = open(path, 'r')
548 line = f.readline()
549 f.close()
550 return line.rstrip()
553def get_fs_size(path):
554 st = ioretry_stat(path)
555 return st.f_blocks * st.f_frsize
558def get_fs_utilisation(path):
559 st = ioretry_stat(path)
560 return (st.f_blocks - st.f_bfree) * \
561 st.f_frsize
564def ismount(path):
565 """Test whether a path is a mount point"""
566 try:
567 s1 = os.stat(path)
568 s2 = os.stat(os.path.join(path, '..'))
569 except OSError as inst:
570 raise CommandException(inst.errno, "os.stat")
571 dev1 = s1.st_dev
572 dev2 = s2.st_dev
573 if dev1 != dev2:
574 return True # path/.. on a different device as path
575 ino1 = s1.st_ino
576 ino2 = s2.st_ino
577 if ino1 == ino2:
578 return True # path/.. is the same i-node as path
579 return False
582def makedirs(name, mode=0o777):
583 head, tail = os.path.split(name)
584 if not tail: 584 ↛ 585line 584 didn't jump to line 585, because the condition on line 584 was never true
585 head, tail = os.path.split(head)
586 if head and tail and not pathexists(head):
587 makedirs(head, mode)
588 if tail == os.curdir:
589 return
590 try:
591 os.mkdir(name, mode)
592 except OSError as exc:
593 if exc.errno == errno.EEXIST and os.path.isdir(name): 593 ↛ 594line 593 didn't jump to line 594, because the condition on line 593 was never true
594 if mode:
595 os.chmod(name, mode)
596 pass
597 else:
598 raise
601def zeroOut(path, fromByte, bytes):
602 """write 'bytes' zeros to 'path' starting from fromByte (inclusive)"""
603 blockSize = 4096
605 fromBlock = fromByte // blockSize
606 if fromByte % blockSize:
607 fromBlock += 1
608 bytesBefore = fromBlock * blockSize - fromByte
609 if bytesBefore > bytes:
610 bytesBefore = bytes
611 bytes -= bytesBefore
612 cmd = [CMD_DD, "if=/dev/zero", "of=%s" % path, "bs=1",
613 "seek=%s" % fromByte, "count=%s" % bytesBefore]
614 try:
615 pread2(cmd)
616 except CommandException:
617 return False
619 blocks = bytes // blockSize
620 bytes -= blocks * blockSize
621 fromByte = (fromBlock + blocks) * blockSize
622 if blocks:
623 cmd = [CMD_DD, "if=/dev/zero", "of=%s" % path, "bs=%s" % blockSize,
624 "seek=%s" % fromBlock, "count=%s" % blocks]
625 try:
626 pread2(cmd)
627 except CommandException:
628 return False
630 if bytes:
631 cmd = [CMD_DD, "if=/dev/zero", "of=%s" % path, "bs=1",
632 "seek=%s" % fromByte, "count=%s" % bytes]
633 try:
634 pread2(cmd)
635 except CommandException:
636 return False
638 return True
641def match_rootdev(s):
642 regex = re.compile("^PRIMARY_DISK")
643 return regex.search(s, 0)
646def getrootdev():
647 filename = '/etc/xensource-inventory'
648 try:
649 f = open(filename, 'r')
650 except:
651 raise xs_errors.XenError('EIO', \
652 opterr="Unable to open inventory file [%s]" % filename)
653 rootdev = ''
654 for line in filter(match_rootdev, f.readlines()):
655 rootdev = line.split("'")[1]
656 if not rootdev: 656 ↛ 657line 656 didn't jump to line 657, because the condition on line 656 was never true
657 raise xs_errors.XenError('NoRootDev')
658 return rootdev
661def getrootdevID():
662 rootdev = getrootdev()
663 try:
664 rootdevID = scsiutil.getSCSIid(rootdev)
665 except:
666 SMlog("util.getrootdevID: Unable to verify serial or SCSIid of device: %s" \
667 % rootdev)
668 return ''
670 if not len(rootdevID):
671 SMlog("util.getrootdevID: Unable to identify scsi device [%s] via scsiID" \
672 % rootdev)
674 return rootdevID
677def get_localAPI_session():
678 # First acquire a valid session
679 session = XenAPI.xapi_local()
680 try:
681 session.xenapi.login_with_password('root', '', '', 'SM')
682 except:
683 raise xs_errors.XenError('APISession')
684 return session
687def get_this_host():
688 uuid = None
689 f = open("/etc/xensource-inventory", 'r')
690 for line in f.readlines():
691 if line.startswith("INSTALLATION_UUID"):
692 uuid = line.split("'")[1]
693 f.close()
694 return uuid
697def get_master_ref(session):
698 pools = session.xenapi.pool.get_all()
699 return session.xenapi.pool.get_master(pools[0])
702def is_master(session):
703 return get_this_host_ref(session) == get_master_ref(session)
706def get_localhost_ref(session):
707 filename = '/etc/xensource-inventory'
708 try:
709 f = open(filename, 'r')
710 except:
711 raise xs_errors.XenError('EIO', \
712 opterr="Unable to open inventory file [%s]" % filename)
713 domid = ''
714 for line in filter(match_domain_id, f.readlines()):
715 domid = line.split("'")[1]
716 if not domid:
717 raise xs_errors.XenError('APILocalhost')
719 vms = session.xenapi.VM.get_all_records_where('field "uuid" = "%s"' % domid)
720 for vm in vms:
721 record = vms[vm]
722 if record["uuid"] == domid:
723 hostid = record["resident_on"]
724 return hostid
725 raise xs_errors.XenError('APILocalhost')
728def match_domain_id(s):
729 regex = re.compile("^CONTROL_DOMAIN_UUID")
730 return regex.search(s, 0)
733def get_hosts_attached_on(session, vdi_uuids):
734 host_refs = {}
735 for vdi_uuid in vdi_uuids:
736 try:
737 vdi_ref = session.xenapi.VDI.get_by_uuid(vdi_uuid)
738 except XenAPI.Failure:
739 SMlog("VDI %s not in db, ignoring" % vdi_uuid)
740 continue
741 sm_config = session.xenapi.VDI.get_sm_config(vdi_ref)
742 for key in [x for x in sm_config.keys() if x.startswith('host_')]:
743 host_refs[key[len('host_'):]] = True
744 return host_refs.keys()
746def get_this_host_address(session):
747 host_uuid = get_this_host()
748 host_ref = session.xenapi.host.get_by_uuid(host_uuid)
749 return session.xenapi.host.get_record(host_ref)['address']
751def get_host_addresses(session):
752 addresses = []
753 hosts = session.xenapi.host.get_all_records()
754 for record in hosts.values():
755 addresses.append(record['address'])
756 return addresses
758def get_this_host_ref(session):
759 host_uuid = get_this_host()
760 host_ref = session.xenapi.host.get_by_uuid(host_uuid)
761 return host_ref
764def get_slaves_attached_on(session, vdi_uuids):
765 "assume this host is the SR master"
766 host_refs = get_hosts_attached_on(session, vdi_uuids)
767 master_ref = get_this_host_ref(session)
768 return [x for x in host_refs if x != master_ref]
771def get_online_hosts(session):
772 online_hosts = []
773 hosts = session.xenapi.host.get_all_records()
774 for host_ref, host_rec in hosts.items():
775 metricsRef = host_rec["metrics"]
776 metrics = session.xenapi.host_metrics.get_record(metricsRef)
777 if metrics["live"]:
778 online_hosts.append(host_ref)
779 return online_hosts
782def get_all_slaves(session):
783 "assume this host is the SR master"
784 host_refs = get_online_hosts(session)
785 master_ref = get_this_host_ref(session)
786 return [x for x in host_refs if x != master_ref]
789def is_attached_rw(sm_config):
790 for key, val in sm_config.items():
791 if key.startswith("host_") and val == "RW":
792 return True
793 return False
796def attached_as(sm_config):
797 for key, val in sm_config.items():
798 if key.startswith("host_") and (val == "RW" or val == "RO"): 798 ↛ 799line 798 didn't jump to line 799, because the condition on line 798 was never true
799 return val
802def find_my_pbd_record(session, host_ref, sr_ref):
803 try:
804 pbds = session.xenapi.PBD.get_all_records()
805 for pbd_ref in pbds.keys():
806 if pbds[pbd_ref]['host'] == host_ref and pbds[pbd_ref]['SR'] == sr_ref:
807 return [pbd_ref, pbds[pbd_ref]]
808 return None
809 except Exception as e:
810 SMlog("Caught exception while looking up PBD for host %s SR %s: %s" % (str(host_ref), str(sr_ref), str(e)))
811 return None
814def find_my_pbd(session, host_ref, sr_ref):
815 ret = find_my_pbd_record(session, host_ref, sr_ref)
816 if ret is not None:
817 return ret[0]
818 else:
819 return None
822def test_hostPBD_devs(session, sr_uuid, devs):
823 host = get_localhost_ref(session)
824 sr = session.xenapi.SR.get_by_uuid(sr_uuid)
825 try:
826 pbds = session.xenapi.PBD.get_all_records()
827 except:
828 raise xs_errors.XenError('APIPBDQuery')
829 for dev in devs.split(','):
830 for pbd in pbds:
831 record = pbds[pbd]
832 # it's ok if it's *our* PBD
833 if record["SR"] == sr:
834 break
835 if record["host"] == host:
836 devconfig = record["device_config"]
837 if 'device' in devconfig:
838 for device in devconfig['device'].split(','):
839 if os.path.realpath(device) == os.path.realpath(dev):
840 return True
841 return False
844def test_hostPBD_lun(session, targetIQN, LUNid):
845 host = get_localhost_ref(session)
846 try:
847 pbds = session.xenapi.PBD.get_all_records()
848 except:
849 raise xs_errors.XenError('APIPBDQuery')
850 for pbd in pbds:
851 record = pbds[pbd]
852 if record["host"] == host:
853 devconfig = record["device_config"]
854 if 'targetIQN' in devconfig and 'LUNid' in devconfig:
855 if devconfig['targetIQN'] == targetIQN and \
856 devconfig['LUNid'] == LUNid:
857 return True
858 return False
861def test_SCSIid(session, sr_uuid, SCSIid):
862 if sr_uuid is not None:
863 sr = session.xenapi.SR.get_by_uuid(sr_uuid)
864 try:
865 pbds = session.xenapi.PBD.get_all_records()
866 except:
867 raise xs_errors.XenError('APIPBDQuery')
868 for pbd in pbds:
869 record = pbds[pbd]
870 # it's ok if it's *our* PBD
871 # During FC SR creation, devscan.py passes sr_uuid as None
872 if sr_uuid is not None:
873 if record["SR"] == sr:
874 break
875 devconfig = record["device_config"]
876 sm_config = session.xenapi.SR.get_sm_config(record["SR"])
877 if 'SCSIid' in devconfig and devconfig['SCSIid'] == SCSIid:
878 return True
879 elif 'SCSIid' in sm_config and sm_config['SCSIid'] == SCSIid:
880 return True
881 elif 'scsi-' + SCSIid in sm_config:
882 return True
883 return False
886class TimeoutException(SMException):
887 pass
890def timeout_call(timeoutseconds, function, *arguments):
891 def handler(signum, frame):
892 raise TimeoutException()
893 signal.signal(signal.SIGALRM, handler)
894 signal.alarm(timeoutseconds)
895 try:
896 return function(*arguments)
897 finally:
898 signal.alarm(0)
901def _incr_iscsiSR_refcount(targetIQN, uuid):
902 if not os.path.exists(ISCSI_REFDIR):
903 os.mkdir(ISCSI_REFDIR)
904 filename = os.path.join(ISCSI_REFDIR, targetIQN)
905 try:
906 f = open(filename, 'a+')
907 except:
908 raise xs_errors.XenError('LVMRefCount', \
909 opterr='file %s' % filename)
911 f.seek(0)
912 found = False
913 refcount = 0
914 for line in filter(match_uuid, f.readlines()):
915 refcount += 1
916 if line.find(uuid) != -1:
917 found = True
918 if not found:
919 f.write("%s\n" % uuid)
920 refcount += 1
921 f.close()
922 return refcount
925def _decr_iscsiSR_refcount(targetIQN, uuid):
926 filename = os.path.join(ISCSI_REFDIR, targetIQN)
927 if not os.path.exists(filename):
928 return 0
929 try:
930 f = open(filename, 'a+')
931 except:
932 raise xs_errors.XenError('LVMRefCount', \
933 opterr='file %s' % filename)
935 f.seek(0)
936 output = []
937 refcount = 0
938 for line in filter(match_uuid, f.readlines()):
939 if line.find(uuid) == -1:
940 output.append(line.rstrip())
941 refcount += 1
942 if not refcount:
943 os.unlink(filename)
944 return refcount
946 # Re-open file and truncate
947 f.close()
948 f = open(filename, 'w')
949 for i in range(0, refcount):
950 f.write("%s\n" % output[i])
951 f.close()
952 return refcount
955# The agent enforces 1 PBD per SR per host, so we
956# check for active SR entries not attached to this host
957def test_activePoolPBDs(session, host, uuid):
958 try:
959 pbds = session.xenapi.PBD.get_all_records()
960 except:
961 raise xs_errors.XenError('APIPBDQuery')
962 for pbd in pbds:
963 record = pbds[pbd]
964 if record["host"] != host and record["SR"] == uuid \
965 and record["currently_attached"]:
966 return True
967 return False
970def remove_mpathcount_field(session, host_ref, sr_ref, SCSIid):
971 try:
972 pbdref = find_my_pbd(session, host_ref, sr_ref)
973 if pbdref is not None:
974 key = "mpath-" + SCSIid
975 session.xenapi.PBD.remove_from_other_config(pbdref, key)
976 except:
977 pass
981def _testHost(hostname, port, errstring):
982 SMlog("_testHost: Testing host/port: %s,%d" % (hostname, port))
983 try:
984 sockinfo = socket.getaddrinfo(hostname, int(port))[0]
985 except:
986 logException('Exception occured getting IP for %s' % hostname)
987 raise xs_errors.XenError('DNSError')
989 timeout = 5
991 sock = socket.socket(sockinfo[0], socket.SOCK_STREAM)
992 # Only allow the connect to block for up to timeout seconds
993 sock.settimeout(timeout)
994 try:
995 sock.connect(sockinfo[4])
996 # Fix for MS storage server bug
997 sock.send(b'\n')
998 sock.close()
999 except socket.error as reason:
1000 SMlog("_testHost: Connect failed after %d seconds (%s) - %s" \
1001 % (timeout, hostname, reason))
1002 raise xs_errors.XenError(errstring)
1005def match_scsiID(s, id):
1006 regex = re.compile(id)
1007 return regex.search(s, 0)
1010def _isSCSIid(s):
1011 regex = re.compile("^scsi-")
1012 return regex.search(s, 0)
1015def test_scsiserial(session, device):
1016 device = os.path.realpath(device)
1017 if not scsiutil._isSCSIdev(device):
1018 SMlog("util.test_scsiserial: Not a serial device: %s" % device)
1019 return False
1020 serial = ""
1021 try:
1022 serial += scsiutil.getserial(device)
1023 except:
1024 # Error allowed, SCSIid is the important one
1025 pass
1027 try:
1028 scsiID = scsiutil.getSCSIid(device)
1029 except:
1030 SMlog("util.test_scsiserial: Unable to verify serial or SCSIid of device: %s" \
1031 % device)
1032 return False
1033 if not len(scsiID):
1034 SMlog("util.test_scsiserial: Unable to identify scsi device [%s] via scsiID" \
1035 % device)
1036 return False
1038 try:
1039 SRs = session.xenapi.SR.get_all_records()
1040 except:
1041 raise xs_errors.XenError('APIFailure')
1042 for SR in SRs:
1043 record = SRs[SR]
1044 conf = record["sm_config"]
1045 if 'devserial' in conf:
1046 for dev in conf['devserial'].split(','):
1047 if _isSCSIid(dev):
1048 if match_scsiID(dev, scsiID):
1049 return True
1050 elif len(serial) and dev == serial:
1051 return True
1052 return False
1055def default(self, field, thunk):
1056 try:
1057 return getattr(self, field)
1058 except:
1059 return thunk()
1062def list_VDI_records_in_sr(sr):
1063 """Helper function which returns a list of all VDI records for this SR
1064 stored in the XenAPI server, useful for implementing SR.scan"""
1065 sr_ref = sr.session.xenapi.SR.get_by_uuid(sr.uuid)
1066 vdis = sr.session.xenapi.VDI.get_all_records_where("field \"SR\" = \"%s\"" % sr_ref)
1067 return vdis
1070# Given a partition (e.g. sda1), get a disk name:
1071def diskFromPartition(partition):
1072 # check whether this is a device mapper device (e.g. /dev/dm-0)
1073 m = re.match('(/dev/)?(dm-[0-9]+)(p[0-9]+)?$', partition)
1074 if m is not None: 1074 ↛ 1075line 1074 didn't jump to line 1075, because the condition on line 1074 was never true
1075 return m.group(2)
1077 numlen = 0 # number of digit characters
1078 m = re.match("\D+(\d+)", partition)
1079 if m is not None: 1079 ↛ 1080line 1079 didn't jump to line 1080, because the condition on line 1079 was never true
1080 numlen = len(m.group(1))
1082 # is it a cciss?
1083 if True in [partition.startswith(x) for x in ['cciss', 'ida', 'rd']]: 1083 ↛ 1084line 1083 didn't jump to line 1084, because the condition on line 1083 was never true
1084 numlen += 1 # need to get rid of trailing 'p'
1086 # is it a mapper path?
1087 if partition.startswith("mapper"): 1087 ↛ 1088line 1087 didn't jump to line 1088, because the condition on line 1087 was never true
1088 if re.search("p[0-9]*$", partition):
1089 numlen = len(re.match("\d+", partition[::-1]).group(0)) + 1
1090 SMlog("Found mapper part, len %d" % numlen)
1091 else:
1092 numlen = 0
1094 # is it /dev/disk/by-id/XYZ-part<k>?
1095 if partition.startswith("disk/by-id"): 1095 ↛ 1096line 1095 didn't jump to line 1096, because the condition on line 1095 was never true
1096 return partition[:partition.rfind("-part")]
1098 return partition[:len(partition) - numlen]
1101def dom0_disks():
1102 """Disks carrying dom0, e.g. ['/dev/sda']"""
1103 disks = []
1104 with open("/etc/mtab", 'r') as f:
1105 for line in f:
1106 (dev, mountpoint, fstype, opts, freq, passno) = line.split(' ')
1107 if mountpoint == '/':
1108 disk = diskFromPartition(dev)
1109 if not (disk in disks):
1110 disks.append(disk)
1111 SMlog("Dom0 disks: %s" % disks)
1112 return disks
1115def set_scheduler_sysfs_node(node, str):
1116 """Set the scheduler for a sysfs node (e.g. '/sys/block/sda')"""
1118 path = os.path.join(node, "queue", "scheduler")
1119 if not os.path.exists(path):
1120 SMlog("no path %s" % path)
1121 return
1122 try:
1123 f = open(path, 'w')
1124 f.write("%s\n" % str)
1125 f.close()
1126 SMlog("Set scheduler to [%s] on [%s]" % (str, node))
1127 except:
1128 SMlog("Error setting scheduler to [%s] on [%s]" % (str, node))
1129 pass
1132def set_scheduler(dev, str):
1133 devices = []
1134 if not scsiutil.match_dm(dev):
1135 # Remove partition numbers
1136 devices.append(diskFromPartition(dev).replace('/', '!'))
1137 else:
1138 rawdev = diskFromPartition(dev)
1139 devices = [os.path.realpath(x)[5:] for x in scsiutil._genReverseSCSIidmap(rawdev.split('/')[-1])]
1141 for d in devices:
1142 set_scheduler_sysfs_node("/sys/block/%s" % d, str)
1145# This function queries XAPI for the existing VDI records for this SR
1146def _getVDIs(srobj):
1147 VDIs = []
1148 try:
1149 sr_ref = getattr(srobj, 'sr_ref')
1150 except AttributeError:
1151 return VDIs
1153 refs = srobj.session.xenapi.SR.get_VDIs(sr_ref)
1154 for vdi in refs:
1155 ref = srobj.session.xenapi.VDI.get_record(vdi)
1156 ref['vdi_ref'] = vdi
1157 VDIs.append(ref)
1158 return VDIs
1161def _getVDI(srobj, vdi_uuid):
1162 vdi = srobj.session.xenapi.VDI.get_by_uuid(vdi_uuid)
1163 ref = srobj.session.xenapi.VDI.get_record(vdi)
1164 ref['vdi_ref'] = vdi
1165 return ref
1168def _convertDNS(name):
1169 addr = socket.getaddrinfo(name, None)[0][4][0]
1170 return addr
1173def _containsVDIinuse(srobj):
1174 VDIs = _getVDIs(srobj)
1175 for vdi in VDIs:
1176 if not vdi['managed']:
1177 continue
1178 sm_config = vdi['sm_config']
1179 if 'SRRef' in sm_config:
1180 try:
1181 PBDs = srobj.session.xenapi.SR.get_PBDs(sm_config['SRRef'])
1182 for pbd in PBDs:
1183 record = PBDs[pbd]
1184 if record["host"] == srobj.host_ref and \
1185 record["currently_attached"]:
1186 return True
1187 except:
1188 pass
1189 return False
1192def isVDICommand(cmd):
1193 if cmd is None or cmd in ["vdi_attach", "vdi_detach", 1193 ↛ 1196line 1193 didn't jump to line 1196, because the condition on line 1193 was never true
1194 "vdi_activate", "vdi_deactivate",
1195 "vdi_epoch_begin", "vdi_epoch_end"]:
1196 return True
1197 else:
1198 return False
1201#########################
1202# Daemon helper functions
1203def p_id_fork():
1204 try:
1205 p_id = os.fork()
1206 except OSError as e:
1207 print("Fork failed: %s (%d)" % (e.strerror, e.errno))
1208 sys.exit(-1)
1210 if (p_id == 0):
1211 os.setsid()
1212 try:
1213 p_id = os.fork()
1214 except OSError as e:
1215 print("Fork failed: %s (%d)" % (e.strerror, e.errno))
1216 sys.exit(-1)
1217 if (p_id == 0):
1218 os.chdir('/opt/xensource/sm')
1219 os.umask(0)
1220 else:
1221 os._exit(0)
1222 else:
1223 os._exit(0)
1226def daemon():
1227 p_id_fork()
1228 # Query the max file descriptor parameter for this process
1229 maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1231 # Close any fds that are open
1232 for fd in range(0, maxfd):
1233 try:
1234 os.close(fd)
1235 except:
1236 pass
1238 # Redirect STDIN to STDOUT and STDERR
1239 os.open('/dev/null', os.O_RDWR)
1240 os.dup2(0, 1)
1241 os.dup2(0, 2)
1242#########################
1244if __debug__:
1245 try:
1246 XE_IOFI_IORETRY
1247 except NameError:
1248 XE_IOFI_IORETRY = os.environ.get('XE_IOFI_IORETRY', None)
1249 if __name__ == 'util' and XE_IOFI_IORETRY is not None: 1249 ↛ 1250line 1249 didn't jump to line 1250, because the condition on line 1249 was never true
1250 __import__('iofi')
1252################################################################################
1253#
1254# Fist points
1255#
1257# * The global variable 'fistpoint' define the list of all possible fistpoints;
1258#
1259# * To activate a fistpoint called 'name', you need to create the file '/tmp/fist_name'
1260# on the SR master;
1261#
1262# * At the moment, activating a fist point can lead to two possible behaviors:
1263# - if '/tmp/fist_LVHDRT_exit' exists, then the function called during the fistpoint is _exit;
1264# - otherwise, the function called is _pause.
1266def _pause(secs, name):
1267 SMlog("Executing fist point %s: sleeping %d seconds ..." % (name, secs))
1268 time.sleep(secs)
1269 SMlog("Executing fist point %s: done" % name)
1272def _exit(name):
1273 SMlog("Executing fist point %s: exiting the current process ..." % name)
1274 raise xs_errors.XenError('FistPoint', opterr='%s' % name)
1277class FistPoint:
1278 def __init__(self, points):
1279 #SMlog("Fist points loaded")
1280 self.points = points
1282 def is_legal(self, name):
1283 return (name in self.points)
1285 def is_active(self, name):
1286 return os.path.exists("/tmp/fist_%s" % name)
1288 def mark_sr(self, name, sruuid, started):
1289 session = get_localAPI_session()
1290 sr = session.xenapi.SR.get_by_uuid(sruuid)
1291 if started:
1292 session.xenapi.SR.add_to_other_config(sr, name, "active")
1293 else:
1294 session.xenapi.SR.remove_from_other_config(sr, name)
1296 def activate(self, name, sruuid):
1297 if name in self.points: 1297 ↛ 1307line 1297 didn't jump to line 1307, because the condition on line 1297 was never false
1298 if self.is_active(name): 1298 ↛ 1299line 1298 didn't jump to line 1299, because the condition on line 1298 was never true
1299 self.mark_sr(name, sruuid, True)
1300 if self.is_active("LVHDRT_exit"):
1301 self.mark_sr(name, sruuid, False)
1302 _exit(name)
1303 else:
1304 _pause(FIST_PAUSE_PERIOD, name)
1305 self.mark_sr(name, sruuid, False)
1306 else:
1307 SMlog("Unknown fist point: %s" % name)
1309 def activate_custom_fn(self, name, fn):
1310 if name in self.points: 1310 ↛ 1316line 1310 didn't jump to line 1316, because the condition on line 1310 was never false
1311 if self.is_active(name): 1311 ↛ 1312line 1311 didn't jump to line 1312, because the condition on line 1311 was never true
1312 SMlog("Executing fist point %s: starting ..." % name)
1313 fn()
1314 SMlog("Executing fist point %s: done" % name)
1315 else:
1316 SMlog("Unknown fist point: %s" % name)
1319def list_find(f, seq):
1320 for item in seq:
1321 if f(item):
1322 return item
1324GCPAUSE_FISTPOINT = "GCLoop_no_pause"
1326fistpoint = FistPoint(["LVHDRT_finding_a_suitable_pair",
1327 "LVHDRT_inflating_the_parent",
1328 "LVHDRT_resizing_while_vdis_are_paused",
1329 "LVHDRT_coalescing_VHD_data",
1330 "LVHDRT_coalescing_before_inflate_grandparent",
1331 "LVHDRT_relinking_grandchildren",
1332 "LVHDRT_before_create_relink_journal",
1333 "LVHDRT_xapiSM_serialization_tests",
1334 "LVHDRT_clone_vdi_after_create_journal",
1335 "LVHDRT_clone_vdi_after_shrink_parent",
1336 "LVHDRT_clone_vdi_after_first_snap",
1337 "LVHDRT_clone_vdi_after_second_snap",
1338 "LVHDRT_clone_vdi_after_parent_hidden",
1339 "LVHDRT_clone_vdi_after_parent_ro",
1340 "LVHDRT_clone_vdi_before_remove_journal",
1341 "LVHDRT_clone_vdi_after_lvcreate",
1342 "LVHDRT_clone_vdi_before_undo_clone",
1343 "LVHDRT_clone_vdi_after_undo_clone",
1344 "LVHDRT_inflate_after_create_journal",
1345 "LVHDRT_inflate_after_setSize",
1346 "LVHDRT_inflate_after_zeroOut",
1347 "LVHDRT_inflate_after_setSizePhys",
1348 "LVHDRT_inflate_after_setSizePhys",
1349 "LVHDRT_coaleaf_before_coalesce",
1350 "LVHDRT_coaleaf_after_coalesce",
1351 "LVHDRT_coaleaf_one_renamed",
1352 "LVHDRT_coaleaf_both_renamed",
1353 "LVHDRT_coaleaf_after_vdirec",
1354 "LVHDRT_coaleaf_before_delete",
1355 "LVHDRT_coaleaf_after_delete",
1356 "LVHDRT_coaleaf_before_remove_j",
1357 "LVHDRT_coaleaf_undo_after_rename",
1358 "LVHDRT_coaleaf_undo_after_rename2",
1359 "LVHDRT_coaleaf_undo_after_refcount",
1360 "LVHDRT_coaleaf_undo_after_deflate",
1361 "LVHDRT_coaleaf_undo_end",
1362 "LVHDRT_coaleaf_stop_after_recovery",
1363 "LVHDRT_coaleaf_finish_after_inflate",
1364 "LVHDRT_coaleaf_finish_end",
1365 "LVHDRT_coaleaf_delay_1",
1366 "LVHDRT_coaleaf_delay_2",
1367 "LVHDRT_coaleaf_delay_3",
1368 "testsm_clone_allow_raw",
1369 "xenrt_default_vdi_type_legacy",
1370 "blktap_activate_inject_failure",
1371 "blktap_activate_error_handling",
1372 GCPAUSE_FISTPOINT,
1373 "cleanup_coalesceVHD_inject_failure",
1374 "cleanup_tracker_no_progress",
1375 "FileSR_fail_hardlink",
1376 "FileSR_fail_snap1",
1377 "FileSR_fail_snap2"])
1380def set_dirty(session, sr):
1381 try:
1382 session.xenapi.SR.add_to_other_config(sr, "dirty", "")
1383 SMlog("set_dirty %s succeeded" % (repr(sr)))
1384 except:
1385 SMlog("set_dirty %s failed (flag already set?)" % (repr(sr)))
1388def doesFileHaveOpenHandles(fileName):
1389 SMlog("Entering doesFileHaveOpenHandles with file: %s" % fileName)
1390 (retVal, processAndPidTuples) = \
1391 findRunningProcessOrOpenFile(fileName, False)
1393 if not retVal:
1394 SMlog("Failed to determine if file %s has open handles." % \
1395 fileName)
1396 # err on the side of caution
1397 return True
1398 else:
1399 if len(processAndPidTuples) > 0:
1400 return True
1401 else:
1402 return False
1405# extract SR uuid from the passed in devmapper entry and return
1406# /dev/mapper/VG_XenStorage--c3d82e92--cb25--c99b--b83a--482eebab4a93-MGT
1407def extractSRFromDevMapper(path):
1408 try:
1409 path = os.path.basename(path)
1410 path = path[len('VG_XenStorage-') + 1:]
1411 path = path.replace('--', '/')
1412 path = path[0:path.rfind('-')]
1413 return path.replace('/', '-')
1414 except:
1415 return ''
1418# Looks at /proc and figures either
1419# If a process is still running (default), returns open file names
1420# If any running process has open handles to the given file (process = False)
1421# returns process names and pids
1422def findRunningProcessOrOpenFile(name, process=True):
1423 retVal = True
1424 links = []
1425 processandpids = []
1426 sockets = set()
1427 try:
1428 SMlog("Entering findRunningProcessOrOpenFile with params: %s" % \
1429 [name, process])
1431 # Look at all pids
1432 pids = [pid for pid in os.listdir('/proc') if pid.isdigit()]
1433 for pid in sorted(pids):
1434 try:
1435 try:
1436 f = None
1437 f = open(os.path.join('/proc', pid, 'cmdline'), 'r')
1438 prog = f.read()[:-1]
1439 if prog: 1439 ↛ 1448line 1439 didn't jump to line 1448, because the condition on line 1439 was never false
1440 # Just want the process name
1441 argv = prog.split('\x00')
1442 prog = argv[0]
1443 except IOError as e:
1444 if e.errno in (errno.ENOENT, errno.ESRCH):
1445 SMlog("ERROR %s reading %s, ignore" % (e.errno, pid))
1446 continue
1447 finally:
1448 if f is not None: 1448 ↛ 1433, 1448 ↛ 14512 missed branches: 1) line 1448 didn't jump to line 1433, because the continue on line 1446 wasn't executed, 2) line 1448 didn't jump to line 1451, because the condition on line 1448 was never false
1449 f.close() 1449 ↛ 1433line 1449 didn't jump to line 1433, because the continue on line 1446 wasn't executed
1451 try:
1452 fd_dir = os.path.join('/proc', pid, 'fd')
1453 files = os.listdir(fd_dir)
1454 except OSError as e:
1455 if e.errno in (errno.ENOENT, errno.ESRCH):
1456 SMlog("ERROR %s reading fds for %s, ignore" % (e.errno, pid))
1457 # Ignore pid that are no longer valid
1458 continue
1459 else:
1460 raise
1462 for file in files:
1463 try:
1464 link = os.readlink(os.path.join(fd_dir, file))
1465 except OSError:
1466 continue
1468 if process: 1468 ↛ 1473line 1468 didn't jump to line 1473, because the condition on line 1468 was never false
1469 if name == prog: 1469 ↛ 1462line 1469 didn't jump to line 1462, because the condition on line 1469 was never false
1470 links.append(link)
1471 else:
1472 # need to return process name and pid tuples
1473 if link == name:
1474 SMlog("File %s has an open handle with process %s "
1475 "with pid %s" % (name, prog, pid))
1476 processandpids.append((prog, pid))
1478 # Get the connected sockets
1479 if name == prog:
1480 sockets.update(get_connected_sockets(pid))
1481 except Exception as e:
1482 SMlog("Exception checking running process or open file handles. " \
1483 "Error: %s" % str(e))
1484 retVal = False
1486 if process: 1486 ↛ 1489line 1486 didn't jump to line 1489, because the condition on line 1486 was never false
1487 return retVal, links, sockets
1488 else:
1489 return retVal, processandpids
1492def get_connected_sockets(pid):
1493 sockets = set()
1494 try:
1495 # Lines in /proc/<pid>/net/unix are formatted as follows
1496 # (see Linux source net/unix/af_unix.c, unix_seq_show() )
1497 # - Pointer address to socket (hex)
1498 # - Refcount (HEX)
1499 # - 0
1500 # - State (HEX, 0 or __SO_ACCEPTCON)
1501 # - Type (HEX - but only 0001 of interest)
1502 # - Connection state (HEX - but only 03, SS_CONNECTED of interest)
1503 # - Inode number
1504 # - Path (optional)
1505 open_sock_matcher = re.compile(
1506 r'^[0-9a-f]+: [0-9A-Fa-f]+ [0-9A-Fa-f]+ [0-9A-Fa-f]+ 0001 03 \d+ (.*)$')
1507 with open(
1508 os.path.join('/proc', str(pid), 'net', 'unix'), 'r') as f:
1509 lines = f.readlines()
1510 for line in lines:
1511 match = open_sock_matcher.match(line)
1512 if match:
1513 sockets.add(match[1])
1514 except OSError as e:
1515 if e.errno in (errno.ENOENT, errno.ESRCH):
1516 # Ignore pid that are no longer valid
1517 SMlog("ERROR %s reading sockets for %s, ignore" %
1518 (e.errno, pid))
1519 else:
1520 raise
1521 return sockets
1524def retry(f, maxretry=20, period=3, exceptions=[Exception]):
1525 retries = 0
1526 while True:
1527 try:
1528 return f()
1529 except Exception as e:
1530 for exception in exceptions:
1531 if isinstance(e, exception):
1532 SMlog('Got exception: {}. Retry number: {}'.format(
1533 str(e), retries
1534 ))
1535 break
1536 else:
1537 SMlog('Got bad exception: {}. Raising...'.format(e))
1538 raise e
1540 retries += 1
1541 if retries >= maxretry:
1542 break
1544 time.sleep(period)
1546 return f()
1549def getCslDevPath(svid):
1550 basepath = "/dev/disk/by-csldev/"
1551 if svid.startswith("NETAPP_"):
1552 # special attention for NETAPP SVIDs
1553 svid_parts = svid.split("__")
1554 globstr = basepath + "NETAPP__LUN__" + "*" + svid_parts[2] + "*" + svid_parts[-1] + "*"
1555 else:
1556 globstr = basepath + svid + "*"
1558 return globstr
1561# Use device in /dev pointed to by cslg path which consists of svid
1562def get_scsiid_from_svid(md_svid):
1563 cslg_path = getCslDevPath(md_svid)
1564 abs_path = glob.glob(cslg_path)
1565 if abs_path:
1566 real_path = os.path.realpath(abs_path[0])
1567 return scsiutil.getSCSIid(real_path)
1568 else:
1569 return None
1572def get_isl_scsiids(session):
1573 # Get cslg type SRs
1574 SRs = session.xenapi.SR.get_all_records_where('field "type" = "cslg"')
1576 # Iterate through the SR to get the scsi ids
1577 scsi_id_ret = []
1578 for SR in SRs:
1579 sr_rec = SRs[SR]
1580 # Use the md_svid to get the scsi id
1581 scsi_id = get_scsiid_from_svid(sr_rec['sm_config']['md_svid'])
1582 if scsi_id:
1583 scsi_id_ret.append(scsi_id)
1585 # Get the vdis in the SR and do the same procedure
1586 vdi_recs = session.xenapi.VDI.get_all_records_where('field "SR" = "%s"' % SR)
1587 for vdi_rec in vdi_recs:
1588 vdi_rec = vdi_recs[vdi_rec]
1589 scsi_id = get_scsiid_from_svid(vdi_rec['sm_config']['SVID'])
1590 if scsi_id:
1591 scsi_id_ret.append(scsi_id)
1593 return scsi_id_ret
1596class extractXVA:
1597 # streams files as a set of file and checksum, caller should remove
1598 # the files, if not needed. The entire directory (Where the files
1599 # and checksum) will only be deleted as part of class cleanup.
1600 HDR_SIZE = 512
1601 BLOCK_SIZE = 512
1602 SIZE_LEN = 12 - 1 # To remove \0 from tail
1603 SIZE_OFFSET = 124
1604 ZERO_FILLED_REC = 2
1605 NULL_IDEN = '\x00'
1606 DIR_IDEN = '/'
1607 CHECKSUM_IDEN = '.checksum'
1608 OVA_FILE = 'ova.xml'
1610 # Init gunzips the file using a subprocess, and reads stdout later
1611 # as and when needed
1612 def __init__(self, filename):
1613 self.__extract_path = ''
1614 self.__filename = filename
1615 cmd = 'gunzip -cd %s' % filename
1616 try:
1617 self.spawn_p = subprocess.Popen(
1618 cmd, shell=True, \
1619 stdin=subprocess.PIPE, stdout=subprocess.PIPE, \
1620 stderr=subprocess.PIPE, close_fds=True)
1621 except Exception as e:
1622 SMlog("Error: %s. Uncompress failed for %s" % (str(e), filename))
1623 raise Exception(str(e))
1625 # Create dir to extract the files
1626 self.__extract_path = tempfile.mkdtemp()
1628 def __del__(self):
1629 shutil.rmtree(self.__extract_path)
1631 # Class supports Generator expression. 'for f_name, checksum in getTuple()'
1632 # returns filename, checksum content. Returns filename, '' in case
1633 # of checksum file missing. e.g. ova.xml
1634 def getTuple(self):
1635 zerod_record = 0
1636 ret_f_name = ''
1637 ret_base_f_name = ''
1639 try:
1640 # Read tar file as sets of file and checksum.
1641 while True:
1642 # Read the output of spawned process, or output of gunzip
1643 f_hdr = self.spawn_p.stdout.read(self.HDR_SIZE)
1645 # Break out in case of end of file
1646 if f_hdr == '':
1647 if zerod_record == extractXVA.ZERO_FILLED_REC:
1648 break
1649 else:
1650 SMlog('Error. Expects %d zero records', \
1651 extractXVA.ZERO_FILLED_REC)
1652 raise Exception('Unrecognized end of file')
1654 # Watch out for zero records, two zero records
1655 # denote end of file.
1656 if f_hdr == extractXVA.NULL_IDEN * extractXVA.HDR_SIZE:
1657 zerod_record += 1
1658 continue
1660 f_name = f_hdr[:f_hdr.index(extractXVA.NULL_IDEN)]
1661 # File header may be for a folder, if so ignore the header
1662 if not f_name.endswith(extractXVA.DIR_IDEN):
1663 f_size_octal = f_hdr[extractXVA.SIZE_OFFSET: \
1664 extractXVA.SIZE_OFFSET + extractXVA.SIZE_LEN]
1665 f_size = int(f_size_octal, 8)
1666 if f_name.endswith(extractXVA.CHECKSUM_IDEN):
1667 if f_name.rstrip(extractXVA.CHECKSUM_IDEN) == \
1668 ret_base_f_name:
1669 checksum = self.spawn_p.stdout.read(f_size)
1670 yield(ret_f_name, checksum)
1671 else:
1672 # Expects file followed by its checksum
1673 SMlog('Error. Sequence mismatch starting with %s', \
1674 ret_f_name)
1675 raise Exception( \
1676 'Files out of sequence starting with %s', \
1677 ret_f_name)
1678 else:
1679 # In case of ova.xml, read the contents into a file and
1680 # return the file name to the caller. For other files,
1681 # read the contents into a file, it will
1682 # be used when a .checksum file is encountered.
1683 ret_f_name = '%s/%s' % (self.__extract_path, f_name)
1684 ret_base_f_name = f_name
1686 # Check if the folder exists on the target location,
1687 # else create it.
1688 folder_path = ret_f_name[:ret_f_name.rfind('/')]
1689 if not os.path.exists(folder_path):
1690 os.mkdir(folder_path)
1692 # Store the file to the tmp folder, strip the tail \0
1693 f = open(ret_f_name, 'w')
1694 f.write(self.spawn_p.stdout.read(f_size))
1695 f.close()
1696 if f_name == extractXVA.OVA_FILE:
1697 yield(ret_f_name, '')
1699 # Skip zero'd portion of data block
1700 round_off = f_size % extractXVA.BLOCK_SIZE
1701 if round_off != 0:
1702 zeros = self.spawn_p.stdout.read(
1703 extractXVA.BLOCK_SIZE - round_off)
1704 except Exception as e:
1705 SMlog("Error: %s. File set extraction failed %s" % (str(e), \
1706 self.__filename))
1708 # Kill and Drain stdout of the gunzip process,
1709 # else gunzip might block on stdout
1710 os.kill(self.spawn_p.pid, signal.SIGTERM)
1711 self.spawn_p.communicate()
1712 raise Exception(str(e))
1714illegal_xml_chars = [(0x00, 0x08), (0x0B, 0x1F), (0x7F, 0x84), (0x86, 0x9F),
1715 (0xD800, 0xDFFF), (0xFDD0, 0xFDDF), (0xFFFE, 0xFFFF),
1716 (0x1FFFE, 0x1FFFF), (0x2FFFE, 0x2FFFF), (0x3FFFE, 0x3FFFF),
1717 (0x4FFFE, 0x4FFFF), (0x5FFFE, 0x5FFFF), (0x6FFFE, 0x6FFFF),
1718 (0x7FFFE, 0x7FFFF), (0x8FFFE, 0x8FFFF), (0x9FFFE, 0x9FFFF),
1719 (0xAFFFE, 0xAFFFF), (0xBFFFE, 0xBFFFF), (0xCFFFE, 0xCFFFF),
1720 (0xDFFFE, 0xDFFFF), (0xEFFFE, 0xEFFFF), (0xFFFFE, 0xFFFFF),
1721 (0x10FFFE, 0x10FFFF)]
1723illegal_ranges = ["%s-%s" % (chr(low), chr(high))
1724 for (low, high) in illegal_xml_chars
1725 if low < sys.maxunicode]
1727illegal_xml_re = re.compile(u'[%s]' % u''.join(illegal_ranges))
1730def isLegalXMLString(s):
1731 """Tells whether this is a valid XML string (i.e. it does not contain
1732 illegal XML characters specified in
1733 http://www.w3.org/TR/2004/REC-xml-20040204/#charsets).
1734 """
1736 if len(s) > 0:
1737 return re.search(illegal_xml_re, s) is None
1738 else:
1739 return True
1742def unictrunc(string, max_bytes):
1743 """
1744 Returns the number of bytes that is smaller than, or equal to, the number
1745 of bytes specified, such that the UTF-8 encoded string can be correctly
1746 truncated.
1747 string: the string to truncate
1748 max_bytes: the maximum number of bytes the truncated string can be
1749 """
1750 string = string.decode('UTF-8')
1751 cur_bytes = 0
1752 for char in string:
1753 charsize = len(char.encode('UTF-8'))
1754 if cur_bytes + charsize > max_bytes:
1755 break
1756 else:
1757 cur_bytes = cur_bytes + charsize
1758 return cur_bytes
1761def hideValuesInPropMap(propmap, propnames):
1762 """
1763 Worker function: input simple map of prop name/value pairs, and
1764 a list of specific propnames whose values we want to hide.
1765 Loop through the "hide" list, and if any are found, hide the
1766 value and return the altered map.
1767 If none found, return the original map
1768 """
1769 matches = []
1770 for propname in propnames:
1771 if propname in propmap: 1771 ↛ 1772line 1771 didn't jump to line 1772, because the condition on line 1771 was never true
1772 matches.append(propname)
1774 if matches: 1774 ↛ 1775line 1774 didn't jump to line 1775, because the condition on line 1774 was never true
1775 deepCopyRec = copy.deepcopy(propmap)
1776 for match in matches:
1777 deepCopyRec[match] = '******'
1778 return deepCopyRec
1780 return propmap
1781# define the list of propnames whose value we want to hide
1783PASSWD_PROP_KEYS = ['password', 'cifspassword', 'chappassword', 'incoming_chappassword']
1784DEFAULT_SEGMENT_LEN = 950
1787def hidePasswdInConfig(config):
1788 """
1789 Function to hide passwd values in a simple prop map,
1790 for example "device_config"
1791 """
1792 return hideValuesInPropMap(config, PASSWD_PROP_KEYS)
1795def hidePasswdInParams(params, configProp):
1796 """
1797 Function to hide password values in a specified property which
1798 is a simple map of prop name/values, and is itself an prop entry
1799 in a larger property map.
1800 For example, param maps containing "device_config", or
1801 "sm_config", etc
1802 """
1803 params[configProp] = hideValuesInPropMap(params[configProp], PASSWD_PROP_KEYS)
1804 return params
1807def hideMemberValuesInXmlParams(xmlParams, propnames=PASSWD_PROP_KEYS):
1808 """
1809 Function to hide password values in XML params, specifically
1810 for the XML format of incoming params to SR modules.
1811 Uses text parsing: loop through the list of specific propnames
1812 whose values we want to hide, and:
1813 - Assemble a full "prefix" containing each property name, e.g.,
1814 "<member><name>password</name><value>"
1815 - Test the XML if it contains that string, save the index.
1816 - If found, get the index of the ending tag
1817 - Truncate the return string starting with the password value.
1818 - Append the substitute "*******" value string.
1819 - Restore the rest of the original string starting with the end tag.
1820 """
1821 findStrPrefixHead = "<member><name>"
1822 findStrPrefixTail = "</name><value>"
1823 findStrSuffix = "</value>"
1824 strlen = len(xmlParams)
1826 for propname in propnames:
1827 findStrPrefix = findStrPrefixHead + propname + findStrPrefixTail
1828 idx = xmlParams.find(findStrPrefix)
1829 if idx != -1: # if found any of them
1830 idx += len(findStrPrefix)
1831 idx2 = xmlParams.find(findStrSuffix, idx)
1832 if idx2 != -1:
1833 retStr = xmlParams[0:idx]
1834 retStr += "******"
1835 retStr += xmlParams[idx2:strlen]
1836 return retStr
1837 else:
1838 return xmlParams
1839 return xmlParams
1842def splitXmlText(xmlData, segmentLen=DEFAULT_SEGMENT_LEN, showContd=False):
1843 """
1844 Split xml string data into substrings small enough for the
1845 syslog line length limit. Split at tag end markers ( ">" ).
1846 Usage:
1847 strList = []
1848 strList = splitXmlText( longXmlText, maxLineLen ) # maxLineLen is optional
1849 """
1850 remainingData = str(xmlData)
1852 # "Un-pretty-print"
1853 remainingData = remainingData.replace('\n', '')
1854 remainingData = remainingData.replace('\t', '')
1856 remainingChars = len(remainingData)
1857 returnData = ''
1859 thisLineNum = 0
1860 while remainingChars > segmentLen:
1861 thisLineNum = thisLineNum + 1
1862 index = segmentLen
1863 tmpStr = remainingData[:segmentLen]
1864 tmpIndex = tmpStr.rfind('>')
1865 if tmpIndex != -1:
1866 index = tmpIndex + 1
1868 tmpStr = tmpStr[:index]
1869 remainingData = remainingData[index:]
1870 remainingChars = len(remainingData)
1872 if showContd:
1873 if thisLineNum != 1:
1874 tmpStr = '(Cont\'d): ' + tmpStr
1875 tmpStr = tmpStr + ' (Cont\'d):'
1877 returnData += tmpStr + '\n'
1879 if showContd and thisLineNum > 0:
1880 remainingData = '(Cont\'d): ' + remainingData
1881 returnData += remainingData
1883 return returnData
1886def inject_failure():
1887 raise Exception('injected failure')
1890def open_atomic(path, mode=None):
1891 """Atomically creates a file if, and only if it does not already exist.
1892 Leaves the file open and returns the file object.
1894 path: the path to atomically open
1895 mode: "r" (read), "w" (write), or "rw" (read/write)
1896 returns: an open file object"""
1898 assert path
1900 flags = os.O_CREAT | os.O_EXCL
1901 modes = {'r': os.O_RDONLY, 'w': os.O_WRONLY, 'rw': os.O_RDWR}
1902 if mode:
1903 if mode not in modes:
1904 raise Exception('invalid access mode ' + mode)
1905 flags |= modes[mode]
1906 fd = os.open(path, flags)
1907 try:
1908 if mode:
1909 return os.fdopen(fd, mode)
1910 else:
1911 return os.fdopen(fd)
1912 except:
1913 os.close(fd)
1914 raise
1917def isInvalidVDI(exception):
1918 return exception.details[0] == "HANDLE_INVALID" or \
1919 exception.details[0] == "UUID_INVALID"
1922def get_pool_restrictions(session):
1923 """Returns pool restrictions as a map, @session must be already
1924 established."""
1925 return list(session.xenapi.pool.get_all_records().values())[0]['restrictions']
1928def read_caching_is_restricted(session):
1929 """Tells whether read caching is restricted."""
1930 if session is None: 1930 ↛ 1931line 1930 didn't jump to line 1931, because the condition on line 1930 was never true
1931 return True
1932 restrictions = get_pool_restrictions(session)
1933 if 'restrict_read_caching' in restrictions and \ 1933 ↛ 1935line 1933 didn't jump to line 1935, because the condition on line 1933 was never true
1934 restrictions['restrict_read_caching'] == "true":
1935 return True
1936 return False
1939def sessions_less_than_targets(other_config, device_config):
1940 if 'multihomelist' in device_config and 'iscsi_sessions' in other_config:
1941 sessions = int(other_config['iscsi_sessions'])
1942 targets = len(device_config['multihomelist'].split(','))
1943 SMlog("Targets %d and iscsi_sessions %d" % (targets, sessions))
1944 return (sessions < targets)
1945 else:
1946 return False
1949def enable_and_start_service(name, start):
1950 attempt = 0
1951 while True:
1952 attempt += 1
1953 fn = 'enable' if start else 'disable'
1954 args = ('systemctl', fn, '--now', name)
1955 (ret, out, err) = doexec(args)
1956 if ret == 0:
1957 return
1958 elif attempt >= 3:
1959 raise Exception(
1960 'Failed to {} {}: {} {}'.format(fn, name, out, err)
1961 )
1962 time.sleep(1)
1965def stop_service(name):
1966 args = ('systemctl', 'stop', name)
1967 (ret, out, err) = doexec(args)
1968 if ret == 0:
1969 return
1970 raise Exception('Failed to stop {}: {} {}'.format(name, out, err))
1973def restart_service(name):
1974 attempt = 0
1975 while True:
1976 attempt += 1
1977 SMlog('Restarting service {} {}...'.format(name, attempt))
1978 args = ('systemctl', 'restart', name)
1979 (ret, out, err) = doexec(args)
1980 if ret == 0:
1981 return
1982 elif attempt >= 3:
1983 SMlog('Restart service FAILED {} {}'.format(name, attempt))
1984 raise Exception(
1985 'Failed to restart {}: {} {}'.format(name, out, err)
1986 )
1987 time.sleep(1)
1990def check_pid_exists(pid):
1991 try:
1992 os.kill(pid, 0)
1993 except OSError:
1994 return False
1995 else:
1996 return True
1999def make_profile(name, function):
2000 """
2001 Helper to execute cProfile using unique log file.
2002 """
2004 import cProfile
2005 import itertools
2006 import os.path
2007 import time
2009 assert name
2010 assert function
2012 FOLDER = '/tmp/sm-perfs/'
2013 makedirs(FOLDER)
2015 filename = time.strftime('{}_%Y%m%d_%H%M%S.prof'.format(name))
2017 def gen_path(path):
2018 yield path
2019 root, ext = os.path.splitext(path)
2020 for i in itertools.count(start=1, step=1):
2021 yield root + '.{}.'.format(i) + ext
2023 for profile_path in gen_path(FOLDER + filename):
2024 try:
2025 file = open_atomic(profile_path, 'w')
2026 file.close()
2027 break
2028 except OSError as e:
2029 if e.errno == errno.EEXIST:
2030 pass
2031 else:
2032 raise
2034 try:
2035 SMlog('* Start profiling of {} ({}) *'.format(name, filename))
2036 cProfile.runctx('function()', None, locals(), profile_path)
2037 finally:
2038 SMlog('* End profiling of {} ({}) *'.format(name, filename))