Coverage for drivers/util.py : 41%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Copyright (C) Citrix Systems Inc.
2#
3# This program is free software; you can redistribute it and/or modify
4# it under the terms of the GNU Lesser General Public License as published
5# by the Free Software Foundation; version 2.1 only.
6#
7# This program is distributed in the hope that it will be useful,
8# but WITHOUT ANY WARRANTY; without even the implied warranty of
9# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10# GNU Lesser General Public License for more details.
11#
12# You should have received a copy of the GNU Lesser General Public License
13# along with this program; if not, write to the Free Software Foundation, Inc.,
14# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
15#
16# Miscellaneous utility functions
17#
19import os
20import re
21import sys
22import subprocess
23import shutil
24import tempfile
25import signal
26import time
27import datetime
28import errno
29import socket
30import xml.dom.minidom
31import scsiutil
32import stat
33import xs_errors
34import XenAPI # pylint: disable=import-error
35import xmlrpc.client
36import base64
37import syslog
38import resource
39import traceback
40import glob
41import copy
42import tempfile
44from functools import reduce
46NO_LOGGING_STAMPFILE = '/etc/xensource/no_sm_log'
48IORETRY_MAX = 20 # retries
49IORETRY_PERIOD = 1.0 # seconds
51LOGGING = not (os.path.exists(NO_LOGGING_STAMPFILE))
52_SM_SYSLOG_FACILITY = syslog.LOG_LOCAL2
53LOG_EMERG = syslog.LOG_EMERG
54LOG_ALERT = syslog.LOG_ALERT
55LOG_CRIT = syslog.LOG_CRIT
56LOG_ERR = syslog.LOG_ERR
57LOG_WARNING = syslog.LOG_WARNING
58LOG_NOTICE = syslog.LOG_NOTICE
59LOG_INFO = syslog.LOG_INFO
60LOG_DEBUG = syslog.LOG_DEBUG
62ISCSI_REFDIR = '/var/run/sr-ref'
64CMD_DD = "/bin/dd"
66FIST_PAUSE_PERIOD = 30 # seconds
69class SMException(Exception):
70 """Base class for all SM exceptions for easier catching & wrapping in
71 XenError"""
74class CommandException(SMException):
75 def error_message(self, code):
76 if code > 0:
77 return os.strerror(code)
78 elif code < 0:
79 return "Signalled %s" % (abs(code))
80 return "Success"
82 def __init__(self, code, cmd="", reason='exec failed'):
83 self.code = code
84 self.cmd = cmd
85 self.reason = reason
86 Exception.__init__(self, self.error_message(code))
89class SRBusyException(SMException):
90 """The SR could not be locked"""
91 pass
94def logException(tag):
95 info = sys.exc_info()
96 if info[0] == SystemExit: 96 ↛ 98line 96 didn't jump to line 98, because the condition on line 96 was never true
97 # this should not be happening when catching "Exception", but it is
98 sys.exit(0)
99 tb = reduce(lambda a, b: "%s%s" % (a, b), traceback.format_tb(info[2]))
100 str = "***** %s: EXCEPTION %s, %s\n%s" % (tag, info[0], info[1], tb)
101 SMlog(str)
104def roundup(divisor, value):
105 """Retruns the rounded up value so it is divisible by divisor."""
107 if value == 0: 107 ↛ 108line 107 didn't jump to line 108, because the condition on line 107 was never true
108 value = 1
109 if value % divisor != 0:
110 return ((int(value) // divisor) + 1) * divisor
111 return value
114def to_plain_string(obj):
115 if obj is None:
116 return None
117 if type(obj) == str:
118 return obj
119 return str(obj)
122def shellquote(arg):
123 return '"%s"' % arg.replace('"', '\\"')
126def make_WWN(name):
127 hex_prefix = name.find("0x")
128 if (hex_prefix >= 0): 128 ↛ 131line 128 didn't jump to line 131, because the condition on line 128 was never false
129 name = name[name.find("0x") + 2:len(name)]
130 # inject dashes for each nibble
131 if (len(name) == 16): # sanity check 131 ↛ 135line 131 didn't jump to line 135, because the condition on line 131 was never false
132 name = name[0:2] + "-" + name[2:4] + "-" + name[4:6] + "-" + \
133 name[6:8] + "-" + name[8:10] + "-" + name[10:12] + "-" + \
134 name[12:14] + "-" + name[14:16]
135 return name
138def _logToSyslog(ident, facility, priority, message):
139 syslog.openlog(ident, 0, facility)
140 syslog.syslog(priority, "[%d] %s" % (os.getpid(), message))
141 syslog.closelog()
144def SMlog(message, ident="SM", priority=LOG_INFO):
145 if LOGGING: 145 ↛ exitline 145 didn't return from function 'SMlog', because the condition on line 145 was never false
146 for message_line in str(message).split('\n'):
147 _logToSyslog(ident, _SM_SYSLOG_FACILITY, priority, message_line)
150def _getDateString():
151 d = datetime.datetime.now()
152 t = d.timetuple()
153 return "%s-%s-%s:%s:%s:%s" % \
154 (t[0], t[1], t[2], t[3], t[4], t[5])
157def doexec(args, inputtext=None, new_env=None, text=True):
158 """Execute a subprocess, then return its return code, stdout and stderr"""
159 env = None
160 if new_env:
161 env = dict(os.environ)
162 env.update(new_env)
163 proc = subprocess.Popen(args, stdin=subprocess.PIPE,
164 stdout=subprocess.PIPE,
165 stderr=subprocess.PIPE,
166 close_fds=True, env=env,
167 universal_newlines=text)
169 if not text and inputtext is not None: 169 ↛ 170line 169 didn't jump to line 170, because the condition on line 169 was never true
170 inputtext = inputtext.encode()
172 (stdout, stderr) = proc.communicate(inputtext)
174 rc = proc.returncode
175 return rc, stdout, stderr
178def is_string(value):
179 return isinstance(value, str)
182# These are partially tested functions that replicate the behaviour of
183# the original pread,pread2 and pread3 functions. Potentially these can
184# replace the original ones at some later date.
185#
186# cmdlist is a list of either single strings or pairs of strings. For
187# each pair, the first component is passed to exec while the second is
188# written to the logs.
189def pread(cmdlist, close_stdin=False, scramble=None, expect_rc=0,
190 quiet=False, new_env=None, text=True):
191 cmdlist_for_exec = []
192 cmdlist_for_log = []
193 for item in cmdlist:
194 if is_string(item): 194 ↛ 204line 194 didn't jump to line 204, because the condition on line 194 was never false
195 cmdlist_for_exec.append(item)
196 if scramble: 196 ↛ 197line 196 didn't jump to line 197, because the condition on line 196 was never true
197 if item.find(scramble) != -1:
198 cmdlist_for_log.append("<filtered out>")
199 else:
200 cmdlist_for_log.append(item)
201 else:
202 cmdlist_for_log.append(item)
203 else:
204 cmdlist_for_exec.append(item[0])
205 cmdlist_for_log.append(item[1])
207 if not quiet: 207 ↛ 209line 207 didn't jump to line 209, because the condition on line 207 was never false
208 SMlog(cmdlist_for_log)
209 (rc, stdout, stderr) = doexec(cmdlist_for_exec, new_env=new_env, text=text)
210 if rc != expect_rc:
211 SMlog("FAILED in util.pread: (rc %d) stdout: '%s', stderr: '%s'" % \
212 (rc, stdout, stderr))
213 if quiet: 213 ↛ 214line 213 didn't jump to line 214, because the condition on line 213 was never true
214 SMlog("Command was: %s" % cmdlist_for_log)
215 if '' == stderr: 215 ↛ 216line 215 didn't jump to line 216, because the condition on line 215 was never true
216 stderr = stdout
217 raise CommandException(rc, str(cmdlist), stderr.strip())
218 if not quiet: 218 ↛ 220line 218 didn't jump to line 220, because the condition on line 218 was never false
219 SMlog(" pread SUCCESS")
220 return stdout
223# POSIX guaranteed atomic within the same file system.
224# Supply directory to ensure tempfile is created
225# in the same directory.
226def atomicFileWrite(targetFile, directory, text):
228 file = None
229 try:
230 # Create file only current pid can write/read to
231 # our responsibility to clean it up.
232 _, tempPath = tempfile.mkstemp(dir=directory)
233 file = open(tempPath, 'w')
234 file.write(text)
236 # Ensure flushed to disk.
237 file.flush()
238 os.fsync(file.fileno())
239 file.close()
241 os.rename(tempPath, targetFile)
242 except OSError:
243 SMlog("FAILED to atomic write to %s" % (targetFile))
245 finally:
246 if (file is not None) and (not file.closed):
247 file.close()
249 if os.path.isfile(tempPath):
250 os.remove(tempPath)
253#Read STDOUT from cmdlist and discard STDERR output
254def pread2(cmdlist, quiet=False, text=True):
255 return pread(cmdlist, quiet=quiet, text=text)
258#Read STDOUT from cmdlist, feeding 'text' to STDIN
259def pread3(cmdlist, text):
260 SMlog(cmdlist)
261 (rc, stdout, stderr) = doexec(cmdlist, text)
262 if rc:
263 SMlog("FAILED in util.pread3: (errno %d) stdout: '%s', stderr: '%s'" % \
264 (rc, stdout, stderr))
265 if '' == stderr:
266 stderr = stdout
267 raise CommandException(rc, str(cmdlist), stderr.strip())
268 SMlog(" pread3 SUCCESS")
269 return stdout
272def listdir(path, quiet=False):
273 cmd = ["ls", path, "-1", "--color=never"]
274 try:
275 text = pread2(cmd, quiet=quiet)[:-1]
276 if len(text) == 0:
277 return []
278 return text.split('\n')
279 except CommandException as inst:
280 if inst.code == errno.ENOENT:
281 raise CommandException(errno.EIO, inst.cmd, inst.reason)
282 else:
283 raise CommandException(inst.code, inst.cmd, inst.reason)
286def gen_uuid():
287 cmd = ["uuidgen", "-r"]
288 return pread(cmd)[:-1]
291def match_uuid(s):
292 regex = re.compile("^[0-9a-f]{8}-(([0-9a-f]{4})-){3}[0-9a-f]{12}")
293 return regex.search(s, 0)
296def findall_uuid(s):
297 regex = re.compile("[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}")
298 return regex.findall(s, 0)
301def exactmatch_uuid(s):
302 regex = re.compile("^[0-9a-f]{8}-(([0-9a-f]{4})-){3}[0-9a-f]{12}$")
303 return regex.search(s, 0)
306def start_log_entry(srpath, path, args):
307 logstring = str(datetime.datetime.now())
308 logstring += " log: "
309 logstring += srpath
310 logstring += " " + path
311 for element in args:
312 logstring += " " + element
313 try:
314 file = open(srpath + "/filelog.txt", "a")
315 file.write(logstring)
316 file.write("\n")
317 file.close()
318 except:
319 pass
321 # failed to write log ...
323def end_log_entry(srpath, path, args):
324 # for teminating, use "error" or "done"
325 logstring = str(datetime.datetime.now())
326 logstring += " end: "
327 logstring += srpath
328 logstring += " " + path
329 for element in args:
330 logstring += " " + element
331 try:
332 file = open(srpath + "/filelog.txt", "a")
333 file.write(logstring)
334 file.write("\n")
335 file.close()
336 except:
337 pass
339 # failed to write log ...
340 # for now print
341 # print "%s" % logstring
343def ioretry(f, errlist=[errno.EIO], maxretry=IORETRY_MAX, period=IORETRY_PERIOD, **ignored):
344 retries = 0
345 while True:
346 try:
347 return f()
348 except OSError as ose:
349 err = int(ose.errno)
350 if not err in errlist:
351 raise CommandException(err, str(f), "OSError")
352 except CommandException as ce:
353 if not int(ce.code) in errlist:
354 raise
356 retries += 1
357 if retries >= maxretry:
358 break
360 time.sleep(period)
362 raise CommandException(errno.ETIMEDOUT, str(f), "Timeout")
365def ioretry_stat(path, maxretry=IORETRY_MAX):
366 # this ioretry is similar to the previous method, but
367 # stat does not raise an error -- so check its return
368 retries = 0
369 while retries < maxretry:
370 stat = os.statvfs(path)
371 if stat.f_blocks != -1:
372 return stat
373 time.sleep(1)
374 retries += 1
375 raise CommandException(errno.EIO, "os.statvfs")
378def sr_get_capability(sr_uuid):
379 result = []
380 session = get_localAPI_session()
381 sr_ref = session.xenapi.SR.get_by_uuid(sr_uuid)
382 sm_type = session.xenapi.SR.get_record(sr_ref)['type']
383 sm_rec = session.xenapi.SM.get_all_records_where(
384 "field \"type\" = \"%s\"" % sm_type)
386 # SM expects at least one entry of any SR type
387 if len(sm_rec) > 0:
388 result = list(sm_rec.values())[0]['capabilities']
390 session.xenapi.logout()
391 return result
394def sr_get_driver_info(driver_info):
395 results = {}
396 # first add in the vanilla stuff
397 for key in ['name', 'description', 'vendor', 'copyright', \
398 'driver_version', 'required_api_version']:
399 results[key] = driver_info[key]
400 # add the capabilities (xmlrpc array)
401 # enforcing activate/deactivate for blktap2
402 caps = driver_info['capabilities']
403 if "ATOMIC_PAUSE" in caps: 403 ↛ 404line 403 didn't jump to line 404, because the condition on line 403 was never true
404 for cap in ("VDI_ACTIVATE", "VDI_DEACTIVATE"):
405 if not cap in caps:
406 caps.append(cap)
407 elif "VDI_ACTIVATE" in caps or "VDI_DEACTIVATE" in caps: 407 ↛ 408line 407 didn't jump to line 408, because the condition on line 407 was never true
408 SMlog("Warning: vdi_[de]activate present for %s" % driver_info["name"])
410 results['capabilities'] = caps
411 # add in the configuration options
412 options = []
413 for option in driver_info['configuration']:
414 options.append({'key': option[0], 'description': option[1]})
415 results['configuration'] = options
416 return xmlrpc.client.dumps((results, ), "", True)
419def return_nil():
420 return xmlrpc.client.dumps((None, ), "", True, allow_none=True)
423def SRtoXML(SRlist):
424 dom = xml.dom.minidom.Document()
425 driver = dom.createElement("SRlist")
426 dom.appendChild(driver)
428 for key in SRlist.keys():
429 dict = SRlist[key]
430 entry = dom.createElement("SR")
431 driver.appendChild(entry)
433 e = dom.createElement("UUID")
434 entry.appendChild(e)
435 textnode = dom.createTextNode(key)
436 e.appendChild(textnode)
438 if 'size' in dict:
439 e = dom.createElement("Size")
440 entry.appendChild(e)
441 textnode = dom.createTextNode(str(dict['size']))
442 e.appendChild(textnode)
444 if 'storagepool' in dict:
445 e = dom.createElement("StoragePool")
446 entry.appendChild(e)
447 textnode = dom.createTextNode(str(dict['storagepool']))
448 e.appendChild(textnode)
450 if 'aggregate' in dict:
451 e = dom.createElement("Aggregate")
452 entry.appendChild(e)
453 textnode = dom.createTextNode(str(dict['aggregate']))
454 e.appendChild(textnode)
456 return dom.toprettyxml()
459def pathexists(path):
460 try:
461 os.lstat(path)
462 return True
463 except OSError as inst:
464 if inst.errno == errno.EIO: 464 ↛ 465line 464 didn't jump to line 465, because the condition on line 464 was never true
465 time.sleep(1)
466 try:
467 listdir(os.path.realpath(os.path.dirname(path)))
468 os.lstat(path)
469 return True
470 except:
471 pass
472 raise CommandException(errno.EIO, "os.lstat(%s)" % path, "failed")
473 return False
476def force_unlink(path):
477 try:
478 os.unlink(path)
479 except OSError as e:
480 if e.errno != errno.ENOENT:
481 raise
484def create_secret(session, secret):
485 ref = session.xenapi.secret.create({'value': secret})
486 return session.xenapi.secret.get_uuid(ref)
489def get_secret(session, uuid):
490 try:
491 ref = session.xenapi.secret.get_by_uuid(uuid)
492 return session.xenapi.secret.get_value(ref)
493 except:
494 raise xs_errors.XenError('InvalidSecret', opterr='Unable to look up secret [%s]' % uuid)
497def get_real_path(path):
498 "Follow symlinks to the actual file"
499 absPath = path
500 directory = ''
501 while os.path.islink(absPath):
502 directory = os.path.dirname(absPath)
503 absPath = os.readlink(absPath)
504 absPath = os.path.join(directory, absPath)
505 return absPath
508def wait_for_path(path, timeout):
509 for i in range(0, timeout): 509 ↛ 513line 509 didn't jump to line 513, because the loop on line 509 didn't complete
510 if len(glob.glob(path)): 510 ↛ 512line 510 didn't jump to line 512, because the condition on line 510 was never false
511 return True
512 time.sleep(1)
513 return False
516def wait_for_nopath(path, timeout):
517 for i in range(0, timeout):
518 if not os.path.exists(path):
519 return True
520 time.sleep(1)
521 return False
524def wait_for_path_multi(path, timeout):
525 for i in range(0, timeout):
526 paths = glob.glob(path)
527 SMlog("_wait_for_paths_multi: paths = %s" % paths)
528 if len(paths):
529 SMlog("_wait_for_paths_multi: return first path: %s" % paths[0])
530 return paths[0]
531 time.sleep(1)
532 return ""
535def isdir(path):
536 try:
537 st = os.stat(path)
538 return stat.S_ISDIR(st.st_mode)
539 except OSError as inst:
540 if inst.errno == errno.EIO: 540 ↛ 541line 540 didn't jump to line 541, because the condition on line 540 was never true
541 raise CommandException(errno.EIO, "os.stat(%s)" % path, "failed")
542 return False
545def get_single_entry(path):
546 f = open(path, 'r')
547 line = f.readline()
548 f.close()
549 return line.rstrip()
552def get_fs_size(path):
553 st = ioretry_stat(path)
554 return st.f_blocks * st.f_frsize
557def get_fs_utilisation(path):
558 st = ioretry_stat(path)
559 return (st.f_blocks - st.f_bfree) * \
560 st.f_frsize
563def ismount(path):
564 """Test whether a path is a mount point"""
565 try:
566 s1 = os.stat(path)
567 s2 = os.stat(os.path.join(path, '..'))
568 except OSError as inst:
569 raise CommandException(inst.errno, "os.stat")
570 dev1 = s1.st_dev
571 dev2 = s2.st_dev
572 if dev1 != dev2:
573 return True # path/.. on a different device as path
574 ino1 = s1.st_ino
575 ino2 = s2.st_ino
576 if ino1 == ino2:
577 return True # path/.. is the same i-node as path
578 return False
581def makedirs(name, mode=0o777):
582 head, tail = os.path.split(name)
583 if not tail: 583 ↛ 584line 583 didn't jump to line 584, because the condition on line 583 was never true
584 head, tail = os.path.split(head)
585 if head and tail and not pathexists(head):
586 makedirs(head, mode)
587 if tail == os.curdir:
588 return
589 try:
590 os.mkdir(name, mode)
591 except OSError as exc:
592 if exc.errno == errno.EEXIST and os.path.isdir(name): 592 ↛ 593line 592 didn't jump to line 593, because the condition on line 592 was never true
593 if mode:
594 os.chmod(name, mode)
595 pass
596 else:
597 raise
600def zeroOut(path, fromByte, bytes):
601 """write 'bytes' zeros to 'path' starting from fromByte (inclusive)"""
602 blockSize = 4096
604 fromBlock = fromByte // blockSize
605 if fromByte % blockSize:
606 fromBlock += 1
607 bytesBefore = fromBlock * blockSize - fromByte
608 if bytesBefore > bytes:
609 bytesBefore = bytes
610 bytes -= bytesBefore
611 cmd = [CMD_DD, "if=/dev/zero", "of=%s" % path, "bs=1",
612 "seek=%s" % fromByte, "count=%s" % bytesBefore]
613 try:
614 pread2(cmd)
615 except CommandException:
616 return False
618 blocks = bytes // blockSize
619 bytes -= blocks * blockSize
620 fromByte = (fromBlock + blocks) * blockSize
621 if blocks:
622 cmd = [CMD_DD, "if=/dev/zero", "of=%s" % path, "bs=%s" % blockSize,
623 "seek=%s" % fromBlock, "count=%s" % blocks]
624 try:
625 pread2(cmd)
626 except CommandException:
627 return False
629 if bytes:
630 cmd = [CMD_DD, "if=/dev/zero", "of=%s" % path, "bs=1",
631 "seek=%s" % fromByte, "count=%s" % bytes]
632 try:
633 pread2(cmd)
634 except CommandException:
635 return False
637 return True
640def match_rootdev(s):
641 regex = re.compile("^PRIMARY_DISK")
642 return regex.search(s, 0)
645def getrootdev():
646 filename = '/etc/xensource-inventory'
647 try:
648 f = open(filename, 'r')
649 except:
650 raise xs_errors.XenError('EIO', \
651 opterr="Unable to open inventory file [%s]" % filename)
652 rootdev = ''
653 for line in filter(match_rootdev, f.readlines()):
654 rootdev = line.split("'")[1]
655 if not rootdev: 655 ↛ 656line 655 didn't jump to line 656, because the condition on line 655 was never true
656 raise xs_errors.XenError('NoRootDev')
657 return rootdev
660def getrootdevID():
661 rootdev = getrootdev()
662 try:
663 rootdevID = scsiutil.getSCSIid(rootdev)
664 except:
665 SMlog("util.getrootdevID: Unable to verify serial or SCSIid of device: %s" \
666 % rootdev)
667 return ''
669 if not len(rootdevID):
670 SMlog("util.getrootdevID: Unable to identify scsi device [%s] via scsiID" \
671 % rootdev)
673 return rootdevID
676def get_localAPI_session():
677 # First acquire a valid session
678 session = XenAPI.xapi_local()
679 try:
680 session.xenapi.login_with_password('root', '', '', 'SM')
681 except:
682 raise xs_errors.XenError('APISession')
683 return session
686def get_this_host():
687 uuid = None
688 f = open("/etc/xensource-inventory", 'r')
689 for line in f.readlines():
690 if line.startswith("INSTALLATION_UUID"):
691 uuid = line.split("'")[1]
692 f.close()
693 return uuid
696def get_master_ref(session):
697 pools = session.xenapi.pool.get_all()
698 return session.xenapi.pool.get_master(pools[0])
701def is_master(session):
702 return get_this_host_ref(session) == get_master_ref(session)
705def get_localhost_ref(session):
706 filename = '/etc/xensource-inventory'
707 try:
708 f = open(filename, 'r')
709 except:
710 raise xs_errors.XenError('EIO', \
711 opterr="Unable to open inventory file [%s]" % filename)
712 domid = ''
713 for line in filter(match_domain_id, f.readlines()):
714 domid = line.split("'")[1]
715 if not domid:
716 raise xs_errors.XenError('APILocalhost')
718 vms = session.xenapi.VM.get_all_records_where('field "uuid" = "%s"' % domid)
719 for vm in vms:
720 record = vms[vm]
721 if record["uuid"] == domid:
722 hostid = record["resident_on"]
723 return hostid
724 raise xs_errors.XenError('APILocalhost')
727def match_domain_id(s):
728 regex = re.compile("^CONTROL_DOMAIN_UUID")
729 return regex.search(s, 0)
732def get_hosts_attached_on(session, vdi_uuids):
733 host_refs = {}
734 for vdi_uuid in vdi_uuids:
735 try:
736 vdi_ref = session.xenapi.VDI.get_by_uuid(vdi_uuid)
737 except XenAPI.Failure:
738 SMlog("VDI %s not in db, ignoring" % vdi_uuid)
739 continue
740 sm_config = session.xenapi.VDI.get_sm_config(vdi_ref)
741 for key in [x for x in sm_config.keys() if x.startswith('host_')]:
742 host_refs[key[len('host_'):]] = True
743 return host_refs.keys()
745def get_this_host_address(session):
746 host_uuid = get_this_host()
747 host_ref = session.xenapi.host.get_by_uuid(host_uuid)
748 return session.xenapi.host.get_record(host_ref)['address']
750def get_host_addresses(session):
751 addresses = []
752 hosts = session.xenapi.host.get_all_records()
753 for record in hosts.values():
754 addresses.append(record['address'])
755 return addresses
757def get_this_host_ref(session):
758 host_uuid = get_this_host()
759 host_ref = session.xenapi.host.get_by_uuid(host_uuid)
760 return host_ref
763def get_slaves_attached_on(session, vdi_uuids):
764 "assume this host is the SR master"
765 host_refs = get_hosts_attached_on(session, vdi_uuids)
766 master_ref = get_this_host_ref(session)
767 return [x for x in host_refs if x != master_ref]
770def get_online_hosts(session):
771 online_hosts = []
772 hosts = session.xenapi.host.get_all_records()
773 for host_ref, host_rec in hosts.items():
774 metricsRef = host_rec["metrics"]
775 metrics = session.xenapi.host_metrics.get_record(metricsRef)
776 if metrics["live"]:
777 online_hosts.append(host_ref)
778 return online_hosts
781def get_all_slaves(session):
782 "assume this host is the SR master"
783 host_refs = get_online_hosts(session)
784 master_ref = get_this_host_ref(session)
785 return [x for x in host_refs if x != master_ref]
788def is_attached_rw(sm_config):
789 for key, val in sm_config.items():
790 if key.startswith("host_") and val == "RW":
791 return True
792 return False
795def attached_as(sm_config):
796 for key, val in sm_config.items():
797 if key.startswith("host_") and (val == "RW" or val == "RO"): 797 ↛ 798line 797 didn't jump to line 798, because the condition on line 797 was never true
798 return val
801def find_my_pbd_record(session, host_ref, sr_ref):
802 try:
803 pbds = session.xenapi.PBD.get_all_records()
804 for pbd_ref in pbds.keys():
805 if pbds[pbd_ref]['host'] == host_ref and pbds[pbd_ref]['SR'] == sr_ref:
806 return [pbd_ref, pbds[pbd_ref]]
807 return None
808 except Exception as e:
809 SMlog("Caught exception while looking up PBD for host %s SR %s: %s" % (str(host_ref), str(sr_ref), str(e)))
810 return None
813def find_my_pbd(session, host_ref, sr_ref):
814 ret = find_my_pbd_record(session, host_ref, sr_ref)
815 if ret is not None:
816 return ret[0]
817 else:
818 return None
821def test_hostPBD_devs(session, sr_uuid, devs):
822 host = get_localhost_ref(session)
823 sr = session.xenapi.SR.get_by_uuid(sr_uuid)
824 try:
825 pbds = session.xenapi.PBD.get_all_records()
826 except:
827 raise xs_errors.XenError('APIPBDQuery')
828 for dev in devs.split(','):
829 for pbd in pbds:
830 record = pbds[pbd]
831 # it's ok if it's *our* PBD
832 if record["SR"] == sr:
833 break
834 if record["host"] == host:
835 devconfig = record["device_config"]
836 if 'device' in devconfig:
837 for device in devconfig['device'].split(','):
838 if os.path.realpath(device) == os.path.realpath(dev):
839 return True
840 return False
843def test_hostPBD_lun(session, targetIQN, LUNid):
844 host = get_localhost_ref(session)
845 try:
846 pbds = session.xenapi.PBD.get_all_records()
847 except:
848 raise xs_errors.XenError('APIPBDQuery')
849 for pbd in pbds:
850 record = pbds[pbd]
851 if record["host"] == host:
852 devconfig = record["device_config"]
853 if 'targetIQN' in devconfig and 'LUNid' in devconfig:
854 if devconfig['targetIQN'] == targetIQN and \
855 devconfig['LUNid'] == LUNid:
856 return True
857 return False
860def test_SCSIid(session, sr_uuid, SCSIid):
861 if sr_uuid is not None:
862 sr = session.xenapi.SR.get_by_uuid(sr_uuid)
863 try:
864 pbds = session.xenapi.PBD.get_all_records()
865 except:
866 raise xs_errors.XenError('APIPBDQuery')
867 for pbd in pbds:
868 record = pbds[pbd]
869 # it's ok if it's *our* PBD
870 # During FC SR creation, devscan.py passes sr_uuid as None
871 if sr_uuid is not None:
872 if record["SR"] == sr:
873 break
874 devconfig = record["device_config"]
875 sm_config = session.xenapi.SR.get_sm_config(record["SR"])
876 if 'SCSIid' in devconfig and devconfig['SCSIid'] == SCSIid:
877 return True
878 elif 'SCSIid' in sm_config and sm_config['SCSIid'] == SCSIid:
879 return True
880 elif 'scsi-' + SCSIid in sm_config:
881 return True
882 return False
885class TimeoutException(SMException):
886 pass
889def timeout_call(timeoutseconds, function, *arguments):
890 def handler(signum, frame):
891 raise TimeoutException()
892 signal.signal(signal.SIGALRM, handler)
893 signal.alarm(timeoutseconds)
894 try:
895 return function(*arguments)
896 finally:
897 signal.alarm(0)
900def _incr_iscsiSR_refcount(targetIQN, uuid):
901 if not os.path.exists(ISCSI_REFDIR):
902 os.mkdir(ISCSI_REFDIR)
903 filename = os.path.join(ISCSI_REFDIR, targetIQN)
904 try:
905 f = open(filename, 'a+')
906 except:
907 raise xs_errors.XenError('LVMRefCount', \
908 opterr='file %s' % filename)
910 f.seek(0)
911 found = False
912 refcount = 0
913 for line in filter(match_uuid, f.readlines()):
914 refcount += 1
915 if line.find(uuid) != -1:
916 found = True
917 if not found:
918 f.write("%s\n" % uuid)
919 refcount += 1
920 f.close()
921 return refcount
924def _decr_iscsiSR_refcount(targetIQN, uuid):
925 filename = os.path.join(ISCSI_REFDIR, targetIQN)
926 if not os.path.exists(filename):
927 return 0
928 try:
929 f = open(filename, 'a+')
930 except:
931 raise xs_errors.XenError('LVMRefCount', \
932 opterr='file %s' % filename)
934 f.seek(0)
935 output = []
936 refcount = 0
937 for line in filter(match_uuid, f.readlines()):
938 if line.find(uuid) == -1:
939 output.append(line.rstrip())
940 refcount += 1
941 if not refcount:
942 os.unlink(filename)
943 return refcount
945 # Re-open file and truncate
946 f.close()
947 f = open(filename, 'w')
948 for i in range(0, refcount):
949 f.write("%s\n" % output[i])
950 f.close()
951 return refcount
954# The agent enforces 1 PBD per SR per host, so we
955# check for active SR entries not attached to this host
956def test_activePoolPBDs(session, host, uuid):
957 try:
958 pbds = session.xenapi.PBD.get_all_records()
959 except:
960 raise xs_errors.XenError('APIPBDQuery')
961 for pbd in pbds:
962 record = pbds[pbd]
963 if record["host"] != host and record["SR"] == uuid \
964 and record["currently_attached"]:
965 return True
966 return False
969def remove_mpathcount_field(session, host_ref, sr_ref, SCSIid):
970 try:
971 pbdref = find_my_pbd(session, host_ref, sr_ref)
972 if pbdref is not None:
973 key = "mpath-" + SCSIid
974 session.xenapi.PBD.remove_from_other_config(pbdref, key)
975 except:
976 pass
980def _testHost(hostname, port, errstring):
981 SMlog("_testHost: Testing host/port: %s,%d" % (hostname, port))
982 try:
983 sockinfo = socket.getaddrinfo(hostname, int(port))[0]
984 except:
985 logException('Exception occured getting IP for %s' % hostname)
986 raise xs_errors.XenError('DNSError')
988 timeout = 5
990 sock = socket.socket(sockinfo[0], socket.SOCK_STREAM)
991 # Only allow the connect to block for up to timeout seconds
992 sock.settimeout(timeout)
993 try:
994 sock.connect(sockinfo[4])
995 # Fix for MS storage server bug
996 sock.send(b'\n')
997 sock.close()
998 except socket.error as reason:
999 SMlog("_testHost: Connect failed after %d seconds (%s) - %s" \
1000 % (timeout, hostname, reason))
1001 raise xs_errors.XenError(errstring)
1004def match_scsiID(s, id):
1005 regex = re.compile(id)
1006 return regex.search(s, 0)
1009def _isSCSIid(s):
1010 regex = re.compile("^scsi-")
1011 return regex.search(s, 0)
1014def test_scsiserial(session, device):
1015 device = os.path.realpath(device)
1016 if not scsiutil._isSCSIdev(device):
1017 SMlog("util.test_scsiserial: Not a serial device: %s" % device)
1018 return False
1019 serial = ""
1020 try:
1021 serial += scsiutil.getserial(device)
1022 except:
1023 # Error allowed, SCSIid is the important one
1024 pass
1026 try:
1027 scsiID = scsiutil.getSCSIid(device)
1028 except:
1029 SMlog("util.test_scsiserial: Unable to verify serial or SCSIid of device: %s" \
1030 % device)
1031 return False
1032 if not len(scsiID):
1033 SMlog("util.test_scsiserial: Unable to identify scsi device [%s] via scsiID" \
1034 % device)
1035 return False
1037 try:
1038 SRs = session.xenapi.SR.get_all_records()
1039 except:
1040 raise xs_errors.XenError('APIFailure')
1041 for SR in SRs:
1042 record = SRs[SR]
1043 conf = record["sm_config"]
1044 if 'devserial' in conf:
1045 for dev in conf['devserial'].split(','):
1046 if _isSCSIid(dev):
1047 if match_scsiID(dev, scsiID):
1048 return True
1049 elif len(serial) and dev == serial:
1050 return True
1051 return False
1054def default(self, field, thunk):
1055 try:
1056 return getattr(self, field)
1057 except:
1058 return thunk()
1061def list_VDI_records_in_sr(sr):
1062 """Helper function which returns a list of all VDI records for this SR
1063 stored in the XenAPI server, useful for implementing SR.scan"""
1064 sr_ref = sr.session.xenapi.SR.get_by_uuid(sr.uuid)
1065 vdis = sr.session.xenapi.VDI.get_all_records_where("field \"SR\" = \"%s\"" % sr_ref)
1066 return vdis
1069# Given a partition (e.g. sda1), get a disk name:
1070def diskFromPartition(partition):
1071 # check whether this is a device mapper device (e.g. /dev/dm-0)
1072 m = re.match('(/dev/)?(dm-[0-9]+)(p[0-9]+)?$', partition)
1073 if m is not None: 1073 ↛ 1074line 1073 didn't jump to line 1074, because the condition on line 1073 was never true
1074 return m.group(2)
1076 numlen = 0 # number of digit characters
1077 m = re.match("\D+(\d+)", partition)
1078 if m is not None: 1078 ↛ 1079line 1078 didn't jump to line 1079, because the condition on line 1078 was never true
1079 numlen = len(m.group(1))
1081 # is it a cciss?
1082 if True in [partition.startswith(x) for x in ['cciss', 'ida', 'rd']]: 1082 ↛ 1083line 1082 didn't jump to line 1083, because the condition on line 1082 was never true
1083 numlen += 1 # need to get rid of trailing 'p'
1085 # is it a mapper path?
1086 if partition.startswith("mapper"): 1086 ↛ 1087line 1086 didn't jump to line 1087, because the condition on line 1086 was never true
1087 if re.search("p[0-9]*$", partition):
1088 numlen = len(re.match("\d+", partition[::-1]).group(0)) + 1
1089 SMlog("Found mapper part, len %d" % numlen)
1090 else:
1091 numlen = 0
1093 # is it /dev/disk/by-id/XYZ-part<k>?
1094 if partition.startswith("disk/by-id"): 1094 ↛ 1095line 1094 didn't jump to line 1095, because the condition on line 1094 was never true
1095 return partition[:partition.rfind("-part")]
1097 return partition[:len(partition) - numlen]
1100def dom0_disks():
1101 """Disks carrying dom0, e.g. ['/dev/sda']"""
1102 disks = []
1103 with open("/etc/mtab", 'r') as f:
1104 for line in f:
1105 (dev, mountpoint, fstype, opts, freq, passno) = line.split(' ')
1106 if mountpoint == '/':
1107 disk = diskFromPartition(dev)
1108 if not (disk in disks):
1109 disks.append(disk)
1110 SMlog("Dom0 disks: %s" % disks)
1111 return disks
1114def set_scheduler_sysfs_node(node, scheds):
1115 """
1116 Set the scheduler for a sysfs node (e.g. '/sys/block/sda')
1117 according to prioritized list schedulers
1118 Try to set the first item, then fall back to the next on failure
1119 """
1121 path = os.path.join(node, "queue", "scheduler")
1122 if not os.path.exists(path): 1122 ↛ 1125line 1122 didn't jump to line 1125, because the condition on line 1122 was never false
1123 SMlog("no path %s" % path)
1124 return
1125 for sched in scheds:
1126 try:
1127 with open(path, 'w') as file:
1128 file.write("%s\n" % sched)
1129 SMlog("Set scheduler to [%s] on [%s]" % (sched, node))
1130 return
1131 except (OSError, IOError) as err:
1132 SMlog("Setting scheduler to [%s] on [%s] failed with [%s]" % (sched, node, str(err)))
1133 SMlog("Error setting schedulers to [%s] on [%s]" % (scheds, node))
1136def set_scheduler(dev, schedulers=None):
1137 if schedulers is None: 1137 ↛ 1140line 1137 didn't jump to line 1140, because the condition on line 1137 was never false
1138 schedulers = ["none", "noop"]
1140 devices = []
1141 if not scsiutil.match_dm(dev): 1141 ↛ 1145line 1141 didn't jump to line 1145, because the condition on line 1141 was never false
1142 # Remove partition numbers
1143 devices.append(diskFromPartition(dev).replace('/', '!'))
1144 else:
1145 rawdev = diskFromPartition(dev)
1146 devices = [os.path.realpath(x)[5:] for x in scsiutil._genReverseSCSIidmap(rawdev.split('/')[-1])]
1148 for d in devices:
1149 set_scheduler_sysfs_node("/sys/block/%s" % d, schedulers)
1152# This function queries XAPI for the existing VDI records for this SR
1153def _getVDIs(srobj):
1154 VDIs = []
1155 try:
1156 sr_ref = getattr(srobj, 'sr_ref')
1157 except AttributeError:
1158 return VDIs
1160 refs = srobj.session.xenapi.SR.get_VDIs(sr_ref)
1161 for vdi in refs:
1162 ref = srobj.session.xenapi.VDI.get_record(vdi)
1163 ref['vdi_ref'] = vdi
1164 VDIs.append(ref)
1165 return VDIs
1168def _getVDI(srobj, vdi_uuid):
1169 vdi = srobj.session.xenapi.VDI.get_by_uuid(vdi_uuid)
1170 ref = srobj.session.xenapi.VDI.get_record(vdi)
1171 ref['vdi_ref'] = vdi
1172 return ref
1175def _convertDNS(name):
1176 addr = socket.getaddrinfo(name, None)[0][4][0]
1177 return addr
1180def _containsVDIinuse(srobj):
1181 VDIs = _getVDIs(srobj)
1182 for vdi in VDIs:
1183 if not vdi['managed']:
1184 continue
1185 sm_config = vdi['sm_config']
1186 if 'SRRef' in sm_config:
1187 try:
1188 PBDs = srobj.session.xenapi.SR.get_PBDs(sm_config['SRRef'])
1189 for pbd in PBDs:
1190 record = PBDs[pbd]
1191 if record["host"] == srobj.host_ref and \
1192 record["currently_attached"]:
1193 return True
1194 except:
1195 pass
1196 return False
1199def isVDICommand(cmd):
1200 if cmd is None or cmd in ["vdi_attach", "vdi_detach", 1200 ↛ 1203line 1200 didn't jump to line 1203, because the condition on line 1200 was never true
1201 "vdi_activate", "vdi_deactivate",
1202 "vdi_epoch_begin", "vdi_epoch_end"]:
1203 return True
1204 else:
1205 return False
1208#########################
1209# Daemon helper functions
1210def p_id_fork():
1211 try:
1212 p_id = os.fork()
1213 except OSError as e:
1214 print("Fork failed: %s (%d)" % (e.strerror, e.errno))
1215 sys.exit(-1)
1217 if (p_id == 0):
1218 os.setsid()
1219 try:
1220 p_id = os.fork()
1221 except OSError as e:
1222 print("Fork failed: %s (%d)" % (e.strerror, e.errno))
1223 sys.exit(-1)
1224 if (p_id == 0):
1225 os.chdir('/opt/xensource/sm')
1226 os.umask(0)
1227 else:
1228 os._exit(0)
1229 else:
1230 os._exit(0)
1233def daemon():
1234 p_id_fork()
1235 # Query the max file descriptor parameter for this process
1236 maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1238 # Close any fds that are open
1239 for fd in range(0, maxfd):
1240 try:
1241 os.close(fd)
1242 except:
1243 pass
1245 # Redirect STDIN to STDOUT and STDERR
1246 os.open('/dev/null', os.O_RDWR)
1247 os.dup2(0, 1)
1248 os.dup2(0, 2)
1249#########################
1251if __debug__:
1252 try:
1253 XE_IOFI_IORETRY
1254 except NameError:
1255 XE_IOFI_IORETRY = os.environ.get('XE_IOFI_IORETRY', None)
1256 if __name__ == 'util' and XE_IOFI_IORETRY is not None: 1256 ↛ 1257line 1256 didn't jump to line 1257, because the condition on line 1256 was never true
1257 __import__('iofi')
1259################################################################################
1260#
1261# Fist points
1262#
1264# * The global variable 'fistpoint' define the list of all possible fistpoints;
1265#
1266# * To activate a fistpoint called 'name', you need to create the file '/tmp/fist_name'
1267# on the SR master;
1268#
1269# * At the moment, activating a fist point can lead to two possible behaviors:
1270# - if '/tmp/fist_LVHDRT_exit' exists, then the function called during the fistpoint is _exit;
1271# - otherwise, the function called is _pause.
1273def _pause(secs, name):
1274 SMlog("Executing fist point %s: sleeping %d seconds ..." % (name, secs))
1275 time.sleep(secs)
1276 SMlog("Executing fist point %s: done" % name)
1279def _exit(name):
1280 SMlog("Executing fist point %s: exiting the current process ..." % name)
1281 raise xs_errors.XenError('FistPoint', opterr='%s' % name)
1284class FistPoint:
1285 def __init__(self, points):
1286 #SMlog("Fist points loaded")
1287 self.points = points
1289 def is_legal(self, name):
1290 return (name in self.points)
1292 def is_active(self, name):
1293 return os.path.exists("/tmp/fist_%s" % name)
1295 def mark_sr(self, name, sruuid, started):
1296 session = get_localAPI_session()
1297 sr = session.xenapi.SR.get_by_uuid(sruuid)
1298 if started:
1299 session.xenapi.SR.add_to_other_config(sr, name, "active")
1300 else:
1301 session.xenapi.SR.remove_from_other_config(sr, name)
1303 def activate(self, name, sruuid):
1304 if name in self.points: 1304 ↛ 1314line 1304 didn't jump to line 1314, because the condition on line 1304 was never false
1305 if self.is_active(name): 1305 ↛ 1306line 1305 didn't jump to line 1306, because the condition on line 1305 was never true
1306 self.mark_sr(name, sruuid, True)
1307 if self.is_active("LVHDRT_exit"):
1308 self.mark_sr(name, sruuid, False)
1309 _exit(name)
1310 else:
1311 _pause(FIST_PAUSE_PERIOD, name)
1312 self.mark_sr(name, sruuid, False)
1313 else:
1314 SMlog("Unknown fist point: %s" % name)
1316 def activate_custom_fn(self, name, fn):
1317 if name in self.points: 1317 ↛ 1323line 1317 didn't jump to line 1323, because the condition on line 1317 was never false
1318 if self.is_active(name): 1318 ↛ 1319line 1318 didn't jump to line 1319, because the condition on line 1318 was never true
1319 SMlog("Executing fist point %s: starting ..." % name)
1320 fn()
1321 SMlog("Executing fist point %s: done" % name)
1322 else:
1323 SMlog("Unknown fist point: %s" % name)
1326def list_find(f, seq):
1327 for item in seq:
1328 if f(item):
1329 return item
1331GCPAUSE_FISTPOINT = "GCLoop_no_pause"
1333fistpoint = FistPoint(["LVHDRT_finding_a_suitable_pair",
1334 "LVHDRT_inflating_the_parent",
1335 "LVHDRT_resizing_while_vdis_are_paused",
1336 "LVHDRT_coalescing_VHD_data",
1337 "LVHDRT_coalescing_before_inflate_grandparent",
1338 "LVHDRT_relinking_grandchildren",
1339 "LVHDRT_before_create_relink_journal",
1340 "LVHDRT_xapiSM_serialization_tests",
1341 "LVHDRT_clone_vdi_after_create_journal",
1342 "LVHDRT_clone_vdi_after_shrink_parent",
1343 "LVHDRT_clone_vdi_after_first_snap",
1344 "LVHDRT_clone_vdi_after_second_snap",
1345 "LVHDRT_clone_vdi_after_parent_hidden",
1346 "LVHDRT_clone_vdi_after_parent_ro",
1347 "LVHDRT_clone_vdi_before_remove_journal",
1348 "LVHDRT_clone_vdi_after_lvcreate",
1349 "LVHDRT_clone_vdi_before_undo_clone",
1350 "LVHDRT_clone_vdi_after_undo_clone",
1351 "LVHDRT_inflate_after_create_journal",
1352 "LVHDRT_inflate_after_setSize",
1353 "LVHDRT_inflate_after_zeroOut",
1354 "LVHDRT_inflate_after_setSizePhys",
1355 "LVHDRT_inflate_after_setSizePhys",
1356 "LVHDRT_coaleaf_before_coalesce",
1357 "LVHDRT_coaleaf_after_coalesce",
1358 "LVHDRT_coaleaf_one_renamed",
1359 "LVHDRT_coaleaf_both_renamed",
1360 "LVHDRT_coaleaf_after_vdirec",
1361 "LVHDRT_coaleaf_before_delete",
1362 "LVHDRT_coaleaf_after_delete",
1363 "LVHDRT_coaleaf_before_remove_j",
1364 "LVHDRT_coaleaf_undo_after_rename",
1365 "LVHDRT_coaleaf_undo_after_rename2",
1366 "LVHDRT_coaleaf_undo_after_refcount",
1367 "LVHDRT_coaleaf_undo_after_deflate",
1368 "LVHDRT_coaleaf_undo_end",
1369 "LVHDRT_coaleaf_stop_after_recovery",
1370 "LVHDRT_coaleaf_finish_after_inflate",
1371 "LVHDRT_coaleaf_finish_end",
1372 "LVHDRT_coaleaf_delay_1",
1373 "LVHDRT_coaleaf_delay_2",
1374 "LVHDRT_coaleaf_delay_3",
1375 "testsm_clone_allow_raw",
1376 "xenrt_default_vdi_type_legacy",
1377 "blktap_activate_inject_failure",
1378 "blktap_activate_error_handling",
1379 GCPAUSE_FISTPOINT,
1380 "cleanup_coalesceVHD_inject_failure",
1381 "cleanup_tracker_no_progress",
1382 "FileSR_fail_hardlink",
1383 "FileSR_fail_snap1",
1384 "FileSR_fail_snap2"])
1387def set_dirty(session, sr):
1388 try:
1389 session.xenapi.SR.add_to_other_config(sr, "dirty", "")
1390 SMlog("set_dirty %s succeeded" % (repr(sr)))
1391 except:
1392 SMlog("set_dirty %s failed (flag already set?)" % (repr(sr)))
1395def doesFileHaveOpenHandles(fileName):
1396 SMlog("Entering doesFileHaveOpenHandles with file: %s" % fileName)
1397 (retVal, processAndPidTuples) = \
1398 findRunningProcessOrOpenFile(fileName, False)
1400 if not retVal:
1401 SMlog("Failed to determine if file %s has open handles." % \
1402 fileName)
1403 # err on the side of caution
1404 return True
1405 else:
1406 if len(processAndPidTuples) > 0:
1407 return True
1408 else:
1409 return False
1412# extract SR uuid from the passed in devmapper entry and return
1413# /dev/mapper/VG_XenStorage--c3d82e92--cb25--c99b--b83a--482eebab4a93-MGT
1414def extractSRFromDevMapper(path):
1415 try:
1416 path = os.path.basename(path)
1417 path = path[len('VG_XenStorage-') + 1:]
1418 path = path.replace('--', '/')
1419 path = path[0:path.rfind('-')]
1420 return path.replace('/', '-')
1421 except:
1422 return ''
1425# Looks at /proc and figures either
1426# If a process is still running (default), returns open file names
1427# If any running process has open handles to the given file (process = False)
1428# returns process names and pids
1429def findRunningProcessOrOpenFile(name, process=True):
1430 retVal = True
1431 links = []
1432 processandpids = []
1433 sockets = set()
1434 try:
1435 SMlog("Entering findRunningProcessOrOpenFile with params: %s" % \
1436 [name, process])
1438 # Look at all pids
1439 pids = [pid for pid in os.listdir('/proc') if pid.isdigit()]
1440 for pid in sorted(pids):
1441 try:
1442 try:
1443 f = None
1444 f = open(os.path.join('/proc', pid, 'cmdline'), 'r')
1445 prog = f.read()[:-1]
1446 if prog: 1446 ↛ 1455line 1446 didn't jump to line 1455, because the condition on line 1446 was never false
1447 # Just want the process name
1448 argv = prog.split('\x00')
1449 prog = argv[0]
1450 except IOError as e:
1451 if e.errno in (errno.ENOENT, errno.ESRCH):
1452 SMlog("ERROR %s reading %s, ignore" % (e.errno, pid))
1453 continue
1454 finally:
1455 if f is not None: 1455 ↛ 1440, 1455 ↛ 14582 missed branches: 1) line 1455 didn't jump to line 1440, because the continue on line 1453 wasn't executed, 2) line 1455 didn't jump to line 1458, because the condition on line 1455 was never false
1456 f.close() 1456 ↛ 1440line 1456 didn't jump to line 1440, because the continue on line 1453 wasn't executed
1458 try:
1459 fd_dir = os.path.join('/proc', pid, 'fd')
1460 files = os.listdir(fd_dir)
1461 except OSError as e:
1462 if e.errno in (errno.ENOENT, errno.ESRCH):
1463 SMlog("ERROR %s reading fds for %s, ignore" % (e.errno, pid))
1464 # Ignore pid that are no longer valid
1465 continue
1466 else:
1467 raise
1469 for file in files:
1470 try:
1471 link = os.readlink(os.path.join(fd_dir, file))
1472 except OSError:
1473 continue
1475 if process: 1475 ↛ 1480line 1475 didn't jump to line 1480, because the condition on line 1475 was never false
1476 if name == prog: 1476 ↛ 1469line 1476 didn't jump to line 1469, because the condition on line 1476 was never false
1477 links.append(link)
1478 else:
1479 # need to return process name and pid tuples
1480 if link == name:
1481 SMlog("File %s has an open handle with process %s "
1482 "with pid %s" % (name, prog, pid))
1483 processandpids.append((prog, pid))
1485 # Get the connected sockets
1486 if name == prog:
1487 sockets.update(get_connected_sockets(pid))
1488 except Exception as e:
1489 SMlog("Exception checking running process or open file handles. " \
1490 "Error: %s" % str(e))
1491 retVal = False
1493 if process: 1493 ↛ 1496line 1493 didn't jump to line 1496, because the condition on line 1493 was never false
1494 return retVal, links, sockets
1495 else:
1496 return retVal, processandpids
1499def get_connected_sockets(pid):
1500 sockets = set()
1501 try:
1502 # Lines in /proc/<pid>/net/unix are formatted as follows
1503 # (see Linux source net/unix/af_unix.c, unix_seq_show() )
1504 # - Pointer address to socket (hex)
1505 # - Refcount (HEX)
1506 # - 0
1507 # - State (HEX, 0 or __SO_ACCEPTCON)
1508 # - Type (HEX - but only 0001 of interest)
1509 # - Connection state (HEX - but only 03, SS_CONNECTED of interest)
1510 # - Inode number
1511 # - Path (optional)
1512 open_sock_matcher = re.compile(
1513 r'^[0-9a-f]+: [0-9A-Fa-f]+ [0-9A-Fa-f]+ [0-9A-Fa-f]+ 0001 03 \d+ (.*)$')
1514 with open(
1515 os.path.join('/proc', str(pid), 'net', 'unix'), 'r') as f:
1516 lines = f.readlines()
1517 for line in lines:
1518 match = open_sock_matcher.match(line)
1519 if match:
1520 sockets.add(match[1])
1521 except OSError as e:
1522 if e.errno in (errno.ENOENT, errno.ESRCH):
1523 # Ignore pid that are no longer valid
1524 SMlog("ERROR %s reading sockets for %s, ignore" %
1525 (e.errno, pid))
1526 else:
1527 raise
1528 return sockets
1531def retry(f, maxretry=20, period=3, exceptions=[Exception]):
1532 retries = 0
1533 while True:
1534 try:
1535 return f()
1536 except Exception as e:
1537 for exception in exceptions:
1538 if isinstance(e, exception):
1539 SMlog('Got exception: {}. Retry number: {}'.format(
1540 str(e), retries
1541 ))
1542 break
1543 else:
1544 SMlog('Got bad exception: {}. Raising...'.format(e))
1545 raise e
1547 retries += 1
1548 if retries >= maxretry:
1549 break
1551 time.sleep(period)
1553 return f()
1556def getCslDevPath(svid):
1557 basepath = "/dev/disk/by-csldev/"
1558 if svid.startswith("NETAPP_"):
1559 # special attention for NETAPP SVIDs
1560 svid_parts = svid.split("__")
1561 globstr = basepath + "NETAPP__LUN__" + "*" + svid_parts[2] + "*" + svid_parts[-1] + "*"
1562 else:
1563 globstr = basepath + svid + "*"
1565 return globstr
1568# Use device in /dev pointed to by cslg path which consists of svid
1569def get_scsiid_from_svid(md_svid):
1570 cslg_path = getCslDevPath(md_svid)
1571 abs_path = glob.glob(cslg_path)
1572 if abs_path:
1573 real_path = os.path.realpath(abs_path[0])
1574 return scsiutil.getSCSIid(real_path)
1575 else:
1576 return None
1579def get_isl_scsiids(session):
1580 # Get cslg type SRs
1581 SRs = session.xenapi.SR.get_all_records_where('field "type" = "cslg"')
1583 # Iterate through the SR to get the scsi ids
1584 scsi_id_ret = []
1585 for SR in SRs:
1586 sr_rec = SRs[SR]
1587 # Use the md_svid to get the scsi id
1588 scsi_id = get_scsiid_from_svid(sr_rec['sm_config']['md_svid'])
1589 if scsi_id:
1590 scsi_id_ret.append(scsi_id)
1592 # Get the vdis in the SR and do the same procedure
1593 vdi_recs = session.xenapi.VDI.get_all_records_where('field "SR" = "%s"' % SR)
1594 for vdi_rec in vdi_recs:
1595 vdi_rec = vdi_recs[vdi_rec]
1596 scsi_id = get_scsiid_from_svid(vdi_rec['sm_config']['SVID'])
1597 if scsi_id:
1598 scsi_id_ret.append(scsi_id)
1600 return scsi_id_ret
1603class extractXVA:
1604 # streams files as a set of file and checksum, caller should remove
1605 # the files, if not needed. The entire directory (Where the files
1606 # and checksum) will only be deleted as part of class cleanup.
1607 HDR_SIZE = 512
1608 BLOCK_SIZE = 512
1609 SIZE_LEN = 12 - 1 # To remove \0 from tail
1610 SIZE_OFFSET = 124
1611 ZERO_FILLED_REC = 2
1612 NULL_IDEN = '\x00'
1613 DIR_IDEN = '/'
1614 CHECKSUM_IDEN = '.checksum'
1615 OVA_FILE = 'ova.xml'
1617 # Init gunzips the file using a subprocess, and reads stdout later
1618 # as and when needed
1619 def __init__(self, filename):
1620 self.__extract_path = ''
1621 self.__filename = filename
1622 cmd = 'gunzip -cd %s' % filename
1623 try:
1624 self.spawn_p = subprocess.Popen(
1625 cmd, shell=True, \
1626 stdin=subprocess.PIPE, stdout=subprocess.PIPE, \
1627 stderr=subprocess.PIPE, close_fds=True)
1628 except Exception as e:
1629 SMlog("Error: %s. Uncompress failed for %s" % (str(e), filename))
1630 raise Exception(str(e))
1632 # Create dir to extract the files
1633 self.__extract_path = tempfile.mkdtemp()
1635 def __del__(self):
1636 shutil.rmtree(self.__extract_path)
1638 # Class supports Generator expression. 'for f_name, checksum in getTuple()'
1639 # returns filename, checksum content. Returns filename, '' in case
1640 # of checksum file missing. e.g. ova.xml
1641 def getTuple(self):
1642 zerod_record = 0
1643 ret_f_name = ''
1644 ret_base_f_name = ''
1646 try:
1647 # Read tar file as sets of file and checksum.
1648 while True:
1649 # Read the output of spawned process, or output of gunzip
1650 f_hdr = self.spawn_p.stdout.read(self.HDR_SIZE)
1652 # Break out in case of end of file
1653 if f_hdr == '':
1654 if zerod_record == extractXVA.ZERO_FILLED_REC:
1655 break
1656 else:
1657 SMlog('Error. Expects %d zero records', \
1658 extractXVA.ZERO_FILLED_REC)
1659 raise Exception('Unrecognized end of file')
1661 # Watch out for zero records, two zero records
1662 # denote end of file.
1663 if f_hdr == extractXVA.NULL_IDEN * extractXVA.HDR_SIZE:
1664 zerod_record += 1
1665 continue
1667 f_name = f_hdr[:f_hdr.index(extractXVA.NULL_IDEN)]
1668 # File header may be for a folder, if so ignore the header
1669 if not f_name.endswith(extractXVA.DIR_IDEN):
1670 f_size_octal = f_hdr[extractXVA.SIZE_OFFSET: \
1671 extractXVA.SIZE_OFFSET + extractXVA.SIZE_LEN]
1672 f_size = int(f_size_octal, 8)
1673 if f_name.endswith(extractXVA.CHECKSUM_IDEN):
1674 if f_name.rstrip(extractXVA.CHECKSUM_IDEN) == \
1675 ret_base_f_name:
1676 checksum = self.spawn_p.stdout.read(f_size)
1677 yield(ret_f_name, checksum)
1678 else:
1679 # Expects file followed by its checksum
1680 SMlog('Error. Sequence mismatch starting with %s', \
1681 ret_f_name)
1682 raise Exception( \
1683 'Files out of sequence starting with %s', \
1684 ret_f_name)
1685 else:
1686 # In case of ova.xml, read the contents into a file and
1687 # return the file name to the caller. For other files,
1688 # read the contents into a file, it will
1689 # be used when a .checksum file is encountered.
1690 ret_f_name = '%s/%s' % (self.__extract_path, f_name)
1691 ret_base_f_name = f_name
1693 # Check if the folder exists on the target location,
1694 # else create it.
1695 folder_path = ret_f_name[:ret_f_name.rfind('/')]
1696 if not os.path.exists(folder_path):
1697 os.mkdir(folder_path)
1699 # Store the file to the tmp folder, strip the tail \0
1700 f = open(ret_f_name, 'w')
1701 f.write(self.spawn_p.stdout.read(f_size))
1702 f.close()
1703 if f_name == extractXVA.OVA_FILE:
1704 yield(ret_f_name, '')
1706 # Skip zero'd portion of data block
1707 round_off = f_size % extractXVA.BLOCK_SIZE
1708 if round_off != 0:
1709 zeros = self.spawn_p.stdout.read(
1710 extractXVA.BLOCK_SIZE - round_off)
1711 except Exception as e:
1712 SMlog("Error: %s. File set extraction failed %s" % (str(e), \
1713 self.__filename))
1715 # Kill and Drain stdout of the gunzip process,
1716 # else gunzip might block on stdout
1717 os.kill(self.spawn_p.pid, signal.SIGTERM)
1718 self.spawn_p.communicate()
1719 raise Exception(str(e))
1721illegal_xml_chars = [(0x00, 0x08), (0x0B, 0x1F), (0x7F, 0x84), (0x86, 0x9F),
1722 (0xD800, 0xDFFF), (0xFDD0, 0xFDDF), (0xFFFE, 0xFFFF),
1723 (0x1FFFE, 0x1FFFF), (0x2FFFE, 0x2FFFF), (0x3FFFE, 0x3FFFF),
1724 (0x4FFFE, 0x4FFFF), (0x5FFFE, 0x5FFFF), (0x6FFFE, 0x6FFFF),
1725 (0x7FFFE, 0x7FFFF), (0x8FFFE, 0x8FFFF), (0x9FFFE, 0x9FFFF),
1726 (0xAFFFE, 0xAFFFF), (0xBFFFE, 0xBFFFF), (0xCFFFE, 0xCFFFF),
1727 (0xDFFFE, 0xDFFFF), (0xEFFFE, 0xEFFFF), (0xFFFFE, 0xFFFFF),
1728 (0x10FFFE, 0x10FFFF)]
1730illegal_ranges = ["%s-%s" % (chr(low), chr(high))
1731 for (low, high) in illegal_xml_chars
1732 if low < sys.maxunicode]
1734illegal_xml_re = re.compile(u'[%s]' % u''.join(illegal_ranges))
1737def isLegalXMLString(s):
1738 """Tells whether this is a valid XML string (i.e. it does not contain
1739 illegal XML characters specified in
1740 http://www.w3.org/TR/2004/REC-xml-20040204/#charsets).
1741 """
1743 if len(s) > 0:
1744 return re.search(illegal_xml_re, s) is None
1745 else:
1746 return True
1749def unictrunc(string, max_bytes):
1750 """
1751 Returns the number of bytes that is smaller than, or equal to, the number
1752 of bytes specified, such that the UTF-8 encoded string can be correctly
1753 truncated.
1754 string: the string to truncate
1755 max_bytes: the maximum number of bytes the truncated string can be
1756 """
1757 string = string.decode('UTF-8')
1758 cur_bytes = 0
1759 for char in string:
1760 charsize = len(char.encode('UTF-8'))
1761 if cur_bytes + charsize > max_bytes:
1762 break
1763 else:
1764 cur_bytes = cur_bytes + charsize
1765 return cur_bytes
1768def hideValuesInPropMap(propmap, propnames):
1769 """
1770 Worker function: input simple map of prop name/value pairs, and
1771 a list of specific propnames whose values we want to hide.
1772 Loop through the "hide" list, and if any are found, hide the
1773 value and return the altered map.
1774 If none found, return the original map
1775 """
1776 matches = []
1777 for propname in propnames:
1778 if propname in propmap: 1778 ↛ 1779line 1778 didn't jump to line 1779, because the condition on line 1778 was never true
1779 matches.append(propname)
1781 if matches: 1781 ↛ 1782line 1781 didn't jump to line 1782, because the condition on line 1781 was never true
1782 deepCopyRec = copy.deepcopy(propmap)
1783 for match in matches:
1784 deepCopyRec[match] = '******'
1785 return deepCopyRec
1787 return propmap
1788# define the list of propnames whose value we want to hide
1790PASSWD_PROP_KEYS = ['password', 'cifspassword', 'chappassword', 'incoming_chappassword']
1791DEFAULT_SEGMENT_LEN = 950
1794def hidePasswdInConfig(config):
1795 """
1796 Function to hide passwd values in a simple prop map,
1797 for example "device_config"
1798 """
1799 return hideValuesInPropMap(config, PASSWD_PROP_KEYS)
1802def hidePasswdInParams(params, configProp):
1803 """
1804 Function to hide password values in a specified property which
1805 is a simple map of prop name/values, and is itself an prop entry
1806 in a larger property map.
1807 For example, param maps containing "device_config", or
1808 "sm_config", etc
1809 """
1810 params[configProp] = hideValuesInPropMap(params[configProp], PASSWD_PROP_KEYS)
1811 return params
1814def hideMemberValuesInXmlParams(xmlParams, propnames=PASSWD_PROP_KEYS):
1815 """
1816 Function to hide password values in XML params, specifically
1817 for the XML format of incoming params to SR modules.
1818 Uses text parsing: loop through the list of specific propnames
1819 whose values we want to hide, and:
1820 - Assemble a full "prefix" containing each property name, e.g.,
1821 "<member><name>password</name><value>"
1822 - Test the XML if it contains that string, save the index.
1823 - If found, get the index of the ending tag
1824 - Truncate the return string starting with the password value.
1825 - Append the substitute "*******" value string.
1826 - Restore the rest of the original string starting with the end tag.
1827 """
1828 findStrPrefixHead = "<member><name>"
1829 findStrPrefixTail = "</name><value>"
1830 findStrSuffix = "</value>"
1831 strlen = len(xmlParams)
1833 for propname in propnames:
1834 findStrPrefix = findStrPrefixHead + propname + findStrPrefixTail
1835 idx = xmlParams.find(findStrPrefix)
1836 if idx != -1: # if found any of them
1837 idx += len(findStrPrefix)
1838 idx2 = xmlParams.find(findStrSuffix, idx)
1839 if idx2 != -1:
1840 retStr = xmlParams[0:idx]
1841 retStr += "******"
1842 retStr += xmlParams[idx2:strlen]
1843 return retStr
1844 else:
1845 return xmlParams
1846 return xmlParams
1849def splitXmlText(xmlData, segmentLen=DEFAULT_SEGMENT_LEN, showContd=False):
1850 """
1851 Split xml string data into substrings small enough for the
1852 syslog line length limit. Split at tag end markers ( ">" ).
1853 Usage:
1854 strList = []
1855 strList = splitXmlText( longXmlText, maxLineLen ) # maxLineLen is optional
1856 """
1857 remainingData = str(xmlData)
1859 # "Un-pretty-print"
1860 remainingData = remainingData.replace('\n', '')
1861 remainingData = remainingData.replace('\t', '')
1863 remainingChars = len(remainingData)
1864 returnData = ''
1866 thisLineNum = 0
1867 while remainingChars > segmentLen:
1868 thisLineNum = thisLineNum + 1
1869 index = segmentLen
1870 tmpStr = remainingData[:segmentLen]
1871 tmpIndex = tmpStr.rfind('>')
1872 if tmpIndex != -1:
1873 index = tmpIndex + 1
1875 tmpStr = tmpStr[:index]
1876 remainingData = remainingData[index:]
1877 remainingChars = len(remainingData)
1879 if showContd:
1880 if thisLineNum != 1:
1881 tmpStr = '(Cont\'d): ' + tmpStr
1882 tmpStr = tmpStr + ' (Cont\'d):'
1884 returnData += tmpStr + '\n'
1886 if showContd and thisLineNum > 0:
1887 remainingData = '(Cont\'d): ' + remainingData
1888 returnData += remainingData
1890 return returnData
1893def inject_failure():
1894 raise Exception('injected failure')
1897def open_atomic(path, mode=None):
1898 """Atomically creates a file if, and only if it does not already exist.
1899 Leaves the file open and returns the file object.
1901 path: the path to atomically open
1902 mode: "r" (read), "w" (write), or "rw" (read/write)
1903 returns: an open file object"""
1905 assert path
1907 flags = os.O_CREAT | os.O_EXCL
1908 modes = {'r': os.O_RDONLY, 'w': os.O_WRONLY, 'rw': os.O_RDWR}
1909 if mode:
1910 if mode not in modes:
1911 raise Exception('invalid access mode ' + mode)
1912 flags |= modes[mode]
1913 fd = os.open(path, flags)
1914 try:
1915 if mode:
1916 return os.fdopen(fd, mode)
1917 else:
1918 return os.fdopen(fd)
1919 except:
1920 os.close(fd)
1921 raise
1924def isInvalidVDI(exception):
1925 return exception.details[0] == "HANDLE_INVALID" or \
1926 exception.details[0] == "UUID_INVALID"
1929def get_pool_restrictions(session):
1930 """Returns pool restrictions as a map, @session must be already
1931 established."""
1932 return list(session.xenapi.pool.get_all_records().values())[0]['restrictions']
1935def read_caching_is_restricted(session):
1936 """Tells whether read caching is restricted."""
1937 if session is None: 1937 ↛ 1938line 1937 didn't jump to line 1938, because the condition on line 1937 was never true
1938 return True
1939 restrictions = get_pool_restrictions(session)
1940 if 'restrict_read_caching' in restrictions and \ 1940 ↛ 1942line 1940 didn't jump to line 1942, because the condition on line 1940 was never true
1941 restrictions['restrict_read_caching'] == "true":
1942 return True
1943 return False
1946def sessions_less_than_targets(other_config, device_config):
1947 if 'multihomelist' in device_config and 'iscsi_sessions' in other_config: 1947 ↛ 1953line 1947 didn't jump to line 1953, because the condition on line 1947 was never false
1948 sessions = int(other_config['iscsi_sessions'])
1949 targets = len(device_config['multihomelist'].split(','))
1950 SMlog("Targets %d and iscsi_sessions %d" % (targets, sessions))
1951 return (sessions < targets)
1952 else:
1953 return False
1956def enable_and_start_service(name, start):
1957 attempt = 0
1958 while True:
1959 attempt += 1
1960 fn = 'enable' if start else 'disable'
1961 args = ('systemctl', fn, '--now', name)
1962 (ret, out, err) = doexec(args)
1963 if ret == 0:
1964 return
1965 elif attempt >= 3:
1966 raise Exception(
1967 'Failed to {} {}: {} {}'.format(fn, name, out, err)
1968 )
1969 time.sleep(1)
1972def stop_service(name):
1973 args = ('systemctl', 'stop', name)
1974 (ret, out, err) = doexec(args)
1975 if ret == 0:
1976 return
1977 raise Exception('Failed to stop {}: {} {}'.format(name, out, err))
1980def restart_service(name):
1981 attempt = 0
1982 while True:
1983 attempt += 1
1984 SMlog('Restarting service {} {}...'.format(name, attempt))
1985 args = ('systemctl', 'restart', name)
1986 (ret, out, err) = doexec(args)
1987 if ret == 0:
1988 return
1989 elif attempt >= 3:
1990 SMlog('Restart service FAILED {} {}'.format(name, attempt))
1991 raise Exception(
1992 'Failed to restart {}: {} {}'.format(name, out, err)
1993 )
1994 time.sleep(1)
1997def check_pid_exists(pid):
1998 try:
1999 os.kill(pid, 0)
2000 except OSError:
2001 return False
2002 else:
2003 return True
2006def make_profile(name, function):
2007 """
2008 Helper to execute cProfile using unique log file.
2009 """
2011 import cProfile
2012 import itertools
2013 import os.path
2014 import time
2016 assert name
2017 assert function
2019 FOLDER = '/tmp/sm-perfs/'
2020 makedirs(FOLDER)
2022 filename = time.strftime('{}_%Y%m%d_%H%M%S.prof'.format(name))
2024 def gen_path(path):
2025 yield path
2026 root, ext = os.path.splitext(path)
2027 for i in itertools.count(start=1, step=1):
2028 yield root + '.{}.'.format(i) + ext
2030 for profile_path in gen_path(FOLDER + filename):
2031 try:
2032 file = open_atomic(profile_path, 'w')
2033 file.close()
2034 break
2035 except OSError as e:
2036 if e.errno == errno.EEXIST:
2037 pass
2038 else:
2039 raise
2041 try:
2042 SMlog('* Start profiling of {} ({}) *'.format(name, filename))
2043 cProfile.runctx('function()', None, locals(), profile_path)
2044 finally:
2045 SMlog('* End profiling of {} ({}) *'.format(name, filename))