Coverage for drivers/util.py : 44%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Copyright (C) Citrix Systems Inc.
2#
3# This program is free software; you can redistribute it and/or modify
4# it under the terms of the GNU Lesser General Public License as published
5# by the Free Software Foundation; version 2.1 only.
6#
7# This program is distributed in the hope that it will be useful,
8# but WITHOUT ANY WARRANTY; without even the implied warranty of
9# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10# GNU Lesser General Public License for more details.
11#
12# You should have received a copy of the GNU Lesser General Public License
13# along with this program; if not, write to the Free Software Foundation, Inc.,
14# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
15#
16# Miscellaneous utility functions
17#
19import os
20import re
21import sys
22import subprocess
23import shutil
24import tempfile
25import signal
26import time
27import datetime
28import errno
29import socket
30import xml.dom.minidom
31import scsiutil
32import stat
33import xs_errors
34import XenAPI # pylint: disable=import-error
35import xmlrpc.client
36import base64
37import syslog
38import resource
39import traceback
40import glob
41import copy
42import tempfile
44from functools import reduce
46NO_LOGGING_STAMPFILE = '/etc/xensource/no_sm_log'
48IORETRY_MAX = 20 # retries
49IORETRY_PERIOD = 1.0 # seconds
51LOGGING = not (os.path.exists(NO_LOGGING_STAMPFILE))
52_SM_SYSLOG_FACILITY = syslog.LOG_LOCAL2
53LOG_EMERG = syslog.LOG_EMERG
54LOG_ALERT = syslog.LOG_ALERT
55LOG_CRIT = syslog.LOG_CRIT
56LOG_ERR = syslog.LOG_ERR
57LOG_WARNING = syslog.LOG_WARNING
58LOG_NOTICE = syslog.LOG_NOTICE
59LOG_INFO = syslog.LOG_INFO
60LOG_DEBUG = syslog.LOG_DEBUG
62ISCSI_REFDIR = '/var/run/sr-ref'
64CMD_DD = "/bin/dd"
66FIST_PAUSE_PERIOD = 30 # seconds
69class SMException(Exception):
70 """Base class for all SM exceptions for easier catching & wrapping in
71 XenError"""
74class CommandException(SMException):
75 def error_message(self, code):
76 if code > 0:
77 return os.strerror(code)
78 elif code < 0:
79 return "Signalled %s" % (abs(code))
80 return "Success"
82 def __init__(self, code, cmd="", reason='exec failed'):
83 self.code = code
84 self.cmd = cmd
85 self.reason = reason
86 Exception.__init__(self, self.error_message(code))
89class SRBusyException(SMException):
90 """The SR could not be locked"""
91 pass
94def logException(tag):
95 info = sys.exc_info()
96 if info[0] == SystemExit: 96 ↛ 98line 96 didn't jump to line 98, because the condition on line 96 was never true
97 # this should not be happening when catching "Exception", but it is
98 sys.exit(0)
99 tb = reduce(lambda a, b: "%s%s" % (a, b), traceback.format_tb(info[2]))
100 str = "***** %s: EXCEPTION %s, %s\n%s" % (tag, info[0], info[1], tb)
101 SMlog(str)
104def roundup(divisor, value):
105 """Retruns the rounded up value so it is divisible by divisor."""
107 if value == 0: 107 ↛ 108line 107 didn't jump to line 108, because the condition on line 107 was never true
108 value = 1
109 if value % divisor != 0:
110 return ((int(value) // divisor) + 1) * divisor
111 return value
114def to_plain_string(obj):
115 if obj is None:
116 return None
117 if type(obj) == str:
118 return obj
119 return str(obj)
122def shellquote(arg):
123 return '"%s"' % arg.replace('"', '\\"')
126def make_WWN(name):
127 hex_prefix = name.find("0x")
128 if (hex_prefix >= 0): 128 ↛ 131line 128 didn't jump to line 131, because the condition on line 128 was never false
129 name = name[name.find("0x") + 2:len(name)]
130 # inject dashes for each nibble
131 if (len(name) == 16): # sanity check 131 ↛ 135line 131 didn't jump to line 135, because the condition on line 131 was never false
132 name = name[0:2] + "-" + name[2:4] + "-" + name[4:6] + "-" + \
133 name[6:8] + "-" + name[8:10] + "-" + name[10:12] + "-" + \
134 name[12:14] + "-" + name[14:16]
135 return name
138def _logToSyslog(ident, facility, priority, message):
139 syslog.openlog(ident, 0, facility)
140 syslog.syslog(priority, "[%d] %s" % (os.getpid(), message))
141 syslog.closelog()
144def SMlog(message, ident="SM", priority=LOG_INFO):
145 if LOGGING: 145 ↛ exitline 145 didn't return from function 'SMlog', because the condition on line 145 was never false
146 for message_line in str(message).split('\n'):
147 _logToSyslog(ident, _SM_SYSLOG_FACILITY, priority, message_line)
150def _getDateString():
151 d = datetime.datetime.now()
152 t = d.timetuple()
153 return "%s-%s-%s:%s:%s:%s" % \
154 (t[0], t[1], t[2], t[3], t[4], t[5])
157def doexec(args, inputtext=None, new_env=None, text=True):
158 """Execute a subprocess, then return its return code, stdout and stderr"""
159 env = None
160 if new_env:
161 env = dict(os.environ)
162 env.update(new_env)
163 proc = subprocess.Popen(args, stdin=subprocess.PIPE,
164 stdout=subprocess.PIPE,
165 stderr=subprocess.PIPE,
166 close_fds=True, env=env,
167 universal_newlines=text)
169 if not text and inputtext is not None: 169 ↛ 170line 169 didn't jump to line 170, because the condition on line 169 was never true
170 inputtext = inputtext.encode()
172 (stdout, stderr) = proc.communicate(inputtext)
174 rc = proc.returncode
175 return rc, stdout, stderr
178def is_string(value):
179 return isinstance(value, str)
182# These are partially tested functions that replicate the behaviour of
183# the original pread,pread2 and pread3 functions. Potentially these can
184# replace the original ones at some later date.
185#
186# cmdlist is a list of either single strings or pairs of strings. For
187# each pair, the first component is passed to exec while the second is
188# written to the logs.
189def pread(cmdlist, close_stdin=False, scramble=None, expect_rc=0,
190 quiet=False, new_env=None, text=True):
191 cmdlist_for_exec = []
192 cmdlist_for_log = []
193 for item in cmdlist:
194 if is_string(item): 194 ↛ 204line 194 didn't jump to line 204, because the condition on line 194 was never false
195 cmdlist_for_exec.append(item)
196 if scramble: 196 ↛ 197line 196 didn't jump to line 197, because the condition on line 196 was never true
197 if item.find(scramble) != -1:
198 cmdlist_for_log.append("<filtered out>")
199 else:
200 cmdlist_for_log.append(item)
201 else:
202 cmdlist_for_log.append(item)
203 else:
204 cmdlist_for_exec.append(item[0])
205 cmdlist_for_log.append(item[1])
207 if not quiet: 207 ↛ 209line 207 didn't jump to line 209, because the condition on line 207 was never false
208 SMlog(cmdlist_for_log)
209 (rc, stdout, stderr) = doexec(cmdlist_for_exec, new_env=new_env, text=text)
210 if rc != expect_rc:
211 SMlog("FAILED in util.pread: (rc %d) stdout: '%s', stderr: '%s'" % \
212 (rc, stdout, stderr))
213 if quiet: 213 ↛ 214line 213 didn't jump to line 214, because the condition on line 213 was never true
214 SMlog("Command was: %s" % cmdlist_for_log)
215 if '' == stderr: 215 ↛ 216line 215 didn't jump to line 216, because the condition on line 215 was never true
216 stderr = stdout
217 raise CommandException(rc, str(cmdlist), stderr.strip())
218 if not quiet: 218 ↛ 220line 218 didn't jump to line 220, because the condition on line 218 was never false
219 SMlog(" pread SUCCESS")
220 return stdout
223# POSIX guaranteed atomic within the same file system.
224# Supply directory to ensure tempfile is created
225# in the same directory.
226def atomicFileWrite(targetFile, directory, text):
228 file = None
229 try:
230 # Create file only current pid can write/read to
231 # our responsibility to clean it up.
232 _, tempPath = tempfile.mkstemp(dir=directory)
233 file = open(tempPath, 'w')
234 file.write(text)
236 # Ensure flushed to disk.
237 file.flush()
238 os.fsync(file.fileno())
239 file.close()
241 os.rename(tempPath, targetFile)
242 except OSError:
243 SMlog("FAILED to atomic write to %s" % (targetFile))
245 finally:
246 if (file is not None) and (not file.closed):
247 file.close()
249 if os.path.isfile(tempPath):
250 os.remove(tempPath)
253#Read STDOUT from cmdlist and discard STDERR output
254def pread2(cmdlist, quiet=False, text=True):
255 return pread(cmdlist, quiet=quiet, text=text)
258#Read STDOUT from cmdlist, feeding 'text' to STDIN
259def pread3(cmdlist, text):
260 SMlog(cmdlist)
261 (rc, stdout, stderr) = doexec(cmdlist, text)
262 if rc:
263 SMlog("FAILED in util.pread3: (errno %d) stdout: '%s', stderr: '%s'" % \
264 (rc, stdout, stderr))
265 if '' == stderr:
266 stderr = stdout
267 raise CommandException(rc, str(cmdlist), stderr.strip())
268 SMlog(" pread3 SUCCESS")
269 return stdout
272def listdir(path, quiet=False):
273 cmd = ["ls", path, "-1", "--color=never"]
274 try:
275 text = pread2(cmd, quiet=quiet)[:-1]
276 if len(text) == 0:
277 return []
278 return text.split('\n')
279 except CommandException as inst:
280 if inst.code == errno.ENOENT:
281 raise CommandException(errno.EIO, inst.cmd, inst.reason)
282 else:
283 raise CommandException(inst.code, inst.cmd, inst.reason)
286def gen_uuid():
287 cmd = ["uuidgen", "-r"]
288 return pread(cmd)[:-1]
291def match_uuid(s):
292 regex = re.compile("^[0-9a-f]{8}-(([0-9a-f]{4})-){3}[0-9a-f]{12}")
293 return regex.search(s, 0)
296def findall_uuid(s):
297 regex = re.compile("[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}")
298 return regex.findall(s, 0)
301def exactmatch_uuid(s):
302 regex = re.compile("^[0-9a-f]{8}-(([0-9a-f]{4})-){3}[0-9a-f]{12}$")
303 return regex.search(s, 0)
306def start_log_entry(srpath, path, args):
307 logstring = str(datetime.datetime.now())
308 logstring += " log: "
309 logstring += srpath
310 logstring += " " + path
311 for element in args:
312 logstring += " " + element
313 try:
314 file = open(srpath + "/filelog.txt", "a")
315 file.write(logstring)
316 file.write("\n")
317 file.close()
318 except:
319 pass
321 # failed to write log ...
323def end_log_entry(srpath, path, args):
324 # for teminating, use "error" or "done"
325 logstring = str(datetime.datetime.now())
326 logstring += " end: "
327 logstring += srpath
328 logstring += " " + path
329 for element in args:
330 logstring += " " + element
331 try:
332 file = open(srpath + "/filelog.txt", "a")
333 file.write(logstring)
334 file.write("\n")
335 file.close()
336 except:
337 pass
339 # failed to write log ...
340 # for now print
341 # print "%s" % logstring
343def ioretry(f, errlist=[errno.EIO], maxretry=IORETRY_MAX, period=IORETRY_PERIOD, **ignored):
344 retries = 0
345 while True:
346 try:
347 return f()
348 except OSError as ose:
349 err = int(ose.errno)
350 if not err in errlist:
351 raise CommandException(err, str(f), "OSError")
352 except CommandException as ce:
353 if not int(ce.code) in errlist:
354 raise
356 retries += 1
357 if retries >= maxretry:
358 break
360 time.sleep(period)
362 raise CommandException(errno.ETIMEDOUT, str(f), "Timeout")
365def ioretry_stat(path, maxretry=IORETRY_MAX):
366 # this ioretry is similar to the previous method, but
367 # stat does not raise an error -- so check its return
368 retries = 0
369 while retries < maxretry:
370 stat = os.statvfs(path)
371 if stat.f_blocks != -1:
372 return stat
373 time.sleep(1)
374 retries += 1
375 raise CommandException(errno.EIO, "os.statvfs")
378def sr_get_capability(sr_uuid):
379 result = []
380 session = get_localAPI_session()
381 try:
382 sr_ref = session.xenapi.SR.get_by_uuid(sr_uuid)
383 sm_type = session.xenapi.SR.get_record(sr_ref)['type']
384 sm_rec = session.xenapi.SM.get_all_records_where(
385 "field \"type\" = \"%s\"" % sm_type)
387 # SM expects at least one entry of any SR type
388 if len(sm_rec) > 0:
389 result = list(sm_rec.values())[0]['capabilities']
391 return result
392 finally:
393 session.xenapi.session.logout()
395def sr_get_driver_info(driver_info):
396 results = {}
397 # first add in the vanilla stuff
398 for key in ['name', 'description', 'vendor', 'copyright', \
399 'driver_version', 'required_api_version']:
400 results[key] = driver_info[key]
401 # add the capabilities (xmlrpc array)
402 # enforcing activate/deactivate for blktap2
403 caps = driver_info['capabilities']
404 if "ATOMIC_PAUSE" in caps: 404 ↛ 405line 404 didn't jump to line 405, because the condition on line 404 was never true
405 for cap in ("VDI_ACTIVATE", "VDI_DEACTIVATE"):
406 if not cap in caps:
407 caps.append(cap)
408 elif "VDI_ACTIVATE" in caps or "VDI_DEACTIVATE" in caps: 408 ↛ 409line 408 didn't jump to line 409, because the condition on line 408 was never true
409 SMlog("Warning: vdi_[de]activate present for %s" % driver_info["name"])
411 results['capabilities'] = caps
412 # add in the configuration options
413 options = []
414 for option in driver_info['configuration']:
415 options.append({'key': option[0], 'description': option[1]})
416 results['configuration'] = options
417 return xmlrpc.client.dumps((results, ), "", True)
420def return_nil():
421 return xmlrpc.client.dumps((None, ), "", True, allow_none=True)
424def SRtoXML(SRlist):
425 dom = xml.dom.minidom.Document()
426 driver = dom.createElement("SRlist")
427 dom.appendChild(driver)
429 for key in SRlist.keys():
430 dict = SRlist[key]
431 entry = dom.createElement("SR")
432 driver.appendChild(entry)
434 e = dom.createElement("UUID")
435 entry.appendChild(e)
436 textnode = dom.createTextNode(key)
437 e.appendChild(textnode)
439 if 'size' in dict:
440 e = dom.createElement("Size")
441 entry.appendChild(e)
442 textnode = dom.createTextNode(str(dict['size']))
443 e.appendChild(textnode)
445 if 'storagepool' in dict:
446 e = dom.createElement("StoragePool")
447 entry.appendChild(e)
448 textnode = dom.createTextNode(str(dict['storagepool']))
449 e.appendChild(textnode)
451 if 'aggregate' in dict:
452 e = dom.createElement("Aggregate")
453 entry.appendChild(e)
454 textnode = dom.createTextNode(str(dict['aggregate']))
455 e.appendChild(textnode)
457 return dom.toprettyxml()
460def pathexists(path):
461 try:
462 os.lstat(path)
463 return True
464 except OSError as inst:
465 if inst.errno == errno.EIO: 465 ↛ 466line 465 didn't jump to line 466, because the condition on line 465 was never true
466 time.sleep(1)
467 try:
468 listdir(os.path.realpath(os.path.dirname(path)))
469 os.lstat(path)
470 return True
471 except:
472 pass
473 raise CommandException(errno.EIO, "os.lstat(%s)" % path, "failed")
474 return False
477def force_unlink(path):
478 try:
479 os.unlink(path)
480 except OSError as e:
481 if e.errno != errno.ENOENT: 481 ↛ 482line 481 didn't jump to line 482, because the condition on line 481 was never true
482 raise
485def create_secret(session, secret):
486 ref = session.xenapi.secret.create({'value': secret})
487 return session.xenapi.secret.get_uuid(ref)
490def get_secret(session, uuid):
491 try:
492 ref = session.xenapi.secret.get_by_uuid(uuid)
493 return session.xenapi.secret.get_value(ref)
494 except:
495 raise xs_errors.XenError('InvalidSecret', opterr='Unable to look up secret [%s]' % uuid)
498def get_real_path(path):
499 "Follow symlinks to the actual file"
500 absPath = path
501 directory = ''
502 while os.path.islink(absPath):
503 directory = os.path.dirname(absPath)
504 absPath = os.readlink(absPath)
505 absPath = os.path.join(directory, absPath)
506 return absPath
509def wait_for_path(path, timeout):
510 for i in range(0, timeout): 510 ↛ 514line 510 didn't jump to line 514, because the loop on line 510 didn't complete
511 if len(glob.glob(path)): 511 ↛ 513line 511 didn't jump to line 513, because the condition on line 511 was never false
512 return True
513 time.sleep(1)
514 return False
517def wait_for_nopath(path, timeout):
518 for i in range(0, timeout):
519 if not os.path.exists(path):
520 return True
521 time.sleep(1)
522 return False
525def wait_for_path_multi(path, timeout):
526 for i in range(0, timeout):
527 paths = glob.glob(path)
528 SMlog("_wait_for_paths_multi: paths = %s" % paths)
529 if len(paths):
530 SMlog("_wait_for_paths_multi: return first path: %s" % paths[0])
531 return paths[0]
532 time.sleep(1)
533 return ""
536def isdir(path):
537 try:
538 st = os.stat(path)
539 return stat.S_ISDIR(st.st_mode)
540 except OSError as inst:
541 if inst.errno == errno.EIO: 541 ↛ 542line 541 didn't jump to line 542, because the condition on line 541 was never true
542 raise CommandException(errno.EIO, "os.stat(%s)" % path, "failed")
543 return False
546def get_single_entry(path):
547 f = open(path, 'r')
548 line = f.readline()
549 f.close()
550 return line.rstrip()
553def get_fs_size(path):
554 st = ioretry_stat(path)
555 return st.f_blocks * st.f_frsize
558def get_fs_utilisation(path):
559 st = ioretry_stat(path)
560 return (st.f_blocks - st.f_bfree) * \
561 st.f_frsize
564def ismount(path):
565 """Test whether a path is a mount point"""
566 try:
567 s1 = os.stat(path)
568 s2 = os.stat(os.path.join(path, '..'))
569 except OSError as inst:
570 raise CommandException(inst.errno, "os.stat")
571 dev1 = s1.st_dev
572 dev2 = s2.st_dev
573 if dev1 != dev2:
574 return True # path/.. on a different device as path
575 ino1 = s1.st_ino
576 ino2 = s2.st_ino
577 if ino1 == ino2:
578 return True # path/.. is the same i-node as path
579 return False
582def makedirs(name, mode=0o777):
583 head, tail = os.path.split(name)
584 if not tail: 584 ↛ 585line 584 didn't jump to line 585, because the condition on line 584 was never true
585 head, tail = os.path.split(head)
586 if head and tail and not pathexists(head):
587 makedirs(head, mode)
588 if tail == os.curdir: 588 ↛ 589line 588 didn't jump to line 589, because the condition on line 588 was never true
589 return
590 try:
591 os.mkdir(name, mode)
592 except OSError as exc:
593 if exc.errno == errno.EEXIST and os.path.isdir(name): 593 ↛ 594line 593 didn't jump to line 594, because the condition on line 593 was never true
594 if mode:
595 os.chmod(name, mode)
596 pass
597 else:
598 raise
601def zeroOut(path, fromByte, bytes):
602 """write 'bytes' zeros to 'path' starting from fromByte (inclusive)"""
603 blockSize = 4096
605 fromBlock = fromByte // blockSize
606 if fromByte % blockSize:
607 fromBlock += 1
608 bytesBefore = fromBlock * blockSize - fromByte
609 if bytesBefore > bytes:
610 bytesBefore = bytes
611 bytes -= bytesBefore
612 cmd = [CMD_DD, "if=/dev/zero", "of=%s" % path, "bs=1",
613 "seek=%s" % fromByte, "count=%s" % bytesBefore, "conv=fsync"]
614 try:
615 pread2(cmd)
616 except CommandException:
617 return False
619 blocks = bytes // blockSize
620 bytes -= blocks * blockSize
621 fromByte = (fromBlock + blocks) * blockSize
622 if blocks:
623 cmd = [CMD_DD, "if=/dev/zero", "of=%s" % path, "bs=%s" % blockSize,
624 "seek=%s" % fromBlock, "count=%s" % blocks, "conv=fsync"]
625 try:
626 pread2(cmd)
627 except CommandException:
628 return False
630 if bytes:
631 cmd = [CMD_DD, "if=/dev/zero", "of=%s" % path, "bs=1",
632 "seek=%s" % fromByte, "count=%s" % bytes, "conv=fsync"]
633 try:
634 pread2(cmd)
635 except CommandException:
636 return False
638 return True
641def wipefs(blockdev):
642 "Wipe filesystem signatures from `blockdev`"
643 pread2(["/usr/sbin/wipefs", "-a", blockdev])
646def match_rootdev(s):
647 regex = re.compile("^PRIMARY_DISK")
648 return regex.search(s, 0)
651def getrootdev():
652 filename = '/etc/xensource-inventory'
653 try:
654 f = open(filename, 'r')
655 except:
656 raise xs_errors.XenError('EIO', \
657 opterr="Unable to open inventory file [%s]" % filename)
658 rootdev = ''
659 for line in filter(match_rootdev, f.readlines()):
660 rootdev = line.split("'")[1]
661 if not rootdev: 661 ↛ 662line 661 didn't jump to line 662, because the condition on line 661 was never true
662 raise xs_errors.XenError('NoRootDev')
663 return rootdev
666def getrootdevID():
667 rootdev = getrootdev()
668 try:
669 rootdevID = scsiutil.getSCSIid(rootdev)
670 except:
671 SMlog("util.getrootdevID: Unable to verify serial or SCSIid of device: %s" \
672 % rootdev)
673 return ''
675 if not len(rootdevID):
676 SMlog("util.getrootdevID: Unable to identify scsi device [%s] via scsiID" \
677 % rootdev)
679 return rootdevID
682def get_localAPI_session():
683 # First acquire a valid session
684 session = XenAPI.xapi_local()
685 try:
686 session.xenapi.login_with_password('root', '', '', 'SM')
687 except:
688 raise xs_errors.XenError('APISession')
689 return session
692def get_this_host():
693 uuid = None
694 f = open("/etc/xensource-inventory", 'r')
695 for line in f.readlines():
696 if line.startswith("INSTALLATION_UUID"):
697 uuid = line.split("'")[1]
698 f.close()
699 return uuid
702def get_master_ref(session):
703 pools = session.xenapi.pool.get_all()
704 return session.xenapi.pool.get_master(pools[0])
707def is_master(session):
708 return get_this_host_ref(session) == get_master_ref(session)
711def get_localhost_ref(session):
712 filename = '/etc/xensource-inventory'
713 try:
714 f = open(filename, 'r')
715 except:
716 raise xs_errors.XenError('EIO', \
717 opterr="Unable to open inventory file [%s]" % filename)
718 domid = ''
719 for line in filter(match_domain_id, f.readlines()):
720 domid = line.split("'")[1]
721 if not domid:
722 raise xs_errors.XenError('APILocalhost')
724 vms = session.xenapi.VM.get_all_records_where('field "uuid" = "%s"' % domid)
725 for vm in vms:
726 record = vms[vm]
727 if record["uuid"] == domid:
728 hostid = record["resident_on"]
729 return hostid
730 raise xs_errors.XenError('APILocalhost')
733def match_domain_id(s):
734 regex = re.compile("^CONTROL_DOMAIN_UUID")
735 return regex.search(s, 0)
738def get_hosts_attached_on(session, vdi_uuids):
739 host_refs = {}
740 for vdi_uuid in vdi_uuids:
741 try:
742 vdi_ref = session.xenapi.VDI.get_by_uuid(vdi_uuid)
743 except XenAPI.Failure:
744 SMlog("VDI %s not in db, ignoring" % vdi_uuid)
745 continue
746 sm_config = session.xenapi.VDI.get_sm_config(vdi_ref)
747 for key in [x for x in sm_config.keys() if x.startswith('host_')]:
748 host_refs[key[len('host_'):]] = True
749 return host_refs.keys()
751def get_this_host_address(session):
752 host_uuid = get_this_host()
753 host_ref = session.xenapi.host.get_by_uuid(host_uuid)
754 return session.xenapi.host.get_record(host_ref)['address']
756def get_host_addresses(session):
757 addresses = []
758 hosts = session.xenapi.host.get_all_records()
759 for record in hosts.values():
760 addresses.append(record['address'])
761 return addresses
763def get_this_host_ref(session):
764 host_uuid = get_this_host()
765 host_ref = session.xenapi.host.get_by_uuid(host_uuid)
766 return host_ref
769def get_slaves_attached_on(session, vdi_uuids):
770 "assume this host is the SR master"
771 host_refs = get_hosts_attached_on(session, vdi_uuids)
772 master_ref = get_this_host_ref(session)
773 return [x for x in host_refs if x != master_ref]
776def get_online_hosts(session):
777 online_hosts = []
778 hosts = session.xenapi.host.get_all_records()
779 for host_ref, host_rec in hosts.items():
780 metricsRef = host_rec["metrics"]
781 metrics = session.xenapi.host_metrics.get_record(metricsRef)
782 if metrics["live"]:
783 online_hosts.append(host_ref)
784 return online_hosts
787def get_all_slaves(session):
788 "assume this host is the SR master"
789 host_refs = get_online_hosts(session)
790 master_ref = get_this_host_ref(session)
791 return [x for x in host_refs if x != master_ref]
794def is_attached_rw(sm_config):
795 for key, val in sm_config.items():
796 if key.startswith("host_") and val == "RW":
797 return True
798 return False
801def attached_as(sm_config):
802 for key, val in sm_config.items():
803 if key.startswith("host_") and (val == "RW" or val == "RO"): 803 ↛ 804line 803 didn't jump to line 804, because the condition on line 803 was never true
804 return val
807def find_my_pbd_record(session, host_ref, sr_ref):
808 try:
809 pbds = session.xenapi.PBD.get_all_records()
810 for pbd_ref in pbds.keys():
811 if pbds[pbd_ref]['host'] == host_ref and pbds[pbd_ref]['SR'] == sr_ref:
812 return [pbd_ref, pbds[pbd_ref]]
813 return None
814 except Exception as e:
815 SMlog("Caught exception while looking up PBD for host %s SR %s: %s" % (str(host_ref), str(sr_ref), str(e)))
816 return None
819def find_my_pbd(session, host_ref, sr_ref):
820 ret = find_my_pbd_record(session, host_ref, sr_ref)
821 if ret is not None:
822 return ret[0]
823 else:
824 return None
827def test_hostPBD_devs(session, sr_uuid, devs):
828 host = get_localhost_ref(session)
829 sr = session.xenapi.SR.get_by_uuid(sr_uuid)
830 try:
831 pbds = session.xenapi.PBD.get_all_records()
832 except:
833 raise xs_errors.XenError('APIPBDQuery')
834 for dev in devs.split(','):
835 for pbd in pbds:
836 record = pbds[pbd]
837 # it's ok if it's *our* PBD
838 if record["SR"] == sr:
839 break
840 if record["host"] == host:
841 devconfig = record["device_config"]
842 if 'device' in devconfig:
843 for device in devconfig['device'].split(','):
844 if os.path.realpath(device) == os.path.realpath(dev):
845 return True
846 return False
849def test_hostPBD_lun(session, targetIQN, LUNid):
850 host = get_localhost_ref(session)
851 try:
852 pbds = session.xenapi.PBD.get_all_records()
853 except:
854 raise xs_errors.XenError('APIPBDQuery')
855 for pbd in pbds:
856 record = pbds[pbd]
857 if record["host"] == host:
858 devconfig = record["device_config"]
859 if 'targetIQN' in devconfig and 'LUNid' in devconfig:
860 if devconfig['targetIQN'] == targetIQN and \
861 devconfig['LUNid'] == LUNid:
862 return True
863 return False
866def test_SCSIid(session, sr_uuid, SCSIid):
867 if sr_uuid is not None:
868 sr = session.xenapi.SR.get_by_uuid(sr_uuid)
869 try:
870 pbds = session.xenapi.PBD.get_all_records()
871 except:
872 raise xs_errors.XenError('APIPBDQuery')
873 for pbd in pbds:
874 record = pbds[pbd]
875 # it's ok if it's *our* PBD
876 # During FC SR creation, devscan.py passes sr_uuid as None
877 if sr_uuid is not None:
878 if record["SR"] == sr:
879 break
880 devconfig = record["device_config"]
881 sm_config = session.xenapi.SR.get_sm_config(record["SR"])
882 if 'SCSIid' in devconfig and devconfig['SCSIid'] == SCSIid:
883 return True
884 elif 'SCSIid' in sm_config and sm_config['SCSIid'] == SCSIid:
885 return True
886 elif 'scsi-' + SCSIid in sm_config:
887 return True
888 return False
891class TimeoutException(SMException):
892 pass
895def timeout_call(timeoutseconds, function, *arguments):
896 def handler(signum, frame):
897 raise TimeoutException()
898 signal.signal(signal.SIGALRM, handler)
899 signal.alarm(timeoutseconds)
900 try:
901 return function(*arguments)
902 finally:
903 signal.alarm(0)
906def _incr_iscsiSR_refcount(targetIQN, uuid):
907 if not os.path.exists(ISCSI_REFDIR):
908 os.mkdir(ISCSI_REFDIR)
909 filename = os.path.join(ISCSI_REFDIR, targetIQN)
910 try:
911 f = open(filename, 'a+')
912 except:
913 raise xs_errors.XenError('LVMRefCount', \
914 opterr='file %s' % filename)
916 f.seek(0)
917 found = False
918 refcount = 0
919 for line in filter(match_uuid, f.readlines()):
920 refcount += 1
921 if line.find(uuid) != -1:
922 found = True
923 if not found:
924 f.write("%s\n" % uuid)
925 refcount += 1
926 f.close()
927 return refcount
930def _decr_iscsiSR_refcount(targetIQN, uuid):
931 filename = os.path.join(ISCSI_REFDIR, targetIQN)
932 if not os.path.exists(filename):
933 return 0
934 try:
935 f = open(filename, 'a+')
936 except:
937 raise xs_errors.XenError('LVMRefCount', \
938 opterr='file %s' % filename)
940 f.seek(0)
941 output = []
942 refcount = 0
943 for line in filter(match_uuid, f.readlines()):
944 if line.find(uuid) == -1:
945 output.append(line.rstrip())
946 refcount += 1
947 if not refcount:
948 os.unlink(filename)
949 return refcount
951 # Re-open file and truncate
952 f.close()
953 f = open(filename, 'w')
954 for i in range(0, refcount):
955 f.write("%s\n" % output[i])
956 f.close()
957 return refcount
960# The agent enforces 1 PBD per SR per host, so we
961# check for active SR entries not attached to this host
962def test_activePoolPBDs(session, host, uuid):
963 try:
964 pbds = session.xenapi.PBD.get_all_records()
965 except:
966 raise xs_errors.XenError('APIPBDQuery')
967 for pbd in pbds:
968 record = pbds[pbd]
969 if record["host"] != host and record["SR"] == uuid \
970 and record["currently_attached"]:
971 return True
972 return False
975def remove_mpathcount_field(session, host_ref, sr_ref, SCSIid):
976 try:
977 pbdref = find_my_pbd(session, host_ref, sr_ref)
978 if pbdref is not None:
979 key = "mpath-" + SCSIid
980 session.xenapi.PBD.remove_from_other_config(pbdref, key)
981 except:
982 pass
986def _testHost(hostname, port, errstring):
987 SMlog("_testHost: Testing host/port: %s,%d" % (hostname, port))
988 try:
989 sockinfo = socket.getaddrinfo(hostname, int(port))[0]
990 except:
991 logException('Exception occured getting IP for %s' % hostname)
992 raise xs_errors.XenError('DNSError')
994 timeout = 5
996 sock = socket.socket(sockinfo[0], socket.SOCK_STREAM)
997 # Only allow the connect to block for up to timeout seconds
998 sock.settimeout(timeout)
999 try:
1000 sock.connect(sockinfo[4])
1001 # Fix for MS storage server bug
1002 sock.send(b'\n')
1003 sock.close()
1004 except socket.error as reason:
1005 SMlog("_testHost: Connect failed after %d seconds (%s) - %s" \
1006 % (timeout, hostname, reason))
1007 raise xs_errors.XenError(errstring)
1010def match_scsiID(s, id):
1011 regex = re.compile(id)
1012 return regex.search(s, 0)
1015def _isSCSIid(s):
1016 regex = re.compile("^scsi-")
1017 return regex.search(s, 0)
1020def test_scsiserial(session, device):
1021 device = os.path.realpath(device)
1022 if not scsiutil._isSCSIdev(device):
1023 SMlog("util.test_scsiserial: Not a serial device: %s" % device)
1024 return False
1025 serial = ""
1026 try:
1027 serial += scsiutil.getserial(device)
1028 except:
1029 # Error allowed, SCSIid is the important one
1030 pass
1032 try:
1033 scsiID = scsiutil.getSCSIid(device)
1034 except:
1035 SMlog("util.test_scsiserial: Unable to verify serial or SCSIid of device: %s" \
1036 % device)
1037 return False
1038 if not len(scsiID):
1039 SMlog("util.test_scsiserial: Unable to identify scsi device [%s] via scsiID" \
1040 % device)
1041 return False
1043 try:
1044 SRs = session.xenapi.SR.get_all_records()
1045 except:
1046 raise xs_errors.XenError('APIFailure')
1047 for SR in SRs:
1048 record = SRs[SR]
1049 conf = record["sm_config"]
1050 if 'devserial' in conf:
1051 for dev in conf['devserial'].split(','):
1052 if _isSCSIid(dev):
1053 if match_scsiID(dev, scsiID):
1054 return True
1055 elif len(serial) and dev == serial:
1056 return True
1057 return False
1060def default(self, field, thunk):
1061 try:
1062 return getattr(self, field)
1063 except:
1064 return thunk()
1067def list_VDI_records_in_sr(sr):
1068 """Helper function which returns a list of all VDI records for this SR
1069 stored in the XenAPI server, useful for implementing SR.scan"""
1070 sr_ref = sr.session.xenapi.SR.get_by_uuid(sr.uuid)
1071 vdis = sr.session.xenapi.VDI.get_all_records_where("field \"SR\" = \"%s\"" % sr_ref)
1072 return vdis
1075# Given a partition (e.g. sda1), get a disk name:
1076def diskFromPartition(partition):
1077 # check whether this is a device mapper device (e.g. /dev/dm-0)
1078 m = re.match('(/dev/)?(dm-[0-9]+)(p[0-9]+)?$', partition)
1079 if m is not None: 1079 ↛ 1080line 1079 didn't jump to line 1080, because the condition on line 1079 was never true
1080 return m.group(2)
1082 numlen = 0 # number of digit characters
1083 m = re.match("\D+(\d+)", partition)
1084 if m is not None: 1084 ↛ 1085line 1084 didn't jump to line 1085, because the condition on line 1084 was never true
1085 numlen = len(m.group(1))
1087 # is it a cciss?
1088 if True in [partition.startswith(x) for x in ['cciss', 'ida', 'rd']]: 1088 ↛ 1089line 1088 didn't jump to line 1089, because the condition on line 1088 was never true
1089 numlen += 1 # need to get rid of trailing 'p'
1091 # is it a mapper path?
1092 if partition.startswith("mapper"): 1092 ↛ 1093line 1092 didn't jump to line 1093, because the condition on line 1092 was never true
1093 if re.search("p[0-9]*$", partition):
1094 numlen = len(re.match("\d+", partition[::-1]).group(0)) + 1
1095 SMlog("Found mapper part, len %d" % numlen)
1096 else:
1097 numlen = 0
1099 # is it /dev/disk/by-id/XYZ-part<k>?
1100 if partition.startswith("disk/by-id"): 1100 ↛ 1101line 1100 didn't jump to line 1101, because the condition on line 1100 was never true
1101 return partition[:partition.rfind("-part")]
1103 return partition[:len(partition) - numlen]
1106def dom0_disks():
1107 """Disks carrying dom0, e.g. ['/dev/sda']"""
1108 disks = []
1109 with open("/etc/mtab", 'r') as f:
1110 for line in f:
1111 (dev, mountpoint, fstype, opts, freq, passno) = line.split(' ')
1112 if mountpoint == '/':
1113 disk = diskFromPartition(dev)
1114 if not (disk in disks):
1115 disks.append(disk)
1116 SMlog("Dom0 disks: %s" % disks)
1117 return disks
1120def set_scheduler_sysfs_node(node, scheds):
1121 """
1122 Set the scheduler for a sysfs node (e.g. '/sys/block/sda')
1123 according to prioritized list schedulers
1124 Try to set the first item, then fall back to the next on failure
1125 """
1127 path = os.path.join(node, "queue", "scheduler")
1128 if not os.path.exists(path): 1128 ↛ 1132line 1128 didn't jump to line 1132, because the condition on line 1128 was never false
1129 SMlog("no path %s" % path)
1130 return
1132 stored_error = None
1133 for sched in scheds:
1134 try:
1135 with open(path, 'w') as file:
1136 file.write("%s\n" % sched)
1137 SMlog("Set scheduler to [%s] on [%s]" % (sched, node))
1138 return
1139 except (OSError, IOError) as err:
1140 stored_error = err
1142 SMlog("Error setting schedulers to [%s] on [%s], %s" % (scheds, node, str(stored_error)))
1145def set_scheduler(dev, schedulers=None):
1146 if schedulers is None: 1146 ↛ 1149line 1146 didn't jump to line 1149, because the condition on line 1146 was never false
1147 schedulers = ["none", "noop"]
1149 devices = []
1150 if not scsiutil.match_dm(dev): 1150 ↛ 1154line 1150 didn't jump to line 1154, because the condition on line 1150 was never false
1151 # Remove partition numbers
1152 devices.append(diskFromPartition(dev).replace('/', '!'))
1153 else:
1154 rawdev = diskFromPartition(dev)
1155 devices = [os.path.realpath(x)[5:] for x in scsiutil._genReverseSCSIidmap(rawdev.split('/')[-1])]
1157 for d in devices:
1158 set_scheduler_sysfs_node("/sys/block/%s" % d, schedulers)
1161# This function queries XAPI for the existing VDI records for this SR
1162def _getVDIs(srobj):
1163 VDIs = []
1164 try:
1165 sr_ref = getattr(srobj, 'sr_ref')
1166 except AttributeError:
1167 return VDIs
1169 refs = srobj.session.xenapi.SR.get_VDIs(sr_ref)
1170 for vdi in refs:
1171 ref = srobj.session.xenapi.VDI.get_record(vdi)
1172 ref['vdi_ref'] = vdi
1173 VDIs.append(ref)
1174 return VDIs
1177def _getVDI(srobj, vdi_uuid):
1178 vdi = srobj.session.xenapi.VDI.get_by_uuid(vdi_uuid)
1179 ref = srobj.session.xenapi.VDI.get_record(vdi)
1180 ref['vdi_ref'] = vdi
1181 return ref
1184def _convertDNS(name):
1185 addr = socket.getaddrinfo(name, None)[0][4][0]
1186 return addr
1189def _containsVDIinuse(srobj):
1190 VDIs = _getVDIs(srobj)
1191 for vdi in VDIs:
1192 if not vdi['managed']:
1193 continue
1194 sm_config = vdi['sm_config']
1195 if 'SRRef' in sm_config:
1196 try:
1197 PBDs = srobj.session.xenapi.SR.get_PBDs(sm_config['SRRef'])
1198 for pbd in PBDs:
1199 record = PBDs[pbd]
1200 if record["host"] == srobj.host_ref and \
1201 record["currently_attached"]:
1202 return True
1203 except:
1204 pass
1205 return False
1208def isVDICommand(cmd):
1209 if cmd is None or cmd in ["vdi_attach", "vdi_detach",
1210 "vdi_activate", "vdi_deactivate",
1211 "vdi_epoch_begin", "vdi_epoch_end"]:
1212 return True
1213 else:
1214 return False
1217#########################
1218# Daemon helper functions
1219def p_id_fork():
1220 try:
1221 p_id = os.fork()
1222 except OSError as e:
1223 print("Fork failed: %s (%d)" % (e.strerror, e.errno))
1224 sys.exit(-1)
1226 if (p_id == 0):
1227 os.setsid()
1228 try:
1229 p_id = os.fork()
1230 except OSError as e:
1231 print("Fork failed: %s (%d)" % (e.strerror, e.errno))
1232 sys.exit(-1)
1233 if (p_id == 0):
1234 os.chdir('/opt/xensource/sm')
1235 os.umask(0)
1236 else:
1237 os._exit(0)
1238 else:
1239 os._exit(0)
1242def daemon():
1243 p_id_fork()
1244 # Query the max file descriptor parameter for this process
1245 maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1247 # Close any fds that are open
1248 for fd in range(0, maxfd):
1249 try:
1250 os.close(fd)
1251 except:
1252 pass
1254 # Redirect STDIN to STDOUT and STDERR
1255 os.open('/dev/null', os.O_RDWR)
1256 os.dup2(0, 1)
1257 os.dup2(0, 2)
1259################################################################################
1260#
1261# Fist points
1262#
1264# * The global variable 'fistpoint' define the list of all possible fistpoints;
1265#
1266# * To activate a fistpoint called 'name', you need to create the file '/tmp/fist_name'
1267# on the SR master;
1268#
1269# * At the moment, activating a fist point can lead to two possible behaviors:
1270# - if '/tmp/fist_LVHDRT_exit' exists, then the function called during the fistpoint is _exit;
1271# - otherwise, the function called is _pause.
1273def _pause(secs, name):
1274 SMlog("Executing fist point %s: sleeping %d seconds ..." % (name, secs))
1275 time.sleep(secs)
1276 SMlog("Executing fist point %s: done" % name)
1279def _exit(name):
1280 SMlog("Executing fist point %s: exiting the current process ..." % name)
1281 raise xs_errors.XenError('FistPoint', opterr='%s' % name)
1284class FistPoint:
1285 def __init__(self, points):
1286 #SMlog("Fist points loaded")
1287 self.points = points
1289 def is_legal(self, name):
1290 return (name in self.points)
1292 def is_active(self, name):
1293 return os.path.exists("/tmp/fist_%s" % name)
1295 def mark_sr(self, name, sruuid, started):
1296 session = get_localAPI_session()
1297 try:
1298 sr = session.xenapi.SR.get_by_uuid(sruuid)
1300 if started:
1301 session.xenapi.SR.add_to_other_config(sr, name, "active")
1302 else:
1303 session.xenapi.SR.remove_from_other_config(sr, name)
1304 finally:
1305 session.xenapi.session.logout()
1307 def activate(self, name, sruuid):
1308 if name in self.points:
1309 if self.is_active(name):
1310 self.mark_sr(name, sruuid, True)
1311 if self.is_active("LVHDRT_exit"): 1311 ↛ 1312line 1311 didn't jump to line 1312, because the condition on line 1311 was never true
1312 self.mark_sr(name, sruuid, False)
1313 _exit(name)
1314 else:
1315 _pause(FIST_PAUSE_PERIOD, name)
1316 self.mark_sr(name, sruuid, False)
1317 else:
1318 SMlog("Unknown fist point: %s" % name)
1320 def activate_custom_fn(self, name, fn):
1321 if name in self.points: 1321 ↛ 1327line 1321 didn't jump to line 1327, because the condition on line 1321 was never false
1322 if self.is_active(name): 1322 ↛ 1323line 1322 didn't jump to line 1323, because the condition on line 1322 was never true
1323 SMlog("Executing fist point %s: starting ..." % name)
1324 fn()
1325 SMlog("Executing fist point %s: done" % name)
1326 else:
1327 SMlog("Unknown fist point: %s" % name)
1330def list_find(f, seq):
1331 for item in seq:
1332 if f(item):
1333 return item
1335GCPAUSE_FISTPOINT = "GCLoop_no_pause"
1337fistpoint = FistPoint(["LVHDRT_finding_a_suitable_pair",
1338 "LVHDRT_inflating_the_parent",
1339 "LVHDRT_resizing_while_vdis_are_paused",
1340 "LVHDRT_coalescing_VHD_data",
1341 "LVHDRT_coalescing_before_inflate_grandparent",
1342 "LVHDRT_relinking_grandchildren",
1343 "LVHDRT_before_create_relink_journal",
1344 "LVHDRT_xapiSM_serialization_tests",
1345 "LVHDRT_clone_vdi_after_create_journal",
1346 "LVHDRT_clone_vdi_after_shrink_parent",
1347 "LVHDRT_clone_vdi_after_first_snap",
1348 "LVHDRT_clone_vdi_after_second_snap",
1349 "LVHDRT_clone_vdi_after_parent_hidden",
1350 "LVHDRT_clone_vdi_after_parent_ro",
1351 "LVHDRT_clone_vdi_before_remove_journal",
1352 "LVHDRT_clone_vdi_after_lvcreate",
1353 "LVHDRT_clone_vdi_before_undo_clone",
1354 "LVHDRT_clone_vdi_after_undo_clone",
1355 "LVHDRT_inflate_after_create_journal",
1356 "LVHDRT_inflate_after_setSize",
1357 "LVHDRT_inflate_after_zeroOut",
1358 "LVHDRT_inflate_after_setSizePhys",
1359 "LVHDRT_inflate_after_setSizePhys",
1360 "LVHDRT_coaleaf_before_coalesce",
1361 "LVHDRT_coaleaf_after_coalesce",
1362 "LVHDRT_coaleaf_one_renamed",
1363 "LVHDRT_coaleaf_both_renamed",
1364 "LVHDRT_coaleaf_after_vdirec",
1365 "LVHDRT_coaleaf_before_delete",
1366 "LVHDRT_coaleaf_after_delete",
1367 "LVHDRT_coaleaf_before_remove_j",
1368 "LVHDRT_coaleaf_undo_after_rename",
1369 "LVHDRT_coaleaf_undo_after_rename2",
1370 "LVHDRT_coaleaf_undo_after_refcount",
1371 "LVHDRT_coaleaf_undo_after_deflate",
1372 "LVHDRT_coaleaf_undo_end",
1373 "LVHDRT_coaleaf_stop_after_recovery",
1374 "LVHDRT_coaleaf_finish_after_inflate",
1375 "LVHDRT_coaleaf_finish_end",
1376 "LVHDRT_coaleaf_delay_1",
1377 "LVHDRT_coaleaf_delay_2",
1378 "LVHDRT_coaleaf_delay_3",
1379 "testsm_clone_allow_raw",
1380 "xenrt_default_vdi_type_legacy",
1381 "blktap_activate_inject_failure",
1382 "blktap_activate_error_handling",
1383 GCPAUSE_FISTPOINT,
1384 "cleanup_coalesceVHD_inject_failure",
1385 "cleanup_tracker_no_progress",
1386 "FileSR_fail_hardlink",
1387 "FileSR_fail_snap1",
1388 "FileSR_fail_snap2",
1389 "LVM_journaler_exists",
1390 "LVM_journaler_none",
1391 "LVM_journaler_badname",
1392 "LVM_journaler_readfail",
1393 "LVM_journaler_writefail"])
1396def set_dirty(session, sr):
1397 try:
1398 session.xenapi.SR.add_to_other_config(sr, "dirty", "")
1399 SMlog("set_dirty %s succeeded" % (repr(sr)))
1400 except:
1401 SMlog("set_dirty %s failed (flag already set?)" % (repr(sr)))
1404def doesFileHaveOpenHandles(fileName):
1405 SMlog("Entering doesFileHaveOpenHandles with file: %s" % fileName)
1406 (retVal, processAndPidTuples) = \
1407 findRunningProcessOrOpenFile(fileName, False)
1409 if not retVal:
1410 SMlog("Failed to determine if file %s has open handles." % \
1411 fileName)
1412 # err on the side of caution
1413 return True
1414 else:
1415 if len(processAndPidTuples) > 0:
1416 return True
1417 else:
1418 return False
1421# extract SR uuid from the passed in devmapper entry and return
1422# /dev/mapper/VG_XenStorage--c3d82e92--cb25--c99b--b83a--482eebab4a93-MGT
1423def extractSRFromDevMapper(path):
1424 try:
1425 path = os.path.basename(path)
1426 path = path[len('VG_XenStorage-') + 1:]
1427 path = path.replace('--', '/')
1428 path = path[0:path.rfind('-')]
1429 return path.replace('/', '-')
1430 except:
1431 return ''
1434def pid_is_alive(pid):
1435 """
1436 Try to kill PID with signal 0.
1437 If we succeed, the PID is alive, so return True.
1438 If we get an EPERM error, the PID is alive but we are not allowed to
1439 signal it. Still return true.
1440 Any other error (e.g. ESRCH), return False
1441 """
1442 try:
1443 os.kill(pid, 0)
1444 return True
1445 except OSError as e:
1446 if e.errno == errno.EPERM:
1447 return True
1448 return False
1451# Looks at /proc and figures either
1452# If a process is still running (default), returns open file names
1453# If any running process has open handles to the given file (process = False)
1454# returns process names and pids
1455def findRunningProcessOrOpenFile(name, process=True):
1456 retVal = True
1457 links = []
1458 processandpids = []
1459 sockets = set()
1460 try:
1461 SMlog("Entering findRunningProcessOrOpenFile with params: %s" % \
1462 [name, process])
1464 # Look at all pids
1465 pids = [pid for pid in os.listdir('/proc') if pid.isdigit()]
1466 for pid in sorted(pids):
1467 try:
1468 try:
1469 f = None
1470 f = open(os.path.join('/proc', pid, 'cmdline'), 'r')
1471 prog = f.read()[:-1]
1472 if prog: 1472 ↛ 1481line 1472 didn't jump to line 1481, because the condition on line 1472 was never false
1473 # Just want the process name
1474 argv = prog.split('\x00')
1475 prog = argv[0]
1476 except IOError as e:
1477 if e.errno in (errno.ENOENT, errno.ESRCH):
1478 SMlog("ERROR %s reading %s, ignore" % (e.errno, pid))
1479 continue
1480 finally:
1481 if f is not None: 1481 ↛ 1466, 1481 ↛ 14842 missed branches: 1) line 1481 didn't jump to line 1466, because the continue on line 1479 wasn't executed, 2) line 1481 didn't jump to line 1484, because the condition on line 1481 was never false
1482 f.close() 1482 ↛ 1466line 1482 didn't jump to line 1466, because the continue on line 1479 wasn't executed
1484 try:
1485 fd_dir = os.path.join('/proc', pid, 'fd')
1486 files = os.listdir(fd_dir)
1487 except OSError as e:
1488 if e.errno in (errno.ENOENT, errno.ESRCH):
1489 SMlog("ERROR %s reading fds for %s, ignore" % (e.errno, pid))
1490 # Ignore pid that are no longer valid
1491 continue
1492 else:
1493 raise
1495 for file in files:
1496 try:
1497 link = os.readlink(os.path.join(fd_dir, file))
1498 except OSError:
1499 continue
1501 if process: 1501 ↛ 1506line 1501 didn't jump to line 1506, because the condition on line 1501 was never false
1502 if name == prog: 1502 ↛ 1495line 1502 didn't jump to line 1495, because the condition on line 1502 was never false
1503 links.append(link)
1504 else:
1505 # need to return process name and pid tuples
1506 if link == name:
1507 processandpids.append((prog, pid))
1509 # Get the connected sockets
1510 if name == prog:
1511 sockets.update(get_connected_sockets(pid))
1513 # We will only have a non-empty processandpids if some fd entries were found.
1514 # Before returning them, verify that all the PIDs in question are properly alive.
1515 # There is no specific guarantee of when a PID's /proc directory will disappear
1516 # when it exits, particularly relative to filedescriptor cleanup, so we want to
1517 # make sure we're not reporting a false positive.
1518 processandpids = [x for x in processandpids if pid_is_alive(int(x[1]))]
1519 for pp in processandpids: 1519 ↛ 1520line 1519 didn't jump to line 1520, because the loop on line 1519 never started
1520 SMlog(f"File {name} has an open handle with process {pp[0]} with pid {pp[1]}")
1522 except Exception as e:
1523 SMlog("Exception checking running process or open file handles. " \
1524 "Error: %s" % str(e))
1525 retVal = False
1527 if process: 1527 ↛ 1530line 1527 didn't jump to line 1530, because the condition on line 1527 was never false
1528 return retVal, links, sockets
1529 else:
1530 return retVal, processandpids
1533def get_connected_sockets(pid):
1534 sockets = set()
1535 try:
1536 # Lines in /proc/<pid>/net/unix are formatted as follows
1537 # (see Linux source net/unix/af_unix.c, unix_seq_show() )
1538 # - Pointer address to socket (hex)
1539 # - Refcount (HEX)
1540 # - 0
1541 # - State (HEX, 0 or __SO_ACCEPTCON)
1542 # - Type (HEX - but only 0001 of interest)
1543 # - Connection state (HEX - but only 03, SS_CONNECTED of interest)
1544 # - Inode number
1545 # - Path (optional)
1546 open_sock_matcher = re.compile(
1547 r'^[0-9a-f]+: [0-9A-Fa-f]+ [0-9A-Fa-f]+ [0-9A-Fa-f]+ 0001 03 \d+ (.*)$')
1548 with open(
1549 os.path.join('/proc', str(pid), 'net', 'unix'), 'r') as f:
1550 lines = f.readlines()
1551 for line in lines:
1552 match = open_sock_matcher.match(line)
1553 if match:
1554 sockets.add(match[1])
1555 except OSError as e:
1556 if e.errno in (errno.ENOENT, errno.ESRCH):
1557 # Ignore pid that are no longer valid
1558 SMlog("ERROR %s reading sockets for %s, ignore" %
1559 (e.errno, pid))
1560 else:
1561 raise
1562 return sockets
1565def retry(f, maxretry=20, period=3, exceptions=[Exception]):
1566 retries = 0
1567 while True:
1568 try:
1569 return f()
1570 except Exception as e:
1571 for exception in exceptions:
1572 if isinstance(e, exception):
1573 SMlog('Got exception: {}. Retry number: {}'.format(
1574 str(e), retries
1575 ))
1576 break
1577 else:
1578 SMlog('Got bad exception: {}. Raising...'.format(e))
1579 raise e
1581 retries += 1
1582 if retries >= maxretry:
1583 break
1585 time.sleep(period)
1587 return f()
1590def getCslDevPath(svid):
1591 basepath = "/dev/disk/by-csldev/"
1592 if svid.startswith("NETAPP_"):
1593 # special attention for NETAPP SVIDs
1594 svid_parts = svid.split("__")
1595 globstr = basepath + "NETAPP__LUN__" + "*" + svid_parts[2] + "*" + svid_parts[-1] + "*"
1596 else:
1597 globstr = basepath + svid + "*"
1599 return globstr
1602# Use device in /dev pointed to by cslg path which consists of svid
1603def get_scsiid_from_svid(md_svid):
1604 cslg_path = getCslDevPath(md_svid)
1605 abs_path = glob.glob(cslg_path)
1606 if abs_path:
1607 real_path = os.path.realpath(abs_path[0])
1608 return scsiutil.getSCSIid(real_path)
1609 else:
1610 return None
1613def get_isl_scsiids(session):
1614 # Get cslg type SRs
1615 SRs = session.xenapi.SR.get_all_records_where('field "type" = "cslg"')
1617 # Iterate through the SR to get the scsi ids
1618 scsi_id_ret = []
1619 for SR in SRs:
1620 sr_rec = SRs[SR]
1621 # Use the md_svid to get the scsi id
1622 scsi_id = get_scsiid_from_svid(sr_rec['sm_config']['md_svid'])
1623 if scsi_id:
1624 scsi_id_ret.append(scsi_id)
1626 # Get the vdis in the SR and do the same procedure
1627 vdi_recs = session.xenapi.VDI.get_all_records_where('field "SR" = "%s"' % SR)
1628 for vdi_rec in vdi_recs:
1629 vdi_rec = vdi_recs[vdi_rec]
1630 scsi_id = get_scsiid_from_svid(vdi_rec['sm_config']['SVID'])
1631 if scsi_id:
1632 scsi_id_ret.append(scsi_id)
1634 return scsi_id_ret
1637class extractXVA:
1638 # streams files as a set of file and checksum, caller should remove
1639 # the files, if not needed. The entire directory (Where the files
1640 # and checksum) will only be deleted as part of class cleanup.
1641 HDR_SIZE = 512
1642 BLOCK_SIZE = 512
1643 SIZE_LEN = 12 - 1 # To remove \0 from tail
1644 SIZE_OFFSET = 124
1645 ZERO_FILLED_REC = 2
1646 NULL_IDEN = '\x00'
1647 DIR_IDEN = '/'
1648 CHECKSUM_IDEN = '.checksum'
1649 OVA_FILE = 'ova.xml'
1651 # Init gunzips the file using a subprocess, and reads stdout later
1652 # as and when needed
1653 def __init__(self, filename):
1654 self.__extract_path = ''
1655 self.__filename = filename
1656 cmd = 'gunzip -cd %s' % filename
1657 try:
1658 self.spawn_p = subprocess.Popen(
1659 cmd, shell=True, \
1660 stdin=subprocess.PIPE, stdout=subprocess.PIPE, \
1661 stderr=subprocess.PIPE, close_fds=True)
1662 except Exception as e:
1663 SMlog("Error: %s. Uncompress failed for %s" % (str(e), filename))
1664 raise Exception(str(e))
1666 # Create dir to extract the files
1667 self.__extract_path = tempfile.mkdtemp()
1669 def __del__(self):
1670 shutil.rmtree(self.__extract_path)
1672 # Class supports Generator expression. 'for f_name, checksum in getTuple()'
1673 # returns filename, checksum content. Returns filename, '' in case
1674 # of checksum file missing. e.g. ova.xml
1675 def getTuple(self):
1676 zerod_record = 0
1677 ret_f_name = ''
1678 ret_base_f_name = ''
1680 try:
1681 # Read tar file as sets of file and checksum.
1682 while True:
1683 # Read the output of spawned process, or output of gunzip
1684 f_hdr = self.spawn_p.stdout.read(self.HDR_SIZE)
1686 # Break out in case of end of file
1687 if f_hdr == '':
1688 if zerod_record == extractXVA.ZERO_FILLED_REC:
1689 break
1690 else:
1691 SMlog('Error. Expects %d zero records', \
1692 extractXVA.ZERO_FILLED_REC)
1693 raise Exception('Unrecognized end of file')
1695 # Watch out for zero records, two zero records
1696 # denote end of file.
1697 if f_hdr == extractXVA.NULL_IDEN * extractXVA.HDR_SIZE:
1698 zerod_record += 1
1699 continue
1701 f_name = f_hdr[:f_hdr.index(extractXVA.NULL_IDEN)]
1702 # File header may be for a folder, if so ignore the header
1703 if not f_name.endswith(extractXVA.DIR_IDEN):
1704 f_size_octal = f_hdr[extractXVA.SIZE_OFFSET: \
1705 extractXVA.SIZE_OFFSET + extractXVA.SIZE_LEN]
1706 f_size = int(f_size_octal, 8)
1707 if f_name.endswith(extractXVA.CHECKSUM_IDEN):
1708 if f_name.rstrip(extractXVA.CHECKSUM_IDEN) == \
1709 ret_base_f_name:
1710 checksum = self.spawn_p.stdout.read(f_size)
1711 yield(ret_f_name, checksum)
1712 else:
1713 # Expects file followed by its checksum
1714 SMlog('Error. Sequence mismatch starting with %s', \
1715 ret_f_name)
1716 raise Exception( \
1717 'Files out of sequence starting with %s', \
1718 ret_f_name)
1719 else:
1720 # In case of ova.xml, read the contents into a file and
1721 # return the file name to the caller. For other files,
1722 # read the contents into a file, it will
1723 # be used when a .checksum file is encountered.
1724 ret_f_name = '%s/%s' % (self.__extract_path, f_name)
1725 ret_base_f_name = f_name
1727 # Check if the folder exists on the target location,
1728 # else create it.
1729 folder_path = ret_f_name[:ret_f_name.rfind('/')]
1730 if not os.path.exists(folder_path):
1731 os.mkdir(folder_path)
1733 # Store the file to the tmp folder, strip the tail \0
1734 f = open(ret_f_name, 'w')
1735 f.write(self.spawn_p.stdout.read(f_size))
1736 f.close()
1737 if f_name == extractXVA.OVA_FILE:
1738 yield(ret_f_name, '')
1740 # Skip zero'd portion of data block
1741 round_off = f_size % extractXVA.BLOCK_SIZE
1742 if round_off != 0:
1743 zeros = self.spawn_p.stdout.read(
1744 extractXVA.BLOCK_SIZE - round_off)
1745 except Exception as e:
1746 SMlog("Error: %s. File set extraction failed %s" % (str(e), \
1747 self.__filename))
1749 # Kill and Drain stdout of the gunzip process,
1750 # else gunzip might block on stdout
1751 os.kill(self.spawn_p.pid, signal.SIGTERM)
1752 self.spawn_p.communicate()
1753 raise Exception(str(e))
1755illegal_xml_chars = [(0x00, 0x08), (0x0B, 0x1F), (0x7F, 0x84), (0x86, 0x9F),
1756 (0xD800, 0xDFFF), (0xFDD0, 0xFDDF), (0xFFFE, 0xFFFF),
1757 (0x1FFFE, 0x1FFFF), (0x2FFFE, 0x2FFFF), (0x3FFFE, 0x3FFFF),
1758 (0x4FFFE, 0x4FFFF), (0x5FFFE, 0x5FFFF), (0x6FFFE, 0x6FFFF),
1759 (0x7FFFE, 0x7FFFF), (0x8FFFE, 0x8FFFF), (0x9FFFE, 0x9FFFF),
1760 (0xAFFFE, 0xAFFFF), (0xBFFFE, 0xBFFFF), (0xCFFFE, 0xCFFFF),
1761 (0xDFFFE, 0xDFFFF), (0xEFFFE, 0xEFFFF), (0xFFFFE, 0xFFFFF),
1762 (0x10FFFE, 0x10FFFF)]
1764illegal_ranges = ["%s-%s" % (chr(low), chr(high))
1765 for (low, high) in illegal_xml_chars
1766 if low < sys.maxunicode]
1768illegal_xml_re = re.compile(u'[%s]' % u''.join(illegal_ranges))
1771def isLegalXMLString(s):
1772 """Tells whether this is a valid XML string (i.e. it does not contain
1773 illegal XML characters specified in
1774 http://www.w3.org/TR/2004/REC-xml-20040204/#charsets).
1775 """
1777 if len(s) > 0:
1778 return re.search(illegal_xml_re, s) is None
1779 else:
1780 return True
1783def unictrunc(string, max_bytes):
1784 """
1785 Given a string, returns the largest number of elements for a prefix
1786 substring of it, such that the UTF-8 encoding of this substring takes no
1787 more than the given number of bytes.
1789 The string may be given as a unicode string or a UTF-8 encoded byte
1790 string, and the number returned will be in characters or bytes
1791 accordingly. Note that in the latter case, the substring will still be a
1792 valid UTF-8 encoded string (which is to say, it won't have been truncated
1793 part way through a multibyte sequence for a unicode character).
1795 string: the string to truncate
1796 max_bytes: the maximum number of bytes the truncated string can be
1797 """
1798 if isinstance(string, str):
1799 return_chars = True
1800 else:
1801 return_chars = False
1802 string = string.decode('UTF-8')
1804 cur_chars = 0
1805 cur_bytes = 0
1806 for char in string:
1807 charsize = len(char.encode('UTF-8'))
1808 if cur_bytes + charsize > max_bytes:
1809 break
1810 else:
1811 cur_chars += 1
1812 cur_bytes += charsize
1813 return cur_chars if return_chars else cur_bytes
1816def hideValuesInPropMap(propmap, propnames):
1817 """
1818 Worker function: input simple map of prop name/value pairs, and
1819 a list of specific propnames whose values we want to hide.
1820 Loop through the "hide" list, and if any are found, hide the
1821 value and return the altered map.
1822 If none found, return the original map
1823 """
1824 matches = []
1825 for propname in propnames:
1826 if propname in propmap: 1826 ↛ 1827line 1826 didn't jump to line 1827, because the condition on line 1826 was never true
1827 matches.append(propname)
1829 if matches: 1829 ↛ 1830line 1829 didn't jump to line 1830, because the condition on line 1829 was never true
1830 deepCopyRec = copy.deepcopy(propmap)
1831 for match in matches:
1832 deepCopyRec[match] = '******'
1833 return deepCopyRec
1835 return propmap
1836# define the list of propnames whose value we want to hide
1838PASSWD_PROP_KEYS = ['password', 'cifspassword', 'chappassword', 'incoming_chappassword']
1839DEFAULT_SEGMENT_LEN = 950
1842def hidePasswdInConfig(config):
1843 """
1844 Function to hide passwd values in a simple prop map,
1845 for example "device_config"
1846 """
1847 return hideValuesInPropMap(config, PASSWD_PROP_KEYS)
1850def hidePasswdInParams(params, configProp):
1851 """
1852 Function to hide password values in a specified property which
1853 is a simple map of prop name/values, and is itself an prop entry
1854 in a larger property map.
1855 For example, param maps containing "device_config", or
1856 "sm_config", etc
1857 """
1858 params[configProp] = hideValuesInPropMap(params[configProp], PASSWD_PROP_KEYS)
1859 return params
1862def hideMemberValuesInXmlParams(xmlParams, propnames=PASSWD_PROP_KEYS):
1863 """
1864 Function to hide password values in XML params, specifically
1865 for the XML format of incoming params to SR modules.
1866 Uses text parsing: loop through the list of specific propnames
1867 whose values we want to hide, and:
1868 - Assemble a full "prefix" containing each property name, e.g.,
1869 "<member><name>password</name><value>"
1870 - Test the XML if it contains that string, save the index.
1871 - If found, get the index of the ending tag
1872 - Truncate the return string starting with the password value.
1873 - Append the substitute "*******" value string.
1874 - Restore the rest of the original string starting with the end tag.
1875 """
1876 findStrPrefixHead = "<member><name>"
1877 findStrPrefixTail = "</name><value>"
1878 findStrSuffix = "</value>"
1879 strlen = len(xmlParams)
1881 for propname in propnames:
1882 findStrPrefix = findStrPrefixHead + propname + findStrPrefixTail
1883 idx = xmlParams.find(findStrPrefix)
1884 if idx != -1: # if found any of them
1885 idx += len(findStrPrefix)
1886 idx2 = xmlParams.find(findStrSuffix, idx)
1887 if idx2 != -1:
1888 retStr = xmlParams[0:idx]
1889 retStr += "******"
1890 retStr += xmlParams[idx2:strlen]
1891 return retStr
1892 else:
1893 return xmlParams
1894 return xmlParams
1897def splitXmlText(xmlData, segmentLen=DEFAULT_SEGMENT_LEN, showContd=False):
1898 """
1899 Split xml string data into substrings small enough for the
1900 syslog line length limit. Split at tag end markers ( ">" ).
1901 Usage:
1902 strList = []
1903 strList = splitXmlText( longXmlText, maxLineLen ) # maxLineLen is optional
1904 """
1905 remainingData = str(xmlData)
1907 # "Un-pretty-print"
1908 remainingData = remainingData.replace('\n', '')
1909 remainingData = remainingData.replace('\t', '')
1911 remainingChars = len(remainingData)
1912 returnData = ''
1914 thisLineNum = 0
1915 while remainingChars > segmentLen:
1916 thisLineNum = thisLineNum + 1
1917 index = segmentLen
1918 tmpStr = remainingData[:segmentLen]
1919 tmpIndex = tmpStr.rfind('>')
1920 if tmpIndex != -1:
1921 index = tmpIndex + 1
1923 tmpStr = tmpStr[:index]
1924 remainingData = remainingData[index:]
1925 remainingChars = len(remainingData)
1927 if showContd:
1928 if thisLineNum != 1:
1929 tmpStr = '(Cont\'d): ' + tmpStr
1930 tmpStr = tmpStr + ' (Cont\'d):'
1932 returnData += tmpStr + '\n'
1934 if showContd and thisLineNum > 0:
1935 remainingData = '(Cont\'d): ' + remainingData
1936 returnData += remainingData
1938 return returnData
1941def inject_failure():
1942 raise Exception('injected failure')
1945def open_atomic(path, mode=None):
1946 """Atomically creates a file if, and only if it does not already exist.
1947 Leaves the file open and returns the file object.
1949 path: the path to atomically open
1950 mode: "r" (read), "w" (write), or "rw" (read/write)
1951 returns: an open file object"""
1953 assert path
1955 flags = os.O_CREAT | os.O_EXCL
1956 modes = {'r': os.O_RDONLY, 'w': os.O_WRONLY, 'rw': os.O_RDWR}
1957 if mode:
1958 if mode not in modes:
1959 raise Exception('invalid access mode ' + mode)
1960 flags |= modes[mode]
1961 fd = os.open(path, flags)
1962 try:
1963 if mode:
1964 return os.fdopen(fd, mode)
1965 else:
1966 return os.fdopen(fd)
1967 except:
1968 os.close(fd)
1969 raise
1972def isInvalidVDI(exception):
1973 return exception.details[0] == "HANDLE_INVALID" or \
1974 exception.details[0] == "UUID_INVALID"
1977def get_pool_restrictions(session):
1978 """Returns pool restrictions as a map, @session must be already
1979 established."""
1980 return list(session.xenapi.pool.get_all_records().values())[0]['restrictions']
1983def read_caching_is_restricted(session):
1984 """Tells whether read caching is restricted."""
1985 if session is None: 1985 ↛ 1986line 1985 didn't jump to line 1986, because the condition on line 1985 was never true
1986 return True
1987 restrictions = get_pool_restrictions(session)
1988 if 'restrict_read_caching' in restrictions and \ 1988 ↛ 1990line 1988 didn't jump to line 1990, because the condition on line 1988 was never true
1989 restrictions['restrict_read_caching'] == "true":
1990 return True
1991 return False
1994def sessions_less_than_targets(other_config, device_config):
1995 if 'multihomelist' in device_config and 'iscsi_sessions' in other_config:
1996 sessions = int(other_config['iscsi_sessions'])
1997 targets = len(device_config['multihomelist'].split(','))
1998 SMlog("Targets %d and iscsi_sessions %d" % (targets, sessions))
1999 return (sessions < targets)
2000 else:
2001 return False
2004def enable_and_start_service(name, start):
2005 attempt = 0
2006 while True:
2007 attempt += 1
2008 fn = 'enable' if start else 'disable'
2009 args = ('systemctl', fn, '--now', name)
2010 (ret, out, err) = doexec(args)
2011 if ret == 0:
2012 return
2013 elif attempt >= 3:
2014 raise Exception(
2015 'Failed to {} {}: {} {}'.format(fn, name, out, err)
2016 )
2017 time.sleep(1)
2020def stop_service(name):
2021 args = ('systemctl', 'stop', name)
2022 (ret, out, err) = doexec(args)
2023 if ret == 0:
2024 return
2025 raise Exception('Failed to stop {}: {} {}'.format(name, out, err))
2028def restart_service(name):
2029 attempt = 0
2030 while True:
2031 attempt += 1
2032 SMlog('Restarting service {} {}...'.format(name, attempt))
2033 args = ('systemctl', 'restart', name)
2034 (ret, out, err) = doexec(args)
2035 if ret == 0:
2036 return
2037 elif attempt >= 3:
2038 SMlog('Restart service FAILED {} {}'.format(name, attempt))
2039 raise Exception(
2040 'Failed to restart {}: {} {}'.format(name, out, err)
2041 )
2042 time.sleep(1)
2045def check_pid_exists(pid):
2046 try:
2047 os.kill(pid, 0)
2048 except OSError:
2049 return False
2050 else:
2051 return True
2054def make_profile(name, function):
2055 """
2056 Helper to execute cProfile using unique log file.
2057 """
2059 import cProfile
2060 import itertools
2061 import os.path
2062 import time
2064 assert name
2065 assert function
2067 FOLDER = '/tmp/sm-perfs/'
2068 makedirs(FOLDER)
2070 filename = time.strftime('{}_%Y%m%d_%H%M%S.prof'.format(name))
2072 def gen_path(path):
2073 yield path
2074 root, ext = os.path.splitext(path)
2075 for i in itertools.count(start=1, step=1):
2076 yield root + '.{}.'.format(i) + ext
2078 for profile_path in gen_path(FOLDER + filename):
2079 try:
2080 file = open_atomic(profile_path, 'w')
2081 file.close()
2082 break
2083 except OSError as e:
2084 if e.errno == errno.EEXIST:
2085 pass
2086 else:
2087 raise
2089 try:
2090 SMlog('* Start profiling of {} ({}) *'.format(name, filename))
2091 cProfile.runctx('function()', None, locals(), profile_path)
2092 finally:
2093 SMlog('* End profiling of {} ({}) *'.format(name, filename))