Coverage for drivers/util.py : 42%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Copyright (C) Citrix Systems Inc.
2#
3# This program is free software; you can redistribute it and/or modify
4# it under the terms of the GNU Lesser General Public License as published
5# by the Free Software Foundation; version 2.1 only.
6#
7# This program is distributed in the hope that it will be useful,
8# but WITHOUT ANY WARRANTY; without even the implied warranty of
9# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10# GNU Lesser General Public License for more details.
11#
12# You should have received a copy of the GNU Lesser General Public License
13# along with this program; if not, write to the Free Software Foundation, Inc.,
14# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
15#
16# Miscellaneous utility functions
17#
19import os
20import re
21import sys
22import subprocess
23import shutil
24import tempfile
25import signal
26import time
27import datetime
28import errno
29import socket
30import xml.dom.minidom
31import scsiutil
32import stat
33import xs_errors
34import XenAPI # pylint: disable=import-error
35import xmlrpc.client
36import base64
37import syslog
38import resource
39import traceback
40import glob
41import copy
42import tempfile
44from functools import reduce
46NO_LOGGING_STAMPFILE = '/etc/xensource/no_sm_log'
48IORETRY_MAX = 20 # retries
49IORETRY_PERIOD = 1.0 # seconds
51LOGGING = not (os.path.exists(NO_LOGGING_STAMPFILE))
52_SM_SYSLOG_FACILITY = syslog.LOG_LOCAL2
53LOG_EMERG = syslog.LOG_EMERG
54LOG_ALERT = syslog.LOG_ALERT
55LOG_CRIT = syslog.LOG_CRIT
56LOG_ERR = syslog.LOG_ERR
57LOG_WARNING = syslog.LOG_WARNING
58LOG_NOTICE = syslog.LOG_NOTICE
59LOG_INFO = syslog.LOG_INFO
60LOG_DEBUG = syslog.LOG_DEBUG
62ISCSI_REFDIR = '/var/run/sr-ref'
64CMD_DD = "/bin/dd"
66FIST_PAUSE_PERIOD = 30 # seconds
69class SMException(Exception):
70 """Base class for all SM exceptions for easier catching & wrapping in
71 XenError"""
74class CommandException(SMException):
75 def error_message(self, code):
76 if code > 0:
77 return os.strerror(code)
78 elif code < 0:
79 return "Signalled %s" % (abs(code))
80 return "Success"
82 def __init__(self, code, cmd="", reason='exec failed'):
83 self.code = code
84 self.cmd = cmd
85 self.reason = reason
86 Exception.__init__(self, self.error_message(code))
89class SRBusyException(SMException):
90 """The SR could not be locked"""
91 pass
94def logException(tag):
95 info = sys.exc_info()
96 if info[0] == SystemExit: 96 ↛ 98line 96 didn't jump to line 98, because the condition on line 96 was never true
97 # this should not be happening when catching "Exception", but it is
98 sys.exit(0)
99 tb = reduce(lambda a, b: "%s%s" % (a, b), traceback.format_tb(info[2]))
100 str = "***** %s: EXCEPTION %s, %s\n%s" % (tag, info[0], info[1], tb)
101 SMlog(str)
104def roundup(divisor, value):
105 """Retruns the rounded up value so it is divisible by divisor."""
107 if value == 0: 107 ↛ 108line 107 didn't jump to line 108, because the condition on line 107 was never true
108 value = 1
109 if value % divisor != 0:
110 return ((int(value) // divisor) + 1) * divisor
111 return value
114def to_plain_string(obj):
115 if obj is None:
116 return None
117 if type(obj) == str:
118 return obj
119 return str(obj)
122def shellquote(arg):
123 return '"%s"' % arg.replace('"', '\\"')
126def make_WWN(name):
127 hex_prefix = name.find("0x")
128 if (hex_prefix >= 0): 128 ↛ 131line 128 didn't jump to line 131, because the condition on line 128 was never false
129 name = name[name.find("0x") + 2:len(name)]
130 # inject dashes for each nibble
131 if (len(name) == 16): # sanity check 131 ↛ 135line 131 didn't jump to line 135, because the condition on line 131 was never false
132 name = name[0:2] + "-" + name[2:4] + "-" + name[4:6] + "-" + \
133 name[6:8] + "-" + name[8:10] + "-" + name[10:12] + "-" + \
134 name[12:14] + "-" + name[14:16]
135 return name
138def _logToSyslog(ident, facility, priority, message):
139 syslog.openlog(ident, 0, facility)
140 syslog.syslog(priority, "[%d] %s" % (os.getpid(), message))
141 syslog.closelog()
144def SMlog(message, ident="SM", priority=LOG_INFO):
145 if LOGGING: 145 ↛ exitline 145 didn't return from function 'SMlog', because the condition on line 145 was never false
146 for message_line in str(message).split('\n'):
147 _logToSyslog(ident, _SM_SYSLOG_FACILITY, priority, message_line)
150def _getDateString():
151 d = datetime.datetime.now()
152 t = d.timetuple()
153 return "%s-%s-%s:%s:%s:%s" % \
154 (t[0], t[1], t[2], t[3], t[4], t[5])
157def doexec(args, inputtext=None, new_env=None, text=True):
158 """Execute a subprocess, then return its return code, stdout and stderr"""
159 env = None
160 if new_env:
161 env = dict(os.environ)
162 env.update(new_env)
163 proc = subprocess.Popen(args, stdin=subprocess.PIPE,
164 stdout=subprocess.PIPE,
165 stderr=subprocess.PIPE,
166 close_fds=True, env=env,
167 universal_newlines=text)
169 if not text and inputtext is not None: 169 ↛ 170line 169 didn't jump to line 170, because the condition on line 169 was never true
170 inputtext = inputtext.encode()
172 (stdout, stderr) = proc.communicate(inputtext)
174 rc = proc.returncode
175 return rc, stdout, stderr
178def is_string(value):
179 return isinstance(value, str)
182# These are partially tested functions that replicate the behaviour of
183# the original pread,pread2 and pread3 functions. Potentially these can
184# replace the original ones at some later date.
185#
186# cmdlist is a list of either single strings or pairs of strings. For
187# each pair, the first component is passed to exec while the second is
188# written to the logs.
189def pread(cmdlist, close_stdin=False, scramble=None, expect_rc=0,
190 quiet=False, new_env=None, text=True):
191 cmdlist_for_exec = []
192 cmdlist_for_log = []
193 for item in cmdlist:
194 if is_string(item): 194 ↛ 204line 194 didn't jump to line 204, because the condition on line 194 was never false
195 cmdlist_for_exec.append(item)
196 if scramble: 196 ↛ 197line 196 didn't jump to line 197, because the condition on line 196 was never true
197 if item.find(scramble) != -1:
198 cmdlist_for_log.append("<filtered out>")
199 else:
200 cmdlist_for_log.append(item)
201 else:
202 cmdlist_for_log.append(item)
203 else:
204 cmdlist_for_exec.append(item[0])
205 cmdlist_for_log.append(item[1])
207 if not quiet: 207 ↛ 209line 207 didn't jump to line 209, because the condition on line 207 was never false
208 SMlog(cmdlist_for_log)
209 (rc, stdout, stderr) = doexec(cmdlist_for_exec, new_env=new_env, text=text)
210 if rc != expect_rc:
211 SMlog("FAILED in util.pread: (rc %d) stdout: '%s', stderr: '%s'" % \
212 (rc, stdout, stderr))
213 if quiet: 213 ↛ 214line 213 didn't jump to line 214, because the condition on line 213 was never true
214 SMlog("Command was: %s" % cmdlist_for_log)
215 if '' == stderr: 215 ↛ 216line 215 didn't jump to line 216, because the condition on line 215 was never true
216 stderr = stdout
217 raise CommandException(rc, str(cmdlist), stderr.strip())
218 if not quiet: 218 ↛ 220line 218 didn't jump to line 220, because the condition on line 218 was never false
219 SMlog(" pread SUCCESS")
220 return stdout
223# POSIX guaranteed atomic within the same file system.
224# Supply directory to ensure tempfile is created
225# in the same directory.
226def atomicFileWrite(targetFile, directory, text):
228 file = None
229 try:
230 # Create file only current pid can write/read to
231 # our responsibility to clean it up.
232 _, tempPath = tempfile.mkstemp(dir=directory)
233 file = open(tempPath, 'w')
234 file.write(text)
236 # Ensure flushed to disk.
237 file.flush()
238 os.fsync(file.fileno())
239 file.close()
241 os.rename(tempPath, targetFile)
242 except OSError:
243 SMlog("FAILED to atomic write to %s" % (targetFile))
245 finally:
246 if (file is not None) and (not file.closed):
247 file.close()
249 if os.path.isfile(tempPath):
250 os.remove(tempPath)
253#Read STDOUT from cmdlist and discard STDERR output
254def pread2(cmdlist, quiet=False, text=True):
255 return pread(cmdlist, quiet=quiet, text=text)
258#Read STDOUT from cmdlist, feeding 'text' to STDIN
259def pread3(cmdlist, text):
260 SMlog(cmdlist)
261 (rc, stdout, stderr) = doexec(cmdlist, text)
262 if rc:
263 SMlog("FAILED in util.pread3: (errno %d) stdout: '%s', stderr: '%s'" % \
264 (rc, stdout, stderr))
265 if '' == stderr:
266 stderr = stdout
267 raise CommandException(rc, str(cmdlist), stderr.strip())
268 SMlog(" pread3 SUCCESS")
269 return stdout
272def listdir(path, quiet=False):
273 cmd = ["ls", path, "-1", "--color=never"]
274 try:
275 text = pread2(cmd, quiet=quiet)[:-1]
276 if len(text) == 0:
277 return []
278 return text.split('\n')
279 except CommandException as inst:
280 if inst.code == errno.ENOENT:
281 raise CommandException(errno.EIO, inst.cmd, inst.reason)
282 else:
283 raise CommandException(inst.code, inst.cmd, inst.reason)
286def gen_uuid():
287 cmd = ["uuidgen", "-r"]
288 return pread(cmd)[:-1]
291def match_uuid(s):
292 regex = re.compile("^[0-9a-f]{8}-(([0-9a-f]{4})-){3}[0-9a-f]{12}")
293 return regex.search(s, 0)
296def findall_uuid(s):
297 regex = re.compile("[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}")
298 return regex.findall(s, 0)
301def exactmatch_uuid(s):
302 regex = re.compile("^[0-9a-f]{8}-(([0-9a-f]{4})-){3}[0-9a-f]{12}$")
303 return regex.search(s, 0)
306def start_log_entry(srpath, path, args):
307 logstring = str(datetime.datetime.now())
308 logstring += " log: "
309 logstring += srpath
310 logstring += " " + path
311 for element in args:
312 logstring += " " + element
313 try:
314 file = open(srpath + "/filelog.txt", "a")
315 file.write(logstring)
316 file.write("\n")
317 file.close()
318 except:
319 pass
321 # failed to write log ...
323def end_log_entry(srpath, path, args):
324 # for teminating, use "error" or "done"
325 logstring = str(datetime.datetime.now())
326 logstring += " end: "
327 logstring += srpath
328 logstring += " " + path
329 for element in args:
330 logstring += " " + element
331 try:
332 file = open(srpath + "/filelog.txt", "a")
333 file.write(logstring)
334 file.write("\n")
335 file.close()
336 except:
337 pass
339 # failed to write log ...
340 # for now print
341 # print "%s" % logstring
343def ioretry(f, errlist=[errno.EIO], maxretry=IORETRY_MAX, period=IORETRY_PERIOD, **ignored):
344 retries = 0
345 while True:
346 try:
347 return f()
348 except OSError as ose:
349 err = int(ose.errno)
350 if not err in errlist:
351 raise CommandException(err, str(f), "OSError")
352 except CommandException as ce:
353 if not int(ce.code) in errlist:
354 raise
356 retries += 1
357 if retries >= maxretry:
358 break
360 time.sleep(period)
362 raise CommandException(errno.ETIMEDOUT, str(f), "Timeout")
365def ioretry_stat(path, maxretry=IORETRY_MAX):
366 # this ioretry is similar to the previous method, but
367 # stat does not raise an error -- so check its return
368 retries = 0
369 while retries < maxretry:
370 stat = os.statvfs(path)
371 if stat.f_blocks != -1:
372 return stat
373 time.sleep(1)
374 retries += 1
375 raise CommandException(errno.EIO, "os.statvfs")
378def sr_get_capability(sr_uuid):
379 result = []
380 session = get_localAPI_session()
381 sr_ref = session.xenapi.SR.get_by_uuid(sr_uuid)
382 sm_type = session.xenapi.SR.get_record(sr_ref)['type']
383 sm_rec = session.xenapi.SM.get_all_records_where(
384 "field \"type\" = \"%s\"" % sm_type)
386 # SM expects at least one entry of any SR type
387 if len(sm_rec) > 0:
388 result = list(sm_rec.values())[0]['capabilities']
390 session.xenapi.logout()
391 return result
394def sr_get_driver_info(driver_info):
395 results = {}
396 # first add in the vanilla stuff
397 for key in ['name', 'description', 'vendor', 'copyright', \
398 'driver_version', 'required_api_version']:
399 results[key] = driver_info[key]
400 # add the capabilities (xmlrpc array)
401 # enforcing activate/deactivate for blktap2
402 caps = driver_info['capabilities']
403 if "ATOMIC_PAUSE" in caps: 403 ↛ 404line 403 didn't jump to line 404, because the condition on line 403 was never true
404 for cap in ("VDI_ACTIVATE", "VDI_DEACTIVATE"):
405 if not cap in caps:
406 caps.append(cap)
407 elif "VDI_ACTIVATE" in caps or "VDI_DEACTIVATE" in caps: 407 ↛ 408line 407 didn't jump to line 408, because the condition on line 407 was never true
408 SMlog("Warning: vdi_[de]activate present for %s" % driver_info["name"])
410 results['capabilities'] = caps
411 # add in the configuration options
412 options = []
413 for option in driver_info['configuration']:
414 options.append({'key': option[0], 'description': option[1]})
415 results['configuration'] = options
416 return xmlrpc.client.dumps((results, ), "", True)
419def return_nil():
420 return xmlrpc.client.dumps((None, ), "", True, allow_none=True)
423def SRtoXML(SRlist):
424 dom = xml.dom.minidom.Document()
425 driver = dom.createElement("SRlist")
426 dom.appendChild(driver)
428 for key in SRlist.keys():
429 dict = SRlist[key]
430 entry = dom.createElement("SR")
431 driver.appendChild(entry)
433 e = dom.createElement("UUID")
434 entry.appendChild(e)
435 textnode = dom.createTextNode(key)
436 e.appendChild(textnode)
438 if 'size' in dict:
439 e = dom.createElement("Size")
440 entry.appendChild(e)
441 textnode = dom.createTextNode(str(dict['size']))
442 e.appendChild(textnode)
444 if 'storagepool' in dict:
445 e = dom.createElement("StoragePool")
446 entry.appendChild(e)
447 textnode = dom.createTextNode(str(dict['storagepool']))
448 e.appendChild(textnode)
450 if 'aggregate' in dict:
451 e = dom.createElement("Aggregate")
452 entry.appendChild(e)
453 textnode = dom.createTextNode(str(dict['aggregate']))
454 e.appendChild(textnode)
456 return dom.toprettyxml()
459def pathexists(path):
460 try:
461 os.lstat(path)
462 return True
463 except OSError as inst:
464 if inst.errno == errno.EIO: 464 ↛ 465line 464 didn't jump to line 465, because the condition on line 464 was never true
465 time.sleep(1)
466 try:
467 listdir(os.path.realpath(os.path.dirname(path)))
468 os.lstat(path)
469 return True
470 except:
471 pass
472 raise CommandException(errno.EIO, "os.lstat(%s)" % path, "failed")
473 return False
476def force_unlink(path):
477 try:
478 os.unlink(path)
479 except OSError as e:
480 if e.errno != errno.ENOENT:
481 raise
484def create_secret(session, secret):
485 ref = session.xenapi.secret.create({'value': secret})
486 return session.xenapi.secret.get_uuid(ref)
489def get_secret(session, uuid):
490 try:
491 ref = session.xenapi.secret.get_by_uuid(uuid)
492 return session.xenapi.secret.get_value(ref)
493 except:
494 raise xs_errors.XenError('InvalidSecret', opterr='Unable to look up secret [%s]' % uuid)
497def get_real_path(path):
498 "Follow symlinks to the actual file"
499 absPath = path
500 directory = ''
501 while os.path.islink(absPath):
502 directory = os.path.dirname(absPath)
503 absPath = os.readlink(absPath)
504 absPath = os.path.join(directory, absPath)
505 return absPath
508def wait_for_path(path, timeout):
509 for i in range(0, timeout): 509 ↛ 513line 509 didn't jump to line 513, because the loop on line 509 didn't complete
510 if len(glob.glob(path)): 510 ↛ 512line 510 didn't jump to line 512, because the condition on line 510 was never false
511 return True
512 time.sleep(1)
513 return False
516def wait_for_nopath(path, timeout):
517 for i in range(0, timeout):
518 if not os.path.exists(path):
519 return True
520 time.sleep(1)
521 return False
524def wait_for_path_multi(path, timeout):
525 for i in range(0, timeout):
526 paths = glob.glob(path)
527 SMlog("_wait_for_paths_multi: paths = %s" % paths)
528 if len(paths):
529 SMlog("_wait_for_paths_multi: return first path: %s" % paths[0])
530 return paths[0]
531 time.sleep(1)
532 return ""
535def isdir(path):
536 try:
537 st = os.stat(path)
538 return stat.S_ISDIR(st.st_mode)
539 except OSError as inst:
540 if inst.errno == errno.EIO: 540 ↛ 541line 540 didn't jump to line 541, because the condition on line 540 was never true
541 raise CommandException(errno.EIO, "os.stat(%s)" % path, "failed")
542 return False
545def get_single_entry(path):
546 f = open(path, 'r')
547 line = f.readline()
548 f.close()
549 return line.rstrip()
552def get_fs_size(path):
553 st = ioretry_stat(path)
554 return st.f_blocks * st.f_frsize
557def get_fs_utilisation(path):
558 st = ioretry_stat(path)
559 return (st.f_blocks - st.f_bfree) * \
560 st.f_frsize
563def ismount(path):
564 """Test whether a path is a mount point"""
565 try:
566 s1 = os.stat(path)
567 s2 = os.stat(os.path.join(path, '..'))
568 except OSError as inst:
569 raise CommandException(inst.errno, "os.stat")
570 dev1 = s1.st_dev
571 dev2 = s2.st_dev
572 if dev1 != dev2:
573 return True # path/.. on a different device as path
574 ino1 = s1.st_ino
575 ino2 = s2.st_ino
576 if ino1 == ino2:
577 return True # path/.. is the same i-node as path
578 return False
581def makedirs(name, mode=0o777):
582 head, tail = os.path.split(name)
583 if not tail: 583 ↛ 584line 583 didn't jump to line 584, because the condition on line 583 was never true
584 head, tail = os.path.split(head)
585 if head and tail and not pathexists(head):
586 makedirs(head, mode)
587 if tail == os.curdir: 587 ↛ 588line 587 didn't jump to line 588, because the condition on line 587 was never true
588 return
589 try:
590 os.mkdir(name, mode)
591 except OSError as exc:
592 if exc.errno == errno.EEXIST and os.path.isdir(name): 592 ↛ 593line 592 didn't jump to line 593, because the condition on line 592 was never true
593 if mode:
594 os.chmod(name, mode)
595 pass
596 else:
597 raise
600def zeroOut(path, fromByte, bytes):
601 """write 'bytes' zeros to 'path' starting from fromByte (inclusive)"""
602 blockSize = 4096
604 fromBlock = fromByte // blockSize
605 if fromByte % blockSize:
606 fromBlock += 1
607 bytesBefore = fromBlock * blockSize - fromByte
608 if bytesBefore > bytes:
609 bytesBefore = bytes
610 bytes -= bytesBefore
611 cmd = [CMD_DD, "if=/dev/zero", "of=%s" % path, "bs=1",
612 "seek=%s" % fromByte, "count=%s" % bytesBefore]
613 try:
614 pread2(cmd)
615 except CommandException:
616 return False
618 blocks = bytes // blockSize
619 bytes -= blocks * blockSize
620 fromByte = (fromBlock + blocks) * blockSize
621 if blocks:
622 cmd = [CMD_DD, "if=/dev/zero", "of=%s" % path, "bs=%s" % blockSize,
623 "seek=%s" % fromBlock, "count=%s" % blocks]
624 try:
625 pread2(cmd)
626 except CommandException:
627 return False
629 if bytes:
630 cmd = [CMD_DD, "if=/dev/zero", "of=%s" % path, "bs=1",
631 "seek=%s" % fromByte, "count=%s" % bytes]
632 try:
633 pread2(cmd)
634 except CommandException:
635 return False
637 return True
640def wipefs(blockdev):
641 "Wipe filesystem signatures from `blockdev`"
642 pread2(["/usr/sbin/wipefs", "-a", blockdev])
645def match_rootdev(s):
646 regex = re.compile("^PRIMARY_DISK")
647 return regex.search(s, 0)
650def getrootdev():
651 filename = '/etc/xensource-inventory'
652 try:
653 f = open(filename, 'r')
654 except:
655 raise xs_errors.XenError('EIO', \
656 opterr="Unable to open inventory file [%s]" % filename)
657 rootdev = ''
658 for line in filter(match_rootdev, f.readlines()):
659 rootdev = line.split("'")[1]
660 if not rootdev: 660 ↛ 661line 660 didn't jump to line 661, because the condition on line 660 was never true
661 raise xs_errors.XenError('NoRootDev')
662 return rootdev
665def getrootdevID():
666 rootdev = getrootdev()
667 try:
668 rootdevID = scsiutil.getSCSIid(rootdev)
669 except:
670 SMlog("util.getrootdevID: Unable to verify serial or SCSIid of device: %s" \
671 % rootdev)
672 return ''
674 if not len(rootdevID):
675 SMlog("util.getrootdevID: Unable to identify scsi device [%s] via scsiID" \
676 % rootdev)
678 return rootdevID
681def get_localAPI_session():
682 # First acquire a valid session
683 session = XenAPI.xapi_local()
684 try:
685 session.xenapi.login_with_password('root', '', '', 'SM')
686 except:
687 raise xs_errors.XenError('APISession')
688 return session
691def get_this_host():
692 uuid = None
693 f = open("/etc/xensource-inventory", 'r')
694 for line in f.readlines():
695 if line.startswith("INSTALLATION_UUID"):
696 uuid = line.split("'")[1]
697 f.close()
698 return uuid
701def get_master_ref(session):
702 pools = session.xenapi.pool.get_all()
703 return session.xenapi.pool.get_master(pools[0])
706def is_master(session):
707 return get_this_host_ref(session) == get_master_ref(session)
710def get_localhost_ref(session):
711 filename = '/etc/xensource-inventory'
712 try:
713 f = open(filename, 'r')
714 except:
715 raise xs_errors.XenError('EIO', \
716 opterr="Unable to open inventory file [%s]" % filename)
717 domid = ''
718 for line in filter(match_domain_id, f.readlines()):
719 domid = line.split("'")[1]
720 if not domid:
721 raise xs_errors.XenError('APILocalhost')
723 vms = session.xenapi.VM.get_all_records_where('field "uuid" = "%s"' % domid)
724 for vm in vms:
725 record = vms[vm]
726 if record["uuid"] == domid:
727 hostid = record["resident_on"]
728 return hostid
729 raise xs_errors.XenError('APILocalhost')
732def match_domain_id(s):
733 regex = re.compile("^CONTROL_DOMAIN_UUID")
734 return regex.search(s, 0)
737def get_hosts_attached_on(session, vdi_uuids):
738 host_refs = {}
739 for vdi_uuid in vdi_uuids:
740 try:
741 vdi_ref = session.xenapi.VDI.get_by_uuid(vdi_uuid)
742 except XenAPI.Failure:
743 SMlog("VDI %s not in db, ignoring" % vdi_uuid)
744 continue
745 sm_config = session.xenapi.VDI.get_sm_config(vdi_ref)
746 for key in [x for x in sm_config.keys() if x.startswith('host_')]:
747 host_refs[key[len('host_'):]] = True
748 return host_refs.keys()
750def get_this_host_address(session):
751 host_uuid = get_this_host()
752 host_ref = session.xenapi.host.get_by_uuid(host_uuid)
753 return session.xenapi.host.get_record(host_ref)['address']
755def get_host_addresses(session):
756 addresses = []
757 hosts = session.xenapi.host.get_all_records()
758 for record in hosts.values():
759 addresses.append(record['address'])
760 return addresses
762def get_this_host_ref(session):
763 host_uuid = get_this_host()
764 host_ref = session.xenapi.host.get_by_uuid(host_uuid)
765 return host_ref
768def get_slaves_attached_on(session, vdi_uuids):
769 "assume this host is the SR master"
770 host_refs = get_hosts_attached_on(session, vdi_uuids)
771 master_ref = get_this_host_ref(session)
772 return [x for x in host_refs if x != master_ref]
775def get_online_hosts(session):
776 online_hosts = []
777 hosts = session.xenapi.host.get_all_records()
778 for host_ref, host_rec in hosts.items():
779 metricsRef = host_rec["metrics"]
780 metrics = session.xenapi.host_metrics.get_record(metricsRef)
781 if metrics["live"]:
782 online_hosts.append(host_ref)
783 return online_hosts
786def get_all_slaves(session):
787 "assume this host is the SR master"
788 host_refs = get_online_hosts(session)
789 master_ref = get_this_host_ref(session)
790 return [x for x in host_refs if x != master_ref]
793def is_attached_rw(sm_config):
794 for key, val in sm_config.items():
795 if key.startswith("host_") and val == "RW":
796 return True
797 return False
800def attached_as(sm_config):
801 for key, val in sm_config.items():
802 if key.startswith("host_") and (val == "RW" or val == "RO"): 802 ↛ 803line 802 didn't jump to line 803, because the condition on line 802 was never true
803 return val
806def find_my_pbd_record(session, host_ref, sr_ref):
807 try:
808 pbds = session.xenapi.PBD.get_all_records()
809 for pbd_ref in pbds.keys():
810 if pbds[pbd_ref]['host'] == host_ref and pbds[pbd_ref]['SR'] == sr_ref:
811 return [pbd_ref, pbds[pbd_ref]]
812 return None
813 except Exception as e:
814 SMlog("Caught exception while looking up PBD for host %s SR %s: %s" % (str(host_ref), str(sr_ref), str(e)))
815 return None
818def find_my_pbd(session, host_ref, sr_ref):
819 ret = find_my_pbd_record(session, host_ref, sr_ref)
820 if ret is not None:
821 return ret[0]
822 else:
823 return None
826def test_hostPBD_devs(session, sr_uuid, devs):
827 host = get_localhost_ref(session)
828 sr = session.xenapi.SR.get_by_uuid(sr_uuid)
829 try:
830 pbds = session.xenapi.PBD.get_all_records()
831 except:
832 raise xs_errors.XenError('APIPBDQuery')
833 for dev in devs.split(','):
834 for pbd in pbds:
835 record = pbds[pbd]
836 # it's ok if it's *our* PBD
837 if record["SR"] == sr:
838 break
839 if record["host"] == host:
840 devconfig = record["device_config"]
841 if 'device' in devconfig:
842 for device in devconfig['device'].split(','):
843 if os.path.realpath(device) == os.path.realpath(dev):
844 return True
845 return False
848def test_hostPBD_lun(session, targetIQN, LUNid):
849 host = get_localhost_ref(session)
850 try:
851 pbds = session.xenapi.PBD.get_all_records()
852 except:
853 raise xs_errors.XenError('APIPBDQuery')
854 for pbd in pbds:
855 record = pbds[pbd]
856 if record["host"] == host:
857 devconfig = record["device_config"]
858 if 'targetIQN' in devconfig and 'LUNid' in devconfig:
859 if devconfig['targetIQN'] == targetIQN and \
860 devconfig['LUNid'] == LUNid:
861 return True
862 return False
865def test_SCSIid(session, sr_uuid, SCSIid):
866 if sr_uuid is not None:
867 sr = session.xenapi.SR.get_by_uuid(sr_uuid)
868 try:
869 pbds = session.xenapi.PBD.get_all_records()
870 except:
871 raise xs_errors.XenError('APIPBDQuery')
872 for pbd in pbds:
873 record = pbds[pbd]
874 # it's ok if it's *our* PBD
875 # During FC SR creation, devscan.py passes sr_uuid as None
876 if sr_uuid is not None:
877 if record["SR"] == sr:
878 break
879 devconfig = record["device_config"]
880 sm_config = session.xenapi.SR.get_sm_config(record["SR"])
881 if 'SCSIid' in devconfig and devconfig['SCSIid'] == SCSIid:
882 return True
883 elif 'SCSIid' in sm_config and sm_config['SCSIid'] == SCSIid:
884 return True
885 elif 'scsi-' + SCSIid in sm_config:
886 return True
887 return False
890class TimeoutException(SMException):
891 pass
894def timeout_call(timeoutseconds, function, *arguments):
895 def handler(signum, frame):
896 raise TimeoutException()
897 signal.signal(signal.SIGALRM, handler)
898 signal.alarm(timeoutseconds)
899 try:
900 return function(*arguments)
901 finally:
902 signal.alarm(0)
905def _incr_iscsiSR_refcount(targetIQN, uuid):
906 if not os.path.exists(ISCSI_REFDIR):
907 os.mkdir(ISCSI_REFDIR)
908 filename = os.path.join(ISCSI_REFDIR, targetIQN)
909 try:
910 f = open(filename, 'a+')
911 except:
912 raise xs_errors.XenError('LVMRefCount', \
913 opterr='file %s' % filename)
915 f.seek(0)
916 found = False
917 refcount = 0
918 for line in filter(match_uuid, f.readlines()):
919 refcount += 1
920 if line.find(uuid) != -1:
921 found = True
922 if not found:
923 f.write("%s\n" % uuid)
924 refcount += 1
925 f.close()
926 return refcount
929def _decr_iscsiSR_refcount(targetIQN, uuid):
930 filename = os.path.join(ISCSI_REFDIR, targetIQN)
931 if not os.path.exists(filename):
932 return 0
933 try:
934 f = open(filename, 'a+')
935 except:
936 raise xs_errors.XenError('LVMRefCount', \
937 opterr='file %s' % filename)
939 f.seek(0)
940 output = []
941 refcount = 0
942 for line in filter(match_uuid, f.readlines()):
943 if line.find(uuid) == -1:
944 output.append(line.rstrip())
945 refcount += 1
946 if not refcount:
947 os.unlink(filename)
948 return refcount
950 # Re-open file and truncate
951 f.close()
952 f = open(filename, 'w')
953 for i in range(0, refcount):
954 f.write("%s\n" % output[i])
955 f.close()
956 return refcount
959# The agent enforces 1 PBD per SR per host, so we
960# check for active SR entries not attached to this host
961def test_activePoolPBDs(session, host, uuid):
962 try:
963 pbds = session.xenapi.PBD.get_all_records()
964 except:
965 raise xs_errors.XenError('APIPBDQuery')
966 for pbd in pbds:
967 record = pbds[pbd]
968 if record["host"] != host and record["SR"] == uuid \
969 and record["currently_attached"]:
970 return True
971 return False
974def remove_mpathcount_field(session, host_ref, sr_ref, SCSIid):
975 try:
976 pbdref = find_my_pbd(session, host_ref, sr_ref)
977 if pbdref is not None:
978 key = "mpath-" + SCSIid
979 session.xenapi.PBD.remove_from_other_config(pbdref, key)
980 except:
981 pass
985def _testHost(hostname, port, errstring):
986 SMlog("_testHost: Testing host/port: %s,%d" % (hostname, port))
987 try:
988 sockinfo = socket.getaddrinfo(hostname, int(port))[0]
989 except:
990 logException('Exception occured getting IP for %s' % hostname)
991 raise xs_errors.XenError('DNSError')
993 timeout = 5
995 sock = socket.socket(sockinfo[0], socket.SOCK_STREAM)
996 # Only allow the connect to block for up to timeout seconds
997 sock.settimeout(timeout)
998 try:
999 sock.connect(sockinfo[4])
1000 # Fix for MS storage server bug
1001 sock.send(b'\n')
1002 sock.close()
1003 except socket.error as reason:
1004 SMlog("_testHost: Connect failed after %d seconds (%s) - %s" \
1005 % (timeout, hostname, reason))
1006 raise xs_errors.XenError(errstring)
1009def match_scsiID(s, id):
1010 regex = re.compile(id)
1011 return regex.search(s, 0)
1014def _isSCSIid(s):
1015 regex = re.compile("^scsi-")
1016 return regex.search(s, 0)
1019def test_scsiserial(session, device):
1020 device = os.path.realpath(device)
1021 if not scsiutil._isSCSIdev(device):
1022 SMlog("util.test_scsiserial: Not a serial device: %s" % device)
1023 return False
1024 serial = ""
1025 try:
1026 serial += scsiutil.getserial(device)
1027 except:
1028 # Error allowed, SCSIid is the important one
1029 pass
1031 try:
1032 scsiID = scsiutil.getSCSIid(device)
1033 except:
1034 SMlog("util.test_scsiserial: Unable to verify serial or SCSIid of device: %s" \
1035 % device)
1036 return False
1037 if not len(scsiID):
1038 SMlog("util.test_scsiserial: Unable to identify scsi device [%s] via scsiID" \
1039 % device)
1040 return False
1042 try:
1043 SRs = session.xenapi.SR.get_all_records()
1044 except:
1045 raise xs_errors.XenError('APIFailure')
1046 for SR in SRs:
1047 record = SRs[SR]
1048 conf = record["sm_config"]
1049 if 'devserial' in conf:
1050 for dev in conf['devserial'].split(','):
1051 if _isSCSIid(dev):
1052 if match_scsiID(dev, scsiID):
1053 return True
1054 elif len(serial) and dev == serial:
1055 return True
1056 return False
1059def default(self, field, thunk):
1060 try:
1061 return getattr(self, field)
1062 except:
1063 return thunk()
1066def list_VDI_records_in_sr(sr):
1067 """Helper function which returns a list of all VDI records for this SR
1068 stored in the XenAPI server, useful for implementing SR.scan"""
1069 sr_ref = sr.session.xenapi.SR.get_by_uuid(sr.uuid)
1070 vdis = sr.session.xenapi.VDI.get_all_records_where("field \"SR\" = \"%s\"" % sr_ref)
1071 return vdis
1074# Given a partition (e.g. sda1), get a disk name:
1075def diskFromPartition(partition):
1076 # check whether this is a device mapper device (e.g. /dev/dm-0)
1077 m = re.match('(/dev/)?(dm-[0-9]+)(p[0-9]+)?$', partition)
1078 if m is not None: 1078 ↛ 1079line 1078 didn't jump to line 1079, because the condition on line 1078 was never true
1079 return m.group(2)
1081 numlen = 0 # number of digit characters
1082 m = re.match("\D+(\d+)", partition)
1083 if m is not None: 1083 ↛ 1084line 1083 didn't jump to line 1084, because the condition on line 1083 was never true
1084 numlen = len(m.group(1))
1086 # is it a cciss?
1087 if True in [partition.startswith(x) for x in ['cciss', 'ida', 'rd']]: 1087 ↛ 1088line 1087 didn't jump to line 1088, because the condition on line 1087 was never true
1088 numlen += 1 # need to get rid of trailing 'p'
1090 # is it a mapper path?
1091 if partition.startswith("mapper"): 1091 ↛ 1092line 1091 didn't jump to line 1092, because the condition on line 1091 was never true
1092 if re.search("p[0-9]*$", partition):
1093 numlen = len(re.match("\d+", partition[::-1]).group(0)) + 1
1094 SMlog("Found mapper part, len %d" % numlen)
1095 else:
1096 numlen = 0
1098 # is it /dev/disk/by-id/XYZ-part<k>?
1099 if partition.startswith("disk/by-id"): 1099 ↛ 1100line 1099 didn't jump to line 1100, because the condition on line 1099 was never true
1100 return partition[:partition.rfind("-part")]
1102 return partition[:len(partition) - numlen]
1105def dom0_disks():
1106 """Disks carrying dom0, e.g. ['/dev/sda']"""
1107 disks = []
1108 with open("/etc/mtab", 'r') as f:
1109 for line in f:
1110 (dev, mountpoint, fstype, opts, freq, passno) = line.split(' ')
1111 if mountpoint == '/':
1112 disk = diskFromPartition(dev)
1113 if not (disk in disks):
1114 disks.append(disk)
1115 SMlog("Dom0 disks: %s" % disks)
1116 return disks
1119def set_scheduler_sysfs_node(node, scheds):
1120 """
1121 Set the scheduler for a sysfs node (e.g. '/sys/block/sda')
1122 according to prioritized list schedulers
1123 Try to set the first item, then fall back to the next on failure
1124 """
1126 path = os.path.join(node, "queue", "scheduler")
1127 if not os.path.exists(path): 1127 ↛ 1130line 1127 didn't jump to line 1130, because the condition on line 1127 was never false
1128 SMlog("no path %s" % path)
1129 return
1130 for sched in scheds:
1131 try:
1132 with open(path, 'w') as file:
1133 file.write("%s\n" % sched)
1134 SMlog("Set scheduler to [%s] on [%s]" % (sched, node))
1135 return
1136 except (OSError, IOError) as err:
1137 SMlog("Setting scheduler to [%s] on [%s] failed with [%s]" % (sched, node, str(err)))
1138 SMlog("Error setting schedulers to [%s] on [%s]" % (scheds, node))
1141def set_scheduler(dev, schedulers=None):
1142 if schedulers is None: 1142 ↛ 1145line 1142 didn't jump to line 1145, because the condition on line 1142 was never false
1143 schedulers = ["none", "noop"]
1145 devices = []
1146 if not scsiutil.match_dm(dev): 1146 ↛ 1150line 1146 didn't jump to line 1150, because the condition on line 1146 was never false
1147 # Remove partition numbers
1148 devices.append(diskFromPartition(dev).replace('/', '!'))
1149 else:
1150 rawdev = diskFromPartition(dev)
1151 devices = [os.path.realpath(x)[5:] for x in scsiutil._genReverseSCSIidmap(rawdev.split('/')[-1])]
1153 for d in devices:
1154 set_scheduler_sysfs_node("/sys/block/%s" % d, schedulers)
1157# This function queries XAPI for the existing VDI records for this SR
1158def _getVDIs(srobj):
1159 VDIs = []
1160 try:
1161 sr_ref = getattr(srobj, 'sr_ref')
1162 except AttributeError:
1163 return VDIs
1165 refs = srobj.session.xenapi.SR.get_VDIs(sr_ref)
1166 for vdi in refs:
1167 ref = srobj.session.xenapi.VDI.get_record(vdi)
1168 ref['vdi_ref'] = vdi
1169 VDIs.append(ref)
1170 return VDIs
1173def _getVDI(srobj, vdi_uuid):
1174 vdi = srobj.session.xenapi.VDI.get_by_uuid(vdi_uuid)
1175 ref = srobj.session.xenapi.VDI.get_record(vdi)
1176 ref['vdi_ref'] = vdi
1177 return ref
1180def _convertDNS(name):
1181 addr = socket.getaddrinfo(name, None)[0][4][0]
1182 return addr
1185def _containsVDIinuse(srobj):
1186 VDIs = _getVDIs(srobj)
1187 for vdi in VDIs:
1188 if not vdi['managed']:
1189 continue
1190 sm_config = vdi['sm_config']
1191 if 'SRRef' in sm_config:
1192 try:
1193 PBDs = srobj.session.xenapi.SR.get_PBDs(sm_config['SRRef'])
1194 for pbd in PBDs:
1195 record = PBDs[pbd]
1196 if record["host"] == srobj.host_ref and \
1197 record["currently_attached"]:
1198 return True
1199 except:
1200 pass
1201 return False
1204def isVDICommand(cmd):
1205 if cmd is None or cmd in ["vdi_attach", "vdi_detach", 1205 ↛ 1208line 1205 didn't jump to line 1208, because the condition on line 1205 was never true
1206 "vdi_activate", "vdi_deactivate",
1207 "vdi_epoch_begin", "vdi_epoch_end"]:
1208 return True
1209 else:
1210 return False
1213#########################
1214# Daemon helper functions
1215def p_id_fork():
1216 try:
1217 p_id = os.fork()
1218 except OSError as e:
1219 print("Fork failed: %s (%d)" % (e.strerror, e.errno))
1220 sys.exit(-1)
1222 if (p_id == 0):
1223 os.setsid()
1224 try:
1225 p_id = os.fork()
1226 except OSError as e:
1227 print("Fork failed: %s (%d)" % (e.strerror, e.errno))
1228 sys.exit(-1)
1229 if (p_id == 0):
1230 os.chdir('/opt/xensource/sm')
1231 os.umask(0)
1232 else:
1233 os._exit(0)
1234 else:
1235 os._exit(0)
1238def daemon():
1239 p_id_fork()
1240 # Query the max file descriptor parameter for this process
1241 maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1243 # Close any fds that are open
1244 for fd in range(0, maxfd):
1245 try:
1246 os.close(fd)
1247 except:
1248 pass
1250 # Redirect STDIN to STDOUT and STDERR
1251 os.open('/dev/null', os.O_RDWR)
1252 os.dup2(0, 1)
1253 os.dup2(0, 2)
1255################################################################################
1256#
1257# Fist points
1258#
1260# * The global variable 'fistpoint' define the list of all possible fistpoints;
1261#
1262# * To activate a fistpoint called 'name', you need to create the file '/tmp/fist_name'
1263# on the SR master;
1264#
1265# * At the moment, activating a fist point can lead to two possible behaviors:
1266# - if '/tmp/fist_LVHDRT_exit' exists, then the function called during the fistpoint is _exit;
1267# - otherwise, the function called is _pause.
1269def _pause(secs, name):
1270 SMlog("Executing fist point %s: sleeping %d seconds ..." % (name, secs))
1271 time.sleep(secs)
1272 SMlog("Executing fist point %s: done" % name)
1275def _exit(name):
1276 SMlog("Executing fist point %s: exiting the current process ..." % name)
1277 raise xs_errors.XenError('FistPoint', opterr='%s' % name)
1280class FistPoint:
1281 def __init__(self, points):
1282 #SMlog("Fist points loaded")
1283 self.points = points
1285 def is_legal(self, name):
1286 return (name in self.points)
1288 def is_active(self, name):
1289 return os.path.exists("/tmp/fist_%s" % name)
1291 def mark_sr(self, name, sruuid, started):
1292 session = get_localAPI_session()
1293 sr = session.xenapi.SR.get_by_uuid(sruuid)
1294 if started:
1295 session.xenapi.SR.add_to_other_config(sr, name, "active")
1296 else:
1297 session.xenapi.SR.remove_from_other_config(sr, name)
1299 def activate(self, name, sruuid):
1300 if name in self.points: 1300 ↛ 1310line 1300 didn't jump to line 1310, because the condition on line 1300 was never false
1301 if self.is_active(name): 1301 ↛ 1302line 1301 didn't jump to line 1302, because the condition on line 1301 was never true
1302 self.mark_sr(name, sruuid, True)
1303 if self.is_active("LVHDRT_exit"):
1304 self.mark_sr(name, sruuid, False)
1305 _exit(name)
1306 else:
1307 _pause(FIST_PAUSE_PERIOD, name)
1308 self.mark_sr(name, sruuid, False)
1309 else:
1310 SMlog("Unknown fist point: %s" % name)
1312 def activate_custom_fn(self, name, fn):
1313 if name in self.points: 1313 ↛ 1319line 1313 didn't jump to line 1319, because the condition on line 1313 was never false
1314 if self.is_active(name): 1314 ↛ 1315line 1314 didn't jump to line 1315, because the condition on line 1314 was never true
1315 SMlog("Executing fist point %s: starting ..." % name)
1316 fn()
1317 SMlog("Executing fist point %s: done" % name)
1318 else:
1319 SMlog("Unknown fist point: %s" % name)
1322def list_find(f, seq):
1323 for item in seq:
1324 if f(item):
1325 return item
1327GCPAUSE_FISTPOINT = "GCLoop_no_pause"
1329fistpoint = FistPoint(["LVHDRT_finding_a_suitable_pair",
1330 "LVHDRT_inflating_the_parent",
1331 "LVHDRT_resizing_while_vdis_are_paused",
1332 "LVHDRT_coalescing_VHD_data",
1333 "LVHDRT_coalescing_before_inflate_grandparent",
1334 "LVHDRT_relinking_grandchildren",
1335 "LVHDRT_before_create_relink_journal",
1336 "LVHDRT_xapiSM_serialization_tests",
1337 "LVHDRT_clone_vdi_after_create_journal",
1338 "LVHDRT_clone_vdi_after_shrink_parent",
1339 "LVHDRT_clone_vdi_after_first_snap",
1340 "LVHDRT_clone_vdi_after_second_snap",
1341 "LVHDRT_clone_vdi_after_parent_hidden",
1342 "LVHDRT_clone_vdi_after_parent_ro",
1343 "LVHDRT_clone_vdi_before_remove_journal",
1344 "LVHDRT_clone_vdi_after_lvcreate",
1345 "LVHDRT_clone_vdi_before_undo_clone",
1346 "LVHDRT_clone_vdi_after_undo_clone",
1347 "LVHDRT_inflate_after_create_journal",
1348 "LVHDRT_inflate_after_setSize",
1349 "LVHDRT_inflate_after_zeroOut",
1350 "LVHDRT_inflate_after_setSizePhys",
1351 "LVHDRT_inflate_after_setSizePhys",
1352 "LVHDRT_coaleaf_before_coalesce",
1353 "LVHDRT_coaleaf_after_coalesce",
1354 "LVHDRT_coaleaf_one_renamed",
1355 "LVHDRT_coaleaf_both_renamed",
1356 "LVHDRT_coaleaf_after_vdirec",
1357 "LVHDRT_coaleaf_before_delete",
1358 "LVHDRT_coaleaf_after_delete",
1359 "LVHDRT_coaleaf_before_remove_j",
1360 "LVHDRT_coaleaf_undo_after_rename",
1361 "LVHDRT_coaleaf_undo_after_rename2",
1362 "LVHDRT_coaleaf_undo_after_refcount",
1363 "LVHDRT_coaleaf_undo_after_deflate",
1364 "LVHDRT_coaleaf_undo_end",
1365 "LVHDRT_coaleaf_stop_after_recovery",
1366 "LVHDRT_coaleaf_finish_after_inflate",
1367 "LVHDRT_coaleaf_finish_end",
1368 "LVHDRT_coaleaf_delay_1",
1369 "LVHDRT_coaleaf_delay_2",
1370 "LVHDRT_coaleaf_delay_3",
1371 "testsm_clone_allow_raw",
1372 "xenrt_default_vdi_type_legacy",
1373 "blktap_activate_inject_failure",
1374 "blktap_activate_error_handling",
1375 GCPAUSE_FISTPOINT,
1376 "cleanup_coalesceVHD_inject_failure",
1377 "cleanup_tracker_no_progress",
1378 "FileSR_fail_hardlink",
1379 "FileSR_fail_snap1",
1380 "FileSR_fail_snap2",
1381 "LVM_journaler_exists",
1382 "LVM_journaler_none",
1383 "LVM_journaler_badname",
1384 "LVM_journaler_readfail",
1385 "LVM_journaler_writefail"])
1388def set_dirty(session, sr):
1389 try:
1390 session.xenapi.SR.add_to_other_config(sr, "dirty", "")
1391 SMlog("set_dirty %s succeeded" % (repr(sr)))
1392 except:
1393 SMlog("set_dirty %s failed (flag already set?)" % (repr(sr)))
1396def doesFileHaveOpenHandles(fileName):
1397 SMlog("Entering doesFileHaveOpenHandles with file: %s" % fileName)
1398 (retVal, processAndPidTuples) = \
1399 findRunningProcessOrOpenFile(fileName, False)
1401 if not retVal:
1402 SMlog("Failed to determine if file %s has open handles." % \
1403 fileName)
1404 # err on the side of caution
1405 return True
1406 else:
1407 if len(processAndPidTuples) > 0:
1408 return True
1409 else:
1410 return False
1413# extract SR uuid from the passed in devmapper entry and return
1414# /dev/mapper/VG_XenStorage--c3d82e92--cb25--c99b--b83a--482eebab4a93-MGT
1415def extractSRFromDevMapper(path):
1416 try:
1417 path = os.path.basename(path)
1418 path = path[len('VG_XenStorage-') + 1:]
1419 path = path.replace('--', '/')
1420 path = path[0:path.rfind('-')]
1421 return path.replace('/', '-')
1422 except:
1423 return ''
1426# Looks at /proc and figures either
1427# If a process is still running (default), returns open file names
1428# If any running process has open handles to the given file (process = False)
1429# returns process names and pids
1430def findRunningProcessOrOpenFile(name, process=True):
1431 retVal = True
1432 links = []
1433 processandpids = []
1434 sockets = set()
1435 try:
1436 SMlog("Entering findRunningProcessOrOpenFile with params: %s" % \
1437 [name, process])
1439 # Look at all pids
1440 pids = [pid for pid in os.listdir('/proc') if pid.isdigit()]
1441 for pid in sorted(pids):
1442 try:
1443 try:
1444 f = None
1445 f = open(os.path.join('/proc', pid, 'cmdline'), 'r')
1446 prog = f.read()[:-1]
1447 if prog: 1447 ↛ 1456line 1447 didn't jump to line 1456, because the condition on line 1447 was never false
1448 # Just want the process name
1449 argv = prog.split('\x00')
1450 prog = argv[0]
1451 except IOError as e:
1452 if e.errno in (errno.ENOENT, errno.ESRCH):
1453 SMlog("ERROR %s reading %s, ignore" % (e.errno, pid))
1454 continue
1455 finally:
1456 if f is not None: 1456 ↛ 1441, 1456 ↛ 14592 missed branches: 1) line 1456 didn't jump to line 1441, because the continue on line 1454 wasn't executed, 2) line 1456 didn't jump to line 1459, because the condition on line 1456 was never false
1457 f.close() 1457 ↛ 1441line 1457 didn't jump to line 1441, because the continue on line 1454 wasn't executed
1459 try:
1460 fd_dir = os.path.join('/proc', pid, 'fd')
1461 files = os.listdir(fd_dir)
1462 except OSError as e:
1463 if e.errno in (errno.ENOENT, errno.ESRCH):
1464 SMlog("ERROR %s reading fds for %s, ignore" % (e.errno, pid))
1465 # Ignore pid that are no longer valid
1466 continue
1467 else:
1468 raise
1470 for file in files:
1471 try:
1472 link = os.readlink(os.path.join(fd_dir, file))
1473 except OSError:
1474 continue
1476 if process: 1476 ↛ 1481line 1476 didn't jump to line 1481, because the condition on line 1476 was never false
1477 if name == prog: 1477 ↛ 1470line 1477 didn't jump to line 1470, because the condition on line 1477 was never false
1478 links.append(link)
1479 else:
1480 # need to return process name and pid tuples
1481 if link == name:
1482 SMlog("File %s has an open handle with process %s "
1483 "with pid %s" % (name, prog, pid))
1484 processandpids.append((prog, pid))
1486 # Get the connected sockets
1487 if name == prog:
1488 sockets.update(get_connected_sockets(pid))
1489 except Exception as e:
1490 SMlog("Exception checking running process or open file handles. " \
1491 "Error: %s" % str(e))
1492 retVal = False
1494 if process: 1494 ↛ 1497line 1494 didn't jump to line 1497, because the condition on line 1494 was never false
1495 return retVal, links, sockets
1496 else:
1497 return retVal, processandpids
1500def get_connected_sockets(pid):
1501 sockets = set()
1502 try:
1503 # Lines in /proc/<pid>/net/unix are formatted as follows
1504 # (see Linux source net/unix/af_unix.c, unix_seq_show() )
1505 # - Pointer address to socket (hex)
1506 # - Refcount (HEX)
1507 # - 0
1508 # - State (HEX, 0 or __SO_ACCEPTCON)
1509 # - Type (HEX - but only 0001 of interest)
1510 # - Connection state (HEX - but only 03, SS_CONNECTED of interest)
1511 # - Inode number
1512 # - Path (optional)
1513 open_sock_matcher = re.compile(
1514 r'^[0-9a-f]+: [0-9A-Fa-f]+ [0-9A-Fa-f]+ [0-9A-Fa-f]+ 0001 03 \d+ (.*)$')
1515 with open(
1516 os.path.join('/proc', str(pid), 'net', 'unix'), 'r') as f:
1517 lines = f.readlines()
1518 for line in lines:
1519 match = open_sock_matcher.match(line)
1520 if match:
1521 sockets.add(match[1])
1522 except OSError as e:
1523 if e.errno in (errno.ENOENT, errno.ESRCH):
1524 # Ignore pid that are no longer valid
1525 SMlog("ERROR %s reading sockets for %s, ignore" %
1526 (e.errno, pid))
1527 else:
1528 raise
1529 return sockets
1532def retry(f, maxretry=20, period=3, exceptions=[Exception]):
1533 retries = 0
1534 while True:
1535 try:
1536 return f()
1537 except Exception as e:
1538 for exception in exceptions:
1539 if isinstance(e, exception):
1540 SMlog('Got exception: {}. Retry number: {}'.format(
1541 str(e), retries
1542 ))
1543 break
1544 else:
1545 SMlog('Got bad exception: {}. Raising...'.format(e))
1546 raise e
1548 retries += 1
1549 if retries >= maxretry:
1550 break
1552 time.sleep(period)
1554 return f()
1557def getCslDevPath(svid):
1558 basepath = "/dev/disk/by-csldev/"
1559 if svid.startswith("NETAPP_"):
1560 # special attention for NETAPP SVIDs
1561 svid_parts = svid.split("__")
1562 globstr = basepath + "NETAPP__LUN__" + "*" + svid_parts[2] + "*" + svid_parts[-1] + "*"
1563 else:
1564 globstr = basepath + svid + "*"
1566 return globstr
1569# Use device in /dev pointed to by cslg path which consists of svid
1570def get_scsiid_from_svid(md_svid):
1571 cslg_path = getCslDevPath(md_svid)
1572 abs_path = glob.glob(cslg_path)
1573 if abs_path:
1574 real_path = os.path.realpath(abs_path[0])
1575 return scsiutil.getSCSIid(real_path)
1576 else:
1577 return None
1580def get_isl_scsiids(session):
1581 # Get cslg type SRs
1582 SRs = session.xenapi.SR.get_all_records_where('field "type" = "cslg"')
1584 # Iterate through the SR to get the scsi ids
1585 scsi_id_ret = []
1586 for SR in SRs:
1587 sr_rec = SRs[SR]
1588 # Use the md_svid to get the scsi id
1589 scsi_id = get_scsiid_from_svid(sr_rec['sm_config']['md_svid'])
1590 if scsi_id:
1591 scsi_id_ret.append(scsi_id)
1593 # Get the vdis in the SR and do the same procedure
1594 vdi_recs = session.xenapi.VDI.get_all_records_where('field "SR" = "%s"' % SR)
1595 for vdi_rec in vdi_recs:
1596 vdi_rec = vdi_recs[vdi_rec]
1597 scsi_id = get_scsiid_from_svid(vdi_rec['sm_config']['SVID'])
1598 if scsi_id:
1599 scsi_id_ret.append(scsi_id)
1601 return scsi_id_ret
1604class extractXVA:
1605 # streams files as a set of file and checksum, caller should remove
1606 # the files, if not needed. The entire directory (Where the files
1607 # and checksum) will only be deleted as part of class cleanup.
1608 HDR_SIZE = 512
1609 BLOCK_SIZE = 512
1610 SIZE_LEN = 12 - 1 # To remove \0 from tail
1611 SIZE_OFFSET = 124
1612 ZERO_FILLED_REC = 2
1613 NULL_IDEN = '\x00'
1614 DIR_IDEN = '/'
1615 CHECKSUM_IDEN = '.checksum'
1616 OVA_FILE = 'ova.xml'
1618 # Init gunzips the file using a subprocess, and reads stdout later
1619 # as and when needed
1620 def __init__(self, filename):
1621 self.__extract_path = ''
1622 self.__filename = filename
1623 cmd = 'gunzip -cd %s' % filename
1624 try:
1625 self.spawn_p = subprocess.Popen(
1626 cmd, shell=True, \
1627 stdin=subprocess.PIPE, stdout=subprocess.PIPE, \
1628 stderr=subprocess.PIPE, close_fds=True)
1629 except Exception as e:
1630 SMlog("Error: %s. Uncompress failed for %s" % (str(e), filename))
1631 raise Exception(str(e))
1633 # Create dir to extract the files
1634 self.__extract_path = tempfile.mkdtemp()
1636 def __del__(self):
1637 shutil.rmtree(self.__extract_path)
1639 # Class supports Generator expression. 'for f_name, checksum in getTuple()'
1640 # returns filename, checksum content. Returns filename, '' in case
1641 # of checksum file missing. e.g. ova.xml
1642 def getTuple(self):
1643 zerod_record = 0
1644 ret_f_name = ''
1645 ret_base_f_name = ''
1647 try:
1648 # Read tar file as sets of file and checksum.
1649 while True:
1650 # Read the output of spawned process, or output of gunzip
1651 f_hdr = self.spawn_p.stdout.read(self.HDR_SIZE)
1653 # Break out in case of end of file
1654 if f_hdr == '':
1655 if zerod_record == extractXVA.ZERO_FILLED_REC:
1656 break
1657 else:
1658 SMlog('Error. Expects %d zero records', \
1659 extractXVA.ZERO_FILLED_REC)
1660 raise Exception('Unrecognized end of file')
1662 # Watch out for zero records, two zero records
1663 # denote end of file.
1664 if f_hdr == extractXVA.NULL_IDEN * extractXVA.HDR_SIZE:
1665 zerod_record += 1
1666 continue
1668 f_name = f_hdr[:f_hdr.index(extractXVA.NULL_IDEN)]
1669 # File header may be for a folder, if so ignore the header
1670 if not f_name.endswith(extractXVA.DIR_IDEN):
1671 f_size_octal = f_hdr[extractXVA.SIZE_OFFSET: \
1672 extractXVA.SIZE_OFFSET + extractXVA.SIZE_LEN]
1673 f_size = int(f_size_octal, 8)
1674 if f_name.endswith(extractXVA.CHECKSUM_IDEN):
1675 if f_name.rstrip(extractXVA.CHECKSUM_IDEN) == \
1676 ret_base_f_name:
1677 checksum = self.spawn_p.stdout.read(f_size)
1678 yield(ret_f_name, checksum)
1679 else:
1680 # Expects file followed by its checksum
1681 SMlog('Error. Sequence mismatch starting with %s', \
1682 ret_f_name)
1683 raise Exception( \
1684 'Files out of sequence starting with %s', \
1685 ret_f_name)
1686 else:
1687 # In case of ova.xml, read the contents into a file and
1688 # return the file name to the caller. For other files,
1689 # read the contents into a file, it will
1690 # be used when a .checksum file is encountered.
1691 ret_f_name = '%s/%s' % (self.__extract_path, f_name)
1692 ret_base_f_name = f_name
1694 # Check if the folder exists on the target location,
1695 # else create it.
1696 folder_path = ret_f_name[:ret_f_name.rfind('/')]
1697 if not os.path.exists(folder_path):
1698 os.mkdir(folder_path)
1700 # Store the file to the tmp folder, strip the tail \0
1701 f = open(ret_f_name, 'w')
1702 f.write(self.spawn_p.stdout.read(f_size))
1703 f.close()
1704 if f_name == extractXVA.OVA_FILE:
1705 yield(ret_f_name, '')
1707 # Skip zero'd portion of data block
1708 round_off = f_size % extractXVA.BLOCK_SIZE
1709 if round_off != 0:
1710 zeros = self.spawn_p.stdout.read(
1711 extractXVA.BLOCK_SIZE - round_off)
1712 except Exception as e:
1713 SMlog("Error: %s. File set extraction failed %s" % (str(e), \
1714 self.__filename))
1716 # Kill and Drain stdout of the gunzip process,
1717 # else gunzip might block on stdout
1718 os.kill(self.spawn_p.pid, signal.SIGTERM)
1719 self.spawn_p.communicate()
1720 raise Exception(str(e))
1722illegal_xml_chars = [(0x00, 0x08), (0x0B, 0x1F), (0x7F, 0x84), (0x86, 0x9F),
1723 (0xD800, 0xDFFF), (0xFDD0, 0xFDDF), (0xFFFE, 0xFFFF),
1724 (0x1FFFE, 0x1FFFF), (0x2FFFE, 0x2FFFF), (0x3FFFE, 0x3FFFF),
1725 (0x4FFFE, 0x4FFFF), (0x5FFFE, 0x5FFFF), (0x6FFFE, 0x6FFFF),
1726 (0x7FFFE, 0x7FFFF), (0x8FFFE, 0x8FFFF), (0x9FFFE, 0x9FFFF),
1727 (0xAFFFE, 0xAFFFF), (0xBFFFE, 0xBFFFF), (0xCFFFE, 0xCFFFF),
1728 (0xDFFFE, 0xDFFFF), (0xEFFFE, 0xEFFFF), (0xFFFFE, 0xFFFFF),
1729 (0x10FFFE, 0x10FFFF)]
1731illegal_ranges = ["%s-%s" % (chr(low), chr(high))
1732 for (low, high) in illegal_xml_chars
1733 if low < sys.maxunicode]
1735illegal_xml_re = re.compile(u'[%s]' % u''.join(illegal_ranges))
1738def isLegalXMLString(s):
1739 """Tells whether this is a valid XML string (i.e. it does not contain
1740 illegal XML characters specified in
1741 http://www.w3.org/TR/2004/REC-xml-20040204/#charsets).
1742 """
1744 if len(s) > 0:
1745 return re.search(illegal_xml_re, s) is None
1746 else:
1747 return True
1750def unictrunc(string, max_bytes):
1751 """
1752 Given a string, returns the largest number of elements for a prefix
1753 substring of it, such that the UTF-8 encoding of this substring takes no
1754 more than the given number of bytes.
1756 The string may be given as a unicode string or a UTF-8 encoded byte
1757 string, and the number returned will be in characters or bytes
1758 accordingly. Note that in the latter case, the substring will still be a
1759 valid UTF-8 encoded string (which is to say, it won't have been truncated
1760 part way through a multibyte sequence for a unicode character).
1762 string: the string to truncate
1763 max_bytes: the maximum number of bytes the truncated string can be
1764 """
1765 if isinstance(string, str):
1766 return_chars = True
1767 else:
1768 return_chars = False
1769 string = string.decode('UTF-8')
1771 cur_chars = 0
1772 cur_bytes = 0
1773 for char in string:
1774 charsize = len(char.encode('UTF-8'))
1775 if cur_bytes + charsize > max_bytes:
1776 break
1777 else:
1778 cur_chars += 1
1779 cur_bytes += charsize
1780 return cur_chars if return_chars else cur_bytes
1783def hideValuesInPropMap(propmap, propnames):
1784 """
1785 Worker function: input simple map of prop name/value pairs, and
1786 a list of specific propnames whose values we want to hide.
1787 Loop through the "hide" list, and if any are found, hide the
1788 value and return the altered map.
1789 If none found, return the original map
1790 """
1791 matches = []
1792 for propname in propnames:
1793 if propname in propmap: 1793 ↛ 1794line 1793 didn't jump to line 1794, because the condition on line 1793 was never true
1794 matches.append(propname)
1796 if matches: 1796 ↛ 1797line 1796 didn't jump to line 1797, because the condition on line 1796 was never true
1797 deepCopyRec = copy.deepcopy(propmap)
1798 for match in matches:
1799 deepCopyRec[match] = '******'
1800 return deepCopyRec
1802 return propmap
1803# define the list of propnames whose value we want to hide
1805PASSWD_PROP_KEYS = ['password', 'cifspassword', 'chappassword', 'incoming_chappassword']
1806DEFAULT_SEGMENT_LEN = 950
1809def hidePasswdInConfig(config):
1810 """
1811 Function to hide passwd values in a simple prop map,
1812 for example "device_config"
1813 """
1814 return hideValuesInPropMap(config, PASSWD_PROP_KEYS)
1817def hidePasswdInParams(params, configProp):
1818 """
1819 Function to hide password values in a specified property which
1820 is a simple map of prop name/values, and is itself an prop entry
1821 in a larger property map.
1822 For example, param maps containing "device_config", or
1823 "sm_config", etc
1824 """
1825 params[configProp] = hideValuesInPropMap(params[configProp], PASSWD_PROP_KEYS)
1826 return params
1829def hideMemberValuesInXmlParams(xmlParams, propnames=PASSWD_PROP_KEYS):
1830 """
1831 Function to hide password values in XML params, specifically
1832 for the XML format of incoming params to SR modules.
1833 Uses text parsing: loop through the list of specific propnames
1834 whose values we want to hide, and:
1835 - Assemble a full "prefix" containing each property name, e.g.,
1836 "<member><name>password</name><value>"
1837 - Test the XML if it contains that string, save the index.
1838 - If found, get the index of the ending tag
1839 - Truncate the return string starting with the password value.
1840 - Append the substitute "*******" value string.
1841 - Restore the rest of the original string starting with the end tag.
1842 """
1843 findStrPrefixHead = "<member><name>"
1844 findStrPrefixTail = "</name><value>"
1845 findStrSuffix = "</value>"
1846 strlen = len(xmlParams)
1848 for propname in propnames:
1849 findStrPrefix = findStrPrefixHead + propname + findStrPrefixTail
1850 idx = xmlParams.find(findStrPrefix)
1851 if idx != -1: # if found any of them
1852 idx += len(findStrPrefix)
1853 idx2 = xmlParams.find(findStrSuffix, idx)
1854 if idx2 != -1:
1855 retStr = xmlParams[0:idx]
1856 retStr += "******"
1857 retStr += xmlParams[idx2:strlen]
1858 return retStr
1859 else:
1860 return xmlParams
1861 return xmlParams
1864def splitXmlText(xmlData, segmentLen=DEFAULT_SEGMENT_LEN, showContd=False):
1865 """
1866 Split xml string data into substrings small enough for the
1867 syslog line length limit. Split at tag end markers ( ">" ).
1868 Usage:
1869 strList = []
1870 strList = splitXmlText( longXmlText, maxLineLen ) # maxLineLen is optional
1871 """
1872 remainingData = str(xmlData)
1874 # "Un-pretty-print"
1875 remainingData = remainingData.replace('\n', '')
1876 remainingData = remainingData.replace('\t', '')
1878 remainingChars = len(remainingData)
1879 returnData = ''
1881 thisLineNum = 0
1882 while remainingChars > segmentLen:
1883 thisLineNum = thisLineNum + 1
1884 index = segmentLen
1885 tmpStr = remainingData[:segmentLen]
1886 tmpIndex = tmpStr.rfind('>')
1887 if tmpIndex != -1:
1888 index = tmpIndex + 1
1890 tmpStr = tmpStr[:index]
1891 remainingData = remainingData[index:]
1892 remainingChars = len(remainingData)
1894 if showContd:
1895 if thisLineNum != 1:
1896 tmpStr = '(Cont\'d): ' + tmpStr
1897 tmpStr = tmpStr + ' (Cont\'d):'
1899 returnData += tmpStr + '\n'
1901 if showContd and thisLineNum > 0:
1902 remainingData = '(Cont\'d): ' + remainingData
1903 returnData += remainingData
1905 return returnData
1908def inject_failure():
1909 raise Exception('injected failure')
1912def open_atomic(path, mode=None):
1913 """Atomically creates a file if, and only if it does not already exist.
1914 Leaves the file open and returns the file object.
1916 path: the path to atomically open
1917 mode: "r" (read), "w" (write), or "rw" (read/write)
1918 returns: an open file object"""
1920 assert path
1922 flags = os.O_CREAT | os.O_EXCL
1923 modes = {'r': os.O_RDONLY, 'w': os.O_WRONLY, 'rw': os.O_RDWR}
1924 if mode:
1925 if mode not in modes:
1926 raise Exception('invalid access mode ' + mode)
1927 flags |= modes[mode]
1928 fd = os.open(path, flags)
1929 try:
1930 if mode:
1931 return os.fdopen(fd, mode)
1932 else:
1933 return os.fdopen(fd)
1934 except:
1935 os.close(fd)
1936 raise
1939def isInvalidVDI(exception):
1940 return exception.details[0] == "HANDLE_INVALID" or \
1941 exception.details[0] == "UUID_INVALID"
1944def get_pool_restrictions(session):
1945 """Returns pool restrictions as a map, @session must be already
1946 established."""
1947 return list(session.xenapi.pool.get_all_records().values())[0]['restrictions']
1950def read_caching_is_restricted(session):
1951 """Tells whether read caching is restricted."""
1952 if session is None: 1952 ↛ 1953line 1952 didn't jump to line 1953, because the condition on line 1952 was never true
1953 return True
1954 restrictions = get_pool_restrictions(session)
1955 if 'restrict_read_caching' in restrictions and \ 1955 ↛ 1957line 1955 didn't jump to line 1957, because the condition on line 1955 was never true
1956 restrictions['restrict_read_caching'] == "true":
1957 return True
1958 return False
1961def sessions_less_than_targets(other_config, device_config):
1962 if 'multihomelist' in device_config and 'iscsi_sessions' in other_config: 1962 ↛ 1968line 1962 didn't jump to line 1968, because the condition on line 1962 was never false
1963 sessions = int(other_config['iscsi_sessions'])
1964 targets = len(device_config['multihomelist'].split(','))
1965 SMlog("Targets %d and iscsi_sessions %d" % (targets, sessions))
1966 return (sessions < targets)
1967 else:
1968 return False
1971def enable_and_start_service(name, start):
1972 attempt = 0
1973 while True:
1974 attempt += 1
1975 fn = 'enable' if start else 'disable'
1976 args = ('systemctl', fn, '--now', name)
1977 (ret, out, err) = doexec(args)
1978 if ret == 0:
1979 return
1980 elif attempt >= 3:
1981 raise Exception(
1982 'Failed to {} {}: {} {}'.format(fn, name, out, err)
1983 )
1984 time.sleep(1)
1987def stop_service(name):
1988 args = ('systemctl', 'stop', name)
1989 (ret, out, err) = doexec(args)
1990 if ret == 0:
1991 return
1992 raise Exception('Failed to stop {}: {} {}'.format(name, out, err))
1995def restart_service(name):
1996 attempt = 0
1997 while True:
1998 attempt += 1
1999 SMlog('Restarting service {} {}...'.format(name, attempt))
2000 args = ('systemctl', 'restart', name)
2001 (ret, out, err) = doexec(args)
2002 if ret == 0:
2003 return
2004 elif attempt >= 3:
2005 SMlog('Restart service FAILED {} {}'.format(name, attempt))
2006 raise Exception(
2007 'Failed to restart {}: {} {}'.format(name, out, err)
2008 )
2009 time.sleep(1)
2012def check_pid_exists(pid):
2013 try:
2014 os.kill(pid, 0)
2015 except OSError:
2016 return False
2017 else:
2018 return True
2021def make_profile(name, function):
2022 """
2023 Helper to execute cProfile using unique log file.
2024 """
2026 import cProfile
2027 import itertools
2028 import os.path
2029 import time
2031 assert name
2032 assert function
2034 FOLDER = '/tmp/sm-perfs/'
2035 makedirs(FOLDER)
2037 filename = time.strftime('{}_%Y%m%d_%H%M%S.prof'.format(name))
2039 def gen_path(path):
2040 yield path
2041 root, ext = os.path.splitext(path)
2042 for i in itertools.count(start=1, step=1):
2043 yield root + '.{}.'.format(i) + ext
2045 for profile_path in gen_path(FOLDER + filename):
2046 try:
2047 file = open_atomic(profile_path, 'w')
2048 file.close()
2049 break
2050 except OSError as e:
2051 if e.errno == errno.EEXIST:
2052 pass
2053 else:
2054 raise
2056 try:
2057 SMlog('* Start profiling of {} ({}) *'.format(name, filename))
2058 cProfile.runctx('function()', None, locals(), profile_path)
2059 finally:
2060 SMlog('* End profiling of {} ({}) *'.format(name, filename))