Coverage for drivers/util.py : 42%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Copyright (C) Citrix Systems Inc.
2#
3# This program is free software; you can redistribute it and/or modify
4# it under the terms of the GNU Lesser General Public License as published
5# by the Free Software Foundation; version 2.1 only.
6#
7# This program is distributed in the hope that it will be useful,
8# but WITHOUT ANY WARRANTY; without even the implied warranty of
9# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10# GNU Lesser General Public License for more details.
11#
12# You should have received a copy of the GNU Lesser General Public License
13# along with this program; if not, write to the Free Software Foundation, Inc.,
14# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
15#
16# Miscellaneous utility functions
17#
19import os
20import re
21import sys
22import subprocess
23import shutil
24import tempfile
25import signal
26import time
27import datetime
28import errno
29import socket
30import xml.dom.minidom
31import scsiutil
32import stat
33import xs_errors
34import XenAPI
35import xmlrpc.client
36import base64
37import syslog
38import resource
39import traceback
40import glob
41import copy
42import tempfile
44from functools import reduce
46NO_LOGGING_STAMPFILE = '/etc/xensource/no_sm_log'
48IORETRY_MAX = 20 # retries
49IORETRY_PERIOD = 1.0 # seconds
51LOGGING = not (os.path.exists(NO_LOGGING_STAMPFILE))
52_SM_SYSLOG_FACILITY = syslog.LOG_LOCAL2
53LOG_EMERG = syslog.LOG_EMERG
54LOG_ALERT = syslog.LOG_ALERT
55LOG_CRIT = syslog.LOG_CRIT
56LOG_ERR = syslog.LOG_ERR
57LOG_WARNING = syslog.LOG_WARNING
58LOG_NOTICE = syslog.LOG_NOTICE
59LOG_INFO = syslog.LOG_INFO
60LOG_DEBUG = syslog.LOG_DEBUG
62ISCSI_REFDIR = '/var/run/sr-ref'
64CMD_DD = "/bin/dd"
66FIST_PAUSE_PERIOD = 30 # seconds
69class SMException(Exception):
70 """Base class for all SM exceptions for easier catching & wrapping in
71 XenError"""
72 pass
75class CommandException(SMException):
76 def error_message(self, code):
77 if code > 0:
78 return os.strerror(code)
79 elif code < 0:
80 return "Signalled %s" % (abs(code))
81 return "Success"
83 def __init__(self, code, cmd="", reason='exec failed'):
84 self.code = code
85 self.cmd = cmd
86 self.reason = reason
87 Exception.__init__(self, self.error_message(code))
90class SRBusyException(SMException):
91 """The SR could not be locked"""
92 pass
95def logException(tag):
96 info = sys.exc_info()
97 if info[0] == SystemExit: 97 ↛ 99line 97 didn't jump to line 99, because the condition on line 97 was never true
98 # this should not be happening when catching "Exception", but it is
99 sys.exit(0)
100 tb = reduce(lambda a, b: "%s%s" % (a, b), traceback.format_tb(info[2]))
101 str = "***** %s: EXCEPTION %s, %s\n%s" % (tag, info[0], info[1], tb)
102 SMlog(str)
105def roundup(divisor, value):
106 """Retruns the rounded up value so it is divisible by divisor."""
108 if value == 0: 108 ↛ 109line 108 didn't jump to line 109, because the condition on line 108 was never true
109 value = 1
110 if value % divisor != 0:
111 return ((int(value) // divisor) + 1) * divisor
112 return value
115def to_plain_string(obj):
116 if obj is None:
117 return None
118 if type(obj) == str:
119 return obj
120 return str(obj)
123def shellquote(arg):
124 return '"%s"' % arg.replace('"', '\\"')
127def make_WWN(name):
128 hex_prefix = name.find("0x")
129 if (hex_prefix >= 0): 129 ↛ 132line 129 didn't jump to line 132, because the condition on line 129 was never false
130 name = name[name.find("0x") + 2:len(name)]
131 # inject dashes for each nibble
132 if (len(name) == 16): # sanity check 132 ↛ 136line 132 didn't jump to line 136, because the condition on line 132 was never false
133 name = name[0:2] + "-" + name[2:4] + "-" + name[4:6] + "-" + \
134 name[6:8] + "-" + name[8:10] + "-" + name[10:12] + "-" + \
135 name[12:14] + "-" + name[14:16]
136 return name
139def _logToSyslog(ident, facility, priority, message):
140 syslog.openlog(ident, 0, facility)
141 syslog.syslog(priority, "[%d] %s" % (os.getpid(), message))
142 syslog.closelog()
145def SMlog(message, ident="SM", priority=LOG_INFO):
146 if LOGGING: 146 ↛ exitline 146 didn't return from function 'SMlog', because the condition on line 146 was never false
147 for message_line in str(message).split('\n'):
148 _logToSyslog(ident, _SM_SYSLOG_FACILITY, priority, message_line)
151def _getDateString():
152 d = datetime.datetime.now()
153 t = d.timetuple()
154 return "%s-%s-%s:%s:%s:%s" % \
155 (t[0], t[1], t[2], t[3], t[4], t[5])
158def doexec(args, inputtext=None, new_env=None, text=True):
159 """Execute a subprocess, then return its return code, stdout and stderr"""
160 env = None
161 if new_env:
162 env = dict(os.environ)
163 env.update(new_env)
164 proc = subprocess.Popen(args, stdin=subprocess.PIPE,
165 stdout=subprocess.PIPE,
166 stderr=subprocess.PIPE,
167 close_fds=True, env=env,
168 universal_newlines=text)
170 if not text and inputtext is not None: 170 ↛ 171line 170 didn't jump to line 171, because the condition on line 170 was never true
171 inputtext = inputtext.encode()
173 (stdout, stderr) = proc.communicate(inputtext)
175 rc = proc.returncode
176 return rc, stdout, stderr
179def is_string(value):
180 return isinstance(value, str)
183# These are partially tested functions that replicate the behaviour of
184# the original pread,pread2 and pread3 functions. Potentially these can
185# replace the original ones at some later date.
186#
187# cmdlist is a list of either single strings or pairs of strings. For
188# each pair, the first component is passed to exec while the second is
189# written to the logs.
190def pread(cmdlist, close_stdin=False, scramble=None, expect_rc=0,
191 quiet=False, new_env=None, text=True):
192 cmdlist_for_exec = []
193 cmdlist_for_log = []
194 for item in cmdlist:
195 if is_string(item): 195 ↛ 205line 195 didn't jump to line 205, because the condition on line 195 was never false
196 cmdlist_for_exec.append(item)
197 if scramble: 197 ↛ 198line 197 didn't jump to line 198, because the condition on line 197 was never true
198 if item.find(scramble) != -1:
199 cmdlist_for_log.append("<filtered out>")
200 else:
201 cmdlist_for_log.append(item)
202 else:
203 cmdlist_for_log.append(item)
204 else:
205 cmdlist_for_exec.append(item[0])
206 cmdlist_for_log.append(item[1])
208 if not quiet: 208 ↛ 210line 208 didn't jump to line 210, because the condition on line 208 was never false
209 SMlog(cmdlist_for_log)
210 (rc, stdout, stderr) = doexec(cmdlist_for_exec, new_env=new_env, text=text)
211 if rc != expect_rc:
212 SMlog("FAILED in util.pread: (rc %d) stdout: '%s', stderr: '%s'" % \
213 (rc, stdout, stderr))
214 if quiet: 214 ↛ 215line 214 didn't jump to line 215, because the condition on line 214 was never true
215 SMlog("Command was: %s" % cmdlist_for_log)
216 if '' == stderr: 216 ↛ 217line 216 didn't jump to line 217, because the condition on line 216 was never true
217 stderr = stdout
218 raise CommandException(rc, str(cmdlist), stderr.strip())
219 if not quiet: 219 ↛ 221line 219 didn't jump to line 221, because the condition on line 219 was never false
220 SMlog(" pread SUCCESS")
221 return stdout
224# POSIX guaranteed atomic within the same file system.
225# Supply directory to ensure tempfile is created
226# in the same directory.
227def atomicFileWrite(targetFile, directory, text):
229 file = None
230 try:
231 # Create file only current pid can write/read to
232 # our responsibility to clean it up.
233 _, tempPath = tempfile.mkstemp(dir=directory)
234 file = open(tempPath, 'w')
235 file.write(text)
237 # Ensure flushed to disk.
238 file.flush()
239 os.fsync(file.fileno())
240 file.close()
242 os.rename(tempPath, targetFile)
243 except OSError:
244 SMlog("FAILED to atomic write to %s" % (targetFile))
246 finally:
247 if (file is not None) and (not file.closed):
248 file.close()
250 if os.path.isfile(tempPath):
251 os.remove(tempPath)
254#Read STDOUT from cmdlist and discard STDERR output
255def pread2(cmdlist, quiet=False, text=True):
256 return pread(cmdlist, quiet=quiet, text=text)
259#Read STDOUT from cmdlist, feeding 'text' to STDIN
260def pread3(cmdlist, text):
261 SMlog(cmdlist)
262 (rc, stdout, stderr) = doexec(cmdlist, text)
263 if rc:
264 SMlog("FAILED in util.pread3: (errno %d) stdout: '%s', stderr: '%s'" % \
265 (rc, stdout, stderr))
266 if '' == stderr:
267 stderr = stdout
268 raise CommandException(rc, str(cmdlist), stderr.strip())
269 SMlog(" pread3 SUCCESS")
270 return stdout
273def listdir(path, quiet=False):
274 cmd = ["ls", path, "-1", "--color=never"]
275 try:
276 text = pread2(cmd, quiet=quiet)[:-1]
277 if len(text) == 0:
278 return []
279 return text.split('\n')
280 except CommandException as inst:
281 if inst.code == errno.ENOENT:
282 raise CommandException(errno.EIO, inst.cmd, inst.reason)
283 else:
284 raise CommandException(inst.code, inst.cmd, inst.reason)
287def gen_uuid():
288 cmd = ["uuidgen", "-r"]
289 return pread(cmd)[:-1]
292def match_uuid(s):
293 regex = re.compile("^[0-9a-f]{8}-(([0-9a-f]{4})-){3}[0-9a-f]{12}")
294 return regex.search(s, 0)
297def findall_uuid(s):
298 regex = re.compile("[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}")
299 return regex.findall(s, 0)
302def exactmatch_uuid(s):
303 regex = re.compile("^[0-9a-f]{8}-(([0-9a-f]{4})-){3}[0-9a-f]{12}$")
304 return regex.search(s, 0)
307def start_log_entry(srpath, path, args):
308 logstring = str(datetime.datetime.now())
309 logstring += " log: "
310 logstring += srpath
311 logstring += " " + path
312 for element in args:
313 logstring += " " + element
314 try:
315 file = open(srpath + "/filelog.txt", "a")
316 file.write(logstring)
317 file.write("\n")
318 file.close()
319 except:
320 pass
322 # failed to write log ...
324def end_log_entry(srpath, path, args):
325 # for teminating, use "error" or "done"
326 logstring = str(datetime.datetime.now())
327 logstring += " end: "
328 logstring += srpath
329 logstring += " " + path
330 for element in args:
331 logstring += " " + element
332 try:
333 file = open(srpath + "/filelog.txt", "a")
334 file.write(logstring)
335 file.write("\n")
336 file.close()
337 except:
338 pass
340 # failed to write log ...
341 # for now print
342 # print "%s" % logstring
344def ioretry(f, errlist=[errno.EIO], maxretry=IORETRY_MAX, period=IORETRY_PERIOD, **ignored):
345 retries = 0
346 while True:
347 try:
348 return f()
349 except OSError as ose:
350 err = int(ose.errno)
351 if not err in errlist:
352 raise CommandException(err, str(f), "OSError")
353 except CommandException as ce:
354 if not int(ce.code) in errlist:
355 raise
357 retries += 1
358 if retries >= maxretry:
359 break
361 time.sleep(period)
363 raise CommandException(errno.ETIMEDOUT, str(f), "Timeout")
366def ioretry_stat(path, maxretry=IORETRY_MAX):
367 # this ioretry is similar to the previous method, but
368 # stat does not raise an error -- so check its return
369 retries = 0
370 while retries < maxretry:
371 stat = os.statvfs(path)
372 if stat.f_blocks != -1:
373 return stat
374 time.sleep(1)
375 retries += 1
376 raise CommandException(errno.EIO, "os.statvfs")
379def sr_get_capability(sr_uuid):
380 result = []
381 session = get_localAPI_session()
382 sr_ref = session.xenapi.SR.get_by_uuid(sr_uuid)
383 sm_type = session.xenapi.SR.get_record(sr_ref)['type']
384 sm_rec = session.xenapi.SM.get_all_records_where(
385 "field \"type\" = \"%s\"" % sm_type)
387 # SM expects at least one entry of any SR type
388 if len(sm_rec) > 0:
389 result = list(sm_rec.values())[0]['capabilities']
391 session.xenapi.logout()
392 return result
395def sr_get_driver_info(driver_info):
396 results = {}
397 # first add in the vanilla stuff
398 for key in ['name', 'description', 'vendor', 'copyright', \
399 'driver_version', 'required_api_version']:
400 results[key] = driver_info[key]
401 # add the capabilities (xmlrpc array)
402 # enforcing activate/deactivate for blktap2
403 caps = driver_info['capabilities']
404 if "ATOMIC_PAUSE" in caps: 404 ↛ 405line 404 didn't jump to line 405, because the condition on line 404 was never true
405 for cap in ("VDI_ACTIVATE", "VDI_DEACTIVATE"):
406 if not cap in caps:
407 caps.append(cap)
408 elif "VDI_ACTIVATE" in caps or "VDI_DEACTIVATE" in caps: 408 ↛ 409line 408 didn't jump to line 409, because the condition on line 408 was never true
409 SMlog("Warning: vdi_[de]activate present for %s" % driver_info["name"])
411 results['capabilities'] = caps
412 # add in the configuration options
413 options = []
414 for option in driver_info['configuration']:
415 options.append({'key': option[0], 'description': option[1]})
416 results['configuration'] = options
417 return xmlrpc.client.dumps((results, ), "", True)
420def return_nil():
421 return xmlrpc.client.dumps((None, ), "", True, allow_none=True)
424def SRtoXML(SRlist):
425 dom = xml.dom.minidom.Document()
426 driver = dom.createElement("SRlist")
427 dom.appendChild(driver)
429 for key in SRlist.keys():
430 dict = SRlist[key]
431 entry = dom.createElement("SR")
432 driver.appendChild(entry)
434 e = dom.createElement("UUID")
435 entry.appendChild(e)
436 textnode = dom.createTextNode(key)
437 e.appendChild(textnode)
439 if 'size' in dict:
440 e = dom.createElement("Size")
441 entry.appendChild(e)
442 textnode = dom.createTextNode(str(dict['size']))
443 e.appendChild(textnode)
445 if 'storagepool' in dict:
446 e = dom.createElement("StoragePool")
447 entry.appendChild(e)
448 textnode = dom.createTextNode(str(dict['storagepool']))
449 e.appendChild(textnode)
451 if 'aggregate' in dict:
452 e = dom.createElement("Aggregate")
453 entry.appendChild(e)
454 textnode = dom.createTextNode(str(dict['aggregate']))
455 e.appendChild(textnode)
457 return dom.toprettyxml()
460def pathexists(path):
461 try:
462 os.lstat(path)
463 return True
464 except OSError as inst:
465 if inst.errno == errno.EIO: 465 ↛ 466line 465 didn't jump to line 466, because the condition on line 465 was never true
466 time.sleep(1)
467 try:
468 listdir(os.path.realpath(os.path.dirname(path)))
469 os.lstat(path)
470 return True
471 except:
472 pass
473 raise CommandException(errno.EIO, "os.lstat(%s)" % path, "failed")
474 return False
477def force_unlink(path):
478 try:
479 os.unlink(path)
480 except OSError as e:
481 if e.errno != errno.ENOENT:
482 raise
485def create_secret(session, secret):
486 ref = session.xenapi.secret.create({'value': secret})
487 return session.xenapi.secret.get_uuid(ref)
490def get_secret(session, uuid):
491 try:
492 ref = session.xenapi.secret.get_by_uuid(uuid)
493 return session.xenapi.secret.get_value(ref)
494 except:
495 raise xs_errors.XenError('InvalidSecret', opterr='Unable to look up secret [%s]' % uuid)
498def get_real_path(path):
499 "Follow symlinks to the actual file"
500 absPath = path
501 directory = ''
502 while os.path.islink(absPath):
503 directory = os.path.dirname(absPath)
504 absPath = os.readlink(absPath)
505 absPath = os.path.join(directory, absPath)
506 return absPath
509def wait_for_path(path, timeout):
510 for i in range(0, timeout): 510 ↛ 514line 510 didn't jump to line 514, because the loop on line 510 didn't complete
511 if len(glob.glob(path)): 511 ↛ 513line 511 didn't jump to line 513, because the condition on line 511 was never false
512 return True
513 time.sleep(1)
514 return False
517def wait_for_nopath(path, timeout):
518 for i in range(0, timeout):
519 if not os.path.exists(path):
520 return True
521 time.sleep(1)
522 return False
525def wait_for_path_multi(path, timeout):
526 for i in range(0, timeout):
527 paths = glob.glob(path)
528 SMlog("_wait_for_paths_multi: paths = %s" % paths)
529 if len(paths):
530 SMlog("_wait_for_paths_multi: return first path: %s" % paths[0])
531 return paths[0]
532 time.sleep(1)
533 return ""
536def isdir(path):
537 try:
538 st = os.stat(path)
539 return stat.S_ISDIR(st.st_mode)
540 except OSError as inst:
541 if inst.errno == errno.EIO: 541 ↛ 542line 541 didn't jump to line 542, because the condition on line 541 was never true
542 raise CommandException(errno.EIO, "os.stat(%s)" % path, "failed")
543 return False
546def get_single_entry(path):
547 f = open(path, 'r')
548 line = f.readline()
549 f.close()
550 return line.rstrip()
553def get_fs_size(path):
554 st = ioretry_stat(path)
555 return st.f_blocks * st.f_frsize
558def get_fs_utilisation(path):
559 st = ioretry_stat(path)
560 return (st.f_blocks - st.f_bfree) * \
561 st.f_frsize
564def ismount(path):
565 """Test whether a path is a mount point"""
566 try:
567 s1 = os.stat(path)
568 s2 = os.stat(os.path.join(path, '..'))
569 except OSError as inst:
570 raise CommandException(inst.errno, "os.stat")
571 dev1 = s1.st_dev
572 dev2 = s2.st_dev
573 if dev1 != dev2:
574 return True # path/.. on a different device as path
575 ino1 = s1.st_ino
576 ino2 = s2.st_ino
577 if ino1 == ino2:
578 return True # path/.. is the same i-node as path
579 return False
582def makedirs(name, mode=0o777):
583 head, tail = os.path.split(name)
584 if not tail: 584 ↛ 585line 584 didn't jump to line 585, because the condition on line 584 was never true
585 head, tail = os.path.split(head)
586 if head and tail and not pathexists(head):
587 makedirs(head, mode)
588 if tail == os.curdir:
589 return
590 try:
591 os.mkdir(name, mode)
592 except OSError as exc:
593 if exc.errno == errno.EEXIST and os.path.isdir(name): 593 ↛ 594line 593 didn't jump to line 594, because the condition on line 593 was never true
594 if mode:
595 os.chmod(name, mode)
596 pass
597 else:
598 raise
601def zeroOut(path, fromByte, bytes):
602 """write 'bytes' zeros to 'path' starting from fromByte (inclusive)"""
603 blockSize = 4096
605 fromBlock = fromByte // blockSize
606 if fromByte % blockSize:
607 fromBlock += 1
608 bytesBefore = fromBlock * blockSize - fromByte
609 if bytesBefore > bytes:
610 bytesBefore = bytes
611 bytes -= bytesBefore
612 cmd = [CMD_DD, "if=/dev/zero", "of=%s" % path, "bs=1",
613 "seek=%s" % fromByte, "count=%s" % bytesBefore]
614 try:
615 pread2(cmd)
616 except CommandException:
617 return False
619 blocks = bytes // blockSize
620 bytes -= blocks * blockSize
621 fromByte = (fromBlock + blocks) * blockSize
622 if blocks:
623 cmd = [CMD_DD, "if=/dev/zero", "of=%s" % path, "bs=%s" % blockSize,
624 "seek=%s" % fromBlock, "count=%s" % blocks]
625 try:
626 pread2(cmd)
627 except CommandException:
628 return False
630 if bytes:
631 cmd = [CMD_DD, "if=/dev/zero", "of=%s" % path, "bs=1",
632 "seek=%s" % fromByte, "count=%s" % bytes]
633 try:
634 pread2(cmd)
635 except CommandException:
636 return False
638 return True
641def match_rootdev(s):
642 regex = re.compile("^PRIMARY_DISK")
643 return regex.search(s, 0)
646def getrootdev():
647 filename = '/etc/xensource-inventory'
648 try:
649 f = open(filename, 'r')
650 except:
651 raise xs_errors.XenError('EIO', \
652 opterr="Unable to open inventory file [%s]" % filename)
653 rootdev = ''
654 for line in filter(match_rootdev, f.readlines()):
655 rootdev = line.split("'")[1]
656 if not rootdev: 656 ↛ 657line 656 didn't jump to line 657, because the condition on line 656 was never true
657 raise xs_errors.XenError('NoRootDev')
658 return rootdev
661def getrootdevID():
662 rootdev = getrootdev()
663 try:
664 rootdevID = scsiutil.getSCSIid(rootdev)
665 except:
666 SMlog("util.getrootdevID: Unable to verify serial or SCSIid of device: %s" \
667 % rootdev)
668 return ''
670 if not len(rootdevID):
671 SMlog("util.getrootdevID: Unable to identify scsi device [%s] via scsiID" \
672 % rootdev)
674 return rootdevID
677def get_localAPI_session():
678 # First acquire a valid session
679 session = XenAPI.xapi_local()
680 try:
681 session.xenapi.login_with_password('root', '', '', 'SM')
682 except:
683 raise xs_errors.XenError('APISession')
684 return session
687def get_this_host():
688 uuid = None
689 f = open("/etc/xensource-inventory", 'r')
690 for line in f.readlines():
691 if line.startswith("INSTALLATION_UUID"):
692 uuid = line.split("'")[1]
693 f.close()
694 return uuid
697def get_master_ref(session):
698 pools = session.xenapi.pool.get_all()
699 return session.xenapi.pool.get_master(pools[0])
702def get_master_rec(session):
703 return session.xenapi.host.get_record(get_master_ref(session))
706def is_master(session):
707 return get_this_host_ref(session) == get_master_ref(session)
710def get_master_address():
711 address = None
712 try:
713 fd = open('/etc/xensource/pool.conf', 'r')
714 try:
715 items = fd.readline().split(':')
716 if items[0].strip() == 'master':
717 address = 'localhost'
718 else:
719 address = items[1].strip()
720 finally:
721 fd.close()
722 except Exception:
723 pass
724 return address
728def get_localhost_ref(session):
729 filename = '/etc/xensource-inventory'
730 try:
731 f = open(filename, 'r')
732 except:
733 raise xs_errors.XenError('EIO', \
734 opterr="Unable to open inventory file [%s]" % filename)
735 domid = ''
736 for line in filter(match_domain_id, f.readlines()):
737 domid = line.split("'")[1]
738 if not domid:
739 raise xs_errors.XenError('APILocalhost')
741 vms = session.xenapi.VM.get_all_records_where('field "uuid" = "%s"' % domid)
742 for vm in vms:
743 record = vms[vm]
744 if record["uuid"] == domid:
745 hostid = record["resident_on"]
746 return hostid
747 raise xs_errors.XenError('APILocalhost')
750def match_domain_id(s):
751 regex = re.compile("^CONTROL_DOMAIN_UUID")
752 return regex.search(s, 0)
755def get_hosts_attached_on(session, vdi_uuids):
756 host_refs = {}
757 for vdi_uuid in vdi_uuids:
758 try:
759 vdi_ref = session.xenapi.VDI.get_by_uuid(vdi_uuid)
760 except XenAPI.Failure:
761 SMlog("VDI %s not in db, ignoring" % vdi_uuid)
762 continue
763 sm_config = session.xenapi.VDI.get_sm_config(vdi_ref)
764 for key in [x for x in sm_config.keys() if x.startswith('host_')]:
765 host_refs[key[len('host_'):]] = True
766 return host_refs.keys()
769def get_this_host_ref(session):
770 host_uuid = get_this_host()
771 host_ref = session.xenapi.host.get_by_uuid(host_uuid)
772 return host_ref
775def get_slaves_attached_on(session, vdi_uuids):
776 "assume this host is the SR master"
777 host_refs = get_hosts_attached_on(session, vdi_uuids)
778 master_ref = get_this_host_ref(session)
779 return [x for x in host_refs if x != master_ref]
782def get_online_hosts(session):
783 online_hosts = []
784 hosts = session.xenapi.host.get_all_records()
785 for host_ref, host_rec in hosts.items():
786 metricsRef = host_rec["metrics"]
787 metrics = session.xenapi.host_metrics.get_record(metricsRef)
788 if metrics["live"]:
789 online_hosts.append(host_ref)
790 return online_hosts
793def get_all_slaves(session):
794 "assume this host is the SR master"
795 host_refs = get_online_hosts(session)
796 master_ref = get_this_host_ref(session)
797 return [x for x in host_refs if x != master_ref]
800def is_attached_rw(sm_config):
801 for key, val in sm_config.items():
802 if key.startswith("host_") and val == "RW":
803 return True
804 return False
807def attached_as(sm_config):
808 for key, val in sm_config.items():
809 if key.startswith("host_") and (val == "RW" or val == "RO"): 809 ↛ 810line 809 didn't jump to line 810, because the condition on line 809 was never true
810 return val
813def find_my_pbd_record(session, host_ref, sr_ref):
814 try:
815 pbds = session.xenapi.PBD.get_all_records()
816 for pbd_ref in pbds.keys():
817 if pbds[pbd_ref]['host'] == host_ref and pbds[pbd_ref]['SR'] == sr_ref:
818 return [pbd_ref, pbds[pbd_ref]]
819 return None
820 except Exception as e:
821 SMlog("Caught exception while looking up PBD for host %s SR %s: %s" % (str(host_ref), str(sr_ref), str(e)))
822 return None
825def find_my_pbd(session, host_ref, sr_ref):
826 ret = find_my_pbd_record(session, host_ref, sr_ref)
827 if ret is not None:
828 return ret[0]
829 else:
830 return None
833def test_hostPBD_devs(session, sr_uuid, devs):
834 host = get_localhost_ref(session)
835 sr = session.xenapi.SR.get_by_uuid(sr_uuid)
836 try:
837 pbds = session.xenapi.PBD.get_all_records()
838 except:
839 raise xs_errors.XenError('APIPBDQuery')
840 for dev in devs.split(','):
841 for pbd in pbds:
842 record = pbds[pbd]
843 # it's ok if it's *our* PBD
844 if record["SR"] == sr:
845 break
846 if record["host"] == host:
847 devconfig = record["device_config"]
848 if 'device' in devconfig:
849 for device in devconfig['device'].split(','):
850 if os.path.realpath(device) == os.path.realpath(dev):
851 return True
852 return False
855def test_hostPBD_lun(session, targetIQN, LUNid):
856 host = get_localhost_ref(session)
857 try:
858 pbds = session.xenapi.PBD.get_all_records()
859 except:
860 raise xs_errors.XenError('APIPBDQuery')
861 for pbd in pbds:
862 record = pbds[pbd]
863 if record["host"] == host:
864 devconfig = record["device_config"]
865 if 'targetIQN' in devconfig and 'LUNid' in devconfig:
866 if devconfig['targetIQN'] == targetIQN and \
867 devconfig['LUNid'] == LUNid:
868 return True
869 return False
872def test_SCSIid(session, sr_uuid, SCSIid):
873 if sr_uuid is not None:
874 sr = session.xenapi.SR.get_by_uuid(sr_uuid)
875 try:
876 pbds = session.xenapi.PBD.get_all_records()
877 except:
878 raise xs_errors.XenError('APIPBDQuery')
879 for pbd in pbds:
880 record = pbds[pbd]
881 # it's ok if it's *our* PBD
882 # During FC SR creation, devscan.py passes sr_uuid as None
883 if sr_uuid is not None:
884 if record["SR"] == sr:
885 break
886 devconfig = record["device_config"]
887 sm_config = session.xenapi.SR.get_sm_config(record["SR"])
888 if 'SCSIid' in devconfig and devconfig['SCSIid'] == SCSIid:
889 return True
890 elif 'SCSIid' in sm_config and sm_config['SCSIid'] == SCSIid:
891 return True
892 elif 'scsi-' + SCSIid in sm_config:
893 return True
894 return False
897class TimeoutException(SMException):
898 pass
901def timeout_call(timeoutseconds, function, *arguments):
902 def handler(signum, frame):
903 raise TimeoutException()
904 signal.signal(signal.SIGALRM, handler)
905 signal.alarm(timeoutseconds)
906 try:
907 return function(*arguments)
908 finally:
909 signal.alarm(0)
912def _incr_iscsiSR_refcount(targetIQN, uuid):
913 if not os.path.exists(ISCSI_REFDIR):
914 os.mkdir(ISCSI_REFDIR)
915 filename = os.path.join(ISCSI_REFDIR, targetIQN)
916 try:
917 f = open(filename, 'a+')
918 except:
919 raise xs_errors.XenError('LVMRefCount', \
920 opterr='file %s' % filename)
922 f.seek(0)
923 found = False
924 refcount = 0
925 for line in filter(match_uuid, f.readlines()):
926 refcount += 1
927 if line.find(uuid) != -1:
928 found = True
929 if not found:
930 f.write("%s\n" % uuid)
931 refcount += 1
932 f.close()
933 return refcount
936def _decr_iscsiSR_refcount(targetIQN, uuid):
937 filename = os.path.join(ISCSI_REFDIR, targetIQN)
938 if not os.path.exists(filename):
939 return 0
940 try:
941 f = open(filename, 'a+')
942 except:
943 raise xs_errors.XenError('LVMRefCount', \
944 opterr='file %s' % filename)
946 f.seek(0)
947 output = []
948 refcount = 0
949 for line in filter(match_uuid, f.readlines()):
950 if line.find(uuid) == -1:
951 output.append(line.rstrip())
952 refcount += 1
953 if not refcount:
954 os.unlink(filename)
955 return refcount
957 # Re-open file and truncate
958 f.close()
959 f = open(filename, 'w')
960 for i in range(0, refcount):
961 f.write("%s\n" % output[i])
962 f.close()
963 return refcount
966# The agent enforces 1 PBD per SR per host, so we
967# check for active SR entries not attached to this host
968def test_activePoolPBDs(session, host, uuid):
969 try:
970 pbds = session.xenapi.PBD.get_all_records()
971 except:
972 raise xs_errors.XenError('APIPBDQuery')
973 for pbd in pbds:
974 record = pbds[pbd]
975 if record["host"] != host and record["SR"] == uuid \
976 and record["currently_attached"]:
977 return True
978 return False
981def remove_mpathcount_field(session, host_ref, sr_ref, SCSIid):
982 try:
983 pbdref = find_my_pbd(session, host_ref, sr_ref)
984 if pbdref is not None:
985 key = "mpath-" + SCSIid
986 session.xenapi.PBD.remove_from_other_config(pbdref, key)
987 except:
988 pass
992def _testHost(hostname, port, errstring):
993 SMlog("_testHost: Testing host/port: %s,%d" % (hostname, port))
994 try:
995 sockinfo = socket.getaddrinfo(hostname, int(port))[0]
996 except:
997 logException('Exception occured getting IP for %s' % hostname)
998 raise xs_errors.XenError('DNSError')
1000 timeout = 5
1002 sock = socket.socket(sockinfo[0], socket.SOCK_STREAM)
1003 # Only allow the connect to block for up to timeout seconds
1004 sock.settimeout(timeout)
1005 try:
1006 sock.connect(sockinfo[4])
1007 # Fix for MS storage server bug
1008 sock.send(b'\n')
1009 sock.close()
1010 except socket.error as reason:
1011 SMlog("_testHost: Connect failed after %d seconds (%s) - %s" \
1012 % (timeout, hostname, reason))
1013 raise xs_errors.XenError(errstring)
1016def match_scsiID(s, id):
1017 regex = re.compile(id)
1018 return regex.search(s, 0)
1021def _isSCSIid(s):
1022 regex = re.compile("^scsi-")
1023 return regex.search(s, 0)
1026def test_scsiserial(session, device):
1027 device = os.path.realpath(device)
1028 if not scsiutil._isSCSIdev(device):
1029 SMlog("util.test_scsiserial: Not a serial device: %s" % device)
1030 return False
1031 serial = ""
1032 try:
1033 serial += scsiutil.getserial(device)
1034 except:
1035 # Error allowed, SCSIid is the important one
1036 pass
1038 try:
1039 scsiID = scsiutil.getSCSIid(device)
1040 except:
1041 SMlog("util.test_scsiserial: Unable to verify serial or SCSIid of device: %s" \
1042 % device)
1043 return False
1044 if not len(scsiID):
1045 SMlog("util.test_scsiserial: Unable to identify scsi device [%s] via scsiID" \
1046 % device)
1047 return False
1049 try:
1050 SRs = session.xenapi.SR.get_all_records()
1051 except:
1052 raise xs_errors.XenError('APIFailure')
1053 for SR in SRs:
1054 record = SRs[SR]
1055 conf = record["sm_config"]
1056 if 'devserial' in conf:
1057 for dev in conf['devserial'].split(','):
1058 if _isSCSIid(dev):
1059 if match_scsiID(dev, scsiID):
1060 return True
1061 elif len(serial) and dev == serial:
1062 return True
1063 return False
1066def default(self, field, thunk):
1067 try:
1068 return getattr(self, field)
1069 except:
1070 return thunk()
1073def list_VDI_records_in_sr(sr):
1074 """Helper function which returns a list of all VDI records for this SR
1075 stored in the XenAPI server, useful for implementing SR.scan"""
1076 sr_ref = sr.session.xenapi.SR.get_by_uuid(sr.uuid)
1077 vdis = sr.session.xenapi.VDI.get_all_records_where("field \"SR\" = \"%s\"" % sr_ref)
1078 return vdis
1081# Given a partition (e.g. sda1), get a disk name:
1082def diskFromPartition(partition):
1083 # check whether this is a device mapper device (e.g. /dev/dm-0)
1084 m = re.match('(/dev/)?(dm-[0-9]+)(p[0-9]+)?$', partition)
1085 if m is not None: 1085 ↛ 1086line 1085 didn't jump to line 1086, because the condition on line 1085 was never true
1086 return m.group(2)
1088 numlen = 0 # number of digit characters
1089 m = re.match("\D+(\d+)", partition)
1090 if m is not None: 1090 ↛ 1091line 1090 didn't jump to line 1091, because the condition on line 1090 was never true
1091 numlen = len(m.group(1))
1093 # is it a cciss?
1094 if True in [partition.startswith(x) for x in ['cciss', 'ida', 'rd']]: 1094 ↛ 1095line 1094 didn't jump to line 1095, because the condition on line 1094 was never true
1095 numlen += 1 # need to get rid of trailing 'p'
1097 # is it a mapper path?
1098 if partition.startswith("mapper"): 1098 ↛ 1099line 1098 didn't jump to line 1099, because the condition on line 1098 was never true
1099 if re.search("p[0-9]*$", partition):
1100 numlen = len(re.match("\d+", partition[::-1]).group(0)) + 1
1101 SMlog("Found mapper part, len %d" % numlen)
1102 else:
1103 numlen = 0
1105 # is it /dev/disk/by-id/XYZ-part<k>?
1106 if partition.startswith("disk/by-id"): 1106 ↛ 1107line 1106 didn't jump to line 1107, because the condition on line 1106 was never true
1107 return partition[:partition.rfind("-part")]
1109 return partition[:len(partition) - numlen]
1112def dom0_disks():
1113 """Disks carrying dom0, e.g. ['/dev/sda']"""
1114 disks = []
1115 with open("/etc/mtab", 'r') as f:
1116 for line in f:
1117 (dev, mountpoint, fstype, opts, freq, passno) = line.split(' ')
1118 if mountpoint == '/':
1119 disk = diskFromPartition(dev)
1120 if not (disk in disks):
1121 disks.append(disk)
1122 SMlog("Dom0 disks: %s" % disks)
1123 return disks
1126def set_scheduler_sysfs_node(node, str):
1127 """Set the scheduler for a sysfs node (e.g. '/sys/block/sda')"""
1129 path = os.path.join(node, "queue", "scheduler")
1130 if not os.path.exists(path):
1131 SMlog("no path %s" % path)
1132 return
1133 try:
1134 f = open(path, 'w')
1135 f.write("%s\n" % str)
1136 f.close()
1137 SMlog("Set scheduler to [%s] on [%s]" % (str, node))
1138 except:
1139 SMlog("Error setting scheduler to [%s] on [%s]" % (str, node))
1140 pass
1143def set_scheduler(dev, str):
1144 devices = []
1145 if not scsiutil.match_dm(dev):
1146 # Remove partition numbers
1147 devices.append(diskFromPartition(dev).replace('/', '!'))
1148 else:
1149 rawdev = diskFromPartition(dev)
1150 devices = [os.path.realpath(x)[5:] for x in scsiutil._genReverseSCSIidmap(rawdev.split('/')[-1])]
1152 for d in devices:
1153 set_scheduler_sysfs_node("/sys/block/%s" % d, str)
1156# This function queries XAPI for the existing VDI records for this SR
1157def _getVDIs(srobj):
1158 VDIs = []
1159 try:
1160 sr_ref = getattr(srobj, 'sr_ref')
1161 except AttributeError:
1162 return VDIs
1164 refs = srobj.session.xenapi.SR.get_VDIs(sr_ref)
1165 for vdi in refs:
1166 ref = srobj.session.xenapi.VDI.get_record(vdi)
1167 ref['vdi_ref'] = vdi
1168 VDIs.append(ref)
1169 return VDIs
1172def _getVDI(srobj, vdi_uuid):
1173 vdi = srobj.session.xenapi.VDI.get_by_uuid(vdi_uuid)
1174 ref = srobj.session.xenapi.VDI.get_record(vdi)
1175 ref['vdi_ref'] = vdi
1176 return ref
1179def _convertDNS(name):
1180 addr = socket.getaddrinfo(name, None)[0][4][0]
1181 return addr
1184def _containsVDIinuse(srobj):
1185 VDIs = _getVDIs(srobj)
1186 for vdi in VDIs:
1187 if not vdi['managed']:
1188 continue
1189 sm_config = vdi['sm_config']
1190 if 'SRRef' in sm_config:
1191 try:
1192 PBDs = srobj.session.xenapi.SR.get_PBDs(sm_config['SRRef'])
1193 for pbd in PBDs:
1194 record = PBDs[pbd]
1195 if record["host"] == srobj.host_ref and \
1196 record["currently_attached"]:
1197 return True
1198 except:
1199 pass
1200 return False
1203def isVDICommand(cmd):
1204 if cmd is None or cmd in ["vdi_attach", "vdi_detach", 1204 ↛ 1207line 1204 didn't jump to line 1207, because the condition on line 1204 was never true
1205 "vdi_activate", "vdi_deactivate",
1206 "vdi_epoch_begin", "vdi_epoch_end"]:
1207 return True
1208 else:
1209 return False
1212#########################
1213# Daemon helper functions
1214def p_id_fork():
1215 try:
1216 p_id = os.fork()
1217 except OSError as e:
1218 print("Fork failed: %s (%d)" % (e.strerror, e.errno))
1219 sys.exit(-1)
1221 if (p_id == 0):
1222 os.setsid()
1223 try:
1224 p_id = os.fork()
1225 except OSError as e:
1226 print("Fork failed: %s (%d)" % (e.strerror, e.errno))
1227 sys.exit(-1)
1228 if (p_id == 0):
1229 os.chdir('/opt/xensource/sm')
1230 os.umask(0)
1231 else:
1232 os._exit(0)
1233 else:
1234 os._exit(0)
1237def daemon():
1238 p_id_fork()
1239 # Query the max file descriptor parameter for this process
1240 maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1242 # Close any fds that are open
1243 for fd in range(0, maxfd):
1244 try:
1245 os.close(fd)
1246 except:
1247 pass
1249 # Redirect STDIN to STDOUT and STDERR
1250 os.open('/dev/null', os.O_RDWR)
1251 os.dup2(0, 1)
1252 os.dup2(0, 2)
1253#########################
1255if __debug__:
1256 try:
1257 XE_IOFI_IORETRY
1258 except NameError:
1259 XE_IOFI_IORETRY = os.environ.get('XE_IOFI_IORETRY', None)
1260 if __name__ == 'util' and XE_IOFI_IORETRY is not None: 1260 ↛ 1261line 1260 didn't jump to line 1261, because the condition on line 1260 was never true
1261 __import__('iofi')
1263################################################################################
1264#
1265# Fist points
1266#
1268# * The global variable 'fistpoint' define the list of all possible fistpoints;
1269#
1270# * To activate a fistpoint called 'name', you need to create the file '/tmp/fist_name'
1271# on the SR master;
1272#
1273# * At the moment, activating a fist point can lead to two possible behaviors:
1274# - if '/tmp/fist_LVHDRT_exit' exists, then the function called during the fistpoint is _exit;
1275# - otherwise, the function called is _pause.
1277def _pause(secs, name):
1278 SMlog("Executing fist point %s: sleeping %d seconds ..." % (name, secs))
1279 time.sleep(secs)
1280 SMlog("Executing fist point %s: done" % name)
1283def _exit(name):
1284 SMlog("Executing fist point %s: exiting the current process ..." % name)
1285 raise xs_errors.XenError('FistPoint', opterr='%s' % name)
1288class FistPoint:
1289 def __init__(self, points):
1290 #SMlog("Fist points loaded")
1291 self.points = points
1293 def is_legal(self, name):
1294 return (name in self.points)
1296 def is_active(self, name):
1297 return os.path.exists("/tmp/fist_%s" % name)
1299 def mark_sr(self, name, sruuid, started):
1300 session = get_localAPI_session()
1301 sr = session.xenapi.SR.get_by_uuid(sruuid)
1302 if started:
1303 session.xenapi.SR.add_to_other_config(sr, name, "active")
1304 else:
1305 session.xenapi.SR.remove_from_other_config(sr, name)
1307 def activate(self, name, sruuid):
1308 if name in self.points: 1308 ↛ 1318line 1308 didn't jump to line 1318, because the condition on line 1308 was never false
1309 if self.is_active(name): 1309 ↛ 1310line 1309 didn't jump to line 1310, because the condition on line 1309 was never true
1310 self.mark_sr(name, sruuid, True)
1311 if self.is_active("LVHDRT_exit"):
1312 self.mark_sr(name, sruuid, False)
1313 _exit(name)
1314 else:
1315 _pause(FIST_PAUSE_PERIOD, name)
1316 self.mark_sr(name, sruuid, False)
1317 else:
1318 SMlog("Unknown fist point: %s" % name)
1320 def activate_custom_fn(self, name, fn):
1321 if name in self.points: 1321 ↛ 1327line 1321 didn't jump to line 1327, because the condition on line 1321 was never false
1322 if self.is_active(name): 1322 ↛ 1323line 1322 didn't jump to line 1323, because the condition on line 1322 was never true
1323 SMlog("Executing fist point %s: starting ..." % name)
1324 fn()
1325 SMlog("Executing fist point %s: done" % name)
1326 else:
1327 SMlog("Unknown fist point: %s" % name)
1330def list_find(f, seq):
1331 for item in seq:
1332 if f(item):
1333 return item
1335GCPAUSE_FISTPOINT = "GCLoop_no_pause"
1337fistpoint = FistPoint(["LVHDRT_finding_a_suitable_pair",
1338 "LVHDRT_inflating_the_parent",
1339 "LVHDRT_resizing_while_vdis_are_paused",
1340 "LVHDRT_coalescing_VHD_data",
1341 "LVHDRT_coalescing_before_inflate_grandparent",
1342 "LVHDRT_relinking_grandchildren",
1343 "LVHDRT_before_create_relink_journal",
1344 "LVHDRT_xapiSM_serialization_tests",
1345 "LVHDRT_clone_vdi_after_create_journal",
1346 "LVHDRT_clone_vdi_after_shrink_parent",
1347 "LVHDRT_clone_vdi_after_first_snap",
1348 "LVHDRT_clone_vdi_after_second_snap",
1349 "LVHDRT_clone_vdi_after_parent_hidden",
1350 "LVHDRT_clone_vdi_after_parent_ro",
1351 "LVHDRT_clone_vdi_before_remove_journal",
1352 "LVHDRT_clone_vdi_after_lvcreate",
1353 "LVHDRT_clone_vdi_before_undo_clone",
1354 "LVHDRT_clone_vdi_after_undo_clone",
1355 "LVHDRT_inflate_after_create_journal",
1356 "LVHDRT_inflate_after_setSize",
1357 "LVHDRT_inflate_after_zeroOut",
1358 "LVHDRT_inflate_after_setSizePhys",
1359 "LVHDRT_inflate_after_setSizePhys",
1360 "LVHDRT_coaleaf_before_coalesce",
1361 "LVHDRT_coaleaf_after_coalesce",
1362 "LVHDRT_coaleaf_one_renamed",
1363 "LVHDRT_coaleaf_both_renamed",
1364 "LVHDRT_coaleaf_after_vdirec",
1365 "LVHDRT_coaleaf_before_delete",
1366 "LVHDRT_coaleaf_after_delete",
1367 "LVHDRT_coaleaf_before_remove_j",
1368 "LVHDRT_coaleaf_undo_after_rename",
1369 "LVHDRT_coaleaf_undo_after_rename2",
1370 "LVHDRT_coaleaf_undo_after_refcount",
1371 "LVHDRT_coaleaf_undo_after_deflate",
1372 "LVHDRT_coaleaf_undo_end",
1373 "LVHDRT_coaleaf_stop_after_recovery",
1374 "LVHDRT_coaleaf_finish_after_inflate",
1375 "LVHDRT_coaleaf_finish_end",
1376 "LVHDRT_coaleaf_delay_1",
1377 "LVHDRT_coaleaf_delay_2",
1378 "LVHDRT_coaleaf_delay_3",
1379 "testsm_clone_allow_raw",
1380 "xenrt_default_vdi_type_legacy",
1381 "blktap_activate_inject_failure",
1382 "blktap_activate_error_handling",
1383 GCPAUSE_FISTPOINT,
1384 "cleanup_coalesceVHD_inject_failure",
1385 "cleanup_tracker_no_progress",
1386 "FileSR_fail_hardlink",
1387 "FileSR_fail_snap1",
1388 "FileSR_fail_snap2"])
1391def set_dirty(session, sr):
1392 try:
1393 session.xenapi.SR.add_to_other_config(sr, "dirty", "")
1394 SMlog("set_dirty %s succeeded" % (repr(sr)))
1395 except:
1396 SMlog("set_dirty %s failed (flag already set?)" % (repr(sr)))
1399def doesFileHaveOpenHandles(fileName):
1400 SMlog("Entering doesFileHaveOpenHandles with file: %s" % fileName)
1401 (retVal, processAndPidTuples) = \
1402 findRunningProcessOrOpenFile(fileName, False)
1404 if not retVal:
1405 SMlog("Failed to determine if file %s has open handles." % \
1406 fileName)
1407 # err on the side of caution
1408 return True
1409 else:
1410 if len(processAndPidTuples) > 0:
1411 return True
1412 else:
1413 return False
1416# extract SR uuid from the passed in devmapper entry and return
1417# /dev/mapper/VG_XenStorage--c3d82e92--cb25--c99b--b83a--482eebab4a93-MGT
1418def extractSRFromDevMapper(path):
1419 try:
1420 path = os.path.basename(path)
1421 path = path[len('VG_XenStorage-') + 1:]
1422 path = path.replace('--', '/')
1423 path = path[0:path.rfind('-')]
1424 return path.replace('/', '-')
1425 except:
1426 return ''
1429# Looks at /proc and figures either
1430# If a process is still running (default), returns open file names
1431# If any running process has open handles to the given file (process = False)
1432# returns process names and pids
1433def findRunningProcessOrOpenFile(name, process=True):
1434 retVal = True
1435 links = []
1436 processandpids = []
1437 sockets = set()
1438 try:
1439 SMlog("Entering findRunningProcessOrOpenFile with params: %s" % \
1440 [name, process])
1442 # Look at all pids
1443 pids = [pid for pid in os.listdir('/proc') if pid.isdigit()]
1444 for pid in sorted(pids):
1445 try:
1446 try:
1447 f = None
1448 f = open(os.path.join('/proc', pid, 'cmdline'), 'r')
1449 prog = f.read()[:-1]
1450 if prog: 1450 ↛ 1459line 1450 didn't jump to line 1459, because the condition on line 1450 was never false
1451 # Just want the process name
1452 argv = prog.split('\x00')
1453 prog = argv[0]
1454 except IOError as e:
1455 if e.errno in (errno.ENOENT, errno.ESRCH):
1456 SMlog("ERROR %s reading %s, ignore" % (e.errno, pid))
1457 continue
1458 finally:
1459 if f is not None: 1459 ↛ 1444, 1459 ↛ 14622 missed branches: 1) line 1459 didn't jump to line 1444, because the continue on line 1457 wasn't executed, 2) line 1459 didn't jump to line 1462, because the condition on line 1459 was never false
1460 f.close() 1460 ↛ 1444line 1460 didn't jump to line 1444, because the continue on line 1457 wasn't executed
1462 try:
1463 fd_dir = os.path.join('/proc', pid, 'fd')
1464 files = os.listdir(fd_dir)
1465 except OSError as e:
1466 if e.errno in (errno.ENOENT, errno.ESRCH):
1467 SMlog("ERROR %s reading fds for %s, ignore" % (e.errno, pid))
1468 # Ignore pid that are no longer valid
1469 continue
1470 else:
1471 raise
1473 for file in files:
1474 try:
1475 link = os.readlink(os.path.join(fd_dir, file))
1476 except OSError:
1477 continue
1479 if process: 1479 ↛ 1484line 1479 didn't jump to line 1484, because the condition on line 1479 was never false
1480 if name == prog: 1480 ↛ 1473line 1480 didn't jump to line 1473, because the condition on line 1480 was never false
1481 links.append(link)
1482 else:
1483 # need to return process name and pid tuples
1484 if link == name:
1485 SMlog("File %s has an open handle with process %s "
1486 "with pid %s" % (name, prog, pid))
1487 processandpids.append((prog, pid))
1489 # Get the connected sockets
1490 if name == prog:
1491 sockets.update(get_connected_sockets(pid))
1492 except Exception as e:
1493 SMlog("Exception checking running process or open file handles. " \
1494 "Error: %s" % str(e))
1495 retVal = False
1497 if process: 1497 ↛ 1500line 1497 didn't jump to line 1500, because the condition on line 1497 was never false
1498 return retVal, links, sockets
1499 else:
1500 return retVal, processandpids
1503def get_connected_sockets(pid):
1504 sockets = set()
1505 try:
1506 # Lines in /proc/<pid>/net/unix are formatted as follows
1507 # (see Linux source net/unix/af_unix.c, unix_seq_show() )
1508 # - Pointer address to socket (hex)
1509 # - Refcount (HEX)
1510 # - 0
1511 # - State (HEX, 0 or __SO_ACCEPTCON)
1512 # - Type (HEX - but only 0001 of interest)
1513 # - Connection state (HEX - but only 03, SS_CONNECTED of interest)
1514 # - Inode number
1515 # - Path (optional)
1516 open_sock_matcher = re.compile(
1517 r'^[0-9a-f]+: [0-9A-Fa-f]+ [0-9A-Fa-f]+ [0-9A-Fa-f]+ 0001 03 \d+ (.*)$')
1518 with open(
1519 os.path.join('/proc', str(pid), 'net', 'unix'), 'r') as f:
1520 lines = f.readlines()
1521 for line in lines:
1522 match = open_sock_matcher.match(line)
1523 if match:
1524 sockets.add(match[1])
1525 except OSError as e:
1526 if e.errno in (errno.ENOENT, errno.ESRCH):
1527 # Ignore pid that are no longer valid
1528 SMlog("ERROR %s reading sockets for %s, ignore" %
1529 (e.errno, pid))
1530 else:
1531 raise
1532 return sockets
1535def retry(f, maxretry=20, period=3, exceptions=[Exception]):
1536 retries = 0
1537 while True:
1538 try:
1539 return f()
1540 except Exception as e:
1541 for exception in exceptions:
1542 if isinstance(e, exception):
1543 SMlog('Got exception: {}. Retry number: {}'.format(
1544 str(e), retries
1545 ))
1546 break
1547 else:
1548 SMlog('Got bad exception: {}. Raising...'.format(e))
1549 raise e
1551 retries += 1
1552 if retries >= maxretry:
1553 break
1555 time.sleep(period)
1557 return f()
1560def getCslDevPath(svid):
1561 basepath = "/dev/disk/by-csldev/"
1562 if svid.startswith("NETAPP_"):
1563 # special attention for NETAPP SVIDs
1564 svid_parts = svid.split("__")
1565 globstr = basepath + "NETAPP__LUN__" + "*" + svid_parts[2] + "*" + svid_parts[-1] + "*"
1566 else:
1567 globstr = basepath + svid + "*"
1569 return globstr
1572# Use device in /dev pointed to by cslg path which consists of svid
1573def get_scsiid_from_svid(md_svid):
1574 cslg_path = getCslDevPath(md_svid)
1575 abs_path = glob.glob(cslg_path)
1576 if abs_path:
1577 real_path = os.path.realpath(abs_path[0])
1578 return scsiutil.getSCSIid(real_path)
1579 else:
1580 return None
1583def get_isl_scsiids(session):
1584 # Get cslg type SRs
1585 SRs = session.xenapi.SR.get_all_records_where('field "type" = "cslg"')
1587 # Iterate through the SR to get the scsi ids
1588 scsi_id_ret = []
1589 for SR in SRs:
1590 sr_rec = SRs[SR]
1591 # Use the md_svid to get the scsi id
1592 scsi_id = get_scsiid_from_svid(sr_rec['sm_config']['md_svid'])
1593 if scsi_id:
1594 scsi_id_ret.append(scsi_id)
1596 # Get the vdis in the SR and do the same procedure
1597 vdi_recs = session.xenapi.VDI.get_all_records_where('field "SR" = "%s"' % SR)
1598 for vdi_rec in vdi_recs:
1599 vdi_rec = vdi_recs[vdi_rec]
1600 scsi_id = get_scsiid_from_svid(vdi_rec['sm_config']['SVID'])
1601 if scsi_id:
1602 scsi_id_ret.append(scsi_id)
1604 return scsi_id_ret
1607class extractXVA:
1608 # streams files as a set of file and checksum, caller should remove
1609 # the files, if not needed. The entire directory (Where the files
1610 # and checksum) will only be deleted as part of class cleanup.
1611 HDR_SIZE = 512
1612 BLOCK_SIZE = 512
1613 SIZE_LEN = 12 - 1 # To remove \0 from tail
1614 SIZE_OFFSET = 124
1615 ZERO_FILLED_REC = 2
1616 NULL_IDEN = '\x00'
1617 DIR_IDEN = '/'
1618 CHECKSUM_IDEN = '.checksum'
1619 OVA_FILE = 'ova.xml'
1621 # Init gunzips the file using a subprocess, and reads stdout later
1622 # as and when needed
1623 def __init__(self, filename):
1624 self.__extract_path = ''
1625 self.__filename = filename
1626 cmd = 'gunzip -cd %s' % filename
1627 try:
1628 self.spawn_p = subprocess.Popen(
1629 cmd, shell=True, \
1630 stdin=subprocess.PIPE, stdout=subprocess.PIPE, \
1631 stderr=subprocess.PIPE, close_fds=True)
1632 except Exception as e:
1633 SMlog("Error: %s. Uncompress failed for %s" % (str(e), filename))
1634 raise Exception(str(e))
1636 # Create dir to extract the files
1637 self.__extract_path = tempfile.mkdtemp()
1639 def __del__(self):
1640 shutil.rmtree(self.__extract_path)
1642 # Class supports Generator expression. 'for f_name, checksum in getTuple()'
1643 # returns filename, checksum content. Returns filename, '' in case
1644 # of checksum file missing. e.g. ova.xml
1645 def getTuple(self):
1646 zerod_record = 0
1647 ret_f_name = ''
1648 ret_base_f_name = ''
1650 try:
1651 # Read tar file as sets of file and checksum.
1652 while True:
1653 # Read the output of spawned process, or output of gunzip
1654 f_hdr = self.spawn_p.stdout.read(self.HDR_SIZE)
1656 # Break out in case of end of file
1657 if f_hdr == '':
1658 if zerod_record == extractXVA.ZERO_FILLED_REC:
1659 break
1660 else:
1661 SMlog('Error. Expects %d zero records', \
1662 extractXVA.ZERO_FILLED_REC)
1663 raise Exception('Unrecognized end of file')
1665 # Watch out for zero records, two zero records
1666 # denote end of file.
1667 if f_hdr == extractXVA.NULL_IDEN * extractXVA.HDR_SIZE:
1668 zerod_record += 1
1669 continue
1671 f_name = f_hdr[:f_hdr.index(extractXVA.NULL_IDEN)]
1672 # File header may be for a folder, if so ignore the header
1673 if not f_name.endswith(extractXVA.DIR_IDEN):
1674 f_size_octal = f_hdr[extractXVA.SIZE_OFFSET: \
1675 extractXVA.SIZE_OFFSET + extractXVA.SIZE_LEN]
1676 f_size = int(f_size_octal, 8)
1677 if f_name.endswith(extractXVA.CHECKSUM_IDEN):
1678 if f_name.rstrip(extractXVA.CHECKSUM_IDEN) == \
1679 ret_base_f_name:
1680 checksum = self.spawn_p.stdout.read(f_size)
1681 yield(ret_f_name, checksum)
1682 else:
1683 # Expects file followed by its checksum
1684 SMlog('Error. Sequence mismatch starting with %s', \
1685 ret_f_name)
1686 raise Exception( \
1687 'Files out of sequence starting with %s', \
1688 ret_f_name)
1689 else:
1690 # In case of ova.xml, read the contents into a file and
1691 # return the file name to the caller. For other files,
1692 # read the contents into a file, it will
1693 # be used when a .checksum file is encountered.
1694 ret_f_name = '%s/%s' % (self.__extract_path, f_name)
1695 ret_base_f_name = f_name
1697 # Check if the folder exists on the target location,
1698 # else create it.
1699 folder_path = ret_f_name[:ret_f_name.rfind('/')]
1700 if not os.path.exists(folder_path):
1701 os.mkdir(folder_path)
1703 # Store the file to the tmp folder, strip the tail \0
1704 f = open(ret_f_name, 'w')
1705 f.write(self.spawn_p.stdout.read(f_size))
1706 f.close()
1707 if f_name == extractXVA.OVA_FILE:
1708 yield(ret_f_name, '')
1710 # Skip zero'd portion of data block
1711 round_off = f_size % extractXVA.BLOCK_SIZE
1712 if round_off != 0:
1713 zeros = self.spawn_p.stdout.read(
1714 extractXVA.BLOCK_SIZE - round_off)
1715 except Exception as e:
1716 SMlog("Error: %s. File set extraction failed %s" % (str(e), \
1717 self.__filename))
1719 # Kill and Drain stdout of the gunzip process,
1720 # else gunzip might block on stdout
1721 os.kill(self.spawn_p.pid, signal.SIGTERM)
1722 self.spawn_p.communicate()
1723 raise Exception(str(e))
1725illegal_xml_chars = [(0x00, 0x08), (0x0B, 0x1F), (0x7F, 0x84), (0x86, 0x9F),
1726 (0xD800, 0xDFFF), (0xFDD0, 0xFDDF), (0xFFFE, 0xFFFF),
1727 (0x1FFFE, 0x1FFFF), (0x2FFFE, 0x2FFFF), (0x3FFFE, 0x3FFFF),
1728 (0x4FFFE, 0x4FFFF), (0x5FFFE, 0x5FFFF), (0x6FFFE, 0x6FFFF),
1729 (0x7FFFE, 0x7FFFF), (0x8FFFE, 0x8FFFF), (0x9FFFE, 0x9FFFF),
1730 (0xAFFFE, 0xAFFFF), (0xBFFFE, 0xBFFFF), (0xCFFFE, 0xCFFFF),
1731 (0xDFFFE, 0xDFFFF), (0xEFFFE, 0xEFFFF), (0xFFFFE, 0xFFFFF),
1732 (0x10FFFE, 0x10FFFF)]
1734illegal_ranges = ["%s-%s" % (chr(low), chr(high))
1735 for (low, high) in illegal_xml_chars
1736 if low < sys.maxunicode]
1738illegal_xml_re = re.compile(u'[%s]' % u''.join(illegal_ranges))
1741def isLegalXMLString(s):
1742 """Tells whether this is a valid XML string (i.e. it does not contain
1743 illegal XML characters specified in
1744 http://www.w3.org/TR/2004/REC-xml-20040204/#charsets).
1745 """
1747 if len(s) > 0:
1748 return re.search(illegal_xml_re, s) is None
1749 else:
1750 return True
1753def unictrunc(string, max_bytes):
1754 """
1755 Returns the number of bytes that is smaller than, or equal to, the number
1756 of bytes specified, such that the UTF-8 encoded string can be correctly
1757 truncated.
1758 string: the string to truncate
1759 max_bytes: the maximum number of bytes the truncated string can be
1760 """
1761 string = string.decode('UTF-8')
1762 cur_bytes = 0
1763 for char in string:
1764 charsize = len(char.encode('UTF-8'))
1765 if cur_bytes + charsize > max_bytes:
1766 break
1767 else:
1768 cur_bytes = cur_bytes + charsize
1769 return cur_bytes
1772def hideValuesInPropMap(propmap, propnames):
1773 """
1774 Worker function: input simple map of prop name/value pairs, and
1775 a list of specific propnames whose values we want to hide.
1776 Loop through the "hide" list, and if any are found, hide the
1777 value and return the altered map.
1778 If none found, return the original map
1779 """
1780 matches = []
1781 for propname in propnames:
1782 if propname in propmap: 1782 ↛ 1783line 1782 didn't jump to line 1783, because the condition on line 1782 was never true
1783 matches.append(propname)
1785 if matches: 1785 ↛ 1786line 1785 didn't jump to line 1786, because the condition on line 1785 was never true
1786 deepCopyRec = copy.deepcopy(propmap)
1787 for match in matches:
1788 deepCopyRec[match] = '******'
1789 return deepCopyRec
1791 return propmap
1792# define the list of propnames whose value we want to hide
1794PASSWD_PROP_KEYS = ['password', 'cifspassword', 'chappassword', 'incoming_chappassword']
1795DEFAULT_SEGMENT_LEN = 950
1798def hidePasswdInConfig(config):
1799 """
1800 Function to hide passwd values in a simple prop map,
1801 for example "device_config"
1802 """
1803 return hideValuesInPropMap(config, PASSWD_PROP_KEYS)
1806def hidePasswdInParams(params, configProp):
1807 """
1808 Function to hide password values in a specified property which
1809 is a simple map of prop name/values, and is itself an prop entry
1810 in a larger property map.
1811 For example, param maps containing "device_config", or
1812 "sm_config", etc
1813 """
1814 params[configProp] = hideValuesInPropMap(params[configProp], PASSWD_PROP_KEYS)
1815 return params
1818def hideMemberValuesInXmlParams(xmlParams, propnames=PASSWD_PROP_KEYS):
1819 """
1820 Function to hide password values in XML params, specifically
1821 for the XML format of incoming params to SR modules.
1822 Uses text parsing: loop through the list of specific propnames
1823 whose values we want to hide, and:
1824 - Assemble a full "prefix" containing each property name, e.g.,
1825 "<member><name>password</name><value>"
1826 - Test the XML if it contains that string, save the index.
1827 - If found, get the index of the ending tag
1828 - Truncate the return string starting with the password value.
1829 - Append the substitute "*******" value string.
1830 - Restore the rest of the original string starting with the end tag.
1831 """
1832 findStrPrefixHead = "<member><name>"
1833 findStrPrefixTail = "</name><value>"
1834 findStrSuffix = "</value>"
1835 strlen = len(xmlParams)
1837 for propname in propnames:
1838 findStrPrefix = findStrPrefixHead + propname + findStrPrefixTail
1839 idx = xmlParams.find(findStrPrefix)
1840 if idx != -1: # if found any of them
1841 idx += len(findStrPrefix)
1842 idx2 = xmlParams.find(findStrSuffix, idx)
1843 if idx2 != -1:
1844 retStr = xmlParams[0:idx]
1845 retStr += "******"
1846 retStr += xmlParams[idx2:strlen]
1847 return retStr
1848 else:
1849 return xmlParams
1850 return xmlParams
1853def splitXmlText(xmlData, segmentLen=DEFAULT_SEGMENT_LEN, showContd=False):
1854 """
1855 Split xml string data into substrings small enough for the
1856 syslog line length limit. Split at tag end markers ( ">" ).
1857 Usage:
1858 strList = []
1859 strList = splitXmlText( longXmlText, maxLineLen ) # maxLineLen is optional
1860 """
1861 remainingData = str(xmlData)
1863 # "Un-pretty-print"
1864 remainingData = remainingData.replace('\n', '')
1865 remainingData = remainingData.replace('\t', '')
1867 remainingChars = len(remainingData)
1868 returnData = ''
1870 thisLineNum = 0
1871 while remainingChars > segmentLen:
1872 thisLineNum = thisLineNum + 1
1873 index = segmentLen
1874 tmpStr = remainingData[:segmentLen]
1875 tmpIndex = tmpStr.rfind('>')
1876 if tmpIndex != -1:
1877 index = tmpIndex + 1
1879 tmpStr = tmpStr[:index]
1880 remainingData = remainingData[index:]
1881 remainingChars = len(remainingData)
1883 if showContd:
1884 if thisLineNum != 1:
1885 tmpStr = '(Cont\'d): ' + tmpStr
1886 tmpStr = tmpStr + ' (Cont\'d):'
1888 returnData += tmpStr + '\n'
1890 if showContd and thisLineNum > 0:
1891 remainingData = '(Cont\'d): ' + remainingData
1892 returnData += remainingData
1894 return returnData
1897def inject_failure():
1898 raise Exception('injected failure')
1901def open_atomic(path, mode=None):
1902 """Atomically creates a file if, and only if it does not already exist.
1903 Leaves the file open and returns the file object.
1905 path: the path to atomically open
1906 mode: "r" (read), "w" (write), or "rw" (read/write)
1907 returns: an open file object"""
1909 assert path
1911 flags = os.O_CREAT | os.O_EXCL
1912 modes = {'r': os.O_RDONLY, 'w': os.O_WRONLY, 'rw': os.O_RDWR}
1913 if mode:
1914 if mode not in modes:
1915 raise Exception('invalid access mode ' + mode)
1916 flags |= modes[mode]
1917 fd = os.open(path, flags)
1918 try:
1919 if mode:
1920 return os.fdopen(fd, mode)
1921 else:
1922 return os.fdopen(fd)
1923 except:
1924 os.close(fd)
1925 raise
1928def isInvalidVDI(exception):
1929 return exception.details[0] == "HANDLE_INVALID" or \
1930 exception.details[0] == "UUID_INVALID"
1933def get_pool_restrictions(session):
1934 """Returns pool restrictions as a map, @session must be already
1935 established."""
1936 return list(session.xenapi.pool.get_all_records().values())[0]['restrictions']
1939def read_caching_is_restricted(session):
1940 """Tells whether read caching is restricted."""
1941 if session is None: 1941 ↛ 1942line 1941 didn't jump to line 1942, because the condition on line 1941 was never true
1942 return True
1943 restrictions = get_pool_restrictions(session)
1944 if 'restrict_read_caching' in restrictions and \ 1944 ↛ 1946line 1944 didn't jump to line 1946, because the condition on line 1944 was never true
1945 restrictions['restrict_read_caching'] == "true":
1946 return True
1947 return False
1950def sessions_less_than_targets(other_config, device_config):
1951 if 'multihomelist' in device_config and 'iscsi_sessions' in other_config:
1952 sessions = int(other_config['iscsi_sessions'])
1953 targets = len(device_config['multihomelist'].split(','))
1954 SMlog("Targets %d and iscsi_sessions %d" % (targets, sessions))
1955 return (sessions < targets)
1956 else:
1957 return False