Coverage for drivers/SR.py : 55%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#!/usr/bin/python3
2#
3# Copyright (C) Citrix Systems Inc.
4#
5# This program is free software; you can redistribute it and/or modify
6# it under the terms of the GNU Lesser General Public License as published
7# by the Free Software Foundation; version 2.1 only.
8#
9# This program is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU Lesser General Public License for more details.
13#
14# You should have received a copy of the GNU Lesser General Public License
15# along with this program; if not, write to the Free Software Foundation, Inc.,
16# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17#
18# SR: Base class for storage repositories
19#
21import VDI
22import xml.dom.minidom
23import errno
24import xs_errors
25import XenAPI # pylint: disable=import-error
26import xmlrpc.client
27import util
28import copy
29import os
30import traceback
32MOUNT_BASE = '/var/run/sr-mount'
33DEFAULT_TAP = 'vhd'
34TAPDISK_UTIL = '/usr/sbin/td-util'
35MASTER_LVM_CONF = '/etc/lvm/master'
37# LUN per VDI key for XenCenter
38LUNPERVDI = "LUNperVDI"
41class SRException(Exception):
42 """Exception raised by storage repository operations"""
43 errno = errno.EINVAL
45 def __init__(self, reason):
46 Exception.__init__(self, reason)
48 def toxml(self):
49 return xmlrpc.client.dumps(xmlrpc.client.Fault(int(self.errno), str(self)), "", True)
52class SROSError(SRException):
53 """Wrapper for OSError"""
55 def __init__(self, errno, reason):
56 self.errno = errno
57 Exception.__init__(self, reason)
60def deviceCheck(op):
61 def wrapper(self, *args):
62 if 'device' not in self.dconf:
63 raise xs_errors.XenError('ConfigDeviceMissing')
64 return op(self, *args)
65 return wrapper
68backends = []
71def registerSR(SRClass):
72 """Register SR with handler. All SR subclasses should call this in
73 the module file
74 """
75 backends.append(SRClass)
78def driver(type):
79 """Find the SR for the given dconf string"""
80 for d in backends: 80 ↛ 83line 80 didn't jump to line 83, because the loop on line 80 didn't complete
81 if d.handles(type):
82 return d
83 raise xs_errors.XenError('SRUnknownType')
86class SR(object):
87 """Semi-abstract storage repository object.
89 Attributes:
90 uuid: string, UUID
91 label: string
92 description: string
93 vdis: dictionary, VDI objects indexed by UUID
94 physical_utilisation: int, bytes consumed by VDIs
95 virtual_allocation: int, bytes allocated to this repository (virtual)
96 physical_size: int, bytes consumed by this repository
97 sr_vditype: string, repository type
98 """
100 def handles(type):
101 """Returns True if this SR class understands the given dconf string"""
102 return False
103 handles = staticmethod(handles)
105 def __init__(self, srcmd, sr_uuid):
106 """Base class initializer. All subclasses should call SR.__init__
107 in their own
108 initializers.
110 Arguments:
111 srcmd: SRCommand instance, contains parsed arguments
112 """
113 try:
114 self.other_config = {}
115 self.srcmd = srcmd
116 self.dconf = srcmd.dconf
117 if 'session_ref' in srcmd.params:
118 self.session_ref = srcmd.params['session_ref']
119 self.session = XenAPI.xapi_local()
120 self.session._session = self.session_ref
121 if 'subtask_of' in self.srcmd.params: 121 ↛ 122line 121 didn't jump to line 122, because the condition on line 121 was never true
122 self.session.transport.add_extra_header('Subtask-of', self.srcmd.params['subtask_of'])
123 else:
124 self.session = None
126 if 'host_ref' not in self.srcmd.params:
127 self.host_ref = ""
128 else:
129 self.host_ref = self.srcmd.params['host_ref']
131 self.sr_ref = self.srcmd.params.get('sr_ref')
133 if 'device_config' in self.srcmd.params:
134 if self.dconf.get("SRmaster") == "true":
135 os.environ['LVM_SYSTEM_DIR'] = MASTER_LVM_CONF
137 if 'device_config' in self.srcmd.params:
138 if 'SCSIid' in self.srcmd.params['device_config']:
139 dev_path = '/dev/disk/by-scsid/' + self.srcmd.params['device_config']['SCSIid']
140 os.environ['LVM_DEVICE'] = dev_path
141 util.SMlog('Setting LVM_DEVICE to %s' % dev_path)
143 except TypeError:
144 raise Exception(traceback.format_exc())
145 except Exception as e:
146 raise e
147 raise xs_errors.XenError('SRBadXML')
149 self.uuid = sr_uuid
151 self.label = ''
152 self.description = ''
153 self.cmd = srcmd.params['command']
154 self.vdis = {}
155 self.physical_utilisation = 0
156 self.virtual_allocation = 0
157 self.physical_size = 0
158 self.sr_vditype = ''
159 self.passthrough = False
160 # XXX: if this is really needed then we must make a deep copy
161 self.original_srcmd = copy.deepcopy(self.srcmd)
162 self.default_vdi_visibility = True
163 self.scheds = ['none', 'noop']
164 self._mpathinit()
165 self.direct = False
166 self.ops_exclusive = []
167 self.driver_config = {}
169 self.load(sr_uuid)
171 @staticmethod
172 def from_uuid(session, sr_uuid):
173 import imp
175 _SR = session.xenapi.SR
176 sr_ref = _SR.get_by_uuid(sr_uuid)
177 sm_type = _SR.get_type(sr_ref)
178 # NB. load the SM driver module
180 _SM = session.xenapi.SM
181 sms = _SM.get_all_records_where('field "type" = "%s"' % sm_type)
182 sm_ref, sm = sms.popitem()
183 assert not sms
185 driver_path = _SM.get_driver_filename(sm_ref)
186 driver_real = os.path.realpath(driver_path)
187 module_name = os.path.basename(driver_path)
189 module = imp.load_source(module_name, driver_real)
190 target = driver(sm_type)
191 # NB. get the host pbd's device_config
193 host_ref = util.get_localhost_ref(session)
195 _PBD = session.xenapi.PBD
196 pbds = _PBD.get_all_records_where('field "SR" = "%s" and' % sr_ref +
197 'field "host" = "%s"' % host_ref)
198 pbd_ref, pbd = pbds.popitem()
199 assert not pbds
201 device_config = _PBD.get_device_config(pbd_ref)
202 # NB. make srcmd, to please our supersized SR constructor.
203 # FIXME
205 from SRCommand import SRCommand
206 cmd = SRCommand(module.DRIVER_INFO)
207 cmd.dconf = device_config
208 cmd.params = {'session_ref': session._session,
209 'host_ref': host_ref,
210 'device_config': device_config,
211 'sr_ref': sr_ref,
212 'sr_uuid': sr_uuid,
213 'command': 'nop'}
215 return target(cmd, sr_uuid)
217 def block_setscheduler(self, dev):
218 try:
219 realdev = os.path.realpath(dev)
220 disk = util.diskFromPartition(realdev)
222 # the normal case: the sr default scheduler (typically none/noop),
223 # potentially overridden by SR.other_config:scheduler
224 other_config = self.session.xenapi.SR.get_other_config(self.sr_ref)
225 sched = other_config.get('scheduler')
226 if not sched or sched in self.scheds: 226 ↛ 227line 226 didn't jump to line 227, because the condition on line 226 was never true
227 scheds = self.scheds
228 else:
229 scheds = [sched]
231 # special case: BFQ/CFQ if the underlying disk holds dom0's file systems.
232 if disk in util.dom0_disks(): 232 ↛ 233, 232 ↛ 2352 missed branches: 1) line 232 didn't jump to line 233, because the condition on line 232 was never true, 2) line 232 didn't jump to line 235, because the condition on line 232 was never false
233 scheds = ['bfq', 'cfq']
235 util.SMlog("Block scheduler: %s (%s) wants %s" % (dev, disk, scheds))
236 util.set_scheduler(realdev[5:], scheds)
237 except Exception as e:
238 util.SMlog("Failed to set block scheduler on %s: %s" % (dev, e))
240 def _addLUNperVDIkey(self):
241 try:
242 self.session.xenapi.SR.add_to_sm_config(self.sr_ref, LUNPERVDI, "true")
243 except:
244 pass
246 def create(self, uuid, size):
247 """Create this repository.
248 This operation may delete existing data.
250 The operation is NOT idempotent. The operation will fail
251 if an SR of the same UUID and driver type already exits.
253 Returns:
254 None
255 Raises:
256 SRUnimplementedMethod
257 """
258 raise xs_errors.XenError('Unimplemented')
260 def delete(self, uuid):
261 """Delete this repository and its contents.
263 This operation IS idempotent -- it will succeed if the repository
264 exists and can be deleted or if the repository does not exist.
265 The caller must ensure that all VDIs are deactivated and detached
266 and that the SR itself has been detached before delete().
267 The call will FAIL if any VDIs in the SR are in use.
269 Returns:
270 None
271 Raises:
272 SRUnimplementedMethod
273 """
274 raise xs_errors.XenError('Unimplemented')
276 def update(self, uuid):
277 """Refresh the fields in the SR object
279 Returns:
280 None
281 Raises:
282 SRUnimplementedMethod
283 """
284 # no-op unless individual backends implement it
285 return
287 def attach(self, uuid):
288 """Initiate local access to the SR. Initialises any
289 device state required to access the substrate.
291 Idempotent.
293 Returns:
294 None
295 Raises:
296 SRUnimplementedMethod
297 """
298 raise xs_errors.XenError('Unimplemented')
300 def after_master_attach(self, uuid):
301 """Perform actions required after attaching on the pool master
302 Return:
303 None
304 """
305 try:
306 self.scan(uuid)
307 except Exception as e:
308 util.SMlog("Error in SR.after_master_attach %s" % e)
309 msg_name = "POST_ATTACH_SCAN_FAILED"
310 msg_body = "Failed to scan SR %s after attaching, " \
311 "error %s" % (uuid, e)
312 self.session.xenapi.message.create(
313 msg_name, 2, "SR", uuid, msg_body)
315 def detach(self, uuid):
316 """Remove local access to the SR. Destroys any device
317 state initiated by the sr_attach() operation.
319 Idempotent. All VDIs must be detached in order for the operation
320 to succeed.
322 Returns:
323 None
324 Raises:
325 SRUnimplementedMethod
326 """
327 raise xs_errors.XenError('Unimplemented')
329 def probe(self):
330 """Perform a backend-specific scan, using the current dconf. If the
331 dconf is complete, then this will return a list of the SRs present of
332 this type on the device, if any. If the dconf is partial, then a
333 backend-specific scan will be performed, returning results that will
334 guide the user in improving the dconf.
336 Idempotent.
338 xapi will ensure that this is serialised wrt any other probes, or
339 attach or detach operations on this host.
341 Returns:
342 An XML fragment containing the scan results. These are specific
343 to the scan being performed, and the current backend.
344 Raises:
345 SRUnimplementedMethod
346 """
347 raise xs_errors.XenError('Unimplemented')
349 def scan(self, uuid):
350 """
351 Returns:
352 """
353 # Update SR parameters
354 self._db_update()
355 # Synchronise VDI list
356 scanrecord = ScanRecord(self)
357 scanrecord.synchronise()
359 def replay(self, uuid):
360 """Replay a multi-stage log entry
362 Returns:
363 None
364 Raises:
365 SRUnimplementedMethod
366 """
367 raise xs_errors.XenError('Unimplemented')
369 def content_type(self, uuid):
370 """Returns the 'content_type' of an SR as a string"""
371 return xmlrpc.client.dumps((str(self.sr_vditype), ), "", True)
373 def load(self, sr_uuid):
374 """Post-init hook"""
375 pass
377 def check_sr(self, sr_uuid):
378 """Hook to check SR health"""
379 pass
381 def vdi(self, uuid):
382 """Return VDI object owned by this repository"""
383 if uuid not in self.vdis:
384 self.vdis[uuid] = VDI.VDI(self, uuid)
385 raise xs_errors.XenError('Unimplemented')
386 return self.vdis[uuid]
388 def forget_vdi(self, uuid):
389 vdi = self.session.xenapi.VDI.get_by_uuid(uuid)
390 self.session.xenapi.VDI.db_forget(vdi)
392 def cleanup(self):
393 # callback after the op is done
394 pass
396 def _db_update(self):
397 sr = self.session.xenapi.SR.get_by_uuid(self.uuid)
398 self.session.xenapi.SR.set_virtual_allocation(sr, str(self.virtual_allocation))
399 self.session.xenapi.SR.set_physical_size(sr, str(self.physical_size))
400 self.session.xenapi.SR.set_physical_utilisation(sr, str(self.physical_utilisation))
402 def _toxml(self):
403 dom = xml.dom.minidom.Document()
404 element = dom.createElement("sr")
405 dom.appendChild(element)
407 # Add default uuid, physical_utilisation, physical_size and
408 # virtual_allocation entries
409 for attr in ('uuid', 'physical_utilisation', 'virtual_allocation',
410 'physical_size'):
411 try:
412 aval = getattr(self, attr)
413 except AttributeError:
414 raise xs_errors.XenError(
415 'InvalidArg', opterr='Missing required field [%s]' % attr)
417 entry = dom.createElement(attr)
418 element.appendChild(entry)
419 textnode = dom.createTextNode(str(aval))
420 entry.appendChild(textnode)
422 # Add the default_vdi_visibility entry
423 entry = dom.createElement('default_vdi_visibility')
424 element.appendChild(entry)
425 if not self.default_vdi_visibility:
426 textnode = dom.createTextNode('False')
427 else:
428 textnode = dom.createTextNode('True')
429 entry.appendChild(textnode)
431 # Add optional label and description entries
432 for attr in ('label', 'description'):
433 try:
434 aval = getattr(self, attr)
435 except AttributeError:
436 continue
437 if aval:
438 entry = dom.createElement(attr)
439 element.appendChild(entry)
440 textnode = dom.createTextNode(str(aval))
441 entry.appendChild(textnode)
443 # Create VDI sub-list
444 if self.vdis:
445 for uuid in self.vdis:
446 if not self.vdis[uuid].deleted:
447 vdinode = dom.createElement("vdi")
448 element.appendChild(vdinode)
449 self.vdis[uuid]._toxml(dom, vdinode)
451 return dom
453 def _fromxml(self, str, tag):
454 dom = xml.dom.minidom.parseString(str)
455 objectlist = dom.getElementsByTagName(tag)[0]
456 taglist = {}
457 for node in objectlist.childNodes:
458 taglist[node.nodeName] = ""
459 for n in node.childNodes:
460 if n.nodeType == n.TEXT_NODE:
461 taglist[node.nodeName] += n.data
462 return taglist
464 def _splitstring(self, str):
465 elementlist = []
466 for i in range(0, len(str)):
467 elementlist.append(str[i])
468 return elementlist
470 def _mpathinit(self):
471 self.mpath = "false"
472 try:
473 if 'multipathing' in self.dconf and \ 473 ↛ 475line 473 didn't jump to line 475, because the condition on line 473 was never true
474 'multipathhandle' in self.dconf:
475 self.mpath = self.dconf['multipathing']
476 self.mpathhandle = self.dconf['multipathhandle']
477 else:
478 hconf = self.session.xenapi.host.get_other_config(self.host_ref)
479 self.mpath = hconf['multipathing']
480 self.mpathhandle = hconf.get('multipathhandle', 'dmp')
482 if self.mpath != "true": 482 ↛ 486line 482 didn't jump to line 486, because the condition on line 482 was never false
483 self.mpath = "false"
484 self.mpathhandle = "null"
486 if not os.path.exists("/opt/xensource/sm/mpath_%s.py" % self.mpathhandle): 486 ↛ 491line 486 didn't jump to line 491, because the condition on line 486 was never false
487 raise IOError("File does not exist = %s" % self.mpathhandle)
488 except:
489 self.mpath = "false"
490 self.mpathhandle = "null"
491 module_name = "mpath_%s" % self.mpathhandle
492 self.mpathmodule = __import__(module_name)
494 def _mpathHandle(self):
495 if self.mpath == "true": 495 ↛ 496line 495 didn't jump to line 496, because the condition on line 495 was never true
496 self.mpathmodule.activate()
497 else:
498 self.mpathmodule.deactivate()
500 def _pathrefresh(self, obj):
501 SCSIid = getattr(self, 'SCSIid')
502 self.dconf['device'] = self.mpathmodule.path(SCSIid)
503 super(obj, self).load(self.uuid)
505 def _setMultipathableFlag(self, SCSIid=''):
506 try:
507 sm_config = self.session.xenapi.SR.get_sm_config(self.sr_ref)
508 sm_config['multipathable'] = 'true'
509 self.session.xenapi.SR.set_sm_config(self.sr_ref, sm_config)
511 if self.mpath == "true" and len(SCSIid):
512 cmd = ['/opt/xensource/sm/mpathcount.py', SCSIid]
513 util.pread2(cmd)
514 except:
515 pass
517 def check_dconf(self, key_list, raise_flag=True):
518 """ Checks if all keys in 'key_list' exist in 'self.dconf'.
520 Input:
521 key_list: a list of keys to check if they exist in self.dconf
522 raise_flag: if true, raise an exception if there are 1 or more
523 keys missing
525 Return: set() containing the missing keys (empty set() if all exist)
526 Raise: xs_errors.XenError('ConfigParamsMissing')
527 """
529 missing_keys = {key for key in key_list if key not in self.dconf}
531 if missing_keys and raise_flag:
532 errstr = 'device-config is missing the following parameters: ' + \
533 ', '.join([key for key in missing_keys])
534 raise xs_errors.XenError('ConfigParamsMissing', opterr=errstr)
536 return missing_keys
539class ScanRecord:
540 def __init__(self, sr):
541 self.sr = sr
542 self.__xenapi_locations = {}
543 self.__xenapi_records = util.list_VDI_records_in_sr(sr)
544 for vdi in list(self.__xenapi_records.keys()): 544 ↛ 545line 544 didn't jump to line 545, because the loop on line 544 never started
545 self.__xenapi_locations[util.to_plain_string(self.__xenapi_records[vdi]['location'])] = vdi
546 self.__sm_records = {}
547 for vdi in list(sr.vdis.values()):
548 # We initialise the sm_config field with the values from the database
549 # The sm_config_overrides contains any new fields we want to add to
550 # sm_config, and also any field to delete (by virtue of having
551 # sm_config_overrides[key]=None)
552 try:
553 if not hasattr(vdi, "sm_config"): 553 ↛ 559line 553 didn't jump to line 559, because the condition on line 553 was never false
554 vdi.sm_config = self.__xenapi_records[self.__xenapi_locations[vdi.location]]['sm_config'].copy()
555 except:
556 util.SMlog("missing config for vdi: %s" % vdi.location)
557 vdi.sm_config = {}
559 vdi._override_sm_config(vdi.sm_config)
561 self.__sm_records[vdi.location] = vdi
563 xenapi_locations = set(self.__xenapi_locations.keys())
564 sm_locations = set(self.__sm_records.keys())
566 # These ones are new on disk
567 self.new = sm_locations.difference(xenapi_locations)
568 # These have disappeared from the disk
569 self.gone = xenapi_locations.difference(sm_locations)
570 # These are the ones which are still present but might have changed...
571 existing = sm_locations.intersection(xenapi_locations)
572 # Synchronise the uuid fields using the location as the primary key
573 # This ensures we know what the UUIDs are even though they aren't stored
574 # in the storage backend.
575 for location in existing: 575 ↛ 576line 575 didn't jump to line 576, because the loop on line 575 never started
576 sm_vdi = self.get_sm_vdi(location)
577 xenapi_vdi = self.get_xenapi_vdi(location)
578 sm_vdi.uuid = util.default(sm_vdi, "uuid", lambda: xenapi_vdi['uuid'])
580 # Only consider those whose configuration looks different
581 self.existing = [x for x in existing if not(self.get_sm_vdi(x).in_sync_with_xenapi_record(self.get_xenapi_vdi(x)))]
583 if len(self.new) != 0:
584 util.SMlog("new VDIs on disk: " + repr(self.new))
585 if len(self.gone) != 0: 585 ↛ 586line 585 didn't jump to line 586, because the condition on line 585 was never true
586 util.SMlog("VDIs missing from disk: " + repr(self.gone))
587 if len(self.existing) != 0: 587 ↛ 588line 587 didn't jump to line 588, because the condition on line 587 was never true
588 util.SMlog("VDIs changed on disk: " + repr(self.existing))
590 def get_sm_vdi(self, location):
591 return self.__sm_records[location]
593 def get_xenapi_vdi(self, location):
594 return self.__xenapi_records[self.__xenapi_locations[location]]
596 def all_xenapi_locations(self):
597 return set(self.__xenapi_locations.keys())
599 def synchronise_new(self):
600 """Add XenAPI records for new disks"""
601 for location in self.new:
602 vdi = self.get_sm_vdi(location)
603 util.SMlog("Introducing VDI with location=%s" % (vdi.location))
604 vdi._db_introduce()
606 def synchronise_gone(self):
607 """Delete XenAPI record for old disks"""
608 for location in self.gone: 608 ↛ 609line 608 didn't jump to line 609, because the loop on line 608 never started
609 vdi = self.get_xenapi_vdi(location)
610 util.SMlog("Forgetting VDI with location=%s uuid=%s" % (util.to_plain_string(vdi['location']), vdi['uuid']))
611 try:
612 self.sr.forget_vdi(vdi['uuid'])
613 except XenAPI.Failure as e:
614 if util.isInvalidVDI(e):
615 util.SMlog("VDI %s not found, ignoring exception" %
616 vdi['uuid'])
617 else:
618 raise
620 def synchronise_existing(self):
621 """Update existing XenAPI records"""
622 for location in self.existing: 622 ↛ 623line 622 didn't jump to line 623, because the loop on line 622 never started
623 vdi = self.get_sm_vdi(location)
625 util.SMlog("Updating VDI with location=%s uuid=%s" % (vdi.location, vdi.uuid))
626 vdi._db_update()
628 def synchronise(self):
629 """Perform the default SM -> xenapi synchronisation; ought to be good enough
630 for most plugins."""
631 self.synchronise_new()
632 self.synchronise_gone()
633 self.synchronise_existing()