Coverage for drivers/SR.py : 54%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#!/usr/bin/python3
2#
3# Copyright (C) Citrix Systems Inc.
4#
5# This program is free software; you can redistribute it and/or modify
6# it under the terms of the GNU Lesser General Public License as published
7# by the Free Software Foundation; version 2.1 only.
8#
9# This program is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU Lesser General Public License for more details.
13#
14# You should have received a copy of the GNU Lesser General Public License
15# along with this program; if not, write to the Free Software Foundation, Inc.,
16# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17#
18# SR: Base class for storage repositories
19#
21import VDI
22import xml.dom.minidom
23import errno
24import xs_errors
25import XenAPI
26import xmlrpc.client
27import util
28import copy
29import os
30import traceback
32MOUNT_BASE = '/var/run/sr-mount'
33DEFAULT_TAP = 'vhd'
34TAPDISK_UTIL = '/usr/sbin/td-util'
35MASTER_LVM_CONF = '/etc/lvm/master'
37# LUN per VDI key for XenCenter
38LUNPERVDI = "LUNperVDI"
41class SRException(Exception):
42 """Exception raised by storage repository operations"""
43 errno = errno.EINVAL
45 def __init__(self, reason):
46 Exception.__init__(self, reason)
48 def toxml(self):
49 return xmlrpc.client.dumps(xmlrpc.client.Fault(int(self.errno), str(self)), "", True)
52class SROSError(SRException):
53 """Wrapper for OSError"""
55 def __init__(self, errno, reason):
56 self.errno = errno
57 Exception.__init__(self, reason)
60def deviceCheck(op):
61 def wrapper(self, *args):
62 if 'device' not in self.dconf:
63 raise xs_errors.XenError('ConfigDeviceMissing')
64 return op(self, *args)
65 return wrapper
68backends = []
71def registerSR(SRClass):
72 """Register SR with handler. All SR subclasses should call this in
73 the module file
74 """
75 backends.append(SRClass)
78def driver(type):
79 """Find the SR for the given dconf string"""
80 for d in backends: 80 ↛ 83line 80 didn't jump to line 83, because the loop on line 80 didn't complete
81 if d.handles(type):
82 return d
83 raise xs_errors.XenError('SRUnknownType')
86class SR(object):
87 """Semi-abstract storage repository object.
89 Attributes:
90 uuid: string, UUID
91 label: string
92 description: string
93 vdis: dictionary, VDI objects indexed by UUID
94 physical_utilisation: int, bytes consumed by VDIs
95 virtual_allocation: int, bytes allocated to this repository (virtual)
96 physical_size: int, bytes consumed by this repository
97 sr_vditype: string, repository type
98 """
100 def handles(type):
101 """Returns True if this SR class understands the given dconf string"""
102 return False
103 handles = staticmethod(handles)
105 def __init__(self, srcmd, sr_uuid):
106 """Base class initializer. All subclasses should call SR.__init__
107 in their own
108 initializers.
110 Arguments:
111 srcmd: SRCommand instance, contains parsed arguments
112 """
113 try:
114 self.srcmd = srcmd
115 self.dconf = srcmd.dconf
116 if 'session_ref' in srcmd.params:
117 self.session_ref = srcmd.params['session_ref']
118 self.session = XenAPI.xapi_local()
119 self.session._session = self.session_ref
120 if 'subtask_of' in self.srcmd.params: 120 ↛ 121line 120 didn't jump to line 121, because the condition on line 120 was never true
121 self.session.transport.add_extra_header('Subtask-of', self.srcmd.params['subtask_of'])
122 else:
123 self.session = None
125 if 'host_ref' not in self.srcmd.params:
126 self.host_ref = ""
127 else:
128 self.host_ref = self.srcmd.params['host_ref']
130 self.sr_ref = self.srcmd.params.get('sr_ref')
132 if 'device_config' in self.srcmd.params:
133 if self.dconf.get("SRmaster") == "true":
134 os.environ['LVM_SYSTEM_DIR'] = MASTER_LVM_CONF
136 if 'device_config' in self.srcmd.params:
137 if 'SCSIid' in self.srcmd.params['device_config']:
138 dev_path = '/dev/disk/by-scsid/' + self.srcmd.params['device_config']['SCSIid']
139 os.environ['LVM_DEVICE'] = dev_path
140 util.SMlog('Setting LVM_DEVICE to %s' % dev_path)
142 except TypeError:
143 raise Exception(traceback.format_exc())
144 except Exception as e:
145 raise e
146 raise xs_errors.XenError('SRBadXML')
148 self.uuid = sr_uuid
150 self.label = ''
151 self.description = ''
152 self.cmd = srcmd.params['command']
153 self.vdis = {}
154 self.physical_utilisation = 0
155 self.virtual_allocation = 0
156 self.physical_size = 0
157 self.sr_vditype = ''
158 self.passthrough = False
159 # XXX: if this is really needed then we must make a deep copy
160 self.original_srcmd = copy.deepcopy(self.srcmd)
161 self.default_vdi_visibility = True
162 self.sched = 'noop'
163 self._mpathinit()
164 self.direct = False
165 self.ops_exclusive = []
166 self.driver_config = {}
168 self.load(sr_uuid)
170 @staticmethod
171 def from_uuid(session, sr_uuid):
172 import imp
174 _SR = session.xenapi.SR
175 sr_ref = _SR.get_by_uuid(sr_uuid)
176 sm_type = _SR.get_type(sr_ref)
177 # NB. load the SM driver module
179 _SM = session.xenapi.SM
180 sms = _SM.get_all_records_where('field "type" = "%s"' % sm_type)
181 sm_ref, sm = sms.popitem()
182 assert not sms
184 driver_path = _SM.get_driver_filename(sm_ref)
185 driver_real = os.path.realpath(driver_path)
186 module_name = os.path.basename(driver_path)
188 module = imp.load_source(module_name, driver_real)
189 target = driver(sm_type)
190 # NB. get the host pbd's device_config
192 host_ref = util.get_localhost_ref(session)
194 _PBD = session.xenapi.PBD
195 pbds = _PBD.get_all_records_where('field "SR" = "%s" and' % sr_ref +
196 'field "host" = "%s"' % host_ref)
197 pbd_ref, pbd = pbds.popitem()
198 assert not pbds
200 device_config = _PBD.get_device_config(pbd_ref)
201 # NB. make srcmd, to please our supersized SR constructor.
202 # FIXME
204 from SRCommand import SRCommand
205 cmd = SRCommand(module.DRIVER_INFO)
206 cmd.dconf = device_config
207 cmd.params = {'session_ref': session._session,
208 'host_ref': host_ref,
209 'device_config': device_config,
210 'sr_ref': sr_ref,
211 'sr_uuid': sr_uuid,
212 'command': 'nop'}
214 return target(cmd, sr_uuid)
216 def block_setscheduler(self, dev):
217 try:
218 realdev = os.path.realpath(dev)
219 disk = util.diskFromPartition(realdev)
221 # the normal case: the sr default scheduler (typically noop),
222 # potentially overridden by SR.other_config:scheduler
223 other_config = self.session.xenapi.SR.get_other_config(self.sr_ref)
224 sched = other_config.get('scheduler')
225 if not sched: 225 ↛ 226line 225 didn't jump to line 226, because the condition on line 225 was never true
226 sched = self.sched
228 # special case: CFQ if the underlying disk holds dom0's file systems.
229 if disk in util.dom0_disks(): 229 ↛ 230, 229 ↛ 2322 missed branches: 1) line 229 didn't jump to line 230, because the condition on line 229 was never true, 2) line 229 didn't jump to line 232, because the condition on line 229 was never false
230 sched = 'cfq'
232 util.SMlog("Block scheduler: %s (%s) wants %s" % (dev, disk, sched))
233 util.set_scheduler(realdev[5:], sched)
235 except Exception as e:
236 util.SMlog("Failed to set block scheduler on %s: %s" % (dev, e))
238 def _addLUNperVDIkey(self):
239 try:
240 self.session.xenapi.SR.add_to_sm_config(self.sr_ref, LUNPERVDI, "true")
241 except:
242 pass
244 def create(self, uuid, size):
245 """Create this repository.
246 This operation may delete existing data.
248 The operation is NOT idempotent. The operation will fail
249 if an SR of the same UUID and driver type already exits.
251 Returns:
252 None
253 Raises:
254 SRUnimplementedMethod
255 """
256 raise xs_errors.XenError('Unimplemented')
258 def delete(self, uuid):
259 """Delete this repository and its contents.
261 This operation IS idempotent -- it will succeed if the repository
262 exists and can be deleted or if the repository does not exist.
263 The caller must ensure that all VDIs are deactivated and detached
264 and that the SR itself has been detached before delete().
265 The call will FAIL if any VDIs in the SR are in use.
267 Returns:
268 None
269 Raises:
270 SRUnimplementedMethod
271 """
272 raise xs_errors.XenError('Unimplemented')
274 def update(self, uuid):
275 """Refresh the fields in the SR object
277 Returns:
278 None
279 Raises:
280 SRUnimplementedMethod
281 """
282 # no-op unless individual backends implement it
283 return
285 def attach(self, uuid):
286 """Initiate local access to the SR. Initialises any
287 device state required to access the substrate.
289 Idempotent.
291 Returns:
292 None
293 Raises:
294 SRUnimplementedMethod
295 """
296 raise xs_errors.XenError('Unimplemented')
298 def after_master_attach(self, uuid):
299 """Perform actions required after attaching on the pool master
300 Return:
301 None
302 """
303 try:
304 self.scan(uuid)
305 except Exception as e:
306 util.SMlog("Error in SR.after_master_attach %s" % e)
307 msg_name = "POST_ATTACH_SCAN_FAILED"
308 msg_body = "Failed to scan SR %s after attaching, " \
309 "error %s" % (uuid, e)
310 self.session.xenapi.message.create(
311 msg_name, 2, "SR", uuid, msg_body)
313 def detach(self, uuid):
314 """Remove local access to the SR. Destroys any device
315 state initiated by the sr_attach() operation.
317 Idempotent. All VDIs must be detached in order for the operation
318 to succeed.
320 Returns:
321 None
322 Raises:
323 SRUnimplementedMethod
324 """
325 raise xs_errors.XenError('Unimplemented')
327 def probe(self):
328 """Perform a backend-specific scan, using the current dconf. If the
329 dconf is complete, then this will return a list of the SRs present of
330 this type on the device, if any. If the dconf is partial, then a
331 backend-specific scan will be performed, returning results that will
332 guide the user in improving the dconf.
334 Idempotent.
336 xapi will ensure that this is serialised wrt any other probes, or
337 attach or detach operations on this host.
339 Returns:
340 An XML fragment containing the scan results. These are specific
341 to the scan being performed, and the current backend.
342 Raises:
343 SRUnimplementedMethod
344 """
345 raise xs_errors.XenError('Unimplemented')
347 def scan(self, uuid):
348 """
349 Returns:
350 """
351 # Update SR parameters
352 self._db_update()
353 # Synchronise VDI list
354 scanrecord = ScanRecord(self)
355 scanrecord.synchronise()
357 def replay(self, uuid):
358 """Replay a multi-stage log entry
360 Returns:
361 None
362 Raises:
363 SRUnimplementedMethod
364 """
365 raise xs_errors.XenError('Unimplemented')
367 def content_type(self, uuid):
368 """Returns the 'content_type' of an SR as a string"""
369 return xmlrpc.client.dumps((str(self.sr_vditype), ), "", True)
371 def load(self, sr_uuid):
372 """Post-init hook"""
373 pass
375 def vdi(self, uuid):
376 """Return VDI object owned by this repository"""
377 if uuid not in self.vdis:
378 self.vdis[uuid] = VDI.VDI(self, uuid)
379 raise xs_errors.XenError('Unimplemented')
380 return self.vdis[uuid]
382 def forget_vdi(self, uuid):
383 vdi = self.session.xenapi.VDI.get_by_uuid(uuid)
384 self.session.xenapi.VDI.db_forget(vdi)
386 def cleanup(self):
387 # callback after the op is done
388 pass
390 def _db_update(self):
391 sr = self.session.xenapi.SR.get_by_uuid(self.uuid)
392 self.session.xenapi.SR.set_virtual_allocation(sr, str(self.virtual_allocation))
393 self.session.xenapi.SR.set_physical_size(sr, str(self.physical_size))
394 self.session.xenapi.SR.set_physical_utilisation(sr, str(self.physical_utilisation))
396 def _toxml(self):
397 dom = xml.dom.minidom.Document()
398 element = dom.createElement("sr")
399 dom.appendChild(element)
401 # Add default uuid, physical_utilisation, physical_size and
402 # virtual_allocation entries
403 for attr in ('uuid', 'physical_utilisation', 'virtual_allocation',
404 'physical_size'):
405 try:
406 aval = getattr(self, attr)
407 except AttributeError:
408 raise xs_errors.XenError(
409 'InvalidArg', opterr='Missing required field [%s]' % attr)
411 entry = dom.createElement(attr)
412 element.appendChild(entry)
413 textnode = dom.createTextNode(str(aval))
414 entry.appendChild(textnode)
416 # Add the default_vdi_visibility entry
417 entry = dom.createElement('default_vdi_visibility')
418 element.appendChild(entry)
419 if not self.default_vdi_visibility:
420 textnode = dom.createTextNode('False')
421 else:
422 textnode = dom.createTextNode('True')
423 entry.appendChild(textnode)
425 # Add optional label and description entries
426 for attr in ('label', 'description'):
427 try:
428 aval = getattr(self, attr)
429 except AttributeError:
430 continue
431 if aval:
432 entry = dom.createElement(attr)
433 element.appendChild(entry)
434 textnode = dom.createTextNode(str(aval))
435 entry.appendChild(textnode)
437 # Create VDI sub-list
438 if self.vdis:
439 for uuid in self.vdis:
440 if not self.vdis[uuid].deleted:
441 vdinode = dom.createElement("vdi")
442 element.appendChild(vdinode)
443 self.vdis[uuid]._toxml(dom, vdinode)
445 return dom
447 def _fromxml(self, str, tag):
448 dom = xml.dom.minidom.parseString(str)
449 objectlist = dom.getElementsByTagName(tag)[0]
450 taglist = {}
451 for node in objectlist.childNodes:
452 taglist[node.nodeName] = ""
453 for n in node.childNodes:
454 if n.nodeType == n.TEXT_NODE:
455 taglist[node.nodeName] += n.data
456 return taglist
458 def _splitstring(self, str):
459 elementlist = []
460 for i in range(0, len(str)):
461 elementlist.append(str[i])
462 return elementlist
464 def _mpathinit(self):
465 self.mpath = "false"
466 try:
467 if 'multipathing' in self.dconf and \ 467 ↛ 469line 467 didn't jump to line 469, because the condition on line 467 was never true
468 'multipathhandle' in self.dconf:
469 self.mpath = self.dconf['multipathing']
470 self.mpathhandle = self.dconf['multipathhandle']
471 else:
472 hconf = self.session.xenapi.host.get_other_config(self.host_ref)
473 self.mpath = hconf['multipathing']
474 self.mpathhandle = hconf.get('multipathhandle', 'dmp')
476 if self.mpath != "true": 476 ↛ 480line 476 didn't jump to line 480, because the condition on line 476 was never false
477 self.mpath = "false"
478 self.mpathhandle = "null"
480 if not os.path.exists("/opt/xensource/sm/mpath_%s.py" % self.mpathhandle): 480 ↛ 485line 480 didn't jump to line 485, because the condition on line 480 was never false
481 raise IOError("File does not exist = %s" % self.mpathhandle)
482 except:
483 self.mpath = "false"
484 self.mpathhandle = "null"
485 module_name = "mpath_%s" % self.mpathhandle
486 self.mpathmodule = __import__(module_name)
488 def _mpathHandle(self):
489 if self.mpath == "true":
490 self.mpathmodule.activate()
491 else:
492 self.mpathmodule.deactivate()
494 def _pathrefresh(self, obj):
495 SCSIid = getattr(self, 'SCSIid')
496 self.dconf['device'] = self.mpathmodule.path(SCSIid)
497 super(obj, self).load(self.uuid)
499 def _setMultipathableFlag(self, SCSIid=''):
500 try:
501 sm_config = self.session.xenapi.SR.get_sm_config(self.sr_ref)
502 sm_config['multipathable'] = 'true'
503 self.session.xenapi.SR.set_sm_config(self.sr_ref, sm_config)
505 if self.mpath == "true" and len(SCSIid):
506 cmd = ['/opt/xensource/sm/mpathcount.py', SCSIid]
507 util.pread2(cmd)
508 except:
509 pass
511 def check_dconf(self, key_list, raise_flag=True):
512 """ Checks if all keys in 'key_list' exist in 'self.dconf'.
514 Input:
515 key_list: a list of keys to check if they exist in self.dconf
516 raise_flag: if true, raise an exception if there are 1 or more
517 keys missing
519 Return: set() containing the missing keys (empty set() if all exist)
520 Raise: xs_errors.XenError('ConfigParamsMissing')
521 """
523 missing_keys = {key for key in key_list if key not in self.dconf}
525 if missing_keys and raise_flag:
526 errstr = 'device-config is missing the following parameters: ' + \
527 ', '.join([key for key in missing_keys])
528 raise xs_errors.XenError('ConfigParamsMissing', opterr=errstr)
530 return missing_keys
533class ScanRecord:
534 def __init__(self, sr):
535 self.sr = sr
536 self.__xenapi_locations = {}
537 self.__xenapi_records = util.list_VDI_records_in_sr(sr)
538 for vdi in list(self.__xenapi_records.keys()): 538 ↛ 539line 538 didn't jump to line 539, because the loop on line 538 never started
539 self.__xenapi_locations[util.to_plain_string(self.__xenapi_records[vdi]['location'])] = vdi
540 self.__sm_records = {}
541 for vdi in list(sr.vdis.values()):
542 # We initialise the sm_config field with the values from the database
543 # The sm_config_overrides contains any new fields we want to add to
544 # sm_config, and also any field to delete (by virtue of having
545 # sm_config_overrides[key]=None)
546 try:
547 if not hasattr(vdi, "sm_config"): 547 ↛ 553line 547 didn't jump to line 553, because the condition on line 547 was never false
548 vdi.sm_config = self.__xenapi_records[self.__xenapi_locations[vdi.location]]['sm_config'].copy()
549 except:
550 util.SMlog("missing config for vdi: %s" % vdi.location)
551 vdi.sm_config = {}
553 vdi._override_sm_config(vdi.sm_config)
555 self.__sm_records[vdi.location] = vdi
557 xenapi_locations = set(self.__xenapi_locations.keys())
558 sm_locations = set(self.__sm_records.keys())
560 # These ones are new on disk
561 self.new = sm_locations.difference(xenapi_locations)
562 # These have disappeared from the disk
563 self.gone = xenapi_locations.difference(sm_locations)
564 # These are the ones which are still present but might have changed...
565 existing = sm_locations.intersection(xenapi_locations)
566 # Synchronise the uuid fields using the location as the primary key
567 # This ensures we know what the UUIDs are even though they aren't stored
568 # in the storage backend.
569 for location in existing: 569 ↛ 570line 569 didn't jump to line 570, because the loop on line 569 never started
570 sm_vdi = self.get_sm_vdi(location)
571 xenapi_vdi = self.get_xenapi_vdi(location)
572 sm_vdi.uuid = util.default(sm_vdi, "uuid", lambda: xenapi_vdi['uuid'])
574 # Only consider those whose configuration looks different
575 self.existing = [x for x in existing if not(self.get_sm_vdi(x).in_sync_with_xenapi_record(self.get_xenapi_vdi(x)))]
577 if len(self.new) != 0:
578 util.SMlog("new VDIs on disk: " + repr(self.new))
579 if len(self.gone) != 0: 579 ↛ 580line 579 didn't jump to line 580, because the condition on line 579 was never true
580 util.SMlog("VDIs missing from disk: " + repr(self.gone))
581 if len(self.existing) != 0: 581 ↛ 582line 581 didn't jump to line 582, because the condition on line 581 was never true
582 util.SMlog("VDIs changed on disk: " + repr(self.existing))
584 def get_sm_vdi(self, location):
585 return self.__sm_records[location]
587 def get_xenapi_vdi(self, location):
588 return self.__xenapi_records[self.__xenapi_locations[location]]
590 def all_xenapi_locations(self):
591 return set(self.__xenapi_locations.keys())
593 def synchronise_new(self):
594 """Add XenAPI records for new disks"""
595 for location in self.new:
596 vdi = self.get_sm_vdi(location)
597 util.SMlog("Introducing VDI with location=%s" % (vdi.location))
598 vdi._db_introduce()
600 def synchronise_gone(self):
601 """Delete XenAPI record for old disks"""
602 for location in self.gone: 602 ↛ 603line 602 didn't jump to line 603, because the loop on line 602 never started
603 vdi = self.get_xenapi_vdi(location)
604 util.SMlog("Forgetting VDI with location=%s uuid=%s" % (util.to_plain_string(vdi['location']), vdi['uuid']))
605 try:
606 self.sr.forget_vdi(vdi['uuid'])
607 except XenAPI.Failure as e:
608 if util.isInvalidVDI(e):
609 util.SMlog("VDI %s not found, ignoring exception" %
610 vdi['uuid'])
611 else:
612 raise
614 def synchronise_existing(self):
615 """Update existing XenAPI records"""
616 for location in self.existing: 616 ↛ 617line 616 didn't jump to line 617, because the loop on line 616 never started
617 vdi = self.get_sm_vdi(location)
619 util.SMlog("Updating VDI with location=%s uuid=%s" % (vdi.location, vdi.uuid))
620 vdi._db_update()
622 def synchronise(self):
623 """Perform the default SM -> xenapi synchronisation; ought to be good enough
624 for most plugins."""
625 self.synchronise_new()
626 self.synchronise_gone()
627 self.synchronise_existing()