Coverage for drivers/SR.py : 56%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#!/usr/bin/python3
2#
3# Copyright (C) Citrix Systems Inc.
4#
5# This program is free software; you can redistribute it and/or modify
6# it under the terms of the GNU Lesser General Public License as published
7# by the Free Software Foundation; version 2.1 only.
8#
9# This program is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU Lesser General Public License for more details.
13#
14# You should have received a copy of the GNU Lesser General Public License
15# along with this program; if not, write to the Free Software Foundation, Inc.,
16# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17#
18# SR: Base class for storage repositories
19#
21import VDI
22import xml.dom.minidom
23import xs_errors
24import XenAPI # pylint: disable=import-error
25import xmlrpc.client
26import util
27import copy
28import os
29import traceback
31MOUNT_BASE = '/var/run/sr-mount'
32DEFAULT_TAP = 'vhd'
33TAPDISK_UTIL = '/usr/sbin/td-util'
34MASTER_LVM_CONF = '/etc/lvm/master'
36# LUN per VDI key for XenCenter
37LUNPERVDI = "LUNperVDI"
43def deviceCheck(op):
44 def wrapper(self, *args):
45 if 'device' not in self.dconf:
46 raise xs_errors.XenError('ConfigDeviceMissing')
47 return op(self, *args)
48 return wrapper
51backends = []
54def registerSR(SRClass):
55 """Register SR with handler. All SR subclasses should call this in
56 the module file
57 """
58 backends.append(SRClass)
61def driver(type):
62 """Find the SR for the given dconf string"""
63 for d in backends: 63 ↛ 66line 63 didn't jump to line 66, because the loop on line 63 didn't complete
64 if d.handles(type):
65 return d
66 raise xs_errors.XenError('SRUnknownType')
69class SR(object):
70 """Semi-abstract storage repository object.
72 Attributes:
73 uuid: string, UUID
74 label: string
75 description: string
76 vdis: dictionary, VDI objects indexed by UUID
77 physical_utilisation: int, bytes consumed by VDIs
78 virtual_allocation: int, bytes allocated to this repository (virtual)
79 physical_size: int, bytes consumed by this repository
80 sr_vditype: string, repository type
81 """
83 @staticmethod
84 def handles(type) -> bool:
85 """Returns True if this SR class understands the given dconf string"""
86 return False
88 def __init__(self, srcmd, sr_uuid):
89 """Base class initializer. All subclasses should call SR.__init__
90 in their own
91 initializers.
93 Arguments:
94 srcmd: SRCommand instance, contains parsed arguments
95 """
96 try:
97 self.other_config = {}
98 self.srcmd = srcmd
99 self.dconf = srcmd.dconf
100 if 'session_ref' in srcmd.params:
101 self.session_ref = srcmd.params['session_ref']
102 self.session = XenAPI.xapi_local()
103 self.session._session = self.session_ref
104 if 'subtask_of' in self.srcmd.params: 104 ↛ 105line 104 didn't jump to line 105, because the condition on line 104 was never true
105 self.session.transport.add_extra_header('Subtask-of', self.srcmd.params['subtask_of'])
106 else:
107 self.session = None
109 if 'host_ref' not in self.srcmd.params:
110 self.host_ref = ""
111 else:
112 self.host_ref = self.srcmd.params['host_ref']
114 self.sr_ref = self.srcmd.params.get('sr_ref')
116 if 'device_config' in self.srcmd.params:
117 if self.dconf.get("SRmaster") == "true":
118 os.environ['LVM_SYSTEM_DIR'] = MASTER_LVM_CONF
120 if 'device_config' in self.srcmd.params:
121 if 'SCSIid' in self.srcmd.params['device_config']:
122 dev_path = '/dev/disk/by-scsid/' + self.srcmd.params['device_config']['SCSIid']
123 os.environ['LVM_DEVICE'] = dev_path
124 util.SMlog('Setting LVM_DEVICE to %s' % dev_path)
126 except TypeError:
127 raise Exception(traceback.format_exc())
128 except Exception as e:
129 raise e
130 raise xs_errors.XenError('SRBadXML')
132 self.uuid = sr_uuid
134 self.label = ''
135 self.description = ''
136 self.cmd = srcmd.params['command']
137 self.vdis = {}
138 self.physical_utilisation = 0
139 self.virtual_allocation = 0
140 self.physical_size = 0
141 self.sr_vditype = ''
142 self.passthrough = False
143 # XXX: if this is really needed then we must make a deep copy
144 self.original_srcmd = copy.deepcopy(self.srcmd)
145 self.default_vdi_visibility = True
146 self.scheds = ['none', 'noop']
147 self._mpathinit()
148 self.direct = False
149 self.ops_exclusive = []
150 self.driver_config = {}
152 self.load(sr_uuid)
154 @staticmethod
155 def from_uuid(session, sr_uuid):
156 import importlib.util
158 _SR = session.xenapi.SR
159 sr_ref = _SR.get_by_uuid(sr_uuid)
160 sm_type = _SR.get_type(sr_ref)
161 # NB. load the SM driver module
163 _SM = session.xenapi.SM
164 sms = _SM.get_all_records_where('field "type" = "%s"' % sm_type)
165 sm_ref, sm = sms.popitem()
166 assert not sms
168 driver_path = _SM.get_driver_filename(sm_ref)
169 driver_real = os.path.realpath(driver_path)
170 module_name = os.path.basename(driver_path)
172 spec = importlib.util.spec_from_file_location(module_name, driver_real)
173 module = importlib.util.module_from_spec(spec)
174 spec.loader.exec_module(module)
176 target = driver(sm_type)
177 # NB. get the host pbd's device_config
179 host_ref = util.get_localhost_ref(session)
181 _PBD = session.xenapi.PBD
182 pbds = _PBD.get_all_records_where('field "SR" = "%s" and' % sr_ref +
183 'field "host" = "%s"' % host_ref)
184 pbd_ref, pbd = pbds.popitem()
185 assert not pbds
187 device_config = _PBD.get_device_config(pbd_ref)
188 # NB. make srcmd, to please our supersized SR constructor.
189 # FIXME
191 from SRCommand import SRCommand
192 cmd = SRCommand(module.DRIVER_INFO)
193 cmd.dconf = device_config
194 cmd.params = {'session_ref': session._session,
195 'host_ref': host_ref,
196 'device_config': device_config,
197 'sr_ref': sr_ref,
198 'sr_uuid': sr_uuid,
199 'command': 'nop'}
201 return target(cmd, sr_uuid)
203 def block_setscheduler(self, dev):
204 try:
205 realdev = os.path.realpath(dev)
206 disk = util.diskFromPartition(realdev)
208 # the normal case: the sr default scheduler (typically none/noop),
209 # potentially overridden by SR.other_config:scheduler
210 other_config = self.session.xenapi.SR.get_other_config(self.sr_ref)
211 sched = other_config.get('scheduler')
212 if not sched or sched in self.scheds: 212 ↛ 213line 212 didn't jump to line 213, because the condition on line 212 was never true
213 scheds = self.scheds
214 else:
215 scheds = [sched]
217 # special case: BFQ/CFQ if the underlying disk holds dom0's file systems.
218 if disk in util.dom0_disks(): 218 ↛ 219, 218 ↛ 2212 missed branches: 1) line 218 didn't jump to line 219, because the condition on line 218 was never true, 2) line 218 didn't jump to line 221, because the condition on line 218 was never false
219 scheds = ['bfq', 'cfq']
221 util.SMlog("Block scheduler: %s (%s) wants %s" % (dev, disk, scheds))
222 util.set_scheduler(realdev[5:], scheds)
223 except Exception as e:
224 util.SMlog("Failed to set block scheduler on %s: %s" % (dev, e))
226 def _addLUNperVDIkey(self):
227 try:
228 self.session.xenapi.SR.add_to_sm_config(self.sr_ref, LUNPERVDI, "true")
229 except:
230 pass
232 def create(self, uuid, size) -> None:
233 """Create this repository.
234 This operation may delete existing data.
236 The operation is NOT idempotent. The operation will fail
237 if an SR of the same UUID and driver type already exits.
239 Returns:
240 None
241 Raises:
242 SRUnimplementedMethod
243 """
244 raise xs_errors.XenError('Unimplemented')
246 def delete(self, uuid) -> None:
247 """Delete this repository and its contents.
249 This operation IS idempotent -- it will succeed if the repository
250 exists and can be deleted or if the repository does not exist.
251 The caller must ensure that all VDIs are deactivated and detached
252 and that the SR itself has been detached before delete().
253 The call will FAIL if any VDIs in the SR are in use.
255 Returns:
256 None
257 Raises:
258 SRUnimplementedMethod
259 """
260 raise xs_errors.XenError('Unimplemented')
262 def update(self, uuid) -> None:
263 """Refresh the fields in the SR object
265 Returns:
266 None
267 Raises:
268 SRUnimplementedMethod
269 """
270 # no-op unless individual backends implement it
271 return
273 def attach(self, uuid) -> None:
274 """Initiate local access to the SR. Initialises any
275 device state required to access the substrate.
277 Idempotent.
279 Returns:
280 None
281 Raises:
282 SRUnimplementedMethod
283 """
284 raise xs_errors.XenError('Unimplemented')
286 def after_master_attach(self, uuid) -> None:
287 """Perform actions required after attaching on the pool master
288 Return:
289 None
290 """
291 try:
292 self.scan(uuid)
293 except Exception as e:
294 util.SMlog("Error in SR.after_master_attach %s" % e)
295 msg_name = "POST_ATTACH_SCAN_FAILED"
296 msg_body = "Failed to scan SR %s after attaching, " \
297 "error %s" % (uuid, e)
298 self.session.xenapi.message.create(
299 msg_name, 2, "SR", uuid, msg_body)
301 def detach(self, uuid) -> None:
302 """Remove local access to the SR. Destroys any device
303 state initiated by the sr_attach() operation.
305 Idempotent. All VDIs must be detached in order for the operation
306 to succeed.
308 Returns:
309 None
310 Raises:
311 SRUnimplementedMethod
312 """
313 raise xs_errors.XenError('Unimplemented')
315 def probe(self) -> str:
316 """Perform a backend-specific scan, using the current dconf. If the
317 dconf is complete, then this will return a list of the SRs present of
318 this type on the device, if any. If the dconf is partial, then a
319 backend-specific scan will be performed, returning results that will
320 guide the user in improving the dconf.
322 Idempotent.
324 xapi will ensure that this is serialised wrt any other probes, or
325 attach or detach operations on this host.
327 Returns:
328 An XML fragment containing the scan results. These are specific
329 to the scan being performed, and the current backend.
330 Raises:
331 SRUnimplementedMethod
332 """
333 raise xs_errors.XenError('Unimplemented')
335 def scan(self, uuid) -> None:
336 """
337 Returns:
338 """
339 # Update SR parameters
340 self._db_update()
341 # Synchronise VDI list
342 scanrecord = ScanRecord(self)
343 scanrecord.synchronise()
345 def replay(self, uuid) -> None:
346 """Replay a multi-stage log entry
348 Returns:
349 None
350 Raises:
351 SRUnimplementedMethod
352 """
353 raise xs_errors.XenError('Unimplemented')
355 def content_type(self, uuid) -> str:
356 """Returns the 'content_type' of an SR as a string"""
357 return xmlrpc.client.dumps((str(self.sr_vditype), ), "", True)
359 def load(self, sr_uuid) -> None:
360 """Post-init hook"""
361 pass
363 def check_sr(self, sr_uuid) -> None:
364 """Hook to check SR health"""
365 pass
367 def vdi(self, uuid) -> 'VDI.VDI':
368 """Return VDI object owned by this repository"""
369 raise xs_errors.XenError('Unimplemented')
371 def forget_vdi(self, uuid) -> None:
372 vdi = self.session.xenapi.VDI.get_by_uuid(uuid)
373 self.session.xenapi.VDI.db_forget(vdi)
375 def cleanup(self) -> None:
376 # callback after the op is done
377 pass
379 def _db_update(self):
380 sr = self.session.xenapi.SR.get_by_uuid(self.uuid)
381 self.session.xenapi.SR.set_virtual_allocation(sr, str(self.virtual_allocation))
382 self.session.xenapi.SR.set_physical_size(sr, str(self.physical_size))
383 self.session.xenapi.SR.set_physical_utilisation(sr, str(self.physical_utilisation))
385 def _toxml(self):
386 dom = xml.dom.minidom.Document()
387 element = dom.createElement("sr")
388 dom.appendChild(element)
390 # Add default uuid, physical_utilisation, physical_size and
391 # virtual_allocation entries
392 for attr in ('uuid', 'physical_utilisation', 'virtual_allocation',
393 'physical_size'):
394 try:
395 aval = getattr(self, attr)
396 except AttributeError:
397 raise xs_errors.XenError(
398 'InvalidArg', opterr='Missing required field [%s]' % attr)
400 entry = dom.createElement(attr)
401 element.appendChild(entry)
402 textnode = dom.createTextNode(str(aval))
403 entry.appendChild(textnode)
405 # Add the default_vdi_visibility entry
406 entry = dom.createElement('default_vdi_visibility')
407 element.appendChild(entry)
408 if not self.default_vdi_visibility:
409 textnode = dom.createTextNode('False')
410 else:
411 textnode = dom.createTextNode('True')
412 entry.appendChild(textnode)
414 # Add optional label and description entries
415 for attr in ('label', 'description'):
416 try:
417 aval = getattr(self, attr)
418 except AttributeError:
419 continue
420 if aval:
421 entry = dom.createElement(attr)
422 element.appendChild(entry)
423 textnode = dom.createTextNode(str(aval))
424 entry.appendChild(textnode)
426 # Create VDI sub-list
427 if self.vdis:
428 for uuid in self.vdis:
429 if not self.vdis[uuid].deleted:
430 vdinode = dom.createElement("vdi")
431 element.appendChild(vdinode)
432 self.vdis[uuid]._toxml(dom, vdinode)
434 return dom
436 def _fromxml(self, str, tag):
437 dom = xml.dom.minidom.parseString(str)
438 objectlist = dom.getElementsByTagName(tag)[0]
439 taglist = {}
440 for node in objectlist.childNodes:
441 taglist[node.nodeName] = ""
442 for n in node.childNodes:
443 if n.nodeType == n.TEXT_NODE:
444 taglist[node.nodeName] += n.data
445 return taglist
447 def _splitstring(self, str):
448 elementlist = []
449 for i in range(0, len(str)):
450 elementlist.append(str[i])
451 return elementlist
453 def _mpathinit(self):
454 self.mpath = "false"
455 try:
456 if 'multipathing' in self.dconf and \ 456 ↛ 458line 456 didn't jump to line 458, because the condition on line 456 was never true
457 'multipathhandle' in self.dconf:
458 self.mpath = self.dconf['multipathing']
459 self.mpathhandle = self.dconf['multipathhandle']
460 else:
461 hconf = self.session.xenapi.host.get_other_config(self.host_ref)
462 self.mpath = hconf['multipathing']
463 self.mpathhandle = hconf.get('multipathhandle', 'dmp')
465 if self.mpath != "true": 465 ↛ 469line 465 didn't jump to line 469, because the condition on line 465 was never false
466 self.mpath = "false"
467 self.mpathhandle = "null"
469 if not os.path.exists("/opt/xensource/sm/mpath_%s.py" % self.mpathhandle): 469 ↛ 474line 469 didn't jump to line 474, because the condition on line 469 was never false
470 raise IOError("File does not exist = %s" % self.mpathhandle)
471 except:
472 self.mpath = "false"
473 self.mpathhandle = "null"
474 module_name = "mpath_%s" % self.mpathhandle
475 self.mpathmodule = __import__(module_name)
477 def _mpathHandle(self):
478 if self.mpath == "true": 478 ↛ 479line 478 didn't jump to line 479, because the condition on line 478 was never true
479 self.mpathmodule.activate()
480 else:
481 self.mpathmodule.deactivate()
483 def _pathrefresh(self, obj):
484 SCSIid = getattr(self, 'SCSIid')
485 self.dconf['device'] = self.mpathmodule.path(SCSIid)
486 super(obj, self).load(self.uuid)
488 def _setMultipathableFlag(self, SCSIid=''):
489 try:
490 sm_config = self.session.xenapi.SR.get_sm_config(self.sr_ref)
491 sm_config['multipathable'] = 'true'
492 self.session.xenapi.SR.set_sm_config(self.sr_ref, sm_config)
494 if self.mpath == "true" and len(SCSIid): 494 ↛ 495line 494 didn't jump to line 495, because the condition on line 494 was never true
495 cmd = ['/opt/xensource/sm/mpathcount.py', SCSIid]
496 util.pread2(cmd)
497 except:
498 pass
500 def check_dconf(self, key_list, raise_flag=True):
501 """ Checks if all keys in 'key_list' exist in 'self.dconf'.
503 Input:
504 key_list: a list of keys to check if they exist in self.dconf
505 raise_flag: if true, raise an exception if there are 1 or more
506 keys missing
508 Return: set() containing the missing keys (empty set() if all exist)
509 Raise: xs_errors.XenError('ConfigParamsMissing')
510 """
512 missing_keys = {key for key in key_list if key not in self.dconf}
514 if missing_keys and raise_flag:
515 errstr = 'device-config is missing the following parameters: ' + \
516 ', '.join([key for key in missing_keys])
517 raise xs_errors.XenError('ConfigParamsMissing', opterr=errstr)
519 return missing_keys
522class ScanRecord:
523 def __init__(self, sr):
524 self.sr = sr
525 self.__xenapi_locations = {}
526 self.__xenapi_records = util.list_VDI_records_in_sr(sr)
527 for vdi in list(self.__xenapi_records.keys()): 527 ↛ 528line 527 didn't jump to line 528, because the loop on line 527 never started
528 self.__xenapi_locations[util.to_plain_string(self.__xenapi_records[vdi]['location'])] = vdi
529 self.__sm_records = {}
530 for vdi in list(sr.vdis.values()):
531 # We initialise the sm_config field with the values from the database
532 # The sm_config_overrides contains any new fields we want to add to
533 # sm_config, and also any field to delete (by virtue of having
534 # sm_config_overrides[key]=None)
535 try:
536 if not hasattr(vdi, "sm_config"): 536 ↛ 542line 536 didn't jump to line 542, because the condition on line 536 was never false
537 vdi.sm_config = self.__xenapi_records[self.__xenapi_locations[vdi.location]]['sm_config'].copy()
538 except:
539 util.SMlog("missing config for vdi: %s" % vdi.location)
540 vdi.sm_config = {}
542 vdi._override_sm_config(vdi.sm_config)
544 self.__sm_records[vdi.location] = vdi
546 xenapi_locations = set(self.__xenapi_locations.keys())
547 sm_locations = set(self.__sm_records.keys())
549 # These ones are new on disk
550 self.new = sm_locations.difference(xenapi_locations)
551 # These have disappeared from the disk
552 self.gone = xenapi_locations.difference(sm_locations)
553 # These are the ones which are still present but might have changed...
554 existing = sm_locations.intersection(xenapi_locations)
555 # Synchronise the uuid fields using the location as the primary key
556 # This ensures we know what the UUIDs are even though they aren't stored
557 # in the storage backend.
558 for location in existing: 558 ↛ 559line 558 didn't jump to line 559, because the loop on line 558 never started
559 sm_vdi = self.get_sm_vdi(location)
560 xenapi_vdi = self.get_xenapi_vdi(location)
561 sm_vdi.uuid = util.default(sm_vdi, "uuid", lambda: xenapi_vdi['uuid'])
563 # Only consider those whose configuration looks different
564 self.existing = [x for x in existing if not(self.get_sm_vdi(x).in_sync_with_xenapi_record(self.get_xenapi_vdi(x)))]
566 if len(self.new) != 0:
567 util.SMlog("new VDIs on disk: " + repr(self.new))
568 if len(self.gone) != 0: 568 ↛ 569line 568 didn't jump to line 569, because the condition on line 568 was never true
569 util.SMlog("VDIs missing from disk: " + repr(self.gone))
570 if len(self.existing) != 0: 570 ↛ 571line 570 didn't jump to line 571, because the condition on line 570 was never true
571 util.SMlog("VDIs changed on disk: " + repr(self.existing))
573 def get_sm_vdi(self, location):
574 return self.__sm_records[location]
576 def get_xenapi_vdi(self, location):
577 return self.__xenapi_records[self.__xenapi_locations[location]]
579 def all_xenapi_locations(self):
580 return set(self.__xenapi_locations.keys())
582 def synchronise_new(self):
583 """Add XenAPI records for new disks"""
584 for location in self.new:
585 vdi = self.get_sm_vdi(location)
586 util.SMlog("Introducing VDI with location=%s" % (vdi.location))
587 vdi._db_introduce()
589 def synchronise_gone(self):
590 """Delete XenAPI record for old disks"""
591 for location in self.gone: 591 ↛ 592line 591 didn't jump to line 592, because the loop on line 591 never started
592 vdi = self.get_xenapi_vdi(location)
593 util.SMlog("Forgetting VDI with location=%s uuid=%s" % (util.to_plain_string(vdi['location']), vdi['uuid']))
594 try:
595 self.sr.forget_vdi(vdi['uuid'])
596 except XenAPI.Failure as e:
597 if util.isInvalidVDI(e):
598 util.SMlog("VDI %s not found, ignoring exception" %
599 vdi['uuid'])
600 else:
601 raise
603 def synchronise_existing(self):
604 """Update existing XenAPI records"""
605 for location in self.existing: 605 ↛ 606line 605 didn't jump to line 606, because the loop on line 605 never started
606 vdi = self.get_sm_vdi(location)
608 util.SMlog("Updating VDI with location=%s uuid=%s" % (vdi.location, vdi.uuid))
609 vdi._db_update()
611 def synchronise(self):
612 """Perform the default SM -> xenapi synchronisation; ought to be good enough
613 for most plugins."""
614 self.synchronise_new()
615 self.synchronise_gone()
616 self.synchronise_existing()