Coverage for drivers/linstorvolumemanager.py : 10%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#!/usr/bin/env python3
2#
3# Copyright (C) 2020 Vates SAS - ronan.abhamon@vates.fr
4#
5# This program is free software: you can redistribute it and/or modify
6# it under the terms of the GNU General Public License as published by
7# the Free Software Foundation, either version 3 of the License, or
8# (at your option) any later version.
9# This program is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU General Public License for more details.
13#
14# You should have received a copy of the GNU General Public License
15# along with this program. If not, see <https://www.gnu.org/licenses/>.
16#
19import distutils.util
20import errno
21import glob
22import json
23import linstor
24import os.path
25import re
26import shutil
27import socket
28import stat
29import time
30import util
31import uuid
34# Contains the data of the "/var/lib/linstor" directory.
35DATABASE_VOLUME_NAME = 'xcp-persistent-database'
36DATABASE_SIZE = 1 << 30 # 1GB.
37DATABASE_PATH = '/var/lib/linstor'
38DATABASE_MKFS = 'mkfs.ext4'
40REG_DRBDADM_PRIMARY = re.compile("([^\\s]+)\\s+role:Primary")
41REG_DRBDSETUP_IP = re.compile('[^\\s]+\\s+(.*):.*$')
43DRBD_BY_RES_PATH = '/dev/drbd/by-res/'
45PLUGIN = 'linstor-manager'
48# ==============================================================================
50def get_local_volume_openers(resource_name, volume):
51 if not resource_name or volume is None:
52 raise Exception('Cannot get DRBD openers without resource name and/or volume.')
54 path = '/sys/kernel/debug/drbd/resources/{}/volumes/{}/openers'.format(
55 resource_name, volume
56 )
58 with open(path, 'r') as openers:
59 # Not a big cost, so read all lines directly.
60 lines = openers.readlines()
62 result = {}
64 opener_re = re.compile('(.*)\\s+([0-9]+)\\s+([0-9]+)')
65 for line in lines:
66 match = opener_re.match(line)
67 assert match
69 groups = match.groups()
70 process_name = groups[0]
71 pid = groups[1]
72 open_duration_ms = groups[2]
73 result[pid] = {
74 'process-name': process_name,
75 'open-duration': open_duration_ms
76 }
78 return json.dumps(result)
80def get_all_volume_openers(resource_name, volume):
81 PLUGIN_CMD = 'getDrbdOpeners'
83 volume = str(volume)
84 openers = {}
86 # Make sure this call never stucks because this function can be called
87 # during HA init and in this case we can wait forever.
88 session = util.timeout_call(10, util.get_localAPI_session)
90 hosts = session.xenapi.host.get_all_records()
91 for host_ref, host_record in hosts.items():
92 node_name = host_record['hostname']
93 try:
94 if not session.xenapi.host_metrics.get_record(
95 host_record['metrics']
96 )['live']:
97 # Ensure we call plugin on online hosts only.
98 continue
100 openers[node_name] = json.loads(
101 session.xenapi.host.call_plugin(host_ref, PLUGIN, PLUGIN_CMD, {
102 'resourceName': resource_name,
103 'volume': volume
104 })
105 )
106 except Exception as e:
107 util.SMlog('Failed to get openers of `{}` on `{}`: {}'.format(
108 resource_name, node_name, e
109 ))
111 return openers
114# ==============================================================================
116def round_up(value, divisor):
117 assert divisor
118 divisor = int(divisor)
119 return ((int(value) + divisor - 1) // divisor) * divisor
122def round_down(value, divisor):
123 assert divisor
124 value = int(value)
125 return value - (value % int(divisor))
128# ==============================================================================
130def get_remote_host_ip(node_name):
131 (ret, stdout, stderr) = util.doexec([
132 'drbdsetup', 'show', DATABASE_VOLUME_NAME, '--json'
133 ])
134 if ret != 0:
135 return
137 try:
138 conf = json.loads(stdout)
139 if not conf:
140 return
142 for connection in conf[0]['connections']:
143 if connection['net']['_name'] == node_name:
144 value = connection['path']['_remote_host']
145 res = REG_DRBDSETUP_IP.match(value)
146 if res:
147 return res.groups()[0]
148 break
149 except Exception:
150 pass
153def _get_controller_uri():
154 PLUGIN_CMD = 'hasControllerRunning'
156 # Try to find controller using drbdadm.
157 (ret, stdout, stderr) = util.doexec([
158 'drbdadm', 'status', DATABASE_VOLUME_NAME
159 ])
160 if ret == 0:
161 # If we are here, the database device exists locally.
163 if stdout.startswith('{} role:Primary'.format(DATABASE_VOLUME_NAME)):
164 # Nice case, we have the controller running on this local host.
165 return 'linstor://localhost'
167 # Try to find the host using DRBD connections.
168 res = REG_DRBDADM_PRIMARY.search(stdout)
169 if res:
170 node_name = res.groups()[0]
171 ip = get_remote_host_ip(node_name)
172 if ip:
173 return 'linstor://' + ip
175 # Worst case: we use many hosts in the pool (>= 4), so we can't find the
176 # primary using drbdadm because we don't have all connections to the
177 # replicated volume. `drbdadm status xcp-persistent-database` returns
178 # 3 connections by default.
179 try:
180 session = util.timeout_call(10, util.get_localAPI_session)
182 for host_ref, host_record in session.xenapi.host.get_all_records().items():
183 node_name = host_record['hostname']
184 try:
185 if distutils.util.strtobool(
186 session.xenapi.host.call_plugin(host_ref, PLUGIN, PLUGIN_CMD, {})
187 ):
188 return 'linstor://' + host_record['address']
189 except Exception as e:
190 # Can throw and exception if a host is offline. So catch it.
191 util.SMlog('Unable to search controller on `{}`: {}'.format(
192 node_name, e
193 ))
194 except:
195 # Not found, maybe we are trying to create the SR...
196 pass
198def get_controller_uri():
199 retries = 0
200 while True:
201 uri = _get_controller_uri()
202 if uri:
203 return uri
205 retries += 1
206 if retries >= 10:
207 break
208 time.sleep(1)
211def get_controller_node_name():
212 PLUGIN_CMD = 'hasControllerRunning'
214 (ret, stdout, stderr) = util.doexec([
215 'drbdadm', 'status', DATABASE_VOLUME_NAME
216 ])
218 if ret == 0:
219 if stdout.startswith('{} role:Primary'.format(DATABASE_VOLUME_NAME)):
220 return 'localhost'
222 res = REG_DRBDADM_PRIMARY.search(stdout)
223 if res:
224 return res.groups()[0]
226 session = util.timeout_call(5, util.get_localAPI_session)
228 for host_ref, host_record in session.xenapi.host.get_all_records().items():
229 node_name = host_record['hostname']
230 try:
231 if not session.xenapi.host_metrics.get_record(
232 host_record['metrics']
233 )['live']:
234 continue
236 if distutils.util.strtobool(session.xenapi.host.call_plugin(
237 host_ref, PLUGIN, PLUGIN_CMD, {}
238 )):
239 return node_name
240 except Exception as e:
241 util.SMlog('Failed to call plugin to get controller on `{}`: {}'.format(
242 node_name, e
243 ))
246def demote_drbd_resource(node_name, resource_name):
247 PLUGIN_CMD = 'demoteDrbdResource'
249 session = util.timeout_call(5, util.get_localAPI_session)
251 for host_ref, host_record in session.xenapi.host.get_all_records().items():
252 if host_record['hostname'] != node_name:
253 continue
255 try:
256 session.xenapi.host.call_plugin(
257 host_ref, PLUGIN, PLUGIN_CMD, {'resource_name': resource_name}
258 )
259 except Exception as e:
260 util.SMlog('Failed to demote resource `{}` on `{}`: {}'.format(
261 resource_name, node_name, e
262 ))
263 raise Exception(
264 'Can\'t demote resource `{}`, unable to find node `{}`'
265 .format(resource_name, node_name)
266 )
268# ==============================================================================
270class LinstorVolumeManagerError(Exception):
271 ERR_GENERIC = 0,
272 ERR_VOLUME_EXISTS = 1,
273 ERR_VOLUME_NOT_EXISTS = 2
275 def __init__(self, message, code=ERR_GENERIC):
276 super(LinstorVolumeManagerError, self).__init__(message)
277 self._code = code
279 @property
280 def code(self):
281 return self._code
284# ==============================================================================
286# Note:
287# If a storage pool is not accessible after a network change:
288# linstor node interface modify <NODE> default --ip <IP>
291class LinstorVolumeManager(object):
292 """
293 API to manager LINSTOR volumes in XCP-ng.
294 A volume in this context is a physical part of the storage layer.
295 """
297 __slots__ = (
298 '_linstor', '_logger',
299 '_uri', '_base_group_name',
300 '_redundancy', '_group_name',
301 '_volumes', '_storage_pools',
302 '_storage_pools_time',
303 '_kv_cache', '_resource_cache', '_volume_info_cache',
304 '_kv_cache_dirty', '_resource_cache_dirty', '_volume_info_cache_dirty'
305 )
307 DEV_ROOT_PATH = DRBD_BY_RES_PATH
309 # Default LVM extent size.
310 BLOCK_SIZE = 4 * 1024 * 1024
312 # List of volume properties.
313 PROP_METADATA = 'metadata'
314 PROP_NOT_EXISTS = 'not-exists'
315 PROP_VOLUME_NAME = 'volume-name'
316 PROP_IS_READONLY_TIMESTAMP = 'readonly-timestamp'
318 # A volume can only be locked for a limited duration.
319 # The goal is to give enough time to slaves to execute some actions on
320 # a device before an UUID update or a coalesce for example.
321 # Expiration is expressed in seconds.
322 LOCKED_EXPIRATION_DELAY = 1 * 60
324 # Used when volume uuid is being updated.
325 PROP_UPDATING_UUID_SRC = 'updating-uuid-src'
327 # States of property PROP_NOT_EXISTS.
328 STATE_EXISTS = '0'
329 STATE_NOT_EXISTS = '1'
330 STATE_CREATING = '2'
332 # Property namespaces.
333 NAMESPACE_SR = 'xcp/sr'
334 NAMESPACE_VOLUME = 'xcp/volume'
336 # Regex to match properties.
337 REG_PROP = '^([^/]+)/{}$'
339 REG_METADATA = re.compile(REG_PROP.format(PROP_METADATA))
340 REG_NOT_EXISTS = re.compile(REG_PROP.format(PROP_NOT_EXISTS))
341 REG_VOLUME_NAME = re.compile(REG_PROP.format(PROP_VOLUME_NAME))
342 REG_UPDATING_UUID_SRC = re.compile(REG_PROP.format(PROP_UPDATING_UUID_SRC))
344 # Prefixes of SR/VOLUME in the LINSTOR DB.
345 # A LINSTOR (resource, group, ...) name cannot start with a number.
346 # So we add a prefix behind our SR/VOLUME uuids.
347 PREFIX_SR = 'xcp-sr-'
348 PREFIX_VOLUME = 'xcp-volume-'
350 # Limit request number when storage pool info is asked, we fetch
351 # the current pool status after N elapsed seconds.
352 STORAGE_POOLS_FETCH_INTERVAL = 15
354 @staticmethod
355 def default_logger(*args):
356 print(args)
358 # --------------------------------------------------------------------------
359 # API.
360 # --------------------------------------------------------------------------
362 class VolumeInfo(object):
363 __slots__ = (
364 'name',
365 'allocated_size', # Allocated size, place count is not used.
366 'virtual_size', # Total virtual available size of this volume
367 # (i.e. the user size at creation).
368 'diskful' # Array of nodes that have a diskful volume.
369 )
371 def __init__(self, name):
372 self.name = name
373 self.allocated_size = 0
374 self.virtual_size = 0
375 self.diskful = []
377 def __repr__(self):
378 return 'VolumeInfo("{}", {}, {}, {})'.format(
379 self.name, self.allocated_size, self.virtual_size,
380 self.diskful
381 )
383 # --------------------------------------------------------------------------
385 def __init__(
386 self, uri, group_name, repair=False, logger=default_logger.__func__,
387 attempt_count=30
388 ):
389 """
390 Create a new LinstorVolumeManager object.
391 :param str uri: URI to communicate with the LINSTOR controller.
392 :param str group_name: The SR goup name to use.
393 :param bool repair: If true we try to remove bad volumes due to a crash
394 or unexpected behavior.
395 :param function logger: Function to log messages.
396 :param int attempt_count: Number of attempts to join the controller.
397 """
399 self._linstor = self._create_linstor_instance(
400 uri, attempt_count=attempt_count
401 )
402 self._base_group_name = group_name
404 # Ensure group exists.
405 group_name = self._build_group_name(group_name)
406 groups = self._linstor.resource_group_list_raise([group_name])
407 groups = groups.resource_groups
408 if not groups:
409 raise LinstorVolumeManagerError(
410 'Unable to find `{}` Linstor SR'.format(group_name)
411 )
413 # Ok. ;)
414 self._logger = logger
415 self._redundancy = groups[0].select_filter.place_count
416 self._group_name = group_name
417 self._volumes = set()
418 self._storage_pools_time = 0
420 # To increate performance and limit request count to LINSTOR services,
421 # we use caches.
422 self._kv_cache = self._create_kv_cache()
423 self._resource_cache = None
424 self._resource_cache_dirty = True
425 self._volume_info_cache = None
426 self._volume_info_cache_dirty = True
427 self._build_volumes(repair=repair)
429 @property
430 def group_name(self):
431 """
432 Give the used group name.
433 :return: The group name.
434 :rtype: str
435 """
436 return self._base_group_name
438 @property
439 def redundancy(self):
440 """
441 Give the used redundancy.
442 :return: The redundancy.
443 :rtype: int
444 """
445 return self._redundancy
447 @property
448 def volumes(self):
449 """
450 Give the volumes uuid set.
451 :return: The volumes uuid set.
452 :rtype: set(str)
453 """
454 return self._volumes
456 @property
457 def max_volume_size_allowed(self):
458 """
459 Give the max volume size currently available in B.
460 :return: The current size.
461 :rtype: int
462 """
464 candidates = self._find_best_size_candidates()
465 if not candidates:
466 raise LinstorVolumeManagerError(
467 'Failed to get max volume size allowed'
468 )
470 size = candidates[0].max_volume_size
471 if size < 0:
472 raise LinstorVolumeManagerError(
473 'Invalid max volume size allowed given: {}'.format(size)
474 )
475 return self.round_down_volume_size(size * 1024)
477 @property
478 def physical_size(self):
479 """
480 Give the total physical size of the SR.
481 :return: The physical size.
482 :rtype: int
483 """
484 return self._compute_size('total_capacity')
486 @property
487 def physical_free_size(self):
488 """
489 Give the total free physical size of the SR.
490 :return: The physical free size.
491 :rtype: int
492 """
493 return self._compute_size('free_capacity')
495 @property
496 def allocated_volume_size(self):
497 """
498 Give the allocated size for all volumes. The place count is not
499 used here. When thick lvm is used, the size for one volume should
500 be equal to the virtual volume size. With thin lvm, the size is equal
501 or lower to the volume size.
502 :return: The allocated size of all volumes.
503 :rtype: int
504 """
506 # Paths: /res_name/vol_number/size
507 sizes = {}
509 for resource in self._get_resource_cache().resources:
510 if resource.name not in sizes:
511 current = sizes[resource.name] = {}
512 else:
513 current = sizes[resource.name]
515 for volume in resource.volumes:
516 # We ignore diskless pools of the form "DfltDisklessStorPool".
517 if volume.storage_pool_name != self._group_name:
518 continue
520 current_size = volume.allocated_size
521 if current_size < 0:
522 raise LinstorVolumeManagerError(
523 'Failed to get allocated size of `{}` on `{}`'
524 .format(resource.name, volume.storage_pool_name)
525 )
526 current[volume.number] = max(current_size, current.get(volume.number) or 0)
528 total_size = 0
529 for volumes in sizes.values():
530 for size in volumes.values():
531 total_size += size
533 return total_size * 1024
535 def get_min_physical_size(self):
536 """
537 Give the minimum physical size of the SR.
538 I.e. the size of the smallest disk + the number of pools.
539 :return: The physical min size.
540 :rtype: tuple(int, int)
541 """
542 size = None
543 pool_count = 0
544 for pool in self._get_storage_pools(force=True):
545 space = pool.free_space
546 if space:
547 pool_count += 1
548 current_size = space.total_capacity
549 if current_size < 0:
550 raise LinstorVolumeManagerError(
551 'Failed to get pool total_capacity attr of `{}`'
552 .format(pool.node_name)
553 )
554 if size is None or current_size < size:
555 size = current_size
556 return (pool_count, (size or 0) * 1024)
558 @property
559 def metadata(self):
560 """
561 Get the metadata of the SR.
562 :return: Dictionary that contains metadata.
563 :rtype: dict(str, dict)
564 """
566 sr_properties = self._get_sr_properties()
567 metadata = sr_properties.get(self.PROP_METADATA)
568 if metadata is not None:
569 metadata = json.loads(metadata)
570 if isinstance(metadata, dict):
571 return metadata
572 raise LinstorVolumeManagerError(
573 'Expected dictionary in SR metadata: {}'.format(
574 self._group_name
575 )
576 )
578 return {}
580 @metadata.setter
581 def metadata(self, metadata):
582 """
583 Set the metadata of the SR.
584 :param dict metadata: Dictionary that contains metadata.
585 """
587 assert isinstance(metadata, dict)
588 sr_properties = self._get_sr_properties()
589 sr_properties[self.PROP_METADATA] = json.dumps(metadata)
591 @property
592 def disconnected_hosts(self):
593 """
594 Get the list of disconnected hosts.
595 :return: Set that contains disconnected hosts.
596 :rtype: set(str)
597 """
599 disconnected_hosts = set()
600 for pool in self._get_storage_pools():
601 for report in pool.reports:
602 if report.ret_code & linstor.consts.WARN_NOT_CONNECTED == \
603 linstor.consts.WARN_NOT_CONNECTED:
604 disconnected_hosts.add(pool.node_name)
605 break
606 return disconnected_hosts
608 def check_volume_exists(self, volume_uuid):
609 """
610 Check if a volume exists in the SR.
611 :return: True if volume exists.
612 :rtype: bool
613 """
614 return volume_uuid in self._volumes
616 def create_volume(
617 self, volume_uuid, size, persistent=True, volume_name=None,
618 no_diskless=False
619 ):
620 """
621 Create a new volume on the SR.
622 :param str volume_uuid: The volume uuid to use.
623 :param int size: volume size in B.
624 :param bool persistent: If false the volume will be unavailable
625 on the next constructor call LinstorSR(...).
626 :param str volume_name: If set, this name is used in the LINSTOR
627 database instead of a generated name.
628 :param bool no_diskless: If set, the default group redundancy is not
629 used, instead the volume is created on all nodes.
630 :return: The current device path of the volume.
631 :rtype: str
632 """
634 self._logger('Creating LINSTOR volume {}...'.format(volume_uuid))
635 if not volume_name:
636 volume_name = self.build_volume_name(util.gen_uuid())
637 volume_properties = self._create_volume_with_properties(
638 volume_uuid, volume_name, size, place_resources=True,
639 no_diskless=no_diskless
640 )
642 # Volume created! Now try to find the device path.
643 try:
644 self._logger(
645 'Find device path of LINSTOR volume {}...'.format(volume_uuid)
646 )
647 device_path = self._find_device_path(volume_uuid, volume_name)
648 if persistent:
649 volume_properties[self.PROP_NOT_EXISTS] = self.STATE_EXISTS
650 self._volumes.add(volume_uuid)
651 self._logger(
652 'LINSTOR volume {} created!'.format(volume_uuid)
653 )
654 return device_path
655 except Exception as e:
656 # There is an issue to find the path.
657 # At this point the volume has just been created, so force flag can be used.
658 self._destroy_volume(volume_uuid, force=True)
659 raise
661 def mark_volume_as_persistent(self, volume_uuid):
662 """
663 Mark volume as persistent if created with persistent=False.
664 :param str volume_uuid: The volume uuid to mark.
665 """
667 self._ensure_volume_exists(volume_uuid)
669 # Mark volume as persistent.
670 volume_properties = self._get_volume_properties(volume_uuid)
671 volume_properties[self.PROP_NOT_EXISTS] = self.STATE_EXISTS
673 def destroy_volume(self, volume_uuid):
674 """
675 Destroy a volume.
676 :param str volume_uuid: The volume uuid to destroy.
677 """
679 self._ensure_volume_exists(volume_uuid)
680 self.ensure_volume_is_not_locked(volume_uuid)
682 # Mark volume as destroyed.
683 volume_properties = self._get_volume_properties(volume_uuid)
684 volume_properties[self.PROP_NOT_EXISTS] = self.STATE_NOT_EXISTS
686 self._volumes.remove(volume_uuid)
687 self._destroy_volume(volume_uuid)
689 def lock_volume(self, volume_uuid, locked=True):
690 """
691 Prevent modifications of the volume properties during
692 "self.LOCKED_EXPIRATION_DELAY" seconds. The SR must be locked
693 when used. This method is useful to attach/detach correctly a volume on
694 a slave. Without it the GC can rename a volume, in this case the old
695 volume path can be used by a slave...
696 :param str volume_uuid: The volume uuid to protect/unprotect.
697 :param bool locked: Lock/unlock the volume.
698 """
700 self._ensure_volume_exists(volume_uuid)
702 self._logger(
703 '{} volume {} as locked'.format(
704 'Mark' if locked else 'Unmark',
705 volume_uuid
706 )
707 )
709 volume_properties = self._get_volume_properties(volume_uuid)
710 if locked:
711 volume_properties[
712 self.PROP_IS_READONLY_TIMESTAMP
713 ] = str(time.time())
714 elif self.PROP_IS_READONLY_TIMESTAMP in volume_properties:
715 volume_properties.pop(self.PROP_IS_READONLY_TIMESTAMP)
717 def ensure_volume_is_not_locked(self, volume_uuid, timeout=None):
718 """
719 Ensure a volume is not locked. Wait if necessary.
720 :param str volume_uuid: The volume uuid to check.
721 :param int timeout: If the volume is always locked after the expiration
722 of the timeout, an exception is thrown.
723 """
724 return self.ensure_volume_list_is_not_locked([volume_uuid], timeout)
726 def ensure_volume_list_is_not_locked(self, volume_uuids, timeout=None):
727 checked = set()
728 for volume_uuid in volume_uuids:
729 if volume_uuid in self._volumes:
730 checked.add(volume_uuid)
732 if not checked:
733 return
735 waiting = False
737 volume_properties = self._get_kv_cache()
739 start = time.time()
740 while True:
741 # Can't delete in for loop, use a copy of the list.
742 remaining = checked.copy()
743 for volume_uuid in checked:
744 volume_properties.namespace = \
745 self._build_volume_namespace(volume_uuid)
746 timestamp = volume_properties.get(
747 self.PROP_IS_READONLY_TIMESTAMP
748 )
749 if timestamp is None:
750 remaining.remove(volume_uuid)
751 continue
753 now = time.time()
754 if now - float(timestamp) > self.LOCKED_EXPIRATION_DELAY:
755 self._logger(
756 'Remove readonly timestamp on {}'.format(volume_uuid)
757 )
758 volume_properties.pop(self.PROP_IS_READONLY_TIMESTAMP)
759 remaining.remove(volume_uuid)
760 continue
762 if not waiting:
763 self._logger(
764 'Volume {} is locked, waiting...'.format(volume_uuid)
765 )
766 waiting = True
767 break
769 if not remaining:
770 break
771 checked = remaining
773 if timeout is not None and now - start > timeout:
774 raise LinstorVolumeManagerError(
775 'volume `{}` is locked and timeout has been reached'
776 .format(volume_uuid),
777 LinstorVolumeManagerError.ERR_VOLUME_NOT_EXISTS
778 )
780 # We must wait to use the volume. After that we can modify it
781 # ONLY if the SR is locked to avoid bad reads on the slaves.
782 time.sleep(1)
783 volume_properties = self._create_kv_cache()
785 if waiting:
786 self._logger('No volume locked now!')
788 def introduce_volume(self, volume_uuid):
789 pass # TODO: Implement me.
791 def resize_volume(self, volume_uuid, new_size):
792 """
793 Resize a volume.
794 :param str volume_uuid: The volume uuid to resize.
795 :param int new_size: New size in B.
796 """
798 volume_name = self.get_volume_name(volume_uuid)
799 self.ensure_volume_is_not_locked(volume_uuid)
800 new_size = self.round_up_volume_size(new_size)
802 result = self._linstor.volume_dfn_modify(
803 rsc_name=volume_name,
804 volume_nr=0,
805 size=new_size // 1024
806 )
808 self._mark_resource_cache_as_dirty()
810 error_str = self._get_error_str(result)
811 if error_str:
812 raise LinstorVolumeManagerError(
813 'Could not resize volume `{}` from SR `{}`: {}'
814 .format(volume_uuid, self._group_name, error_str)
815 )
817 def get_volume_name(self, volume_uuid):
818 """
819 Get the name of a particular volume.
820 :param str volume_uuid: The volume uuid of the name to get.
821 :return: The volume name.
822 :rtype: str
823 """
825 self._ensure_volume_exists(volume_uuid)
826 volume_properties = self._get_volume_properties(volume_uuid)
827 volume_name = volume_properties.get(self.PROP_VOLUME_NAME)
828 if volume_name:
829 return volume_name
830 raise LinstorVolumeManagerError(
831 'Failed to get volume name of {}'.format(volume_uuid)
832 )
834 def get_volume_size(self, volume_uuid):
835 """
836 Get the size of a particular volume.
837 :param str volume_uuid: The volume uuid of the size to get.
838 :return: The volume size.
839 :rtype: int
840 """
842 volume_name = self.get_volume_name(volume_uuid)
843 dfns = self._linstor.resource_dfn_list_raise(
844 query_volume_definitions=True,
845 filter_by_resource_definitions=[volume_name]
846 ).resource_definitions
848 size = dfns[0].volume_definitions[0].size
849 if size < 0:
850 raise LinstorVolumeManagerError(
851 'Failed to get volume size of: {}'.format(volume_uuid)
852 )
853 return size * 1024
856 def set_auto_promote_timeout(self, volume_uuid, timeout):
857 """
858 Define the blocking time of open calls when a DRBD
859 is already open on another host.
860 :param str volume_uuid: The volume uuid to modify.
861 """
863 volume_name = self.get_volume_name(volume_uuid)
864 result = self._linstor.resource_dfn_modify(volume_name, {
865 'DrbdOptions/Resource/auto-promote-timeout': timeout
866 })
867 error_str = self._get_error_str(result)
868 if error_str:
869 raise LinstorVolumeManagerError(
870 'Could not change the auto promote timeout of `{}`: {}'
871 .format(volume_uuid, error_str)
872 )
874 def get_volume_info(self, volume_uuid):
875 """
876 Get the volume info of a particular volume.
877 :param str volume_uuid: The volume uuid of the volume info to get.
878 :return: The volume info.
879 :rtype: VolumeInfo
880 """
882 volume_name = self.get_volume_name(volume_uuid)
883 return self._get_volumes_info()[volume_name]
885 def get_device_path(self, volume_uuid):
886 """
887 Get the dev path of a volume.
888 :param str volume_uuid: The volume uuid to get the dev path.
889 :return: The current device path of the volume.
890 :rtype: str
891 """
893 volume_name = self.get_volume_name(volume_uuid)
894 return self._find_device_path(volume_uuid, volume_name)
896 def get_volume_uuid_from_device_path(self, device_path):
897 """
898 Get the volume uuid of a device_path.
899 :param str device_path: The dev path to find the volume uuid.
900 :return: The volume uuid of the local device path.
901 :rtype: str
902 """
904 expected_volume_name = \
905 self.get_volume_name_from_device_path(device_path)
907 volume_names = self.get_volumes_with_name()
908 for volume_uuid, volume_name in volume_names.items():
909 if volume_name == expected_volume_name:
910 return volume_uuid
912 raise LinstorVolumeManagerError(
913 'Unable to find volume uuid from dev path `{}`'.format(device_path)
914 )
916 def get_volume_name_from_device_path(self, device_path):
917 """
918 Get the volume name of a device_path.
919 :param str device_path: The dev path to find the volume name.
920 :return: The volume name of the device path.
921 :rtype: str
922 """
924 # Assume that we have a path like this:
925 # - "/dev/drbd/by-res/xcp-volume-<UUID>/0"
926 # - "../xcp-volume-<UUID>/0"
927 if device_path.startswith(DRBD_BY_RES_PATH):
928 prefix_len = len(DRBD_BY_RES_PATH)
929 else:
930 assert device_path.startswith('../')
931 prefix_len = 3
933 res_name_end = device_path.find('/', prefix_len)
934 assert res_name_end != -1
935 return device_path[prefix_len:res_name_end]
937 def update_volume_uuid(self, volume_uuid, new_volume_uuid, force=False):
938 """
939 Change the uuid of a volume.
940 :param str volume_uuid: The volume to modify.
941 :param str new_volume_uuid: The new volume uuid to use.
942 :param bool force: If true we doesn't check if volume_uuid is in the
943 volume list. I.e. the volume can be marked as deleted but the volume
944 can still be in the LINSTOR KV store if the deletion has failed.
945 In specific cases like "undo" after a failed clone we must rename a bad
946 deleted VDI.
947 """
949 assert volume_uuid != new_volume_uuid
951 self._logger(
952 'Trying to update volume UUID {} to {}...'
953 .format(volume_uuid, new_volume_uuid)
954 )
955 if not force:
956 self._ensure_volume_exists(volume_uuid)
957 self.ensure_volume_is_not_locked(volume_uuid)
959 if new_volume_uuid in self._volumes:
960 raise LinstorVolumeManagerError(
961 'Volume `{}` already exists'.format(new_volume_uuid),
962 LinstorVolumeManagerError.ERR_VOLUME_EXISTS
963 )
965 volume_properties = self._get_volume_properties(volume_uuid)
966 if volume_properties.get(self.PROP_UPDATING_UUID_SRC):
967 raise LinstorVolumeManagerError(
968 'Cannot update volume uuid {}: invalid state'
969 .format(volume_uuid)
970 )
972 # 1. Copy in temp variables metadata and volume_name.
973 metadata = volume_properties.get(self.PROP_METADATA)
974 volume_name = volume_properties.get(self.PROP_VOLUME_NAME)
976 # 2. Switch to new volume namespace.
977 volume_properties.namespace = self._build_volume_namespace(
978 new_volume_uuid
979 )
981 if list(volume_properties.items()):
982 raise LinstorVolumeManagerError(
983 'Cannot update volume uuid {} to {}: '
984 .format(volume_uuid, new_volume_uuid) +
985 'this last one is not empty'
986 )
988 try:
989 # 3. Mark new volume properties with PROP_UPDATING_UUID_SRC.
990 # If we crash after that, the new properties can be removed
991 # properly.
992 volume_properties[self.PROP_NOT_EXISTS] = self.STATE_NOT_EXISTS
993 volume_properties[self.PROP_UPDATING_UUID_SRC] = volume_uuid
995 # 4. Copy the properties.
996 # Note: On new volumes, during clone for example, the metadata
997 # may be missing. So we must test it to avoid this error:
998 # "None has to be a str/unicode, but is <type 'NoneType'>"
999 if metadata:
1000 volume_properties[self.PROP_METADATA] = metadata
1001 volume_properties[self.PROP_VOLUME_NAME] = volume_name
1003 # 5. Ok!
1004 volume_properties[self.PROP_NOT_EXISTS] = self.STATE_EXISTS
1005 except Exception as e:
1006 try:
1007 # Clear the new volume properties in case of failure.
1008 assert volume_properties.namespace == \
1009 self._build_volume_namespace(new_volume_uuid)
1010 volume_properties.clear()
1011 except Exception as e:
1012 self._logger(
1013 'Failed to clear new volume properties: {} (ignoring...)'
1014 .format(e)
1015 )
1016 raise LinstorVolumeManagerError(
1017 'Failed to copy volume properties: {}'.format(e)
1018 )
1020 try:
1021 # 6. After this point, it's ok we can remove the
1022 # PROP_UPDATING_UUID_SRC property and clear the src properties
1023 # without problems.
1025 # 7. Switch to old volume namespace.
1026 volume_properties.namespace = self._build_volume_namespace(
1027 volume_uuid
1028 )
1029 volume_properties.clear()
1031 # 8. Switch a last time to new volume namespace.
1032 volume_properties.namespace = self._build_volume_namespace(
1033 new_volume_uuid
1034 )
1035 volume_properties.pop(self.PROP_UPDATING_UUID_SRC)
1036 except Exception as e:
1037 raise LinstorVolumeManagerError(
1038 'Failed to clear volume properties '
1039 'after volume uuid update: {}'.format(e)
1040 )
1042 self._volumes.remove(volume_uuid)
1043 self._volumes.add(new_volume_uuid)
1045 self._logger(
1046 'UUID update succeeded of {} to {}! (properties={})'
1047 .format(
1048 volume_uuid, new_volume_uuid,
1049 self._get_filtered_properties(volume_properties)
1050 )
1051 )
1053 def update_volume_name(self, volume_uuid, volume_name):
1054 """
1055 Change the volume name of a volume.
1056 :param str volume_uuid: The volume to modify.
1057 :param str volume_name: The volume_name to use.
1058 """
1060 self._ensure_volume_exists(volume_uuid)
1061 self.ensure_volume_is_not_locked(volume_uuid)
1062 if not volume_name.startswith(self.PREFIX_VOLUME):
1063 raise LinstorVolumeManagerError(
1064 'Volume name `{}` must be start with `{}`'
1065 .format(volume_name, self.PREFIX_VOLUME)
1066 )
1068 if volume_name not in self._fetch_resource_names():
1069 raise LinstorVolumeManagerError(
1070 'Volume `{}` doesn\'t exist'.format(volume_name)
1071 )
1073 volume_properties = self._get_volume_properties(volume_uuid)
1074 volume_properties[self.PROP_VOLUME_NAME] = volume_name
1076 def get_usage_states(self, volume_uuid):
1077 """
1078 Check if a volume is currently used.
1079 :param str volume_uuid: The volume uuid to check.
1080 :return: A dictionnary that contains states.
1081 :rtype: dict(str, bool or None)
1082 """
1084 states = {}
1086 volume_name = self.get_volume_name(volume_uuid)
1087 for resource_state in self._linstor.resource_list_raise(
1088 filter_by_resources=[volume_name]
1089 ).resource_states:
1090 states[resource_state.node_name] = resource_state.in_use
1092 return states
1094 def get_volume_openers(self, volume_uuid):
1095 """
1096 Get openers of a volume.
1097 :param str volume_uuid: The volume uuid to monitor.
1098 :return: A dictionnary that contains openers.
1099 :rtype: dict(str, obj)
1100 """
1101 return get_all_volume_openers(self.get_volume_name(volume_uuid), '0')
1104 def get_volumes_with_name(self):
1105 """
1106 Give a volume dictionnary that contains names actually owned.
1107 :return: A volume/name dict.
1108 :rtype: dict(str, str)
1109 """
1110 return self._get_volumes_by_property(self.REG_VOLUME_NAME)
1112 def get_volumes_with_info(self):
1113 """
1114 Give a volume dictionnary that contains VolumeInfos.
1115 :return: A volume/VolumeInfo dict.
1116 :rtype: dict(str, VolumeInfo)
1117 """
1119 volumes = {}
1121 all_volume_info = self._get_volumes_info()
1122 volume_names = self.get_volumes_with_name()
1123 for volume_uuid, volume_name in volume_names.items():
1124 if volume_name:
1125 volume_info = all_volume_info.get(volume_name)
1126 if volume_info:
1127 volumes[volume_uuid] = volume_info
1128 continue
1130 # Well I suppose if this volume is not available,
1131 # LINSTOR has been used directly without using this API.
1132 volumes[volume_uuid] = self.VolumeInfo('')
1134 return volumes
1136 def get_volumes_with_metadata(self):
1137 """
1138 Give a volume dictionnary that contains metadata.
1139 :return: A volume/metadata dict.
1140 :rtype: dict(str, dict)
1141 """
1143 volumes = {}
1145 metadata = self._get_volumes_by_property(self.REG_METADATA)
1146 for volume_uuid, volume_metadata in metadata.items():
1147 if volume_metadata:
1148 volume_metadata = json.loads(volume_metadata)
1149 if isinstance(volume_metadata, dict):
1150 volumes[volume_uuid] = volume_metadata
1151 continue
1152 raise LinstorVolumeManagerError(
1153 'Expected dictionary in volume metadata: {}'
1154 .format(volume_uuid)
1155 )
1157 volumes[volume_uuid] = {}
1159 return volumes
1161 def get_volume_metadata(self, volume_uuid):
1162 """
1163 Get the metadata of a volume.
1164 :return: Dictionary that contains metadata.
1165 :rtype: dict
1166 """
1168 self._ensure_volume_exists(volume_uuid)
1169 volume_properties = self._get_volume_properties(volume_uuid)
1170 metadata = volume_properties.get(self.PROP_METADATA)
1171 if metadata:
1172 metadata = json.loads(metadata)
1173 if isinstance(metadata, dict):
1174 return metadata
1175 raise LinstorVolumeManagerError(
1176 'Expected dictionary in volume metadata: {}'
1177 .format(volume_uuid)
1178 )
1179 return {}
1181 def set_volume_metadata(self, volume_uuid, metadata):
1182 """
1183 Set the metadata of a volume.
1184 :param dict metadata: Dictionary that contains metadata.
1185 """
1187 self._ensure_volume_exists(volume_uuid)
1188 self.ensure_volume_is_not_locked(volume_uuid)
1190 assert isinstance(metadata, dict)
1191 volume_properties = self._get_volume_properties(volume_uuid)
1192 volume_properties[self.PROP_METADATA] = json.dumps(metadata)
1194 def update_volume_metadata(self, volume_uuid, metadata):
1195 """
1196 Update the metadata of a volume. It modify only the given keys.
1197 It doesn't remove unreferenced key instead of set_volume_metadata.
1198 :param dict metadata: Dictionary that contains metadata.
1199 """
1201 self._ensure_volume_exists(volume_uuid)
1202 self.ensure_volume_is_not_locked(volume_uuid)
1204 assert isinstance(metadata, dict)
1205 volume_properties = self._get_volume_properties(volume_uuid)
1207 current_metadata = json.loads(
1208 volume_properties.get(self.PROP_METADATA, '{}')
1209 )
1210 if not isinstance(metadata, dict):
1211 raise LinstorVolumeManagerError(
1212 'Expected dictionary in volume metadata: {}'
1213 .format(volume_uuid)
1214 )
1216 for key, value in metadata.items():
1217 current_metadata[key] = value
1218 volume_properties[self.PROP_METADATA] = json.dumps(current_metadata)
1220 def shallow_clone_volume(self, volume_uuid, clone_uuid, persistent=True):
1221 """
1222 Clone a volume. Do not copy the data, this method creates a new volume
1223 with the same size. It tries to create the volume on the same host
1224 than volume source.
1225 :param str volume_uuid: The volume to clone.
1226 :param str clone_uuid: The cloned volume.
1227 :param bool persistent: If false the volume will be unavailable
1228 on the next constructor call LinstorSR(...).
1229 :return: The current device path of the cloned volume.
1230 :rtype: str
1231 """
1233 volume_name = self.get_volume_name(volume_uuid)
1234 self.ensure_volume_is_not_locked(volume_uuid)
1236 # 1. Find ideal nodes + size to use.
1237 ideal_node_names, size = self._get_volume_node_names_and_size(
1238 volume_name
1239 )
1240 if size <= 0:
1241 raise LinstorVolumeManagerError(
1242 'Invalid size of {} for volume `{}`'.format(size, volume_name)
1243 )
1245 # 2. Find the node(s) with the maximum space.
1246 candidates = self._find_best_size_candidates()
1247 if not candidates:
1248 raise LinstorVolumeManagerError(
1249 'Unable to shallow clone volume `{}`, no free space found.'
1250 )
1252 # 3. Compute node names and search if we can try to clone
1253 # on the same nodes than volume.
1254 def find_best_nodes():
1255 for candidate in candidates:
1256 for node_name in candidate.node_names:
1257 if node_name in ideal_node_names:
1258 return candidate.node_names
1260 node_names = find_best_nodes()
1261 if not node_names:
1262 node_names = candidates[0].node_names
1264 if len(node_names) < self._redundancy:
1265 raise LinstorVolumeManagerError(
1266 'Unable to shallow clone volume `{}`, '.format(volume_uuid) +
1267 '{} are required to clone, found: {}'.format(
1268 self._redundancy, len(node_names)
1269 )
1270 )
1272 # 4. Compute resources to create.
1273 clone_volume_name = self.build_volume_name(util.gen_uuid())
1274 diskless_node_names = self._get_node_names()
1275 resources = []
1276 for node_name in node_names:
1277 diskless_node_names.remove(node_name)
1278 resources.append(linstor.ResourceData(
1279 node_name=node_name,
1280 rsc_name=clone_volume_name,
1281 storage_pool=self._group_name
1282 ))
1284 # 5. Create resources!
1285 def clean():
1286 try:
1287 self._destroy_volume(clone_uuid, force=True)
1288 except Exception as e:
1289 self._logger(
1290 'Unable to destroy volume {} after shallow clone fail: {}'
1291 .format(clone_uuid, e)
1292 )
1294 def create():
1295 # Note: placed outside try/except block because we create only definition first.
1296 # There is no reason to call `clean` before the real resource creation.
1297 volume_properties = self._create_volume_with_properties(
1298 clone_uuid, clone_volume_name, size,
1299 place_resources=False
1300 )
1302 # After this point, `clean` can be called for any fail because the clone UUID
1303 # is really unique. No risk to remove existing data.
1304 try:
1305 result = self._linstor.resource_create(resources)
1306 error_str = self._get_error_str(result)
1307 if error_str:
1308 raise LinstorVolumeManagerError(
1309 'Could not create cloned volume `{}` of `{}` from '
1310 'SR `{}`: {}'.format(
1311 clone_uuid, volume_uuid, self._group_name,
1312 error_str
1313 )
1314 )
1315 return volume_properties
1316 except Exception:
1317 clean()
1318 raise
1320 # Retry because we can get errors like this:
1321 # "Resource disappeared while waiting for it to be ready" or
1322 # "Resource did not became ready on node 'XXX' within reasonable time, check Satellite for errors."
1323 # in the LINSTOR server.
1324 volume_properties = util.retry(create, maxretry=5)
1326 try:
1327 device_path = self._find_device_path(clone_uuid, clone_volume_name)
1328 if persistent:
1329 volume_properties[self.PROP_NOT_EXISTS] = self.STATE_EXISTS
1330 self._volumes.add(clone_uuid)
1331 return device_path
1332 except Exception as e:
1333 clean()
1334 raise
1336 def remove_resourceless_volumes(self):
1337 """
1338 Remove all volumes without valid or non-empty name
1339 (i.e. without LINSTOR resource). It's different than
1340 LinstorVolumeManager constructor that takes a `repair` param that
1341 removes volumes with `PROP_NOT_EXISTS` to 1.
1342 """
1344 resource_names = self._fetch_resource_names()
1345 for volume_uuid, volume_name in self.get_volumes_with_name().items():
1346 if not volume_name or volume_name not in resource_names:
1347 # Don't force, we can be sure of what's happening.
1348 self.destroy_volume(volume_uuid)
1350 def destroy(self):
1351 """
1352 Destroy this SR. Object should not be used after that.
1353 :param bool force: Try to destroy volumes before if true.
1354 """
1356 if self._volumes:
1357 raise LinstorVolumeManagerError(
1358 'Cannot destroy LINSTOR volume manager: '
1359 'It exists remaining volumes'
1360 )
1362 controller_is_running = self._controller_is_running()
1363 uri = 'linstor://localhost'
1364 try:
1365 if controller_is_running:
1366 self._start_controller(start=False)
1368 # 1. Umount LINSTOR database.
1369 self._mount_database_volume(
1370 self.build_device_path(DATABASE_VOLUME_NAME),
1371 mount=False,
1372 force=True
1373 )
1375 # 2. Refresh instance.
1376 self._start_controller(start=True)
1377 self._linstor = self._create_linstor_instance(
1378 uri, keep_uri_unmodified=True
1379 )
1381 # 3. Destroy database volume.
1382 self._destroy_resource(DATABASE_VOLUME_NAME)
1384 # 4. Destroy group and storage pools.
1385 self._destroy_resource_group(self._linstor, self._group_name)
1386 for pool in self._get_storage_pools(force=True):
1387 self._destroy_storage_pool(
1388 self._linstor, pool.name, pool.node_name
1389 )
1390 except Exception as e:
1391 self._start_controller(start=controller_is_running)
1392 raise e
1394 try:
1395 self._start_controller(start=False)
1396 for file in glob.glob(DATABASE_PATH + '/'):
1397 os.remove(file)
1398 except Exception as e:
1399 util.SMlog(
1400 'Ignoring failure after LINSTOR SR destruction: {}'
1401 .format(e)
1402 )
1404 def find_up_to_date_diskful_nodes(self, volume_uuid):
1405 """
1406 Find all nodes that contain a specific volume using diskful disks.
1407 The disk must be up to data to be used.
1408 :param str volume_uuid: The volume to use.
1409 :return: The available nodes.
1410 :rtype: tuple(set(str), str)
1411 """
1413 volume_name = self.get_volume_name(volume_uuid)
1415 in_use_by = None
1416 node_names = set()
1418 resource_states = filter(
1419 lambda resource_state: resource_state.name == volume_name,
1420 self._get_resource_cache().resource_states
1421 )
1423 for resource_state in resource_states:
1424 volume_state = resource_state.volume_states[0]
1425 if volume_state.disk_state == 'UpToDate':
1426 node_names.add(resource_state.node_name)
1427 if resource_state.in_use:
1428 in_use_by = resource_state.node_name
1430 return (node_names, in_use_by)
1432 def invalidate_resource_cache(self):
1433 """
1434 If resources are impacted by external commands like vhdutil,
1435 it's necessary to call this function to invalidate current resource
1436 cache.
1437 """
1438 self._mark_resource_cache_as_dirty()
1440 def has_node(self, node_name):
1441 """
1442 Check if a node exists in the LINSTOR database.
1443 :rtype: bool
1444 """
1445 result = self._linstor.node_list()
1446 error_str = self._get_error_str(result)
1447 if error_str:
1448 raise LinstorVolumeManagerError(
1449 'Failed to list nodes using `{}`: {}'
1450 .format(node_name, error_str)
1451 )
1452 return bool(result[0].node(node_name))
1454 def create_node(self, node_name, ip):
1455 """
1456 Create a new node in the LINSTOR database.
1457 :param str node_name: Node name to use.
1458 :param str ip: Host IP to communicate.
1459 """
1460 result = self._linstor.node_create(
1461 node_name,
1462 linstor.consts.VAL_NODE_TYPE_CMBD,
1463 ip
1464 )
1465 errors = self._filter_errors(result)
1466 if errors:
1467 error_str = self._get_error_str(errors)
1468 raise LinstorVolumeManagerError(
1469 'Failed to create node `{}`: {}'.format(node_name, error_str)
1470 )
1472 def destroy_node(self, node_name):
1473 """
1474 Destroy a node in the LINSTOR database.
1475 :param str node_name: Node name to remove.
1476 """
1477 result = self._linstor.node_delete(node_name)
1478 errors = self._filter_errors(result)
1479 if errors:
1480 error_str = self._get_error_str(errors)
1481 raise LinstorVolumeManagerError(
1482 'Failed to destroy node `{}`: {}'.format(node_name, error_str)
1483 )
1485 def get_nodes_info(self):
1486 """
1487 Get all nodes + statuses, used or not by the pool.
1488 :rtype: dict(str, dict)
1489 """
1490 try:
1491 nodes = {}
1492 for node in self._linstor.node_list_raise().nodes:
1493 nodes[node.name] = node.connection_status
1494 return nodes
1495 except Exception as e:
1496 raise LinstorVolumeManagerError(
1497 'Failed to get all nodes: `{}`'.format(e)
1498 )
1500 def get_storage_pools_info(self):
1501 """
1502 Give all storage pools of current group name.
1503 :rtype: dict(str, list)
1504 """
1505 storage_pools = {}
1506 for pool in self._get_storage_pools(force=True):
1507 if pool.node_name not in storage_pools:
1508 storage_pools[pool.node_name] = []
1510 size = -1
1511 capacity = -1
1513 space = pool.free_space
1514 if space:
1515 size = space.free_capacity
1516 if size < 0:
1517 size = -1
1518 else:
1519 size *= 1024
1520 capacity = space.total_capacity
1521 if capacity <= 0:
1522 capacity = -1
1523 else:
1524 capacity *= 1024
1526 storage_pools[pool.node_name].append({
1527 'storage-pool-name': pool.name,
1528 'uuid': pool.uuid,
1529 'free-size': size,
1530 'capacity': capacity
1531 })
1533 return storage_pools
1535 def get_resources_info(self):
1536 """
1537 Give all resources of current group name.
1538 :rtype: dict(str, list)
1539 """
1540 resources = {}
1541 resource_list = self._linstor.resource_list_raise()
1542 for resource in resource_list.resources:
1543 if resource.name not in resources:
1544 resources[resource.name] = {}
1546 resources[resource.name][resource.node_name] = {
1547 'volumes': [],
1548 'diskful': linstor.consts.FLAG_DISKLESS not in resource.flags,
1549 'tie-breaker': linstor.consts.FLAG_TIE_BREAKER in resource.flags
1550 }
1552 for volume in resource.volumes:
1553 # We ignore diskless pools of the form "DfltDisklessStorPool".
1554 if volume.storage_pool_name != self._group_name:
1555 continue
1557 usable_size = volume.usable_size
1558 if usable_size < 0:
1559 usable_size = -1
1560 else:
1561 usable_size *= 1024
1563 allocated_size = volume.allocated_size
1564 if allocated_size < 0:
1565 allocated_size = -1
1566 else:
1567 allocated_size *= 1024
1569 resources[resource.name][resource.node_name]['volumes'].append({
1570 'storage-pool-name': volume.storage_pool_name,
1571 'uuid': volume.uuid,
1572 'number': volume.number,
1573 'device-path': volume.device_path,
1574 'usable-size': usable_size,
1575 'allocated-size': allocated_size
1576 })
1578 for resource_state in resource_list.resource_states:
1579 resource = resources[resource_state.rsc_name][resource_state.node_name]
1580 resource['in-use'] = resource_state.in_use
1582 volumes = resource['volumes']
1583 for volume_state in resource_state.volume_states:
1584 volume = next((x for x in volumes if x['number'] == volume_state.number), None)
1585 if volume:
1586 volume['disk-state'] = volume_state.disk_state
1588 return resources
1590 @classmethod
1591 def create_sr(
1592 cls, group_name, ips, redundancy,
1593 thin_provisioning, auto_quorum,
1594 logger=default_logger.__func__
1595 ):
1596 """
1597 Create a new SR on the given nodes.
1598 :param str group_name: The SR group_name to use.
1599 :param set(str) ips: Node ips.
1600 :param int redundancy: How many copy of volumes should we store?
1601 :param bool thin_provisioning: Use thin or thick provisioning.
1602 :param bool auto_quorum: DB quorum is monitored by LINSTOR.
1603 :param function logger: Function to log messages.
1604 :return: A new LinstorSr instance.
1605 :rtype: LinstorSr
1606 """
1608 try:
1609 cls._start_controller(start=True)
1610 sr = cls._create_sr(
1611 group_name,
1612 ips,
1613 redundancy,
1614 thin_provisioning,
1615 auto_quorum,
1616 logger
1617 )
1618 finally:
1619 # Controller must be stopped and volume unmounted because
1620 # it is the role of the drbd-reactor daemon to do the right
1621 # actions.
1622 cls._start_controller(start=False)
1623 cls._mount_volume(
1624 cls.build_device_path(DATABASE_VOLUME_NAME),
1625 DATABASE_PATH,
1626 mount=False
1627 )
1628 return sr
1630 @classmethod
1631 def _create_sr(
1632 cls, group_name, ips, redundancy,
1633 thin_provisioning, auto_quorum,
1634 logger=default_logger.__func__
1635 ):
1636 # 1. Check if SR already exists.
1637 uri = 'linstor://localhost'
1639 lin = cls._create_linstor_instance(uri, keep_uri_unmodified=True)
1641 node_names = list(ips.keys())
1642 for node_name, ip in ips.items():
1643 while True:
1644 # Try to create node.
1645 result = lin.node_create(
1646 node_name,
1647 linstor.consts.VAL_NODE_TYPE_CMBD,
1648 ip
1649 )
1651 errors = cls._filter_errors(result)
1652 if cls._check_errors(
1653 errors, [linstor.consts.FAIL_EXISTS_NODE]
1654 ):
1655 # If it already exists, remove, then recreate.
1656 result = lin.node_delete(node_name)
1657 error_str = cls._get_error_str(result)
1658 if error_str:
1659 raise LinstorVolumeManagerError(
1660 'Failed to remove old node `{}`: {}'
1661 .format(node_name, error_str)
1662 )
1663 elif not errors:
1664 break # Created!
1665 else:
1666 raise LinstorVolumeManagerError(
1667 'Failed to create node `{}` with ip `{}`: {}'.format(
1668 node_name, ip, cls._get_error_str(errors)
1669 )
1670 )
1672 driver_pool_name = group_name
1673 base_group_name = group_name
1674 group_name = cls._build_group_name(group_name)
1675 pools = lin.storage_pool_list_raise(filter_by_stor_pools=[group_name])
1676 pools = pools.storage_pools
1677 if pools:
1678 existing_node_names = [pool.node_name for pool in pools]
1679 raise LinstorVolumeManagerError(
1680 'Unable to create SR `{}`. It already exists on node(s): {}'
1681 .format(group_name, existing_node_names)
1682 )
1684 if lin.resource_group_list_raise(
1685 [group_name]
1686 ).resource_groups:
1687 raise LinstorVolumeManagerError(
1688 'Unable to create SR `{}`: The group name already exists'
1689 .format(group_name)
1690 )
1692 if thin_provisioning:
1693 driver_pool_parts = driver_pool_name.split('/')
1694 if not len(driver_pool_parts) == 2:
1695 raise LinstorVolumeManagerError(
1696 'Invalid group name using thin provisioning. '
1697 'Expected format: \'VG/LV`\''
1698 )
1700 # 2. Create storage pool on each node + resource group.
1701 reg_volume_group_not_found = re.compile(
1702 ".*Volume group '.*' not found$"
1703 )
1705 i = 0
1706 try:
1707 # 2.a. Create storage pools.
1708 storage_pool_count = 0
1709 while i < len(node_names):
1710 node_name = node_names[i]
1712 result = lin.storage_pool_create(
1713 node_name=node_name,
1714 storage_pool_name=group_name,
1715 storage_driver='LVM_THIN' if thin_provisioning else 'LVM',
1716 driver_pool_name=driver_pool_name
1717 )
1719 errors = linstor.Linstor.filter_api_call_response_errors(
1720 result
1721 )
1722 if errors:
1723 if len(errors) == 1 and errors[0].is_error(
1724 linstor.consts.FAIL_STOR_POOL_CONFIGURATION_ERROR
1725 ) and reg_volume_group_not_found.match(errors[0].message):
1726 logger(
1727 'Volume group `{}` not found on `{}`. Ignoring...'
1728 .format(group_name, node_name)
1729 )
1730 cls._destroy_storage_pool(lin, group_name, node_name)
1731 else:
1732 error_str = cls._get_error_str(result)
1733 raise LinstorVolumeManagerError(
1734 'Could not create SP `{}` on node `{}`: {}'
1735 .format(group_name, node_name, error_str)
1736 )
1737 else:
1738 storage_pool_count += 1
1739 i += 1
1741 if not storage_pool_count:
1742 raise LinstorVolumeManagerError(
1743 'Unable to create SR `{}`: No VG group found'.format(
1744 group_name,
1745 )
1746 )
1748 # 2.b. Create resource group.
1749 result = lin.resource_group_create(
1750 name=group_name,
1751 place_count=redundancy,
1752 storage_pool=group_name,
1753 diskless_on_remaining=True
1754 )
1755 error_str = cls._get_error_str(result)
1756 if error_str:
1757 raise LinstorVolumeManagerError(
1758 'Could not create RG `{}`: {}'.format(
1759 group_name, error_str
1760 )
1761 )
1763 # 2.c. Create volume group.
1764 result = lin.volume_group_create(group_name)
1765 error_str = cls._get_error_str(result)
1766 if error_str:
1767 raise LinstorVolumeManagerError(
1768 'Could not create VG `{}`: {}'.format(
1769 group_name, error_str
1770 )
1771 )
1773 # 3. Create the LINSTOR database volume and mount it.
1774 try:
1775 logger('Creating database volume...')
1776 volume_path = cls._create_database_volume(
1777 lin, group_name, node_names, redundancy, auto_quorum
1778 )
1779 except LinstorVolumeManagerError as e:
1780 if e.code != LinstorVolumeManagerError.ERR_VOLUME_EXISTS:
1781 logger('Destroying database volume after creation fail...')
1782 cls._force_destroy_database_volume(lin, group_name)
1783 raise
1785 try:
1786 logger('Mounting database volume...')
1788 # First we must disable the controller to move safely the
1789 # LINSTOR config.
1790 cls._start_controller(start=False)
1792 cls._mount_database_volume(volume_path)
1793 except Exception as e:
1794 # Ensure we are connected because controller has been
1795 # restarted during mount call.
1796 logger('Destroying database volume after mount fail...')
1798 try:
1799 cls._start_controller(start=True)
1800 except Exception:
1801 pass
1803 lin = cls._create_linstor_instance(
1804 uri, keep_uri_unmodified=True
1805 )
1806 cls._force_destroy_database_volume(lin, group_name)
1807 raise e
1809 cls._start_controller(start=True)
1810 lin = cls._create_linstor_instance(uri, keep_uri_unmodified=True)
1812 # 4. Remove storage pools/resource/volume group in the case of errors.
1813 except Exception as e:
1814 logger('Destroying resource group and storage pools after fail...')
1815 try:
1816 cls._destroy_resource_group(lin, group_name)
1817 except Exception as e2:
1818 logger('Failed to destroy resource group: {}'.format(e2))
1819 pass
1820 j = 0
1821 i = min(i, len(node_names) - 1)
1822 while j <= i:
1823 try:
1824 cls._destroy_storage_pool(lin, group_name, node_names[j])
1825 except Exception as e2:
1826 logger('Failed to destroy resource group: {}'.format(e2))
1827 pass
1828 j += 1
1829 raise e
1831 # 5. Return new instance.
1832 instance = cls.__new__(cls)
1833 instance._linstor = lin
1834 instance._logger = logger
1835 instance._redundancy = redundancy
1836 instance._base_group_name = base_group_name
1837 instance._group_name = group_name
1838 instance._volumes = set()
1839 instance._storage_pools_time = 0
1840 instance._kv_cache = instance._create_kv_cache()
1841 instance._resource_cache = None
1842 instance._resource_cache_dirty = True
1843 instance._volume_info_cache = None
1844 instance._volume_info_cache_dirty = True
1845 return instance
1847 @classmethod
1848 def build_device_path(cls, volume_name):
1849 """
1850 Build a device path given a volume name.
1851 :param str volume_name: The volume name to use.
1852 :return: A valid or not device path.
1853 :rtype: str
1854 """
1856 return '{}{}/0'.format(cls.DEV_ROOT_PATH, volume_name)
1858 @classmethod
1859 def build_volume_name(cls, base_name):
1860 """
1861 Build a volume name given a base name (i.e. a UUID).
1862 :param str volume_name: The volume name to use.
1863 :return: A valid or not device path.
1864 :rtype: str
1865 """
1866 return '{}{}'.format(cls.PREFIX_VOLUME, base_name)
1868 @classmethod
1869 def round_up_volume_size(cls, volume_size):
1870 """
1871 Align volume size on higher multiple of BLOCK_SIZE.
1872 :param int volume_size: The volume size to align.
1873 :return: An aligned volume size.
1874 :rtype: int
1875 """
1876 return round_up(volume_size, cls.BLOCK_SIZE)
1878 @classmethod
1879 def round_down_volume_size(cls, volume_size):
1880 """
1881 Align volume size on lower multiple of BLOCK_SIZE.
1882 :param int volume_size: The volume size to align.
1883 :return: An aligned volume size.
1884 :rtype: int
1885 """
1886 return round_down(volume_size, cls.BLOCK_SIZE)
1888 # --------------------------------------------------------------------------
1889 # Private helpers.
1890 # --------------------------------------------------------------------------
1892 def _create_kv_cache(self):
1893 self._kv_cache = self._create_linstor_kv('/')
1894 self._kv_cache_dirty = False
1895 return self._kv_cache
1897 def _get_kv_cache(self):
1898 if self._kv_cache_dirty:
1899 self._kv_cache = self._create_kv_cache()
1900 return self._kv_cache
1902 def _create_resource_cache(self):
1903 self._resource_cache = self._linstor.resource_list_raise()
1904 self._resource_cache_dirty = False
1905 return self._resource_cache
1907 def _get_resource_cache(self):
1908 if self._resource_cache_dirty:
1909 self._resource_cache = self._create_resource_cache()
1910 return self._resource_cache
1912 def _mark_resource_cache_as_dirty(self):
1913 self._resource_cache_dirty = True
1914 self._volume_info_cache_dirty = True
1916 # --------------------------------------------------------------------------
1918 def _ensure_volume_exists(self, volume_uuid):
1919 if volume_uuid not in self._volumes:
1920 raise LinstorVolumeManagerError(
1921 'volume `{}` doesn\'t exist'.format(volume_uuid),
1922 LinstorVolumeManagerError.ERR_VOLUME_NOT_EXISTS
1923 )
1925 def _find_best_size_candidates(self):
1926 result = self._linstor.resource_group_qmvs(self._group_name)
1927 error_str = self._get_error_str(result)
1928 if error_str:
1929 raise LinstorVolumeManagerError(
1930 'Failed to get max volume size allowed of SR `{}`: {}'.format(
1931 self._group_name,
1932 error_str
1933 )
1934 )
1935 return result[0].candidates
1937 def _fetch_resource_names(self):
1938 resource_names = set()
1939 dfns = self._linstor.resource_dfn_list_raise().resource_definitions
1940 for dfn in dfns:
1941 if dfn.resource_group_name == self._group_name and \
1942 linstor.consts.FLAG_DELETE not in dfn.flags:
1943 resource_names.add(dfn.name)
1944 return resource_names
1946 def _get_volumes_info(self, volume_name=None):
1947 all_volume_info = {}
1949 if not self._volume_info_cache_dirty:
1950 return self._volume_info_cache
1952 for resource in self._get_resource_cache().resources:
1953 if resource.name not in all_volume_info:
1954 current = all_volume_info[resource.name] = self.VolumeInfo(
1955 resource.name
1956 )
1957 else:
1958 current = all_volume_info[resource.name]
1960 if linstor.consts.FLAG_DISKLESS not in resource.flags:
1961 current.diskful.append(resource.node_name)
1963 for volume in resource.volumes:
1964 # We ignore diskless pools of the form "DfltDisklessStorPool".
1965 if volume.storage_pool_name == self._group_name:
1966 if volume.allocated_size < 0:
1967 raise LinstorVolumeManagerError(
1968 'Failed to get allocated size of `{}` on `{}`'
1969 .format(resource.name, volume.storage_pool_name)
1970 )
1971 allocated_size = volume.allocated_size
1973 current.allocated_size = current.allocated_size and \
1974 max(current.allocated_size, allocated_size) or \
1975 allocated_size
1977 usable_size = volume.usable_size
1978 if usable_size > 0 and (
1979 usable_size < current.virtual_size or
1980 not current.virtual_size
1981 ):
1982 current.virtual_size = usable_size
1984 if current.virtual_size <= 0:
1985 raise LinstorVolumeManagerError(
1986 'Failed to get usable size of `{}` on `{}`'
1987 .format(resource.name, volume.storage_pool_name)
1988 )
1990 for current in all_volume_info.values():
1991 current.allocated_size *= 1024
1992 current.virtual_size *= 1024
1994 self._volume_info_cache_dirty = False
1995 self._volume_info_cache = all_volume_info
1997 return all_volume_info
1999 def _get_volume_node_names_and_size(self, volume_name):
2000 node_names = set()
2001 size = -1
2002 for resource in self._linstor.resource_list_raise(
2003 filter_by_resources=[volume_name]
2004 ).resources:
2005 for volume in resource.volumes:
2006 # We ignore diskless pools of the form "DfltDisklessStorPool".
2007 if volume.storage_pool_name == self._group_name:
2008 node_names.add(resource.node_name)
2010 current_size = volume.usable_size
2011 if current_size < 0:
2012 raise LinstorVolumeManagerError(
2013 'Failed to get usable size of `{}` on `{}`'
2014 .format(resource.name, volume.storage_pool_name)
2015 )
2017 if size < 0:
2018 size = current_size
2019 else:
2020 size = min(size, current_size)
2022 return (node_names, size * 1024)
2024 def _compute_size(self, attr):
2025 capacity = 0
2026 for pool in self._get_storage_pools(force=True):
2027 space = pool.free_space
2028 if space:
2029 size = getattr(space, attr)
2030 if size < 0:
2031 raise LinstorVolumeManagerError(
2032 'Failed to get pool {} attr of `{}`'
2033 .format(attr, pool.node_name)
2034 )
2035 capacity += size
2036 return capacity * 1024
2038 def _get_node_names(self):
2039 node_names = set()
2040 for pool in self._get_storage_pools():
2041 node_names.add(pool.node_name)
2042 return node_names
2044 def _get_storage_pools(self, force=False):
2045 cur_time = time.time()
2046 elsaped_time = cur_time - self._storage_pools_time
2048 if force or elsaped_time >= self.STORAGE_POOLS_FETCH_INTERVAL:
2049 self._storage_pools = self._linstor.storage_pool_list_raise(
2050 filter_by_stor_pools=[self._group_name]
2051 ).storage_pools
2052 self._storage_pools_time = time.time()
2054 return self._storage_pools
2056 def _create_volume(
2057 self, volume_uuid, volume_name, size, place_resources,
2058 no_diskless=False
2059 ):
2060 if no_diskless and not place_resources:
2061 raise LinstorVolumeManagerError(
2062 'Could not create volume `{}` from SR `{}`: it\'s impossible '
2063 .format(volume_uuid, self._group_name) +
2064 'to force no diskless without placing resources'
2065 )
2067 size = self.round_up_volume_size(size)
2068 self._mark_resource_cache_as_dirty()
2070 resources = []
2071 if no_diskless:
2072 for node_name in self._get_node_names():
2073 resources.append(linstor.ResourceData(
2074 node_name=node_name,
2075 rsc_name=volume_name,
2076 storage_pool=self._group_name
2077 ))
2079 def create_definition():
2080 self._check_volume_creation_errors(
2081 self._linstor.resource_group_spawn(
2082 rsc_grp_name=self._group_name,
2083 rsc_dfn_name=volume_name,
2084 vlm_sizes=['{}B'.format(size)],
2085 definitions_only=True
2086 ),
2087 volume_uuid,
2088 self._group_name
2089 )
2090 self._configure_volume_peer_slots(self._linstor, volume_name)
2092 def clean():
2093 try:
2094 self._destroy_volume(volume_uuid, force=True)
2095 except Exception as e:
2096 self._logger(
2097 'Unable to destroy volume {} after creation fail: {}'
2098 .format(volume_uuid, e)
2099 )
2101 def create():
2102 try:
2103 create_definition()
2104 if no_diskless:
2105 # Create a physical resource on each node.
2106 result = self._linstor.resource_create(resources)
2107 error_str = self._get_error_str(result)
2108 if error_str:
2109 raise LinstorVolumeManagerError(
2110 'Could not create volume `{}` from SR `{}`: {}'.format(
2111 volume_uuid, self._group_name, error_str
2112 )
2113 )
2114 elif place_resources:
2115 # Basic case when we use the default redundancy of the group.
2116 self._check_volume_creation_errors(
2117 self._linstor.resource_auto_place(
2118 rsc_name=volume_name,
2119 place_count=self._redundancy,
2120 diskless_on_remaining=not no_diskless
2121 ),
2122 volume_uuid,
2123 self._group_name
2124 )
2125 except LinstorVolumeManagerError as e:
2126 if e.code != LinstorVolumeManagerError.ERR_VOLUME_EXISTS:
2127 clean()
2128 raise
2129 except Exception:
2130 clean()
2131 raise
2133 util.retry(create, maxretry=5)
2135 def _create_volume_with_properties(
2136 self, volume_uuid, volume_name, size, place_resources,
2137 no_diskless=False
2138 ):
2139 if self.check_volume_exists(volume_uuid):
2140 raise LinstorVolumeManagerError(
2141 'Could not create volume `{}` from SR `{}`, it already exists'
2142 .format(volume_uuid, self._group_name) + ' in properties',
2143 LinstorVolumeManagerError.ERR_VOLUME_EXISTS
2144 )
2146 if volume_name in self._fetch_resource_names():
2147 raise LinstorVolumeManagerError(
2148 'Could not create volume `{}` from SR `{}`, '.format(
2149 volume_uuid, self._group_name
2150 ) + 'resource of the same name already exists in LINSTOR'
2151 )
2153 # I am paranoid.
2154 volume_properties = self._get_volume_properties(volume_uuid)
2155 if (volume_properties.get(self.PROP_NOT_EXISTS) is not None):
2156 raise LinstorVolumeManagerError(
2157 'Could not create volume `{}`, '.format(volume_uuid) +
2158 'properties already exist'
2159 )
2161 try:
2162 volume_properties[self.PROP_NOT_EXISTS] = self.STATE_CREATING
2163 volume_properties[self.PROP_VOLUME_NAME] = volume_name
2165 self._create_volume(
2166 volume_uuid, volume_name, size, place_resources, no_diskless
2167 )
2169 assert volume_properties.namespace == \
2170 self._build_volume_namespace(volume_uuid)
2171 return volume_properties
2172 except LinstorVolumeManagerError as e:
2173 # Do not destroy existing resource!
2174 # In theory we can't get this error because we check this event
2175 # before the `self._create_volume` case.
2176 # It can only happen if the same volume uuid is used in the same
2177 # call in another host.
2178 if e.code != LinstorVolumeManagerError.ERR_VOLUME_EXISTS:
2179 self._destroy_volume(volume_uuid, force=True)
2180 raise
2182 def _find_device_path(self, volume_uuid, volume_name):
2183 current_device_path = self._request_device_path(
2184 volume_uuid, volume_name, activate=True
2185 )
2187 # We use realpath here to get the /dev/drbd<id> path instead of
2188 # /dev/drbd/by-res/<resource_name>.
2189 expected_device_path = self.build_device_path(volume_name)
2190 util.wait_for_path(expected_device_path, 5)
2192 device_realpath = os.path.realpath(expected_device_path)
2193 if current_device_path != device_realpath:
2194 raise LinstorVolumeManagerError(
2195 'Invalid path, current={}, expected={} (realpath={})'
2196 .format(
2197 current_device_path,
2198 expected_device_path,
2199 device_realpath
2200 )
2201 )
2202 return expected_device_path
2204 def _request_device_path(self, volume_uuid, volume_name, activate=False):
2205 node_name = socket.gethostname()
2207 resource = next(filter(
2208 lambda resource: resource.node_name == node_name and
2209 resource.name == volume_name,
2210 self._get_resource_cache().resources
2211 ), None)
2213 if not resource:
2214 if activate:
2215 self._mark_resource_cache_as_dirty()
2216 self._activate_device_path(
2217 self._linstor, node_name, volume_name
2218 )
2219 return self._request_device_path(volume_uuid, volume_name)
2220 raise LinstorVolumeManagerError(
2221 'Empty dev path for `{}`, but definition "seems" to exist'
2222 .format(volume_uuid)
2223 )
2224 # Contains a path of the /dev/drbd<id> form.
2225 return resource.volumes[0].device_path
2227 def _destroy_resource(self, resource_name, force=False):
2228 result = self._linstor.resource_dfn_delete(resource_name)
2229 error_str = self._get_error_str(result)
2230 if not error_str:
2231 self._mark_resource_cache_as_dirty()
2232 return
2234 if not force:
2235 self._mark_resource_cache_as_dirty()
2236 raise LinstorVolumeManagerError(
2237 'Could not destroy resource `{}` from SR `{}`: {}'
2238 .format(resource_name, self._group_name, error_str)
2239 )
2241 # If force is used, ensure there is no opener.
2242 all_openers = get_all_volume_openers(resource_name, '0')
2243 for openers in all_openers.values():
2244 if openers:
2245 self._mark_resource_cache_as_dirty()
2246 raise LinstorVolumeManagerError(
2247 'Could not force destroy resource `{}` from SR `{}`: {} (openers=`{}`)'
2248 .format(resource_name, self._group_name, error_str, all_openers)
2249 )
2251 # Maybe the resource is blocked in primary mode. DRBD/LINSTOR issue?
2252 resource_states = filter(
2253 lambda resource_state: resource_state.name == resource_name,
2254 self._get_resource_cache().resource_states
2255 )
2257 # Mark only after computation of states.
2258 self._mark_resource_cache_as_dirty()
2260 for resource_state in resource_states:
2261 volume_state = resource_state.volume_states[0]
2262 if resource_state.in_use:
2263 demote_drbd_resource(resource_state.node_name, resource_name)
2264 break
2265 self._destroy_resource(resource_name)
2267 def _destroy_volume(self, volume_uuid, force=False):
2268 volume_properties = self._get_volume_properties(volume_uuid)
2269 try:
2270 volume_name = volume_properties.get(self.PROP_VOLUME_NAME)
2271 if volume_name in self._fetch_resource_names():
2272 self._destroy_resource(volume_name, force)
2274 # Assume this call is atomic.
2275 volume_properties.clear()
2276 except Exception as e:
2277 raise LinstorVolumeManagerError(
2278 'Cannot destroy volume `{}`: {}'.format(volume_uuid, e)
2279 )
2281 def _build_volumes(self, repair):
2282 properties = self._kv_cache
2283 resource_names = self._fetch_resource_names()
2285 self._volumes = set()
2287 updating_uuid_volumes = self._get_volumes_by_property(
2288 self.REG_UPDATING_UUID_SRC, ignore_inexisting_volumes=False
2289 )
2290 if updating_uuid_volumes and not repair:
2291 raise LinstorVolumeManagerError(
2292 'Cannot build LINSTOR volume list: '
2293 'It exists invalid "updating uuid volumes", repair is required'
2294 )
2296 existing_volumes = self._get_volumes_by_property(
2297 self.REG_NOT_EXISTS, ignore_inexisting_volumes=False
2298 )
2299 for volume_uuid, not_exists in existing_volumes.items():
2300 properties.namespace = self._build_volume_namespace(volume_uuid)
2302 src_uuid = properties.get(self.PROP_UPDATING_UUID_SRC)
2303 if src_uuid:
2304 self._logger(
2305 'Ignoring volume during manager initialization with prop '
2306 ' PROP_UPDATING_UUID_SRC: {} (properties={})'
2307 .format(
2308 volume_uuid,
2309 self._get_filtered_properties(properties)
2310 )
2311 )
2312 continue
2314 # Insert volume in list if the volume exists. Or if the volume
2315 # is being created and a slave wants to use it (repair = False).
2316 #
2317 # If we are on the master and if repair is True and state is
2318 # Creating, it's probably a bug or crash: the creation process has
2319 # been stopped.
2320 if not_exists == self.STATE_EXISTS or (
2321 not repair and not_exists == self.STATE_CREATING
2322 ):
2323 self._volumes.add(volume_uuid)
2324 continue
2326 if not repair:
2327 self._logger(
2328 'Ignoring bad volume during manager initialization: {} '
2329 '(properties={})'.format(
2330 volume_uuid,
2331 self._get_filtered_properties(properties)
2332 )
2333 )
2334 continue
2336 # Remove bad volume.
2337 try:
2338 self._logger(
2339 'Removing bad volume during manager initialization: {} '
2340 '(properties={})'.format(
2341 volume_uuid,
2342 self._get_filtered_properties(properties)
2343 )
2344 )
2345 volume_name = properties.get(self.PROP_VOLUME_NAME)
2347 # Little optimization, don't call `self._destroy_volume`,
2348 # we already have resource name list.
2349 if volume_name in resource_names:
2350 self._destroy_resource(volume_name, force=True)
2352 # Assume this call is atomic.
2353 properties.clear()
2354 except Exception as e:
2355 # Do not raise, we don't want to block user action.
2356 self._logger(
2357 'Cannot clean volume {}: {}'.format(volume_uuid, e)
2358 )
2360 # The volume can't be removed, maybe it's still in use,
2361 # in this case rename it with the "DELETED_" prefix.
2362 # This prefix is mandatory if it exists a snap transaction to
2363 # rollback because the original VDI UUID can try to be renamed
2364 # with the UUID we are trying to delete...
2365 if not volume_uuid.startswith('DELETED_'):
2366 self.update_volume_uuid(
2367 volume_uuid, 'DELETED_' + volume_uuid, force=True
2368 )
2370 for dest_uuid, src_uuid in updating_uuid_volumes.items():
2371 dest_namespace = self._build_volume_namespace(dest_uuid)
2373 properties.namespace = dest_namespace
2374 if int(properties.get(self.PROP_NOT_EXISTS)):
2375 properties.clear()
2376 continue
2378 properties.namespace = self._build_volume_namespace(src_uuid)
2379 properties.clear()
2381 properties.namespace = dest_namespace
2382 properties.pop(self.PROP_UPDATING_UUID_SRC)
2384 if src_uuid in self._volumes:
2385 self._volumes.remove(src_uuid)
2386 self._volumes.add(dest_uuid)
2388 def _get_sr_properties(self):
2389 return self._create_linstor_kv(self._build_sr_namespace())
2391 def _get_volumes_by_property(
2392 self, reg_prop, ignore_inexisting_volumes=True
2393 ):
2394 base_properties = self._get_kv_cache()
2395 base_properties.namespace = self._build_volume_namespace()
2397 volume_properties = {}
2398 for volume_uuid in self._volumes:
2399 volume_properties[volume_uuid] = ''
2401 for key, value in base_properties.items():
2402 res = reg_prop.match(key)
2403 if res:
2404 volume_uuid = res.groups()[0]
2405 if not ignore_inexisting_volumes or \
2406 volume_uuid in self._volumes:
2407 volume_properties[volume_uuid] = value
2409 return volume_properties
2411 def _create_linstor_kv(self, namespace):
2412 return linstor.KV(
2413 self._group_name,
2414 uri=self._linstor.controller_host(),
2415 namespace=namespace
2416 )
2418 def _get_volume_properties(self, volume_uuid):
2419 properties = self._get_kv_cache()
2420 properties.namespace = self._build_volume_namespace(volume_uuid)
2421 return properties
2423 @classmethod
2424 def _build_sr_namespace(cls):
2425 return '/{}/'.format(cls.NAMESPACE_SR)
2427 @classmethod
2428 def _build_volume_namespace(cls, volume_uuid=None):
2429 # Return a path to all volumes if `volume_uuid` is not given.
2430 if volume_uuid is None:
2431 return '/{}/'.format(cls.NAMESPACE_VOLUME)
2432 return '/{}/{}/'.format(cls.NAMESPACE_VOLUME, volume_uuid)
2434 @classmethod
2435 def _get_error_str(cls, result):
2436 return ', '.join([
2437 err.message for err in cls._filter_errors(result)
2438 ])
2440 @classmethod
2441 def _create_linstor_instance(
2442 cls, uri, keep_uri_unmodified=False, attempt_count=30
2443 ):
2444 retry = False
2446 def connect(uri):
2447 if not uri:
2448 uri = get_controller_uri()
2449 if not uri:
2450 raise LinstorVolumeManagerError(
2451 'Unable to find controller uri...'
2452 )
2453 instance = linstor.Linstor(uri, keep_alive=True)
2454 instance.connect()
2455 return instance
2457 try:
2458 return connect(uri)
2459 except (linstor.errors.LinstorNetworkError, LinstorVolumeManagerError):
2460 pass
2462 if not keep_uri_unmodified:
2463 uri = None
2465 return util.retry(
2466 lambda: connect(uri),
2467 maxretry=attempt_count,
2468 period=1,
2469 exceptions=[
2470 linstor.errors.LinstorNetworkError,
2471 LinstorVolumeManagerError
2472 ]
2473 )
2475 @classmethod
2476 def _configure_volume_peer_slots(cls, lin, volume_name):
2477 result = lin.resource_dfn_modify(volume_name, {}, peer_slots=3)
2478 error_str = cls._get_error_str(result)
2479 if error_str:
2480 raise LinstorVolumeManagerError(
2481 'Could not configure volume peer slots of {}: {}'
2482 .format(volume_name, error_str)
2483 )
2485 @classmethod
2486 def _activate_device_path(cls, lin, node_name, volume_name):
2487 result = lin.resource_create([
2488 linstor.ResourceData(node_name, volume_name, diskless=True)
2489 ])
2490 if linstor.Linstor.all_api_responses_no_error(result):
2491 return
2492 errors = linstor.Linstor.filter_api_call_response_errors(result)
2493 if len(errors) == 1 and errors[0].is_error(
2494 linstor.consts.FAIL_EXISTS_RSC
2495 ):
2496 return
2498 raise LinstorVolumeManagerError(
2499 'Unable to activate device path of `{}` on node `{}`: {}'
2500 .format(volume_name, node_name, ', '.join(
2501 [str(x) for x in result]))
2502 )
2504 @classmethod
2505 def _request_database_path(cls, lin, activate=False):
2506 node_name = socket.gethostname()
2508 try:
2509 resource = next(filter(
2510 lambda resource: resource.node_name == node_name and
2511 resource.name == DATABASE_VOLUME_NAME,
2512 lin.resource_list_raise().resources
2513 ), None)
2514 except Exception as e:
2515 raise LinstorVolumeManagerError(
2516 'Unable to get resources during database creation: {}'
2517 .format(e)
2518 )
2520 if not resource:
2521 if activate:
2522 cls._activate_device_path(
2523 lin, node_name, DATABASE_VOLUME_NAME
2524 )
2525 return cls._request_database_path(
2526 DATABASE_VOLUME_NAME, DATABASE_VOLUME_NAME
2527 )
2528 raise LinstorVolumeManagerError(
2529 'Empty dev path for `{}`, but definition "seems" to exist'
2530 .format(DATABASE_PATH)
2531 )
2532 # Contains a path of the /dev/drbd<id> form.
2533 return resource.volumes[0].device_path
2535 @classmethod
2536 def _create_database_volume(
2537 cls, lin, group_name, node_names, redundancy, auto_quorum
2538 ):
2539 try:
2540 dfns = lin.resource_dfn_list_raise().resource_definitions
2541 except Exception as e:
2542 raise LinstorVolumeManagerError(
2543 'Unable to get definitions during database creation: {}'
2544 .format(e)
2545 )
2547 if dfns:
2548 raise LinstorVolumeManagerError(
2549 'Could not create volume `{}` from SR `{}`, '.format(
2550 DATABASE_VOLUME_NAME, group_name
2551 ) + 'LINSTOR volume list must be empty.'
2552 )
2554 # Workaround to use thin lvm. Without this line an error is returned:
2555 # "Not enough available nodes"
2556 # I don't understand why but this command protect against this bug.
2557 try:
2558 pools = lin.storage_pool_list_raise(
2559 filter_by_stor_pools=[group_name]
2560 )
2561 except Exception as e:
2562 raise LinstorVolumeManagerError(
2563 'Failed to get storage pool list before database creation: {}'
2564 .format(e)
2565 )
2567 # Ensure we have a correct list of storage pools.
2568 nodes_with_pool = [pool.node_name for pool in pools.storage_pools]
2569 assert nodes_with_pool # We must have at least one storage pool!
2570 for node_name in nodes_with_pool:
2571 assert node_name in node_names
2572 util.SMlog('Nodes with storage pool: {}'.format(nodes_with_pool))
2574 # Create the database definition.
2575 size = cls.round_up_volume_size(DATABASE_SIZE)
2576 cls._check_volume_creation_errors(lin.resource_group_spawn(
2577 rsc_grp_name=group_name,
2578 rsc_dfn_name=DATABASE_VOLUME_NAME,
2579 vlm_sizes=['{}B'.format(size)],
2580 definitions_only=True
2581 ), DATABASE_VOLUME_NAME, group_name)
2582 cls._configure_volume_peer_slots(lin, DATABASE_VOLUME_NAME)
2584 # Create real resources on the first nodes.
2585 resources = []
2587 diskful_nodes = []
2588 diskless_nodes = []
2589 for node_name in node_names:
2590 if node_name in nodes_with_pool:
2591 diskful_nodes.append(node_name)
2592 else:
2593 diskless_nodes.append(node_name)
2595 assert diskful_nodes
2596 for node_name in diskful_nodes[:redundancy]:
2597 util.SMlog('Create database diskful on {}'.format(node_name))
2598 resources.append(linstor.ResourceData(
2599 node_name=node_name,
2600 rsc_name=DATABASE_VOLUME_NAME,
2601 storage_pool=group_name
2602 ))
2603 # Create diskless resources on the remaining set.
2604 for node_name in diskful_nodes[redundancy:] + diskless_nodes:
2605 util.SMlog('Create database diskless on {}'.format(node_name))
2606 resources.append(linstor.ResourceData(
2607 node_name=node_name,
2608 rsc_name=DATABASE_VOLUME_NAME,
2609 diskless=True
2610 ))
2612 result = lin.resource_create(resources)
2613 error_str = cls._get_error_str(result)
2614 if error_str:
2615 raise LinstorVolumeManagerError(
2616 'Could not create database volume from SR `{}`: {}'.format(
2617 group_name, error_str
2618 )
2619 )
2621 # We must modify the quorum. Otherwise we can't use correctly the
2622 # drbd-reactor daemon.
2623 if auto_quorum:
2624 result = lin.resource_dfn_modify(DATABASE_VOLUME_NAME, {
2625 'DrbdOptions/auto-quorum': 'disabled',
2626 'DrbdOptions/Resource/quorum': 'majority'
2627 })
2628 error_str = cls._get_error_str(result)
2629 if error_str:
2630 raise LinstorVolumeManagerError(
2631 'Could not activate quorum on database volume: {}'
2632 .format(error_str)
2633 )
2635 # Create database and ensure path exists locally and
2636 # on replicated devices.
2637 current_device_path = cls._request_database_path(lin, activate=True)
2639 # Ensure diskless paths exist on other hosts. Otherwise PBDs can't be
2640 # plugged.
2641 for node_name in node_names:
2642 cls._activate_device_path(lin, node_name, DATABASE_VOLUME_NAME)
2644 # We use realpath here to get the /dev/drbd<id> path instead of
2645 # /dev/drbd/by-res/<resource_name>.
2646 expected_device_path = cls.build_device_path(DATABASE_VOLUME_NAME)
2647 util.wait_for_path(expected_device_path, 5)
2649 device_realpath = os.path.realpath(expected_device_path)
2650 if current_device_path != device_realpath:
2651 raise LinstorVolumeManagerError(
2652 'Invalid path, current={}, expected={} (realpath={})'
2653 .format(
2654 current_device_path,
2655 expected_device_path,
2656 device_realpath
2657 )
2658 )
2660 try:
2661 util.pread2([DATABASE_MKFS, expected_device_path])
2662 except Exception as e:
2663 raise LinstorVolumeManagerError(
2664 'Failed to execute {} on database volume: {}'
2665 .format(DATABASE_MKFS, e)
2666 )
2668 return expected_device_path
2670 @classmethod
2671 def _destroy_database_volume(cls, lin, group_name):
2672 error_str = cls._get_error_str(
2673 lin.resource_dfn_delete(DATABASE_VOLUME_NAME)
2674 )
2675 if error_str:
2676 raise LinstorVolumeManagerError(
2677 'Could not destroy resource `{}` from SR `{}`: {}'
2678 .format(DATABASE_VOLUME_NAME, group_name, error_str)
2679 )
2681 @classmethod
2682 def _mount_database_volume(cls, volume_path, mount=True, force=False):
2683 backup_path = DATABASE_PATH + '-' + str(uuid.uuid4())
2685 try:
2686 # 1. Create a backup config folder.
2687 database_not_empty = bool(os.listdir(DATABASE_PATH))
2688 if database_not_empty:
2689 try:
2690 os.mkdir(backup_path)
2691 except Exception as e:
2692 raise LinstorVolumeManagerError(
2693 'Failed to create backup path {} of LINSTOR config: {}'
2694 .format(backup_path, e)
2695 )
2697 # 2. Move the config in the mounted volume.
2698 if database_not_empty:
2699 cls._move_files(DATABASE_PATH, backup_path)
2701 cls._mount_volume(volume_path, DATABASE_PATH, mount)
2703 if database_not_empty:
2704 cls._move_files(backup_path, DATABASE_PATH, force)
2706 # 3. Remove useless backup directory.
2707 try:
2708 os.rmdir(backup_path)
2709 except Exception:
2710 raise LinstorVolumeManagerError(
2711 'Failed to remove backup path {} of LINSTOR config {}'
2712 .format(backup_path, e)
2713 )
2714 except Exception as e:
2715 def force_exec(fn):
2716 try:
2717 fn()
2718 except Exception:
2719 pass
2721 if mount == cls._is_mounted(DATABASE_PATH):
2722 force_exec(lambda: cls._move_files(
2723 DATABASE_PATH, backup_path
2724 ))
2725 force_exec(lambda: cls._mount_volume(
2726 volume_path, DATABASE_PATH, not mount
2727 ))
2729 if mount != cls._is_mounted(DATABASE_PATH):
2730 force_exec(lambda: cls._move_files(
2731 backup_path, DATABASE_PATH
2732 ))
2734 force_exec(lambda: os.rmdir(backup_path))
2735 raise e
2737 @classmethod
2738 def _force_destroy_database_volume(cls, lin, group_name):
2739 try:
2740 cls._destroy_database_volume(lin, group_name)
2741 except Exception:
2742 pass
2744 @classmethod
2745 def _destroy_storage_pool(cls, lin, group_name, node_name):
2746 def destroy():
2747 result = lin.storage_pool_delete(node_name, group_name)
2748 errors = cls._filter_errors(result)
2749 if cls._check_errors(errors, [
2750 linstor.consts.FAIL_NOT_FOUND_STOR_POOL,
2751 linstor.consts.FAIL_NOT_FOUND_STOR_POOL_DFN
2752 ]):
2753 return
2755 if errors:
2756 raise LinstorVolumeManagerError(
2757 'Failed to destroy SP `{}` on node `{}`: {}'.format(
2758 group_name,
2759 node_name,
2760 cls._get_error_str(errors)
2761 )
2762 )
2764 # We must retry to avoid errors like:
2765 # "can not be deleted as volumes / snapshot-volumes are still using it"
2766 # after LINSTOR database volume destruction.
2767 return util.retry(destroy, maxretry=10)
2769 @classmethod
2770 def _destroy_resource_group(cls, lin, group_name):
2771 def destroy():
2772 result = lin.resource_group_delete(group_name)
2773 errors = cls._filter_errors(result)
2774 if cls._check_errors(errors, [
2775 linstor.consts.FAIL_NOT_FOUND_RSC_GRP
2776 ]):
2777 return
2779 if errors:
2780 raise LinstorVolumeManagerError(
2781 'Failed to destroy RG `{}`: {}'
2782 .format(group_name, cls._get_error_str(errors))
2783 )
2785 return util.retry(destroy, maxretry=10)
2787 @classmethod
2788 def _build_group_name(cls, base_name):
2789 # If thin provisioning is used we have a path like this:
2790 # `VG/LV`. "/" is not accepted by LINSTOR.
2791 return '{}{}'.format(cls.PREFIX_SR, base_name.replace('/', '_'))
2793 @classmethod
2794 def _check_volume_creation_errors(cls, result, volume_uuid, group_name):
2795 errors = cls._filter_errors(result)
2796 if cls._check_errors(errors, [
2797 linstor.consts.FAIL_EXISTS_RSC, linstor.consts.FAIL_EXISTS_RSC_DFN
2798 ]):
2799 raise LinstorVolumeManagerError(
2800 'Failed to create volume `{}` from SR `{}`, it already exists'
2801 .format(volume_uuid, group_name),
2802 LinstorVolumeManagerError.ERR_VOLUME_EXISTS
2803 )
2805 if errors:
2806 raise LinstorVolumeManagerError(
2807 'Failed to create volume `{}` from SR `{}`: {}'.format(
2808 volume_uuid,
2809 group_name,
2810 cls._get_error_str(errors)
2811 )
2812 )
2814 @classmethod
2815 def _move_files(cls, src_dir, dest_dir, force=False):
2816 def listdir(dir):
2817 ignored = ['lost+found']
2818 return [file for file in os.listdir(dir) if file not in ignored]
2820 try:
2821 if not force:
2822 files = listdir(dest_dir)
2823 if files:
2824 raise LinstorVolumeManagerError(
2825 'Cannot move files from {} to {} because destination '
2826 'contains: {}'.format(src_dir, dest_dir, files)
2827 )
2828 except LinstorVolumeManagerError:
2829 raise
2830 except Exception as e:
2831 raise LinstorVolumeManagerError(
2832 'Cannot list dir {}: {}'.format(dest_dir, e)
2833 )
2835 try:
2836 for file in listdir(src_dir):
2837 try:
2838 dest_file = os.path.join(dest_dir, file)
2839 if not force and os.path.exists(dest_file):
2840 raise LinstorVolumeManagerError(
2841 'Cannot move {} because it already exists in the '
2842 'destination'.format(file)
2843 )
2844 shutil.move(os.path.join(src_dir, file), dest_file)
2845 except LinstorVolumeManagerError:
2846 raise
2847 except Exception as e:
2848 raise LinstorVolumeManagerError(
2849 'Cannot move {}: {}'.format(file, e)
2850 )
2851 except Exception as e:
2852 if not force:
2853 try:
2854 cls._move_files(dest_dir, src_dir, force=True)
2855 except Exception:
2856 pass
2858 raise LinstorVolumeManagerError(
2859 'Failed to move files from {} to {}: {}'.format(
2860 src_dir, dest_dir, e
2861 )
2862 )
2864 @staticmethod
2865 def _get_filtered_properties(properties):
2866 return dict(properties.items())
2868 @staticmethod
2869 def _filter_errors(result):
2870 return [
2871 err for err in result
2872 if hasattr(err, 'is_error') and err.is_error()
2873 ]
2875 @staticmethod
2876 def _check_errors(result, codes):
2877 for err in result:
2878 for code in codes:
2879 if err.is_error(code):
2880 return True
2881 return False
2883 @classmethod
2884 def _controller_is_running(cls):
2885 return cls._service_is_running('linstor-controller')
2887 @classmethod
2888 def _start_controller(cls, start=True):
2889 return cls._start_service('linstor-controller', start)
2891 @staticmethod
2892 def _start_service(name, start=True):
2893 action = 'start' if start else 'stop'
2894 (ret, out, err) = util.doexec([
2895 'systemctl', action, name
2896 ])
2897 if ret != 0:
2898 raise LinstorVolumeManagerError(
2899 'Failed to {} {}: {} {}'
2900 .format(action, name, out, err)
2901 )
2903 @staticmethod
2904 def _service_is_running(name):
2905 (ret, out, err) = util.doexec([
2906 'systemctl', 'is-active', '--quiet', name
2907 ])
2908 return not ret
2910 @staticmethod
2911 def _is_mounted(mountpoint):
2912 (ret, out, err) = util.doexec(['mountpoint', '-q', mountpoint])
2913 return ret == 0
2915 @classmethod
2916 def _mount_volume(cls, volume_path, mountpoint, mount=True):
2917 if mount:
2918 try:
2919 util.pread(['mount', volume_path, mountpoint])
2920 except Exception as e:
2921 raise LinstorVolumeManagerError(
2922 'Failed to mount volume {} on {}: {}'
2923 .format(volume_path, mountpoint, e)
2924 )
2925 else:
2926 try:
2927 if cls._is_mounted(mountpoint):
2928 util.pread(['umount', mountpoint])
2929 except Exception as e:
2930 raise LinstorVolumeManagerError(
2931 'Failed to umount volume {} on {}: {}'
2932 .format(volume_path, mountpoint, e)
2933 )
2936# ==============================================================================
2938# Check if a path is a DRBD resource and log the process name/pid
2939# that opened it.
2940def log_drbd_openers(path):
2941 # Ignore if it's not a symlink to DRBD resource.
2942 if not path.startswith(DRBD_BY_RES_PATH):
2943 return
2945 # Compute resource name.
2946 res_name_end = path.find('/', len(DRBD_BY_RES_PATH))
2947 if res_name_end == -1:
2948 return
2949 res_name = path[len(DRBD_BY_RES_PATH):res_name_end]
2951 volume_end = path.rfind('/')
2952 if volume_end == res_name_end:
2953 return
2954 volume = path[volume_end + 1:]
2956 try:
2957 # Ensure path is a DRBD.
2958 drbd_path = os.path.realpath(path)
2959 stats = os.stat(drbd_path)
2960 if not stat.S_ISBLK(stats.st_mode) or os.major(stats.st_rdev) != 147:
2961 return
2963 # Find where the device is open.
2964 (ret, stdout, stderr) = util.doexec(['drbdadm', 'status', res_name])
2965 if ret != 0:
2966 util.SMlog('Failed to execute `drbdadm status` on `{}`: {}'.format(
2967 res_name, stderr
2968 ))
2969 return
2971 # Is it a local device?
2972 if stdout.startswith('{} role:Primary'.format(res_name)):
2973 util.SMlog(
2974 'DRBD resource `{}` is open on local host: {}'
2975 .format(path, get_local_volume_openers(res_name, volume))
2976 )
2977 return
2979 # Is it a remote device?
2980 util.SMlog(
2981 'DRBD resource `{}` is open on hosts: {}'
2982 .format(path, get_all_volume_openers(res_name, volume))
2983 )
2984 except Exception as e:
2985 util.SMlog(
2986 'Got exception while trying to determine where DRBD resource ' +
2987 '`{}` is open: {}'.format(path, e)
2988 )