Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1#!/usr/bin/python3 

2# 

3# Copyright (C) Citrix Systems Inc. 

4# 

5# This program is free software; you can redistribute it and/or modify 

6# it under the terms of the GNU Lesser General Public License as published 

7# by the Free Software Foundation; version 2.1 only. 

8# 

9# This program is distributed in the hope that it will be useful, 

10# but WITHOUT ANY WARRANTY; without even the implied warranty of 

11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

12# GNU Lesser General Public License for more details. 

13# 

14# You should have received a copy of the GNU Lesser General Public License 

15# along with this program; if not, write to the Free Software Foundation, Inc., 

16# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 

17# 

18# Script to coalesce and garbage collect VHD-based SR's in the background 

19# 

20 

21from sm_typing import Optional, override 

22 

23import os 

24import os.path 

25import sys 

26import time 

27import signal 

28import subprocess 

29import getopt 

30import datetime 

31import traceback 

32import base64 

33import zlib 

34import errno 

35import stat 

36 

37import XenAPI # pylint: disable=import-error 

38import util 

39import lvutil 

40import vhdutil 

41import lvhdutil 

42import lvmcache 

43import journaler 

44import fjournaler 

45import lock 

46import blktap2 

47import xs_errors 

48from refcounter import RefCounter 

49from ipc import IPCFlag 

50from lvmanager import LVActivator 

51from srmetadata import LVMMetadataHandler, VDI_TYPE_TAG 

52from functools import reduce 

53from time import monotonic as _time 

54 

55try: 

56 from linstorjournaler import LinstorJournaler 

57 from linstorvhdutil import LinstorVhdUtil 

58 from linstorvolumemanager import get_controller_uri 

59 from linstorvolumemanager import LinstorVolumeManager 

60 from linstorvolumemanager import LinstorVolumeManagerError 

61 from linstorvolumemanager import PERSISTENT_PREFIX as LINSTOR_PERSISTENT_PREFIX 

62 

63 LINSTOR_AVAILABLE = True 

64except ImportError: 

65 LINSTOR_AVAILABLE = False 

66 

67# Disable automatic leaf-coalescing. Online leaf-coalesce is currently not 

68# possible due to lvhd_stop_using_() not working correctly. However, we leave 

69# this option available through the explicit LEAFCLSC_FORCE flag in the VDI 

70# record for use by the offline tool (which makes the operation safe by pausing 

71# the VM first) 

72AUTO_ONLINE_LEAF_COALESCE_ENABLED = True 

73 

74FLAG_TYPE_ABORT = "abort" # flag to request aborting of GC/coalesce 

75 

76# process "lock", used simply as an indicator that a process already exists 

77# that is doing GC/coalesce on this SR (such a process holds the lock, and we 

78# check for the fact by trying the lock). 

79lockGCRunning = None 

80 

81# process "lock" to indicate that the GC process has been activated but may not 

82# yet be running, stops a second process from being started. 

83LOCK_TYPE_GC_ACTIVE = "gc_active" 

84lockGCActive = None 

85 

86# Default coalesce error rate limit, in messages per minute. A zero value 

87# disables throttling, and a negative value disables error reporting. 

88DEFAULT_COALESCE_ERR_RATE = 1.0 / 60 

89 

90COALESCE_LAST_ERR_TAG = 'last-coalesce-error' 

91COALESCE_ERR_RATE_TAG = 'coalesce-error-rate' 

92VAR_RUN = "/var/run/" 

93SPEED_LOG_ROOT = VAR_RUN + "{uuid}.speed_log" 

94 

95N_RUNNING_AVERAGE = 10 

96 

97NON_PERSISTENT_DIR = '/run/nonpersistent/sm' 

98 

99 

100class AbortException(util.SMException): 

101 pass 

102 

103 

104################################################################################ 

105# 

106# Util 

107# 

108class Util: 

109 RET_RC = 1 

110 RET_STDOUT = 2 

111 RET_STDERR = 4 

112 

113 UUID_LEN = 36 

114 

115 PREFIX = {"G": 1024 * 1024 * 1024, "M": 1024 * 1024, "K": 1024} 

116 

117 @staticmethod 

118 def log(text) -> None: 

119 util.SMlog(text, ident="SMGC") 

120 

121 @staticmethod 

122 def logException(tag): 

123 info = sys.exc_info() 

124 if info[0] == SystemExit: 124 ↛ 126line 124 didn't jump to line 126, because the condition on line 124 was never true

125 # this should not be happening when catching "Exception", but it is 

126 sys.exit(0) 

127 tb = reduce(lambda a, b: "%s%s" % (a, b), traceback.format_tb(info[2])) 

128 Util.log("*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*") 

129 Util.log(" ***********************") 

130 Util.log(" * E X C E P T I O N *") 

131 Util.log(" ***********************") 

132 Util.log("%s: EXCEPTION %s, %s" % (tag, info[0], info[1])) 

133 Util.log(tb) 

134 Util.log("*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*") 

135 

136 @staticmethod 

137 def doexec(args, expectedRC, inputtext=None, ret=None, log=True): 

138 "Execute a subprocess, then return its return code, stdout, stderr" 

139 proc = subprocess.Popen(args, 

140 stdin=subprocess.PIPE, \ 

141 stdout=subprocess.PIPE, \ 

142 stderr=subprocess.PIPE, \ 

143 shell=True, \ 

144 close_fds=True) 

145 (stdout, stderr) = proc.communicate(inputtext) 

146 stdout = str(stdout) 

147 stderr = str(stderr) 

148 rc = proc.returncode 

149 if log: 

150 Util.log("`%s`: %s" % (args, rc)) 

151 if type(expectedRC) != type([]): 

152 expectedRC = [expectedRC] 

153 if not rc in expectedRC: 

154 reason = stderr.strip() 

155 if stdout.strip(): 

156 reason = "%s (stdout: %s)" % (reason, stdout.strip()) 

157 Util.log("Failed: %s" % reason) 

158 raise util.CommandException(rc, args, reason) 

159 

160 if ret == Util.RET_RC: 

161 return rc 

162 if ret == Util.RET_STDERR: 

163 return stderr 

164 return stdout 

165 

166 @staticmethod 

167 def runAbortable(func, ret, ns, abortTest, pollInterval, timeOut): 

168 """execute func in a separate thread and kill it if abortTest signals 

169 so""" 

170 abortSignaled = abortTest() # check now before we clear resultFlag 

171 resultFlag = IPCFlag(ns) 

172 resultFlag.clearAll() 

173 pid = os.fork() 

174 if pid: 

175 startTime = _time() 

176 try: 

177 while True: 

178 if resultFlag.test("success"): 

179 Util.log(" Child process completed successfully") 

180 resultFlag.clear("success") 

181 return 

182 if resultFlag.test("failure"): 

183 resultFlag.clear("failure") 

184 raise util.SMException("Child process exited with error") 

185 if abortTest() or abortSignaled: 

186 os.killpg(pid, signal.SIGKILL) 

187 raise AbortException("Aborting due to signal") 

188 if timeOut and _time() - startTime > timeOut: 

189 os.killpg(pid, signal.SIGKILL) 

190 resultFlag.clearAll() 

191 raise util.SMException("Timed out") 

192 time.sleep(pollInterval) 

193 finally: 

194 wait_pid = 0 

195 rc = -1 

196 count = 0 

197 while wait_pid == 0 and count < 10: 

198 wait_pid, rc = os.waitpid(pid, os.WNOHANG) 

199 if wait_pid == 0: 

200 time.sleep(2) 

201 count += 1 

202 

203 if wait_pid == 0: 

204 Util.log("runAbortable: wait for process completion timed out") 

205 else: 

206 os.setpgrp() 

207 try: 

208 if func() == ret: 

209 resultFlag.set("success") 

210 else: 

211 resultFlag.set("failure") 

212 except Exception as e: 

213 Util.log("Child process failed with : (%s)" % e) 

214 resultFlag.set("failure") 

215 Util.logException("This exception has occured") 

216 os._exit(0) 

217 

218 @staticmethod 

219 def num2str(number): 

220 for prefix in ("G", "M", "K"): 220 ↛ 223line 220 didn't jump to line 223, because the loop on line 220 didn't complete

221 if number >= Util.PREFIX[prefix]: 

222 return "%.3f%s" % (float(number) / Util.PREFIX[prefix], prefix) 

223 return "%s" % number 

224 

225 @staticmethod 

226 def numBits(val): 

227 count = 0 

228 while val: 

229 count += val & 1 

230 val = val >> 1 

231 return count 

232 

233 @staticmethod 

234 def countBits(bitmap1, bitmap2): 

235 """return bit count in the bitmap produced by ORing the two bitmaps""" 

236 len1 = len(bitmap1) 

237 len2 = len(bitmap2) 

238 lenLong = len1 

239 lenShort = len2 

240 bitmapLong = bitmap1 

241 if len2 > len1: 

242 lenLong = len2 

243 lenShort = len1 

244 bitmapLong = bitmap2 

245 

246 count = 0 

247 for i in range(lenShort): 

248 val = bitmap1[i] | bitmap2[i] 

249 count += Util.numBits(val) 

250 

251 for i in range(i + 1, lenLong): 

252 val = bitmapLong[i] 

253 count += Util.numBits(val) 

254 return count 

255 

256 @staticmethod 

257 def getThisScript(): 

258 thisScript = util.get_real_path(__file__) 

259 if thisScript.endswith(".pyc"): 

260 thisScript = thisScript[:-1] 

261 return thisScript 

262 

263 

264################################################################################ 

265# 

266# XAPI 

267# 

268class XAPI: 

269 USER = "root" 

270 PLUGIN_ON_SLAVE = "on-slave" 

271 

272 CONFIG_SM = 0 

273 CONFIG_OTHER = 1 

274 CONFIG_ON_BOOT = 2 

275 CONFIG_ALLOW_CACHING = 3 

276 

277 CONFIG_NAME = { 

278 CONFIG_SM: "sm-config", 

279 CONFIG_OTHER: "other-config", 

280 CONFIG_ON_BOOT: "on-boot", 

281 CONFIG_ALLOW_CACHING: "allow_caching" 

282 } 

283 

284 class LookupError(util.SMException): 

285 pass 

286 

287 @staticmethod 

288 def getSession(): 

289 session = XenAPI.xapi_local() 

290 session.xenapi.login_with_password(XAPI.USER, '', '', 'SM') 

291 return session 

292 

293 def __init__(self, session, srUuid): 

294 self.sessionPrivate = False 

295 self.session = session 

296 if self.session is None: 

297 self.session = self.getSession() 

298 self.sessionPrivate = True 

299 self._srRef = self.session.xenapi.SR.get_by_uuid(srUuid) 

300 self.srRecord = self.session.xenapi.SR.get_record(self._srRef) 

301 self.hostUuid = util.get_this_host() 

302 self._hostRef = self.session.xenapi.host.get_by_uuid(self.hostUuid) 

303 self.task = None 

304 self.task_progress = {"coalescable": 0, "done": 0} 

305 

306 def __del__(self): 

307 if self.sessionPrivate: 

308 self.session.xenapi.session.logout() 

309 

310 def isPluggedHere(self): 

311 pbds = self.getAttachedPBDs() 

312 for pbdRec in pbds: 

313 if pbdRec["host"] == self._hostRef: 

314 return True 

315 return False 

316 

317 def poolOK(self): 

318 host_recs = self.session.xenapi.host.get_all_records() 

319 for host_ref, host_rec in host_recs.items(): 

320 if not host_rec["enabled"]: 

321 Util.log("Host %s not enabled" % host_rec["uuid"]) 

322 return False 

323 return True 

324 

325 def isMaster(self): 

326 if self.srRecord["shared"]: 

327 pool = list(self.session.xenapi.pool.get_all_records().values())[0] 

328 return pool["master"] == self._hostRef 

329 else: 

330 pbds = self.getAttachedPBDs() 

331 if len(pbds) < 1: 

332 raise util.SMException("Local SR not attached") 

333 elif len(pbds) > 1: 

334 raise util.SMException("Local SR multiply attached") 

335 return pbds[0]["host"] == self._hostRef 

336 

337 def getAttachedPBDs(self): 

338 """Return PBD records for all PBDs of this SR that are currently 

339 attached""" 

340 attachedPBDs = [] 

341 pbds = self.session.xenapi.PBD.get_all_records() 

342 for pbdRec in pbds.values(): 

343 if pbdRec["SR"] == self._srRef and pbdRec["currently_attached"]: 

344 attachedPBDs.append(pbdRec) 

345 return attachedPBDs 

346 

347 def getOnlineHosts(self): 

348 return util.get_online_hosts(self.session) 

349 

350 def ensureInactive(self, hostRef, args): 

351 text = self.session.xenapi.host.call_plugin( \ 

352 hostRef, self.PLUGIN_ON_SLAVE, "multi", args) 

353 Util.log("call-plugin returned: '%s'" % text) 

354 

355 def getRecordHost(self, hostRef): 

356 return self.session.xenapi.host.get_record(hostRef) 

357 

358 def _getRefVDI(self, uuid): 

359 return self.session.xenapi.VDI.get_by_uuid(uuid) 

360 

361 def getRefVDI(self, vdi): 

362 return self._getRefVDI(vdi.uuid) 

363 

364 def getRecordVDI(self, uuid): 

365 try: 

366 ref = self._getRefVDI(uuid) 

367 return self.session.xenapi.VDI.get_record(ref) 

368 except XenAPI.Failure: 

369 return None 

370 

371 def singleSnapshotVDI(self, vdi): 

372 return self.session.xenapi.VDI.snapshot(vdi.getRef(), 

373 {"type": "internal"}) 

374 

375 def forgetVDI(self, srUuid, vdiUuid): 

376 """Forget the VDI, but handle the case where the VDI has already been 

377 forgotten (i.e. ignore errors)""" 

378 try: 

379 vdiRef = self.session.xenapi.VDI.get_by_uuid(vdiUuid) 

380 self.session.xenapi.VDI.forget(vdiRef) 

381 except XenAPI.Failure: 

382 pass 

383 

384 def getConfigVDI(self, vdi, key): 

385 kind = vdi.CONFIG_TYPE[key] 

386 if kind == self.CONFIG_SM: 

387 cfg = self.session.xenapi.VDI.get_sm_config(vdi.getRef()) 

388 elif kind == self.CONFIG_OTHER: 

389 cfg = self.session.xenapi.VDI.get_other_config(vdi.getRef()) 

390 elif kind == self.CONFIG_ON_BOOT: 

391 cfg = self.session.xenapi.VDI.get_on_boot(vdi.getRef()) 

392 elif kind == self.CONFIG_ALLOW_CACHING: 

393 cfg = self.session.xenapi.VDI.get_allow_caching(vdi.getRef()) 

394 else: 

395 assert(False) 

396 Util.log("Got %s for %s: %s" % (self.CONFIG_NAME[kind], vdi, repr(cfg))) 

397 return cfg 

398 

399 def removeFromConfigVDI(self, vdi, key): 

400 kind = vdi.CONFIG_TYPE[key] 

401 if kind == self.CONFIG_SM: 

402 self.session.xenapi.VDI.remove_from_sm_config(vdi.getRef(), key) 

403 elif kind == self.CONFIG_OTHER: 

404 self.session.xenapi.VDI.remove_from_other_config(vdi.getRef(), key) 

405 else: 

406 assert(False) 

407 

408 def addToConfigVDI(self, vdi, key, val): 

409 kind = vdi.CONFIG_TYPE[key] 

410 if kind == self.CONFIG_SM: 

411 self.session.xenapi.VDI.add_to_sm_config(vdi.getRef(), key, val) 

412 elif kind == self.CONFIG_OTHER: 

413 self.session.xenapi.VDI.add_to_other_config(vdi.getRef(), key, val) 

414 else: 

415 assert(False) 

416 

417 def isSnapshot(self, vdi): 

418 return self.session.xenapi.VDI.get_is_a_snapshot(vdi.getRef()) 

419 

420 def markCacheSRsDirty(self): 

421 sr_refs = self.session.xenapi.SR.get_all_records_where( \ 

422 'field "local_cache_enabled" = "true"') 

423 for sr_ref in sr_refs: 

424 Util.log("Marking SR %s dirty" % sr_ref) 

425 util.set_dirty(self.session, sr_ref) 

426 

427 def srUpdate(self): 

428 Util.log("Starting asynch srUpdate for SR %s" % self.srRecord["uuid"]) 

429 abortFlag = IPCFlag(self.srRecord["uuid"]) 

430 task = self.session.xenapi.Async.SR.update(self._srRef) 

431 cancelTask = True 

432 try: 

433 for i in range(60): 

434 status = self.session.xenapi.task.get_status(task) 

435 if not status == "pending": 

436 Util.log("SR.update_asynch status changed to [%s]" % status) 

437 cancelTask = False 

438 return 

439 if abortFlag.test(FLAG_TYPE_ABORT): 

440 Util.log("Abort signalled during srUpdate, cancelling task...") 

441 try: 

442 self.session.xenapi.task.cancel(task) 

443 cancelTask = False 

444 Util.log("Task cancelled") 

445 except: 

446 pass 

447 return 

448 time.sleep(1) 

449 finally: 

450 if cancelTask: 

451 self.session.xenapi.task.cancel(task) 

452 self.session.xenapi.task.destroy(task) 

453 Util.log("Asynch srUpdate still running, but timeout exceeded.") 

454 

455 def update_task(self): 

456 self.session.xenapi.task.set_other_config( 

457 self.task, 

458 { 

459 "applies_to": self._srRef 

460 }) 

461 total = self.task_progress['coalescable'] + self.task_progress['done'] 

462 if (total > 0): 

463 self.session.xenapi.task.set_progress( 

464 self.task, float(self.task_progress['done']) / total) 

465 

466 def create_task(self, label, description): 

467 self.task = self.session.xenapi.task.create(label, description) 

468 self.update_task() 

469 

470 def update_task_progress(self, key, value): 

471 self.task_progress[key] = value 

472 if self.task: 

473 self.update_task() 

474 

475 def set_task_status(self, status): 

476 if self.task: 

477 self.session.xenapi.task.set_status(self.task, status) 

478 

479 

480################################################################################ 

481# 

482# VDI 

483# 

484class VDI(object): 

485 """Object representing a VDI of a VHD-based SR""" 

486 

487 POLL_INTERVAL = 1 

488 POLL_TIMEOUT = 30 

489 DEVICE_MAJOR = 202 

490 DRIVER_NAME_VHD = "vhd" 

491 

492 # config keys & values 

493 DB_VHD_PARENT = "vhd-parent" 

494 DB_VDI_TYPE = "vdi_type" 

495 DB_VHD_BLOCKS = "vhd-blocks" 

496 DB_VDI_PAUSED = "paused" 

497 DB_VDI_RELINKING = "relinking" 

498 DB_VDI_ACTIVATING = "activating" 

499 DB_GC = "gc" 

500 DB_COALESCE = "coalesce" 

501 DB_LEAFCLSC = "leaf-coalesce" # config key 

502 LEAFCLSC_DISABLED = "false" # set by user; means do not leaf-coalesce 

503 LEAFCLSC_FORCE = "force" # set by user; means skip snap-coalesce 

504 LEAFCLSC_OFFLINE = "offline" # set here for informational purposes: means 

505 # no space to snap-coalesce or unable to keep 

506 # up with VDI. This is not used by the SM, it 

507 # might be used by external components. 

508 DB_ONBOOT = "on-boot" 

509 ONBOOT_RESET = "reset" 

510 DB_ALLOW_CACHING = "allow_caching" 

511 

512 CONFIG_TYPE = { 

513 DB_VHD_PARENT: XAPI.CONFIG_SM, 

514 DB_VDI_TYPE: XAPI.CONFIG_SM, 

515 DB_VHD_BLOCKS: XAPI.CONFIG_SM, 

516 DB_VDI_PAUSED: XAPI.CONFIG_SM, 

517 DB_VDI_RELINKING: XAPI.CONFIG_SM, 

518 DB_VDI_ACTIVATING: XAPI.CONFIG_SM, 

519 DB_GC: XAPI.CONFIG_OTHER, 

520 DB_COALESCE: XAPI.CONFIG_OTHER, 

521 DB_LEAFCLSC: XAPI.CONFIG_OTHER, 

522 DB_ONBOOT: XAPI.CONFIG_ON_BOOT, 

523 DB_ALLOW_CACHING: XAPI.CONFIG_ALLOW_CACHING, 

524 } 

525 

526 LIVE_LEAF_COALESCE_MAX_SIZE = 20 * 1024 * 1024 # bytes 

527 LIVE_LEAF_COALESCE_TIMEOUT = 10 # seconds 

528 TIMEOUT_SAFETY_MARGIN = 0.5 # extra margin when calculating 

529 # feasibility of leaf coalesce 

530 

531 JRN_RELINK = "relink" # journal entry type for relinking children 

532 JRN_COALESCE = "coalesce" # to communicate which VDI is being coalesced 

533 JRN_LEAF = "leaf" # used in coalesce-leaf 

534 

535 STR_TREE_INDENT = 4 

536 

537 def __init__(self, sr, uuid, raw): 

538 self.sr = sr 

539 self.scanError = True 

540 self.uuid = uuid 

541 self.raw = raw 

542 self.fileName = "" 

543 self.parentUuid = "" 

544 self.sizeVirt = -1 

545 self._sizeVHD = -1 

546 self._sizeAllocated = -1 

547 self.hidden = False 

548 self.parent = None 

549 self.children = [] 

550 self._vdiRef = None 

551 self._clearRef() 

552 

553 @staticmethod 

554 def extractUuid(path): 

555 raise NotImplementedError("Implement in sub class") 

556 

557 def load(self, info=None) -> None: 

558 """Load VDI info""" 

559 pass 

560 

561 def getDriverName(self) -> str: 

562 return self.DRIVER_NAME_VHD 

563 

564 def getRef(self): 

565 if self._vdiRef is None: 

566 self._vdiRef = self.sr.xapi.getRefVDI(self) 

567 return self._vdiRef 

568 

569 def getConfig(self, key, default=None): 

570 config = self.sr.xapi.getConfigVDI(self, key) 

571 if key == self.DB_ONBOOT or key == self.DB_ALLOW_CACHING: 571 ↛ 572line 571 didn't jump to line 572, because the condition on line 571 was never true

572 val = config 

573 else: 

574 val = config.get(key) 

575 if val: 

576 return val 

577 return default 

578 

579 def setConfig(self, key, val): 

580 self.sr.xapi.removeFromConfigVDI(self, key) 

581 self.sr.xapi.addToConfigVDI(self, key, val) 

582 Util.log("Set %s = %s for %s" % (key, val, self)) 

583 

584 def delConfig(self, key): 

585 self.sr.xapi.removeFromConfigVDI(self, key) 

586 Util.log("Removed %s from %s" % (key, self)) 

587 

588 def ensureUnpaused(self): 

589 if self.getConfig(self.DB_VDI_PAUSED) == "true": 

590 Util.log("Unpausing VDI %s" % self) 

591 self.unpause() 

592 

593 def pause(self, failfast=False) -> None: 

594 if not blktap2.VDI.tap_pause(self.sr.xapi.session, self.sr.uuid, 

595 self.uuid, failfast): 

596 raise util.SMException("Failed to pause VDI %s" % self) 

597 

598 def _report_tapdisk_unpause_error(self): 

599 try: 

600 xapi = self.sr.xapi.session.xenapi 

601 sr_ref = xapi.SR.get_by_uuid(self.sr.uuid) 

602 msg_name = "failed to unpause tapdisk" 

603 msg_body = "Failed to unpause tapdisk for VDI %s, " \ 

604 "VMs using this tapdisk have lost access " \ 

605 "to the corresponding disk(s)" % self.uuid 

606 xapi.message.create(msg_name, "4", "SR", self.sr.uuid, msg_body) 

607 except Exception as e: 

608 util.SMlog("failed to generate message: %s" % e) 

609 

610 def unpause(self): 

611 if not blktap2.VDI.tap_unpause(self.sr.xapi.session, self.sr.uuid, 

612 self.uuid): 

613 self._report_tapdisk_unpause_error() 

614 raise util.SMException("Failed to unpause VDI %s" % self) 

615 

616 def refresh(self, ignoreNonexistent=True): 

617 """Pause-unpause in one step""" 

618 self.sr.lock() 

619 try: 

620 try: 

621 if not blktap2.VDI.tap_refresh(self.sr.xapi.session, 621 ↛ 623line 621 didn't jump to line 623, because the condition on line 621 was never true

622 self.sr.uuid, self.uuid): 

623 self._report_tapdisk_unpause_error() 

624 raise util.SMException("Failed to refresh %s" % self) 

625 except XenAPI.Failure as e: 

626 if util.isInvalidVDI(e) and ignoreNonexistent: 

627 Util.log("VDI %s not found, ignoring" % self) 

628 return 

629 raise 

630 finally: 

631 self.sr.unlock() 

632 

633 def isSnapshot(self): 

634 return self.sr.xapi.isSnapshot(self) 

635 

636 def isAttachedRW(self): 

637 return util.is_attached_rw( 

638 self.sr.xapi.session.xenapi.VDI.get_sm_config(self.getRef())) 

639 

640 def getVHDBlocks(self): 

641 val = self.updateBlockInfo() 

642 bitmap = zlib.decompress(base64.b64decode(val)) 

643 return bitmap 

644 

645 def isCoalesceable(self): 

646 """A VDI is coalesceable if it has no siblings and is not a leaf""" 

647 return not self.scanError and \ 

648 self.parent and \ 

649 len(self.parent.children) == 1 and \ 

650 self.hidden and \ 

651 len(self.children) > 0 

652 

653 def isLeafCoalesceable(self): 

654 """A VDI is leaf-coalesceable if it has no siblings and is a leaf""" 

655 return not self.scanError and \ 

656 self.parent and \ 

657 len(self.parent.children) == 1 and \ 

658 not self.hidden and \ 

659 len(self.children) == 0 

660 

661 def canLiveCoalesce(self, speed): 

662 """Can we stop-and-leaf-coalesce this VDI? The VDI must be 

663 isLeafCoalesceable() already""" 

664 feasibleSize = False 

665 allowedDownTime = \ 

666 self.TIMEOUT_SAFETY_MARGIN * self.LIVE_LEAF_COALESCE_TIMEOUT 

667 vhd_size = self.getAllocatedSize() 

668 if speed: 

669 feasibleSize = \ 

670 vhd_size // speed < allowedDownTime 

671 else: 

672 feasibleSize = \ 

673 vhd_size < self.LIVE_LEAF_COALESCE_MAX_SIZE 

674 

675 return (feasibleSize or 

676 self.getConfig(self.DB_LEAFCLSC) == self.LEAFCLSC_FORCE) 

677 

678 def getAllPrunable(self): 

679 if len(self.children) == 0: # base case 

680 # it is possible to have a hidden leaf that was recently coalesced 

681 # onto its parent, its children already relinked but not yet 

682 # reloaded - in which case it may not be garbage collected yet: 

683 # some tapdisks could still be using the file. 

684 if self.sr.journaler.get(self.JRN_RELINK, self.uuid): 

685 return [] 

686 if not self.scanError and self.hidden: 

687 return [self] 

688 return [] 

689 

690 thisPrunable = True 

691 vdiList = [] 

692 for child in self.children: 

693 childList = child.getAllPrunable() 

694 vdiList.extend(childList) 

695 if child not in childList: 

696 thisPrunable = False 

697 

698 # We can destroy the current VDI if all childs are hidden BUT the 

699 # current VDI must be hidden too to do that! 

700 # Example in this case (after a failed live leaf coalesce): 

701 # 

702 # SMGC: [32436] SR 07ed ('linstor-nvme-sr') (2 VDIs in 1 VHD trees): 

703 # SMGC: [32436] b5458d61(1.000G/4.127M) 

704 # SMGC: [32436] *OLD_b545(1.000G/4.129M) 

705 # 

706 # OLD_b545 is hidden and must be removed, but b5458d61 not. 

707 # Normally we are not in this function when the delete action is 

708 # executed but in `_liveLeafCoalesce`. 

709 

710 if not self.scanError and not self.hidden and thisPrunable: 

711 vdiList.append(self) 

712 return vdiList 

713 

714 def getSizeVHD(self) -> int: 

715 return self._sizeVHD 

716 

717 def getAllocatedSize(self) -> int: 

718 return self._sizeAllocated 

719 

720 def getTreeRoot(self): 

721 "Get the root of the tree that self belongs to" 

722 root = self 

723 while root.parent: 

724 root = root.parent 

725 return root 

726 

727 def getTreeHeight(self): 

728 "Get the height of the subtree rooted at self" 

729 if len(self.children) == 0: 

730 return 1 

731 

732 maxChildHeight = 0 

733 for child in self.children: 

734 childHeight = child.getTreeHeight() 

735 if childHeight > maxChildHeight: 

736 maxChildHeight = childHeight 

737 

738 return maxChildHeight + 1 

739 

740 def getAllLeaves(self): 

741 "Get all leaf nodes in the subtree rooted at self" 

742 if len(self.children) == 0: 

743 return [self] 

744 

745 leaves = [] 

746 for child in self.children: 

747 leaves.extend(child.getAllLeaves()) 

748 return leaves 

749 

750 def updateBlockInfo(self) -> Optional[str]: 

751 val = base64.b64encode(self._queryVHDBlocks()).decode() 

752 self.setConfig(VDI.DB_VHD_BLOCKS, val) 

753 return val 

754 

755 def rename(self, uuid) -> None: 

756 "Rename the VDI file" 

757 assert(not self.sr.vdis.get(uuid)) 

758 self._clearRef() 

759 oldUuid = self.uuid 

760 self.uuid = uuid 

761 self.children = [] 

762 # updating the children themselves is the responsibility of the caller 

763 del self.sr.vdis[oldUuid] 

764 self.sr.vdis[self.uuid] = self 

765 

766 def delete(self) -> None: 

767 "Physically delete the VDI" 

768 lock.Lock.cleanup(self.uuid, lvhdutil.NS_PREFIX_LVM + self.sr.uuid) 

769 lock.Lock.cleanupAll(self.uuid) 

770 self._clear() 

771 

772 def getParent(self) -> str: 

773 return vhdutil.getParent(self.path, lambda x: x.strip()) 773 ↛ exitline 773 didn't run the lambda on line 773

774 

775 def repair(self, parent) -> None: 

776 vhdutil.repair(parent) 

777 

778 @override 

779 def __str__(self) -> str: 

780 strHidden = "" 

781 if self.hidden: 781 ↛ 782line 781 didn't jump to line 782, because the condition on line 781 was never true

782 strHidden = "*" 

783 strSizeVirt = "?" 

784 if self.sizeVirt > 0: 784 ↛ 785line 784 didn't jump to line 785, because the condition on line 784 was never true

785 strSizeVirt = Util.num2str(self.sizeVirt) 

786 strSizeVHD = "?" 

787 if self._sizeVHD > 0: 787 ↛ 788line 787 didn't jump to line 788, because the condition on line 787 was never true

788 strSizeVHD = "/%s" % Util.num2str(self._sizeVHD) 

789 strSizeAllocated = "?" 

790 if self._sizeAllocated >= 0: 

791 strSizeAllocated = "/%s" % Util.num2str(self._sizeAllocated) 

792 strType = "" 

793 if self.raw: 

794 strType = "[RAW]" 

795 strSizeVHD = "" 

796 

797 return "%s%s(%s%s%s)%s" % (strHidden, self.uuid[0:8], strSizeVirt, 

798 strSizeVHD, strSizeAllocated, strType) 

799 

800 def validate(self, fast=False) -> None: 

801 if not vhdutil.check(self.path, fast=fast): 801 ↛ 802line 801 didn't jump to line 802, because the condition on line 801 was never true

802 raise util.SMException("VHD %s corrupted" % self) 

803 

804 def _clear(self): 

805 self.uuid = "" 

806 self.path = "" 

807 self.parentUuid = "" 

808 self.parent = None 

809 self._clearRef() 

810 

811 def _clearRef(self): 

812 self._vdiRef = None 

813 

814 def _doCoalesce(self) -> None: 

815 """Coalesce self onto parent. Only perform the actual coalescing of 

816 VHD, but not the subsequent relinking. We'll do that as the next step, 

817 after reloading the entire SR in case things have changed while we 

818 were coalescing""" 

819 self.validate() 

820 self.parent.validate(True) 

821 self.parent._increaseSizeVirt(self.sizeVirt) 

822 self.sr._updateSlavesOnResize(self.parent) 

823 self._coalesceVHD(0) 

824 self.parent.validate(True) 

825 #self._verifyContents(0) 

826 self.parent.updateBlockInfo() 

827 

828 def _verifyContents(self, timeOut): 

829 Util.log(" Coalesce verification on %s" % self) 

830 abortTest = lambda: IPCFlag(self.sr.uuid).test(FLAG_TYPE_ABORT) 

831 Util.runAbortable(lambda: self._runTapdiskDiff(), True, 

832 self.sr.uuid, abortTest, VDI.POLL_INTERVAL, timeOut) 

833 Util.log(" Coalesce verification succeeded") 

834 

835 def _runTapdiskDiff(self): 

836 cmd = "tapdisk-diff -n %s:%s -m %s:%s" % \ 

837 (self.getDriverName(), self.path, \ 

838 self.parent.getDriverName(), self.parent.path) 

839 Util.doexec(cmd, 0) 

840 return True 

841 

842 @staticmethod 

843 def _reportCoalesceError(vdi, ce): 

844 """Reports a coalesce error to XenCenter. 

845 

846 vdi: the VDI object on which the coalesce error occured 

847 ce: the CommandException that was raised""" 

848 

849 msg_name = os.strerror(ce.code) 

850 if ce.code == errno.ENOSPC: 

851 # TODO We could add more information here, e.g. exactly how much 

852 # space is required for the particular coalesce, as well as actions 

853 # to be taken by the user and consequences of not taking these 

854 # actions. 

855 msg_body = 'Run out of space while coalescing.' 

856 elif ce.code == errno.EIO: 

857 msg_body = 'I/O error while coalescing.' 

858 else: 

859 msg_body = '' 

860 util.SMlog('Coalesce failed on SR %s: %s (%s)' 

861 % (vdi.sr.uuid, msg_name, msg_body)) 

862 

863 # Create a XenCenter message, but don't spam. 

864 xapi = vdi.sr.xapi.session.xenapi 

865 sr_ref = xapi.SR.get_by_uuid(vdi.sr.uuid) 

866 oth_cfg = xapi.SR.get_other_config(sr_ref) 

867 if COALESCE_ERR_RATE_TAG in oth_cfg: 

868 coalesce_err_rate = float(oth_cfg[COALESCE_ERR_RATE_TAG]) 

869 else: 

870 coalesce_err_rate = DEFAULT_COALESCE_ERR_RATE 

871 

872 xcmsg = False 

873 if coalesce_err_rate == 0: 

874 xcmsg = True 

875 elif coalesce_err_rate > 0: 

876 now = datetime.datetime.now() 

877 sm_cfg = xapi.SR.get_sm_config(sr_ref) 

878 if COALESCE_LAST_ERR_TAG in sm_cfg: 

879 # seconds per message (minimum distance in time between two 

880 # messages in seconds) 

881 spm = datetime.timedelta(seconds=(1.0 / coalesce_err_rate) * 60) 

882 last = datetime.datetime.fromtimestamp( 

883 float(sm_cfg[COALESCE_LAST_ERR_TAG])) 

884 if now - last >= spm: 

885 xapi.SR.remove_from_sm_config(sr_ref, 

886 COALESCE_LAST_ERR_TAG) 

887 xcmsg = True 

888 else: 

889 xcmsg = True 

890 if xcmsg: 

891 xapi.SR.add_to_sm_config(sr_ref, COALESCE_LAST_ERR_TAG, 

892 str(now.strftime('%s'))) 

893 if xcmsg: 

894 xapi.message.create(msg_name, "3", "SR", vdi.sr.uuid, msg_body) 

895 

896 def coalesce(self) -> int: 

897 # size is returned in sectors 

898 return vhdutil.coalesce(self.path) * 512 

899 

900 @staticmethod 

901 def _doCoalesceVHD(vdi): 

902 try: 

903 startTime = time.time() 

904 vhdSize = vdi.getAllocatedSize() 

905 coalesced_size = vdi.coalesce() 

906 endTime = time.time() 

907 vdi.sr.recordStorageSpeed(startTime, endTime, coalesced_size) 

908 except util.CommandException as ce: 

909 # We use try/except for the following piece of code because it runs 

910 # in a separate process context and errors will not be caught and 

911 # reported by anyone. 

912 try: 

913 # Report coalesce errors back to user via XC 

914 VDI._reportCoalesceError(vdi, ce) 

915 except Exception as e: 

916 util.SMlog('failed to create XenCenter message: %s' % e) 

917 raise ce 

918 except: 

919 raise 

920 

921 def _vdi_is_raw(self, vdi_path): 

922 """ 

923 Given path to vdi determine if it is raw 

924 """ 

925 uuid = self.extractUuid(vdi_path) 

926 return self.sr.vdis[uuid].raw 

927 

928 def _coalesceVHD(self, timeOut): 

929 Util.log(" Running VHD coalesce on %s" % self) 

930 abortTest = lambda: IPCFlag(self.sr.uuid).test(FLAG_TYPE_ABORT) 930 ↛ exitline 930 didn't run the lambda on line 930

931 try: 

932 util.fistpoint.activate_custom_fn( 

933 "cleanup_coalesceVHD_inject_failure", 

934 util.inject_failure) 

935 Util.runAbortable(lambda: VDI._doCoalesceVHD(self), None, 

936 self.sr.uuid, abortTest, VDI.POLL_INTERVAL, timeOut) 

937 except: 

938 #exception at this phase could indicate a failure in vhd coalesce 

939 # or a kill of vhd coalesce by runAbortable due to timeOut 

940 # Try a repair and reraise the exception 

941 parent = "" 

942 try: 

943 parent = self.getParent() 

944 if not self._vdi_is_raw(parent): 

945 # Repair error is logged and ignored. Error reraised later 

946 util.SMlog('Coalesce failed on %s, attempting repair on ' \ 

947 'parent %s' % (self.uuid, parent)) 

948 self.repair(parent) 

949 except Exception as e: 

950 util.SMlog('(error ignored) Failed to repair parent %s ' \ 

951 'after failed coalesce on %s, err: %s' % 

952 (parent, self.path, e)) 

953 raise 

954 

955 util.fistpoint.activate("LVHDRT_coalescing_VHD_data", self.sr.uuid) 

956 

957 def _relinkSkip(self) -> None: 

958 """Relink children of this VDI to point to the parent of this VDI""" 

959 abortFlag = IPCFlag(self.sr.uuid) 

960 for child in self.children: 

961 if abortFlag.test(FLAG_TYPE_ABORT): 961 ↛ 962line 961 didn't jump to line 962, because the condition on line 961 was never true

962 raise AbortException("Aborting due to signal") 

963 Util.log(" Relinking %s from %s to %s" % \ 

964 (child, self, self.parent)) 

965 util.fistpoint.activate("LVHDRT_relinking_grandchildren", self.sr.uuid) 

966 child._setParent(self.parent) 

967 self.children = [] 

968 

969 def _reloadChildren(self, vdiSkip): 

970 """Pause & unpause all VDIs in the subtree to cause blktap to reload 

971 the VHD metadata for this file in any online VDI""" 

972 abortFlag = IPCFlag(self.sr.uuid) 

973 for child in self.children: 

974 if child == vdiSkip: 

975 continue 

976 if abortFlag.test(FLAG_TYPE_ABORT): 976 ↛ 977line 976 didn't jump to line 977, because the condition on line 976 was never true

977 raise AbortException("Aborting due to signal") 

978 Util.log(" Reloading VDI %s" % child) 

979 child._reload() 

980 

981 def _reload(self): 

982 """Pause & unpause to cause blktap to reload the VHD metadata""" 

983 for child in self.children: 983 ↛ 984line 983 didn't jump to line 984, because the loop on line 983 never started

984 child._reload() 

985 

986 # only leaves can be attached 

987 if len(self.children) == 0: 987 ↛ exitline 987 didn't return from function '_reload', because the condition on line 987 was never false

988 try: 

989 self.delConfig(VDI.DB_VDI_RELINKING) 

990 except XenAPI.Failure as e: 

991 if not util.isInvalidVDI(e): 

992 raise 

993 self.refresh() 

994 

995 def _tagChildrenForRelink(self): 

996 if len(self.children) == 0: 

997 retries = 0 

998 try: 

999 while retries < 15: 

1000 retries += 1 

1001 if self.getConfig(VDI.DB_VDI_ACTIVATING) is not None: 

1002 Util.log("VDI %s is activating, wait to relink" % 

1003 self.uuid) 

1004 else: 

1005 self.setConfig(VDI.DB_VDI_RELINKING, "True") 

1006 

1007 if self.getConfig(VDI.DB_VDI_ACTIVATING): 

1008 self.delConfig(VDI.DB_VDI_RELINKING) 

1009 Util.log("VDI %s started activating while tagging" % 

1010 self.uuid) 

1011 else: 

1012 return 

1013 time.sleep(2) 

1014 

1015 raise util.SMException("Failed to tag vdi %s for relink" % self) 

1016 except XenAPI.Failure as e: 

1017 if not util.isInvalidVDI(e): 

1018 raise 

1019 

1020 for child in self.children: 

1021 child._tagChildrenForRelink() 

1022 

1023 def _loadInfoParent(self): 

1024 ret = vhdutil.getParent(self.path, lvhdutil.extractUuid) 

1025 if ret: 

1026 self.parentUuid = ret 

1027 

1028 def _setParent(self, parent) -> None: 

1029 vhdutil.setParent(self.path, parent.path, False) 

1030 self.parent = parent 

1031 self.parentUuid = parent.uuid 

1032 parent.children.append(self) 

1033 try: 

1034 self.setConfig(self.DB_VHD_PARENT, self.parentUuid) 

1035 Util.log("Updated the vhd-parent field for child %s with %s" % \ 

1036 (self.uuid, self.parentUuid)) 

1037 except: 

1038 Util.log("Failed to update %s with vhd-parent field %s" % \ 

1039 (self.uuid, self.parentUuid)) 

1040 

1041 def _loadInfoHidden(self) -> None: 

1042 hidden = vhdutil.getHidden(self.path) 

1043 self.hidden = (hidden != 0) 

1044 

1045 def _setHidden(self, hidden=True) -> None: 

1046 vhdutil.setHidden(self.path, hidden) 

1047 self.hidden = hidden 

1048 

1049 def _increaseSizeVirt(self, size, atomic=True) -> None: 

1050 """ensure the virtual size of 'self' is at least 'size'. Note that 

1051 resizing a VHD must always be offline and atomically: the file must 

1052 not be open by anyone and no concurrent operations may take place. 

1053 Thus we use the Agent API call for performing paused atomic 

1054 operations. If the caller is already in the atomic context, it must 

1055 call with atomic = False""" 

1056 if self.sizeVirt >= size: 1056 ↛ 1058line 1056 didn't jump to line 1058, because the condition on line 1056 was never false

1057 return 

1058 Util.log(" Expanding VHD virt size for VDI %s: %s -> %s" % \ 

1059 (self, Util.num2str(self.sizeVirt), Util.num2str(size))) 

1060 

1061 msize = vhdutil.getMaxResizeSize(self.path) * 1024 * 1024 

1062 if (size <= msize): 

1063 vhdutil.setSizeVirtFast(self.path, size) 

1064 else: 

1065 if atomic: 

1066 vdiList = self._getAllSubtree() 

1067 self.sr.lock() 

1068 try: 

1069 self.sr.pauseVDIs(vdiList) 

1070 try: 

1071 self._setSizeVirt(size) 

1072 finally: 

1073 self.sr.unpauseVDIs(vdiList) 

1074 finally: 

1075 self.sr.unlock() 

1076 else: 

1077 self._setSizeVirt(size) 

1078 

1079 self.sizeVirt = vhdutil.getSizeVirt(self.path) 

1080 

1081 def _setSizeVirt(self, size) -> None: 

1082 """WARNING: do not call this method directly unless all VDIs in the 

1083 subtree are guaranteed to be unplugged (and remain so for the duration 

1084 of the operation): this operation is only safe for offline VHDs""" 

1085 jFile = os.path.join(self.sr.path, self.uuid) 

1086 vhdutil.setSizeVirt(self.path, size, jFile) 

1087 

1088 def _queryVHDBlocks(self) -> bytes: 

1089 return vhdutil.getBlockBitmap(self.path) 

1090 

1091 def _getCoalescedSizeData(self): 

1092 """Get the data size of the resulting VHD if we coalesce self onto 

1093 parent. We calculate the actual size by using the VHD block allocation 

1094 information (as opposed to just adding up the two VHD sizes to get an 

1095 upper bound)""" 

1096 # make sure we don't use stale BAT info from vdi_rec since the child 

1097 # was writable all this time 

1098 self.delConfig(VDI.DB_VHD_BLOCKS) 

1099 blocksChild = self.getVHDBlocks() 

1100 blocksParent = self.parent.getVHDBlocks() 

1101 numBlocks = Util.countBits(blocksChild, blocksParent) 

1102 Util.log("Num combined blocks = %d" % numBlocks) 

1103 sizeData = numBlocks * vhdutil.VHD_BLOCK_SIZE 

1104 assert(sizeData <= self.sizeVirt) 

1105 return sizeData 

1106 

1107 def _calcExtraSpaceForCoalescing(self) -> int: 

1108 sizeData = self._getCoalescedSizeData() 

1109 sizeCoalesced = sizeData + vhdutil.calcOverheadBitmap(sizeData) + \ 

1110 vhdutil.calcOverheadEmpty(self.sizeVirt) 

1111 Util.log("Coalesced size = %s" % Util.num2str(sizeCoalesced)) 

1112 return sizeCoalesced - self.parent.getSizeVHD() 

1113 

1114 def _calcExtraSpaceForLeafCoalescing(self) -> int: 

1115 """How much extra space in the SR will be required to 

1116 [live-]leaf-coalesce this VDI""" 

1117 # the space requirements are the same as for inline coalesce 

1118 return self._calcExtraSpaceForCoalescing() 

1119 

1120 def _calcExtraSpaceForSnapshotCoalescing(self) -> int: 

1121 """How much extra space in the SR will be required to 

1122 snapshot-coalesce this VDI""" 

1123 return self._calcExtraSpaceForCoalescing() + \ 

1124 vhdutil.calcOverheadEmpty(self.sizeVirt) # extra snap leaf 

1125 

1126 def _getAllSubtree(self): 

1127 """Get self and all VDIs in the subtree of self as a flat list""" 

1128 vdiList = [self] 

1129 for child in self.children: 

1130 vdiList.extend(child._getAllSubtree()) 

1131 return vdiList 

1132 

1133 

1134class FileVDI(VDI): 

1135 """Object representing a VDI in a file-based SR (EXT or NFS)""" 

1136 

1137 @staticmethod 

1138 def extractUuid(path): 

1139 path = os.path.basename(path.strip()) 

1140 if not (path.endswith(vhdutil.FILE_EXTN_VHD) or \ 1140 ↛ 1142line 1140 didn't jump to line 1142, because the condition on line 1140 was never true

1141 path.endswith(vhdutil.FILE_EXTN_RAW)): 

1142 return None 

1143 uuid = path.replace(vhdutil.FILE_EXTN_VHD, "").replace( \ 

1144 vhdutil.FILE_EXTN_RAW, "") 

1145 # TODO: validate UUID format 

1146 return uuid 

1147 

1148 def __init__(self, sr, uuid, raw): 

1149 VDI.__init__(self, sr, uuid, raw) 

1150 if self.raw: 1150 ↛ 1151line 1150 didn't jump to line 1151, because the condition on line 1150 was never true

1151 self.fileName = "%s%s" % (self.uuid, vhdutil.FILE_EXTN_RAW) 

1152 else: 

1153 self.fileName = "%s%s" % (self.uuid, vhdutil.FILE_EXTN_VHD) 

1154 

1155 @override 

1156 def load(self, info=None) -> None: 

1157 if not info: 

1158 if not util.pathexists(self.path): 

1159 raise util.SMException("%s not found" % self.path) 

1160 try: 

1161 info = vhdutil.getVHDInfo(self.path, self.extractUuid) 

1162 except util.SMException: 

1163 Util.log(" [VDI %s: failed to read VHD metadata]" % self.uuid) 

1164 return 

1165 self.parent = None 

1166 self.children = [] 

1167 self.parentUuid = info.parentUuid 

1168 self.sizeVirt = info.sizeVirt 

1169 self._sizeVHD = info.sizePhys 

1170 self._sizeAllocated = info.sizeAllocated 

1171 self.hidden = info.hidden 

1172 self.scanError = False 

1173 self.path = os.path.join(self.sr.path, "%s%s" % \ 

1174 (self.uuid, vhdutil.FILE_EXTN_VHD)) 

1175 

1176 @override 

1177 def rename(self, uuid) -> None: 

1178 oldPath = self.path 

1179 VDI.rename(self, uuid) 

1180 self.fileName = "%s%s" % (self.uuid, vhdutil.FILE_EXTN_VHD) 

1181 self.path = os.path.join(self.sr.path, self.fileName) 

1182 assert(not util.pathexists(self.path)) 

1183 Util.log("Renaming %s -> %s" % (oldPath, self.path)) 

1184 os.rename(oldPath, self.path) 

1185 

1186 @override 

1187 def delete(self) -> None: 

1188 if len(self.children) > 0: 1188 ↛ 1189line 1188 didn't jump to line 1189, because the condition on line 1188 was never true

1189 raise util.SMException("VDI %s has children, can't delete" % \ 

1190 self.uuid) 

1191 try: 

1192 self.sr.lock() 

1193 try: 

1194 os.unlink(self.path) 

1195 self.sr.forgetVDI(self.uuid) 

1196 finally: 

1197 self.sr.unlock() 

1198 except OSError: 

1199 raise util.SMException("os.unlink(%s) failed" % self.path) 

1200 VDI.delete(self) 

1201 

1202 @override 

1203 def getAllocatedSize(self) -> int: 

1204 if self._sizeAllocated == -1: 1204 ↛ 1205line 1204 didn't jump to line 1205, because the condition on line 1204 was never true

1205 self._sizeAllocated = vhdutil.getAllocatedSize(self.path) 

1206 return self._sizeAllocated 

1207 

1208 

1209class LVHDVDI(VDI): 

1210 """Object representing a VDI in an LVHD SR""" 

1211 

1212 JRN_ZERO = "zero" # journal entry type for zeroing out end of parent 

1213 DRIVER_NAME_RAW = "aio" 

1214 

1215 @override 

1216 def load(self, info=None) -> None: 

1217 # `info` is always set. `None` default value is only here to match parent method. 

1218 assert info, "No info given to LVHDVDI.load" 

1219 self.parent = None 

1220 self.children = [] 

1221 self._sizeVHD = -1 

1222 self._sizeAllocated = -1 

1223 self.scanError = info.scanError 

1224 self.sizeLV = info.sizeLV 

1225 self.sizeVirt = info.sizeVirt 

1226 self.fileName = info.lvName 

1227 self.lvActive = info.lvActive 

1228 self.lvOpen = info.lvOpen 

1229 self.lvReadonly = info.lvReadonly 

1230 self.hidden = info.hidden 

1231 self.parentUuid = info.parentUuid 

1232 self.path = os.path.join(self.sr.path, self.fileName) 

1233 

1234 @staticmethod 

1235 def extractUuid(path): 

1236 return lvhdutil.extractUuid(path) 

1237 

1238 @override 

1239 def getDriverName(self) -> str: 

1240 if self.raw: 

1241 return self.DRIVER_NAME_RAW 

1242 return self.DRIVER_NAME_VHD 

1243 

1244 def inflate(self, size): 

1245 """inflate the LV containing the VHD to 'size'""" 

1246 if self.raw: 

1247 return 

1248 self._activate() 

1249 self.sr.lock() 

1250 try: 

1251 lvhdutil.inflate(self.sr.journaler, self.sr.uuid, self.uuid, size) 

1252 util.fistpoint.activate("LVHDRT_inflating_the_parent", self.sr.uuid) 

1253 finally: 

1254 self.sr.unlock() 

1255 self.sizeLV = self.sr.lvmCache.getSize(self.fileName) 

1256 self._sizeVHD = -1 

1257 self._sizeAllocated = -1 

1258 

1259 def deflate(self): 

1260 """deflate the LV containing the VHD to minimum""" 

1261 if self.raw: 

1262 return 

1263 self._activate() 

1264 self.sr.lock() 

1265 try: 

1266 lvhdutil.deflate(self.sr.lvmCache, self.fileName, self.getSizeVHD()) 

1267 finally: 

1268 self.sr.unlock() 

1269 self.sizeLV = self.sr.lvmCache.getSize(self.fileName) 

1270 self._sizeVHD = -1 

1271 self._sizeAllocated = -1 

1272 

1273 def inflateFully(self): 

1274 self.inflate(lvhdutil.calcSizeVHDLV(self.sizeVirt)) 

1275 

1276 def inflateParentForCoalesce(self): 

1277 """Inflate the parent only as much as needed for the purposes of 

1278 coalescing""" 

1279 if self.parent.raw: 

1280 return 

1281 inc = self._calcExtraSpaceForCoalescing() 

1282 if inc > 0: 

1283 util.fistpoint.activate("LVHDRT_coalescing_before_inflate_grandparent", self.sr.uuid) 

1284 self.parent.inflate(self.parent.sizeLV + inc) 

1285 

1286 @override 

1287 def updateBlockInfo(self) -> Optional[str]: 

1288 if not self.raw: 

1289 return VDI.updateBlockInfo(self) 

1290 return None 

1291 

1292 @override 

1293 def rename(self, uuid) -> None: 

1294 oldUuid = self.uuid 

1295 oldLVName = self.fileName 

1296 VDI.rename(self, uuid) 

1297 self.fileName = lvhdutil.LV_PREFIX[vhdutil.VDI_TYPE_VHD] + self.uuid 

1298 if self.raw: 

1299 self.fileName = lvhdutil.LV_PREFIX[vhdutil.VDI_TYPE_RAW] + self.uuid 

1300 self.path = os.path.join(self.sr.path, self.fileName) 

1301 assert(not self.sr.lvmCache.checkLV(self.fileName)) 

1302 

1303 self.sr.lvmCache.rename(oldLVName, self.fileName) 

1304 if self.sr.lvActivator.get(oldUuid, False): 

1305 self.sr.lvActivator.replace(oldUuid, self.uuid, self.fileName, False) 

1306 

1307 ns = lvhdutil.NS_PREFIX_LVM + self.sr.uuid 

1308 (cnt, bcnt) = RefCounter.check(oldUuid, ns) 

1309 RefCounter.set(self.uuid, cnt, bcnt, ns) 

1310 RefCounter.reset(oldUuid, ns) 

1311 

1312 @override 

1313 def delete(self) -> None: 

1314 if len(self.children) > 0: 

1315 raise util.SMException("VDI %s has children, can't delete" % \ 

1316 self.uuid) 

1317 self.sr.lock() 

1318 try: 

1319 self.sr.lvmCache.remove(self.fileName) 

1320 self.sr.forgetVDI(self.uuid) 

1321 finally: 

1322 self.sr.unlock() 

1323 RefCounter.reset(self.uuid, lvhdutil.NS_PREFIX_LVM + self.sr.uuid) 

1324 VDI.delete(self) 

1325 

1326 @override 

1327 def getSizeVHD(self) -> int: 

1328 if self._sizeVHD == -1: 

1329 self._loadInfoSizeVHD() 

1330 return self._sizeVHD 

1331 

1332 def _loadInfoSizeVHD(self): 

1333 """Get the physical utilization of the VHD file. We do it individually 

1334 (and not using the VHD batch scanner) as an optimization: this info is 

1335 relatively expensive and we need it only for VDI's involved in 

1336 coalescing.""" 

1337 if self.raw: 

1338 return 

1339 self._activate() 

1340 self._sizeVHD = vhdutil.getSizePhys(self.path) 

1341 if self._sizeVHD <= 0: 

1342 raise util.SMException("phys size of %s = %d" % \ 

1343 (self, self._sizeVHD)) 

1344 

1345 @override 

1346 def getAllocatedSize(self) -> int: 

1347 if self._sizeAllocated == -1: 

1348 self._loadInfoSizeAllocated() 

1349 return self._sizeAllocated 

1350 

1351 def _loadInfoSizeAllocated(self): 

1352 """ 

1353 Get the allocated size of the VHD volume. 

1354 """ 

1355 if self.raw: 

1356 return 

1357 self._activate() 

1358 self._sizeAllocated = vhdutil.getAllocatedSize(self.path) 

1359 

1360 @override 

1361 def _loadInfoHidden(self) -> None: 

1362 if self.raw: 

1363 self.hidden = self.sr.lvmCache.getHidden(self.fileName) 

1364 else: 

1365 VDI._loadInfoHidden(self) 

1366 

1367 @override 

1368 def _setHidden(self, hidden=True) -> None: 

1369 if self.raw: 

1370 self.sr.lvmCache.setHidden(self.fileName, hidden) 

1371 self.hidden = hidden 

1372 else: 

1373 VDI._setHidden(self, hidden) 

1374 

1375 @override 

1376 def __str__(self) -> str: 

1377 strType = "VHD" 

1378 if self.raw: 

1379 strType = "RAW" 

1380 strHidden = "" 

1381 if self.hidden: 

1382 strHidden = "*" 

1383 strSizeVHD = "" 

1384 if self._sizeVHD > 0: 

1385 strSizeVHD = Util.num2str(self._sizeVHD) 

1386 strSizeAllocated = "" 

1387 if self._sizeAllocated >= 0: 

1388 strSizeAllocated = Util.num2str(self._sizeAllocated) 

1389 strActive = "n" 

1390 if self.lvActive: 

1391 strActive = "a" 

1392 if self.lvOpen: 

1393 strActive += "o" 

1394 return "%s%s[%s](%s/%s/%s/%s|%s)" % (strHidden, self.uuid[0:8], strType, 

1395 Util.num2str(self.sizeVirt), strSizeVHD, strSizeAllocated, 

1396 Util.num2str(self.sizeLV), strActive) 

1397 

1398 @override 

1399 def validate(self, fast=False) -> None: 

1400 if not self.raw: 

1401 VDI.validate(self, fast) 

1402 

1403 @override 

1404 def _doCoalesce(self) -> None: 

1405 """LVHD parents must first be activated, inflated, and made writable""" 

1406 try: 

1407 self._activateChain() 

1408 self.sr.lvmCache.setReadonly(self.parent.fileName, False) 

1409 self.parent.validate() 

1410 self.inflateParentForCoalesce() 

1411 VDI._doCoalesce(self) 

1412 finally: 

1413 self.parent._loadInfoSizeVHD() 

1414 self.parent.deflate() 

1415 self.sr.lvmCache.setReadonly(self.parent.fileName, True) 

1416 

1417 @override 

1418 def _setParent(self, parent) -> None: 

1419 self._activate() 

1420 if self.lvReadonly: 

1421 self.sr.lvmCache.setReadonly(self.fileName, False) 

1422 

1423 try: 

1424 vhdutil.setParent(self.path, parent.path, parent.raw) 

1425 finally: 

1426 if self.lvReadonly: 

1427 self.sr.lvmCache.setReadonly(self.fileName, True) 

1428 self._deactivate() 

1429 self.parent = parent 

1430 self.parentUuid = parent.uuid 

1431 parent.children.append(self) 

1432 try: 

1433 self.setConfig(self.DB_VHD_PARENT, self.parentUuid) 

1434 Util.log("Updated the vhd-parent field for child %s with %s" % \ 

1435 (self.uuid, self.parentUuid)) 

1436 except: 

1437 Util.log("Failed to update the vhd-parent with %s for child %s" % \ 

1438 (self.parentUuid, self.uuid)) 

1439 

1440 def _activate(self): 

1441 self.sr.lvActivator.activate(self.uuid, self.fileName, False) 

1442 

1443 def _activateChain(self): 

1444 vdi = self 

1445 while vdi: 

1446 vdi._activate() 

1447 vdi = vdi.parent 

1448 

1449 def _deactivate(self): 

1450 self.sr.lvActivator.deactivate(self.uuid, False) 

1451 

1452 @override 

1453 def _increaseSizeVirt(self, size, atomic=True) -> None: 

1454 "ensure the virtual size of 'self' is at least 'size'" 

1455 self._activate() 

1456 if not self.raw: 

1457 VDI._increaseSizeVirt(self, size, atomic) 

1458 return 

1459 

1460 # raw VDI case 

1461 offset = self.sizeLV 

1462 if self.sizeVirt < size: 

1463 oldSize = self.sizeLV 

1464 self.sizeLV = util.roundup(lvutil.LVM_SIZE_INCREMENT, size) 

1465 Util.log(" Growing %s: %d->%d" % (self.path, oldSize, self.sizeLV)) 

1466 self.sr.lvmCache.setSize(self.fileName, self.sizeLV) 

1467 offset = oldSize 

1468 unfinishedZero = False 

1469 jval = self.sr.journaler.get(self.JRN_ZERO, self.uuid) 

1470 if jval: 

1471 unfinishedZero = True 

1472 offset = int(jval) 

1473 length = self.sizeLV - offset 

1474 if not length: 

1475 return 

1476 

1477 if unfinishedZero: 

1478 Util.log(" ==> Redoing unfinished zeroing out") 

1479 else: 

1480 self.sr.journaler.create(self.JRN_ZERO, self.uuid, \ 

1481 str(offset)) 

1482 Util.log(" Zeroing %s: from %d, %dB" % (self.path, offset, length)) 

1483 abortTest = lambda: IPCFlag(self.sr.uuid).test(FLAG_TYPE_ABORT) 

1484 func = lambda: util.zeroOut(self.path, offset, length) 

1485 Util.runAbortable(func, True, self.sr.uuid, abortTest, 

1486 VDI.POLL_INTERVAL, 0) 

1487 self.sr.journaler.remove(self.JRN_ZERO, self.uuid) 

1488 

1489 @override 

1490 def _setSizeVirt(self, size) -> None: 

1491 """WARNING: do not call this method directly unless all VDIs in the 

1492 subtree are guaranteed to be unplugged (and remain so for the duration 

1493 of the operation): this operation is only safe for offline VHDs""" 

1494 self._activate() 

1495 jFile = lvhdutil.createVHDJournalLV(self.sr.lvmCache, self.uuid, 

1496 vhdutil.MAX_VHD_JOURNAL_SIZE) 

1497 try: 

1498 lvhdutil.setSizeVirt(self.sr.journaler, self.sr.uuid, self.uuid, 

1499 size, jFile) 

1500 finally: 

1501 lvhdutil.deleteVHDJournalLV(self.sr.lvmCache, self.uuid) 

1502 

1503 @override 

1504 def _queryVHDBlocks(self) -> bytes: 

1505 self._activate() 

1506 return VDI._queryVHDBlocks(self) 

1507 

1508 @override 

1509 def _calcExtraSpaceForCoalescing(self) -> int: 

1510 if self.parent.raw: 

1511 return 0 # raw parents are never deflated in the first place 

1512 sizeCoalesced = lvhdutil.calcSizeVHDLV(self._getCoalescedSizeData()) 

1513 Util.log("Coalesced size = %s" % Util.num2str(sizeCoalesced)) 

1514 return sizeCoalesced - self.parent.sizeLV 

1515 

1516 @override 

1517 def _calcExtraSpaceForLeafCoalescing(self) -> int: 

1518 """How much extra space in the SR will be required to 

1519 [live-]leaf-coalesce this VDI""" 

1520 # we can deflate the leaf to minimize the space requirements 

1521 deflateDiff = self.sizeLV - lvhdutil.calcSizeLV(self.getSizeVHD()) 

1522 return self._calcExtraSpaceForCoalescing() - deflateDiff 

1523 

1524 @override 

1525 def _calcExtraSpaceForSnapshotCoalescing(self) -> int: 

1526 return self._calcExtraSpaceForCoalescing() + \ 

1527 lvhdutil.calcSizeLV(self.getSizeVHD()) 

1528 

1529 

1530class LinstorVDI(VDI): 

1531 """Object representing a VDI in a LINSTOR SR""" 

1532 

1533 VOLUME_LOCK_TIMEOUT = 30 

1534 

1535 @override 

1536 def load(self, info=None) -> None: 

1537 self.parentUuid = info.parentUuid 

1538 self.scanError = True 

1539 self.parent = None 

1540 self.children = [] 

1541 

1542 self.fileName = self.sr._linstor.get_volume_name(self.uuid) 

1543 self.path = self.sr._linstor.build_device_path(self.fileName) 

1544 

1545 if not info: 

1546 try: 

1547 info = self.sr._vhdutil.get_vhd_info(self.uuid) 

1548 except util.SMException: 

1549 Util.log( 

1550 ' [VDI {}: failed to read VHD metadata]'.format(self.uuid) 

1551 ) 

1552 return 

1553 

1554 self.parentUuid = info.parentUuid 

1555 self.sizeVirt = info.sizeVirt 

1556 self._sizeVHD = -1 

1557 self._sizeAllocated = -1 

1558 self.drbd_size = -1 

1559 self.hidden = info.hidden 

1560 self.scanError = False 

1561 self.vdi_type = vhdutil.VDI_TYPE_VHD 

1562 

1563 @override 

1564 def getSizeVHD(self, fetch=False) -> int: 

1565 if self._sizeVHD < 0 or fetch: 

1566 self._sizeVHD = self.sr._vhdutil.get_size_phys(self.uuid) 

1567 return self._sizeVHD 

1568 

1569 def getDrbdSize(self, fetch=False): 

1570 if self.drbd_size < 0 or fetch: 

1571 self.drbd_size = self.sr._vhdutil.get_drbd_size(self.uuid) 

1572 return self.drbd_size 

1573 

1574 @override 

1575 def getAllocatedSize(self) -> int: 

1576 if self._sizeAllocated == -1: 

1577 if not self.raw: 

1578 self._sizeAllocated = self.sr._vhdutil.get_allocated_size(self.uuid) 

1579 return self._sizeAllocated 

1580 

1581 def inflate(self, size): 

1582 if self.raw: 

1583 return 

1584 self.sr.lock() 

1585 try: 

1586 # Ensure we use the real DRBD size and not the cached one. 

1587 # Why? Because this attribute can be changed if volume is resized by user. 

1588 self.drbd_size = self.getDrbdSize(fetch=True) 

1589 self.sr._vhdutil.inflate(self.sr.journaler, self.uuid, self.path, size, self.drbd_size) 

1590 finally: 

1591 self.sr.unlock() 

1592 self.drbd_size = -1 

1593 self._sizeVHD = -1 

1594 self._sizeAllocated = -1 

1595 

1596 def deflate(self): 

1597 if self.raw: 

1598 return 

1599 self.sr.lock() 

1600 try: 

1601 # Ensure we use the real sizes and not the cached info. 

1602 self.drbd_size = self.getDrbdSize(fetch=True) 

1603 self._sizeVHD = self.getSizeVHD(fetch=True) 

1604 self.sr._vhdutil.force_deflate(self.path, self._sizeVHD, self.drbd_size, zeroize=False) 

1605 finally: 

1606 self.sr.unlock() 

1607 self.drbd_size = -1 

1608 self._sizeVHD = -1 

1609 self._sizeAllocated = -1 

1610 

1611 def inflateFully(self): 

1612 if not self.raw: 

1613 self.inflate(LinstorVhdUtil.compute_volume_size(self.sizeVirt, self.vdi_type)) 

1614 

1615 @override 

1616 def rename(self, uuid) -> None: 

1617 Util.log('Renaming {} -> {} (path={})'.format( 

1618 self.uuid, uuid, self.path 

1619 )) 

1620 self.sr._linstor.update_volume_uuid(self.uuid, uuid) 

1621 VDI.rename(self, uuid) 

1622 

1623 @override 

1624 def delete(self) -> None: 

1625 if len(self.children) > 0: 

1626 raise util.SMException( 

1627 'VDI {} has children, can\'t delete'.format(self.uuid) 

1628 ) 

1629 self.sr.lock() 

1630 try: 

1631 self.sr._linstor.destroy_volume(self.uuid) 

1632 self.sr.forgetVDI(self.uuid) 

1633 finally: 

1634 self.sr.unlock() 

1635 VDI.delete(self) 

1636 

1637 @override 

1638 def validate(self, fast=False) -> None: 

1639 if not self.raw and not self.sr._vhdutil.check(self.uuid, fast=fast): 

1640 raise util.SMException('VHD {} corrupted'.format(self)) 

1641 

1642 @override 

1643 def pause(self, failfast=False) -> None: 

1644 self.sr._linstor.ensure_volume_is_not_locked( 

1645 self.uuid, timeout=self.VOLUME_LOCK_TIMEOUT 

1646 ) 

1647 return super(LinstorVDI, self).pause(failfast) 

1648 

1649 @override 

1650 def coalesce(self) -> int: 

1651 # Note: We raise `SMException` here to skip the current coalesce in case of failure. 

1652 # Using another exception we can't execute the next coalesce calls. 

1653 return self.sr._vhdutil.force_coalesce(self.path) * 512 

1654 

1655 @override 

1656 def getParent(self) -> str: 

1657 return self.sr._vhdutil.get_parent( 

1658 self.sr._linstor.get_volume_uuid_from_device_path(self.path) 

1659 ) 

1660 

1661 @override 

1662 def repair(self, parent_uuid) -> None: 

1663 self.sr._vhdutil.force_repair( 

1664 self.sr._linstor.get_device_path(parent_uuid) 

1665 ) 

1666 

1667 @override 

1668 def _relinkSkip(self) -> None: 

1669 abortFlag = IPCFlag(self.sr.uuid) 

1670 for child in self.children: 

1671 if abortFlag.test(FLAG_TYPE_ABORT): 

1672 raise AbortException('Aborting due to signal') 

1673 Util.log( 

1674 ' Relinking {} from {} to {}'.format( 

1675 child, self, self.parent 

1676 ) 

1677 ) 

1678 

1679 session = child.sr.xapi.session 

1680 sr_uuid = child.sr.uuid 

1681 vdi_uuid = child.uuid 

1682 try: 

1683 self.sr._linstor.ensure_volume_is_not_locked( 

1684 vdi_uuid, timeout=self.VOLUME_LOCK_TIMEOUT 

1685 ) 

1686 blktap2.VDI.tap_pause(session, sr_uuid, vdi_uuid) 

1687 child._setParent(self.parent) 

1688 finally: 

1689 blktap2.VDI.tap_unpause(session, sr_uuid, vdi_uuid) 

1690 self.children = [] 

1691 

1692 @override 

1693 def _setParent(self, parent) -> None: 

1694 self.sr._linstor.get_device_path(self.uuid) 

1695 self.sr._vhdutil.force_parent(self.path, parent.path) 

1696 self.parent = parent 

1697 self.parentUuid = parent.uuid 

1698 parent.children.append(self) 

1699 try: 

1700 self.setConfig(self.DB_VHD_PARENT, self.parentUuid) 

1701 Util.log("Updated the vhd-parent field for child %s with %s" % \ 

1702 (self.uuid, self.parentUuid)) 

1703 except: 

1704 Util.log("Failed to update %s with vhd-parent field %s" % \ 

1705 (self.uuid, self.parentUuid)) 

1706 

1707 @override 

1708 def _doCoalesce(self) -> None: 

1709 try: 

1710 self._activateChain() 

1711 self.parent.validate() 

1712 self._inflateParentForCoalesce() 

1713 VDI._doCoalesce(self) 

1714 finally: 

1715 self.parent.deflate() 

1716 

1717 def _activateChain(self): 

1718 vdi = self 

1719 while vdi: 

1720 try: 

1721 p = self.sr._linstor.get_device_path(vdi.uuid) 

1722 except Exception as e: 

1723 # Use SMException to skip coalesce. 

1724 # Otherwise the GC is stopped... 

1725 raise util.SMException(str(e)) 

1726 vdi = vdi.parent 

1727 

1728 @override 

1729 def _setHidden(self, hidden=True) -> None: 

1730 HIDDEN_TAG = 'hidden' 

1731 

1732 if self.raw: 

1733 self.sr._linstor.update_volume_metadata(self.uuid, { 

1734 HIDDEN_TAG: hidden 

1735 }) 

1736 self.hidden = hidden 

1737 else: 

1738 VDI._setHidden(self, hidden) 

1739 

1740 @override 

1741 def _setSizeVirt(self, size) -> None: 

1742 jfile = self.uuid + '-jvhd' 

1743 self.sr._linstor.create_volume( 

1744 jfile, vhdutil.MAX_VHD_JOURNAL_SIZE, persistent=False, volume_name=jfile 

1745 ) 

1746 try: 

1747 self.inflate(LinstorVhdUtil.compute_volume_size(size, self.vdi_type)) 

1748 self.sr._vhdutil.set_size_virt(size, jfile) 

1749 finally: 

1750 try: 

1751 self.sr._linstor.destroy_volume(jfile) 

1752 except Exception: 

1753 # We can ignore it, in any case this volume is not persistent. 

1754 pass 

1755 

1756 @override 

1757 def _queryVHDBlocks(self) -> bytes: 

1758 return self.sr._vhdutil.get_block_bitmap(self.uuid) 

1759 

1760 def _inflateParentForCoalesce(self): 

1761 if self.parent.raw: 

1762 return 

1763 inc = self._calcExtraSpaceForCoalescing() 

1764 if inc > 0: 

1765 self.parent.inflate(self.parent.getDrbdSize() + inc) 

1766 

1767 @override 

1768 def _calcExtraSpaceForCoalescing(self) -> int: 

1769 if self.parent.raw: 

1770 return 0 

1771 size_coalesced = LinstorVhdUtil.compute_volume_size( 

1772 self._getCoalescedSizeData(), self.vdi_type 

1773 ) 

1774 Util.log("Coalesced size = %s" % Util.num2str(size_coalesced)) 

1775 return size_coalesced - self.parent.getDrbdSize() 

1776 

1777 @override 

1778 def _calcExtraSpaceForLeafCoalescing(self) -> int: 

1779 assert self.getDrbdSize() > 0 

1780 assert self.getSizeVHD() > 0 

1781 deflate_diff = self.getDrbdSize() - LinstorVolumeManager.round_up_volume_size(self.getSizeVHD()) 

1782 assert deflate_diff >= 0 

1783 return self._calcExtraSpaceForCoalescing() - deflate_diff 

1784 

1785 @override 

1786 def _calcExtraSpaceForSnapshotCoalescing(self) -> int: 

1787 assert self.getSizeVHD() > 0 

1788 return self._calcExtraSpaceForCoalescing() + \ 

1789 LinstorVolumeManager.round_up_volume_size(self.getSizeVHD()) 

1790 

1791################################################################################ 

1792# 

1793# SR 

1794# 

1795class SR(object): 

1796 class LogFilter: 

1797 def __init__(self, sr): 

1798 self.sr = sr 

1799 self.stateLogged = False 

1800 self.prevState = {} 

1801 self.currState = {} 

1802 

1803 def logState(self): 

1804 changes = "" 

1805 self.currState.clear() 

1806 for vdi in self.sr.vdiTrees: 

1807 self.currState[vdi.uuid] = self._getTreeStr(vdi) 

1808 if not self.prevState.get(vdi.uuid) or \ 

1809 self.prevState[vdi.uuid] != self.currState[vdi.uuid]: 

1810 changes += self.currState[vdi.uuid] 

1811 

1812 for uuid in self.prevState: 

1813 if not self.currState.get(uuid): 

1814 changes += "Tree %s gone\n" % uuid 

1815 

1816 result = "SR %s (%d VDIs in %d VHD trees): " % \ 

1817 (self.sr, len(self.sr.vdis), len(self.sr.vdiTrees)) 

1818 

1819 if len(changes) > 0: 

1820 if self.stateLogged: 

1821 result += "showing only VHD trees that changed:" 

1822 result += "\n%s" % changes 

1823 else: 

1824 result += "no changes" 

1825 

1826 for line in result.split("\n"): 

1827 Util.log("%s" % line) 

1828 self.prevState.clear() 

1829 for key, val in self.currState.items(): 

1830 self.prevState[key] = val 

1831 self.stateLogged = True 

1832 

1833 def logNewVDI(self, uuid): 

1834 if self.stateLogged: 

1835 Util.log("Found new VDI when scanning: %s" % uuid) 

1836 

1837 def _getTreeStr(self, vdi, indent=8): 

1838 treeStr = "%s%s\n" % (" " * indent, vdi) 

1839 for child in vdi.children: 

1840 treeStr += self._getTreeStr(child, indent + VDI.STR_TREE_INDENT) 

1841 return treeStr 

1842 

1843 TYPE_FILE = "file" 

1844 TYPE_LVHD = "lvhd" 

1845 TYPE_LINSTOR = "linstor" 

1846 TYPES = [TYPE_LVHD, TYPE_FILE, TYPE_LINSTOR] 

1847 

1848 LOCK_RETRY_INTERVAL = 3 

1849 LOCK_RETRY_ATTEMPTS = 20 

1850 LOCK_RETRY_ATTEMPTS_LOCK = 100 

1851 

1852 SCAN_RETRY_ATTEMPTS = 3 

1853 

1854 JRN_CLONE = "clone" # journal entry type for the clone operation (from SM) 

1855 TMP_RENAME_PREFIX = "OLD_" 

1856 

1857 KEY_OFFLINE_COALESCE_NEEDED = "leaf_coalesce_need_offline" 

1858 KEY_OFFLINE_COALESCE_OVERRIDE = "leaf_coalesce_offline_override" 

1859 

1860 @staticmethod 

1861 def getInstance(uuid, xapiSession, createLock=True, force=False): 

1862 xapi = XAPI(xapiSession, uuid) 

1863 type = normalizeType(xapi.srRecord["type"]) 

1864 if type == SR.TYPE_FILE: 

1865 return FileSR(uuid, xapi, createLock, force) 

1866 elif type == SR.TYPE_LVHD: 

1867 return LVHDSR(uuid, xapi, createLock, force) 

1868 elif type == SR.TYPE_LINSTOR: 

1869 return LinstorSR(uuid, xapi, createLock, force) 

1870 raise util.SMException("SR type %s not recognized" % type) 

1871 

1872 def __init__(self, uuid, xapi, createLock, force): 

1873 self.logFilter = self.LogFilter(self) 

1874 self.uuid = uuid 

1875 self.path = "" 

1876 self.name = "" 

1877 self.vdis = {} 

1878 self.vdiTrees = [] 

1879 self.journaler = None 

1880 self.xapi = xapi 

1881 self._locked = 0 

1882 self._srLock = None 

1883 if createLock: 1883 ↛ 1884line 1883 didn't jump to line 1884, because the condition on line 1883 was never true

1884 self._srLock = lock.Lock(vhdutil.LOCK_TYPE_SR, self.uuid) 

1885 else: 

1886 Util.log("Requested no SR locking") 

1887 self.name = self.xapi.srRecord["name_label"] 

1888 self._failedCoalesceTargets = [] 

1889 

1890 if not self.xapi.isPluggedHere(): 

1891 if force: 1891 ↛ 1892line 1891 didn't jump to line 1892, because the condition on line 1891 was never true

1892 Util.log("SR %s not attached on this host, ignoring" % uuid) 

1893 else: 

1894 if not self.wait_for_plug(): 

1895 raise util.SMException("SR %s not attached on this host" % uuid) 

1896 

1897 if force: 1897 ↛ 1898line 1897 didn't jump to line 1898, because the condition on line 1897 was never true

1898 Util.log("Not checking if we are Master (SR %s)" % uuid) 

1899 elif not self.xapi.isMaster(): 1899 ↛ 1900line 1899 didn't jump to line 1900, because the condition on line 1899 was never true

1900 raise util.SMException("This host is NOT master, will not run") 

1901 

1902 def wait_for_plug(self): 

1903 for _ in range(1, 10): 

1904 time.sleep(2) 

1905 if self.xapi.isPluggedHere(): 

1906 return True 

1907 return False 

1908 

1909 def gcEnabled(self, refresh=True): 

1910 if refresh: 

1911 self.xapi.srRecord = \ 

1912 self.xapi.session.xenapi.SR.get_record(self.xapi._srRef) 

1913 if self.xapi.srRecord["other_config"].get(VDI.DB_GC) == "false": 

1914 Util.log("GC is disabled for this SR, abort") 

1915 return False 

1916 return True 

1917 

1918 def scan(self, force=False) -> None: 

1919 """Scan the SR and load VDI info for each VDI. If called repeatedly, 

1920 update VDI objects if they already exist""" 

1921 pass 

1922 

1923 def scanLocked(self, force=False): 

1924 self.lock() 

1925 try: 

1926 self.scan(force) 

1927 finally: 

1928 self.unlock() 

1929 

1930 def getVDI(self, uuid): 

1931 return self.vdis.get(uuid) 

1932 

1933 def hasWork(self): 

1934 if len(self.findGarbage()) > 0: 

1935 return True 

1936 if self.findCoalesceable(): 

1937 return True 

1938 if self.findLeafCoalesceable(): 

1939 return True 

1940 if self.needUpdateBlockInfo(): 

1941 return True 

1942 return False 

1943 

1944 def findCoalesceable(self): 

1945 """Find a coalesceable VDI. Return a vdi that should be coalesced 

1946 (choosing one among all coalesceable candidates according to some 

1947 criteria) or None if there is no VDI that could be coalesced""" 

1948 

1949 candidates = [] 

1950 

1951 srSwitch = self.xapi.srRecord["other_config"].get(VDI.DB_COALESCE) 

1952 if srSwitch == "false": 

1953 Util.log("Coalesce disabled for this SR") 

1954 return candidates 

1955 

1956 # finish any VDI for which a relink journal entry exists first 

1957 journals = self.journaler.getAll(VDI.JRN_RELINK) 

1958 for uuid in journals: 

1959 vdi = self.getVDI(uuid) 

1960 if vdi and vdi not in self._failedCoalesceTargets: 

1961 return vdi 

1962 

1963 for vdi in self.vdis.values(): 

1964 if vdi.isCoalesceable() and vdi not in self._failedCoalesceTargets: 

1965 candidates.append(vdi) 

1966 Util.log("%s is coalescable" % vdi.uuid) 

1967 

1968 self.xapi.update_task_progress("coalescable", len(candidates)) 

1969 

1970 # pick one in the tallest tree 

1971 treeHeight = dict() 

1972 for c in candidates: 

1973 height = c.getTreeRoot().getTreeHeight() 

1974 if treeHeight.get(height): 

1975 treeHeight[height].append(c) 

1976 else: 

1977 treeHeight[height] = [c] 

1978 

1979 freeSpace = self.getFreeSpace() 

1980 heights = list(treeHeight.keys()) 

1981 heights.sort(reverse=True) 

1982 for h in heights: 

1983 for c in treeHeight[h]: 

1984 spaceNeeded = c._calcExtraSpaceForCoalescing() 

1985 if spaceNeeded <= freeSpace: 

1986 Util.log("Coalesce candidate: %s (tree height %d)" % (c, h)) 

1987 return c 

1988 else: 

1989 Util.log("No space to coalesce %s (free space: %d)" % \ 

1990 (c, freeSpace)) 

1991 return None 

1992 

1993 def getSwitch(self, key): 

1994 return self.xapi.srRecord["other_config"].get(key) 

1995 

1996 def forbiddenBySwitch(self, switch, condition, fail_msg): 

1997 srSwitch = self.getSwitch(switch) 

1998 ret = False 

1999 if srSwitch: 

2000 ret = srSwitch == condition 

2001 

2002 if ret: 

2003 Util.log(fail_msg) 

2004 

2005 return ret 

2006 

2007 def leafCoalesceForbidden(self): 

2008 return (self.forbiddenBySwitch(VDI.DB_COALESCE, 

2009 "false", 

2010 "Coalesce disabled for this SR") or 

2011 self.forbiddenBySwitch(VDI.DB_LEAFCLSC, 

2012 VDI.LEAFCLSC_DISABLED, 

2013 "Leaf-coalesce disabled for this SR")) 

2014 

2015 def findLeafCoalesceable(self): 

2016 """Find leaf-coalesceable VDIs in each VHD tree""" 

2017 

2018 candidates = [] 

2019 if self.leafCoalesceForbidden(): 

2020 return candidates 

2021 

2022 self.gatherLeafCoalesceable(candidates) 

2023 

2024 self.xapi.update_task_progress("coalescable", len(candidates)) 

2025 

2026 freeSpace = self.getFreeSpace() 

2027 for candidate in candidates: 

2028 # check the space constraints to see if leaf-coalesce is actually 

2029 # feasible for this candidate 

2030 spaceNeeded = candidate._calcExtraSpaceForSnapshotCoalescing() 

2031 spaceNeededLive = spaceNeeded 

2032 if spaceNeeded > freeSpace: 

2033 spaceNeededLive = candidate._calcExtraSpaceForLeafCoalescing() 

2034 if candidate.canLiveCoalesce(self.getStorageSpeed()): 

2035 spaceNeeded = spaceNeededLive 

2036 

2037 if spaceNeeded <= freeSpace: 

2038 Util.log("Leaf-coalesce candidate: %s" % candidate) 

2039 return candidate 

2040 else: 

2041 Util.log("No space to leaf-coalesce %s (free space: %d)" % \ 

2042 (candidate, freeSpace)) 

2043 if spaceNeededLive <= freeSpace: 

2044 Util.log("...but enough space if skip snap-coalesce") 

2045 candidate.setConfig(VDI.DB_LEAFCLSC, 

2046 VDI.LEAFCLSC_OFFLINE) 

2047 

2048 return None 

2049 

2050 def gatherLeafCoalesceable(self, candidates): 

2051 for vdi in self.vdis.values(): 

2052 if not vdi.isLeafCoalesceable(): 

2053 continue 

2054 if vdi in self._failedCoalesceTargets: 

2055 continue 

2056 if vdi.getConfig(vdi.DB_ONBOOT) == vdi.ONBOOT_RESET: 

2057 Util.log("Skipping reset-on-boot %s" % vdi) 

2058 continue 

2059 if vdi.getConfig(vdi.DB_ALLOW_CACHING): 

2060 Util.log("Skipping allow_caching=true %s" % vdi) 

2061 continue 

2062 if vdi.getConfig(vdi.DB_LEAFCLSC) == vdi.LEAFCLSC_DISABLED: 

2063 Util.log("Leaf-coalesce disabled for %s" % vdi) 

2064 continue 

2065 if not (AUTO_ONLINE_LEAF_COALESCE_ENABLED or 

2066 vdi.getConfig(vdi.DB_LEAFCLSC) == vdi.LEAFCLSC_FORCE): 

2067 continue 

2068 candidates.append(vdi) 

2069 

2070 def coalesce(self, vdi, dryRun=False): 

2071 """Coalesce vdi onto parent""" 

2072 Util.log("Coalescing %s -> %s" % (vdi, vdi.parent)) 

2073 if dryRun: 2073 ↛ 2074line 2073 didn't jump to line 2074, because the condition on line 2073 was never true

2074 return 

2075 

2076 try: 

2077 self._coalesce(vdi) 

2078 except util.SMException as e: 

2079 if isinstance(e, AbortException): 2079 ↛ 2080line 2079 didn't jump to line 2080, because the condition on line 2079 was never true

2080 self.cleanup() 

2081 raise 

2082 else: 

2083 self._failedCoalesceTargets.append(vdi) 

2084 Util.logException("coalesce") 

2085 Util.log("Coalesce failed, skipping") 

2086 self.cleanup() 

2087 

2088 def coalesceLeaf(self, vdi, dryRun=False): 

2089 """Leaf-coalesce vdi onto parent""" 

2090 Util.log("Leaf-coalescing %s -> %s" % (vdi, vdi.parent)) 

2091 if dryRun: 

2092 return 

2093 

2094 try: 

2095 uuid = vdi.uuid 

2096 try: 

2097 # "vdi" object will no longer be valid after this call 

2098 self._coalesceLeaf(vdi) 

2099 finally: 

2100 vdi = self.getVDI(uuid) 

2101 if vdi: 

2102 vdi.delConfig(vdi.DB_LEAFCLSC) 

2103 except AbortException: 

2104 self.cleanup() 

2105 raise 

2106 except (util.SMException, XenAPI.Failure) as e: 

2107 self._failedCoalesceTargets.append(vdi) 

2108 Util.logException("leaf-coalesce") 

2109 Util.log("Leaf-coalesce failed on %s, skipping" % vdi) 

2110 self.cleanup() 

2111 

2112 def garbageCollect(self, dryRun=False): 

2113 vdiList = self.findGarbage() 

2114 Util.log("Found %d VDIs for deletion:" % len(vdiList)) 

2115 for vdi in vdiList: 

2116 Util.log(" %s" % vdi) 

2117 if not dryRun: 

2118 self.deleteVDIs(vdiList) 

2119 self.cleanupJournals(dryRun) 

2120 

2121 def findGarbage(self): 

2122 vdiList = [] 

2123 for vdi in self.vdiTrees: 

2124 vdiList.extend(vdi.getAllPrunable()) 

2125 return vdiList 

2126 

2127 def deleteVDIs(self, vdiList) -> None: 

2128 for vdi in vdiList: 

2129 if IPCFlag(self.uuid).test(FLAG_TYPE_ABORT): 

2130 raise AbortException("Aborting due to signal") 

2131 Util.log("Deleting unlinked VDI %s" % vdi) 

2132 self.deleteVDI(vdi) 

2133 

2134 def deleteVDI(self, vdi) -> None: 

2135 assert(len(vdi.children) == 0) 

2136 del self.vdis[vdi.uuid] 

2137 if vdi.parent: 2137 ↛ 2139line 2137 didn't jump to line 2139, because the condition on line 2137 was never false

2138 vdi.parent.children.remove(vdi) 

2139 if vdi in self.vdiTrees: 2139 ↛ 2140line 2139 didn't jump to line 2140, because the condition on line 2139 was never true

2140 self.vdiTrees.remove(vdi) 

2141 vdi.delete() 

2142 

2143 def forgetVDI(self, vdiUuid) -> None: 

2144 self.xapi.forgetVDI(self.uuid, vdiUuid) 

2145 

2146 def pauseVDIs(self, vdiList) -> None: 

2147 paused = [] 

2148 failed = False 

2149 for vdi in vdiList: 

2150 try: 

2151 vdi.pause() 

2152 paused.append(vdi) 

2153 except: 

2154 Util.logException("pauseVDIs") 

2155 failed = True 

2156 break 

2157 

2158 if failed: 

2159 self.unpauseVDIs(paused) 

2160 raise util.SMException("Failed to pause VDIs") 

2161 

2162 def unpauseVDIs(self, vdiList): 

2163 failed = False 

2164 for vdi in vdiList: 

2165 try: 

2166 vdi.unpause() 

2167 except: 

2168 Util.log("ERROR: Failed to unpause VDI %s" % vdi) 

2169 failed = True 

2170 if failed: 

2171 raise util.SMException("Failed to unpause VDIs") 

2172 

2173 def getFreeSpace(self) -> int: 

2174 return 0 

2175 

2176 def cleanup(self): 

2177 Util.log("In cleanup") 

2178 return 

2179 

2180 @override 

2181 def __str__(self) -> str: 

2182 if self.name: 

2183 ret = "%s ('%s')" % (self.uuid[0:4], self.name) 

2184 else: 

2185 ret = "%s" % self.uuid 

2186 return ret 

2187 

2188 def lock(self): 

2189 """Acquire the SR lock. Nested acquire()'s are ok. Check for Abort 

2190 signal to avoid deadlocking (trying to acquire the SR lock while the 

2191 lock is held by a process that is trying to abort us)""" 

2192 if not self._srLock: 

2193 return 

2194 

2195 if self._locked == 0: 

2196 abortFlag = IPCFlag(self.uuid) 

2197 for i in range(SR.LOCK_RETRY_ATTEMPTS_LOCK): 

2198 if self._srLock.acquireNoblock(): 

2199 self._locked += 1 

2200 return 

2201 if abortFlag.test(FLAG_TYPE_ABORT): 

2202 raise AbortException("Abort requested") 

2203 time.sleep(SR.LOCK_RETRY_INTERVAL) 

2204 raise util.SMException("Unable to acquire the SR lock") 

2205 

2206 self._locked += 1 

2207 

2208 def unlock(self): 

2209 if not self._srLock: 2209 ↛ 2211line 2209 didn't jump to line 2211, because the condition on line 2209 was never false

2210 return 

2211 assert(self._locked > 0) 

2212 self._locked -= 1 

2213 if self._locked == 0: 

2214 self._srLock.release() 

2215 

2216 def needUpdateBlockInfo(self) -> bool: 

2217 for vdi in self.vdis.values(): 

2218 if vdi.scanError or len(vdi.children) == 0: 

2219 continue 

2220 if not vdi.getConfig(vdi.DB_VHD_BLOCKS): 

2221 return True 

2222 return False 

2223 

2224 def updateBlockInfo(self) -> None: 

2225 for vdi in self.vdis.values(): 

2226 if vdi.scanError or len(vdi.children) == 0: 

2227 continue 

2228 if not vdi.getConfig(vdi.DB_VHD_BLOCKS): 

2229 vdi.updateBlockInfo() 

2230 

2231 def cleanupCoalesceJournals(self): 

2232 """Remove stale coalesce VDI indicators""" 

2233 entries = self.journaler.getAll(VDI.JRN_COALESCE) 

2234 for uuid, jval in entries.items(): 

2235 self.journaler.remove(VDI.JRN_COALESCE, uuid) 

2236 

2237 def cleanupJournals(self, dryRun=False): 

2238 """delete journal entries for non-existing VDIs""" 

2239 for t in [LVHDVDI.JRN_ZERO, VDI.JRN_RELINK, SR.JRN_CLONE]: 

2240 entries = self.journaler.getAll(t) 

2241 for uuid, jval in entries.items(): 

2242 if self.getVDI(uuid): 

2243 continue 

2244 if t == SR.JRN_CLONE: 

2245 baseUuid, clonUuid = jval.split("_") 

2246 if self.getVDI(baseUuid): 

2247 continue 

2248 Util.log(" Deleting stale '%s' journal entry for %s " 

2249 "(%s)" % (t, uuid, jval)) 

2250 if not dryRun: 

2251 self.journaler.remove(t, uuid) 

2252 

2253 def cleanupCache(self, maxAge=-1) -> int: 

2254 return 0 

2255 

2256 def _coalesce(self, vdi): 

2257 if self.journaler.get(vdi.JRN_RELINK, vdi.uuid): 2257 ↛ 2260line 2257 didn't jump to line 2260, because the condition on line 2257 was never true

2258 # this means we had done the actual coalescing already and just 

2259 # need to finish relinking and/or refreshing the children 

2260 Util.log("==> Coalesce apparently already done: skipping") 

2261 else: 

2262 # JRN_COALESCE is used to check which VDI is being coalesced in 

2263 # order to decide whether to abort the coalesce. We remove the 

2264 # journal as soon as the VHD coalesce step is done, because we 

2265 # don't expect the rest of the process to take long 

2266 self.journaler.create(vdi.JRN_COALESCE, vdi.uuid, "1") 

2267 vdi._doCoalesce() 

2268 self.journaler.remove(vdi.JRN_COALESCE, vdi.uuid) 

2269 

2270 util.fistpoint.activate("LVHDRT_before_create_relink_journal", self.uuid) 

2271 

2272 # we now need to relink the children: lock the SR to prevent ops 

2273 # like SM.clone from manipulating the VDIs we'll be relinking and 

2274 # rescan the SR first in case the children changed since the last 

2275 # scan 

2276 self.journaler.create(vdi.JRN_RELINK, vdi.uuid, "1") 

2277 

2278 self.lock() 

2279 try: 

2280 vdi.parent._tagChildrenForRelink() 

2281 self.scan() 

2282 vdi._relinkSkip() 

2283 finally: 

2284 self.unlock() 

2285 # Reload the children to leave things consistent 

2286 vdi.parent._reloadChildren(vdi) 

2287 

2288 self.journaler.remove(vdi.JRN_RELINK, vdi.uuid) 

2289 self.deleteVDI(vdi) 

2290 

2291 class CoalesceTracker: 

2292 GRACE_ITERATIONS = 1 

2293 MAX_ITERATIONS_NO_PROGRESS = 3 

2294 MAX_ITERATIONS = 10 

2295 MAX_INCREASE_FROM_MINIMUM = 1.2 

2296 HISTORY_STRING = "Iteration: {its} -- Initial size {initSize}" \ 

2297 " --> Final size {finSize}" 

2298 

2299 def __init__(self, sr): 

2300 self.itsNoProgress = 0 

2301 self.its = 0 

2302 self.minSize = float("inf") 

2303 self.history = [] 

2304 self.reason = "" 

2305 self.startSize = None 

2306 self.finishSize = None 

2307 self.sr = sr 

2308 

2309 def abortCoalesce(self, prevSize, curSize): 

2310 res = False 

2311 

2312 self.its += 1 

2313 self.history.append(self.HISTORY_STRING.format(its=self.its, 

2314 initSize=prevSize, 

2315 finSize=curSize)) 

2316 

2317 self.finishSize = curSize 

2318 

2319 if self.startSize is None: 

2320 self.startSize = prevSize 

2321 

2322 if curSize < self.minSize: 

2323 self.minSize = curSize 

2324 

2325 if prevSize < self.minSize: 

2326 self.minSize = prevSize 

2327 

2328 if prevSize < curSize: 

2329 self.itsNoProgress += 1 

2330 Util.log("No progress, attempt:" 

2331 " {attempt}".format(attempt=self.itsNoProgress)) 

2332 util.fistpoint.activate("cleanup_tracker_no_progress", self.sr.uuid) 

2333 

2334 if (not res) and (self.its > self.MAX_ITERATIONS): 

2335 max = self.MAX_ITERATIONS 

2336 self.reason = \ 

2337 "Max iterations ({max}) exceeded".format(max=max) 

2338 res = True 

2339 

2340 if (not res) and (self.itsNoProgress > 

2341 self.MAX_ITERATIONS_NO_PROGRESS): 

2342 max = self.MAX_ITERATIONS_NO_PROGRESS 

2343 self.reason = \ 

2344 "No progress made for {max} iterations".format(max=max) 

2345 res = True 

2346 

2347 maxSizeFromMin = self.MAX_INCREASE_FROM_MINIMUM * self.minSize 

2348 if (self.its > self.GRACE_ITERATIONS and 

2349 (not res) and (curSize > maxSizeFromMin)): 

2350 self.reason = "Unexpected bump in size," \ 

2351 " compared to minimum acheived" 

2352 res = True 

2353 

2354 return res 

2355 

2356 def printReasoning(self): 

2357 Util.log("Aborted coalesce") 

2358 for hist in self.history: 

2359 Util.log(hist) 

2360 Util.log(self.reason) 

2361 Util.log("Starting size was {size}" 

2362 .format(size=self.startSize)) 

2363 Util.log("Final size was {size}" 

2364 .format(size=self.finishSize)) 

2365 Util.log("Minimum size acheived was {size}" 

2366 .format(size=self.minSize)) 

2367 

2368 def _coalesceLeaf(self, vdi): 

2369 """Leaf-coalesce VDI vdi. Return true if we succeed, false if we cannot 

2370 complete due to external changes, namely vdi_delete and vdi_snapshot 

2371 that alter leaf-coalescibility of vdi""" 

2372 tracker = self.CoalesceTracker(self) 

2373 while not vdi.canLiveCoalesce(self.getStorageSpeed()): 

2374 prevSizeVHD = vdi.getSizeVHD() 

2375 if not self._snapshotCoalesce(vdi): 2375 ↛ 2376line 2375 didn't jump to line 2376, because the condition on line 2375 was never true

2376 return False 

2377 if tracker.abortCoalesce(prevSizeVHD, vdi.getSizeVHD()): 

2378 tracker.printReasoning() 

2379 raise util.SMException("VDI {uuid} could not be coalesced" 

2380 .format(uuid=vdi.uuid)) 

2381 return self._liveLeafCoalesce(vdi) 

2382 

2383 def calcStorageSpeed(self, startTime, endTime, vhdSize): 

2384 speed = None 

2385 total_time = endTime - startTime 

2386 if total_time > 0: 

2387 speed = float(vhdSize) / float(total_time) 

2388 return speed 

2389 

2390 def writeSpeedToFile(self, speed): 

2391 content = [] 

2392 speedFile = None 

2393 path = SPEED_LOG_ROOT.format(uuid=self.uuid) 

2394 self.lock() 

2395 try: 

2396 Util.log("Writing to file: {myfile}".format(myfile=path)) 

2397 lines = "" 

2398 if not os.path.isfile(path): 

2399 lines = str(speed) + "\n" 

2400 else: 

2401 speedFile = open(path, "r+") 

2402 content = speedFile.readlines() 

2403 content.append(str(speed) + "\n") 

2404 if len(content) > N_RUNNING_AVERAGE: 

2405 del content[0] 

2406 lines = "".join(content) 

2407 

2408 util.atomicFileWrite(path, VAR_RUN, lines) 

2409 finally: 

2410 if speedFile is not None: 

2411 speedFile.close() 

2412 Util.log("Closing file: {myfile}".format(myfile=path)) 

2413 self.unlock() 

2414 

2415 def recordStorageSpeed(self, startTime, endTime, vhdSize): 

2416 speed = self.calcStorageSpeed(startTime, endTime, vhdSize) 

2417 if speed is None: 

2418 return 

2419 

2420 self.writeSpeedToFile(speed) 

2421 

2422 def getStorageSpeed(self): 

2423 speedFile = None 

2424 path = SPEED_LOG_ROOT.format(uuid=self.uuid) 

2425 self.lock() 

2426 try: 

2427 speed = None 

2428 if os.path.isfile(path): 

2429 speedFile = open(path) 

2430 content = speedFile.readlines() 

2431 try: 

2432 content = [float(i) for i in content] 

2433 except ValueError: 

2434 Util.log("Something bad in the speed log:{log}". 

2435 format(log=speedFile.readlines())) 

2436 return speed 

2437 

2438 if len(content): 

2439 speed = sum(content) / float(len(content)) 

2440 if speed <= 0: 2440 ↛ 2442line 2440 didn't jump to line 2442, because the condition on line 2440 was never true

2441 # Defensive, should be impossible. 

2442 Util.log("Bad speed: {speed} calculated for SR: {uuid}". 

2443 format(speed=speed, uuid=self.uuid)) 

2444 speed = None 

2445 else: 

2446 Util.log("Speed file empty for SR: {uuid}". 

2447 format(uuid=self.uuid)) 

2448 else: 

2449 Util.log("Speed log missing for SR: {uuid}". 

2450 format(uuid=self.uuid)) 

2451 return speed 

2452 finally: 

2453 if not (speedFile is None): 

2454 speedFile.close() 

2455 self.unlock() 

2456 

2457 def _snapshotCoalesce(self, vdi): 

2458 # Note that because we are not holding any locks here, concurrent SM 

2459 # operations may change this tree under our feet. In particular, vdi 

2460 # can be deleted, or it can be snapshotted. 

2461 assert(AUTO_ONLINE_LEAF_COALESCE_ENABLED) 

2462 Util.log("Single-snapshotting %s" % vdi) 

2463 util.fistpoint.activate("LVHDRT_coaleaf_delay_1", self.uuid) 

2464 try: 

2465 ret = self.xapi.singleSnapshotVDI(vdi) 

2466 Util.log("Single-snapshot returned: %s" % ret) 

2467 except XenAPI.Failure as e: 

2468 if util.isInvalidVDI(e): 

2469 Util.log("The VDI appears to have been concurrently deleted") 

2470 return False 

2471 raise 

2472 self.scanLocked() 

2473 tempSnap = vdi.parent 

2474 if not tempSnap.isCoalesceable(): 

2475 Util.log("The VDI appears to have been concurrently snapshotted") 

2476 return False 

2477 Util.log("Coalescing parent %s" % tempSnap) 

2478 util.fistpoint.activate("LVHDRT_coaleaf_delay_2", self.uuid) 

2479 vhdSize = vdi.getSizeVHD() 

2480 self._coalesce(tempSnap) 

2481 if not vdi.isLeafCoalesceable(): 

2482 Util.log("The VDI tree appears to have been altered since") 

2483 return False 

2484 return True 

2485 

2486 def _liveLeafCoalesce(self, vdi) -> bool: 

2487 util.fistpoint.activate("LVHDRT_coaleaf_delay_3", self.uuid) 

2488 self.lock() 

2489 try: 

2490 self.scan() 

2491 if not self.getVDI(vdi.uuid): 

2492 Util.log("The VDI appears to have been deleted meanwhile") 

2493 return False 

2494 if not vdi.isLeafCoalesceable(): 

2495 Util.log("The VDI is no longer leaf-coalesceable") 

2496 return False 

2497 

2498 uuid = vdi.uuid 

2499 vdi.pause(failfast=True) 

2500 try: 

2501 try: 

2502 # "vdi" object will no longer be valid after this call 

2503 self._doCoalesceLeaf(vdi) 

2504 except: 

2505 Util.logException("_doCoalesceLeaf") 

2506 self._handleInterruptedCoalesceLeaf() 

2507 raise 

2508 finally: 

2509 vdi = self.getVDI(uuid) 

2510 if vdi: 

2511 vdi.ensureUnpaused() 

2512 vdiOld = self.getVDI(self.TMP_RENAME_PREFIX + uuid) 

2513 if vdiOld: 

2514 util.fistpoint.activate("LVHDRT_coaleaf_before_delete", self.uuid) 

2515 self.deleteVDI(vdiOld) 

2516 util.fistpoint.activate("LVHDRT_coaleaf_after_delete", self.uuid) 

2517 finally: 

2518 self.cleanup() 

2519 self.unlock() 

2520 self.logFilter.logState() 

2521 return True 

2522 

2523 def _doCoalesceLeaf(self, vdi): 

2524 """Actual coalescing of a leaf VDI onto parent. Must be called in an 

2525 offline/atomic context""" 

2526 self.journaler.create(VDI.JRN_LEAF, vdi.uuid, vdi.parent.uuid) 

2527 self._prepareCoalesceLeaf(vdi) 

2528 vdi.parent._setHidden(False) 

2529 vdi.parent._increaseSizeVirt(vdi.sizeVirt, False) 

2530 vdi.validate(True) 

2531 vdi.parent.validate(True) 

2532 util.fistpoint.activate("LVHDRT_coaleaf_before_coalesce", self.uuid) 

2533 timeout = vdi.LIVE_LEAF_COALESCE_TIMEOUT 

2534 if vdi.getConfig(vdi.DB_LEAFCLSC) == vdi.LEAFCLSC_FORCE: 

2535 Util.log("Leaf-coalesce forced, will not use timeout") 

2536 timeout = 0 

2537 vdi._coalesceVHD(timeout) 

2538 util.fistpoint.activate("LVHDRT_coaleaf_after_coalesce", self.uuid) 

2539 vdi.parent.validate(True) 

2540 #vdi._verifyContents(timeout / 2) 

2541 

2542 # rename 

2543 vdiUuid = vdi.uuid 

2544 oldName = vdi.fileName 

2545 origParentUuid = vdi.parent.uuid 

2546 vdi.rename(self.TMP_RENAME_PREFIX + vdiUuid) 

2547 util.fistpoint.activate("LVHDRT_coaleaf_one_renamed", self.uuid) 

2548 vdi.parent.rename(vdiUuid) 

2549 util.fistpoint.activate("LVHDRT_coaleaf_both_renamed", self.uuid) 

2550 self._updateSlavesOnRename(vdi.parent, oldName, origParentUuid) 

2551 

2552 # Note that "vdi.parent" is now the single remaining leaf and "vdi" is 

2553 # garbage 

2554 

2555 # update the VDI record 

2556 vdi.parent.delConfig(VDI.DB_VHD_PARENT) 

2557 if vdi.parent.raw: 

2558 vdi.parent.setConfig(VDI.DB_VDI_TYPE, vhdutil.VDI_TYPE_RAW) 

2559 vdi.parent.delConfig(VDI.DB_VHD_BLOCKS) 

2560 util.fistpoint.activate("LVHDRT_coaleaf_after_vdirec", self.uuid) 

2561 

2562 self._updateNode(vdi) 

2563 

2564 # delete the obsolete leaf & inflate the parent (in that order, to 

2565 # minimize free space requirements) 

2566 parent = vdi.parent 

2567 vdi._setHidden(True) 

2568 vdi.parent.children = [] 

2569 vdi.parent = None 

2570 

2571 extraSpace = self._calcExtraSpaceNeeded(vdi, parent) 

2572 freeSpace = self.getFreeSpace() 

2573 if freeSpace < extraSpace: 

2574 # don't delete unless we need the space: deletion is time-consuming 

2575 # because it requires contacting the slaves, and we're paused here 

2576 util.fistpoint.activate("LVHDRT_coaleaf_before_delete", self.uuid) 

2577 self.deleteVDI(vdi) 

2578 util.fistpoint.activate("LVHDRT_coaleaf_after_delete", self.uuid) 

2579 

2580 util.fistpoint.activate("LVHDRT_coaleaf_before_remove_j", self.uuid) 

2581 self.journaler.remove(VDI.JRN_LEAF, vdiUuid) 

2582 

2583 self.forgetVDI(origParentUuid) 

2584 self._finishCoalesceLeaf(parent) 

2585 self._updateSlavesOnResize(parent) 

2586 

2587 def _calcExtraSpaceNeeded(self, child, parent) -> int: 

2588 assert(not parent.raw) # raw parents not supported 

2589 extra = child.getSizeVHD() - parent.getSizeVHD() 

2590 if extra < 0: 

2591 extra = 0 

2592 return extra 

2593 

2594 def _prepareCoalesceLeaf(self, vdi) -> None: 

2595 pass 

2596 

2597 def _updateNode(self, vdi) -> None: 

2598 pass 

2599 

2600 def _finishCoalesceLeaf(self, parent) -> None: 

2601 pass 

2602 

2603 def _updateSlavesOnUndoLeafCoalesce(self, parent, child) -> None: 

2604 pass 

2605 

2606 def _updateSlavesOnRename(self, vdi, oldName, origParentUuid) -> None: 

2607 pass 

2608 

2609 def _updateSlavesOnResize(self, vdi) -> None: 

2610 pass 

2611 

2612 def _removeStaleVDIs(self, uuidsPresent) -> None: 

2613 for uuid in list(self.vdis.keys()): 

2614 if not uuid in uuidsPresent: 

2615 Util.log("VDI %s disappeared since last scan" % \ 

2616 self.vdis[uuid]) 

2617 del self.vdis[uuid] 

2618 

2619 def _handleInterruptedCoalesceLeaf(self) -> None: 

2620 """An interrupted leaf-coalesce operation may leave the VHD tree in an 

2621 inconsistent state. If the old-leaf VDI is still present, we revert the 

2622 operation (in case the original error is persistent); otherwise we must 

2623 finish the operation""" 

2624 pass 

2625 

2626 def _buildTree(self, force): 

2627 self.vdiTrees = [] 

2628 for vdi in self.vdis.values(): 

2629 if vdi.parentUuid: 

2630 parent = self.getVDI(vdi.parentUuid) 

2631 if not parent: 

2632 if vdi.uuid.startswith(self.TMP_RENAME_PREFIX): 

2633 self.vdiTrees.append(vdi) 

2634 continue 

2635 if force: 

2636 Util.log("ERROR: Parent VDI %s not found! (for %s)" % \ 

2637 (vdi.parentUuid, vdi.uuid)) 

2638 self.vdiTrees.append(vdi) 

2639 continue 

2640 else: 

2641 raise util.SMException("Parent VDI %s of %s not " \ 

2642 "found" % (vdi.parentUuid, vdi.uuid)) 

2643 vdi.parent = parent 

2644 parent.children.append(vdi) 

2645 else: 

2646 self.vdiTrees.append(vdi) 

2647 

2648 

2649class FileSR(SR): 

2650 TYPE = SR.TYPE_FILE 

2651 CACHE_FILE_EXT = ".vhdcache" 

2652 # cache cleanup actions 

2653 CACHE_ACTION_KEEP = 0 

2654 CACHE_ACTION_REMOVE = 1 

2655 CACHE_ACTION_REMOVE_IF_INACTIVE = 2 

2656 

2657 def __init__(self, uuid, xapi, createLock, force): 

2658 SR.__init__(self, uuid, xapi, createLock, force) 

2659 self.path = "/var/run/sr-mount/%s" % self.uuid 

2660 self.journaler = fjournaler.Journaler(self.path) 

2661 

2662 @override 

2663 def scan(self, force=False) -> None: 

2664 if not util.pathexists(self.path): 

2665 raise util.SMException("directory %s not found!" % self.uuid) 

2666 vhds = self._scan(force) 

2667 for uuid, vhdInfo in vhds.items(): 

2668 vdi = self.getVDI(uuid) 

2669 if not vdi: 

2670 self.logFilter.logNewVDI(uuid) 

2671 vdi = FileVDI(self, uuid, False) 

2672 self.vdis[uuid] = vdi 

2673 vdi.load(vhdInfo) 

2674 uuidsPresent = list(vhds.keys()) 

2675 rawList = [x for x in os.listdir(self.path) if x.endswith(vhdutil.FILE_EXTN_RAW)] 

2676 for rawName in rawList: 

2677 uuid = FileVDI.extractUuid(rawName) 

2678 uuidsPresent.append(uuid) 

2679 vdi = self.getVDI(uuid) 

2680 if not vdi: 

2681 self.logFilter.logNewVDI(uuid) 

2682 vdi = FileVDI(self, uuid, True) 

2683 self.vdis[uuid] = vdi 

2684 self._removeStaleVDIs(uuidsPresent) 

2685 self._buildTree(force) 

2686 self.logFilter.logState() 

2687 self._handleInterruptedCoalesceLeaf() 

2688 

2689 @override 

2690 def getFreeSpace(self) -> int: 

2691 return util.get_fs_size(self.path) - util.get_fs_utilisation(self.path) 

2692 

2693 @override 

2694 def deleteVDIs(self, vdiList) -> None: 

2695 rootDeleted = False 

2696 for vdi in vdiList: 

2697 if not vdi.parent: 

2698 rootDeleted = True 

2699 break 

2700 SR.deleteVDIs(self, vdiList) 

2701 if self.xapi.srRecord["type"] == "nfs" and rootDeleted: 

2702 self.xapi.markCacheSRsDirty() 

2703 

2704 @override 

2705 def cleanupCache(self, maxAge=-1) -> int: 

2706 """Clean up IntelliCache cache files. Caches for leaf nodes are 

2707 removed when the leaf node no longer exists or its allow-caching 

2708 attribute is not set. Caches for parent nodes are removed when the 

2709 parent node no longer exists or it hasn't been used in more than 

2710 <maxAge> hours. 

2711 Return number of caches removed. 

2712 """ 

2713 numRemoved = 0 

2714 cacheFiles = [x for x in os.listdir(self.path) if self._isCacheFileName(x)] 

2715 Util.log("Found %d cache files" % len(cacheFiles)) 

2716 cutoff = datetime.datetime.now() - datetime.timedelta(hours=maxAge) 

2717 for cacheFile in cacheFiles: 

2718 uuid = cacheFile[:-len(self.CACHE_FILE_EXT)] 

2719 action = self.CACHE_ACTION_KEEP 

2720 rec = self.xapi.getRecordVDI(uuid) 

2721 if not rec: 

2722 Util.log("Cache %s: VDI doesn't exist" % uuid) 

2723 action = self.CACHE_ACTION_REMOVE 

2724 elif rec["managed"] and not rec["allow_caching"]: 

2725 Util.log("Cache %s: caching disabled" % uuid) 

2726 action = self.CACHE_ACTION_REMOVE 

2727 elif not rec["managed"] and maxAge >= 0: 

2728 lastAccess = datetime.datetime.fromtimestamp( \ 

2729 os.path.getatime(os.path.join(self.path, cacheFile))) 

2730 if lastAccess < cutoff: 

2731 Util.log("Cache %s: older than %d hrs" % (uuid, maxAge)) 

2732 action = self.CACHE_ACTION_REMOVE_IF_INACTIVE 

2733 

2734 if action == self.CACHE_ACTION_KEEP: 

2735 Util.log("Keeping cache %s" % uuid) 

2736 continue 

2737 

2738 lockId = uuid 

2739 parentUuid = None 

2740 if rec and rec["managed"]: 

2741 parentUuid = rec["sm_config"].get("vhd-parent") 

2742 if parentUuid: 

2743 lockId = parentUuid 

2744 

2745 cacheLock = lock.Lock(blktap2.VDI.LOCK_CACHE_SETUP, lockId) 

2746 cacheLock.acquire() 

2747 try: 

2748 if self._cleanupCache(uuid, action): 

2749 numRemoved += 1 

2750 finally: 

2751 cacheLock.release() 

2752 return numRemoved 

2753 

2754 def _cleanupCache(self, uuid, action): 

2755 assert(action != self.CACHE_ACTION_KEEP) 

2756 rec = self.xapi.getRecordVDI(uuid) 

2757 if rec and rec["allow_caching"]: 

2758 Util.log("Cache %s appears to have become valid" % uuid) 

2759 return False 

2760 

2761 fullPath = os.path.join(self.path, uuid + self.CACHE_FILE_EXT) 

2762 tapdisk = blktap2.Tapdisk.find_by_path(fullPath) 

2763 if tapdisk: 

2764 if action == self.CACHE_ACTION_REMOVE_IF_INACTIVE: 

2765 Util.log("Cache %s still in use" % uuid) 

2766 return False 

2767 Util.log("Shutting down tapdisk for %s" % fullPath) 

2768 tapdisk.shutdown() 

2769 

2770 Util.log("Deleting file %s" % fullPath) 

2771 os.unlink(fullPath) 

2772 return True 

2773 

2774 def _isCacheFileName(self, name): 

2775 return (len(name) == Util.UUID_LEN + len(self.CACHE_FILE_EXT)) and \ 

2776 name.endswith(self.CACHE_FILE_EXT) 

2777 

2778 def _scan(self, force): 

2779 for i in range(SR.SCAN_RETRY_ATTEMPTS): 

2780 error = False 

2781 pattern = os.path.join(self.path, "*%s" % vhdutil.FILE_EXTN_VHD) 

2782 vhds = vhdutil.getAllVHDs(pattern, FileVDI.extractUuid) 

2783 for uuid, vhdInfo in vhds.items(): 

2784 if vhdInfo.error: 

2785 error = True 

2786 break 

2787 if not error: 

2788 return vhds 

2789 Util.log("Scan error on attempt %d" % i) 

2790 if force: 

2791 return vhds 

2792 raise util.SMException("Scan error") 

2793 

2794 @override 

2795 def deleteVDI(self, vdi) -> None: 

2796 self._checkSlaves(vdi) 

2797 SR.deleteVDI(self, vdi) 

2798 

2799 def _checkSlaves(self, vdi): 

2800 onlineHosts = self.xapi.getOnlineHosts() 

2801 abortFlag = IPCFlag(self.uuid) 

2802 for pbdRecord in self.xapi.getAttachedPBDs(): 

2803 hostRef = pbdRecord["host"] 

2804 if hostRef == self.xapi._hostRef: 

2805 continue 

2806 if abortFlag.test(FLAG_TYPE_ABORT): 

2807 raise AbortException("Aborting due to signal") 

2808 try: 

2809 self._checkSlave(hostRef, vdi) 

2810 except util.CommandException: 

2811 if hostRef in onlineHosts: 

2812 raise 

2813 

2814 def _checkSlave(self, hostRef, vdi): 

2815 call = (hostRef, "nfs-on-slave", "check", {'path': vdi.path}) 

2816 Util.log("Checking with slave: %s" % repr(call)) 

2817 _host = self.xapi.session.xenapi.host 

2818 text = _host.call_plugin( * call) 

2819 

2820 @override 

2821 def _handleInterruptedCoalesceLeaf(self) -> None: 

2822 entries = self.journaler.getAll(VDI.JRN_LEAF) 

2823 for uuid, parentUuid in entries.items(): 

2824 fileList = os.listdir(self.path) 

2825 childName = uuid + vhdutil.FILE_EXTN_VHD 

2826 tmpChildName = self.TMP_RENAME_PREFIX + uuid + vhdutil.FILE_EXTN_VHD 

2827 parentName1 = parentUuid + vhdutil.FILE_EXTN_VHD 

2828 parentName2 = parentUuid + vhdutil.FILE_EXTN_RAW 

2829 parentPresent = (parentName1 in fileList or parentName2 in fileList) 

2830 if parentPresent or tmpChildName in fileList: 

2831 self._undoInterruptedCoalesceLeaf(uuid, parentUuid) 

2832 else: 

2833 self._finishInterruptedCoalesceLeaf(uuid, parentUuid) 

2834 self.journaler.remove(VDI.JRN_LEAF, uuid) 

2835 vdi = self.getVDI(uuid) 

2836 if vdi: 

2837 vdi.ensureUnpaused() 

2838 

2839 def _undoInterruptedCoalesceLeaf(self, childUuid, parentUuid): 

2840 Util.log("*** UNDO LEAF-COALESCE") 

2841 parent = self.getVDI(parentUuid) 

2842 if not parent: 

2843 parent = self.getVDI(childUuid) 

2844 if not parent: 

2845 raise util.SMException("Neither %s nor %s found" % \ 

2846 (parentUuid, childUuid)) 

2847 Util.log("Renaming parent back: %s -> %s" % (childUuid, parentUuid)) 

2848 parent.rename(parentUuid) 

2849 util.fistpoint.activate("LVHDRT_coaleaf_undo_after_rename", self.uuid) 

2850 

2851 child = self.getVDI(childUuid) 

2852 if not child: 

2853 child = self.getVDI(self.TMP_RENAME_PREFIX + childUuid) 

2854 if not child: 

2855 raise util.SMException("Neither %s nor %s found" % \ 

2856 (childUuid, self.TMP_RENAME_PREFIX + childUuid)) 

2857 Util.log("Renaming child back to %s" % childUuid) 

2858 child.rename(childUuid) 

2859 Util.log("Updating the VDI record") 

2860 child.setConfig(VDI.DB_VHD_PARENT, parentUuid) 

2861 child.setConfig(VDI.DB_VDI_TYPE, vhdutil.VDI_TYPE_VHD) 

2862 util.fistpoint.activate("LVHDRT_coaleaf_undo_after_rename2", self.uuid) 

2863 

2864 if child.hidden: 

2865 child._setHidden(False) 

2866 if not parent.hidden: 

2867 parent._setHidden(True) 

2868 self._updateSlavesOnUndoLeafCoalesce(parent, child) 

2869 util.fistpoint.activate("LVHDRT_coaleaf_undo_end", self.uuid) 

2870 Util.log("*** leaf-coalesce undo successful") 

2871 if util.fistpoint.is_active("LVHDRT_coaleaf_stop_after_recovery"): 

2872 child.setConfig(VDI.DB_LEAFCLSC, VDI.LEAFCLSC_DISABLED) 

2873 

2874 def _finishInterruptedCoalesceLeaf(self, childUuid, parentUuid): 

2875 Util.log("*** FINISH LEAF-COALESCE") 

2876 vdi = self.getVDI(childUuid) 

2877 if not vdi: 

2878 raise util.SMException("VDI %s not found" % childUuid) 

2879 try: 

2880 self.forgetVDI(parentUuid) 

2881 except XenAPI.Failure: 

2882 pass 

2883 self._updateSlavesOnResize(vdi) 

2884 util.fistpoint.activate("LVHDRT_coaleaf_finish_end", self.uuid) 

2885 Util.log("*** finished leaf-coalesce successfully") 

2886 

2887 

2888class LVHDSR(SR): 

2889 TYPE = SR.TYPE_LVHD 

2890 SUBTYPES = ["lvhdoiscsi", "lvhdohba"] 

2891 

2892 def __init__(self, uuid, xapi, createLock, force): 

2893 SR.__init__(self, uuid, xapi, createLock, force) 

2894 self.vgName = "%s%s" % (lvhdutil.VG_PREFIX, self.uuid) 

2895 self.path = os.path.join(lvhdutil.VG_LOCATION, self.vgName) 

2896 

2897 sr_ref = self.xapi.session.xenapi.SR.get_by_uuid(self.uuid) 

2898 other_conf = self.xapi.session.xenapi.SR.get_other_config(sr_ref) 

2899 lvm_conf = other_conf.get('lvm-conf') if other_conf else None 

2900 self.lvmCache = lvmcache.LVMCache(self.vgName, lvm_conf) 

2901 

2902 self.lvActivator = LVActivator(self.uuid, self.lvmCache) 

2903 self.journaler = journaler.Journaler(self.lvmCache) 

2904 

2905 @override 

2906 def deleteVDI(self, vdi) -> None: 

2907 if self.lvActivator.get(vdi.uuid, False): 

2908 self.lvActivator.deactivate(vdi.uuid, False) 

2909 self._checkSlaves(vdi) 

2910 SR.deleteVDI(self, vdi) 

2911 

2912 @override 

2913 def forgetVDI(self, vdiUuid) -> None: 

2914 SR.forgetVDI(self, vdiUuid) 

2915 mdpath = os.path.join(self.path, lvutil.MDVOLUME_NAME) 

2916 LVMMetadataHandler(mdpath).deleteVdiFromMetadata(vdiUuid) 

2917 

2918 @override 

2919 def getFreeSpace(self) -> int: 

2920 stats = lvutil._getVGstats(self.vgName) 

2921 return stats['physical_size'] - stats['physical_utilisation'] 

2922 

2923 @override 

2924 def cleanup(self): 

2925 if not self.lvActivator.deactivateAll(): 

2926 Util.log("ERROR deactivating LVs while cleaning up") 

2927 

2928 @override 

2929 def needUpdateBlockInfo(self) -> bool: 

2930 for vdi in self.vdis.values(): 

2931 if vdi.scanError or vdi.raw or len(vdi.children) == 0: 

2932 continue 

2933 if not vdi.getConfig(vdi.DB_VHD_BLOCKS): 

2934 return True 

2935 return False 

2936 

2937 @override 

2938 def updateBlockInfo(self) -> None: 

2939 numUpdated = 0 

2940 for vdi in self.vdis.values(): 

2941 if vdi.scanError or vdi.raw or len(vdi.children) == 0: 

2942 continue 

2943 if not vdi.getConfig(vdi.DB_VHD_BLOCKS): 

2944 vdi.updateBlockInfo() 

2945 numUpdated += 1 

2946 if numUpdated: 

2947 # deactivate the LVs back sooner rather than later. If we don't 

2948 # now, by the time this thread gets to deactivations, another one 

2949 # might have leaf-coalesced a node and deleted it, making the child 

2950 # inherit the refcount value and preventing the correct decrement 

2951 self.cleanup() 

2952 

2953 @override 

2954 def scan(self, force=False) -> None: 

2955 vdis = self._scan(force) 

2956 for uuid, vdiInfo in vdis.items(): 

2957 vdi = self.getVDI(uuid) 

2958 if not vdi: 

2959 self.logFilter.logNewVDI(uuid) 

2960 vdi = LVHDVDI(self, uuid, 

2961 vdiInfo.vdiType == vhdutil.VDI_TYPE_RAW) 

2962 self.vdis[uuid] = vdi 

2963 vdi.load(vdiInfo) 

2964 self._removeStaleVDIs(vdis.keys()) 

2965 self._buildTree(force) 

2966 self.logFilter.logState() 

2967 self._handleInterruptedCoalesceLeaf() 

2968 

2969 def _scan(self, force): 

2970 for i in range(SR.SCAN_RETRY_ATTEMPTS): 

2971 error = False 

2972 self.lvmCache.refresh() 

2973 vdis = lvhdutil.getVDIInfo(self.lvmCache) 

2974 for uuid, vdiInfo in vdis.items(): 

2975 if vdiInfo.scanError: 

2976 error = True 

2977 break 

2978 if not error: 

2979 return vdis 

2980 Util.log("Scan error, retrying (%d)" % i) 

2981 if force: 

2982 return vdis 

2983 raise util.SMException("Scan error") 

2984 

2985 @override 

2986 def _removeStaleVDIs(self, uuidsPresent) -> None: 

2987 for uuid in list(self.vdis.keys()): 

2988 if not uuid in uuidsPresent: 

2989 Util.log("VDI %s disappeared since last scan" % \ 

2990 self.vdis[uuid]) 

2991 del self.vdis[uuid] 

2992 if self.lvActivator.get(uuid, False): 

2993 self.lvActivator.remove(uuid, False) 

2994 

2995 @override 

2996 def _liveLeafCoalesce(self, vdi) -> bool: 

2997 """If the parent is raw and the child was resized (virt. size), then 

2998 we'll need to resize the parent, which can take a while due to zeroing 

2999 out of the extended portion of the LV. Do it before pausing the child 

3000 to avoid a protracted downtime""" 

3001 if vdi.parent.raw and vdi.sizeVirt > vdi.parent.sizeVirt: 

3002 self.lvmCache.setReadonly(vdi.parent.fileName, False) 

3003 vdi.parent._increaseSizeVirt(vdi.sizeVirt) 

3004 

3005 return SR._liveLeafCoalesce(self, vdi) 

3006 

3007 @override 

3008 def _prepareCoalesceLeaf(self, vdi) -> None: 

3009 vdi._activateChain() 

3010 self.lvmCache.setReadonly(vdi.parent.fileName, False) 

3011 vdi.deflate() 

3012 vdi.inflateParentForCoalesce() 

3013 

3014 @override 

3015 def _updateNode(self, vdi) -> None: 

3016 # fix the refcounts: the remaining node should inherit the binary 

3017 # refcount from the leaf (because if it was online, it should remain 

3018 # refcounted as such), but the normal refcount from the parent (because 

3019 # this node is really the parent node) - minus 1 if it is online (since 

3020 # non-leaf nodes increment their normal counts when they are online and 

3021 # we are now a leaf, storing that 1 in the binary refcount). 

3022 ns = lvhdutil.NS_PREFIX_LVM + self.uuid 

3023 cCnt, cBcnt = RefCounter.check(vdi.uuid, ns) 

3024 pCnt, pBcnt = RefCounter.check(vdi.parent.uuid, ns) 

3025 pCnt = pCnt - cBcnt 

3026 assert(pCnt >= 0) 

3027 RefCounter.set(vdi.parent.uuid, pCnt, cBcnt, ns) 

3028 

3029 @override 

3030 def _finishCoalesceLeaf(self, parent) -> None: 

3031 if not parent.isSnapshot() or parent.isAttachedRW(): 

3032 parent.inflateFully() 

3033 else: 

3034 parent.deflate() 

3035 

3036 @override 

3037 def _calcExtraSpaceNeeded(self, child, parent) -> int: 

3038 return lvhdutil.calcSizeVHDLV(parent.sizeVirt) - parent.sizeLV 

3039 

3040 @override 

3041 def _handleInterruptedCoalesceLeaf(self) -> None: 

3042 entries = self.journaler.getAll(VDI.JRN_LEAF) 

3043 for uuid, parentUuid in entries.items(): 

3044 childLV = lvhdutil.LV_PREFIX[vhdutil.VDI_TYPE_VHD] + uuid 

3045 tmpChildLV = lvhdutil.LV_PREFIX[vhdutil.VDI_TYPE_VHD] + \ 

3046 self.TMP_RENAME_PREFIX + uuid 

3047 parentLV1 = lvhdutil.LV_PREFIX[vhdutil.VDI_TYPE_VHD] + parentUuid 

3048 parentLV2 = lvhdutil.LV_PREFIX[vhdutil.VDI_TYPE_RAW] + parentUuid 

3049 parentPresent = (self.lvmCache.checkLV(parentLV1) or \ 

3050 self.lvmCache.checkLV(parentLV2)) 

3051 if parentPresent or self.lvmCache.checkLV(tmpChildLV): 

3052 self._undoInterruptedCoalesceLeaf(uuid, parentUuid) 

3053 else: 

3054 self._finishInterruptedCoalesceLeaf(uuid, parentUuid) 

3055 self.journaler.remove(VDI.JRN_LEAF, uuid) 

3056 vdi = self.getVDI(uuid) 

3057 if vdi: 

3058 vdi.ensureUnpaused() 

3059 

3060 def _undoInterruptedCoalesceLeaf(self, childUuid, parentUuid): 

3061 Util.log("*** UNDO LEAF-COALESCE") 

3062 parent = self.getVDI(parentUuid) 

3063 if not parent: 

3064 parent = self.getVDI(childUuid) 

3065 if not parent: 

3066 raise util.SMException("Neither %s nor %s found" % \ 

3067 (parentUuid, childUuid)) 

3068 Util.log("Renaming parent back: %s -> %s" % (childUuid, parentUuid)) 

3069 parent.rename(parentUuid) 

3070 util.fistpoint.activate("LVHDRT_coaleaf_undo_after_rename", self.uuid) 

3071 

3072 child = self.getVDI(childUuid) 

3073 if not child: 

3074 child = self.getVDI(self.TMP_RENAME_PREFIX + childUuid) 

3075 if not child: 

3076 raise util.SMException("Neither %s nor %s found" % \ 

3077 (childUuid, self.TMP_RENAME_PREFIX + childUuid)) 

3078 Util.log("Renaming child back to %s" % childUuid) 

3079 child.rename(childUuid) 

3080 Util.log("Updating the VDI record") 

3081 child.setConfig(VDI.DB_VHD_PARENT, parentUuid) 

3082 child.setConfig(VDI.DB_VDI_TYPE, vhdutil.VDI_TYPE_VHD) 

3083 util.fistpoint.activate("LVHDRT_coaleaf_undo_after_rename2", self.uuid) 

3084 

3085 # refcount (best effort - assume that it had succeeded if the 

3086 # second rename succeeded; if not, this adjustment will be wrong, 

3087 # leading to a non-deactivation of the LV) 

3088 ns = lvhdutil.NS_PREFIX_LVM + self.uuid 

3089 cCnt, cBcnt = RefCounter.check(child.uuid, ns) 

3090 pCnt, pBcnt = RefCounter.check(parent.uuid, ns) 

3091 pCnt = pCnt + cBcnt 

3092 RefCounter.set(parent.uuid, pCnt, 0, ns) 

3093 util.fistpoint.activate("LVHDRT_coaleaf_undo_after_refcount", self.uuid) 

3094 

3095 parent.deflate() 

3096 child.inflateFully() 

3097 util.fistpoint.activate("LVHDRT_coaleaf_undo_after_deflate", self.uuid) 

3098 if child.hidden: 

3099 child._setHidden(False) 

3100 if not parent.hidden: 

3101 parent._setHidden(True) 

3102 if not parent.lvReadonly: 

3103 self.lvmCache.setReadonly(parent.fileName, True) 

3104 self._updateSlavesOnUndoLeafCoalesce(parent, child) 

3105 util.fistpoint.activate("LVHDRT_coaleaf_undo_end", self.uuid) 

3106 Util.log("*** leaf-coalesce undo successful") 

3107 if util.fistpoint.is_active("LVHDRT_coaleaf_stop_after_recovery"): 

3108 child.setConfig(VDI.DB_LEAFCLSC, VDI.LEAFCLSC_DISABLED) 

3109 

3110 def _finishInterruptedCoalesceLeaf(self, childUuid, parentUuid): 

3111 Util.log("*** FINISH LEAF-COALESCE") 

3112 vdi = self.getVDI(childUuid) 

3113 if not vdi: 

3114 raise util.SMException("VDI %s not found" % childUuid) 

3115 vdi.inflateFully() 

3116 util.fistpoint.activate("LVHDRT_coaleaf_finish_after_inflate", self.uuid) 

3117 try: 

3118 self.forgetVDI(parentUuid) 

3119 except XenAPI.Failure: 

3120 pass 

3121 self._updateSlavesOnResize(vdi) 

3122 util.fistpoint.activate("LVHDRT_coaleaf_finish_end", self.uuid) 

3123 Util.log("*** finished leaf-coalesce successfully") 

3124 

3125 def _checkSlaves(self, vdi): 

3126 """Confirm with all slaves in the pool that 'vdi' is not in use. We 

3127 try to check all slaves, including those that the Agent believes are 

3128 offline, but ignore failures for offline hosts. This is to avoid cases 

3129 where the Agent thinks a host is offline but the host is up.""" 

3130 args = {"vgName": self.vgName, 

3131 "action1": "deactivateNoRefcount", 

3132 "lvName1": vdi.fileName, 

3133 "action2": "cleanupLockAndRefcount", 

3134 "uuid2": vdi.uuid, 

3135 "ns2": lvhdutil.NS_PREFIX_LVM + self.uuid} 

3136 onlineHosts = self.xapi.getOnlineHosts() 

3137 abortFlag = IPCFlag(self.uuid) 

3138 for pbdRecord in self.xapi.getAttachedPBDs(): 

3139 hostRef = pbdRecord["host"] 

3140 if hostRef == self.xapi._hostRef: 

3141 continue 

3142 if abortFlag.test(FLAG_TYPE_ABORT): 

3143 raise AbortException("Aborting due to signal") 

3144 Util.log("Checking with slave %s (path %s)" % ( 

3145 self.xapi.getRecordHost(hostRef)['hostname'], vdi.path)) 

3146 try: 

3147 self.xapi.ensureInactive(hostRef, args) 

3148 except XenAPI.Failure: 

3149 if hostRef in onlineHosts: 

3150 raise 

3151 

3152 @override 

3153 def _updateSlavesOnUndoLeafCoalesce(self, parent, child) -> None: 

3154 slaves = util.get_slaves_attached_on(self.xapi.session, [child.uuid]) 

3155 if not slaves: 

3156 Util.log("Update-on-leaf-undo: VDI %s not attached on any slave" % \ 

3157 child) 

3158 return 

3159 

3160 tmpName = lvhdutil.LV_PREFIX[vhdutil.VDI_TYPE_VHD] + \ 

3161 self.TMP_RENAME_PREFIX + child.uuid 

3162 args = {"vgName": self.vgName, 

3163 "action1": "deactivateNoRefcount", 

3164 "lvName1": tmpName, 

3165 "action2": "deactivateNoRefcount", 

3166 "lvName2": child.fileName, 

3167 "action3": "refresh", 

3168 "lvName3": child.fileName, 

3169 "action4": "refresh", 

3170 "lvName4": parent.fileName} 

3171 for slave in slaves: 

3172 Util.log("Updating %s, %s, %s on slave %s" % \ 

3173 (tmpName, child.fileName, parent.fileName, 

3174 self.xapi.getRecordHost(slave)['hostname'])) 

3175 text = self.xapi.session.xenapi.host.call_plugin( \ 

3176 slave, self.xapi.PLUGIN_ON_SLAVE, "multi", args) 

3177 Util.log("call-plugin returned: '%s'" % text) 

3178 

3179 @override 

3180 def _updateSlavesOnRename(self, vdi, oldNameLV, origParentUuid) -> None: 

3181 slaves = util.get_slaves_attached_on(self.xapi.session, [vdi.uuid]) 

3182 if not slaves: 

3183 Util.log("Update-on-rename: VDI %s not attached on any slave" % vdi) 

3184 return 

3185 

3186 args = {"vgName": self.vgName, 

3187 "action1": "deactivateNoRefcount", 

3188 "lvName1": oldNameLV, 

3189 "action2": "refresh", 

3190 "lvName2": vdi.fileName, 

3191 "action3": "cleanupLockAndRefcount", 

3192 "uuid3": origParentUuid, 

3193 "ns3": lvhdutil.NS_PREFIX_LVM + self.uuid} 

3194 for slave in slaves: 

3195 Util.log("Updating %s to %s on slave %s" % \ 

3196 (oldNameLV, vdi.fileName, 

3197 self.xapi.getRecordHost(slave)['hostname'])) 

3198 text = self.xapi.session.xenapi.host.call_plugin( \ 

3199 slave, self.xapi.PLUGIN_ON_SLAVE, "multi", args) 

3200 Util.log("call-plugin returned: '%s'" % text) 

3201 

3202 @override 

3203 def _updateSlavesOnResize(self, vdi) -> None: 

3204 uuids = [x.uuid for x in vdi.getAllLeaves()] 

3205 slaves = util.get_slaves_attached_on(self.xapi.session, uuids) 

3206 if not slaves: 

3207 util.SMlog("Update-on-resize: %s not attached on any slave" % vdi) 

3208 return 

3209 lvhdutil.lvRefreshOnSlaves(self.xapi.session, self.uuid, self.vgName, 

3210 vdi.fileName, vdi.uuid, slaves) 

3211 

3212 

3213class LinstorSR(SR): 

3214 TYPE = SR.TYPE_LINSTOR 

3215 

3216 def __init__(self, uuid, xapi, createLock, force): 

3217 if not LINSTOR_AVAILABLE: 

3218 raise util.SMException( 

3219 'Can\'t load cleanup LinstorSR: LINSTOR libraries are missing' 

3220 ) 

3221 

3222 SR.__init__(self, uuid, xapi, createLock, force) 

3223 self.path = LinstorVolumeManager.DEV_ROOT_PATH 

3224 self._reloadLinstor(journaler_only=True) 

3225 

3226 @override 

3227 def deleteVDI(self, vdi) -> None: 

3228 self._checkSlaves(vdi) 

3229 SR.deleteVDI(self, vdi) 

3230 

3231 @override 

3232 def getFreeSpace(self) -> int: 

3233 return self._linstor.max_volume_size_allowed 

3234 

3235 @override 

3236 def scan(self, force=False) -> None: 

3237 all_vdi_info = self._scan(force) 

3238 for uuid, vdiInfo in all_vdi_info.items(): 

3239 # When vdiInfo is None, the VDI is RAW. 

3240 vdi = self.getVDI(uuid) 

3241 if not vdi: 

3242 self.logFilter.logNewVDI(uuid) 

3243 vdi = LinstorVDI(self, uuid, not vdiInfo) 

3244 self.vdis[uuid] = vdi 

3245 if vdiInfo: 

3246 vdi.load(vdiInfo) 

3247 self._removeStaleVDIs(all_vdi_info.keys()) 

3248 self._buildTree(force) 

3249 self.logFilter.logState() 

3250 self._handleInterruptedCoalesceLeaf() 

3251 

3252 @override 

3253 def pauseVDIs(self, vdiList) -> None: 

3254 self._linstor.ensure_volume_list_is_not_locked( 

3255 vdiList, timeout=LinstorVDI.VOLUME_LOCK_TIMEOUT 

3256 ) 

3257 return super(LinstorSR, self).pauseVDIs(vdiList) 

3258 

3259 def _reloadLinstor(self, journaler_only=False): 

3260 session = self.xapi.session 

3261 host_ref = util.get_this_host_ref(session) 

3262 sr_ref = session.xenapi.SR.get_by_uuid(self.uuid) 

3263 

3264 pbd = util.find_my_pbd(session, host_ref, sr_ref) 

3265 if pbd is None: 

3266 raise util.SMException('Failed to find PBD') 

3267 

3268 dconf = session.xenapi.PBD.get_device_config(pbd) 

3269 group_name = dconf['group-name'] 

3270 

3271 controller_uri = get_controller_uri() 

3272 self.journaler = LinstorJournaler( 

3273 controller_uri, group_name, logger=util.SMlog 

3274 ) 

3275 

3276 if journaler_only: 

3277 return 

3278 

3279 self._linstor = LinstorVolumeManager( 

3280 controller_uri, 

3281 group_name, 

3282 repair=True, 

3283 logger=util.SMlog 

3284 ) 

3285 self._vhdutil = LinstorVhdUtil(session, self._linstor) 

3286 

3287 def _scan(self, force): 

3288 for i in range(SR.SCAN_RETRY_ATTEMPTS): 

3289 self._reloadLinstor() 

3290 error = False 

3291 try: 

3292 all_vdi_info = self._load_vdi_info() 

3293 for uuid, vdiInfo in all_vdi_info.items(): 

3294 if vdiInfo and vdiInfo.error: 

3295 error = True 

3296 break 

3297 if not error: 

3298 return all_vdi_info 

3299 Util.log('Scan error, retrying ({})'.format(i)) 

3300 except Exception as e: 

3301 Util.log('Scan exception, retrying ({}): {}'.format(i, e)) 

3302 Util.log(traceback.format_exc()) 

3303 

3304 if force: 

3305 return all_vdi_info 

3306 raise util.SMException('Scan error') 

3307 

3308 def _load_vdi_info(self): 

3309 all_vdi_info = {} 

3310 

3311 # TODO: Ensure metadata contains the right info. 

3312 

3313 all_volume_info = self._linstor.get_volumes_with_info() 

3314 volumes_metadata = self._linstor.get_volumes_with_metadata() 

3315 for vdi_uuid, volume_info in all_volume_info.items(): 

3316 try: 

3317 volume_metadata = volumes_metadata[vdi_uuid] 

3318 if not volume_info.name and not list(volume_metadata.items()): 

3319 continue # Ignore it, probably deleted. 

3320 

3321 if vdi_uuid.startswith('DELETED_'): 

3322 # Assume it's really a RAW volume of a failed snap without VHD header/footer. 

3323 # We must remove this VDI now without adding it in the VDI list. 

3324 # Otherwise `Relinking` calls and other actions can be launched on it. 

3325 # We don't want that... 

3326 Util.log('Deleting bad VDI {}'.format(vdi_uuid)) 

3327 

3328 self.lock() 

3329 try: 

3330 self._linstor.destroy_volume(vdi_uuid) 

3331 try: 

3332 self.forgetVDI(vdi_uuid) 

3333 except: 

3334 pass 

3335 except Exception as e: 

3336 Util.log('Cannot delete bad VDI: {}'.format(e)) 

3337 finally: 

3338 self.unlock() 

3339 continue 

3340 

3341 vdi_type = volume_metadata.get(VDI_TYPE_TAG) 

3342 volume_name = self._linstor.get_volume_name(vdi_uuid) 

3343 if volume_name.startswith(LINSTOR_PERSISTENT_PREFIX): 

3344 # Always RAW! 

3345 info = None 

3346 elif vdi_type == vhdutil.VDI_TYPE_VHD: 

3347 info = self._vhdutil.get_vhd_info(vdi_uuid) 

3348 else: 

3349 # Ensure it's not a VHD... 

3350 try: 

3351 info = self._vhdutil.get_vhd_info(vdi_uuid) 

3352 except: 

3353 try: 

3354 self._vhdutil.force_repair( 

3355 self._linstor.get_device_path(vdi_uuid) 

3356 ) 

3357 info = self._vhdutil.get_vhd_info(vdi_uuid) 

3358 except: 

3359 info = None 

3360 

3361 except Exception as e: 

3362 Util.log( 

3363 ' [VDI {}: failed to load VDI info]: {}' 

3364 .format(vdi_uuid, e) 

3365 ) 

3366 info = vhdutil.VHDInfo(vdi_uuid) 

3367 info.error = 1 

3368 

3369 all_vdi_info[vdi_uuid] = info 

3370 

3371 return all_vdi_info 

3372 

3373 @override 

3374 def _prepareCoalesceLeaf(self, vdi) -> None: 

3375 vdi._activateChain() 

3376 vdi.deflate() 

3377 vdi._inflateParentForCoalesce() 

3378 

3379 @override 

3380 def _finishCoalesceLeaf(self, parent) -> None: 

3381 if not parent.isSnapshot() or parent.isAttachedRW(): 

3382 parent.inflateFully() 

3383 else: 

3384 parent.deflate() 

3385 

3386 @override 

3387 def _calcExtraSpaceNeeded(self, child, parent) -> int: 

3388 return LinstorVhdUtil.compute_volume_size(parent.sizeVirt, parent.vdi_type) - parent.getDrbdSize() 

3389 

3390 def _hasValidDevicePath(self, uuid): 

3391 try: 

3392 self._linstor.get_device_path(uuid) 

3393 except Exception: 

3394 # TODO: Maybe log exception. 

3395 return False 

3396 return True 

3397 

3398 @override 

3399 def _liveLeafCoalesce(self, vdi) -> bool: 

3400 self.lock() 

3401 try: 

3402 self._linstor.ensure_volume_is_not_locked( 

3403 vdi.uuid, timeout=LinstorVDI.VOLUME_LOCK_TIMEOUT 

3404 ) 

3405 return super(LinstorSR, self)._liveLeafCoalesce(vdi) 

3406 finally: 

3407 self.unlock() 

3408 

3409 @override 

3410 def _handleInterruptedCoalesceLeaf(self) -> None: 

3411 entries = self.journaler.get_all(VDI.JRN_LEAF) 

3412 for uuid, parentUuid in entries.items(): 

3413 if self._hasValidDevicePath(parentUuid) or \ 

3414 self._hasValidDevicePath(self.TMP_RENAME_PREFIX + uuid): 

3415 self._undoInterruptedCoalesceLeaf(uuid, parentUuid) 

3416 else: 

3417 self._finishInterruptedCoalesceLeaf(uuid, parentUuid) 

3418 self.journaler.remove(VDI.JRN_LEAF, uuid) 

3419 vdi = self.getVDI(uuid) 

3420 if vdi: 

3421 vdi.ensureUnpaused() 

3422 

3423 def _undoInterruptedCoalesceLeaf(self, childUuid, parentUuid): 

3424 Util.log('*** UNDO LEAF-COALESCE') 

3425 parent = self.getVDI(parentUuid) 

3426 if not parent: 

3427 parent = self.getVDI(childUuid) 

3428 if not parent: 

3429 raise util.SMException( 

3430 'Neither {} nor {} found'.format(parentUuid, childUuid) 

3431 ) 

3432 Util.log( 

3433 'Renaming parent back: {} -> {}'.format(childUuid, parentUuid) 

3434 ) 

3435 parent.rename(parentUuid) 

3436 

3437 child = self.getVDI(childUuid) 

3438 if not child: 

3439 child = self.getVDI(self.TMP_RENAME_PREFIX + childUuid) 

3440 if not child: 

3441 raise util.SMException( 

3442 'Neither {} nor {} found'.format( 

3443 childUuid, self.TMP_RENAME_PREFIX + childUuid 

3444 ) 

3445 ) 

3446 Util.log('Renaming child back to {}'.format(childUuid)) 

3447 child.rename(childUuid) 

3448 Util.log('Updating the VDI record') 

3449 child.setConfig(VDI.DB_VHD_PARENT, parentUuid) 

3450 child.setConfig(VDI.DB_VDI_TYPE, vhdutil.VDI_TYPE_VHD) 

3451 

3452 # TODO: Maybe deflate here. 

3453 

3454 if child.hidden: 

3455 child._setHidden(False) 

3456 if not parent.hidden: 

3457 parent._setHidden(True) 

3458 self._updateSlavesOnUndoLeafCoalesce(parent, child) 

3459 Util.log('*** leaf-coalesce undo successful') 

3460 

3461 def _finishInterruptedCoalesceLeaf(self, childUuid, parentUuid): 

3462 Util.log('*** FINISH LEAF-COALESCE') 

3463 vdi = self.getVDI(childUuid) 

3464 if not vdi: 

3465 raise util.SMException('VDI {} not found'.format(childUuid)) 

3466 # TODO: Maybe inflate. 

3467 try: 

3468 self.forgetVDI(parentUuid) 

3469 except XenAPI.Failure: 

3470 pass 

3471 self._updateSlavesOnResize(vdi) 

3472 Util.log('*** finished leaf-coalesce successfully') 

3473 

3474 def _checkSlaves(self, vdi): 

3475 try: 

3476 all_openers = self._linstor.get_volume_openers(vdi.uuid) 

3477 for openers in all_openers.values(): 

3478 for opener in openers.values(): 

3479 if opener['process-name'] != 'tapdisk': 

3480 raise util.SMException( 

3481 'VDI {} is in use: {}'.format(vdi.uuid, all_openers) 

3482 ) 

3483 except LinstorVolumeManagerError as e: 

3484 if e.code != LinstorVolumeManagerError.ERR_VOLUME_NOT_EXISTS: 

3485 raise 

3486 

3487 

3488################################################################################ 

3489# 

3490# Helpers 

3491# 

3492def daemonize(): 

3493 pid = os.fork() 

3494 if pid: 

3495 os.waitpid(pid, 0) 

3496 Util.log("New PID [%d]" % pid) 

3497 return False 

3498 os.chdir("/") 

3499 os.setsid() 

3500 pid = os.fork() 

3501 if pid: 

3502 Util.log("Will finish as PID [%d]" % pid) 

3503 os._exit(0) 

3504 for fd in [0, 1, 2]: 

3505 try: 

3506 os.close(fd) 

3507 except OSError: 

3508 pass 

3509 # we need to fill those special fd numbers or pread won't work 

3510 sys.stdin = open("/dev/null", 'r') 

3511 sys.stderr = open("/dev/null", 'w') 

3512 sys.stdout = open("/dev/null", 'w') 

3513 # As we're a new process we need to clear the lock objects 

3514 lock.Lock.clearAll() 

3515 return True 

3516 

3517 

3518def normalizeType(type): 

3519 if type in LVHDSR.SUBTYPES: 

3520 type = SR.TYPE_LVHD 

3521 if type in ["lvm", "lvmoiscsi", "lvmohba", "lvmofcoe"]: 

3522 # temporary while LVHD is symlinked as LVM 

3523 type = SR.TYPE_LVHD 

3524 if type in [ 

3525 "ext", "nfs", "ocfsoiscsi", "ocfsohba", "smb", "cephfs", "glusterfs", 

3526 "moosefs", "xfs", "zfs", "largeblock" 

3527 ]: 

3528 type = SR.TYPE_FILE 

3529 if type in ["linstor"]: 

3530 type = SR.TYPE_LINSTOR 

3531 if type not in SR.TYPES: 

3532 raise util.SMException("Unsupported SR type: %s" % type) 

3533 return type 

3534 

3535GCPAUSE_DEFAULT_SLEEP = 5 * 60 

3536 

3537 

3538def _gc_init_file(sr_uuid): 

3539 return os.path.join(NON_PERSISTENT_DIR, str(sr_uuid), 'gc_init') 

3540 

3541 

3542def _create_init_file(sr_uuid): 

3543 util.makedirs(os.path.join(NON_PERSISTENT_DIR, str(sr_uuid))) 

3544 with open(os.path.join( 

3545 NON_PERSISTENT_DIR, str(sr_uuid), 'gc_init'), 'w+') as f: 

3546 f.write('1') 

3547 

3548 

3549def _gcLoopPause(sr, dryRun=False, immediate=False): 

3550 if immediate: 

3551 return 

3552 

3553 # Check to see if the GCPAUSE_FISTPOINT is present. If so the fist 

3554 # point will just return. Otherwise, fall back on an abortable sleep. 

3555 

3556 if util.fistpoint.is_active(util.GCPAUSE_FISTPOINT): 

3557 

3558 util.fistpoint.activate_custom_fn(util.GCPAUSE_FISTPOINT, 3558 ↛ exitline 3558 didn't jump to the function exit

3559 lambda *args: None) 

3560 elif os.path.exists(_gc_init_file(sr.uuid)): 

3561 def abortTest(): 

3562 return IPCFlag(sr.uuid).test(FLAG_TYPE_ABORT) 

3563 

3564 # If time.sleep hangs we are in deep trouble, however for 

3565 # completeness we set the timeout of the abort thread to 

3566 # 110% of GCPAUSE_DEFAULT_SLEEP. 

3567 Util.log("GC active, about to go quiet") 

3568 Util.runAbortable(lambda: time.sleep(GCPAUSE_DEFAULT_SLEEP), 3568 ↛ exitline 3568 didn't run the lambda on line 3568

3569 None, sr.uuid, abortTest, VDI.POLL_INTERVAL, 

3570 GCPAUSE_DEFAULT_SLEEP * 1.1) 

3571 Util.log("GC active, quiet period ended") 

3572 

3573 

3574def _gcLoop(sr, dryRun=False, immediate=False): 

3575 if not lockGCActive.acquireNoblock(): 3575 ↛ 3576line 3575 didn't jump to line 3576, because the condition on line 3575 was never true

3576 Util.log("Another GC instance already active, exiting") 

3577 return 

3578 

3579 # Check we're still attached after acquiring locks 

3580 if not sr.xapi.isPluggedHere(): 

3581 Util.log("SR no longer attached, exiting") 

3582 return 

3583 

3584 # Clean up Intellicache files 

3585 sr.cleanupCache() 

3586 

3587 # Track how many we do 

3588 coalesced = 0 

3589 task_status = "success" 

3590 try: 

3591 # Check if any work needs to be done 

3592 if not sr.xapi.isPluggedHere(): 3592 ↛ 3593line 3592 didn't jump to line 3593, because the condition on line 3592 was never true

3593 Util.log("SR no longer attached, exiting") 

3594 return 

3595 sr.scanLocked() 

3596 if not sr.hasWork(): 

3597 Util.log("No work, exiting") 

3598 return 

3599 sr.xapi.create_task( 

3600 "Garbage Collection", 

3601 "Garbage collection for SR %s" % sr.uuid) 

3602 _gcLoopPause(sr, dryRun, immediate=immediate) 

3603 while True: 

3604 if not sr.xapi.isPluggedHere(): 3604 ↛ 3605line 3604 didn't jump to line 3605, because the condition on line 3604 was never true

3605 Util.log("SR no longer attached, exiting") 

3606 break 

3607 sr.scanLocked() 

3608 if not sr.hasWork(): 

3609 Util.log("No work, exiting") 

3610 break 

3611 

3612 if not lockGCRunning.acquireNoblock(): 3612 ↛ 3613line 3612 didn't jump to line 3613, because the condition on line 3612 was never true

3613 Util.log("Unable to acquire GC running lock.") 

3614 return 

3615 try: 

3616 if not sr.gcEnabled(): 3616 ↛ 3617line 3616 didn't jump to line 3617, because the condition on line 3616 was never true

3617 break 

3618 

3619 sr.xapi.update_task_progress("done", coalesced) 

3620 

3621 sr.cleanupCoalesceJournals() 

3622 # Create the init file here in case startup is waiting on it 

3623 _create_init_file(sr.uuid) 

3624 sr.scanLocked() 

3625 sr.updateBlockInfo() 

3626 

3627 howmany = len(sr.findGarbage()) 

3628 if howmany > 0: 

3629 Util.log("Found %d orphaned vdis" % howmany) 

3630 sr.lock() 

3631 try: 

3632 sr.garbageCollect(dryRun) 

3633 finally: 

3634 sr.unlock() 

3635 sr.xapi.srUpdate() 

3636 

3637 candidate = sr.findCoalesceable() 

3638 if candidate: 

3639 util.fistpoint.activate( 

3640 "LVHDRT_finding_a_suitable_pair", sr.uuid) 

3641 sr.coalesce(candidate, dryRun) 

3642 sr.xapi.srUpdate() 

3643 coalesced += 1 

3644 continue 

3645 

3646 candidate = sr.findLeafCoalesceable() 

3647 if candidate: 3647 ↛ 3654line 3647 didn't jump to line 3654, because the condition on line 3647 was never false

3648 sr.coalesceLeaf(candidate, dryRun) 

3649 sr.xapi.srUpdate() 

3650 coalesced += 1 

3651 continue 

3652 

3653 finally: 

3654 lockGCRunning.release() 3654 ↛ 3659line 3654 didn't jump to line 3659, because the break on line 3617 wasn't executed

3655 except: 

3656 task_status = "failure" 

3657 raise 

3658 finally: 

3659 sr.xapi.set_task_status(task_status) 

3660 Util.log("GC process exiting, no work left") 

3661 _create_init_file(sr.uuid) 

3662 lockGCActive.release() 

3663 

3664 

3665def _xapi_enabled(session, hostref): 

3666 host = session.xenapi.host.get_record(hostref) 

3667 return host['enabled'] 

3668 

3669 

3670def _ensure_xapi_initialised(session): 

3671 """ 

3672 Don't want to start GC until Xapi is fully initialised 

3673 """ 

3674 local_session = None 

3675 if session is None: 

3676 local_session = util.get_localAPI_session() 

3677 session = local_session 

3678 

3679 try: 

3680 hostref = session.xenapi.host.get_by_uuid(util.get_this_host()) 

3681 while not _xapi_enabled(session, hostref): 

3682 util.SMlog("Xapi not ready, GC waiting") 

3683 time.sleep(15) 

3684 finally: 

3685 if local_session is not None: 

3686 local_session.xenapi.session.logout() 

3687 

3688def _gc(session, srUuid, dryRun=False, immediate=False): 

3689 init(srUuid) 

3690 _ensure_xapi_initialised(session) 

3691 sr = SR.getInstance(srUuid, session) 

3692 if not sr.gcEnabled(False): 3692 ↛ 3693line 3692 didn't jump to line 3693, because the condition on line 3692 was never true

3693 return 

3694 

3695 try: 

3696 _gcLoop(sr, dryRun, immediate=immediate) 

3697 finally: 

3698 sr.cleanup() 

3699 sr.logFilter.logState() 

3700 del sr.xapi 

3701 

3702 

3703def _abort(srUuid, soft=False): 

3704 """Aborts an GC/coalesce. 

3705 

3706 srUuid: the UUID of the SR whose GC/coalesce must be aborted 

3707 soft: If set to True and there is a pending abort signal, the function 

3708 doesn't do anything. If set to False, a new abort signal is issued. 

3709 

3710 returns: If soft is set to False, we return True holding lockGCActive. If 

3711 soft is set to False and an abort signal is pending, we return False 

3712 without holding lockGCActive. An exception is raised in case of error.""" 

3713 Util.log("=== SR %s: abort ===" % (srUuid)) 

3714 init(srUuid) 

3715 if not lockGCActive.acquireNoblock(): 

3716 gotLock = False 

3717 Util.log("Aborting currently-running instance (SR %s)" % srUuid) 

3718 abortFlag = IPCFlag(srUuid) 

3719 if not abortFlag.set(FLAG_TYPE_ABORT, soft): 

3720 return False 

3721 for i in range(SR.LOCK_RETRY_ATTEMPTS): 

3722 gotLock = lockGCActive.acquireNoblock() 

3723 if gotLock: 

3724 break 

3725 time.sleep(SR.LOCK_RETRY_INTERVAL) 

3726 abortFlag.clear(FLAG_TYPE_ABORT) 

3727 if not gotLock: 

3728 raise util.CommandException(code=errno.ETIMEDOUT, 

3729 reason="SR %s: error aborting existing process" % srUuid) 

3730 return True 

3731 

3732 

3733def init(srUuid): 

3734 global lockGCRunning 

3735 if not lockGCRunning: 3735 ↛ 3736line 3735 didn't jump to line 3736, because the condition on line 3735 was never true

3736 lockGCRunning = lock.Lock(lock.LOCK_TYPE_GC_RUNNING, srUuid) 

3737 global lockGCActive 

3738 if not lockGCActive: 3738 ↛ 3739line 3738 didn't jump to line 3739, because the condition on line 3738 was never true

3739 lockGCActive = LockActive(srUuid) 

3740 

3741 

3742class LockActive: 

3743 """ 

3744 Wraps the use of LOCK_TYPE_GC_ACTIVE such that the lock cannot be acquired 

3745 if another process holds the SR lock. 

3746 """ 

3747 def __init__(self, srUuid): 

3748 self._lock = lock.Lock(LOCK_TYPE_GC_ACTIVE, srUuid) 

3749 self._srLock = lock.Lock(vhdutil.LOCK_TYPE_SR, srUuid) 

3750 

3751 def acquireNoblock(self): 

3752 self._srLock.acquire() 

3753 

3754 try: 

3755 return self._lock.acquireNoblock() 

3756 finally: 

3757 self._srLock.release() 

3758 

3759 def release(self): 

3760 self._lock.release() 

3761 

3762 

3763def usage(): 

3764 output = """Garbage collect and/or coalesce VHDs in a VHD-based SR 

3765 

3766Parameters: 

3767 -u --uuid UUID SR UUID 

3768 and one of: 

3769 -g --gc garbage collect, coalesce, and repeat while there is work 

3770 -G --gc_force garbage collect once, aborting any current operations 

3771 -c --cache-clean <max_age> clean up IntelliCache cache files older than 

3772 max_age hours 

3773 -a --abort abort any currently running operation (GC or coalesce) 

3774 -q --query query the current state (GC'ing, coalescing or not running) 

3775 -x --disable disable GC/coalesce (will be in effect until you exit) 

3776 -t --debug see Debug below 

3777 

3778Options: 

3779 -b --background run in background (return immediately) (valid for -g only) 

3780 -f --force continue in the presence of VHDs with errors (when doing 

3781 GC, this might cause removal of any such VHDs) (only valid 

3782 for -G) (DANGEROUS) 

3783 

3784Debug: 

3785 The --debug parameter enables manipulation of LVHD VDIs for debugging 

3786 purposes. ** NEVER USE IT ON A LIVE VM ** 

3787 The following parameters are required: 

3788 -t --debug <cmd> <cmd> is one of "activate", "deactivate", "inflate", 

3789 "deflate". 

3790 -v --vdi_uuid VDI UUID 

3791 """ 

3792 #-d --dry-run don't actually perform any SR-modifying operations 

3793 print(output) 

3794 Util.log("(Invalid usage)") 

3795 sys.exit(1) 

3796 

3797 

3798############################################################################## 

3799# 

3800# API 

3801# 

3802def abort(srUuid, soft=False): 

3803 """Abort GC/coalesce if we are currently GC'ing or coalescing a VDI pair. 

3804 """ 

3805 if _abort(srUuid, soft): 

3806 Util.log("abort: releasing the process lock") 

3807 lockGCActive.release() 

3808 return True 

3809 else: 

3810 return False 

3811 

3812 

3813def gc(session, srUuid, inBackground, dryRun=False): 

3814 """Garbage collect all deleted VDIs in SR "srUuid". Fork & return 

3815 immediately if inBackground=True. 

3816 

3817 The following algorithm is used: 

3818 1. If we are already GC'ing in this SR, return 

3819 2. If we are already coalescing a VDI pair: 

3820 a. Scan the SR and determine if the VDI pair is GC'able 

3821 b. If the pair is not GC'able, return 

3822 c. If the pair is GC'able, abort coalesce 

3823 3. Scan the SR 

3824 4. If there is nothing to collect, nor to coalesce, return 

3825 5. If there is something to collect, GC all, then goto 3 

3826 6. If there is something to coalesce, coalesce one pair, then goto 3 

3827 """ 

3828 Util.log("=== SR %s: gc ===" % srUuid) 

3829 if inBackground: 

3830 if daemonize(): 3830 ↛ exitline 3830 didn't return from function 'gc', because the condition on line 3830 was never false

3831 # we are now running in the background. Catch & log any errors 

3832 # because there is no other way to propagate them back at this 

3833 # point 

3834 

3835 try: 

3836 _gc(None, srUuid, dryRun) 

3837 except AbortException: 

3838 Util.log("Aborted") 

3839 except Exception: 

3840 Util.logException("gc") 

3841 Util.log("* * * * * SR %s: ERROR\n" % srUuid) 

3842 os._exit(0) 

3843 else: 

3844 _gc(session, srUuid, dryRun, immediate=True) 

3845 

3846 

3847def start_gc(session, sr_uuid): 

3848 """ 

3849 This function is used to try to start a backgrounded GC session by forking 

3850 the current process. If using the systemd version, call start_gc_service() instead. 

3851 """ 

3852 # don't bother if an instance already running (this is just an 

3853 # optimization to reduce the overhead of forking a new process if we 

3854 # don't have to, but the process will check the lock anyways) 

3855 lockRunning = lock.Lock(lock.LOCK_TYPE_GC_RUNNING, sr_uuid) 

3856 if not lockRunning.acquireNoblock(): 

3857 if should_preempt(session, sr_uuid): 

3858 util.SMlog("Aborting currently-running coalesce of garbage VDI") 

3859 try: 

3860 if not abort(sr_uuid, soft=True): 

3861 util.SMlog("The GC has already been scheduled to re-start") 

3862 except util.CommandException as e: 

3863 if e.code != errno.ETIMEDOUT: 

3864 raise 

3865 util.SMlog('failed to abort the GC') 

3866 else: 

3867 util.SMlog("A GC instance already running, not kicking") 

3868 return 

3869 else: 

3870 lockRunning.release() 

3871 

3872 util.SMlog(f"Starting GC file is {__file__}") 

3873 subprocess.run([__file__, '-b', '-u', sr_uuid, '-g'], 

3874 stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True) 

3875 

3876def start_gc_service(sr_uuid, wait=False): 

3877 """ 

3878 This starts the templated systemd service which runs GC on the given SR UUID. 

3879 If the service was already started, this is a no-op. 

3880 

3881 Because the service is a one-shot with RemainAfterExit=no, when called with 

3882 wait=True this will run the service synchronously and will not return until the 

3883 run has finished. This is used to force a run of the GC instead of just kicking it 

3884 in the background. 

3885 """ 

3886 sr_uuid_esc = sr_uuid.replace("-", "\\x2d") 

3887 util.SMlog(f"Kicking SMGC@{sr_uuid}...") 

3888 cmd=[ "/usr/bin/systemctl", "--quiet" ] 

3889 if not wait: 3889 ↛ 3891line 3889 didn't jump to line 3891, because the condition on line 3889 was never false

3890 cmd.append("--no-block") 

3891 cmd += ["start", f"SMGC@{sr_uuid_esc}"] 

3892 subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True) 

3893 

3894 

3895def gc_force(session, srUuid, force=False, dryRun=False, lockSR=False): 

3896 """Garbage collect all deleted VDIs in SR "srUuid". The caller must ensure 

3897 the SR lock is held. 

3898 The following algorithm is used: 

3899 1. If we are already GC'ing or coalescing a VDI pair, abort GC/coalesce 

3900 2. Scan the SR 

3901 3. GC 

3902 4. return 

3903 """ 

3904 Util.log("=== SR %s: gc_force ===" % srUuid) 

3905 init(srUuid) 

3906 sr = SR.getInstance(srUuid, session, lockSR, True) 

3907 if not lockGCActive.acquireNoblock(): 

3908 abort(srUuid) 

3909 else: 

3910 Util.log("Nothing was running, clear to proceed") 

3911 

3912 if force: 

3913 Util.log("FORCED: will continue even if there are VHD errors") 

3914 sr.scanLocked(force) 

3915 sr.cleanupCoalesceJournals() 

3916 

3917 try: 

3918 sr.cleanupCache() 

3919 sr.garbageCollect(dryRun) 

3920 finally: 

3921 sr.cleanup() 

3922 sr.logFilter.logState() 

3923 lockGCActive.release() 

3924 

3925 

3926def get_state(srUuid): 

3927 """Return whether GC/coalesce is currently running or not. This asks systemd for 

3928 the state of the templated SMGC service and will return True if it is "activating" 

3929 or "running" (for completeness, as in practice it will never achieve the latter state) 

3930 """ 

3931 sr_uuid_esc = srUuid.replace("-", "\\x2d") 

3932 cmd=[ "/usr/bin/systemctl", "is-active", f"SMGC@{sr_uuid_esc}"] 

3933 result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True) 

3934 state = result.stdout.decode('utf-8').rstrip() 

3935 if state == "activating" or state == "running": 

3936 return True 

3937 return False 

3938 

3939 

3940def should_preempt(session, srUuid): 

3941 sr = SR.getInstance(srUuid, session) 

3942 entries = sr.journaler.getAll(VDI.JRN_COALESCE) 

3943 if len(entries) == 0: 

3944 return False 

3945 elif len(entries) > 1: 

3946 raise util.SMException("More than one coalesce entry: " + str(entries)) 

3947 sr.scanLocked() 

3948 coalescedUuid = entries.popitem()[0] 

3949 garbage = sr.findGarbage() 

3950 for vdi in garbage: 

3951 if vdi.uuid == coalescedUuid: 

3952 return True 

3953 return False 

3954 

3955 

3956def get_coalesceable_leaves(session, srUuid, vdiUuids): 

3957 coalesceable = [] 

3958 sr = SR.getInstance(srUuid, session) 

3959 sr.scanLocked() 

3960 for uuid in vdiUuids: 

3961 vdi = sr.getVDI(uuid) 

3962 if not vdi: 

3963 raise util.SMException("VDI %s not found" % uuid) 

3964 if vdi.isLeafCoalesceable(): 

3965 coalesceable.append(uuid) 

3966 return coalesceable 

3967 

3968 

3969def cache_cleanup(session, srUuid, maxAge): 

3970 sr = SR.getInstance(srUuid, session) 

3971 return sr.cleanupCache(maxAge) 

3972 

3973 

3974def debug(sr_uuid, cmd, vdi_uuid): 

3975 Util.log("Debug command: %s" % cmd) 

3976 sr = SR.getInstance(sr_uuid, None) 

3977 if not isinstance(sr, LVHDSR): 

3978 print("Error: not an LVHD SR") 

3979 return 

3980 sr.scanLocked() 

3981 vdi = sr.getVDI(vdi_uuid) 

3982 if not vdi: 

3983 print("Error: VDI %s not found") 

3984 return 

3985 print("Running %s on SR %s" % (cmd, sr)) 

3986 print("VDI before: %s" % vdi) 

3987 if cmd == "activate": 

3988 vdi._activate() 

3989 print("VDI file: %s" % vdi.path) 

3990 if cmd == "deactivate": 

3991 ns = lvhdutil.NS_PREFIX_LVM + sr.uuid 

3992 sr.lvmCache.deactivate(ns, vdi.uuid, vdi.fileName, False) 

3993 if cmd == "inflate": 

3994 vdi.inflateFully() 

3995 sr.cleanup() 

3996 if cmd == "deflate": 

3997 vdi.deflate() 

3998 sr.cleanup() 

3999 sr.scanLocked() 

4000 print("VDI after: %s" % vdi) 

4001 

4002 

4003def abort_optional_reenable(uuid): 

4004 print("Disabling GC/coalesce for %s" % uuid) 

4005 ret = _abort(uuid) 

4006 input("Press enter to re-enable...") 

4007 print("GC/coalesce re-enabled") 

4008 lockGCRunning.release() 

4009 if ret: 

4010 lockGCActive.release() 

4011 

4012 

4013############################################################################## 

4014# 

4015# CLI 

4016# 

4017def main(): 

4018 action = "" 

4019 uuid = "" 

4020 background = False 

4021 force = False 

4022 dryRun = False 

4023 debug_cmd = "" 

4024 vdi_uuid = "" 

4025 shortArgs = "gGc:aqxu:bfdt:v:" 

4026 longArgs = ["gc", "gc_force", "clean_cache", "abort", "query", "disable", 

4027 "uuid=", "background", "force", "dry-run", "debug=", "vdi_uuid="] 

4028 

4029 try: 

4030 opts, args = getopt.getopt(sys.argv[1:], shortArgs, longArgs) 

4031 except getopt.GetoptError: 

4032 usage() 

4033 for o, a in opts: 

4034 if o in ("-g", "--gc"): 

4035 action = "gc" 

4036 if o in ("-G", "--gc_force"): 

4037 action = "gc_force" 

4038 if o in ("-c", "--clean_cache"): 

4039 action = "clean_cache" 

4040 maxAge = int(a) 

4041 if o in ("-a", "--abort"): 

4042 action = "abort" 

4043 if o in ("-q", "--query"): 

4044 action = "query" 

4045 if o in ("-x", "--disable"): 

4046 action = "disable" 

4047 if o in ("-u", "--uuid"): 

4048 uuid = a 

4049 if o in ("-b", "--background"): 

4050 background = True 

4051 if o in ("-f", "--force"): 

4052 force = True 

4053 if o in ("-d", "--dry-run"): 

4054 Util.log("Dry run mode") 

4055 dryRun = True 

4056 if o in ("-t", "--debug"): 

4057 action = "debug" 

4058 debug_cmd = a 

4059 if o in ("-v", "--vdi_uuid"): 

4060 vdi_uuid = a 

4061 

4062 if not action or not uuid: 

4063 usage() 

4064 if action == "debug" and not (debug_cmd and vdi_uuid) or \ 

4065 action != "debug" and (debug_cmd or vdi_uuid): 

4066 usage() 

4067 

4068 if action != "query" and action != "debug": 

4069 print("All output goes to log") 

4070 

4071 if action == "gc": 

4072 gc(None, uuid, background, dryRun) 

4073 elif action == "gc_force": 

4074 gc_force(None, uuid, force, dryRun, True) 

4075 elif action == "clean_cache": 

4076 cache_cleanup(None, uuid, maxAge) 

4077 elif action == "abort": 

4078 abort(uuid) 

4079 elif action == "query": 

4080 print("Currently running: %s" % get_state(uuid)) 

4081 elif action == "disable": 

4082 abort_optional_reenable(uuid) 

4083 elif action == "debug": 

4084 debug(uuid, debug_cmd, vdi_uuid) 

4085 

4086 

4087if __name__ == '__main__': 4087 ↛ 4088line 4087 didn't jump to line 4088, because the condition on line 4087 was never true

4088 main()