Package CedarBackup2 :: Module peer
[hide private]
[frames] | no frames]

Source Code for Module CedarBackup2.peer

   1  # -*- coding: iso-8859-1 -*- 
   2  # vim: set ft=python ts=3 sw=3 expandtab: 
   3  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
   4  # 
   5  #              C E D A R 
   6  #          S O L U T I O N S       "Software done right." 
   7  #           S O F T W A R E 
   8  # 
   9  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
  10  # 
  11  # Copyright (c) 2004-2008 Kenneth J. Pronovici. 
  12  # All rights reserved. 
  13  # 
  14  # This program is free software; you can redistribute it and/or 
  15  # modify it under the terms of the GNU General Public License, 
  16  # Version 2, as published by the Free Software Foundation. 
  17  # 
  18  # This program is distributed in the hope that it will be useful, 
  19  # but WITHOUT ANY WARRANTY; without even the implied warranty of 
  20  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 
  21  # 
  22  # Copies of the GNU General Public License are available from 
  23  # the Free Software Foundation website, http://www.gnu.org/. 
  24  # 
  25  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
  26  # 
  27  # Author   : Kenneth J. Pronovici <pronovic@ieee.org> 
  28  # Language : Python (>= 2.3) 
  29  # Project  : Cedar Backup, release 2 
  30  # Revision : $Id: peer.py 856 2008-03-17 02:43:34Z pronovic $ 
  31  # Purpose  : Provides backup peer-related objects. 
  32  # 
  33  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
  34   
  35  ######################################################################## 
  36  # Module documentation 
  37  ######################################################################## 
  38   
  39  """ 
  40  Provides backup peer-related objects and utility functions. 
  41   
  42  @sort: LocalPeer, Remote Peer 
  43   
  44  @var DEF_COLLECT_INDICATOR: Name of the default collect indicator file. 
  45  @var DEF_STAGE_INDICATOR: Name of the default stage indicator file. 
  46   
  47  @author: Kenneth J. Pronovici <pronovic@ieee.org> 
  48  """ 
  49   
  50   
  51  ######################################################################## 
  52  # Imported modules 
  53  ######################################################################## 
  54   
  55  # System modules 
  56  import os 
  57  import logging 
  58  import shutil 
  59  import sets 
  60  import re 
  61   
  62  # Cedar Backup modules 
  63  from CedarBackup2.filesystem import FilesystemList 
  64  from CedarBackup2.util import resolveCommand, executeCommand 
  65  from CedarBackup2.util import splitCommandLine, encodePath 
  66   
  67   
  68  ######################################################################## 
  69  # Module-wide constants and variables 
  70  ######################################################################## 
  71   
  72  logger                  = logging.getLogger("CedarBackup2.log.peer") 
  73   
  74  DEF_RCP_COMMAND         = [ "/usr/bin/scp", "-B", "-q", "-C" ] 
  75  DEF_RSH_COMMAND         = [ "/usr/bin/ssh", ] 
  76  DEF_CBACK_COMMAND       = "/usr/bin/cback" 
  77   
  78  DEF_COLLECT_INDICATOR   = "cback.collect" 
  79  DEF_STAGE_INDICATOR     = "cback.stage" 
  80   
  81  SU_COMMAND              = [ "su" ] 
  82   
  83   
  84  ######################################################################## 
  85  # LocalPeer class definition 
  86  ######################################################################## 
  87   
88 -class LocalPeer(object):
89 90 ###################### 91 # Class documentation 92 ###################### 93 94 """ 95 Backup peer representing a local peer in a backup pool. 96 97 This is a class representing a local (non-network) peer in a backup pool. 98 Local peers are backed up by simple filesystem copy operations. A local 99 peer has associated with it a name (typically, but not necessarily, a 100 hostname) and a collect directory. 101 102 The public methods other than the constructor are part of a "backup peer" 103 interface shared with the C{RemotePeer} class. 104 105 @sort: __init__, stagePeer, checkCollectIndicator, writeStageIndicator, 106 _copyLocalDir, _copyLocalFile, name, collectDir 107 """ 108 109 ############## 110 # Constructor 111 ############## 112
113 - def __init__(self, name, collectDir):
114 """ 115 Initializes a local backup peer. 116 117 Note that the collect directory must be an absolute path, but does not 118 have to exist when the object is instantiated. We do a lazy validation 119 on this value since we could (potentially) be creating peer objects 120 before an ongoing backup completed. 121 122 @param name: Name of the backup peer 123 @type name: String, typically a hostname 124 125 @param collectDir: Path to the peer's collect directory 126 @type collectDir: String representing an absolute local path on disk 127 128 @raise ValueError: If the name is empty. 129 @raise ValueError: If collect directory is not an absolute path. 130 """ 131 self._name = None 132 self._collectDir = None 133 self.name = name 134 self.collectDir = collectDir
135 136 137 ############# 138 # Properties 139 ############# 140
141 - def _setName(self, value):
142 """ 143 Property target used to set the peer name. 144 The value must be a non-empty string and cannot be C{None}. 145 @raise ValueError: If the value is an empty string or C{None}. 146 """ 147 if value is None or len(value) < 1: 148 raise ValueError("Peer name must be a non-empty string.") 149 self._name = value
150
151 - def _getName(self):
152 """ 153 Property target used to get the peer name. 154 """ 155 return self._name
156
157 - def _setCollectDir(self, value):
158 """ 159 Property target used to set the collect directory. 160 The value must be an absolute path and cannot be C{None}. 161 It does not have to exist on disk at the time of assignment. 162 @raise ValueError: If the value is C{None} or is not an absolute path. 163 @raise ValueError: If a path cannot be encoded properly. 164 """ 165 if value is None or not os.path.isabs(value): 166 raise ValueError("Collect directory must be an absolute path.") 167 self._collectDir = encodePath(value)
168
169 - def _getCollectDir(self):
170 """ 171 Property target used to get the collect directory. 172 """ 173 return self._collectDir
174 175 name = property(_getName, _setName, None, "Name of the peer.") 176 collectDir = property(_getCollectDir, _setCollectDir, None, "Path to the peer's collect directory (an absolute local path).") 177 178 179 ################# 180 # Public methods 181 ################# 182
183 - def stagePeer(self, targetDir, ownership=None, permissions=None):
184 """ 185 Stages data from the peer into the indicated local target directory. 186 187 The collect and target directories must both already exist before this 188 method is called. If passed in, ownership and permissions will be 189 applied to the files that are copied. 190 191 @note: The caller is responsible for checking that the indicator exists, 192 if they care. This function only stages the files within the directory. 193 194 @note: If you have user/group as strings, call the L{util.getUidGid} function 195 to get the associated uid/gid as an ownership tuple. 196 197 @param targetDir: Target directory to write data into 198 @type targetDir: String representing a directory on disk 199 200 @param ownership: Owner and group that the staged files should have 201 @type ownership: Tuple of numeric ids C{(uid, gid)} 202 203 @param permissions: Permissions that the staged files should have 204 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}). 205 206 @return: Number of files copied from the source directory to the target directory. 207 208 @raise ValueError: If collect directory is not a directory or does not exist 209 @raise ValueError: If target directory is not a directory, does not exist or is not absolute. 210 @raise ValueError: If a path cannot be encoded properly. 211 @raise IOError: If there were no files to stage (i.e. the directory was empty) 212 @raise IOError: If there is an IO error copying a file. 213 @raise OSError: If there is an OS error copying or changing permissions on a file 214 """ 215 targetDir = encodePath(targetDir) 216 if not os.path.isabs(targetDir): 217 logger.debug("Target directory [%s] not an absolute path." % targetDir) 218 raise ValueError("Target directory must be an absolute path.") 219 if not os.path.exists(self.collectDir) or not os.path.isdir(self.collectDir): 220 logger.debug("Collect directory [%s] is not a directory or does not exist on disk." % self.collectDir) 221 raise ValueError("Collect directory is not a directory or does not exist on disk.") 222 if not os.path.exists(targetDir) or not os.path.isdir(targetDir): 223 logger.debug("Target directory [%s] is not a directory or does not exist on disk." % targetDir) 224 raise ValueError("Target directory is not a directory or does not exist on disk.") 225 count = LocalPeer._copyLocalDir(self.collectDir, targetDir, ownership, permissions) 226 if count == 0: 227 raise IOError("Did not copy any files from local peer.") 228 return count
229
230 - def checkCollectIndicator(self, collectIndicator=None):
231 """ 232 Checks the collect indicator in the peer's staging directory. 233 234 When a peer has completed collecting its backup files, it will write an 235 empty indicator file into its collect directory. This method checks to 236 see whether that indicator has been written. We're "stupid" here - if 237 the collect directory doesn't exist, you'll naturally get back C{False}. 238 239 If you need to, you can override the name of the collect indicator file 240 by passing in a different name. 241 242 @param collectIndicator: Name of the collect indicator file to check 243 @type collectIndicator: String representing name of a file in the collect directory 244 245 @return: Boolean true/false depending on whether the indicator exists. 246 @raise ValueError: If a path cannot be encoded properly. 247 """ 248 collectIndicator = encodePath(collectIndicator) 249 if collectIndicator is None: 250 return os.path.exists(os.path.join(self.collectDir, DEF_COLLECT_INDICATOR)) 251 else: 252 return os.path.exists(os.path.join(self.collectDir, collectIndicator))
253
254 - def writeStageIndicator(self, stageIndicator=None, ownership=None, permissions=None):
255 """ 256 Writes the stage indicator in the peer's staging directory. 257 258 When the master has completed collecting its backup files, it will write 259 an empty indicator file into the peer's collect directory. The presence 260 of this file implies that the staging process is complete. 261 262 If you need to, you can override the name of the stage indicator file by 263 passing in a different name. 264 265 @note: If you have user/group as strings, call the L{util.getUidGid} 266 function to get the associated uid/gid as an ownership tuple. 267 268 @param stageIndicator: Name of the indicator file to write 269 @type stageIndicator: String representing name of a file in the collect directory 270 271 @param ownership: Owner and group that the indicator file should have 272 @type ownership: Tuple of numeric ids C{(uid, gid)} 273 274 @param permissions: Permissions that the indicator file should have 275 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}). 276 277 @raise ValueError: If collect directory is not a directory or does not exist 278 @raise ValueError: If a path cannot be encoded properly. 279 @raise IOError: If there is an IO error creating the file. 280 @raise OSError: If there is an OS error creating or changing permissions on the file 281 """ 282 stageIndicator = encodePath(stageIndicator) 283 if not os.path.exists(self.collectDir) or not os.path.isdir(self.collectDir): 284 logger.debug("Collect directory [%s] is not a directory or does not exist on disk." % self.collectDir) 285 raise ValueError("Collect directory is not a directory or does not exist on disk.") 286 if stageIndicator is None: 287 fileName = os.path.join(self.collectDir, DEF_STAGE_INDICATOR) 288 else: 289 fileName = os.path.join(self.collectDir, stageIndicator) 290 LocalPeer._copyLocalFile(None, fileName, ownership, permissions) # None for sourceFile results in an empty target
291 292 293 ################## 294 # Private methods 295 ################## 296
297 - def _copyLocalDir(sourceDir, targetDir, ownership=None, permissions=None):
298 """ 299 Copies files from the source directory to the target directory. 300 301 This function is not recursive. Only the files in the directory will be 302 copied. Ownership and permissions will be left at their default values 303 if new values are not specified. The source and target directories are 304 allowed to be soft links to a directory, but besides that soft links are 305 ignored. 306 307 @note: If you have user/group as strings, call the L{util.getUidGid} 308 function to get the associated uid/gid as an ownership tuple. 309 310 @param sourceDir: Source directory 311 @type sourceDir: String representing a directory on disk 312 313 @param targetDir: Target directory 314 @type targetDir: String representing a directory on disk 315 316 @param ownership: Owner and group that the copied files should have 317 @type ownership: Tuple of numeric ids C{(uid, gid)} 318 319 @param permissions: Permissions that the staged files should have 320 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}). 321 322 @return: Number of files copied from the source directory to the target directory. 323 324 @raise ValueError: If source or target is not a directory or does not exist. 325 @raise ValueError: If a path cannot be encoded properly. 326 @raise IOError: If there is an IO error copying the files. 327 @raise OSError: If there is an OS error copying or changing permissions on a files 328 """ 329 filesCopied = 0 330 sourceDir = encodePath(sourceDir) 331 targetDir = encodePath(targetDir) 332 for fileName in os.listdir(sourceDir): 333 sourceFile = os.path.join(sourceDir, fileName) 334 targetFile = os.path.join(targetDir, fileName) 335 LocalPeer._copyLocalFile(sourceFile, targetFile, ownership, permissions) 336 filesCopied += 1 337 return filesCopied
338 _copyLocalDir = staticmethod(_copyLocalDir) 339
340 - def _copyLocalFile(sourceFile=None, targetFile=None, ownership=None, permissions=None, overwrite=True):
341 """ 342 Copies a source file to a target file. 343 344 If the source file is C{None} then the target file will be created or 345 overwritten as an empty file. If the target file is C{None}, this method 346 is a no-op. Attempting to copy a soft link or a directory will result in 347 an exception. 348 349 @note: If you have user/group as strings, call the L{util.getUidGid} 350 function to get the associated uid/gid as an ownership tuple. 351 352 @note: We will not overwrite a target file that exists when this method 353 is invoked. If the target already exists, we'll raise an exception. 354 355 @param sourceFile: Source file to copy 356 @type sourceFile: String representing a file on disk, as an absolute path 357 358 @param targetFile: Target file to create 359 @type targetFile: String representing a file on disk, as an absolute path 360 361 @param ownership: Owner and group that the copied should have 362 @type ownership: Tuple of numeric ids C{(uid, gid)} 363 364 @param permissions: Permissions that the staged files should have 365 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}). 366 367 @param overwrite: Indicates whether it's OK to overwrite the target file. 368 @type overwrite: Boolean true/false. 369 370 @raise ValueError: If the passed-in source file is not a regular file. 371 @raise ValueError: If a path cannot be encoded properly. 372 @raise IOError: If the target file already exists. 373 @raise IOError: If there is an IO error copying the file 374 @raise OSError: If there is an OS error copying or changing permissions on a file 375 """ 376 targetFile = encodePath(targetFile) 377 sourceFile = encodePath(sourceFile) 378 if targetFile is None: 379 return 380 if not overwrite: 381 if os.path.exists(targetFile): 382 raise IOError("Target file [%s] already exists." % targetFile) 383 if sourceFile is None: 384 open(targetFile, "w").write("") 385 else: 386 if os.path.isfile(sourceFile) and not os.path.islink(sourceFile): 387 shutil.copy(sourceFile, targetFile) 388 else: 389 logger.debug("Source [%s] is not a regular file." % sourceFile) 390 raise ValueError("Source is not a regular file.") 391 if ownership is not None: 392 os.chown(targetFile, ownership[0], ownership[1]) 393 if permissions is not None: 394 os.chmod(targetFile, permissions)
395 _copyLocalFile = staticmethod(_copyLocalFile)
396 397 398 ######################################################################## 399 # RemotePeer class definition 400 ######################################################################## 401
402 -class RemotePeer(object):
403 404 ###################### 405 # Class documentation 406 ###################### 407 408 """ 409 Backup peer representing a remote peer in a backup pool. 410 411 This is a class representing a remote (networked) peer in a backup pool. 412 Remote peers are backed up using an rcp-compatible copy command. A remote 413 peer has associated with it a name (which must be a valid hostname), a 414 collect directory, a working directory and a copy method (an rcp-compatible 415 command). 416 417 You can also set an optional local user value. This username will be used 418 as the local user for any remote copies that are required. It can only be 419 used if the root user is executing the backup. The root user will C{su} to 420 the local user and execute the remote copies as that user. 421 422 The copy method is associated with the peer and not with the actual request 423 to copy, because we can envision that each remote host might have a 424 different connect method. 425 426 The public methods other than the constructor are part of a "backup peer" 427 interface shared with the C{LocalPeer} class. 428 429 @sort: __init__, stagePeer, checkCollectIndicator, writeStageIndicator, 430 executeRemoteCommand, executeManagedAction, _getDirContents, 431 _copyRemoteDir, _copyRemoteFile, _pushLocalFile, name, collectDir, 432 remoteUser, rcpCommand, rshCommand, cbackCommand 433 """ 434 435 ############## 436 # Constructor 437 ############## 438
439 - def __init__(self, name=None, collectDir=None, workingDir=None, remoteUser=None, 440 rcpCommand=None, localUser=None, rshCommand=None, cbackCommand=None):
441 """ 442 Initializes a remote backup peer. 443 444 @note: If provided, each command will eventually be parsed into a list of 445 strings suitable for passing to C{util.executeCommand} in order to avoid 446 security holes related to shell interpolation. This parsing will be 447 done by the L{util.splitCommandLine} function. See the documentation for 448 that function for some important notes about its limitations. 449 450 @param name: Name of the backup peer 451 @type name: String, must be a valid DNS hostname 452 453 @param collectDir: Path to the peer's collect directory 454 @type collectDir: String representing an absolute path on the remote peer 455 456 @param workingDir: Working directory that can be used to create temporary files, etc. 457 @type workingDir: String representing an absolute path on the current host. 458 459 @param remoteUser: Name of the Cedar Backup user on the remote peer 460 @type remoteUser: String representing a username, valid via remote shell to the peer 461 462 @param localUser: Name of the Cedar Backup user on the current host 463 @type localUser: String representing a username, valid on the current host 464 465 @param rcpCommand: An rcp-compatible copy command to use for copying files from the peer 466 @type rcpCommand: String representing a system command including required arguments 467 468 @param rshCommand: An rsh-compatible copy command to use for remote shells to the peer 469 @type rshCommand: String representing a system command including required arguments 470 471 @param cbackCommand: A chack-compatible command to use for executing managed actions 472 @type cbackCommand: String representing a system command including required arguments 473 474 @raise ValueError: If collect directory is not an absolute path 475 """ 476 self._name = None 477 self._collectDir = None 478 self._workingDir = None 479 self._remoteUser = None 480 self._localUser = None 481 self._rcpCommand = None 482 self._rcpCommandList = None 483 self._rshCommand = None 484 self._rshCommandList = None 485 self._cbackCommand = None 486 self.name = name 487 self.collectDir = collectDir 488 self.workingDir = workingDir 489 self.remoteUser = remoteUser 490 self.localUser = localUser 491 self.rcpCommand = rcpCommand 492 self.rshCommand = rshCommand 493 self.cbackCommand = cbackCommand
494 495 496 ############# 497 # Properties 498 ############# 499
500 - def _setName(self, value):
501 """ 502 Property target used to set the peer name. 503 The value must be a non-empty string and cannot be C{None}. 504 @raise ValueError: If the value is an empty string or C{None}. 505 """ 506 if value is None or len(value) < 1: 507 raise ValueError("Peer name must be a non-empty string.") 508 self._name = value
509
510 - def _getName(self):
511 """ 512 Property target used to get the peer name. 513 """ 514 return self._name
515
516 - def _setCollectDir(self, value):
517 """ 518 Property target used to set the collect directory. 519 The value must be an absolute path and cannot be C{None}. 520 It does not have to exist on disk at the time of assignment. 521 @raise ValueError: If the value is C{None} or is not an absolute path. 522 @raise ValueError: If the value cannot be encoded properly. 523 """ 524 if value is not None: 525 if not os.path.isabs(value): 526 raise ValueError("Collect directory must be an absolute path.") 527 self._collectDir = encodePath(value)
528
529 - def _getCollectDir(self):
530 """ 531 Property target used to get the collect directory. 532 """ 533 return self._collectDir
534
535 - def _setWorkingDir(self, value):
536 """ 537 Property target used to set the working directory. 538 The value must be an absolute path and cannot be C{None}. 539 @raise ValueError: If the value is C{None} or is not an absolute path. 540 @raise ValueError: If the value cannot be encoded properly. 541 """ 542 if value is not None: 543 if not os.path.isabs(value): 544 raise ValueError("Working directory must be an absolute path.") 545 self._workingDir = encodePath(value)
546
547 - def _getWorkingDir(self):
548 """ 549 Property target used to get the working directory. 550 """ 551 return self._workingDir
552
553 - def _setRemoteUser(self, value):
554 """ 555 Property target used to set the remote user. 556 The value must be a non-empty string and cannot be C{None}. 557 @raise ValueError: If the value is an empty string or C{None}. 558 """ 559 if value is None or len(value) < 1: 560 raise ValueError("Peer remote user must be a non-empty string.") 561 self._remoteUser = value
562
563 - def _getRemoteUser(self):
564 """ 565 Property target used to get the remote user. 566 """ 567 return self._remoteUser
568
569 - def _setLocalUser(self, value):
570 """ 571 Property target used to set the local user. 572 The value must be a non-empty string if it is not C{None}. 573 @raise ValueError: If the value is an empty string. 574 """ 575 if value is not None: 576 if len(value) < 1: 577 raise ValueError("Peer local user must be a non-empty string.") 578 self._localUser = value
579
580 - def _getLocalUser(self):
581 """ 582 Property target used to get the local user. 583 """ 584 return self._localUser
585
586 - def _setRcpCommand(self, value):
587 """ 588 Property target to set the rcp command. 589 590 The value must be a non-empty string or C{None}. Its value is stored in 591 the two forms: "raw" as provided by the client, and "parsed" into a list 592 suitable for being passed to L{util.executeCommand} via 593 L{util.splitCommandLine}. 594 595 However, all the caller will ever see via the property is the actual 596 value they set (which includes seeing C{None}, even if we translate that 597 internally to C{DEF_RCP_COMMAND}). Internally, we should always use 598 C{self._rcpCommandList} if we want the actual command list. 599 600 @raise ValueError: If the value is an empty string. 601 """ 602 if value is None: 603 self._rcpCommand = None 604 self._rcpCommandList = DEF_RCP_COMMAND 605 else: 606 if len(value) >= 1: 607 self._rcpCommand = value 608 self._rcpCommandList = splitCommandLine(self._rcpCommand) 609 else: 610 raise ValueError("The rcp command must be a non-empty string.")
611
612 - def _getRcpCommand(self):
613 """ 614 Property target used to get the rcp command. 615 """ 616 return self._rcpCommand
617
618 - def _setRshCommand(self, value):
619 """ 620 Property target to set the rsh command. 621 622 The value must be a non-empty string or C{None}. Its value is stored in 623 the two forms: "raw" as provided by the client, and "parsed" into a list 624 suitable for being passed to L{util.executeCommand} via 625 L{util.splitCommandLine}. 626 627 However, all the caller will ever see via the property is the actual 628 value they set (which includes seeing C{None}, even if we translate that 629 internally to C{DEF_RSH_COMMAND}). Internally, we should always use 630 C{self._rshCommandList} if we want the actual command list. 631 632 @raise ValueError: If the value is an empty string. 633 """ 634 if value is None: 635 self._rshCommand = None 636 self._rshCommandList = DEF_RSH_COMMAND 637 else: 638 if len(value) >= 1: 639 self._rshCommand = value 640 self._rshCommandList = splitCommandLine(self._rshCommand) 641 else: 642 raise ValueError("The rsh command must be a non-empty string.")
643
644 - def _getRshCommand(self):
645 """ 646 Property target used to get the rsh command. 647 """ 648 return self._rshCommand
649
650 - def _setCbackCommand(self, value):
651 """ 652 Property target to set the cback command. 653 654 The value must be a non-empty string or C{None}. Unlike the other 655 command, this value is only stored in the "raw" form provided by the 656 client. 657 658 @raise ValueError: If the value is an empty string. 659 """ 660 if value is None: 661 self._cbackCommand = None 662 else: 663 if len(value) >= 1: 664 self._cbackCommand = value 665 else: 666 raise ValueError("The cback command must be a non-empty string.")
667
668 - def _getCbackCommand(self):
669 """ 670 Property target used to get the cback command. 671 """ 672 return self._cbackCommand
673 674 name = property(_getName, _setName, None, "Name of the peer (a valid DNS hostname).") 675 collectDir = property(_getCollectDir, _setCollectDir, None, "Path to the peer's collect directory (an absolute local path).") 676 workingDir = property(_getWorkingDir, _setWorkingDir, None, "Path to the peer's working directory (an absolute local path).") 677 remoteUser = property(_getRemoteUser, _setRemoteUser, None, "Name of the Cedar Backup user on the remote peer.") 678 localUser = property(_getLocalUser, _setLocalUser, None, "Name of the Cedar Backup user on the current host.") 679 rcpCommand = property(_getRcpCommand, _setRcpCommand, None, "An rcp-compatible copy command to use for copying files.") 680 rshCommand = property(_getRshCommand, _setRshCommand, None, "An rsh-compatible command to use for remote shells to the peer.") 681 cbackCommand = property(_getCbackCommand, _setCbackCommand, None, "A chack-compatible command to use for executing managed actions.") 682 683 684 ################# 685 # Public methods 686 ################# 687
688 - def stagePeer(self, targetDir, ownership=None, permissions=None):
689 """ 690 Stages data from the peer into the indicated local target directory. 691 692 The target directory must already exist before this method is called. If 693 passed in, ownership and permissions will be applied to the files that 694 are copied. 695 696 @note: The returned count of copied files might be inaccurate if some of 697 the copied files already existed in the staging directory prior to the 698 copy taking place. We don't clear the staging directory first, because 699 some extension might also be using it. 700 701 @note: If you have user/group as strings, call the L{util.getUidGid} function 702 to get the associated uid/gid as an ownership tuple. 703 704 @note: Unlike the local peer version of this method, an I/O error might 705 or might not be raised if the directory is empty. Since we're using a 706 remote copy method, we just don't have the fine-grained control over our 707 exceptions that's available when we can look directly at the filesystem, 708 and we can't control whether the remote copy method thinks an empty 709 directory is an error. 710 711 @param targetDir: Target directory to write data into 712 @type targetDir: String representing a directory on disk 713 714 @param ownership: Owner and group that the staged files should have 715 @type ownership: Tuple of numeric ids C{(uid, gid)} 716 717 @param permissions: Permissions that the staged files should have 718 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}). 719 720 @return: Number of files copied from the source directory to the target directory. 721 722 @raise ValueError: If target directory is not a directory, does not exist or is not absolute. 723 @raise ValueError: If a path cannot be encoded properly. 724 @raise IOError: If there were no files to stage (i.e. the directory was empty) 725 @raise IOError: If there is an IO error copying a file. 726 @raise OSError: If there is an OS error copying or changing permissions on a file 727 """ 728 targetDir = encodePath(targetDir) 729 if not os.path.isabs(targetDir): 730 logger.debug("Target directory [%s] not an absolute path." % targetDir) 731 raise ValueError("Target directory must be an absolute path.") 732 if not os.path.exists(targetDir) or not os.path.isdir(targetDir): 733 logger.debug("Target directory [%s] is not a directory or does not exist on disk." % targetDir) 734 raise ValueError("Target directory is not a directory or does not exist on disk.") 735 count = RemotePeer._copyRemoteDir(self.remoteUser, self.localUser, self.name, 736 self._rcpCommand, self._rcpCommandList, 737 self.collectDir, targetDir, 738 ownership, permissions) 739 if count == 0: 740 raise IOError("Did not copy any files from local peer.") 741 return count
742
743 - def checkCollectIndicator(self, collectIndicator=None):
744 """ 745 Checks the collect indicator in the peer's staging directory. 746 747 When a peer has completed collecting its backup files, it will write an 748 empty indicator file into its collect directory. This method checks to 749 see whether that indicator has been written. If the remote copy command 750 fails, we return C{False} as if the file weren't there. 751 752 If you need to, you can override the name of the collect indicator file 753 by passing in a different name. 754 755 @note: Apparently, we can't count on all rcp-compatible implementations 756 to return sensible errors for some error conditions. As an example, the 757 C{scp} command in Debian 'woody' returns a zero (normal) status even when 758 it can't find a host or if the login or path is invalid. Because of 759 this, the implementation of this method is rather convoluted. 760 761 @param collectIndicator: Name of the collect indicator file to check 762 @type collectIndicator: String representing name of a file in the collect directory 763 764 @return: Boolean true/false depending on whether the indicator exists. 765 @raise ValueError: If a path cannot be encoded properly. 766 """ 767 try: 768 if collectIndicator is None: 769 sourceFile = os.path.join(self.collectDir, DEF_COLLECT_INDICATOR) 770 targetFile = os.path.join(self.workingDir, DEF_COLLECT_INDICATOR) 771 else: 772 collectIndicator = encodePath(collectIndicator) 773 sourceFile = os.path.join(self.collectDir, collectIndicator) 774 targetFile = os.path.join(self.workingDir, collectIndicator) 775 logger.debug("Fetch remote [%s] into [%s]." % (sourceFile, targetFile)) 776 if os.path.exists(targetFile): 777 try: 778 os.remove(targetFile) 779 except: 780 raise Exception("Error: collect indicator [%s] already exists!" % targetFile) 781 try: 782 RemotePeer._copyRemoteFile(self.remoteUser, self.localUser, self.name, 783 self._rcpCommand, self._rcpCommandList, 784 sourceFile, targetFile, 785 overwrite=False) 786 if os.path.exists(targetFile): 787 return True 788 else: 789 return False 790 except Exception, e: 791 logger.info("Failed looking for collect indicator: %s" % e) 792 return False 793 finally: 794 if os.path.exists(targetFile): 795 try: 796 os.remove(targetFile) 797 except: pass
798
799 - def writeStageIndicator(self, stageIndicator=None):
800 """ 801 Writes the stage indicator in the peer's staging directory. 802 803 When the master has completed collecting its backup files, it will write 804 an empty indicator file into the peer's collect directory. The presence 805 of this file implies that the staging process is complete. 806 807 If you need to, you can override the name of the stage indicator file by 808 passing in a different name. 809 810 @note: If you have user/group as strings, call the L{util.getUidGid} function 811 to get the associated uid/gid as an ownership tuple. 812 813 @param stageIndicator: Name of the indicator file to write 814 @type stageIndicator: String representing name of a file in the collect directory 815 816 @raise ValueError: If a path cannot be encoded properly. 817 @raise IOError: If there is an IO error creating the file. 818 @raise OSError: If there is an OS error creating or changing permissions on the file 819 """ 820 stageIndicator = encodePath(stageIndicator) 821 if stageIndicator is None: 822 sourceFile = os.path.join(self.workingDir, DEF_STAGE_INDICATOR) 823 targetFile = os.path.join(self.collectDir, DEF_STAGE_INDICATOR) 824 else: 825 sourceFile = os.path.join(self.workingDir, DEF_STAGE_INDICATOR) 826 targetFile = os.path.join(self.collectDir, stageIndicator) 827 try: 828 if not os.path.exists(sourceFile): 829 open(sourceFile, "w").write("") 830 RemotePeer._pushLocalFile(self.remoteUser, self.localUser, self.name, 831 self._rcpCommand, self._rcpCommandList, 832 sourceFile, targetFile) 833 finally: 834 if os.path.exists(sourceFile): 835 try: 836 os.remove(sourceFile) 837 except: pass
838
839 - def executeRemoteCommand(self, command):
840 """ 841 Executes a command on the peer via remote shell. 842 843 @param command: Command to execute 844 @type command: String command-line suitable for use with rsh. 845 846 @raise IOError: If there is an error executing the command on the remote peer. 847 """ 848 RemotePeer._executeRemoteCommand(self.remoteUser, self.localUser, 849 self.name, self._rshCommand, 850 self._rshCommandList, command)
851
852 - def executeManagedAction(self, action, fullBackup):
853 """ 854 Executes a managed action on this peer. 855 856 @param action: Name of the action to execute. 857 @param fullBackup: Whether a full backup should be executed. 858 859 @raise IOError: If there is an error executing the action on the remote peer. 860 """ 861 try: 862 command = RemotePeer._buildCbackCommand(self.cbackCommand, action, fullBackup) 863 self.executeRemoteCommand(command) 864 except IOError, e: 865 logger.info(e) 866 raise IOError("Failed to execute action [%s] on managed client [%s]." % (action, self.name))
867 868 869 ################## 870 # Private methods 871 ################## 872
873 - def _getDirContents(path):
874 """ 875 Returns the contents of a directory in terms of a Set. 876 877 The directory's contents are read as a L{FilesystemList} containing only 878 files, and then the list is converted into a C{sets.Set} object for later 879 use. 880 881 @param path: Directory path to get contents for 882 @type path: String representing a path on disk 883 884 @return: Set of files in the directory 885 @raise ValueError: If path is not a directory or does not exist. 886 """ 887 contents = FilesystemList() 888 contents.excludeDirs = True 889 contents.excludeLinks = True 890 contents.addDirContents(path) 891 return sets.Set(contents)
892 _getDirContents = staticmethod(_getDirContents) 893
894 - def _copyRemoteDir(remoteUser, localUser, remoteHost, rcpCommand, rcpCommandList, 895 sourceDir, targetDir, ownership=None, permissions=None):
896 """ 897 Copies files from the source directory to the target directory. 898 899 This function is not recursive. Only the files in the directory will be 900 copied. Ownership and permissions will be left at their default values 901 if new values are not specified. Behavior when copying soft links from 902 the collect directory is dependent on the behavior of the specified rcp 903 command. 904 905 @note: The returned count of copied files might be inaccurate if some of 906 the copied files already existed in the staging directory prior to the 907 copy taking place. We don't clear the staging directory first, because 908 some extension might also be using it. 909 910 @note: If you have user/group as strings, call the L{util.getUidGid} function 911 to get the associated uid/gid as an ownership tuple. 912 913 @note: We don't have a good way of knowing exactly what files we copied 914 down from the remote peer, unless we want to parse the output of the rcp 915 command (ugh). We could change permissions on everything in the target 916 directory, but that's kind of ugly too. Instead, we use Python's set 917 functionality to figure out what files were added while we executed the 918 rcp command. This isn't perfect - for instance, it's not correct if 919 someone else is messing with the directory at the same time we're doing 920 the remote copy - but it's about as good as we're going to get. 921 922 @note: Apparently, we can't count on all rcp-compatible implementations 923 to return sensible errors for some error conditions. As an example, the 924 C{scp} command in Debian 'woody' returns a zero (normal) status even 925 when it can't find a host or if the login or path is invalid. We try 926 to work around this by issuing C{IOError} if we don't copy any files from 927 the remote host. 928 929 @param remoteUser: Name of the Cedar Backup user on the remote peer 930 @type remoteUser: String representing a username, valid via the copy command 931 932 @param localUser: Name of the Cedar Backup user on the current host 933 @type localUser: String representing a username, valid on the current host 934 935 @param remoteHost: Hostname of the remote peer 936 @type remoteHost: String representing a hostname, accessible via the copy command 937 938 @param rcpCommand: An rcp-compatible copy command to use for copying files from the peer 939 @type rcpCommand: String representing a system command including required arguments 940 941 @param rcpCommandList: An rcp-compatible copy command to use for copying files 942 @type rcpCommandList: Command as a list to be passed to L{util.executeCommand} 943 944 @param sourceDir: Source directory 945 @type sourceDir: String representing a directory on disk 946 947 @param targetDir: Target directory 948 @type targetDir: String representing a directory on disk 949 950 @param ownership: Owner and group that the copied files should have 951 @type ownership: Tuple of numeric ids C{(uid, gid)} 952 953 @param permissions: Permissions that the staged files should have 954 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}). 955 956 @return: Number of files copied from the source directory to the target directory. 957 958 @raise ValueError: If source or target is not a directory or does not exist. 959 @raise IOError: If there is an IO error copying the files. 960 """ 961 beforeSet = RemotePeer._getDirContents(targetDir) 962 if localUser is not None: 963 try: 964 if os.getuid() != 0: 965 raise IOError("Only root can remote copy as another user.") 966 except AttributeError: pass 967 actualCommand = "%s %s@%s:%s/* %s" % (rcpCommand, remoteUser, remoteHost, sourceDir, targetDir) 968 command = resolveCommand(SU_COMMAND) 969 result = executeCommand(command, [localUser, "-c", actualCommand])[0] 970 if result != 0: 971 raise IOError("Error (%d) copying files from remote host as local user [%s]." % (result, localUser)) 972 else: 973 copySource = "%s@%s:%s/*" % (remoteUser, remoteHost, sourceDir) 974 command = resolveCommand(rcpCommandList) 975 result = executeCommand(command, [copySource, targetDir])[0] 976 if result != 0: 977 raise IOError("Error (%d) copying files from remote host." % result) 978 afterSet = RemotePeer._getDirContents(targetDir) 979 if len(afterSet) == 0: 980 raise IOError("Did not copy any files from remote peer.") 981 differenceSet = afterSet.difference(beforeSet) # files we added as part of copy 982 if len(differenceSet) == 0: 983 raise IOError("Apparently did not copy any new files from remote peer.") 984 for targetFile in differenceSet: 985 if ownership is not None: 986 os.chown(targetFile, ownership[0], ownership[1]) 987 if permissions is not None: 988 os.chmod(targetFile, permissions) 989 return len(differenceSet)
990 _copyRemoteDir = staticmethod(_copyRemoteDir) 991
992 - def _copyRemoteFile(remoteUser, localUser, remoteHost, 993 rcpCommand, rcpCommandList, 994 sourceFile, targetFile, ownership=None, 995 permissions=None, overwrite=True):
996 """ 997 Copies a remote source file to a target file. 998 999 @note: Internally, we have to go through and escape any spaces in the 1000 source path with double-backslash, otherwise things get screwed up. It 1001 doesn't seem to be required in the target path. I hope this is portable 1002 to various different rcp methods, but I guess it might not be (all I have 1003 to test with is OpenSSH). 1004 1005 @note: If you have user/group as strings, call the L{util.getUidGid} function 1006 to get the associated uid/gid as an ownership tuple. 1007 1008 @note: We will not overwrite a target file that exists when this method 1009 is invoked. If the target already exists, we'll raise an exception. 1010 1011 @note: Apparently, we can't count on all rcp-compatible implementations 1012 to return sensible errors for some error conditions. As an example, the 1013 C{scp} command in Debian 'woody' returns a zero (normal) status even when 1014 it can't find a host or if the login or path is invalid. We try to work 1015 around this by issuing C{IOError} the target file does not exist when 1016 we're done. 1017 1018 @param remoteUser: Name of the Cedar Backup user on the remote peer 1019 @type remoteUser: String representing a username, valid via the copy command 1020 1021 @param remoteHost: Hostname of the remote peer 1022 @type remoteHost: String representing a hostname, accessible via the copy command 1023 1024 @param localUser: Name of the Cedar Backup user on the current host 1025 @type localUser: String representing a username, valid on the current host 1026 1027 @param rcpCommand: An rcp-compatible copy command to use for copying files from the peer 1028 @type rcpCommand: String representing a system command including required arguments 1029 1030 @param rcpCommandList: An rcp-compatible copy command to use for copying files 1031 @type rcpCommandList: Command as a list to be passed to L{util.executeCommand} 1032 1033 @param sourceFile: Source file to copy 1034 @type sourceFile: String representing a file on disk, as an absolute path 1035 1036 @param targetFile: Target file to create 1037 @type targetFile: String representing a file on disk, as an absolute path 1038 1039 @param ownership: Owner and group that the copied should have 1040 @type ownership: Tuple of numeric ids C{(uid, gid)} 1041 1042 @param permissions: Permissions that the staged files should have 1043 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}). 1044 1045 @param overwrite: Indicates whether it's OK to overwrite the target file. 1046 @type overwrite: Boolean true/false. 1047 1048 @raise IOError: If the target file already exists. 1049 @raise IOError: If there is an IO error copying the file 1050 @raise OSError: If there is an OS error changing permissions on the file 1051 """ 1052 if not overwrite: 1053 if os.path.exists(targetFile): 1054 raise IOError("Target file [%s] already exists." % targetFile) 1055 if localUser is not None: 1056 try: 1057 if os.getuid() != 0: 1058 raise IOError("Only root can remote copy as another user.") 1059 except AttributeError: pass 1060 actualCommand = "%s %s@%s:%s %s" % (rcpCommand, remoteUser, remoteHost, sourceFile.replace(" ", "\\ "), targetFile) 1061 command = resolveCommand(SU_COMMAND) 1062 result = executeCommand(command, [localUser, "-c", actualCommand])[0] 1063 if result != 0: 1064 raise IOError("Error (%d) copying [%s] from remote host as local user [%s]." % (result, sourceFile, localUser)) 1065 else: 1066 copySource = "%s@%s:%s" % (remoteUser, remoteHost, sourceFile.replace(" ", "\\ ")) 1067 command = resolveCommand(rcpCommandList) 1068 result = executeCommand(command, [copySource, targetFile])[0] 1069 if result != 0: 1070 raise IOError("Error (%d) copying [%s] from remote host." % (result, sourceFile)) 1071 if not os.path.exists(targetFile): 1072 raise IOError("Apparently unable to copy file from remote host.") 1073 if ownership is not None: 1074 os.chown(targetFile, ownership[0], ownership[1]) 1075 if permissions is not None: 1076 os.chmod(targetFile, permissions)
1077 _copyRemoteFile = staticmethod(_copyRemoteFile) 1078
1079 - def _pushLocalFile(remoteUser, localUser, remoteHost, 1080 rcpCommand, rcpCommandList, 1081 sourceFile, targetFile, overwrite=True):
1082 """ 1083 Copies a local source file to a remote host. 1084 1085 @note: We will not overwrite a target file that exists when this method 1086 is invoked. If the target already exists, we'll raise an exception. 1087 1088 @note: Internally, we have to go through and escape any spaces in the 1089 source and target paths with double-backslash, otherwise things get 1090 screwed up. I hope this is portable to various different rcp methods, 1091 but I guess it might not be (all I have to test with is OpenSSH). 1092 1093 @note: If you have user/group as strings, call the L{util.getUidGid} function 1094 to get the associated uid/gid as an ownership tuple. 1095 1096 @param remoteUser: Name of the Cedar Backup user on the remote peer 1097 @type remoteUser: String representing a username, valid via the copy command 1098 1099 @param localUser: Name of the Cedar Backup user on the current host 1100 @type localUser: String representing a username, valid on the current host 1101 1102 @param remoteHost: Hostname of the remote peer 1103 @type remoteHost: String representing a hostname, accessible via the copy command 1104 1105 @param rcpCommand: An rcp-compatible copy command to use for copying files from the peer 1106 @type rcpCommand: String representing a system command including required arguments 1107 1108 @param rcpCommandList: An rcp-compatible copy command to use for copying files 1109 @type rcpCommandList: Command as a list to be passed to L{util.executeCommand} 1110 1111 @param sourceFile: Source file to copy 1112 @type sourceFile: String representing a file on disk, as an absolute path 1113 1114 @param targetFile: Target file to create 1115 @type targetFile: String representing a file on disk, as an absolute path 1116 1117 @param overwrite: Indicates whether it's OK to overwrite the target file. 1118 @type overwrite: Boolean true/false. 1119 1120 @raise IOError: If there is an IO error copying the file 1121 @raise OSError: If there is an OS error changing permissions on the file 1122 """ 1123 if not overwrite: 1124 if os.path.exists(targetFile): 1125 raise IOError("Target file [%s] already exists." % targetFile) 1126 if localUser is not None: 1127 try: 1128 if os.getuid() != 0: 1129 raise IOError("Only root can remote copy as another user.") 1130 except AttributeError: pass 1131 actualCommand = '%s "%s" "%s@%s:%s"' % (rcpCommand, sourceFile, remoteUser, remoteHost, targetFile) 1132 command = resolveCommand(SU_COMMAND) 1133 result = executeCommand(command, [localUser, "-c", actualCommand])[0] 1134 if result != 0: 1135 raise IOError("Error (%d) copying [%s] to remote host as local user [%s]." % (result, sourceFile, localUser)) 1136 else: 1137 copyTarget = "%s@%s:%s" % (remoteUser, remoteHost, targetFile.replace(" ", "\\ ")) 1138 command = resolveCommand(rcpCommandList) 1139 result = executeCommand(command, [sourceFile.replace(" ", "\\ "), copyTarget])[0] 1140 if result != 0: 1141 raise IOError("Error (%d) copying [%s] to remote host." % (result, sourceFile))
1142 _pushLocalFile = staticmethod(_pushLocalFile) 1143
1144 - def _executeRemoteCommand(remoteUser, localUser, remoteHost, rshCommand, rshCommandList, remoteCommand):
1145 """ 1146 Executes a command on the peer via remote shell. 1147 1148 @param remoteUser: Name of the Cedar Backup user on the remote peer 1149 @type remoteUser: String representing a username, valid on the remote host 1150 1151 @param localUser: Name of the Cedar Backup user on the current host 1152 @type localUser: String representing a username, valid on the current host 1153 1154 @param remoteHost: Hostname of the remote peer 1155 @type remoteHost: String representing a hostname, accessible via the copy command 1156 1157 @param rshCommand: An rsh-compatible copy command to use for remote shells to the peer 1158 @type rshCommand: String representing a system command including required arguments 1159 1160 @param rshCommandList: An rsh-compatible copy command to use for remote shells to the peer 1161 @type rshCommandList: Command as a list to be passed to L{util.executeCommand} 1162 1163 @param remoteCommand: The command to be executed on the remote host 1164 @type remoteCommand: String command-line, with no special shell characters ($, <, etc.) 1165 1166 @raise IOError: If there is an error executing the remote command 1167 """ 1168 actualCommand = "%s %s@%s '%s'" % (rshCommand, remoteUser, remoteHost, remoteCommand) 1169 if localUser is not None: 1170 try: 1171 if os.getuid() != 0: 1172 raise IOError("Only root can remote shell as another user.") 1173 except AttributeError: pass 1174 command = resolveCommand(SU_COMMAND) 1175 result = executeCommand(command, [localUser, "-c", actualCommand])[0] 1176 if result != 0: 1177 raise IOError("Command failed [su -c %s \"%s\"]" % (localUser, actualCommand)) 1178 else: 1179 command = resolveCommand(rshCommandList) 1180 result = executeCommand(command, ["%s@%s" % (remoteUser, remoteHost), "%s" % remoteCommand])[0] 1181 if result != 0: 1182 raise IOError("Command failed [%s]" % (actualCommand))
1183 _executeRemoteCommand = staticmethod(_executeRemoteCommand) 1184
1185 - def _buildCbackCommand(cbackCommand, action, fullBackup):
1186 """ 1187 Builds a Cedar Backup command line for the named action. 1188 1189 @note: If the cback command is None, then DEF_CBACK_COMMAND is used. 1190 1191 @param cbackCommand: cback command to execute, including required options 1192 @param action: Name of the action to execute. 1193 @param fullBackup: Whether a full backup should be executed. 1194 1195 @return: String suitable for passing to L{_executeRemoteCommand} as remoteCommand. 1196 @raise ValueError: If action is None. 1197 """ 1198 if action is None: 1199 raise ValueError("Action cannot be None.") 1200 if cbackCommand is None: 1201 cbackCommand = DEF_CBACK_COMMAND 1202 if fullBackup: 1203 return "%s --full %s" % (cbackCommand, action) 1204 else: 1205 return "%s %s" % (cbackCommand, action)
1206 _buildCbackCommand = staticmethod(_buildCbackCommand)
1207