vstf/vstf/common/rsync.py

   1 #!/usr/bin/python
   2
   3 # Python conterpart of rsync written by Vivian De Smedt
   4 # Send any comment or bug report to vivian@vdesmedt.com.
   5 # I would like to thanks William Tan for its support in tuning rsync.py to support unicode path.
   6 # I would like to thanks Luc Saffre for its bug reports and fixes.
   7
   8 #from __future__ import nested_scopes
   9
  10 import os, os.path, shutil, glob, re, sys, getopt, stat, string
  11
  12
  13 try:
  14         import win32file
  15 except:
  16         win32file = None
  17
  18 class Cookie:
  19         def __init__(self):
  20                 self.sink_root = ""
  21                 self.target_root = ""
  22                 self.quiet = 0
  23                 self.recursive = 0
  24                 self.relative = 0
  25                 self.dry_run = 0
  26                 self.time = 0
  27                 self.update = 0
  28                 self.cvs_ignore = 0
  29                 self.ignore_time = 0
  30                 self.delete = 0
  31                 self.delete_excluded = 0
  32                 self.delete_from_source = 0
  33                 self.size_only = 0
  34                 self.modify_window = 2
  35                 self.existing = 0
  36                 self.filters = []
  37                 self.case_sensitivity = 0
  38                 if os.name == "nt":
  39                         self.case_sensitivity = re.I
  40
  41 def visit(cookie, dirname, names):
  42         """Copy files names from sink_root + (dirname - sink_root) to target_root + (dirname - sink_root)"""
  43         if os.path.split(cookie.sink_root)[1]: # Should be tested with (C:\Cvs -> C:\)! (C:\Archives\MyDatas\UltraEdit -> C:\Archives\MyDatas) (Cvs -> "")! (Archives\MyDatas\UltraEdit -> Archives\MyDatas) (\Cvs -> \)! (\Archives\MyDatas\UltraEdit -> Archives\MyDatas)
  44                 dirname = dirname[len(cookie.sink_root) + 1:]
  45         else:
  46                 dirname = dirname[len(cookie.sink_root):]
  47         target_dir = os.path.join(cookie.target_root, dirname)
  48         if not os.path.isdir(target_dir):
  49                 makeDir(cookie, target_dir)
  50         sink_dir = os.path.join(cookie.sink_root, dirname)
  51
  52         filters = []
  53         if cookie.cvs_ignore:
  54                 ignore = os.path.join(sink_dir, ".cvsignore")
  55                 if os.path.isfile(ignore):
  56                         filters = convertPatterns(ignore, "-")
  57         filters = filters + cookie.filters
  58
  59         names_excluded = []
  60         if filters:
  61                 # filter sink files (names):
  62                 name_index = 0
  63                 while name_index < len(names):
  64                         name = names[name_index]
  65                         path = os.path.join(dirname, name)
  66                         path = convertPath(path)
  67                         if os.path.isdir(os.path.join(sink_dir, name)):
  68                                 path = path + "/"
  69                         for filter in filters:
  70                                 if re.search(filter[1], path, cookie.case_sensitivity):
  71                                         if filter[0] == '-':
  72                                                 sink = os.path.join(sink_dir, name)
  73                                                 if cookie.delete_from_source:
  74                                                         if os.path.isfile(sink):
  75                                                                 removeFile(cookie, sink)
  76                                                         elif os.path.isdir(sink):
  77                                                                 removeDir(cookie, sink)
  78                                                         else:
  79                                                                 logError("Sink %s is neither a file nor a folder (skip removal)" % sink)
  80                                                 names_excluded += [names[name_index]]
  81                                                 del(names[name_index])
  82                                                 name_index = name_index - 1
  83                                                 break
  84                                         elif filter[0] == '+':
  85                                                 break
  86                         name_index = name_index + 1
  87
  88         if cookie.delete and os.path.isdir(target_dir):
  89                 # Delete files and folder in target not present in filtered sink.
  90                 for name in os.listdir(target_dir):
  91                         if not cookie.delete_excluded and name in names_excluded:
  92                                 continue
  93                         if not name in names:
  94                                 target = os.path.join(target_dir, name)
  95                                 if os.path.isfile(target):
  96                                         removeFile(cookie, target)
  97                                 elif os.path.isdir(target):
  98                                         removeDir(cookie, target)
  99                                 else:
 100                                         pass
 101
 102         for name in names:
 103                 # Copy files and folder from sink to target.
 104                 sink = os.path.join(sink_dir, name)
 105                 #print sink
 106                 target = os.path.join(target_dir, name)
 107                 if os.path.exists(target):
 108                         # When target already exit:
 109                         if os.path.isfile(sink):
 110                                 if os.path.isfile(target):
 111                                         # file-file
 112                                         if shouldUpdate(cookie, sink, target):
 113                                                 updateFile(cookie, sink, target)
 114                                 elif os.path.isdir(target):
 115                                         # file-folder
 116                                         removeDir(cookie, target)
 117                                         copyFile(cookie, sink, target)
 118                                 else:
 119                                         # file-???
 120                                         logError("Target %s is neither a file nor folder (skip update)" % sink)
 121
 122                         elif os.path.isdir(sink):
 123                                 if os.path.isfile(target):
 124                                         # folder-file
 125                                         removeFile(cookie, target)
 126                                         makeDir(cookie, target)
 127                         else:
 128                                 # ???-xxx
 129                                 logError("Sink %s is neither a file nor a folder (skip update)" % sink)
 130
 131                 elif not cookie.existing:
 132                         # When target dont exist:
 133                         if os.path.isfile(sink):
 134                                 # file
 135                                 copyFile(cookie, sink, target)
 136                         elif os.path.isdir(sink):
 137                                 # folder
 138                                 makeDir(cookie, target)
 139                         else:
 140                                 logError("Sink %s is neither a file nor a folder (skip update)" % sink)
 141
 142
 143 def log(cookie, message):
 144         if not cookie.quiet:
 145                 try:
 146                         print message
 147                 except UnicodeEncodeError:
 148                         print message.encode("utf8")
 149
 150
 151 def logError(message):
 152         try:
 153                 sys.stderr.write(message + "\n")
 154         except UnicodeEncodeError:
 155                 sys.stderr.write(message.encode("utf8") + "\n")
 156
 157
 158 def shouldUpdate(cookie, sink, target):
 159         try:
 160                 sink_st = os.stat(sink)
 161                 sink_sz = sink_st.st_size
 162                 sink_mt = sink_st.st_mtime
 163         except:
 164                 logError("Fail to retrieve information about sink %s (skip update)" % sink)
 165                 return 0
 166
 167         try:
 168                 target_st = os.stat(target)
 169                 target_sz = target_st.st_size
 170                 target_mt = target_st.st_mtime
 171         except:
 172                 logError("Fail to retrieve information about target %s (skip update)" % target)
 173                 return 0
 174
 175         if cookie.update:
 176                 return target_mt < sink_mt - cookie.modify_window
 177
 178         if cookie.ignore_time:
 179                 return 1
 180
 181         if target_sz != sink_sz:
 182                 return 1
 183
 184         if cookie.size_only:
 185                 return 0
 186
 187         return abs(target_mt - sink_mt) > cookie.modify_window
 188
 189
 190 def copyFile(cookie, sink, target):
 191         log(cookie, "copy: %s to: %s" % (sink, target))
 192         if not cookie.dry_run:
 193                 try:
 194                         shutil.copyfile(sink, target)
 195                 except:
 196                         logError("Fail to copy %s" % sink)
 197
 198                 if cookie.time:
 199                         try:
 200                                 s = os.stat(sink)
 201                                 os.utime(target, (s.st_atime, s.st_mtime));
 202                         except:
 203                                 logError("Fail to copy timestamp of %s" % sink)
 204
 205
 206 def updateFile(cookie, sink, target):
 207         log(cookie, "update: %s to: %s" % (sink, target))
 208         if not cookie.dry_run:
 209                 # Read only and hidden and system files can not be overridden.
 210                 try:
 211                         try:
 212                                 if win32file:
 213                                         filemode = win32file.GetFileAttributesW(target)
 214                                         win32file.SetFileAttributesW(target, filemode & ~win32file.FILE_ATTRIBUTE_READONLY & ~win32file.FILE_ATTRIBUTE_HIDDEN & ~win32file.FILE_ATTRIBUTE_SYSTEM)
 215                                 else:
 216                                         os.chmod(target, stat.S_IWUSR)
 217                         except:
 218                                 #logError("Fail to allow override of %s" % target)
 219                                 pass
 220
 221                         shutil.copyfile(sink, target)
 222                         if cookie.time:
 223                                 try:
 224                                         s = os.stat(sink)
 225                                         os.utime(target, (s.st_atime, s.st_mtime));
 226                                 except:
 227                                         logError("Fail to copy timestamp of %s" % sink) # The utime api of the 2.3 version of python is not unicode compliant.
 228                 except:
 229                         logError("Fail to override %s" % sink)
 230
 231                 if win32file:
 232                         win32file.SetFileAttributesW(target, filemode)
 233
 234
 235 def prepareRemoveFile(path):
 236         if win32file:
 237                 filemode = win32file.GetFileAttributesW(path)
 238                 win32file.SetFileAttributesW(path, filemode & ~win32file.FILE_ATTRIBUTE_READONLY & ~win32file.FILE_ATTRIBUTE_HIDDEN & ~win32file.FILE_ATTRIBUTE_SYSTEM)
 239         else:
 240                 os.chmod(path, stat.S_IWUSR)
 241
 242
 243 def removeFile(cookie, target):
 244         # Read only files could not be deleted.
 245         log(cookie, "remove: %s" % target)
 246         if not cookie.dry_run:
 247                 try:
 248                         try:
 249                                 prepareRemoveFile(target)
 250                         except:
 251                                 #logError("Fail to allow removal of %s" % target)
 252                                 pass
 253
 254                         os.remove(target)
 255                 except:
 256                         logError("Fail to remove %s" % target)
 257
 258
 259
 260 def makeDir(cookie, target):
 261         log(cookie, "make dir: %s" % target)
 262         if not cookie.dry_run:
 263                 try:
 264                         os.makedirs(target)
 265                 except:
 266                         logError("Fail to make dir %s" % target)
 267
 268
 269 def visitForPrepareRemoveDir(arg, dirname, names):
 270         for name in names:
 271                 path = os.path.join(dirname, name)
 272                 prepareRemoveFile(path)
 273
 274
 275 def prepareRemoveDir(path):
 276         prepareRemoveFile(path)
 277         os.path.walk(path, visitForPrepareRemoveDir, None)
 278
 279
 280 def OnRemoveDirError(func, path, excinfo):
 281         logError("Fail to remove %s" % path)
 282
 283
 284 def removeDir(cookie, target):
 285         # Read only directory could not be deleted.
 286         log(cookie, "remove dir: %s" % target)
 287         if not cookie.dry_run:
 288                 prepareRemoveDir(target)
 289                 try:
 290                         shutil.rmtree(target, False, OnRemoveDirError)
 291                 except:
 292                         logError("Fail to remove dir %s" % target)
 293
 294
 295 def convertPath(path):
 296         # Convert windows, mac path to unix version.
 297         separator = os.path.normpath("/")
 298         if separator != "/":
 299                 path = re.sub(re.escape(separator), "/", path)
 300
 301         # Help file, folder pattern to express that it should match the all file or folder name.
 302         path = "/" + path
 303         return path
 304
 305
 306 def convertPattern(pattern, sign):
 307         """Convert a rsync pattern that match against a path to a filter that match against a converted path."""
 308
 309         # Check for include vs exclude patterns.
 310         if pattern[:2] == "+ ":
 311                 pattern = pattern[2:]
 312                 sign = "+"
 313         elif pattern[:2] == "- ":
 314                 pattern = pattern[2:]
 315                 sign = "-"
 316
 317         # Express windows, mac patterns in unix patterns (rsync.py extension).
 318         separator = os.path.normpath("/")
 319         if separator != "/":
 320                 pattern = re.sub(re.escape(separator), "/", pattern)
 321
 322         # If pattern contains '/' it should match from the start.
 323         temp = pattern
 324         if pattern[0] == "/":
 325                 pattern = pattern[1:]
 326         if temp[-1] == "/":
 327                 temp = temp[:-1]
 328
 329         # Convert pattern rules: ** * ? to regexp rules.
 330         pattern = re.escape(pattern)
 331         pattern = string.replace(pattern, "\\?", ".")
 332         pattern = string.replace(pattern, "\\*\\*", ".*")
 333         pattern = string.replace(pattern, "\\*", "[^/]*")
 334         pattern = string.replace(pattern, "\\*", ".*")
 335
 336         if "/" in temp:
 337                 # If pattern contains '/' it should match from the start.
 338                 pattern = "^\\/" + pattern
 339         else:
 340                 # Else the pattern should match the all file or folder name.
 341                 pattern = "\\/" + pattern
 342
 343         if pattern[-2:] != "\\/" and pattern[-2:] != ".*":
 344                 # File patterns should match also folders.
 345                 pattern = pattern + "\\/?"
 346
 347         # Pattern should match till the end.
 348         pattern = pattern + "$"
 349         return (sign, pattern)
 350
 351
 352 def convertPatterns(path, sign):
 353         """Read the files for pattern and return a vector of filters"""
 354         filters = []
 355         f = open(path, "r")
 356         while 1:
 357                 pattern = f.readline()
 358                 if not pattern:
 359                         break
 360                 if pattern[-1] == "\n":
 361                         pattern = pattern[:-1]
 362
 363                 if re.match("[\t ]*$", pattern):
 364                         continue
 365                 if pattern[0] == "#":
 366                         continue
 367                 filters = filters + [convertPattern(pattern, sign)]
 368         f.close()
 369         return filters
 370
 371
 372 def printUsage():
 373         """Print the help string that should printed by rsync.py -h"""
 374         print "usage: rsync.py [options] source target"
 375         print """
 376  -q, --quiet              decrease verbosity
 377  -r, --recursive          recurse into directories
 378  -R, --relative           use relative path names
 379  -u, --update             update only (don't overwrite newer files)
 380  -t, --times              preserve times
 381  -n, --dry-run            show what would have been transferred
 382      --existing           only update files that already exist
 383      --delete             delete files that don't exist on the sending side
 384      --delete-excluded    also delete excluded files on the receiving side
 385      --delete-from-source delete excluded files on the receiving side
 386  -I, --ignore-times       don't exclude files that match length and time
 387      --size-only          only use file size when determining if a file should
 388                           be transferred
 389      --modify-window=NUM  timestamp window (seconds) for file match (default=2)
 390      --existing           only update existing target files or folders
 391  -C, --cvs-exclude        auto ignore files in the same way CVS does
 392      --exclude=PATTERN    exclude files matching PATTERN
 393      --exclude-from=FILE  exclude patterns listed in FILE
 394      --include=PATTERN    don't exclude files matching PATTERN
 395      --include-from=FILE  don't exclude patterns listed in FILE
 396      --version            print version number
 397  -h, --help               show this help screen
 398
 399 See http://www.vdesmedt.com/~vds2212/rsync.html for informations and updates.
 400 Send an email to vivian@vdesmedt.com for comments and bug reports."""
 401
 402
 403 def printVersion():
 404         print "rsync.py version 2.0.1"
 405
 406
 407 def main(args):
 408         cookie = Cookie()
 409
 410         opts, args = getopt.getopt(args, "qrRntuCIh", ["quiet", "recursive", "relative", "dry-run", "time", "update", "cvs-ignore", "ignore-times", "help", "delete", "delete-excluded", "delete-from-source", "existing", "size-only", "modify-window=", "exclude=", "exclude-from=", "include=", "include-from=", "version"])
 411         for o, v in opts:
 412                 if o in ["-q", "--quiet"]:
 413                         cookie.quiet = 1
 414                 if o in ["-r", "--recursive"]:
 415                         cookie.recursive = 1
 416                 if o in ["-R", "--relative"]:
 417                         cookie.relative = 1
 418                 elif o in ["-n", "--dry-run"]:
 419                         cookie.dry_run = 1
 420                 elif o in ["-t", "--times", "--time"]: # --time is there to guaranty backward compatibility with previous buggy version.
 421                         cookie.time = 1
 422                 elif o in ["-u", "--update"]:
 423                         cookie.update = 1
 424                 elif o in ["-C", "--cvs-ignore"]:
 425                         cookie.cvs_ignore = 1
 426                 elif o in ["-I", "--ignore-time"]:
 427                         cookie.ignore_time = 1
 428                 elif o == "--delete":
 429                         cookie.delete = 1
 430                 elif o == "--delete-excluded":
 431                         cookie.delete = 1
 432                         cookie.delete_excluded = 1
 433                 elif o == "--delete-from-source":
 434                         cookie.delete_from_source = 1
 435                 elif o == "--size-only":
 436                         cookie.size_only = 1
 437                 elif o == "--modify-window":
 438                         cookie.modify_window = int(v)
 439                 elif o == "--existing":
 440                         cookie.existing = 1
 441                 elif o == "--exclude":
 442                         cookie.filters = cookie.filters + [convertPattern(v, "-")]
 443                 elif o == "--exclude-from":
 444                         cookie.filters = cookie.filters + convertPatterns(v, "-")
 445                 elif o == "--include":
 446                         cookie.filters = cookie.filters + [convertPattern(v, "+")]
 447                 elif o == "--include-from":
 448                         cookie.filters = cookie.filters + convertPatterns(v, "+")
 449                 elif o == "--version":
 450                         printVersion()
 451                         return 0
 452                 elif o in ["-h", "--help"]:
 453                         printUsage()
 454                         return 0
 455
 456         if len(args) <= 1:
 457                 printUsage()
 458                 return 1
 459
 460         #print cookie.filters
 461
 462         target_root = args[1]
 463         try: # In order to allow compatibility below 2.3.
 464                 pass
 465                 if os.path.__dict__.has_key("supports_unicode_filenames") and os.path.supports_unicode_filenames:
 466                         target_root = unicode(target_root, sys.getfilesystemencoding())
 467         finally:
 468                 cookie.target_root = target_root
 469
 470         sinks = glob.glob(args[0])
 471         if not sinks:
 472                 return 0
 473
 474         sink_families = {}
 475         for sink in sinks:
 476                 try: # In order to allow compatibility below 2.3.
 477                         if os.path.__dict__.has_key("supports_unicode_filenames") and os.path.supports_unicode_filenames:
 478                                 sink = unicode(sink, sys.getfilesystemencoding())
 479                 except:
 480                         pass
 481                 sink_name = ""
 482                 sink_root = sink
 483                 sink_drive, sink_root = os.path.splitdrive(sink)
 484                 while not sink_name:
 485                         if sink_root == os.path.sep:
 486                                 sink_name = "."
 487                                 break
 488                         sink_root, sink_name = os.path.split(sink_root)
 489                 sink_root = sink_drive + sink_root
 490                 if not sink_families.has_key(sink_root):
 491                         sink_families[sink_root] = []
 492                 sink_families[sink_root] = sink_families[sink_root] + [sink_name]
 493
 494         for sink_root in sink_families.keys():
 495                 if cookie.relative:
 496                         cookie.sink_root = ""
 497                 else:
 498                         cookie.sink_root = sink_root
 499
 500                 global y # In order to allow compatibility below 2.1 (nested scope where used before).
 501                 y = sink_root
 502                 files = filter(lambda x: os.path.isfile(os.path.join(y, x)), sink_families[sink_root])
 503                 if files:
 504                         visit(cookie, sink_root, files)
 505
 506                 #global y # In order to allow compatibility below 2.1 (nested scope where used before).
 507                 y = sink_root
 508                 folders = filter(lambda x: os.path.isdir(os.path.join(y, x)), sink_families[sink_root])
 509                 for folder in folders:
 510                         folder_path = os.path.join(sink_root, folder)
 511                         if not cookie.recursive:
 512                                 visit(cookie, folder_path, os.listdir(folder_path))
 513                         else:
 514                                 os.path.walk(folder_path, visit, cookie)
 515         return 0
 516
 517 if __name__ == "__main__":
 518         sys.exit(main(sys.argv[1:]))