vstf/vstf/common/rsync.py

   1 ##############################################################################
   2 # Copyright (c) 2015 Huawei Technologies Co.,Ltd and others.
   3 #
   4 # All rights reserved. This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 # http://www.apache.org/licenses/LICENSE-2.0
   8 ##############################################################################
   9
  10
  11 # from __future__ import nested_scopes
  12
  13 import os, os.path, shutil, glob, re, sys, getopt, stat, string
  14
  15 try:
  16     import win32file
  17 except:
  18     win32file = None
  19
  20
  21 class Cookie:
  22     def __init__(self):
  23         self.sink_root = ""
  24         self.target_root = ""
  25         self.quiet = 0
  26         self.recursive = 0
  27         self.relative = 0
  28         self.dry_run = 0
  29         self.time = 0
  30         self.update = 0
  31         self.cvs_ignore = 0
  32         self.ignore_time = 0
  33         self.delete = 0
  34         self.delete_excluded = 0
  35         self.delete_from_source = 0
  36         self.size_only = 0
  37         self.modify_window = 2
  38         self.existing = 0
  39         self.filters = []
  40         self.case_sensitivity = 0
  41         if os.name == "nt":
  42             self.case_sensitivity = re.I
  43
  44
  45 def visit(cookie, dirname, names):
  46     """Copy files names from sink_root + (dirname - sink_root) to target_root + (dirname - sink_root)"""
  47     if os.path.split(cookie.sink_root)[
  48         1]:  # Should be tested with (C:\Cvs -> C:\)! (C:\Archives\MyDatas\UltraEdit -> C:\Archives\MyDatas) (Cvs -> "")! (Archives\MyDatas\UltraEdit -> Archives\MyDatas) (\Cvs -> \)! (\Archives\MyDatas\UltraEdit -> Archives\MyDatas)
  49         dirname = dirname[len(cookie.sink_root) + 1:]
  50     else:
  51         dirname = dirname[len(cookie.sink_root):]
  52     target_dir = os.path.join(cookie.target_root, dirname)
  53     if not os.path.isdir(target_dir):
  54         makeDir(cookie, target_dir)
  55     sink_dir = os.path.join(cookie.sink_root, dirname)
  56
  57     filters = []
  58     if cookie.cvs_ignore:
  59         ignore = os.path.join(sink_dir, ".cvsignore")
  60         if os.path.isfile(ignore):
  61             filters = convertPatterns(ignore, "-")
  62     filters = filters + cookie.filters
  63
  64     names_excluded = []
  65     if filters:
  66         # filter sink files (names):
  67         name_index = 0
  68         while name_index < len(names):
  69             name = names[name_index]
  70             path = os.path.join(dirname, name)
  71             path = convertPath(path)
  72             if os.path.isdir(os.path.join(sink_dir, name)):
  73                 path = path + "/"
  74             for filter in filters:
  75                 if re.search(filter[1], path, cookie.case_sensitivity):
  76                     if filter[0] == '-':
  77                         sink = os.path.join(sink_dir, name)
  78                         if cookie.delete_from_source:
  79                             if os.path.isfile(sink):
  80                                 removeFile(cookie, sink)
  81                             elif os.path.isdir(sink):
  82                                 removeDir(cookie, sink)
  83                             else:
  84                                 logError("Sink %s is neither a file nor a folder (skip removal)" % sink)
  85                         names_excluded += [names[name_index]]
  86                         del (names[name_index])
  87                         name_index = name_index - 1
  88                         break
  89                     elif filter[0] == '+':
  90                         break
  91             name_index = name_index + 1
  92
  93     if cookie.delete and os.path.isdir(target_dir):
  94         # Delete files and folder in target not present in filtered sink.
  95         for name in os.listdir(target_dir):
  96             if not cookie.delete_excluded and name in names_excluded:
  97                 continue
  98             if not name in names:
  99                 target = os.path.join(target_dir, name)
 100                 if os.path.isfile(target):
 101                     removeFile(cookie, target)
 102                 elif os.path.isdir(target):
 103                     removeDir(cookie, target)
 104                 else:
 105                     pass
 106
 107     for name in names:
 108         # Copy files and folder from sink to target.
 109         sink = os.path.join(sink_dir, name)
 110         # print sink
 111         target = os.path.join(target_dir, name)
 112         if os.path.exists(target):
 113             # When target already exit:
 114             if os.path.isfile(sink):
 115                 if os.path.isfile(target):
 116                     # file-file
 117                     if shouldUpdate(cookie, sink, target):
 118                         updateFile(cookie, sink, target)
 119                 elif os.path.isdir(target):
 120                     # file-folder
 121                     removeDir(cookie, target)
 122                     copyFile(cookie, sink, target)
 123                 else:
 124                     # file-???
 125                     logError("Target %s is neither a file nor folder (skip update)" % sink)
 126
 127             elif os.path.isdir(sink):
 128                 if os.path.isfile(target):
 129                     # folder-file
 130                     removeFile(cookie, target)
 131                     makeDir(cookie, target)
 132             else:
 133                 # ???-xxx
 134                 logError("Sink %s is neither a file nor a folder (skip update)" % sink)
 135
 136         elif not cookie.existing:
 137             # When target dont exist:
 138             if os.path.isfile(sink):
 139                 # file
 140                 copyFile(cookie, sink, target)
 141             elif os.path.isdir(sink):
 142                 # folder
 143                 makeDir(cookie, target)
 144             else:
 145                 logError("Sink %s is neither a file nor a folder (skip update)" % sink)
 146
 147
 148 def log(cookie, message):
 149     if not cookie.quiet:
 150         try:
 151             print message
 152         except UnicodeEncodeError:
 153             print message.encode("utf8")
 154
 155
 156 def logError(message):
 157     try:
 158         sys.stderr.write(message + "\n")
 159     except UnicodeEncodeError:
 160         sys.stderr.write(message.encode("utf8") + "\n")
 161
 162
 163 def shouldUpdate(cookie, sink, target):
 164     try:
 165         sink_st = os.stat(sink)
 166         sink_sz = sink_st.st_size
 167         sink_mt = sink_st.st_mtime
 168     except:
 169         logError("Fail to retrieve information about sink %s (skip update)" % sink)
 170         return 0
 171
 172     try:
 173         target_st = os.stat(target)
 174         target_sz = target_st.st_size
 175         target_mt = target_st.st_mtime
 176     except:
 177         logError("Fail to retrieve information about target %s (skip update)" % target)
 178         return 0
 179
 180     if cookie.update:
 181         return target_mt < sink_mt - cookie.modify_window
 182
 183     if cookie.ignore_time:
 184         return 1
 185
 186     if target_sz != sink_sz:
 187         return 1
 188
 189     if cookie.size_only:
 190         return 0
 191
 192     return abs(target_mt - sink_mt) > cookie.modify_window
 193
 194
 195 def copyFile(cookie, sink, target):
 196     log(cookie, "copy: %s to: %s" % (sink, target))
 197     if not cookie.dry_run:
 198         try:
 199             shutil.copyfile(sink, target)
 200         except:
 201             logError("Fail to copy %s" % sink)
 202
 203         if cookie.time:
 204             try:
 205                 s = os.stat(sink)
 206                 os.utime(target, (s.st_atime, s.st_mtime));
 207             except:
 208                 logError("Fail to copy timestamp of %s" % sink)
 209
 210
 211 def updateFile(cookie, sink, target):
 212     log(cookie, "update: %s to: %s" % (sink, target))
 213     if not cookie.dry_run:
 214         # Read only and hidden and system files can not be overridden.
 215         try:
 216             try:
 217                 if win32file:
 218                     filemode = win32file.GetFileAttributesW(target)
 219                     win32file.SetFileAttributesW(target,
 220                                                  filemode & ~win32file.FILE_ATTRIBUTE_READONLY & ~win32file.FILE_ATTRIBUTE_HIDDEN & ~win32file.FILE_ATTRIBUTE_SYSTEM)
 221                 else:
 222                     os.chmod(target, stat.S_IWUSR)
 223             except:
 224                 # logError("Fail to allow override of %s" % target)
 225                 pass
 226
 227             shutil.copyfile(sink, target)
 228             if cookie.time:
 229                 try:
 230                     s = os.stat(sink)
 231                     os.utime(target, (s.st_atime, s.st_mtime));
 232                 except:
 233                     logError(
 234                         "Fail to copy timestamp of %s" % sink)  # The utime api of the 2.3 version of python is not unicode compliant.
 235         except:
 236             logError("Fail to override %s" % sink)
 237
 238         if win32file:
 239             win32file.SetFileAttributesW(target, filemode)
 240
 241
 242 def prepareRemoveFile(path):
 243     if win32file:
 244         filemode = win32file.GetFileAttributesW(path)
 245         win32file.SetFileAttributesW(path,
 246                                      filemode & ~win32file.FILE_ATTRIBUTE_READONLY & ~win32file.FILE_ATTRIBUTE_HIDDEN & ~win32file.FILE_ATTRIBUTE_SYSTEM)
 247     else:
 248         os.chmod(path, stat.S_IWUSR)
 249
 250
 251 def removeFile(cookie, target):
 252     # Read only files could not be deleted.
 253     log(cookie, "remove: %s" % target)
 254     if not cookie.dry_run:
 255         try:
 256             try:
 257                 prepareRemoveFile(target)
 258             except:
 259                 # logError("Fail to allow removal of %s" % target)
 260                 pass
 261
 262             os.remove(target)
 263         except:
 264             logError("Fail to remove %s" % target)
 265
 266
 267 def makeDir(cookie, target):
 268     log(cookie, "make dir: %s" % target)
 269     if not cookie.dry_run:
 270         try:
 271             os.makedirs(target)
 272         except:
 273             logError("Fail to make dir %s" % target)
 274
 275
 276 def visitForPrepareRemoveDir(arg, dirname, names):
 277     for name in names:
 278         path = os.path.join(dirname, name)
 279         prepareRemoveFile(path)
 280
 281
 282 def prepareRemoveDir(path):
 283     prepareRemoveFile(path)
 284     os.path.walk(path, visitForPrepareRemoveDir, None)
 285
 286
 287 def OnRemoveDirError(func, path, excinfo):
 288     logError("Fail to remove %s" % path)
 289
 290
 291 def removeDir(cookie, target):
 292     # Read only directory could not be deleted.
 293     log(cookie, "remove dir: %s" % target)
 294     if not cookie.dry_run:
 295         prepareRemoveDir(target)
 296         try:
 297             shutil.rmtree(target, False, OnRemoveDirError)
 298         except:
 299             logError("Fail to remove dir %s" % target)
 300
 301
 302 def convertPath(path):
 303     # Convert windows, mac path to unix version.
 304     separator = os.path.normpath("/")
 305     if separator != "/":
 306         path = re.sub(re.escape(separator), "/", path)
 307
 308     # Help file, folder pattern to express that it should match the all file or folder name.
 309     path = "/" + path
 310     return path
 311
 312
 313 def convertPattern(pattern, sign):
 314     """Convert a rsync pattern that match against a path to a filter that match against a converted path."""
 315
 316     # Check for include vs exclude patterns.
 317     if pattern[:2] == "+ ":
 318         pattern = pattern[2:]
 319         sign = "+"
 320     elif pattern[:2] == "- ":
 321         pattern = pattern[2:]
 322         sign = "-"
 323
 324     # Express windows, mac patterns in unix patterns (rsync.py extension).
 325     separator = os.path.normpath("/")
 326     if separator != "/":
 327         pattern = re.sub(re.escape(separator), "/", pattern)
 328
 329     # If pattern contains '/' it should match from the start.
 330     temp = pattern
 331     if pattern[0] == "/":
 332         pattern = pattern[1:]
 333     if temp[-1] == "/":
 334         temp = temp[:-1]
 335
 336     # Convert pattern rules: ** * ? to regexp rules.
 337     pattern = re.escape(pattern)
 338     pattern = string.replace(pattern, "\\?", ".")
 339     pattern = string.replace(pattern, "\\*\\*", ".*")
 340     pattern = string.replace(pattern, "\\*", "[^/]*")
 341     pattern = string.replace(pattern, "\\*", ".*")
 342
 343     if "/" in temp:
 344         # If pattern contains '/' it should match from the start.
 345         pattern = "^\\/" + pattern
 346     else:
 347         # Else the pattern should match the all file or folder name.
 348         pattern = "\\/" + pattern
 349
 350     if pattern[-2:] != "\\/" and pattern[-2:] != ".*":
 351         # File patterns should match also folders.
 352         pattern = pattern + "\\/?"
 353
 354     # Pattern should match till the end.
 355     pattern = pattern + "$"
 356     return (sign, pattern)
 357
 358
 359 def convertPatterns(path, sign):
 360     """Read the files for pattern and return a vector of filters"""
 361     filters = []
 362     f = open(path, "r")
 363     while 1:
 364         pattern = f.readline()
 365         if not pattern:
 366             break
 367         if pattern[-1] == "\n":
 368             pattern = pattern[:-1]
 369
 370         if re.match("[\t ]*$", pattern):
 371             continue
 372         if pattern[0] == "#":
 373             continue
 374         filters = filters + [convertPattern(pattern, sign)]
 375     f.close()
 376     return filters
 377
 378
 379 def printUsage():
 380     """Print the help string that should printed by rsync.py -h"""
 381     print "usage: rsync.py [options] source target"
 382     print """
 383  -q, --quiet              decrease verbosity
 384  -r, --recursive          recurse into directories
 385  -R, --relative           use relative path names
 386  -u, --update             update only (don't overwrite newer files)
 387  -t, --times              preserve times
 388  -n, --dry-run            show what would have been transferred
 389      --existing           only update files that already exist
 390      --delete             delete files that don't exist on the sending side
 391      --delete-excluded    also delete excluded files on the receiving side
 392      --delete-from-source delete excluded files on the receiving side
 393  -I, --ignore-times       don't exclude files that match length and time
 394      --size-only          only use file size when determining if a file should
 395                           be transferred
 396      --modify-window=NUM  timestamp window (seconds) for file match (default=2)
 397      --existing           only update existing target files or folders
 398  -C, --cvs-exclude        auto ignore files in the same way CVS does
 399      --exclude=PATTERN    exclude files matching PATTERN
 400      --exclude-from=FILE  exclude patterns listed in FILE
 401      --include=PATTERN    don't exclude files matching PATTERN
 402      --include-from=FILE  don't exclude patterns listed in FILE
 403      --version            print version number
 404  -h, --help               show this help screen
 405
 406 See http://www.vdesmedt.com/~vds2212/rsync.html for informations and updates.
 407 Send an email to vivian@vdesmedt.com for comments and bug reports."""
 408
 409
 410 def printVersion():
 411     print "rsync.py version 2.0.1"
 412
 413
 414 def main(args):
 415     cookie = Cookie()
 416
 417     opts, args = getopt.getopt(args, "qrRntuCIh",
 418                                ["quiet", "recursive", "relative", "dry-run", "time", "update", "cvs-ignore",
 419                                 "ignore-times", "help", "delete", "delete-excluded", "delete-from-source", "existing",
 420                                 "size-only", "modify-window=", "exclude=", "exclude-from=", "include=", "include-from=",
 421                                 "version"])
 422     for o, v in opts:
 423         if o in ["-q", "--quiet"]:
 424             cookie.quiet = 1
 425         if o in ["-r", "--recursive"]:
 426             cookie.recursive = 1
 427         if o in ["-R", "--relative"]:
 428             cookie.relative = 1
 429         elif o in ["-n", "--dry-run"]:
 430             cookie.dry_run = 1
 431         elif o in ["-t", "--times",
 432                    "--time"]:  # --time is there to guaranty backward compatibility with previous buggy version.
 433             cookie.time = 1
 434         elif o in ["-u", "--update"]:
 435             cookie.update = 1
 436         elif o in ["-C", "--cvs-ignore"]:
 437             cookie.cvs_ignore = 1
 438         elif o in ["-I", "--ignore-time"]:
 439             cookie.ignore_time = 1
 440         elif o == "--delete":
 441             cookie.delete = 1
 442         elif o == "--delete-excluded":
 443             cookie.delete = 1
 444             cookie.delete_excluded = 1
 445         elif o == "--delete-from-source":
 446             cookie.delete_from_source = 1
 447         elif o == "--size-only":
 448             cookie.size_only = 1
 449         elif o == "--modify-window":
 450             cookie.modify_window = int(v)
 451         elif o == "--existing":
 452             cookie.existing = 1
 453         elif o == "--exclude":
 454             cookie.filters = cookie.filters + [convertPattern(v, "-")]
 455         elif o == "--exclude-from":
 456             cookie.filters = cookie.filters + convertPatterns(v, "-")
 457         elif o == "--include":
 458             cookie.filters = cookie.filters + [convertPattern(v, "+")]
 459         elif o == "--include-from":
 460             cookie.filters = cookie.filters + convertPatterns(v, "+")
 461         elif o == "--version":
 462             printVersion()
 463             return 0
 464         elif o in ["-h", "--help"]:
 465             printUsage()
 466             return 0
 467
 468     if len(args) <= 1:
 469         printUsage()
 470         return 1
 471
 472     # print cookie.filters
 473
 474     target_root = args[1]
 475     try:  # In order to allow compatibility below 2.3.
 476         pass
 477         if os.path.__dict__.has_key("supports_unicode_filenames") and os.path.supports_unicode_filenames:
 478             target_root = unicode(target_root, sys.getfilesystemencoding())
 479     finally:
 480         cookie.target_root = target_root
 481
 482     sinks = glob.glob(args[0])
 483     if not sinks:
 484         return 0
 485
 486     sink_families = {}
 487     for sink in sinks:
 488         try:  # In order to allow compatibility below 2.3.
 489             if os.path.__dict__.has_key("supports_unicode_filenames") and os.path.supports_unicode_filenames:
 490                 sink = unicode(sink, sys.getfilesystemencoding())
 491         except:
 492             pass
 493         sink_name = ""
 494         sink_root = sink
 495         sink_drive, sink_root = os.path.splitdrive(sink)
 496         while not sink_name:
 497             if sink_root == os.path.sep:
 498                 sink_name = "."
 499                 break
 500             sink_root, sink_name = os.path.split(sink_root)
 501         sink_root = sink_drive + sink_root
 502         if not sink_families.has_key(sink_root):
 503             sink_families[sink_root] = []
 504         sink_families[sink_root] = sink_families[sink_root] + [sink_name]
 505
 506     for sink_root in sink_families.keys():
 507         if cookie.relative:
 508             cookie.sink_root = ""
 509         else:
 510             cookie.sink_root = sink_root
 511
 512         global y  # In order to allow compatibility below 2.1 (nested scope where used before).
 513         y = sink_root
 514         files = filter(lambda x: os.path.isfile(os.path.join(y, x)), sink_families[sink_root])
 515         if files:
 516             visit(cookie, sink_root, files)
 517
 518         # global y # In order to allow compatibility below 2.1 (nested scope where used before).
 519         y = sink_root
 520         folders = filter(lambda x: os.path.isdir(os.path.join(y, x)), sink_families[sink_root])
 521         for folder in folders:
 522             folder_path = os.path.join(sink_root, folder)
 523             if not cookie.recursive:
 524                 visit(cookie, folder_path, os.listdir(folder_path))
 525             else:
 526                 os.path.walk(folder_path, visit, cookie)
 527     return 0
 528
 529
 530 if __name__ == "__main__":
 531     sys.exit(main(sys.argv[1:]))