testsuites/vstf/vstf_scripts/vstf/common/rsync.py

   1 ##############################################################################
   2 # Copyright (c) 2015 Huawei Technologies Co.,Ltd and others.
   3 #
   4 # All rights reserved. This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 # http://www.apache.org/licenses/LICENSE-2.0
   8 ##############################################################################
   9
  10
  11 # from __future__ import nested_scopes
  12
  13 import os
  14 import os.path
  15 import shutil
  16 import glob
  17 import re
  18 import sys
  19 import getopt
  20 import stat
  21 import string
  22
  23 try:
  24     import win32file
  25 except:
  26     win32file = None
  27
  28
  29 class Cookie:
  30
  31     def __init__(self):
  32         self.sink_root = ""
  33         self.target_root = ""
  34         self.quiet = 0
  35         self.recursive = 0
  36         self.relative = 0
  37         self.dry_run = 0
  38         self.time = 0
  39         self.update = 0
  40         self.cvs_ignore = 0
  41         self.ignore_time = 0
  42         self.delete = 0
  43         self.delete_excluded = 0
  44         self.delete_from_source = 0
  45         self.size_only = 0
  46         self.modify_window = 2
  47         self.existing = 0
  48         self.filters = []
  49         self.case_sensitivity = 0
  50         if os.name == "nt":
  51             self.case_sensitivity = re.I
  52
  53
  54 def visit(cookie, dirname, names):
  55     """Copy files names from sink_root + (dirname - sink_root) to target_root + (dirname - sink_root)"""
  56     if os.path.split(cookie.sink_root)[
  57             1]:  # Should be tested with (C:\Cvs -> C:\)! (C:\Archives\MyDatas\UltraEdit -> C:\Archives\MyDatas) (Cvs -> "")! (Archives\MyDatas\UltraEdit -> Archives\MyDatas) (\Cvs -> \)! (\Archives\MyDatas\UltraEdit -> Archives\MyDatas)
  58         dirname = dirname[len(cookie.sink_root) + 1:]
  59     else:
  60         dirname = dirname[len(cookie.sink_root):]
  61     target_dir = os.path.join(cookie.target_root, dirname)
  62     if not os.path.isdir(target_dir):
  63         makeDir(cookie, target_dir)
  64     sink_dir = os.path.join(cookie.sink_root, dirname)
  65
  66     filters = []
  67     if cookie.cvs_ignore:
  68         ignore = os.path.join(sink_dir, ".cvsignore")
  69         if os.path.isfile(ignore):
  70             filters = convertPatterns(ignore, "-")
  71     filters = filters + cookie.filters
  72
  73     names_excluded = []
  74     if filters:
  75         # filter sink files (names):
  76         name_index = 0
  77         while name_index < len(names):
  78             name = names[name_index]
  79             path = os.path.join(dirname, name)
  80             path = convertPath(path)
  81             if os.path.isdir(os.path.join(sink_dir, name)):
  82                 path = path + "/"
  83             for filter in filters:
  84                 if re.search(filter[1], path, cookie.case_sensitivity):
  85                     if filter[0] == '-':
  86                         sink = os.path.join(sink_dir, name)
  87                         if cookie.delete_from_source:
  88                             if os.path.isfile(sink):
  89                                 removeFile(cookie, sink)
  90                             elif os.path.isdir(sink):
  91                                 removeDir(cookie, sink)
  92                             else:
  93                                 logError(
  94                                     "Sink %s is neither a file nor a folder (skip removal)" %
  95                                     sink)
  96                         names_excluded += [names[name_index]]
  97                         del (names[name_index])
  98                         name_index = name_index - 1
  99                         break
 100                     elif filter[0] == '+':
 101                         break
 102             name_index = name_index + 1
 103
 104     if cookie.delete and os.path.isdir(target_dir):
 105         # Delete files and folder in target not present in filtered sink.
 106         for name in os.listdir(target_dir):
 107             if not cookie.delete_excluded and name in names_excluded:
 108                 continue
 109             if name not in names:
 110                 target = os.path.join(target_dir, name)
 111                 if os.path.isfile(target):
 112                     removeFile(cookie, target)
 113                 elif os.path.isdir(target):
 114                     removeDir(cookie, target)
 115                 else:
 116                     pass
 117
 118     for name in names:
 119         # Copy files and folder from sink to target.
 120         sink = os.path.join(sink_dir, name)
 121         # print sink
 122         target = os.path.join(target_dir, name)
 123         if os.path.exists(target):
 124             # When target already exit:
 125             if os.path.isfile(sink):
 126                 if os.path.isfile(target):
 127                     # file-file
 128                     if shouldUpdate(cookie, sink, target):
 129                         updateFile(cookie, sink, target)
 130                 elif os.path.isdir(target):
 131                     # file-folder
 132                     removeDir(cookie, target)
 133                     copyFile(cookie, sink, target)
 134                 else:
 135                     # file-???
 136                     logError(
 137                         "Target %s is neither a file nor folder (skip update)" %
 138                         sink)
 139
 140             elif os.path.isdir(sink):
 141                 if os.path.isfile(target):
 142                     # folder-file
 143                     removeFile(cookie, target)
 144                     makeDir(cookie, target)
 145             else:
 146                 # ???-xxx
 147                 logError(
 148                     "Sink %s is neither a file nor a folder (skip update)" %
 149                     sink)
 150
 151         elif not cookie.existing:
 152             # When target dont exist:
 153             if os.path.isfile(sink):
 154                 # file
 155                 copyFile(cookie, sink, target)
 156             elif os.path.isdir(sink):
 157                 # folder
 158                 makeDir(cookie, target)
 159             else:
 160                 logError(
 161                     "Sink %s is neither a file nor a folder (skip update)" %
 162                     sink)
 163
 164
 165 def log(cookie, message):
 166     if not cookie.quiet:
 167         try:
 168             print message
 169         except UnicodeEncodeError:
 170             print message.encode("utf8")
 171
 172
 173 def logError(message):
 174     try:
 175         sys.stderr.write(message + "\n")
 176     except UnicodeEncodeError:
 177         sys.stderr.write(message.encode("utf8") + "\n")
 178
 179
 180 def shouldUpdate(cookie, sink, target):
 181     try:
 182         sink_st = os.stat(sink)
 183         sink_sz = sink_st.st_size
 184         sink_mt = sink_st.st_mtime
 185     except:
 186         logError(
 187             "Fail to retrieve information about sink %s (skip update)" %
 188             sink)
 189         return 0
 190
 191     try:
 192         target_st = os.stat(target)
 193         target_sz = target_st.st_size
 194         target_mt = target_st.st_mtime
 195     except:
 196         logError(
 197             "Fail to retrieve information about target %s (skip update)" %
 198             target)
 199         return 0
 200
 201     if cookie.update:
 202         return target_mt < sink_mt - cookie.modify_window
 203
 204     if cookie.ignore_time:
 205         return 1
 206
 207     if target_sz != sink_sz:
 208         return 1
 209
 210     if cookie.size_only:
 211         return 0
 212
 213     return abs(target_mt - sink_mt) > cookie.modify_window
 214
 215
 216 def copyFile(cookie, sink, target):
 217     log(cookie, "copy: %s to: %s" % (sink, target))
 218     if not cookie.dry_run:
 219         try:
 220             shutil.copyfile(sink, target)
 221         except:
 222             logError("Fail to copy %s" % sink)
 223
 224         if cookie.time:
 225             try:
 226                 s = os.stat(sink)
 227                 os.utime(target, (s.st_atime, s.st_mtime))
 228             except:
 229                 logError("Fail to copy timestamp of %s" % sink)
 230
 231
 232 def updateFile(cookie, sink, target):
 233     log(cookie, "update: %s to: %s" % (sink, target))
 234     if not cookie.dry_run:
 235         # Read only and hidden and system files can not be overridden.
 236         try:
 237             try:
 238                 if win32file:
 239                     filemode = win32file.GetFileAttributesW(target)
 240                     win32file.SetFileAttributesW(
 241                         target,
 242                         filemode & ~win32file.FILE_ATTRIBUTE_READONLY & ~win32file.FILE_ATTRIBUTE_HIDDEN & ~win32file.FILE_ATTRIBUTE_SYSTEM)
 243                 else:
 244                     os.chmod(target, stat.S_IWUSR)
 245             except:
 246                 # logError("Fail to allow override of %s" % target)
 247                 pass
 248
 249             shutil.copyfile(sink, target)
 250             if cookie.time:
 251                 try:
 252                     s = os.stat(sink)
 253                     os.utime(target, (s.st_atime, s.st_mtime))
 254                 except:
 255                     # The utime api of the 2.3 version of python is not unicode
 256                     # compliant.
 257                     logError("Fail to copy timestamp of %s" % sink)
 258         except:
 259             logError("Fail to override %s" % sink)
 260
 261         if win32file:
 262             win32file.SetFileAttributesW(target, filemode)
 263
 264
 265 def prepareRemoveFile(path):
 266     if win32file:
 267         filemode = win32file.GetFileAttributesW(path)
 268         win32file.SetFileAttributesW(path, filemode & ~win32file.FILE_ATTRIBUTE_READONLY &
 269                                      ~win32file.FILE_ATTRIBUTE_HIDDEN & ~win32file.FILE_ATTRIBUTE_SYSTEM)
 270     else:
 271         os.chmod(path, stat.S_IWUSR)
 272
 273
 274 def removeFile(cookie, target):
 275     # Read only files could not be deleted.
 276     log(cookie, "remove: %s" % target)
 277     if not cookie.dry_run:
 278         try:
 279             try:
 280                 prepareRemoveFile(target)
 281             except:
 282                 # logError("Fail to allow removal of %s" % target)
 283                 pass
 284
 285             os.remove(target)
 286         except:
 287             logError("Fail to remove %s" % target)
 288
 289
 290 def makeDir(cookie, target):
 291     log(cookie, "make dir: %s" % target)
 292     if not cookie.dry_run:
 293         try:
 294             os.makedirs(target)
 295         except:
 296             logError("Fail to make dir %s" % target)
 297
 298
 299 def visitForPrepareRemoveDir(arg, dirname, names):
 300     for name in names:
 301         path = os.path.join(dirname, name)
 302         prepareRemoveFile(path)
 303
 304
 305 def prepareRemoveDir(path):
 306     prepareRemoveFile(path)
 307     os.path.walk(path, visitForPrepareRemoveDir, None)
 308
 309
 310 def OnRemoveDirError(func, path, excinfo):
 311     logError("Fail to remove %s" % path)
 312
 313
 314 def removeDir(cookie, target):
 315     # Read only directory could not be deleted.
 316     log(cookie, "remove dir: %s" % target)
 317     if not cookie.dry_run:
 318         prepareRemoveDir(target)
 319         try:
 320             shutil.rmtree(target, False, OnRemoveDirError)
 321         except:
 322             logError("Fail to remove dir %s" % target)
 323
 324
 325 def convertPath(path):
 326     # Convert windows, mac path to unix version.
 327     separator = os.path.normpath("/")
 328     if separator != "/":
 329         path = re.sub(re.escape(separator), "/", path)
 330
 331     # Help file, folder pattern to express that it should match the all file
 332     # or folder name.
 333     path = "/" + path
 334     return path
 335
 336
 337 def convertPattern(pattern, sign):
 338     """Convert a rsync pattern that match against a path to a filter that match against a converted path."""
 339
 340     # Check for include vs exclude patterns.
 341     if pattern[:2] == "+ ":
 342         pattern = pattern[2:]
 343         sign = "+"
 344     elif pattern[:2] == "- ":
 345         pattern = pattern[2:]
 346         sign = "-"
 347
 348     # Express windows, mac patterns in unix patterns (rsync.py extension).
 349     separator = os.path.normpath("/")
 350     if separator != "/":
 351         pattern = re.sub(re.escape(separator), "/", pattern)
 352
 353     # If pattern contains '/' it should match from the start.
 354     temp = pattern
 355     if pattern[0] == "/":
 356         pattern = pattern[1:]
 357     if temp[-1] == "/":
 358         temp = temp[:-1]
 359
 360     # Convert pattern rules: ** * ? to regexp rules.
 361     pattern = re.escape(pattern)
 362     pattern = string.replace(pattern, "\\?", ".")
 363     pattern = string.replace(pattern, "\\*\\*", ".*")
 364     pattern = string.replace(pattern, "\\*", "[^/]*")
 365     pattern = string.replace(pattern, "\\*", ".*")
 366
 367     if "/" in temp:
 368         # If pattern contains '/' it should match from the start.
 369         pattern = "^\\/" + pattern
 370     else:
 371         # Else the pattern should match the all file or folder name.
 372         pattern = "\\/" + pattern
 373
 374     if pattern[-2:] != "\\/" and pattern[-2:] != ".*":
 375         # File patterns should match also folders.
 376         pattern = pattern + "\\/?"
 377
 378     # Pattern should match till the end.
 379     pattern = pattern + "$"
 380     return (sign, pattern)
 381
 382
 383 def convertPatterns(path, sign):
 384     """Read the files for pattern and return a vector of filters"""
 385     filters = []
 386     f = open(path, "r")
 387     while True:
 388         pattern = f.readline()
 389         if not pattern:
 390             break
 391         if pattern[-1] == "\n":
 392             pattern = pattern[:-1]
 393
 394         if re.match("[\t ]*$", pattern):
 395             continue
 396         if pattern[0] == "#":
 397             continue
 398         filters = filters + [convertPattern(pattern, sign)]
 399     f.close()
 400     return filters
 401
 402
 403 def printUsage():
 404     """Print the help string that should printed by rsync.py -h"""
 405     print "usage: rsync.py [options] source target"
 406     print """
 407  -q, --quiet              decrease verbosity
 408  -r, --recursive          recurse into directories
 409  -R, --relative           use relative path names
 410  -u, --update             update only (don't overwrite newer files)
 411  -t, --times              preserve times
 412  -n, --dry-run            show what would have been transferred
 413      --existing           only update files that already exist
 414      --delete             delete files that don't exist on the sending side
 415      --delete-excluded    also delete excluded files on the receiving side
 416      --delete-from-source delete excluded files on the receiving side
 417  -I, --ignore-times       don't exclude files that match length and time
 418      --size-only          only use file size when determining if a file should
 419                           be transferred
 420      --modify-window=NUM  timestamp window (seconds) for file match (default=2)
 421      --existing           only update existing target files or folders
 422  -C, --cvs-exclude        auto ignore files in the same way CVS does
 423      --exclude=PATTERN    exclude files matching PATTERN
 424      --exclude-from=FILE  exclude patterns listed in FILE
 425      --include=PATTERN    don't exclude files matching PATTERN
 426      --include-from=FILE  don't exclude patterns listed in FILE
 427      --version            print version number
 428  -h, --help               show this help screen
 429
 430 See http://www.vdesmedt.com/~vds2212/rsync.html for informations and updates.
 431 Send an email to vivian@vdesmedt.com for comments and bug reports."""
 432
 433
 434 def printVersion():
 435     print "rsync.py version 2.0.1"
 436
 437
 438 def main(args):
 439     cookie = Cookie()
 440
 441     opts, args = getopt.getopt(args, "qrRntuCIh",
 442                                ["quiet", "recursive", "relative", "dry-run", "time", "update", "cvs-ignore",
 443                                 "ignore-times", "help", "delete", "delete-excluded", "delete-from-source", "existing",
 444                                 "size-only", "modify-window=", "exclude=", "exclude-from=", "include=", "include-from=",
 445                                 "version"])
 446     for o, v in opts:
 447         if o in ["-q", "--quiet"]:
 448             cookie.quiet = 1
 449         if o in ["-r", "--recursive"]:
 450             cookie.recursive = 1
 451         if o in ["-R", "--relative"]:
 452             cookie.relative = 1
 453         elif o in ["-n", "--dry-run"]:
 454             cookie.dry_run = 1
 455         # --time is there to guaranty backward compatibility with previous buggy version.
 456         elif o in ["-t", "--times", "--time"]:
 457             cookie.time = 1
 458         elif o in ["-u", "--update"]:
 459             cookie.update = 1
 460         elif o in ["-C", "--cvs-ignore"]:
 461             cookie.cvs_ignore = 1
 462         elif o in ["-I", "--ignore-time"]:
 463             cookie.ignore_time = 1
 464         elif o == "--delete":
 465             cookie.delete = 1
 466         elif o == "--delete-excluded":
 467             cookie.delete = 1
 468             cookie.delete_excluded = 1
 469         elif o == "--delete-from-source":
 470             cookie.delete_from_source = 1
 471         elif o == "--size-only":
 472             cookie.size_only = 1
 473         elif o == "--modify-window":
 474             cookie.modify_window = int(v)
 475         elif o == "--existing":
 476             cookie.existing = 1
 477         elif o == "--exclude":
 478             cookie.filters = cookie.filters + [convertPattern(v, "-")]
 479         elif o == "--exclude-from":
 480             cookie.filters = cookie.filters + convertPatterns(v, "-")
 481         elif o == "--include":
 482             cookie.filters = cookie.filters + [convertPattern(v, "+")]
 483         elif o == "--include-from":
 484             cookie.filters = cookie.filters + convertPatterns(v, "+")
 485         elif o == "--version":
 486             printVersion()
 487             return 0
 488         elif o in ["-h", "--help"]:
 489             printUsage()
 490             return 0
 491
 492     if len(args) <= 1:
 493         printUsage()
 494         return 1
 495
 496     # print cookie.filters
 497
 498     target_root = args[1]
 499     try:  # In order to allow compatibility below 2.3.
 500         pass
 501         if "supports_unicode_filenames" in os.path.__dict__ and os.path.supports_unicode_filenames:
 502             target_root = unicode(target_root, sys.getfilesystemencoding())
 503     finally:
 504         cookie.target_root = target_root
 505
 506     sinks = glob.glob(args[0])
 507     if not sinks:
 508         return 0
 509
 510     sink_families = {}
 511     for sink in sinks:
 512         try:  # In order to allow compatibility below 2.3.
 513             if "supports_unicode_filenames" in os.path.__dict__ and os.path.supports_unicode_filenames:
 514                 sink = unicode(sink, sys.getfilesystemencoding())
 515         except:
 516             pass
 517         sink_name = ""
 518         sink_root = sink
 519         sink_drive, sink_root = os.path.splitdrive(sink)
 520         while not sink_name:
 521             if sink_root == os.path.sep:
 522                 sink_name = "."
 523                 break
 524             sink_root, sink_name = os.path.split(sink_root)
 525         sink_root = sink_drive + sink_root
 526         if sink_root not in sink_families:
 527             sink_families[sink_root] = []
 528         sink_families[sink_root] = sink_families[sink_root] + [sink_name]
 529
 530     for sink_root in sink_families.keys():
 531         if cookie.relative:
 532             cookie.sink_root = ""
 533         else:
 534             cookie.sink_root = sink_root
 535
 536         # In order to allow compatibility below 2.1 (nested scope where used
 537         # before).
 538         global y
 539         y = sink_root
 540         files = filter(
 541             lambda x: os.path.isfile(
 542                 os.path.join(
 543                     y,
 544                     x)),
 545             sink_families[sink_root])
 546         if files:
 547             visit(cookie, sink_root, files)
 548
 549         # global y # In order to allow compatibility below 2.1 (nested scope
 550         # where used before).
 551         y = sink_root
 552         folders = filter(
 553             lambda x: os.path.isdir(
 554                 os.path.join(
 555                     y,
 556                     x)),
 557             sink_families[sink_root])
 558         for folder in folders:
 559             folder_path = os.path.join(sink_root, folder)
 560             if not cookie.recursive:
 561                 visit(cookie, folder_path, os.listdir(folder_path))
 562             else:
 563                 os.path.walk(folder_path, visit, cookie)
 564     return 0
 565
 566
 567 if __name__ == "__main__":
 568     sys.exit(main(sys.argv[1:]))