3 # Tests for incremental drive-backup
5 # Copyright (C) 2015 John Snow for Red Hat, Inc.
9 # This program is free software; you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation; either version 2 of the License, or
12 # (at your option) any later version.
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
27 def io_write_patterns(img, patterns):
28 for pattern in patterns:
29 iotests.qemu_io('-c', 'write -P%s %s %s' % pattern, img)
39 def transaction_action(action, **kwargs):
42 'data': dict((k.replace('_', '-'), v) for k, v in kwargs.iteritems())
46 def transaction_bitmap_clear(node, name, **kwargs):
47 return transaction_action('block-dirty-bitmap-clear',
48 node=node, name=name, **kwargs)
51 def transaction_drive_backup(device, target, **kwargs):
52 return transaction_action('drive-backup', device=device, target=target,
57 def __init__(self, name, drive):
63 def base_target(self):
64 return (self.drive['backup'], None)
66 def new_target(self, num=None):
70 base = os.path.join(iotests.test_dir,
71 "%s.%s." % (self.drive['id'], self.name))
72 suff = "%i.%s" % (num, self.drive['fmt'])
73 target = base + "inc" + suff
74 reference = base + "ref" + suff
75 self.backups.append((target, reference))
76 return (target, reference)
78 def last_target(self):
80 return self.backups[-1]
81 return self.base_target()
84 for image in self.backups.pop():
89 for backup in self.backups:
94 class TestIncrementalBackupBase(iotests.QMPTestCase):
95 def __init__(self, *args):
96 super(TestIncrementalBackupBase, self).__init__(*args)
100 self.vm = iotests.VM()
101 self.err_img = os.path.join(iotests.test_dir, 'err.%s' % iotests.imgfmt)
105 # Create a base image with a distinctive patterning
106 drive0 = self.add_node('drive0')
107 self.img_create(drive0['file'], drive0['fmt'])
108 self.vm.add_drive(drive0['file'])
109 self.write_default_pattern(drive0['file'])
113 def write_default_pattern(self, target):
114 io_write_patterns(target, (('0x41', 0, 512),
115 ('0xd5', '1M', '32k'),
116 ('0xdc', '32M', '124k')))
119 def add_node(self, node_id, fmt=iotests.imgfmt, path=None, backup=None):
121 path = os.path.join(iotests.test_dir, '%s.%s' % (node_id, fmt))
123 backup = os.path.join(iotests.test_dir,
124 '%s.full.backup.%s' % (node_id, fmt))
131 return self.drives[-1]
134 def img_create(self, img, fmt=iotests.imgfmt, size='64M',
135 parent=None, parentFormat=None, **kwargs):
137 for k,v in kwargs.iteritems():
138 optargs = optargs + ['-o', '%s=%s' % (k,v)]
139 args = ['create', '-f', fmt] + optargs + [img, size]
141 if parentFormat is None:
143 args = args + ['-b', parent, '-F', parentFormat]
144 iotests.qemu_img(*args)
145 self.files.append(img)
148 def do_qmp_backup(self, error='Input/output error', **kwargs):
149 res = self.vm.qmp('drive-backup', **kwargs)
150 self.assert_qmp(res, 'return', {})
151 return self.wait_qmp_backup(kwargs['device'], error)
154 def wait_qmp_backup(self, device, error='Input/output error'):
155 event = self.vm.event_wait(name="BLOCK_JOB_COMPLETED",
156 match={'data': {'device': device}})
157 self.assertNotEqual(event, None)
160 failure = self.dictpath(event, 'data/error')
161 except AssertionError:
163 self.assert_qmp(event, 'data/offset', event['data']['len'])
167 self.assert_qmp(event, 'data/error', error)
171 def wait_qmp_backup_cancelled(self, device):
172 event = self.vm.event_wait(name='BLOCK_JOB_CANCELLED',
173 match={'data': {'device': device}})
174 self.assertNotEqual(event, None)
177 def create_anchor_backup(self, drive=None):
179 drive = self.drives[-1]
180 res = self.do_qmp_backup(device=drive['id'], sync='full',
181 format=drive['fmt'], target=drive['backup'])
183 self.files.append(drive['backup'])
184 return drive['backup']
187 def make_reference_backup(self, bitmap=None):
189 bitmap = self.bitmaps[-1]
190 _, reference = bitmap.last_target()
191 res = self.do_qmp_backup(device=bitmap.drive['id'], sync='full',
192 format=bitmap.drive['fmt'], target=reference)
196 def add_bitmap(self, name, drive, **kwargs):
197 bitmap = Bitmap(name, drive)
198 self.bitmaps.append(bitmap)
199 result = self.vm.qmp('block-dirty-bitmap-add', node=drive['id'],
200 name=bitmap.name, **kwargs)
201 self.assert_qmp(result, 'return', {})
205 def prepare_backup(self, bitmap=None, parent=None):
207 bitmap = self.bitmaps[-1]
209 parent, _ = bitmap.last_target()
211 target, _ = bitmap.new_target()
212 self.img_create(target, bitmap.drive['fmt'], parent=parent)
216 def create_incremental(self, bitmap=None, parent=None,
217 parentFormat=None, validate=True):
219 bitmap = self.bitmaps[-1]
221 parent, _ = bitmap.last_target()
223 target = self.prepare_backup(bitmap, parent)
224 res = self.do_qmp_backup(device=bitmap.drive['id'],
225 sync='incremental', bitmap=bitmap.name,
226 format=bitmap.drive['fmt'], target=target,
230 self.assertFalse(validate)
232 self.make_reference_backup(bitmap)
236 def check_backups(self):
237 for bitmap in self.bitmaps:
238 for incremental, reference in bitmap.backups:
239 self.assertTrue(iotests.compare_images(incremental, reference))
240 last = bitmap.last_target()[0]
241 self.assertTrue(iotests.compare_images(last, bitmap.drive['file']))
244 def hmp_io_writes(self, drive, patterns):
245 for pattern in patterns:
246 self.vm.hmp_qemu_io(drive, 'write -P%s %s %s' % pattern)
247 self.vm.hmp_qemu_io(drive, 'flush')
250 def do_incremental_simple(self, **kwargs):
251 self.create_anchor_backup()
252 self.add_bitmap('bitmap0', self.drives[0], **kwargs)
254 # Sanity: Create a "hollow" incremental backup
255 self.create_incremental()
256 # Three writes: One complete overwrite, one new segment,
257 # and one partial overlap.
258 self.hmp_io_writes(self.drives[0]['id'], (('0xab', 0, 512),
259 ('0xfe', '16M', '256k'),
260 ('0x64', '32736k', '64k')))
261 self.create_incremental()
262 # Three more writes, one of each kind, like above
263 self.hmp_io_writes(self.drives[0]['id'], (('0x9a', 0, 512),
264 ('0x55', '8M', '352k'),
265 ('0x78', '15872k', '1M')))
266 self.create_incremental()
273 for bitmap in self.bitmaps:
275 for filename in self.files:
280 class TestIncrementalBackup(TestIncrementalBackupBase):
281 def test_incremental_simple(self):
283 Test: Create and verify three incremental backups.
285 Create a bitmap and a full backup before VM execution begins,
286 then create a series of three incremental backups "during execution,"
287 i.e.; after IO requests begin modifying the drive.
289 return self.do_incremental_simple()
292 def test_small_granularity(self):
294 Test: Create and verify backups made with a small granularity bitmap.
296 Perform the same test as test_incremental_simple, but with a granularity
297 of only 32KiB instead of the present default of 64KiB.
299 return self.do_incremental_simple(granularity=32768)
302 def test_large_granularity(self):
304 Test: Create and verify backups made with a large granularity bitmap.
306 Perform the same test as test_incremental_simple, but with a granularity
307 of 128KiB instead of the present default of 64KiB.
309 return self.do_incremental_simple(granularity=131072)
312 def test_larger_cluster_target(self):
314 Test: Create and verify backups made to a larger cluster size target.
316 With a default granularity of 64KiB, verify that backups made to a
317 larger cluster size target of 128KiB without a backing file works.
319 drive0 = self.drives[0]
321 # Create a cluster_size=128k full backup / "anchor" backup
322 self.img_create(drive0['backup'], cluster_size='128k')
323 self.assertTrue(self.do_qmp_backup(device=drive0['id'], sync='full',
324 format=drive0['fmt'],
325 target=drive0['backup'],
328 # Create bitmap and dirty it with some new writes.
329 # overwrite [32736, 32799] which will dirty bitmap clusters at
330 # 32M-64K and 32M. 32M+64K will be left undirtied.
331 bitmap0 = self.add_bitmap('bitmap0', drive0)
332 self.hmp_io_writes(drive0['id'],
334 ('0xfe', '16M', '256k'),
335 ('0x64', '32736k', '64k')))
338 # Prepare a cluster_size=128k backup target without a backing file.
339 (target, _) = bitmap0.new_target()
340 self.img_create(target, bitmap0.drive['fmt'], cluster_size='128k')
342 # Perform Incremental Backup
343 self.assertTrue(self.do_qmp_backup(device=bitmap0.drive['id'],
346 format=bitmap0.drive['fmt'],
349 self.make_reference_backup(bitmap0)
351 # Add the backing file, then compare and exit.
352 iotests.qemu_img('rebase', '-f', drive0['fmt'], '-u', '-b',
353 drive0['backup'], '-F', drive0['fmt'], target)
358 def test_incremental_transaction(self):
359 '''Test: Verify backups made from transactionally created bitmaps.
361 Create a bitmap "before" VM execution begins, then create a second
362 bitmap AFTER writes have already occurred. Use transactions to create
363 a full backup and synchronize both bitmaps to this backup.
364 Create an incremental backup through both bitmaps and verify that
365 both backups match the current drive0 image.
368 drive0 = self.drives[0]
369 bitmap0 = self.add_bitmap('bitmap0', drive0)
370 self.hmp_io_writes(drive0['id'], (('0xab', 0, 512),
371 ('0xfe', '16M', '256k'),
372 ('0x64', '32736k', '64k')))
373 bitmap1 = self.add_bitmap('bitmap1', drive0)
375 result = self.vm.qmp('transaction', actions=[
376 transaction_bitmap_clear(bitmap0.drive['id'], bitmap0.name),
377 transaction_bitmap_clear(bitmap1.drive['id'], bitmap1.name),
378 transaction_drive_backup(drive0['id'], drive0['backup'],
379 sync='full', format=drive0['fmt'])
381 self.assert_qmp(result, 'return', {})
382 self.wait_until_completed(drive0['id'])
383 self.files.append(drive0['backup'])
385 self.hmp_io_writes(drive0['id'], (('0x9a', 0, 512),
386 ('0x55', '8M', '352k'),
387 ('0x78', '15872k', '1M')))
388 # Both bitmaps should be correctly in sync.
389 self.create_incremental(bitmap0)
390 self.create_incremental(bitmap1)
395 def test_transaction_failure(self):
396 '''Test: Verify backups made from a transaction that partially fails.
398 Add a second drive with its own unique pattern, and add a bitmap to each
399 drive. Use blkdebug to interfere with the backup on just one drive and
400 attempt to create a coherent incremental backup across both drives.
402 verify a failure in one but not both, then delete the failed stubs and
403 re-run the same transaction.
405 verify that both incrementals are created successfully.
408 # Create a second drive, with pattern:
409 drive1 = self.add_node('drive1')
410 self.img_create(drive1['file'], drive1['fmt'])
411 io_write_patterns(drive1['file'], (('0x14', 0, 512),
412 ('0x5d', '1M', '32k'),
413 ('0xcd', '32M', '124k')))
415 # Create a blkdebug interface to this img as 'drive1'
416 result = self.vm.qmp('blockdev-add', options={
418 'driver': drive1['fmt'],
420 'driver': 'blkdebug',
423 'filename': drive1['file']
426 'event': 'flush_to_disk',
434 'immediately': False,
439 self.assert_qmp(result, 'return', {})
441 # Create bitmaps and full backups for both drives
442 drive0 = self.drives[0]
443 dr0bm0 = self.add_bitmap('bitmap0', drive0)
444 dr1bm0 = self.add_bitmap('bitmap0', drive1)
445 self.create_anchor_backup(drive0)
446 self.create_anchor_backup(drive1)
447 self.assert_no_active_block_jobs()
448 self.assertFalse(self.vm.get_qmp_events(wait=False))
450 # Emulate some writes
451 self.hmp_io_writes(drive0['id'], (('0xab', 0, 512),
452 ('0xfe', '16M', '256k'),
453 ('0x64', '32736k', '64k')))
454 self.hmp_io_writes(drive1['id'], (('0xba', 0, 512),
455 ('0xef', '16M', '256k'),
456 ('0x46', '32736k', '64k')))
458 # Create incremental backup targets
459 target0 = self.prepare_backup(dr0bm0)
460 target1 = self.prepare_backup(dr1bm0)
462 # Ask for a new incremental backup per-each drive,
463 # expecting drive1's backup to fail:
465 transaction_drive_backup(drive0['id'], target0, sync='incremental',
466 format=drive0['fmt'], mode='existing',
468 transaction_drive_backup(drive1['id'], target1, sync='incremental',
469 format=drive1['fmt'], mode='existing',
472 result = self.vm.qmp('transaction', actions=transaction,
473 properties={'completion-mode': 'grouped'} )
474 self.assert_qmp(result, 'return', {})
476 # Observe that drive0's backup is cancelled and drive1 completes with
478 self.wait_qmp_backup_cancelled(drive0['id'])
479 self.assertFalse(self.wait_qmp_backup(drive1['id']))
480 error = self.vm.event_wait('BLOCK_JOB_ERROR')
481 self.assert_qmp(error, 'data', {'device': drive1['id'],
483 'operation': 'read'})
484 self.assertFalse(self.vm.get_qmp_events(wait=False))
485 self.assert_no_active_block_jobs()
487 # Delete drive0's successful target and eliminate our record of the
488 # unsuccessful drive1 target. Then re-run the same transaction.
491 target0 = self.prepare_backup(dr0bm0)
492 target1 = self.prepare_backup(dr1bm0)
494 # Re-run the exact same transaction.
495 result = self.vm.qmp('transaction', actions=transaction,
496 properties={'completion-mode':'grouped'})
497 self.assert_qmp(result, 'return', {})
499 # Both should complete successfully this time.
500 self.assertTrue(self.wait_qmp_backup(drive0['id']))
501 self.assertTrue(self.wait_qmp_backup(drive1['id']))
502 self.make_reference_backup(dr0bm0)
503 self.make_reference_backup(dr1bm0)
504 self.assertFalse(self.vm.get_qmp_events(wait=False))
505 self.assert_no_active_block_jobs()
507 # And the images should of course validate.
512 def test_sync_dirty_bitmap_missing(self):
513 self.assert_no_active_block_jobs()
514 self.files.append(self.err_img)
515 result = self.vm.qmp('drive-backup', device=self.drives[0]['id'],
516 sync='incremental', format=self.drives[0]['fmt'],
518 self.assert_qmp(result, 'error/class', 'GenericError')
521 def test_sync_dirty_bitmap_not_found(self):
522 self.assert_no_active_block_jobs()
523 self.files.append(self.err_img)
524 result = self.vm.qmp('drive-backup', device=self.drives[0]['id'],
525 sync='incremental', bitmap='unknown',
526 format=self.drives[0]['fmt'], target=self.err_img)
527 self.assert_qmp(result, 'error/class', 'GenericError')
530 def test_sync_dirty_bitmap_bad_granularity(self):
532 Test: Test what happens if we provide an improper granularity.
534 The granularity must always be a power of 2.
536 self.assert_no_active_block_jobs()
537 self.assertRaises(AssertionError, self.add_bitmap,
538 'bitmap0', self.drives[0],
542 class TestIncrementalBackupBlkdebug(TestIncrementalBackupBase):
543 '''Incremental backup tests that utilize a BlkDebug filter on drive0.'''
546 drive0 = self.add_node('drive0')
547 self.img_create(drive0['file'], drive0['fmt'])
548 self.write_default_pattern(drive0['file'])
551 def test_incremental_failure(self):
552 '''Test: Verify backups made after a failure are correct.
554 Simulate a failure during an incremental backup block job,
555 emulate additional writes, then create another incremental backup
556 afterwards and verify that the backup created is correct.
559 drive0 = self.drives[0]
560 result = self.vm.qmp('blockdev-add', options={
562 'driver': drive0['fmt'],
564 'driver': 'blkdebug',
567 'filename': drive0['file']
570 'event': 'flush_to_disk',
578 'immediately': False,
583 self.assert_qmp(result, 'return', {})
585 self.create_anchor_backup(drive0)
586 self.add_bitmap('bitmap0', drive0)
587 # Note: at this point, during a normal execution,
588 # Assume that the VM resumes and begins issuing IO requests here.
590 self.hmp_io_writes(drive0['id'], (('0xab', 0, 512),
591 ('0xfe', '16M', '256k'),
592 ('0x64', '32736k', '64k')))
594 result = self.create_incremental(validate=False)
595 self.assertFalse(result)
596 self.hmp_io_writes(drive0['id'], (('0x9a', 0, 512),
597 ('0x55', '8M', '352k'),
598 ('0x78', '15872k', '1M')))
599 self.create_incremental()
604 if __name__ == '__main__':
605 iotests.main(supported_fmts=['qcow2'])