src/ceph/src/test/osd/TestPGLog.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3 /*
   4  * Ceph - scalable distributed file system
   5  *
   6  * Copyright (C) 2013 Cloudwatt <libre.licensing@cloudwatt.com>
   7  *
   8  * Author: Loic Dachary <loic@dachary.org>
   9  *
  10  * This program is free software; you can redistribute it and/or modify
  11  * it under the terms of the GNU Library Public License as published by
  12  * the Free Software Foundation; either version 2, or (at your option)
  13  * any later version.
  14  *
  15  * This program is distributed in the hope that it will be useful,
  16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18  * GNU Library Public License for more details.
  19  *
  20  */
  21
  22 #include <stdio.h>
  23 #include <signal.h>
  24 #include "gtest/gtest.h"
  25 #include "osd/PGLog.h"
  26 #include "osd/OSDMap.h"
  27 #include "include/coredumpctl.h"
  28 #include "../objectstore/store_test_fixture.h"
  29
  30
  31 struct PGLogTestBase {
  32   static hobject_t mk_obj(unsigned id) {
  33     hobject_t hoid;
  34     stringstream ss;
  35     ss << "obj_" << id;
  36     hoid.oid = ss.str();
  37     hoid.set_hash(id);
  38     hoid.pool = 1;
  39     return hoid;
  40   }
  41   static eversion_t mk_evt(unsigned ep, unsigned v) {
  42     return eversion_t(ep, v);
  43   }
  44   static pg_log_entry_t mk_ple_mod(
  45     const hobject_t &hoid, eversion_t v, eversion_t pv, osd_reqid_t reqid) {
  46     pg_log_entry_t e;
  47     e.mark_unrollbackable();
  48     e.op = pg_log_entry_t::MODIFY;
  49     e.soid = hoid;
  50     e.version = v;
  51     e.prior_version = pv;
  52     e.reqid = reqid;
  53     return e;
  54   }
  55   static pg_log_entry_t mk_ple_dt(
  56     const hobject_t &hoid, eversion_t v, eversion_t pv, osd_reqid_t reqid) {
  57     pg_log_entry_t e;
  58     e.mark_unrollbackable();
  59     e.op = pg_log_entry_t::DELETE;
  60     e.soid = hoid;
  61     e.version = v;
  62     e.prior_version = pv;
  63     e.reqid = reqid;
  64     return e;
  65   }
  66   static pg_log_entry_t mk_ple_ldt(
  67     const hobject_t &hoid, eversion_t v, eversion_t pv) {
  68     pg_log_entry_t e;
  69     e.mark_unrollbackable();
  70     e.op = pg_log_entry_t::LOST_DELETE;
  71     e.soid = hoid;
  72     e.version = v;
  73     e.prior_version = pv;
  74     return e;
  75   }
  76   static pg_log_entry_t mk_ple_mod_rb(
  77     const hobject_t &hoid, eversion_t v, eversion_t pv, osd_reqid_t reqid) {
  78     pg_log_entry_t e;
  79     e.op = pg_log_entry_t::MODIFY;
  80     e.soid = hoid;
  81     e.version = v;
  82     e.prior_version = pv;
  83     e.reqid = reqid;
  84     return e;
  85   }
  86   static pg_log_entry_t mk_ple_dt_rb(
  87     const hobject_t &hoid, eversion_t v, eversion_t pv, osd_reqid_t reqid) {
  88     pg_log_entry_t e;
  89     e.op = pg_log_entry_t::DELETE;
  90     e.soid = hoid;
  91     e.version = v;
  92     e.prior_version = pv;
  93     e.reqid = reqid;
  94     return e;
  95   }
  96   static pg_log_entry_t mk_ple_err(
  97     const hobject_t &hoid, eversion_t v, osd_reqid_t reqid) {
  98     pg_log_entry_t e;
  99     e.op = pg_log_entry_t::ERROR;
 100     e.soid = hoid;
 101     e.version = v;
 102     e.prior_version = eversion_t(0, 0);
 103     e.reqid = reqid;
 104     return e;
 105   }
 106   static pg_log_entry_t mk_ple_mod(
 107     const hobject_t &hoid, eversion_t v, eversion_t pv) {
 108     return mk_ple_mod(hoid, v, pv, osd_reqid_t());
 109   }
 110   static pg_log_entry_t mk_ple_dt(
 111     const hobject_t &hoid, eversion_t v, eversion_t pv) {
 112     return mk_ple_dt(hoid, v, pv, osd_reqid_t());
 113   }
 114   static pg_log_entry_t mk_ple_mod_rb(
 115     const hobject_t &hoid, eversion_t v, eversion_t pv) {
 116     return mk_ple_mod_rb(hoid, v, pv, osd_reqid_t());
 117   }
 118   static pg_log_entry_t mk_ple_dt_rb(
 119     const hobject_t &hoid, eversion_t v, eversion_t pv) {
 120     return mk_ple_dt_rb(hoid, v, pv, osd_reqid_t());
 121   }
 122   static pg_log_entry_t mk_ple_err(
 123     const hobject_t &hoid, eversion_t v) {
 124     return mk_ple_err(hoid, v, osd_reqid_t());
 125   }
 126 }; // PGLogTestBase
 127
 128
 129 class PGLogTest : virtual public ::testing::Test, protected PGLog, public PGLogTestBase  {
 130 public:
 131   PGLogTest() : PGLog(g_ceph_context) {}
 132   void SetUp() override {
 133     missing.may_include_deletes = true;
 134   }
 135
 136 #include "common/ceph_context.h"
 137 #include "common/config.h"
 138
 139   void TearDown() override {
 140     clear();
 141   }
 142
 143
 144   struct TestCase {
 145     list<pg_log_entry_t> base;
 146     list<pg_log_entry_t> auth;
 147     list<pg_log_entry_t> div;
 148
 149     pg_missing_t init;
 150     pg_missing_t final;
 151
 152     set<hobject_t> toremove;
 153     list<pg_log_entry_t> torollback;
 154     bool deletes_during_peering;
 155
 156   private:
 157     IndexedLog fullauth;
 158     IndexedLog fulldiv;
 159     pg_info_t authinfo;
 160     pg_info_t divinfo;
 161   public:
 162     TestCase() : deletes_during_peering(false) {}
 163     void setup() {
 164       init.may_include_deletes = !deletes_during_peering;
 165       final.may_include_deletes = !deletes_during_peering;
 166       fullauth.log.insert(fullauth.log.end(), base.begin(), base.end());
 167       fullauth.log.insert(fullauth.log.end(), auth.begin(), auth.end());
 168       fulldiv.log.insert(fulldiv.log.end(), base.begin(), base.end());
 169       fulldiv.log.insert(fulldiv.log.end(), div.begin(), div.end());
 170
 171       fullauth.head = authinfo.last_update = fullauth.log.rbegin()->version;
 172       authinfo.last_complete = fullauth.log.rbegin()->version;
 173       authinfo.log_tail = fullauth.log.begin()->version;
 174       authinfo.log_tail.version--;
 175       fullauth.tail = authinfo.log_tail;
 176       authinfo.last_backfill = hobject_t::get_max();
 177
 178       fulldiv.head = divinfo.last_update = fulldiv.log.rbegin()->version;
 179       divinfo.last_complete = eversion_t();
 180       divinfo.log_tail = fulldiv.log.begin()->version;
 181       divinfo.log_tail.version--;
 182       fulldiv.tail = divinfo.log_tail;
 183       divinfo.last_backfill = hobject_t::get_max();
 184
 185       if (init.get_items().empty()) {
 186         divinfo.last_complete = divinfo.last_update;
 187       } else {
 188         eversion_t fmissing = init.get_items().at(init.get_rmissing().begin()->second).need;
 189         for (list<pg_log_entry_t>::const_iterator i = fulldiv.log.begin();
 190              i != fulldiv.log.end();
 191              ++i) {
 192           if (i->version < fmissing)
 193             divinfo.last_complete = i->version;
 194           else
 195             break;
 196         }
 197       }
 198
 199       fullauth.index();
 200       fulldiv.index();
 201     }
 202     void set_div_bounds(eversion_t head, eversion_t tail) {
 203       fulldiv.tail = divinfo.log_tail = tail;
 204       fulldiv.head = divinfo.last_update = head;
 205     }
 206     void set_auth_bounds(eversion_t head, eversion_t tail) {
 207       fullauth.tail = authinfo.log_tail = tail;
 208       fullauth.head = authinfo.last_update = head;
 209     }
 210     const IndexedLog &get_fullauth() const { return fullauth; }
 211     const IndexedLog &get_fulldiv() const { return fulldiv; }
 212     const pg_info_t &get_authinfo() const { return authinfo; }
 213     const pg_info_t &get_divinfo() const { return divinfo; }
 214   }; // struct TestCase
 215
 216   struct LogHandler : public PGLog::LogEntryHandler {
 217     set<hobject_t> removed;
 218     list<pg_log_entry_t> rolledback;
 219
 220     void rollback(
 221       const pg_log_entry_t &entry) override {
 222       rolledback.push_back(entry);
 223     }
 224     void rollforward(
 225       const pg_log_entry_t &entry) override {}
 226     void remove(
 227       const hobject_t &hoid) override {
 228       removed.insert(hoid);
 229     }
 230     void try_stash(const hobject_t &, version_t) override {
 231       // lost/unfound cases are not tested yet
 232     }
 233     void trim(
 234       const pg_log_entry_t &entry) override {}
 235   };
 236
 237   template <typename missing_t>
 238   void verify_missing(
 239     const TestCase &tcase,
 240     const missing_t &missing) {
 241     ASSERT_EQ(tcase.final.get_items().size(), missing.get_items().size());
 242     for (auto i = missing.get_items().begin();
 243          i != missing.get_items().end();
 244          ++i) {
 245       EXPECT_TRUE(tcase.final.get_items().count(i->first));
 246       EXPECT_EQ(tcase.final.get_items().find(i->first)->second.need, i->second.need);
 247       EXPECT_EQ(tcase.final.get_items().find(i->first)->second.have, i->second.have);
 248     }
 249     bool correct = missing.debug_verify_from_init(tcase.init, &(std::cout));
 250     ASSERT_TRUE(correct);
 251   }
 252
 253   void verify_sideeffects(
 254     const TestCase &tcase,
 255     const LogHandler &handler) {
 256     ASSERT_EQ(tcase.toremove.size(), handler.removed.size());
 257     ASSERT_EQ(tcase.torollback.size(), handler.rolledback.size());
 258
 259     {
 260       list<pg_log_entry_t>::const_iterator titer = tcase.torollback.begin();
 261       list<pg_log_entry_t>::const_iterator hiter = handler.rolledback.begin();
 262       for (; titer != tcase.torollback.end(); ++titer, ++hiter) {
 263         EXPECT_EQ(titer->version, hiter->version);
 264       }
 265     }
 266
 267     {
 268       set<hobject_t>::const_iterator titer = tcase.toremove.begin();
 269       set<hobject_t>::const_iterator hiter = handler.removed.begin();
 270       for (; titer != tcase.toremove.end(); ++titer, ++hiter) {
 271         EXPECT_EQ(*titer, *hiter);
 272       }
 273     }
 274   }
 275
 276   void test_merge_log(const TestCase &tcase) {
 277     clear();
 278     log = tcase.get_fulldiv();
 279     pg_info_t info = tcase.get_divinfo();
 280
 281     missing = tcase.init;
 282     missing.flush();
 283
 284     IndexedLog olog;
 285     olog = tcase.get_fullauth();
 286     pg_info_t oinfo = tcase.get_authinfo();
 287
 288     LogHandler h;
 289     bool dirty_info = false;
 290     bool dirty_big_info = false;
 291     merge_log(
 292       oinfo, olog, pg_shard_t(1, shard_id_t(0)), info,
 293       &h, dirty_info, dirty_big_info);
 294
 295     ASSERT_EQ(info.last_update, oinfo.last_update);
 296     verify_missing(tcase, missing);
 297     verify_sideeffects(tcase, h);
 298   }
 299
 300   void test_proc_replica_log(const TestCase &tcase) {
 301     clear();
 302     log = tcase.get_fullauth();
 303     pg_info_t info = tcase.get_authinfo();
 304
 305     pg_missing_t omissing = tcase.init;
 306
 307     IndexedLog olog;
 308     olog = tcase.get_fulldiv();
 309     pg_info_t oinfo = tcase.get_divinfo();
 310
 311     proc_replica_log(
 312       oinfo, olog, omissing, pg_shard_t(1, shard_id_t(0)));
 313
 314     assert(oinfo.last_update >= log.tail);
 315
 316     if (!tcase.base.empty()) {
 317       ASSERT_EQ(tcase.base.rbegin()->version, oinfo.last_update);
 318     }
 319
 320     for (list<pg_log_entry_t>::const_iterator i = tcase.auth.begin();
 321          i != tcase.auth.end();
 322          ++i) {
 323       if (i->version > oinfo.last_update) {
 324         if (i->is_delete() && tcase.deletes_during_peering) {
 325           omissing.rm(i->soid, i->version);
 326         } else {
 327           omissing.add_next_event(*i);
 328         }
 329       }
 330     }
 331     verify_missing(tcase, omissing);
 332   } // test_proc_replica_log
 333
 334   void run_test_case(const TestCase &tcase) {
 335     test_merge_log(tcase);
 336     test_proc_replica_log(tcase);
 337   }
 338 }; // class PGLogTest
 339
 340 struct TestHandler : public PGLog::LogEntryHandler {
 341   list<hobject_t> &removed;
 342   explicit TestHandler(list<hobject_t> &removed) : removed(removed) {}
 343
 344   void rollback(
 345     const pg_log_entry_t &entry) override {}
 346   void rollforward(
 347     const pg_log_entry_t &entry) override {}
 348   void remove(
 349     const hobject_t &hoid) override {
 350     removed.push_back(hoid);
 351   }
 352   void cant_rollback(const pg_log_entry_t &entry) {}
 353   void try_stash(const hobject_t &, version_t) override {
 354     // lost/unfound cases are not tested yet
 355   }
 356   void trim(
 357     const pg_log_entry_t &entry) override {}
 358 };
 359
 360 TEST_F(PGLogTest, rewind_divergent_log) {
 361   /*        +----------------+
 362             |  log           |
 363             +--------+-------+
 364             |        |object |
 365             |version | hash  |
 366             |        |       |
 367        tail > (1,1)  |  x5   |
 368             |        |       |
 369             |        |       |
 370             | (1,4)  |  x9   < newhead
 371             | MODIFY |       |
 372             |        |       |
 373        head > (1,5)  |  x9   |
 374             | DELETE |       |
 375             |        |       |
 376             +--------+-------+
 377
 378   */
 379   {
 380     clear();
 381
 382     pg_info_t info;
 383     list<hobject_t> remove_snap;
 384     bool dirty_info = false;
 385     bool dirty_big_info = false;
 386
 387     hobject_t divergent_object;
 388     eversion_t divergent_version;
 389     eversion_t newhead;
 390
 391     hobject_t divergent;
 392     divergent.set_hash(0x9);
 393
 394     {
 395       pg_log_entry_t e;
 396       e.mark_unrollbackable();
 397
 398       e.version = eversion_t(1, 1);
 399       e.soid.set_hash(0x5);
 400       log.tail = e.version;
 401       log.log.push_back(e);
 402       e.version = newhead = eversion_t(1, 4);
 403       e.soid = divergent;
 404       e.op = pg_log_entry_t::MODIFY;
 405       log.log.push_back(e);
 406       e.version = divergent_version = eversion_t(1, 5);
 407       e.prior_version = eversion_t(1, 4);
 408       e.soid = divergent;
 409       divergent_object = e.soid;
 410       e.op = pg_log_entry_t::DELETE;
 411       log.log.push_back(e);
 412       log.head = e.version;
 413       log.index();
 414
 415       info.last_update = log.head;
 416       info.last_complete = log.head;
 417     }
 418
 419     EXPECT_FALSE(missing.have_missing());
 420     EXPECT_EQ(3U, log.log.size());
 421     EXPECT_TRUE(remove_snap.empty());
 422     EXPECT_EQ(log.head, info.last_update);
 423     EXPECT_EQ(log.head, info.last_complete);
 424     EXPECT_FALSE(is_dirty());
 425     EXPECT_FALSE(dirty_info);
 426     EXPECT_FALSE(dirty_big_info);
 427
 428     TestHandler h(remove_snap);
 429     rewind_divergent_log(newhead, info, &h,
 430                          dirty_info, dirty_big_info);
 431
 432     EXPECT_TRUE(log.objects.count(divergent));
 433     EXPECT_TRUE(missing.is_missing(divergent_object));
 434     EXPECT_EQ(1U, log.objects.count(divergent_object));
 435     EXPECT_EQ(2U, log.log.size());
 436     EXPECT_TRUE(remove_snap.empty());
 437     EXPECT_EQ(newhead, info.last_update);
 438     EXPECT_EQ(newhead, info.last_complete);
 439     EXPECT_TRUE(is_dirty());
 440     EXPECT_TRUE(dirty_info);
 441     EXPECT_TRUE(dirty_big_info);
 442   }
 443
 444   /*        +----------------+
 445             |  log           |
 446             +--------+-------+
 447             |        |object |
 448             |version | hash  |
 449             |        |       |
 450        tail > (1,1)  | NULL  |
 451             |        |       |
 452             | (1,4)  | NULL  < newhead
 453             |        |       |
 454        head > (1,5)  |  x9   |
 455             |        |       |
 456             +--------+-------+
 457
 458   */
 459   {
 460     clear();
 461
 462     pg_info_t info;
 463     list<hobject_t> remove_snap;
 464     bool dirty_info = false;
 465     bool dirty_big_info = false;
 466
 467     hobject_t divergent_object;
 468     eversion_t divergent_version;
 469     eversion_t prior_version;
 470     eversion_t newhead;
 471     {
 472       pg_log_entry_t e;
 473       e.mark_unrollbackable();
 474
 475       info.log_tail = log.tail = eversion_t(1, 1);
 476       newhead = eversion_t(1, 3);
 477       e.version = divergent_version = eversion_t(1, 5);
 478       e.soid.set_hash(0x9);
 479       divergent_object = e.soid;
 480       e.op = pg_log_entry_t::DELETE;
 481       e.prior_version = prior_version = eversion_t(0, 2);
 482       log.log.push_back(e);
 483       log.head = e.version;
 484     }
 485
 486     EXPECT_FALSE(missing.have_missing());
 487     EXPECT_EQ(1U, log.log.size());
 488     EXPECT_TRUE(remove_snap.empty());
 489     EXPECT_FALSE(is_dirty());
 490     EXPECT_FALSE(dirty_info);
 491     EXPECT_FALSE(dirty_big_info);
 492
 493     TestHandler h(remove_snap);
 494     rewind_divergent_log(newhead, info, &h,
 495                          dirty_info, dirty_big_info);
 496
 497     EXPECT_TRUE(missing.is_missing(divergent_object));
 498     EXPECT_EQ(0U, log.objects.count(divergent_object));
 499     EXPECT_TRUE(log.empty());
 500     EXPECT_TRUE(remove_snap.empty());
 501     EXPECT_TRUE(is_dirty());
 502     EXPECT_TRUE(dirty_info);
 503     EXPECT_TRUE(dirty_big_info);
 504   }
 505
 506   // Test for 13965
 507   {
 508     clear();
 509
 510     list<hobject_t> remove_snap;
 511     pg_info_t info;
 512     info.log_tail = log.tail = eversion_t(1, 5);
 513     info.last_update = eversion_t(1, 6);
 514     bool dirty_info = false;
 515     bool dirty_big_info = false;
 516
 517     {
 518       pg_log_entry_t e;
 519       e.mark_unrollbackable();
 520       e.version = eversion_t(1, 5);
 521       e.soid.set_hash(0x9);
 522       add(e);
 523     }
 524     {
 525       pg_log_entry_t e;
 526       e.mark_unrollbackable();
 527       e.version = eversion_t(1, 6);
 528       e.soid.set_hash(0x10);
 529       add(e);
 530     }
 531     TestHandler h(remove_snap);
 532     roll_forward_to(eversion_t(1, 6), &h);
 533     rewind_divergent_log(eversion_t(1, 5), info, &h,
 534                          dirty_info, dirty_big_info);
 535     pg_log_t log;
 536     reset_backfill_claim_log(log, &h);
 537   }
 538 }
 539
 540 TEST_F(PGLogTest, merge_old_entry) {
 541   // entries > last_backfill are silently ignored
 542   {
 543     clear();
 544
 545     ObjectStore::Transaction t;
 546     pg_log_entry_t oe;
 547     oe.mark_unrollbackable();
 548     pg_info_t info;
 549     list<hobject_t> remove_snap;
 550
 551     info.last_backfill = hobject_t();
 552     info.last_backfill.set_hash(100);
 553     oe.soid.set_hash(2);
 554     ASSERT_GT(oe.soid, info.last_backfill);
 555
 556     EXPECT_FALSE(is_dirty());
 557     EXPECT_TRUE(remove_snap.empty());
 558     EXPECT_TRUE(t.empty());
 559     EXPECT_FALSE(missing.have_missing());
 560     EXPECT_TRUE(log.empty());
 561
 562     TestHandler h(remove_snap);
 563     merge_old_entry(t, oe, info, &h);
 564
 565     EXPECT_FALSE(is_dirty());
 566     EXPECT_TRUE(remove_snap.empty());
 567     EXPECT_TRUE(t.empty());
 568     EXPECT_FALSE(missing.have_missing());
 569     EXPECT_TRUE(log.empty());
 570   }
 571
 572   // the new entry (from the logs) has a version that is higher than
 573   // the old entry (from the log entry given in argument) : do
 574   // nothing and return false
 575   {
 576     clear();
 577
 578     ObjectStore::Transaction t;
 579     pg_info_t info;
 580     list<hobject_t> remove_snap;
 581
 582     pg_log_entry_t ne;
 583     ne.mark_unrollbackable();
 584     ne.version = eversion_t(2,1);
 585     log.add(ne);
 586
 587     EXPECT_FALSE(is_dirty());
 588     EXPECT_TRUE(remove_snap.empty());
 589     EXPECT_TRUE(t.empty());
 590     EXPECT_FALSE(missing.have_missing());
 591     EXPECT_EQ(1U, log.log.size());
 592     EXPECT_EQ(ne.version, log.log.front().version);
 593
 594     // the newer entry ( from the logs ) can be DELETE
 595     {
 596       log.log.front().op = pg_log_entry_t::DELETE;
 597       pg_log_entry_t oe;
 598       oe.mark_unrollbackable();
 599       oe.version = eversion_t(1,1);
 600
 601       TestHandler h(remove_snap);
 602       merge_old_entry(t, oe, info, &h);
 603     }
 604
 605     // if the newer entry is not DELETE, the object must be in missing
 606     {
 607       pg_log_entry_t &ne = log.log.front();
 608       ne.op = pg_log_entry_t::MODIFY;
 609       missing.add_next_event(ne);
 610       pg_log_entry_t oe;
 611       oe.mark_unrollbackable();
 612       oe.version = eversion_t(1,1);
 613
 614       TestHandler h(remove_snap);
 615       merge_old_entry(t, oe, info, &h);
 616
 617       missing.rm(ne.soid, ne.version);
 618     }
 619
 620     missing.flush();
 621     EXPECT_FALSE(is_dirty());
 622     EXPECT_FALSE(remove_snap.empty());
 623     EXPECT_TRUE(t.empty());
 624     EXPECT_FALSE(missing.have_missing());
 625     EXPECT_EQ(1U, log.log.size());
 626     EXPECT_EQ(ne.version, log.log.front().version);
 627
 628   }
 629
 630   // the new entry (from the logs) has a version that is lower than
 631   // the old entry (from the log entry given in argument) and
 632   // old and new are delete : do nothing and return false
 633   {
 634     clear();
 635
 636     ObjectStore::Transaction t;
 637     pg_log_entry_t oe;
 638     oe.mark_unrollbackable();
 639     pg_info_t info;
 640     list<hobject_t> remove_snap;
 641
 642     pg_log_entry_t ne;
 643     ne.mark_unrollbackable();
 644     ne.version = eversion_t(1,1);
 645     ne.op = pg_log_entry_t::DELETE;
 646     log.add(ne);
 647
 648     oe.version = eversion_t(2,1);
 649     oe.op = pg_log_entry_t::DELETE;
 650
 651     EXPECT_FALSE(is_dirty());
 652     EXPECT_TRUE(remove_snap.empty());
 653     EXPECT_TRUE(t.empty());
 654     EXPECT_FALSE(missing.have_missing());
 655     EXPECT_EQ(1U, log.log.size());
 656
 657     TestHandler h(remove_snap);
 658     merge_old_entry(t, oe, info, &h);
 659
 660     EXPECT_FALSE(is_dirty());
 661     EXPECT_TRUE(remove_snap.empty());
 662     EXPECT_TRUE(t.empty());
 663     EXPECT_FALSE(missing.have_missing());
 664     EXPECT_EQ(1U, log.log.size());
 665   }
 666
 667   // the new entry (from the logs) has a version that is lower than
 668   // the old entry (from the log entry given in argument) and
 669   // old is update and new is DELETE :
 670   // if the object is in missing, it is removed
 671   {
 672     clear();
 673
 674     ObjectStore::Transaction t;
 675     pg_log_entry_t oe;
 676     oe.mark_unrollbackable();
 677     pg_info_t info;
 678     list<hobject_t> remove_snap;
 679
 680     pg_log_entry_t ne;
 681     ne.mark_unrollbackable();
 682     ne.version = eversion_t(1,1);
 683     ne.op = pg_log_entry_t::DELETE;
 684     log.add(ne);
 685
 686     oe.version = eversion_t(2,1);
 687     oe.op = pg_log_entry_t::MODIFY;
 688     missing.add_next_event(oe);
 689
 690     missing.flush();
 691     EXPECT_FALSE(is_dirty());
 692     EXPECT_TRUE(remove_snap.empty());
 693     EXPECT_TRUE(t.empty());
 694     EXPECT_TRUE(missing.is_missing(oe.soid));
 695     EXPECT_EQ(1U, log.log.size());
 696
 697     TestHandler h(remove_snap);
 698     merge_old_entry(t, oe, info, &h);
 699
 700     missing.flush();
 701     EXPECT_FALSE(is_dirty());
 702     EXPECT_TRUE(remove_snap.size() > 0);
 703     EXPECT_TRUE(t.empty());
 704     EXPECT_FALSE(missing.have_missing());
 705     EXPECT_EQ(1U, log.log.size());
 706   }
 707
 708   // there is no new entry (from the logs) and
 709   // the old entry (from the log entry given in argument) is not a CLONE and
 710   // the old entry prior_version is greater than the tail of the log :
 711   // do nothing and return false
 712   {
 713     clear();
 714
 715     ObjectStore::Transaction t;
 716     pg_log_entry_t oe;
 717     oe.mark_unrollbackable();
 718     pg_info_t info;
 719     list<hobject_t> remove_snap;
 720
 721     info.log_tail = eversion_t(1,1);
 722     oe.op = pg_log_entry_t::MODIFY;
 723     oe.prior_version = eversion_t(2,1);
 724     missing_add(oe.soid, oe.prior_version, eversion_t());
 725
 726     missing.flush();
 727     EXPECT_FALSE(is_dirty());
 728     EXPECT_TRUE(remove_snap.empty());
 729     EXPECT_TRUE(t.empty());
 730     EXPECT_TRUE(log.empty());
 731
 732     TestHandler h(remove_snap);
 733     merge_old_entry(t, oe, info, &h);
 734
 735     missing.flush();
 736     EXPECT_FALSE(is_dirty());
 737     EXPECT_TRUE(remove_snap.empty());
 738     EXPECT_TRUE(t.empty());
 739     EXPECT_TRUE(log.empty());
 740   }
 741
 742   // there is no new entry (from the logs) and
 743   // the old entry (from the log entry given in argument) is not a CLONE and
 744   // the old entry (from the log entry given in argument) is not a DELETE and
 745   // the old entry prior_version is lower than the tail of the log :
 746   //   add the old object to the remove_snap list and
 747   //   add the old object to divergent priors and
 748   //   add or update the prior_version of the object to missing and
 749   //   return false
 750   {
 751     clear();
 752
 753     ObjectStore::Transaction t;
 754     pg_log_entry_t oe;
 755     oe.mark_unrollbackable();
 756     pg_info_t info;
 757     list<hobject_t> remove_snap;
 758
 759     info.log_tail = eversion_t(2,1);
 760     oe.soid.set_hash(1);
 761     oe.op = pg_log_entry_t::MODIFY;
 762     oe.prior_version = eversion_t(1,1);
 763
 764     EXPECT_FALSE(is_dirty());
 765     EXPECT_TRUE(remove_snap.empty());
 766     EXPECT_TRUE(t.empty());
 767     EXPECT_FALSE(missing.have_missing());
 768     EXPECT_TRUE(log.empty());
 769
 770     TestHandler h(remove_snap);
 771     merge_old_entry(t, oe, info, &h);
 772
 773     EXPECT_TRUE(is_dirty());
 774     EXPECT_EQ(oe.soid, remove_snap.front());
 775     EXPECT_TRUE(t.empty());
 776     EXPECT_TRUE(missing.is_missing(oe.soid));
 777     EXPECT_TRUE(log.empty());
 778   }
 779
 780   // there is no new entry (from the logs) and
 781   // the old entry (from the log entry given in argument) is not a CLONE and
 782   // the old entry (from the log entry given in argument) is a DELETE and
 783   // the old entry prior_version is lower than the tail of the log :
 784   //   add the old object to divergent priors and
 785   //   add or update the prior_version of the object to missing and
 786   //   return false
 787   {
 788     clear();
 789
 790     ObjectStore::Transaction t;
 791     pg_log_entry_t oe;
 792     oe.mark_unrollbackable();
 793     pg_info_t info;
 794     list<hobject_t> remove_snap;
 795
 796     info.log_tail = eversion_t(2,1);
 797     oe.soid.set_hash(1);
 798     oe.op = pg_log_entry_t::DELETE;
 799     oe.prior_version = eversion_t(1,1);
 800
 801     EXPECT_FALSE(is_dirty());
 802     EXPECT_TRUE(remove_snap.empty());
 803     EXPECT_TRUE(t.empty());
 804     EXPECT_FALSE(missing.have_missing());
 805     EXPECT_TRUE(log.empty());
 806
 807     TestHandler h(remove_snap);
 808     merge_old_entry(t, oe, info, &h);
 809
 810     EXPECT_TRUE(is_dirty());
 811     EXPECT_TRUE(remove_snap.empty());
 812     EXPECT_TRUE(t.empty());
 813     EXPECT_TRUE(missing.is_missing(oe.soid));
 814     EXPECT_TRUE(log.empty());
 815   }
 816
 817
 818   // there is no new entry (from the logs) and
 819   // the old entry (from the log entry given in argument) is not a CLONE and
 820   // the old entry (from the log entry given in argument) is not a DELETE and
 821   // the old entry prior_version is eversion_t() :
 822   //   add the old object to the remove_snap list and
 823   //   remove the prior_version of the object from missing, if any and
 824   //   return false
 825   {
 826     clear();
 827
 828     ObjectStore::Transaction t;
 829     pg_log_entry_t oe;
 830     oe.mark_unrollbackable();
 831     pg_info_t info;
 832     list<hobject_t> remove_snap;
 833
 834     info.log_tail = eversion_t(10,1);
 835     oe.soid.set_hash(1);
 836     oe.op = pg_log_entry_t::MODIFY;
 837     oe.prior_version = eversion_t();
 838
 839     missing.add(oe.soid, eversion_t(1,1), eversion_t(), false);
 840
 841     missing.flush();
 842     EXPECT_FALSE(is_dirty());
 843     EXPECT_TRUE(remove_snap.empty());
 844     EXPECT_TRUE(t.empty());
 845     EXPECT_TRUE(missing.is_missing(oe.soid));
 846     EXPECT_TRUE(log.empty());
 847
 848     TestHandler h(remove_snap);
 849     merge_old_entry(t, oe, info, &h);
 850
 851     missing.flush();
 852     EXPECT_FALSE(is_dirty());
 853     EXPECT_EQ(oe.soid, remove_snap.front());
 854     EXPECT_TRUE(t.empty());
 855     EXPECT_FALSE(missing.have_missing());
 856     EXPECT_TRUE(log.empty());
 857   }
 858
 859 }
 860
 861 TEST_F(PGLogTest, merge_log) {
 862   // head and tail match, last_backfill is set:
 863   // noop
 864   {
 865     clear();
 866
 867     pg_log_t olog;
 868     pg_info_t oinfo;
 869     pg_shard_t fromosd;
 870     pg_info_t info;
 871     list<hobject_t> remove_snap;
 872     bool dirty_info = false;
 873     bool dirty_big_info = false;
 874
 875     hobject_t last_backfill(object_t("oname"), string("key"), 1, 234, 1, "");
 876     info.last_backfill = last_backfill;
 877     eversion_t stat_version(10, 1);
 878     info.stats.version = stat_version;
 879     log.tail = olog.tail = eversion_t(1, 1);
 880     log.head = olog.head = eversion_t(2, 1);
 881
 882     EXPECT_FALSE(missing.have_missing());
 883     EXPECT_EQ(0U, log.log.size());
 884     EXPECT_EQ(stat_version, info.stats.version);
 885     EXPECT_TRUE(remove_snap.empty());
 886     EXPECT_EQ(last_backfill, info.last_backfill);
 887     EXPECT_TRUE(info.purged_snaps.empty());
 888     EXPECT_FALSE(is_dirty());
 889     EXPECT_FALSE(dirty_info);
 890     EXPECT_FALSE(dirty_big_info);
 891
 892     TestHandler h(remove_snap);
 893     merge_log(oinfo, olog, fromosd, info, &h,
 894               dirty_info, dirty_big_info);
 895
 896     EXPECT_FALSE(missing.have_missing());
 897     EXPECT_EQ(0U, log.log.size());
 898     EXPECT_EQ(stat_version, info.stats.version);
 899     EXPECT_TRUE(remove_snap.empty());
 900     EXPECT_TRUE(info.purged_snaps.empty());
 901     EXPECT_FALSE(is_dirty());
 902     EXPECT_FALSE(dirty_info);
 903     EXPECT_FALSE(dirty_big_info);
 904   }
 905
 906   // head and tail match, last_backfill is not set: info.stats is
 907   // copied from oinfo.stats but info.stats.reported_* is guaranteed to
 908   // never be replaced by a lower version
 909   {
 910     clear();
 911
 912     pg_log_t olog;
 913     pg_info_t oinfo;
 914     pg_shard_t fromosd;
 915     pg_info_t info;
 916     list<hobject_t> remove_snap;
 917     bool dirty_info = false;
 918     bool dirty_big_info = false;
 919
 920     eversion_t stat_version(10, 1);
 921     oinfo.stats.version = stat_version;
 922     info.stats.reported_seq = 1;
 923     info.stats.reported_epoch = 10;
 924     oinfo.stats.reported_seq = 1;
 925     oinfo.stats.reported_epoch = 1;
 926     log.tail = olog.tail = eversion_t(1, 1);
 927     log.head = olog.head = eversion_t(2, 1);
 928     missing.may_include_deletes = false;
 929
 930     EXPECT_FALSE(missing.have_missing());
 931     EXPECT_EQ(0U, log.log.size());
 932     EXPECT_EQ(eversion_t(), info.stats.version);
 933     EXPECT_EQ(1ull, info.stats.reported_seq);
 934     EXPECT_EQ(10u, info.stats.reported_epoch);
 935     EXPECT_TRUE(remove_snap.empty());
 936     EXPECT_TRUE(info.last_backfill.is_max());
 937     EXPECT_TRUE(info.purged_snaps.empty());
 938     EXPECT_FALSE(is_dirty());
 939     EXPECT_FALSE(dirty_info);
 940     EXPECT_FALSE(dirty_big_info);
 941
 942     TestHandler h(remove_snap);
 943     merge_log(oinfo, olog, fromosd, info, &h,
 944               dirty_info, dirty_big_info);
 945
 946     EXPECT_FALSE(missing.have_missing());
 947     EXPECT_EQ(0U, log.log.size());
 948     EXPECT_EQ(stat_version, info.stats.version);
 949     EXPECT_EQ(1ull, info.stats.reported_seq);
 950     EXPECT_EQ(10u, info.stats.reported_epoch);
 951     EXPECT_TRUE(remove_snap.empty());
 952     EXPECT_TRUE(info.purged_snaps.empty());
 953     EXPECT_FALSE(is_dirty());
 954     EXPECT_FALSE(dirty_info);
 955     EXPECT_FALSE(dirty_big_info);
 956   }
 957
 958   /*         Before
 959             +--------------------------+
 960             |  log              olog   |
 961             +--------+-------+---------+
 962             |        |object |         |
 963             |version | hash  | version |
 964             |        |       |         |
 965             |        |  x5   |  (1,1)  < tail
 966             |        |       |         |
 967             |        |       |         |
 968        tail > (1,4)  |  x7   |         |
 969             |        |       |         |
 970             |        |       |         |
 971        head > (1,5)  |  x9   |  (1,5)  < head
 972             |        |       |         |
 973             |        |       |         |
 974             +--------+-------+---------+
 975
 976              After
 977             +-----------------
 978             |  log           |
 979             +--------+-------+
 980             |        |object |
 981             |version | hash  |
 982             |        |       |
 983        tail > (1,1)  |  x5   |
 984             |        |       |
 985             |        |       |
 986             | (1,4)  |  x7   |
 987             |        |       |
 988             |        |       |
 989        head > (1,5)  |  x9   |
 990             |        |       |
 991             |        |       |
 992             +--------+-------+
 993   */
 994   {
 995     clear();
 996
 997     pg_log_t olog;
 998     pg_info_t oinfo;
 999     pg_shard_t fromosd;
1000     pg_info_t info;
1001     list<hobject_t> remove_snap;
1002     bool dirty_info = false;
1003     bool dirty_big_info = false;
1004     missing.may_include_deletes = false;
1005
1006     {
1007       pg_log_entry_t e;
1008       e.mark_unrollbackable();
1009
1010       e.version = eversion_t(1, 4);
1011       e.soid.set_hash(0x5);
1012       log.tail = e.version;
1013       log.log.push_back(e);
1014       e.version = eversion_t(1, 5);
1015       e.soid.set_hash(0x9);
1016       log.log.push_back(e);
1017       log.head = e.version;
1018       log.index();
1019
1020       info.last_update = log.head;
1021
1022       e.version = eversion_t(1, 1);
1023       e.soid.set_hash(0x5);
1024       olog.tail = e.version;
1025       olog.log.push_back(e);
1026       e.version = eversion_t(1, 5);
1027       e.soid.set_hash(0x9);
1028       olog.log.push_back(e);
1029       olog.head = e.version;
1030     }
1031
1032     hobject_t last_backfill(object_t("oname"), string("key"), 1, 234, 1, "");
1033     info.last_backfill = last_backfill;
1034     eversion_t stat_version(10, 1);
1035     info.stats.version = stat_version;
1036
1037     EXPECT_FALSE(missing.have_missing());
1038     EXPECT_EQ(2U, log.log.size());
1039     EXPECT_EQ(stat_version, info.stats.version);
1040     EXPECT_TRUE(remove_snap.empty());
1041     EXPECT_EQ(last_backfill, info.last_backfill);
1042     EXPECT_TRUE(info.purged_snaps.empty());
1043     EXPECT_FALSE(is_dirty());
1044     EXPECT_FALSE(dirty_info);
1045     EXPECT_FALSE(dirty_big_info);
1046
1047     TestHandler h(remove_snap);
1048     merge_log(oinfo, olog, fromosd, info, &h,
1049               dirty_info, dirty_big_info);
1050
1051     EXPECT_FALSE(missing.have_missing());
1052     EXPECT_EQ(3U, log.log.size());
1053     EXPECT_EQ(stat_version, info.stats.version);
1054     EXPECT_TRUE(remove_snap.empty());
1055     EXPECT_TRUE(info.purged_snaps.empty());
1056     EXPECT_TRUE(is_dirty());
1057     EXPECT_TRUE(dirty_info);
1058     EXPECT_TRUE(dirty_big_info);
1059   }
1060
1061   /*        +--------------------------+
1062             |  log              olog   |
1063             +--------+-------+---------+
1064             |        |object |         |
1065             |version | hash  | version |
1066             |        |       |         |
1067        tail > (1,1)  |  x5   |  (1,1)  < tail
1068             |        |       |         |
1069             |        |       |         |
1070             | (1,2)  |  x3   |  (1,2)  < lower_bound
1071             |        |       |         |
1072             |        |       |         |
1073        head > (1,3)  |  x9   |         |
1074             | DELETE |       |         |
1075             |        |       |         |
1076             |        |  x9   |  (2,3)  |
1077             |        |       |  MODIFY |
1078             |        |       |         |
1079             |        |  x7   |  (2,4)  < head
1080             |        |       |  DELETE |
1081             +--------+-------+---------+
1082
1083       The log entry (1,3) deletes the object x9 but the olog entry (2,3) modifies
1084       it and is authoritative : the log entry (1,3) is divergent.
1085
1086   */
1087   {
1088     clear();
1089
1090     pg_log_t olog;
1091     pg_info_t oinfo;
1092     pg_shard_t fromosd;
1093     pg_info_t info;
1094     list<hobject_t> remove_snap;
1095     bool dirty_info = false;
1096     bool dirty_big_info = false;
1097
1098     hobject_t divergent_object;
1099     missing.may_include_deletes = true;
1100
1101     {
1102       pg_log_entry_t e;
1103       e.mark_unrollbackable();
1104
1105       e.version = eversion_t(1, 1);
1106       e.soid.set_hash(0x5);
1107       log.tail = e.version;
1108       log.log.push_back(e);
1109       e.version = eversion_t(1, 2);
1110       e.soid.set_hash(0x3);
1111       log.log.push_back(e);
1112       e.version = eversion_t(1,3);
1113       e.soid.set_hash(0x9);
1114       divergent_object = e.soid;
1115       e.op = pg_log_entry_t::DELETE;
1116       log.log.push_back(e);
1117       log.head = e.version;
1118       log.index();
1119
1120       info.last_update = log.head;
1121
1122       e.version = eversion_t(1, 1);
1123       e.soid.set_hash(0x5);
1124       olog.tail = e.version;
1125       olog.log.push_back(e);
1126       e.version = eversion_t(1, 2);
1127       e.soid.set_hash(0x3);
1128       olog.log.push_back(e);
1129       e.version = eversion_t(2, 3);
1130       e.soid.set_hash(0x9);
1131       e.op = pg_log_entry_t::MODIFY;
1132       olog.log.push_back(e);
1133       e.version = eversion_t(2, 4);
1134       e.soid.set_hash(0x7);
1135       e.op = pg_log_entry_t::DELETE;
1136       olog.log.push_back(e);
1137       olog.head = e.version;
1138     }
1139
1140     snapid_t purged_snap(1);
1141     {
1142       oinfo.last_update = olog.head;
1143       oinfo.purged_snaps.insert(purged_snap);
1144     }
1145
1146     EXPECT_FALSE(missing.have_missing());
1147     EXPECT_EQ(1U, log.objects.count(divergent_object));
1148     EXPECT_EQ(3U, log.log.size());
1149     EXPECT_TRUE(remove_snap.empty());
1150     EXPECT_EQ(log.head, info.last_update);
1151     EXPECT_TRUE(info.purged_snaps.empty());
1152     EXPECT_FALSE(is_dirty());
1153     EXPECT_FALSE(dirty_info);
1154     EXPECT_FALSE(dirty_big_info);
1155
1156     TestHandler h(remove_snap);
1157     merge_log(oinfo, olog, fromosd, info, &h,
1158               dirty_info, dirty_big_info);
1159
1160     /* When the divergent entry is a DELETE and the authoritative
1161        entry is a MODIFY, the object will be added to missing : it is
1162        a verifiable side effect proving the entry was identified
1163        to be divergent.
1164     */
1165     EXPECT_TRUE(missing.is_missing(divergent_object));
1166     EXPECT_EQ(1U, log.objects.count(divergent_object));
1167     EXPECT_EQ(4U, log.log.size());
1168     /* DELETE entries from olog that are appended to the hed of the
1169        log, and the divergent version of the object is removed (added
1170        to remove_snap)
1171      */
1172     EXPECT_EQ(0x9U, remove_snap.front().get_hash());
1173     EXPECT_EQ(log.head, info.last_update);
1174     EXPECT_TRUE(info.purged_snaps.contains(purged_snap));
1175     EXPECT_TRUE(is_dirty());
1176     EXPECT_TRUE(dirty_info);
1177     EXPECT_TRUE(dirty_big_info);
1178   }
1179
1180   /*        +--------------------------+
1181             |  log              olog   |
1182             +--------+-------+---------+
1183             |        |object |         |
1184             |version | hash  | version |
1185             |        |       |         |
1186        tail > (1,1)  |  x5   |  (1,1)  < tail
1187             |        |       |         |
1188             |        |       |         |
1189             | (1,2)  |  x3   |  (1,2)  < lower_bound
1190             |        |       |         |
1191             |        |       |         |
1192        head > (1,3)  |  x9   |         |
1193             | DELETE |       |         |
1194             |        |       |         |
1195             |        |  x9   |  (2,3)  |
1196             |        |       |  MODIFY |
1197             |        |       |         |
1198             |        |  x7   |  (2,4)  < head
1199             |        |       |  DELETE |
1200             +--------+-------+---------+
1201
1202       The log entry (1,3) deletes the object x9 but the olog entry (2,3) modifies
1203       it and is authoritative : the log entry (1,3) is divergent.
1204
1205   */
1206   {
1207     clear();
1208
1209     pg_log_t olog;
1210     pg_info_t oinfo;
1211     pg_shard_t fromosd;
1212     pg_info_t info;
1213     list<hobject_t> remove_snap;
1214     bool dirty_info = false;
1215     bool dirty_big_info = false;
1216
1217     hobject_t divergent_object;
1218
1219     {
1220       pg_log_entry_t e;
1221       e.mark_unrollbackable();
1222
1223       e.version = eversion_t(1, 1);
1224       e.soid.set_hash(0x5);
1225       log.tail = e.version;
1226       log.log.push_back(e);
1227       e.version = eversion_t(1, 2);
1228       e.soid.set_hash(0x3);
1229       log.log.push_back(e);
1230       e.version = eversion_t(1,3);
1231       e.soid.set_hash(0x9);
1232       divergent_object = e.soid;
1233       e.op = pg_log_entry_t::DELETE;
1234       log.log.push_back(e);
1235       log.head = e.version;
1236       log.index();
1237
1238       info.last_update = log.head;
1239
1240       e.version = eversion_t(1, 1);
1241       e.soid.set_hash(0x5);
1242       olog.tail = e.version;
1243       olog.log.push_back(e);
1244       e.version = eversion_t(1, 2);
1245       e.soid.set_hash(0x3);
1246       olog.log.push_back(e);
1247       e.version = eversion_t(2, 3);
1248       e.soid.set_hash(0x9);
1249       e.op = pg_log_entry_t::MODIFY;
1250       olog.log.push_back(e);
1251       e.version = eversion_t(2, 4);
1252       e.soid.set_hash(0x7);
1253       e.op = pg_log_entry_t::DELETE;
1254       olog.log.push_back(e);
1255       olog.head = e.version;
1256     }
1257
1258     snapid_t purged_snap(1);
1259     {
1260       oinfo.last_update = olog.head;
1261       oinfo.purged_snaps.insert(purged_snap);
1262     }
1263
1264     EXPECT_FALSE(missing.have_missing());
1265     EXPECT_EQ(1U, log.objects.count(divergent_object));
1266     EXPECT_EQ(3U, log.log.size());
1267     EXPECT_TRUE(remove_snap.empty());
1268     EXPECT_EQ(log.head, info.last_update);
1269     EXPECT_TRUE(info.purged_snaps.empty());
1270     EXPECT_FALSE(is_dirty());
1271     EXPECT_FALSE(dirty_info);
1272     EXPECT_FALSE(dirty_big_info);
1273
1274     TestHandler h(remove_snap);
1275     missing.may_include_deletes = false;
1276     merge_log(oinfo, olog, fromosd, info, &h,
1277               dirty_info, dirty_big_info);
1278
1279     /* When the divergent entry is a DELETE and the authoritative
1280        entry is a MODIFY, the object will be added to missing : it is
1281        a verifiable side effect proving the entry was identified
1282        to be divergent.
1283     */
1284     EXPECT_TRUE(missing.is_missing(divergent_object));
1285     EXPECT_EQ(1U, log.objects.count(divergent_object));
1286     EXPECT_EQ(4U, log.log.size());
1287     /* DELETE entries from olog that are appended to the hed of the
1288        log, and the divergent version of the object is removed (added
1289        to remove_snap). When peering handles deletes, it is the earlier
1290        version that is in the removed list.
1291      */
1292     EXPECT_EQ(0x7U, remove_snap.front().get_hash());
1293     EXPECT_EQ(log.head, info.last_update);
1294     EXPECT_TRUE(info.purged_snaps.contains(purged_snap));
1295     EXPECT_TRUE(is_dirty());
1296     EXPECT_TRUE(dirty_info);
1297     EXPECT_TRUE(dirty_big_info);
1298   }
1299
1300   /*        +--------------------------+
1301             |  log              olog   |
1302             +--------+-------+---------+
1303             |        |object |         |
1304             |version | hash  | version |
1305             |        |       |         |
1306        tail > (1,1)  |  x5   |  (1,1)  < tail
1307             |        |       |         |
1308             |        |       |         |
1309             | (1,4)  |  x7   |  (1,4)  < head
1310             |        |       |         |
1311             |        |       |         |
1312        head > (1,5)  |  x9   |         |
1313             |        |       |         |
1314             |        |       |         |
1315             +--------+-------+---------+
1316
1317       The head of the log entry (1,5) is divergent because it is greater than the
1318       head of olog.
1319
1320   */
1321   {
1322     clear();
1323
1324     pg_log_t olog;
1325     pg_info_t oinfo;
1326     pg_shard_t fromosd;
1327     pg_info_t info;
1328     list<hobject_t> remove_snap;
1329     bool dirty_info = false;
1330     bool dirty_big_info = false;
1331
1332     {
1333       pg_log_entry_t e;
1334       e.mark_unrollbackable();
1335
1336       e.version = eversion_t(1, 1);
1337       e.soid.set_hash(0x5);
1338       log.tail = e.version;
1339       log.log.push_back(e);
1340       e.version = eversion_t(1, 4);
1341       e.soid.set_hash(0x7);
1342       log.log.push_back(e);
1343       e.version = eversion_t(1, 5);
1344       e.soid.set_hash(0x9);
1345       log.log.push_back(e);
1346       log.head = e.version;
1347       log.index();
1348
1349       info.last_update = log.head;
1350
1351       e.version = eversion_t(1, 1);
1352       e.soid.set_hash(0x5);
1353       olog.tail = e.version;
1354       olog.log.push_back(e);
1355       e.version = eversion_t(1, 4);
1356       e.soid.set_hash(0x7);
1357       olog.log.push_back(e);
1358       olog.head = e.version;
1359     }
1360
1361     hobject_t last_backfill(object_t("oname"), string("key"), 1, 234, 1, "");
1362     info.last_backfill = last_backfill;
1363     eversion_t stat_version(10, 1);
1364     info.stats.version = stat_version;
1365
1366     EXPECT_FALSE(missing.have_missing());
1367     EXPECT_EQ(3U, log.log.size());
1368     EXPECT_EQ(stat_version, info.stats.version);
1369     EXPECT_TRUE(remove_snap.empty());
1370     EXPECT_EQ(last_backfill, info.last_backfill);
1371     EXPECT_TRUE(info.purged_snaps.empty());
1372     EXPECT_FALSE(is_dirty());
1373     EXPECT_FALSE(dirty_info);
1374     EXPECT_FALSE(dirty_big_info);
1375
1376     TestHandler h(remove_snap);
1377     missing.may_include_deletes = false;
1378     merge_log(oinfo, olog, fromosd, info, &h,
1379               dirty_info, dirty_big_info);
1380
1381     EXPECT_FALSE(missing.have_missing());
1382     EXPECT_EQ(2U, log.log.size());
1383     EXPECT_EQ(stat_version, info.stats.version);
1384     EXPECT_EQ(0x9U, remove_snap.front().get_hash());
1385     EXPECT_TRUE(info.purged_snaps.empty());
1386     EXPECT_TRUE(is_dirty());
1387     EXPECT_TRUE(dirty_info);
1388     EXPECT_TRUE(dirty_big_info);
1389   }
1390
1391   // If our log is empty, the incoming log needs to have not been trimmed.
1392   {
1393     clear();
1394
1395     pg_log_t olog;
1396     pg_info_t oinfo;
1397     pg_shard_t fromosd;
1398     pg_info_t info;
1399     list<hobject_t> remove_snap;
1400     bool dirty_info = false;
1401     bool dirty_big_info = false;
1402
1403     // olog has been trimmed
1404     olog.tail = eversion_t(1, 1);
1405
1406     TestHandler h(remove_snap);
1407     PrCtl unset_dumpable;
1408     ASSERT_DEATH(merge_log(oinfo, olog, fromosd, info, &h,
1409                            dirty_info, dirty_big_info), "");
1410   }
1411
1412 }
1413
1414 TEST_F(PGLogTest, proc_replica_log) {
1415   // empty log : no side effect
1416   {
1417     clear();
1418
1419     pg_log_t olog;
1420     pg_info_t oinfo;
1421     pg_missing_t omissing;
1422     pg_shard_t from;
1423
1424     eversion_t last_update(1, 1);
1425     log.head = olog.head = oinfo.last_update = last_update;
1426     eversion_t last_complete(1, 1);
1427     oinfo.last_complete = last_complete;
1428
1429     EXPECT_FALSE(omissing.have_missing());
1430     EXPECT_EQ(last_update, oinfo.last_update);
1431     EXPECT_EQ(last_complete, oinfo.last_complete);
1432
1433     missing.may_include_deletes = false;
1434     proc_replica_log(oinfo, olog, omissing, from);
1435
1436     EXPECT_FALSE(omissing.have_missing());
1437     EXPECT_EQ(last_update, oinfo.last_update);
1438     EXPECT_EQ(last_update, oinfo.last_complete);
1439   }
1440
1441   /*        +--------------------------+
1442             |  log              olog   |
1443             +--------+-------+---------+
1444             |        |object |         |
1445             |version | hash  | version |
1446             |        |       |         |
1447             |        |  x3   |  (1,1)  < tail
1448             |        |       |         |
1449             |        |       |         |
1450        tail > (1,2)  |  x5   |         |
1451             |        |       |         |
1452             |        |       |         |
1453        head > (1,3)  |  x9   |         |
1454             | DELETE |       |         |
1455             |        |       |         |
1456             |        |  x9   |  (2,3)  < head
1457             |        |       |  DELETE |
1458             |        |       |         |
1459             +--------+-------+---------+
1460
1461       The log entry (1,3) deletes the object x9 and the olog entry
1462       (2,3) also deletes it : do nothing. The olog tail is ignored
1463       because it is before the log tail.
1464
1465   */
1466   {
1467     clear();
1468
1469     pg_log_t olog;
1470     pg_info_t oinfo;
1471     pg_missing_t omissing;
1472     pg_shard_t from;
1473
1474     {
1475       pg_log_entry_t e;
1476       e.mark_unrollbackable();
1477
1478       e.version = eversion_t(1, 2);
1479       e.soid.set_hash(0x5);
1480       log.tail = e.version;
1481       log.log.push_back(e);
1482       e.version = eversion_t(1, 3);
1483       e.soid.set_hash(0x9);
1484       e.op = pg_log_entry_t::DELETE;
1485       log.log.push_back(e);
1486       log.head = e.version;
1487       log.index();
1488
1489       e.version = eversion_t(1, 1);
1490       e.soid.set_hash(0x3);
1491       olog.tail = e.version;
1492       olog.log.push_back(e);
1493       e.version = eversion_t(2, 3);
1494       e.soid.set_hash(0x9);
1495       e.op = pg_log_entry_t::DELETE;
1496       olog.log.push_back(e);
1497       olog.head = e.version;
1498
1499       oinfo.last_update = olog.head;
1500       oinfo.last_complete = olog.head;
1501     }
1502
1503     EXPECT_FALSE(omissing.have_missing());
1504     EXPECT_EQ(olog.head, oinfo.last_update);
1505     EXPECT_EQ(olog.head, oinfo.last_complete);
1506
1507     missing.may_include_deletes = false;
1508     proc_replica_log(oinfo, olog, omissing, from);
1509
1510     EXPECT_FALSE(omissing.have_missing());
1511   }
1512
1513  {
1514     clear();
1515
1516     pg_log_t olog;
1517     pg_info_t oinfo;
1518     pg_missing_t omissing;
1519     pg_shard_t from;
1520
1521     hobject_t divergent_object;
1522
1523     {
1524       pg_log_entry_t e;
1525       e.mark_unrollbackable();
1526
1527       {
1528         e.soid = divergent_object;
1529         e.soid.set_hash(0x1);
1530         e.version = eversion_t(1, 1);
1531         log.tail = e.version;
1532         log.log.push_back(e);
1533
1534         e.soid = divergent_object;
1535         e.prior_version = eversion_t(1, 1);
1536         e.version = eversion_t(1, 2);
1537         log.tail = e.version;
1538         log.log.push_back(e);
1539
1540         e.soid.set_hash(0x3);
1541         e.version = eversion_t(1, 4);
1542         log.log.push_back(e);
1543
1544         e.soid.set_hash(0x7);
1545         e.version = eversion_t(1, 5);
1546         log.log.push_back(e);
1547
1548         e.soid.set_hash(0x8);
1549         e.version = eversion_t(1, 6);
1550         log.log.push_back(e);
1551
1552         e.soid.set_hash(0x9);
1553         e.op = pg_log_entry_t::DELETE;
1554         e.version = eversion_t(2, 7);
1555         log.log.push_back(e);
1556
1557         e.soid.set_hash(0xa);
1558         e.version = eversion_t(2, 8);
1559         log.head = e.version;
1560         log.log.push_back(e);
1561       }
1562       log.index();
1563
1564       {
1565         e.soid = divergent_object;
1566         e.soid.set_hash(0x1);
1567         e.version = eversion_t(1, 1);
1568         olog.tail = e.version;
1569         olog.log.push_back(e);
1570
1571         e.soid = divergent_object;
1572         e.prior_version = eversion_t(1, 1);
1573         e.version = eversion_t(1, 2);
1574         olog.log.push_back(e);
1575
1576         e.prior_version = eversion_t(0, 0);
1577         e.soid.set_hash(0x3);
1578         e.version = eversion_t(1, 4);
1579         olog.log.push_back(e);
1580
1581         e.soid.set_hash(0x7);
1582         e.version = eversion_t(1, 5);
1583         olog.log.push_back(e);
1584
1585         e.soid.set_hash(0x8);
1586         e.version = eversion_t(1, 6);
1587         olog.log.push_back(e);
1588
1589         e.soid.set_hash(0x9); // should not be added to missing, create
1590         e.op = pg_log_entry_t::MODIFY;
1591         e.version = eversion_t(1, 7);
1592         olog.log.push_back(e);
1593
1594         e.soid = divergent_object; // should be added to missing at 1,2
1595         e.op = pg_log_entry_t::MODIFY;
1596         e.version = eversion_t(1, 8);
1597         e.prior_version = eversion_t(1, 2);
1598         olog.log.push_back(e);
1599         olog.head = e.version;
1600       }
1601       oinfo.last_update = olog.head;
1602       oinfo.last_complete = olog.head;
1603     }
1604
1605     EXPECT_FALSE(omissing.have_missing());
1606     EXPECT_EQ(olog.head, oinfo.last_update);
1607     EXPECT_EQ(olog.head, oinfo.last_complete);
1608
1609     missing.may_include_deletes = false;
1610     proc_replica_log(oinfo, olog, omissing, from);
1611
1612     EXPECT_TRUE(omissing.have_missing());
1613     EXPECT_TRUE(omissing.is_missing(divergent_object));
1614     EXPECT_EQ(eversion_t(1, 2), omissing.get_items().at(divergent_object).need);
1615     EXPECT_EQ(eversion_t(1, 6), oinfo.last_update);
1616     EXPECT_EQ(eversion_t(1, 1), oinfo.last_complete);
1617   }
1618
1619   /*        +--------------------------+
1620             |  olog              log   |
1621             +--------+-------+---------+
1622             |        |object |         |
1623             |version | hash  | version |
1624             |        |       |         |
1625        tail > (1,1)  |  x9   |  (1,1)  < tail
1626             |        |       |         |
1627             |        |       |         |
1628             | (1,2)  |  x3   |  (1,2)  |
1629             |        |       |         |
1630             |        |       |         |
1631        head > (1,3)  |  x9   |         |
1632             | DELETE |       |         |
1633             |        |       |         |
1634             |        |  x9   |  (2,3)  < head
1635             |        |       |  DELETE |
1636             |        |       |         |
1637             +--------+-------+---------+
1638
1639       The log entry (1,3) deletes the object x9 and the olog entry
1640       (2,3) also deletes it : do nothing.
1641
1642   */
1643   {
1644     clear();
1645
1646     pg_log_t olog;
1647     pg_info_t oinfo;
1648     pg_missing_t omissing;
1649     pg_shard_t from;
1650
1651     eversion_t last_update(1, 2);
1652     hobject_t divergent_object;
1653     divergent_object.set_hash(0x9);
1654
1655     {
1656       pg_log_entry_t e;
1657       e.mark_unrollbackable();
1658
1659       e.version = eversion_t(1, 1);
1660       e.soid = divergent_object;
1661       log.tail = e.version;
1662       log.log.push_back(e);
1663       e.version = last_update;
1664       e.soid.set_hash(0x3);
1665       log.log.push_back(e);
1666       e.version = eversion_t(2, 3);
1667       e.prior_version = eversion_t(1, 1);
1668       e.soid = divergent_object;
1669       e.op = pg_log_entry_t::DELETE;
1670       log.log.push_back(e);
1671       log.head = e.version;
1672       log.index();
1673
1674       e.version = eversion_t(1, 1);
1675       e.soid = divergent_object;
1676       olog.tail = e.version;
1677       olog.log.push_back(e);
1678       e.version = last_update;
1679       e.soid.set_hash(0x3);
1680       olog.log.push_back(e);
1681       e.version = eversion_t(1, 3);
1682       e.prior_version = eversion_t(1, 1);
1683       e.soid = divergent_object;
1684       e.op = pg_log_entry_t::DELETE;
1685       olog.log.push_back(e);
1686       olog.head = e.version;
1687
1688       oinfo.last_update = olog.head;
1689       oinfo.last_complete = olog.head;
1690     }
1691
1692     EXPECT_FALSE(omissing.have_missing());
1693     EXPECT_EQ(olog.head, oinfo.last_update);
1694     EXPECT_EQ(olog.head, oinfo.last_complete);
1695
1696     missing.may_include_deletes = false;
1697     proc_replica_log(oinfo, olog, omissing, from);
1698
1699     EXPECT_TRUE(omissing.have_missing());
1700     EXPECT_TRUE(omissing.is_missing(divergent_object));
1701     EXPECT_EQ(omissing.get_items().at(divergent_object).have, eversion_t(0, 0));
1702     EXPECT_EQ(omissing.get_items().at(divergent_object).need, eversion_t(1, 1));
1703     EXPECT_EQ(last_update, oinfo.last_update);
1704   }
1705
1706   /*        +--------------------------+
1707             |  olog              log   |
1708             +--------+-------+---------+
1709             |        |object |         |
1710             |version | hash  | version |
1711             |        |       |         |
1712        tail > (1,1)  |  x9   |  (1,1)  < tail
1713             |        |       |         |
1714             |        |       |         |
1715             | (1,2)  |  x3   |  (1,2)  |
1716             |        |       |         |
1717             |        |       |         |
1718        head > (1,3)  |  x9   |         |
1719             | MODIFY |       |         |
1720             |        |       |         |
1721             |        |  x9   |  (2,3)  < head
1722             |        |       |  DELETE |
1723             |        |       |         |
1724             +--------+-------+---------+
1725
1726       The log entry (1,3) deletes the object x9 but the olog entry
1727       (2,3) modifies it : remove it from omissing.
1728
1729   */
1730   {
1731     clear();
1732
1733     pg_log_t olog;
1734     pg_info_t oinfo;
1735     pg_missing_t omissing;
1736     pg_shard_t from;
1737
1738     eversion_t last_update(1, 2);
1739     hobject_t divergent_object;
1740
1741     {
1742       pg_log_entry_t e;
1743       e.mark_unrollbackable();
1744
1745       e.version = eversion_t(1, 1);
1746       e.soid = divergent_object;
1747       log.tail = e.version;
1748       log.log.push_back(e);
1749       e.version = last_update;
1750       e.soid.set_hash(0x3);
1751       log.log.push_back(e);
1752       e.version = eversion_t(2, 3);
1753       e.prior_version = eversion_t(1, 1);
1754       e.soid = divergent_object;
1755       e.op = pg_log_entry_t::DELETE;
1756       log.log.push_back(e);
1757       log.head = e.version;
1758       log.index();
1759
1760       e.version = eversion_t(1, 1);
1761       e.soid = divergent_object;
1762       olog.tail = e.version;
1763       olog.log.push_back(e);
1764       e.version = last_update;
1765       e.soid.set_hash(0x3);
1766       olog.log.push_back(e);
1767       e.version = eversion_t(1, 3);
1768       e.prior_version = eversion_t(1, 1);
1769       e.soid = divergent_object;
1770       divergent_object = e.soid;
1771       omissing.add(divergent_object, e.version, eversion_t(), false);
1772       e.op = pg_log_entry_t::MODIFY;
1773       olog.log.push_back(e);
1774       olog.head = e.version;
1775
1776       oinfo.last_update = olog.head;
1777       oinfo.last_complete = olog.head;
1778     }
1779
1780     EXPECT_TRUE(omissing.have_missing());
1781     EXPECT_TRUE(omissing.is_missing(divergent_object));
1782     EXPECT_EQ(eversion_t(1, 3), omissing.get_items().at(divergent_object).need);
1783     EXPECT_EQ(olog.head, oinfo.last_update);
1784     EXPECT_EQ(olog.head, oinfo.last_complete);
1785
1786     missing.may_include_deletes = false;
1787     proc_replica_log(oinfo, olog, omissing, from);
1788
1789     EXPECT_TRUE(omissing.have_missing());
1790     EXPECT_TRUE(omissing.is_missing(divergent_object));
1791     EXPECT_EQ(omissing.get_items().at(divergent_object).have, eversion_t(0, 0));
1792     EXPECT_EQ(omissing.get_items().at(divergent_object).need, eversion_t(1, 1));
1793     EXPECT_EQ(last_update, oinfo.last_update);
1794   }
1795
1796   /*        +--------------------------+
1797             |  log              olog   |
1798             +--------+-------+---------+
1799             |        |object |         |
1800             |version | hash  | version |
1801             |        |       |         |
1802        tail > (1,1)  |  x9   |  (1,1)  < tail
1803             |        |       |         |
1804             |        |       |         |
1805             | (1,2)  |  x3   |  (1,2)  |
1806             |        |       |         |
1807             |        |       |         |
1808             |        |  x9   |  (1,3)  < head
1809             |        |       |  MODIFY |
1810             |        |       |         |
1811        head > (2,3)  |  x9   |         |
1812             | DELETE |       |         |
1813             |        |       |         |
1814             +--------+-------+---------+
1815
1816       The log entry (2,3) deletes the object x9 but the olog entry
1817       (1,3) modifies it : proc_replica_log should adjust missing to
1818       1,1 for that object until add_next_event in PG::activate processes
1819       the delete.
1820   */
1821   {
1822     clear();
1823
1824     pg_log_t olog;
1825     pg_info_t oinfo;
1826     pg_missing_t omissing;
1827     pg_shard_t from;
1828
1829     eversion_t last_update(1, 2);
1830     hobject_t divergent_object;
1831     eversion_t new_version(2, 3);
1832     eversion_t divergent_version(1, 3);
1833
1834     {
1835       pg_log_entry_t e;
1836       e.mark_unrollbackable();
1837
1838       e.version = eversion_t(1, 1);
1839       e.soid.set_hash(0x9);
1840       log.tail = e.version;
1841       log.log.push_back(e);
1842       e.version = last_update;
1843       e.soid.set_hash(0x3);
1844       log.log.push_back(e);
1845       e.version = new_version;
1846       e.prior_version = eversion_t(1, 1);
1847       e.soid.set_hash(0x9);
1848       e.op = pg_log_entry_t::DELETE;
1849       log.log.push_back(e);
1850       log.head = e.version;
1851       log.index();
1852
1853       e.op = pg_log_entry_t::MODIFY;
1854       e.version = eversion_t(1, 1);
1855       e.soid.set_hash(0x9);
1856       olog.tail = e.version;
1857       olog.log.push_back(e);
1858       e.version = last_update;
1859       e.soid.set_hash(0x3);
1860       olog.log.push_back(e);
1861       e.version = divergent_version;
1862       e.prior_version = eversion_t(1, 1);
1863       e.soid.set_hash(0x9);
1864       divergent_object = e.soid;
1865       omissing.add(divergent_object, e.version, eversion_t(), false);
1866       e.op = pg_log_entry_t::MODIFY;
1867       olog.log.push_back(e);
1868       olog.head = e.version;
1869
1870       oinfo.last_update = olog.head;
1871       oinfo.last_complete = olog.head;
1872     }
1873
1874     EXPECT_TRUE(omissing.have_missing());
1875     EXPECT_TRUE(omissing.is_missing(divergent_object));
1876     EXPECT_EQ(divergent_version, omissing.get_items().at(divergent_object).need);
1877     EXPECT_EQ(olog.head, oinfo.last_update);
1878     EXPECT_EQ(olog.head, oinfo.last_complete);
1879
1880     missing.may_include_deletes = false;
1881     proc_replica_log(oinfo, olog, omissing, from);
1882
1883     EXPECT_TRUE(omissing.have_missing());
1884     EXPECT_TRUE(omissing.get_items().begin()->second.need == eversion_t(1, 1));
1885     EXPECT_EQ(last_update, oinfo.last_update);
1886     EXPECT_EQ(eversion_t(0, 0), oinfo.last_complete);
1887   }
1888
1889 }
1890
1891 TEST_F(PGLogTest, merge_log_1) {
1892   TestCase t;
1893   t.base.push_back(mk_ple_mod(mk_obj(1), mk_evt(10, 100), mk_evt(8, 80)));
1894
1895   t.div.push_back(mk_ple_mod(mk_obj(1), mk_evt(10, 101), mk_evt(10, 100)));
1896
1897   t.final.add(mk_obj(1), mk_evt(10, 100), mk_evt(0, 0), false);
1898
1899   t.toremove.insert(mk_obj(1));
1900
1901   t.setup();
1902   run_test_case(t);
1903 }
1904
1905 TEST_F(PGLogTest, merge_log_2) {
1906   TestCase t;
1907   t.base.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 100), mk_evt(8, 80)));
1908
1909   t.div.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 101), mk_evt(10, 100)));
1910   t.div.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 102), mk_evt(10, 101)));
1911
1912   t.torollback.insert(
1913     t.torollback.begin(), t.div.rbegin(), t.div.rend());
1914
1915   t.setup();
1916   run_test_case(t);
1917 }
1918
1919 TEST_F(PGLogTest, merge_log_3) {
1920   TestCase t;
1921   t.base.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 100), mk_evt(8, 80)));
1922
1923   t.div.push_back(mk_ple_mod(mk_obj(1), mk_evt(10, 101), mk_evt(10, 100)));
1924   t.div.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 102), mk_evt(10, 101)));
1925
1926   t.final.add(mk_obj(1), mk_evt(10, 100), mk_evt(0, 0), false);
1927
1928   t.toremove.insert(mk_obj(1));
1929
1930   t.setup();
1931   run_test_case(t);
1932 }
1933
1934 TEST_F(PGLogTest, merge_log_4) {
1935   TestCase t;
1936   t.base.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 100), mk_evt(8, 80)));
1937
1938   t.div.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 101), mk_evt(10, 100)));
1939   t.div.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 102), mk_evt(10, 101)));
1940
1941   t.init.add(mk_obj(1), mk_evt(10, 102), mk_evt(0, 0), false);
1942   t.final.add(mk_obj(1), mk_evt(10, 100), mk_evt(0, 0), false);
1943
1944   t.setup();
1945   run_test_case(t);
1946 }
1947
1948 TEST_F(PGLogTest, merge_log_5) {
1949   TestCase t;
1950   t.base.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 100), mk_evt(8, 80)));
1951
1952   t.div.push_back(mk_ple_mod(mk_obj(1), mk_evt(10, 101), mk_evt(10, 100)));
1953   t.div.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 102), mk_evt(10, 101)));
1954
1955   t.auth.push_back(mk_ple_mod(mk_obj(1), mk_evt(11, 101), mk_evt(10, 100)));
1956
1957   t.final.add(mk_obj(1), mk_evt(11, 101), mk_evt(0, 0), false);
1958
1959   t.toremove.insert(mk_obj(1));
1960
1961   t.setup();
1962   run_test_case(t);
1963 }
1964
1965 TEST_F(PGLogTest, merge_log_6) {
1966   TestCase t;
1967   t.base.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 100), mk_evt(8, 80)));
1968
1969   t.auth.push_back(mk_ple_mod(mk_obj(1), mk_evt(11, 101), mk_evt(10, 100)));
1970
1971   t.final.add(mk_obj(1), mk_evt(11, 101), mk_evt(10, 100), false);
1972
1973   t.setup();
1974   run_test_case(t);
1975 }
1976
1977 TEST_F(PGLogTest, merge_log_7) {
1978   TestCase t;
1979   t.base.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 100), mk_evt(8, 80)));
1980
1981   t.auth.push_back(mk_ple_mod(mk_obj(1), mk_evt(11, 101), mk_evt(10, 100)));
1982
1983   t.init.add(mk_obj(1), mk_evt(10, 100), mk_evt(8, 80), false);
1984   t.final.add(mk_obj(1), mk_evt(11, 101), mk_evt(8, 80), false);
1985
1986   t.setup();
1987   run_test_case(t);
1988 }
1989
1990 TEST_F(PGLogTest, merge_log_8) {
1991   TestCase t;
1992   t.base.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 100), mk_evt(8, 80)));
1993
1994   t.auth.push_back(mk_ple_dt(mk_obj(1), mk_evt(11, 101), mk_evt(10, 100)));
1995
1996   t.init.add(mk_obj(1), mk_evt(10, 100), mk_evt(8, 80), false);
1997   t.final.add(mk_obj(1), mk_evt(11, 101), mk_evt(8, 80), true);
1998
1999   t.setup();
2000   run_test_case(t);
2001 }
2002
2003 TEST_F(PGLogTest, merge_log_9) {
2004   TestCase t;
2005   t.base.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 100), mk_evt(8, 80)));
2006
2007   t.auth.push_back(mk_ple_dt(mk_obj(1), mk_evt(11, 101), mk_evt(10, 100)));
2008
2009   t.init.add(mk_obj(1), mk_evt(10, 100), mk_evt(8, 80), false);
2010   t.toremove.insert(mk_obj(1));
2011   t.deletes_during_peering = true;
2012
2013   t.setup();
2014   run_test_case(t);
2015 }
2016
2017 TEST_F(PGLogTest, merge_log_10) {
2018   TestCase t;
2019   t.base.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 100), mk_evt(8, 80)));
2020
2021   t.auth.push_back(mk_ple_ldt(mk_obj(1), mk_evt(11, 101), mk_evt(10, 100)));
2022
2023   t.init.add(mk_obj(1), mk_evt(10, 100), mk_evt(8, 80), false);
2024   t.final.add(mk_obj(1), mk_evt(11, 101), mk_evt(8, 80), true);
2025
2026   t.setup();
2027   run_test_case(t);
2028 }
2029
2030 TEST_F(PGLogTest, merge_log_prior_version_have) {
2031   TestCase t;
2032   t.base.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 100), mk_evt(8, 80)));
2033
2034   t.div.push_back(mk_ple_mod(mk_obj(1), mk_evt(10, 101), mk_evt(10, 100)));
2035
2036   t.init.add(mk_obj(1), mk_evt(10, 101), mk_evt(10, 100), false);
2037
2038   t.setup();
2039   run_test_case(t);
2040 }
2041
2042 TEST_F(PGLogTest, merge_log_split_missing_entries_at_head) {
2043   TestCase t;
2044   t.auth.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 100), mk_evt(8, 70)));
2045   t.auth.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(15, 150), mk_evt(10, 100)));
2046
2047   t.div.push_back(mk_ple_mod(mk_obj(1), mk_evt(8, 70), mk_evt(8, 65)));
2048
2049   t.setup();
2050   t.set_div_bounds(mk_evt(9, 79), mk_evt(8, 69));
2051   t.set_auth_bounds(mk_evt(15, 160), mk_evt(9, 77));
2052   t.final.add(mk_obj(1), mk_evt(15, 150), mk_evt(8, 70), false);
2053   run_test_case(t);
2054 }
2055
2056 TEST_F(PGLogTest, olog_tail_gt_log_tail_split) {
2057   TestCase t;
2058   t.auth.push_back(mk_ple_mod(mk_obj(1), mk_evt(10, 100), mk_evt(8, 70)));
2059   t.auth.push_back(mk_ple_mod(mk_obj(1), mk_evt(15, 150), mk_evt(10, 100)));
2060   t.auth.push_back(mk_ple_mod(mk_obj(1), mk_evt(15, 155), mk_evt(15, 150)));
2061
2062   t.setup();
2063   t.set_div_bounds(mk_evt(15, 153), mk_evt(15, 151));
2064   t.set_auth_bounds(mk_evt(15, 156), mk_evt(10, 99));
2065   t.final.add(mk_obj(1), mk_evt(15, 155), mk_evt(15, 150), false);
2066   run_test_case(t);
2067 }
2068
2069 TEST_F(PGLogTest, olog_tail_gt_log_tail_split2) {
2070   TestCase t;
2071   t.auth.push_back(mk_ple_mod(mk_obj(1), mk_evt(10, 100), mk_evt(8, 70)));
2072   t.auth.push_back(mk_ple_mod(mk_obj(1), mk_evt(15, 150), mk_evt(10, 100)));
2073   t.auth.push_back(mk_ple_mod(mk_obj(1), mk_evt(16, 155), mk_evt(15, 150)));
2074   t.div.push_back(mk_ple_mod(mk_obj(1), mk_evt(15, 153), mk_evt(15, 150)));
2075
2076   t.setup();
2077   t.set_div_bounds(mk_evt(15, 153), mk_evt(15, 151));
2078   t.set_auth_bounds(mk_evt(16, 156), mk_evt(10, 99));
2079   t.final.add(mk_obj(1), mk_evt(16, 155), mk_evt(0, 0), false);
2080   t.toremove.insert(mk_obj(1));
2081   run_test_case(t);
2082 }
2083
2084 TEST_F(PGLogTest, filter_log_1) {
2085   {
2086     clear();
2087
2088     int osd_id = 1;
2089     epoch_t epoch = 40;
2090     int64_t pool_id = 1;
2091     int bits = 2;
2092     int max_osd = 4;
2093     int pg_num = max_osd << bits;
2094     int num_objects = 1000;
2095     int num_internal = 10;
2096
2097     // Set up splitting map
2098     OSDMap *osdmap = new OSDMap;
2099     uuid_d test_uuid;
2100     test_uuid.generate_random();
2101     osdmap->build_simple_with_pool(g_ceph_context, epoch, test_uuid, max_osd, bits, bits);
2102     osdmap->set_state(osd_id, CEPH_OSD_EXISTS);
2103
2104     const string hit_set_namespace("internal");
2105
2106     {
2107       pg_log_entry_t e;
2108       e.mark_unrollbackable();
2109       e.op = pg_log_entry_t::MODIFY;
2110       e.soid.pool = pool_id;
2111
2112       uuid_d uuid_name;
2113       int i;
2114       for (i = 1; i <= num_objects; ++i) {
2115         e.version = eversion_t(epoch, i);
2116         // Use this to generate random file names
2117         uuid_name.generate_random();
2118         ostringstream name;
2119         name << uuid_name;
2120         e.soid.oid.name = name.str();
2121         // First has no namespace
2122         if (i != 1) {
2123            // num_internal have the internal namspace
2124           if (i <= num_internal + 1) {
2125             e.soid.nspace = hit_set_namespace;
2126           } else { // rest have different namespaces
2127             ostringstream ns;
2128             ns << "ns" << i;
2129             e.soid.nspace = ns.str();
2130           }
2131         }
2132         log.log.push_back(e);
2133         if (i == 1)
2134           log.tail = e.version;
2135       }
2136       log.head = e.version;
2137       log.index();
2138     }
2139
2140     spg_t pgid(pg_t(2, pool_id), shard_id_t::NO_SHARD);
2141
2142     // See if we created the right number of entries
2143     int total = log.log.size();
2144     ASSERT_EQ(total, num_objects);
2145
2146     // Some should be removed
2147     {
2148       pg_log_t filtered, reject;
2149       pg_log_t::filter_log(
2150         pgid, *osdmap, hit_set_namespace, log, filtered, reject);
2151       log = IndexedLog(filtered);
2152     }
2153     EXPECT_LE(log.log.size(), (size_t)total);
2154
2155     // If we filter a second time, there should be the same total
2156     total = log.log.size();
2157     {
2158       pg_log_t filtered, reject;
2159       pg_log_t::filter_log(
2160         pgid, *osdmap, hit_set_namespace, log, filtered, reject);
2161       log = IndexedLog(filtered);
2162     }
2163     EXPECT_EQ(log.log.size(), (size_t)total);
2164
2165     // Increase pg_num as if there would be a split
2166     int new_pg_num = pg_num * 16;
2167     OSDMap::Incremental inc(epoch + 1);
2168     inc.fsid = test_uuid;
2169     const pg_pool_t *pool = osdmap->get_pg_pool(pool_id);
2170     pg_pool_t newpool;
2171     newpool = *pool;
2172     newpool.set_pg_num(new_pg_num);
2173     newpool.set_pgp_num(new_pg_num);
2174     inc.new_pools[pool_id] = newpool;
2175     int ret = osdmap->apply_incremental(inc);
2176     ASSERT_EQ(ret, 0);
2177
2178     // We should have fewer entries after a filter
2179     {
2180       pg_log_t filtered, reject;
2181       pg_log_t::filter_log(
2182         pgid, *osdmap, hit_set_namespace, log, filtered, reject);
2183       log = IndexedLog(filtered);
2184     }
2185     EXPECT_LE(log.log.size(), (size_t)total);
2186
2187     // Make sure all internal entries are retained
2188     int count = 0;
2189     for (list<pg_log_entry_t>::iterator i = log.log.begin();
2190          i != log.log.end(); ++i) {
2191       if (i->soid.nspace == hit_set_namespace) count++;
2192     }
2193     EXPECT_EQ(count, num_internal);
2194   }
2195 }
2196
2197 TEST_F(PGLogTest, get_request) {
2198   clear();
2199
2200   // make sure writes, deletes, and errors are found
2201   vector<pg_log_entry_t> entries;
2202   hobject_t oid(object_t("objname"), "key", 123, 456, 0, "");
2203   entries.push_back(
2204     pg_log_entry_t(pg_log_entry_t::ERROR, oid, eversion_t(6,2), eversion_t(3,4),
2205                    1, osd_reqid_t(entity_name_t::CLIENT(777), 8, 1),
2206                    utime_t(0,1), -ENOENT));
2207   entries.push_back(
2208     pg_log_entry_t(pg_log_entry_t::MODIFY, oid, eversion_t(6,3), eversion_t(3,4),
2209                    2, osd_reqid_t(entity_name_t::CLIENT(777), 8, 2),
2210                    utime_t(1,2), 0));
2211   entries.push_back(
2212     pg_log_entry_t(pg_log_entry_t::DELETE, oid, eversion_t(7,4), eversion_t(7,4),
2213                    3, osd_reqid_t(entity_name_t::CLIENT(777), 8, 3),
2214                    utime_t(10,2), 0));
2215   entries.push_back(
2216     pg_log_entry_t(pg_log_entry_t::ERROR, oid, eversion_t(7,5), eversion_t(7,4),
2217                    3, osd_reqid_t(entity_name_t::CLIENT(777), 8, 4),
2218                    utime_t(20,1), -ENOENT));
2219
2220   for (auto &entry : entries) {
2221     log.add(entry);
2222   }
2223
2224   for (auto &entry : entries) {
2225     eversion_t replay_version;
2226     version_t user_version;
2227     int return_code = 0;
2228     bool got = log.get_request(
2229       entry.reqid, &replay_version, &user_version, &return_code);
2230     EXPECT_TRUE(got);
2231     EXPECT_EQ(entry.return_code, return_code);
2232     EXPECT_EQ(entry.version, replay_version);
2233     EXPECT_EQ(entry.user_version, user_version);
2234   }
2235 }
2236
2237 TEST_F(PGLogTest, ErrorNotIndexedByObject) {
2238   clear();
2239
2240   // make sure writes, deletes, and errors are found
2241   hobject_t oid(object_t("objname"), "key", 123, 456, 0, "");
2242   log.add(
2243     pg_log_entry_t(pg_log_entry_t::ERROR, oid, eversion_t(6,2), eversion_t(3,4),
2244                    1, osd_reqid_t(entity_name_t::CLIENT(777), 8, 1),
2245                    utime_t(0,1), -ENOENT));
2246
2247   EXPECT_FALSE(log.logged_object(oid));
2248
2249   pg_log_entry_t modify(pg_log_entry_t::MODIFY, oid, eversion_t(6,3),
2250                         eversion_t(3,4), 2,
2251                         osd_reqid_t(entity_name_t::CLIENT(777), 8, 2),
2252                         utime_t(1,2), 0);
2253   log.add(modify);
2254
2255   EXPECT_TRUE(log.logged_object(oid));
2256   pg_log_entry_t *entry = log.objects[oid];
2257   EXPECT_EQ(modify.op, entry->op);
2258   EXPECT_EQ(modify.version, entry->version);
2259   EXPECT_EQ(modify.prior_version, entry->prior_version);
2260   EXPECT_EQ(modify.user_version, entry->user_version);
2261   EXPECT_EQ(modify.reqid, entry->reqid);
2262
2263   pg_log_entry_t del(pg_log_entry_t::DELETE, oid, eversion_t(7,4),
2264                      eversion_t(7,4), 3,
2265                      osd_reqid_t(entity_name_t::CLIENT(777), 8, 3),
2266                      utime_t(10,2), 0);
2267   log.add(del);
2268
2269   EXPECT_TRUE(log.logged_object(oid));
2270   entry = log.objects[oid];
2271   EXPECT_EQ(del.op, entry->op);
2272   EXPECT_EQ(del.version, entry->version);
2273   EXPECT_EQ(del.prior_version, entry->prior_version);
2274   EXPECT_EQ(del.user_version, entry->user_version);
2275   EXPECT_EQ(del.reqid, entry->reqid);
2276
2277   log.add(
2278     pg_log_entry_t(pg_log_entry_t::ERROR, oid, eversion_t(7,5), eversion_t(7,4),
2279                    3, osd_reqid_t(entity_name_t::CLIENT(777), 8, 4),
2280                    utime_t(20,1), -ENOENT));
2281
2282   EXPECT_TRUE(log.logged_object(oid));
2283   entry = log.objects[oid];
2284   EXPECT_EQ(del.op, entry->op);
2285   EXPECT_EQ(del.version, entry->version);
2286   EXPECT_EQ(del.prior_version, entry->prior_version);
2287   EXPECT_EQ(del.user_version, entry->user_version);
2288   EXPECT_EQ(del.reqid, entry->reqid);
2289 }
2290
2291 TEST_F(PGLogTest, split_into_preserves_may_include_deletes) {
2292   clear();
2293
2294   {
2295     rebuilt_missing_with_deletes = false;
2296     missing.may_include_deletes = true;
2297     PGLog child_log(cct, prefix_provider);
2298     pg_t child_pg;
2299     split_into(child_pg, 6, &child_log);
2300     ASSERT_TRUE(child_log.get_missing().may_include_deletes);
2301     ASSERT_TRUE(child_log.get_rebuilt_missing_with_deletes());
2302   }
2303
2304   {
2305     rebuilt_missing_with_deletes = false;
2306     missing.may_include_deletes = false;
2307     PGLog child_log(cct, prefix_provider);
2308     pg_t child_pg;
2309     split_into(child_pg, 6, &child_log);
2310     ASSERT_FALSE(child_log.get_missing().may_include_deletes);
2311     ASSERT_FALSE(child_log.get_rebuilt_missing_with_deletes());
2312   }
2313 }
2314
2315 class PGLogTestRebuildMissing : public PGLogTest, public StoreTestFixture {
2316 public:
2317   PGLogTestRebuildMissing() : PGLogTest(), StoreTestFixture("memstore") {}
2318   void SetUp() override {
2319     StoreTestFixture::SetUp();
2320     ObjectStore::Sequencer osr(__func__);
2321     ObjectStore::Transaction t;
2322     test_coll = coll_t(spg_t(pg_t(1, 1)));
2323     t.create_collection(test_coll, 0);
2324     store->apply_transaction(&osr, std::move(t));
2325     existing_oid = mk_obj(0);
2326     nonexistent_oid = mk_obj(1);
2327     ghobject_t existing_ghobj(existing_oid);
2328     object_info_t existing_info;
2329     existing_info.version = eversion_t(6, 2);
2330     bufferlist enc_oi;
2331     ::encode(existing_info, enc_oi, 0);
2332     ObjectStore::Transaction t2;
2333     t2.touch(test_coll, ghobject_t(existing_oid));
2334     t2.setattr(test_coll, ghobject_t(existing_oid), OI_ATTR, enc_oi);
2335     ASSERT_EQ(0u, store->apply_transaction(&osr, std::move(t2)));
2336     info.last_backfill = hobject_t::get_max();
2337     info.last_complete = eversion_t();
2338   }
2339
2340   void TearDown() override {
2341     clear();
2342     missing.may_include_deletes = false;
2343     StoreTestFixture::TearDown();
2344   }
2345
2346   pg_info_t info;
2347   coll_t test_coll;
2348   hobject_t existing_oid, nonexistent_oid;
2349
2350   void run_rebuild_missing_test(const map<hobject_t, pg_missing_item> &expected_missing_items) {
2351     rebuild_missing_set_with_deletes(store.get(), test_coll, info);
2352     ASSERT_EQ(expected_missing_items, missing.get_items());
2353   }
2354 };
2355
2356 TEST_F(PGLogTestRebuildMissing, EmptyLog) {
2357   missing.add(existing_oid, mk_evt(6, 2), mk_evt(6, 3), false);
2358   missing.add(nonexistent_oid, mk_evt(7, 4), mk_evt(0, 0), false);
2359   map<hobject_t, pg_missing_item> orig_missing = missing.get_items();
2360   run_rebuild_missing_test(orig_missing);
2361 }
2362
2363 TEST_F(PGLogTestRebuildMissing, SameVersionMod) {
2364   missing.add(existing_oid, mk_evt(6, 2), mk_evt(6, 1), false);
2365   log.add(mk_ple_mod(existing_oid, mk_evt(6, 2), mk_evt(6, 1)));
2366   map<hobject_t, pg_missing_item> empty_missing;
2367   run_rebuild_missing_test(empty_missing);
2368 }
2369
2370 TEST_F(PGLogTestRebuildMissing, DelExisting) {
2371   missing.add(existing_oid, mk_evt(6, 3), mk_evt(6, 2), false);
2372   log.add(mk_ple_dt(existing_oid, mk_evt(7, 5), mk_evt(7, 4)));
2373   map<hobject_t, pg_missing_item> expected;
2374   expected[existing_oid] = pg_missing_item(mk_evt(7, 5), mk_evt(6, 2), true);
2375   run_rebuild_missing_test(expected);
2376 }
2377
2378 TEST_F(PGLogTestRebuildMissing, DelNonexistent) {
2379   log.add(mk_ple_dt(nonexistent_oid, mk_evt(7, 5), mk_evt(7, 4)));
2380   map<hobject_t, pg_missing_item> expected;
2381   expected[nonexistent_oid] = pg_missing_item(mk_evt(7, 5), mk_evt(0, 0), true);
2382   run_rebuild_missing_test(expected);
2383 }
2384
2385 TEST_F(PGLogTestRebuildMissing, MissingNotInLog) {
2386   missing.add(mk_obj(10), mk_evt(8, 12), mk_evt(8, 10), false);
2387   log.add(mk_ple_dt(nonexistent_oid, mk_evt(7, 5), mk_evt(7, 4)));
2388   map<hobject_t, pg_missing_item> expected;
2389   expected[nonexistent_oid] = pg_missing_item(mk_evt(7, 5), mk_evt(0, 0), true);
2390   expected[mk_obj(10)] = pg_missing_item(mk_evt(8, 12), mk_evt(8, 10), false);
2391   run_rebuild_missing_test(expected);
2392 }
2393
2394
2395 class PGLogMergeDupsTest : protected PGLog, public StoreTestFixture {
2396
2397 public:
2398
2399   PGLogMergeDupsTest() : PGLog(g_ceph_context), StoreTestFixture("memstore") { }
2400
2401   void SetUp() override {
2402     StoreTestFixture::SetUp();
2403     ObjectStore::Sequencer osr(__func__);
2404     ObjectStore::Transaction t;
2405     test_coll = coll_t(spg_t(pg_t(1, 1)));
2406     t.create_collection(test_coll, 0);
2407     store->apply_transaction(&osr, std::move(t));
2408   }
2409
2410   void TearDown() override {
2411     test_disk_roundtrip();
2412     clear();
2413     StoreTestFixture::TearDown();
2414   }
2415
2416   static pg_log_dup_t create_dup_entry(uint a, uint b) {
2417     // make each dup_entry unique by using different client id's
2418     static uint client_id = 777;
2419     return pg_log_dup_t(eversion_t(a, b),
2420                         a,
2421                         osd_reqid_t(entity_name_t::CLIENT(client_id++), 8, 1),
2422                         0);
2423   }
2424
2425   static std::vector<pg_log_dup_t> example_dups_1() {
2426     std::vector<pg_log_dup_t> result = {
2427       create_dup_entry(10, 11),
2428       create_dup_entry(10, 12),
2429       create_dup_entry(11, 1),
2430       create_dup_entry(12, 3),
2431       create_dup_entry(13, 99)
2432     };
2433     return result;
2434   }
2435
2436   static std::vector<pg_log_dup_t> example_dups_2() {
2437     std::vector<pg_log_dup_t> result = {
2438       create_dup_entry(12, 3),
2439       create_dup_entry(13, 99),
2440       create_dup_entry(15, 11),
2441       create_dup_entry(16, 14),
2442       create_dup_entry(16, 32)
2443     };
2444     return result;
2445   }
2446
2447   void add_dups(uint a, uint b) {
2448     log.dups.push_back(create_dup_entry(a, b));
2449     write_from_dups = MIN(write_from_dups, log.dups.back().version);
2450   }
2451
2452   void add_dups(const std::vector<pg_log_dup_t>& l) {
2453     for (auto& i : l) {
2454       log.dups.push_back(i);
2455       write_from_dups = MIN(write_from_dups, log.dups.back().version);
2456     }
2457   }
2458
2459   static void add_dups(IndexedLog& log, const std::vector<pg_log_dup_t>& dups) {
2460     for (auto& i : dups) {
2461       log.dups.push_back(i);
2462     }
2463   }
2464
2465   void check_order() {
2466     eversion_t prev(0, 0);
2467
2468     for (auto& i : log.dups) {
2469       EXPECT_LT(prev, i.version) << "verify versions monotonically increase";
2470       prev = i.version;
2471     }
2472   }
2473
2474   void check_index() {
2475     EXPECT_EQ(log.dups.size(), log.dup_index.size());
2476     for (auto& i : log.dups) {
2477       EXPECT_EQ(1u, log.dup_index.count(i.reqid));
2478     }
2479   }
2480
2481   void test_disk_roundtrip() {
2482     ObjectStore::Sequencer osr(__func__);
2483     ObjectStore::Transaction t;
2484     hobject_t hoid;
2485     hoid.pool = 1;
2486     hoid.oid = "log";
2487     ghobject_t log_oid(hoid);
2488     map<string, bufferlist> km;
2489     write_log_and_missing(t, &km, test_coll, log_oid, false);
2490     if (!km.empty()) {
2491       t.omap_setkeys(test_coll, log_oid, km);
2492     }
2493     ASSERT_EQ(0u, store->apply_transaction(&osr, std::move(t)));
2494
2495     auto orig_dups = log.dups;
2496     clear();
2497     ostringstream err;
2498     read_log_and_missing(store.get(), test_coll, test_coll, log_oid,
2499                          pg_info_t(), false, err, false);
2500     ASSERT_EQ(orig_dups.size(), log.dups.size());
2501     ASSERT_EQ(orig_dups, log.dups);
2502     auto dups_it = log.dups.begin();
2503     for (auto orig_dup : orig_dups) {
2504       ASSERT_EQ(orig_dup, *dups_it);
2505       ++dups_it;
2506     }
2507   }
2508
2509   coll_t test_coll;
2510 };
2511
2512 TEST_F(PGLogMergeDupsTest, OtherEmpty) {
2513   log.tail = eversion_t(14, 5);
2514
2515   IndexedLog olog;
2516
2517   add_dups(example_dups_1());
2518   index();
2519
2520   bool changed = merge_log_dups(olog);
2521
2522   EXPECT_FALSE(changed);
2523   EXPECT_EQ(5u, log.dups.size());
2524
2525   if (5 == log.dups.size()) {
2526     EXPECT_EQ(10u, log.dups.front().version.epoch);
2527     EXPECT_EQ(11u, log.dups.front().version.version);
2528     EXPECT_EQ(13u, log.dups.back().version.epoch);
2529     EXPECT_EQ(99u, log.dups.back().version.version);
2530   }
2531
2532   check_order();
2533   check_index();
2534 }
2535
2536 TEST_F(PGLogMergeDupsTest, AmEmpty) {
2537   log.tail = eversion_t(14, 5);
2538   index();
2539
2540   IndexedLog olog;
2541
2542   add_dups(olog, example_dups_1());
2543
2544   bool changed = merge_log_dups(olog);
2545
2546   EXPECT_TRUE(changed);
2547   EXPECT_EQ(5u, log.dups.size());
2548
2549   if (5 == log.dups.size()) {
2550     EXPECT_EQ(10u, log.dups.front().version.epoch);
2551     EXPECT_EQ(11u, log.dups.front().version.version);
2552
2553     EXPECT_EQ(13u, log.dups.back().version.epoch);
2554     EXPECT_EQ(99u, log.dups.back().version.version);
2555   }
2556
2557   check_order();
2558   check_index();
2559 }
2560
2561 TEST_F(PGLogMergeDupsTest, AmEmptyOverlap) {
2562   log.tail = eversion_t(12, 3);
2563   index();
2564
2565   IndexedLog olog;
2566
2567   add_dups(olog, example_dups_1());
2568
2569   bool changed = merge_log_dups(olog);
2570
2571   EXPECT_TRUE(changed);
2572   EXPECT_EQ(3u, log.dups.size());
2573
2574   if (3 == log.dups.size()) {
2575     EXPECT_EQ(10u, log.dups.front().version.epoch);
2576     EXPECT_EQ(11u, log.dups.front().version.version);
2577
2578     EXPECT_EQ(11u, log.dups.back().version.epoch);
2579     EXPECT_EQ(1u, log.dups.back().version.version);
2580   }
2581
2582   check_order();
2583   check_index();
2584 }
2585
2586 TEST_F(PGLogMergeDupsTest, Same) {
2587   log.tail = eversion_t(14, 1);
2588
2589   IndexedLog olog;
2590
2591   add_dups(example_dups_1());
2592   index();
2593   add_dups(olog, example_dups_1());
2594
2595   bool changed = merge_log_dups(olog);
2596
2597   EXPECT_FALSE(changed);
2598   EXPECT_EQ(5u, log.dups.size());
2599
2600   if (5 == log.dups.size()) {
2601     EXPECT_EQ(10u, log.dups.front().version.epoch);
2602     EXPECT_EQ(11u, log.dups.front().version.version);
2603
2604     EXPECT_EQ(13u, log.dups.back().version.epoch);
2605     EXPECT_EQ(99u, log.dups.back().version.version);
2606   }
2607
2608   check_order();
2609   check_index();
2610 }
2611
2612
2613 TEST_F(PGLogMergeDupsTest, Later) {
2614   log.tail = eversion_t(16, 14);
2615
2616   IndexedLog olog;
2617
2618   add_dups(example_dups_1());
2619   index();
2620   add_dups(olog, example_dups_2());
2621
2622   bool changed = merge_log_dups(olog);
2623
2624   EXPECT_TRUE(changed);
2625   EXPECT_EQ(6u, log.dups.size());
2626
2627   if (6 == log.dups.size()) {
2628     EXPECT_EQ(10u, log.dups.front().version.epoch);
2629     EXPECT_EQ(11u, log.dups.front().version.version);
2630
2631     EXPECT_EQ(15u, log.dups.back().version.epoch);
2632     EXPECT_EQ(11u, log.dups.back().version.version);
2633   }
2634
2635   check_order();
2636   check_index();
2637 }
2638
2639
2640 TEST_F(PGLogMergeDupsTest, Earlier) {
2641   log.tail = eversion_t(17, 2);
2642
2643   IndexedLog olog;
2644
2645   add_dups(example_dups_2());
2646   index();
2647   add_dups(olog, example_dups_1());
2648
2649   bool changed = merge_log_dups(olog);
2650
2651   EXPECT_TRUE(changed);
2652   EXPECT_EQ(8u, log.dups.size());
2653
2654   if (6 == log.dups.size()) {
2655     EXPECT_EQ(10u, log.dups.front().version.epoch);
2656     EXPECT_EQ(11u, log.dups.front().version.version);
2657
2658     EXPECT_EQ(16u, log.dups.back().version.epoch);
2659     EXPECT_EQ(32u, log.dups.back().version.version);
2660   }
2661
2662   check_order();
2663   check_index();
2664 }
2665
2666
2667 TEST_F(PGLogMergeDupsTest, Superset) {
2668   log.tail = eversion_t(17, 2);
2669
2670   IndexedLog olog;
2671
2672   add_dups(example_dups_1());
2673   index();
2674
2675   olog.dups.push_back(create_dup_entry(9, 5));
2676   olog.dups.push_back(create_dup_entry(15, 11));
2677
2678   bool changed = merge_log_dups(olog);
2679
2680   EXPECT_TRUE(changed);
2681   EXPECT_EQ(7u, log.dups.size());
2682
2683   if (7 == log.dups.size()) {
2684     EXPECT_EQ(9u, log.dups.front().version.epoch);
2685     EXPECT_EQ(5u, log.dups.front().version.version);
2686
2687     EXPECT_EQ(15u, log.dups.back().version.epoch);
2688     EXPECT_EQ(11u, log.dups.back().version.version);
2689   }
2690
2691   check_order();
2692   check_index();
2693 }
2694
2695
2696 struct PGLogTrimTest :
2697   public ::testing::Test,
2698   public PGLogTestBase,
2699   public PGLog::IndexedLog
2700 {
2701   CephContext *cct = g_ceph_context;
2702
2703   void SetUp(unsigned min_entries, unsigned max_entries, unsigned dup_track) {
2704     constexpr size_t size = 10;
2705
2706     char min_entries_s[size];
2707     char max_entries_s[size];
2708     char dup_track_s[size];
2709
2710     snprintf(min_entries_s, size, "%u", min_entries);
2711     snprintf(max_entries_s, size, "%u", max_entries);
2712     snprintf(dup_track_s, size, "%u", dup_track);
2713
2714     cct->_conf->set_val_or_die("osd_min_pg_log_entries", min_entries_s);
2715     cct->_conf->set_val_or_die("osd_max_pg_log_entries", max_entries_s);
2716     cct->_conf->set_val_or_die("osd_pg_log_dups_tracked", dup_track_s);
2717   }
2718 }; // struct PGLogTrimTest
2719
2720
2721 TEST_F(PGLogTrimTest, TestMakingCephContext)
2722 {
2723   SetUp(1, 2, 5);
2724
2725   EXPECT_EQ(1u, cct->_conf->osd_min_pg_log_entries);
2726   EXPECT_EQ(2u, cct->_conf->osd_max_pg_log_entries);
2727   EXPECT_EQ(5u, cct->_conf->osd_pg_log_dups_tracked);
2728 }
2729
2730
2731 TEST_F(PGLogTrimTest, TestPartialTrim)
2732 {
2733   SetUp(1, 2, 20);
2734   PGLog::IndexedLog log;
2735   log.head = mk_evt(24, 0);
2736   log.skip_can_rollback_to_to_head();
2737   log.head = mk_evt(9, 0);
2738
2739   log.add(mk_ple_mod(mk_obj(1), mk_evt(10, 100), mk_evt(8, 70)));
2740   log.add(mk_ple_dt(mk_obj(2), mk_evt(15, 150), mk_evt(10, 100)));
2741   log.add(mk_ple_mod_rb(mk_obj(3), mk_evt(15, 155), mk_evt(15, 150)));
2742   log.add(mk_ple_mod(mk_obj(1), mk_evt(19, 160), mk_evt(25, 152)));
2743   log.add(mk_ple_mod(mk_obj(4), mk_evt(21, 165), mk_evt(26, 160)));
2744   log.add(mk_ple_dt_rb(mk_obj(5), mk_evt(21, 167), mk_evt(31, 166)));
2745
2746   std::set<eversion_t> trimmed;
2747   std::set<std::string> trimmed_dups;
2748   eversion_t write_from_dups = eversion_t::max();
2749
2750   log.trim(cct, mk_evt(19, 157), &trimmed, &trimmed_dups, &write_from_dups);
2751
2752   EXPECT_EQ(eversion_t(15, 150), write_from_dups);
2753   EXPECT_EQ(3u, log.log.size());
2754   EXPECT_EQ(3u, trimmed.size());
2755   EXPECT_EQ(2u, log.dups.size());
2756   EXPECT_EQ(0u, trimmed_dups.size());
2757
2758   SetUp(1, 2, 15);
2759
2760   std::set<eversion_t> trimmed2;
2761   std::set<std::string> trimmed_dups2;
2762   eversion_t write_from_dups2 = eversion_t::max();
2763
2764   log.trim(cct, mk_evt(20, 164), &trimmed2, &trimmed_dups2, &write_from_dups2);
2765
2766   EXPECT_EQ(eversion_t(19, 160), write_from_dups2);
2767   EXPECT_EQ(2u, log.log.size());
2768   EXPECT_EQ(1u, trimmed2.size());
2769   EXPECT_EQ(2u, log.dups.size());
2770   EXPECT_EQ(1u, trimmed_dups2.size());
2771 }
2772
2773
2774 TEST_F(PGLogTrimTest, TestTrimNoTrimmed) {
2775   SetUp(1, 2, 20);
2776   PGLog::IndexedLog log;
2777   log.head = mk_evt(20, 0);
2778   log.skip_can_rollback_to_to_head();
2779   log.head = mk_evt(9, 0);
2780
2781   log.add(mk_ple_mod(mk_obj(1), mk_evt(10, 100), mk_evt(8, 70)));
2782   log.add(mk_ple_dt(mk_obj(2), mk_evt(15, 150), mk_evt(10, 100)));
2783   log.add(mk_ple_mod_rb(mk_obj(3), mk_evt(15, 155), mk_evt(15, 150)));
2784   log.add(mk_ple_mod(mk_obj(1), mk_evt(20, 160), mk_evt(25, 152)));
2785   log.add(mk_ple_mod(mk_obj(4), mk_evt(21, 165), mk_evt(26, 160)));
2786   log.add(mk_ple_dt_rb(mk_obj(5), mk_evt(21, 167), mk_evt(31, 166)));
2787
2788   eversion_t write_from_dups = eversion_t::max();
2789
2790   log.trim(cct, mk_evt(19, 157), nullptr, nullptr, &write_from_dups);
2791
2792   EXPECT_EQ(eversion_t(15, 150), write_from_dups);
2793   EXPECT_EQ(3u, log.log.size());
2794   EXPECT_EQ(2u, log.dups.size());
2795 }
2796
2797
2798 TEST_F(PGLogTrimTest, TestTrimNoDups)
2799 {
2800   SetUp(1, 2, 10);
2801   PGLog::IndexedLog log;
2802   log.head = mk_evt(20, 0);
2803   log.skip_can_rollback_to_to_head();
2804   log.head = mk_evt(9, 0);
2805
2806   log.add(mk_ple_mod(mk_obj(1), mk_evt(10, 100), mk_evt(8, 70)));
2807   log.add(mk_ple_dt(mk_obj(2), mk_evt(15, 150), mk_evt(10, 100)));
2808   log.add(mk_ple_mod_rb(mk_obj(3), mk_evt(15, 155), mk_evt(15, 150)));
2809   log.add(mk_ple_mod(mk_obj(1), mk_evt(20, 160), mk_evt(25, 152)));
2810   log.add(mk_ple_mod(mk_obj(4), mk_evt(21, 165), mk_evt(26, 160)));
2811   log.add(mk_ple_dt_rb(mk_obj(5), mk_evt(21, 167), mk_evt(31, 166)));
2812
2813   std::set<eversion_t> trimmed;
2814   std::set<std::string> trimmed_dups;
2815   eversion_t write_from_dups = eversion_t::max();
2816
2817   log.trim(cct, mk_evt(19, 157), &trimmed, &trimmed_dups, &write_from_dups);
2818
2819   EXPECT_EQ(eversion_t::max(), write_from_dups);
2820   EXPECT_EQ(3u, log.log.size());
2821   EXPECT_EQ(3u, trimmed.size());
2822   EXPECT_EQ(0u, log.dups.size());
2823   EXPECT_EQ(0u, trimmed_dups.size());
2824 }
2825
2826 TEST_F(PGLogTrimTest, TestNoTrim)
2827 {
2828   SetUp(1, 2, 20);
2829   PGLog::IndexedLog log;
2830   log.head = mk_evt(24, 0);
2831   log.skip_can_rollback_to_to_head();
2832   log.head = mk_evt(9, 0);
2833
2834   log.add(mk_ple_mod(mk_obj(1), mk_evt(10, 100), mk_evt(8, 70)));
2835   log.add(mk_ple_dt(mk_obj(2), mk_evt(15, 150), mk_evt(10, 100)));
2836   log.add(mk_ple_mod_rb(mk_obj(3), mk_evt(15, 155), mk_evt(15, 150)));
2837   log.add(mk_ple_mod(mk_obj(1), mk_evt(19, 160), mk_evt(25, 152)));
2838   log.add(mk_ple_mod(mk_obj(4), mk_evt(21, 165), mk_evt(26, 160)));
2839   log.add(mk_ple_dt_rb(mk_obj(5), mk_evt(21, 167), mk_evt(31, 166)));
2840
2841   std::set<eversion_t> trimmed;
2842   std::set<std::string> trimmed_dups;
2843   eversion_t write_from_dups = eversion_t::max();
2844
2845   log.trim(cct, mk_evt(9, 99), &trimmed, &trimmed_dups, &write_from_dups);
2846
2847   EXPECT_EQ(eversion_t::max(), write_from_dups);
2848   EXPECT_EQ(6u, log.log.size());
2849   EXPECT_EQ(0u, trimmed.size());
2850   EXPECT_EQ(0u, log.dups.size());
2851   EXPECT_EQ(0u, trimmed_dups.size());
2852 }
2853
2854 TEST_F(PGLogTrimTest, TestTrimAll)
2855 {
2856   SetUp(1, 2, 20);
2857   PGLog::IndexedLog log;
2858   log.head = mk_evt(24, 0);
2859   log.skip_can_rollback_to_to_head();
2860   log.head = mk_evt(9, 0);
2861
2862   log.add(mk_ple_mod(mk_obj(1), mk_evt(10, 100), mk_evt(8, 70)));
2863   log.add(mk_ple_dt(mk_obj(2), mk_evt(15, 150), mk_evt(10, 100)));
2864   log.add(mk_ple_mod_rb(mk_obj(3), mk_evt(15, 155), mk_evt(15, 150)));
2865   log.add(mk_ple_mod(mk_obj(1), mk_evt(19, 160), mk_evt(25, 152)));
2866   log.add(mk_ple_mod(mk_obj(4), mk_evt(21, 165), mk_evt(26, 160)));
2867   log.add(mk_ple_dt_rb(mk_obj(5), mk_evt(21, 167), mk_evt(31, 166)));
2868
2869   std::set<eversion_t> trimmed;
2870   std::set<std::string> trimmed_dups;
2871   eversion_t write_from_dups = eversion_t::max();
2872
2873   log.trim(cct, mk_evt(22, 180), &trimmed, &trimmed_dups, &write_from_dups);
2874
2875   EXPECT_EQ(eversion_t(15, 150), write_from_dups);
2876   EXPECT_EQ(0u, log.log.size());
2877   EXPECT_EQ(6u, trimmed.size());
2878   EXPECT_EQ(5u, log.dups.size());
2879   EXPECT_EQ(0u, trimmed_dups.size());
2880 }
2881
2882
2883 TEST_F(PGLogTrimTest, TestGetRequest) {
2884   SetUp(1, 2, 20);
2885   PGLog::IndexedLog log;
2886   log.head = mk_evt(20, 0);
2887   log.skip_can_rollback_to_to_head();
2888   log.head = mk_evt(9, 0);
2889
2890   entity_name_t client = entity_name_t::CLIENT(777);
2891
2892   log.add(mk_ple_mod(mk_obj(1), mk_evt(10, 100), mk_evt(8, 70),
2893                      osd_reqid_t(client, 8, 1)));
2894   log.add(mk_ple_dt(mk_obj(2), mk_evt(15, 150), mk_evt(10, 100),
2895                     osd_reqid_t(client, 8, 2)));
2896   log.add(mk_ple_mod_rb(mk_obj(3), mk_evt(15, 155), mk_evt(15, 150),
2897                         osd_reqid_t(client, 8, 3)));
2898   log.add(mk_ple_mod(mk_obj(1), mk_evt(20, 160), mk_evt(25, 152),
2899                      osd_reqid_t(client, 8, 4)));
2900   log.add(mk_ple_mod(mk_obj(4), mk_evt(21, 165), mk_evt(26, 160),
2901                      osd_reqid_t(client, 8, 5)));
2902   log.add(mk_ple_dt_rb(mk_obj(5), mk_evt(21, 167), mk_evt(31, 166),
2903                        osd_reqid_t(client, 8, 6)));
2904
2905   eversion_t write_from_dups = eversion_t::max();
2906
2907   log.trim(cct, mk_evt(19, 157), nullptr, nullptr, &write_from_dups);
2908
2909   EXPECT_EQ(eversion_t(15, 150), write_from_dups);
2910   EXPECT_EQ(3u, log.log.size());
2911   EXPECT_EQ(2u, log.dups.size());
2912
2913   eversion_t version;
2914   version_t user_version;
2915   int return_code;
2916
2917   osd_reqid_t log_reqid = osd_reqid_t(client, 8, 5);
2918   osd_reqid_t dup_reqid = osd_reqid_t(client, 8, 3);
2919   osd_reqid_t bad_reqid = osd_reqid_t(client, 8, 1);
2920
2921   bool result;
2922
2923   result = log.get_request(log_reqid, &version, &user_version, &return_code);
2924   EXPECT_EQ(true, result);
2925   EXPECT_EQ(mk_evt(21, 165), version);
2926
2927   result = log.get_request(dup_reqid, &version, &user_version, &return_code);
2928   EXPECT_EQ(true, result);
2929   EXPECT_EQ(mk_evt(15, 155), version);
2930
2931   result = log.get_request(bad_reqid, &version, &user_version, &return_code);
2932   EXPECT_FALSE(result);
2933 }
2934
2935 TEST_F(PGLogTest, _merge_object_divergent_entries) {
2936   {
2937     // Test for issue 20843
2938     clear();
2939     hobject_t hoid(object_t(/*name*/"notify.7"),
2940                    /*key*/string(""),
2941                    /*snap*/7,
2942                    /*hash*/77,
2943                    /*pool*/5,
2944                    /*nspace*/string(""));
2945     mempool::osd_pglog::list<pg_log_entry_t> orig_entries;
2946     orig_entries.push_back(mk_ple_mod(hoid, eversion_t(8336, 957), eversion_t(8336, 952)));
2947     orig_entries.push_back(mk_ple_err(hoid, eversion_t(8336, 958)));
2948     orig_entries.push_back(mk_ple_err(hoid, eversion_t(8336, 959)));
2949     orig_entries.push_back(mk_ple_mod(hoid, eversion_t(8336, 960), eversion_t(8336, 957)));
2950     log.add(mk_ple_mod(hoid, eversion_t(8973, 1075), eversion_t(8971, 1070)));
2951     missing.add(hoid,
2952                 /*need*/eversion_t(8971, 1070),
2953                 /*have*/eversion_t(8336, 952),
2954                 false);
2955     pg_info_t oinfo;
2956     LogHandler rollbacker;
2957     _merge_object_divergent_entries(log, hoid,
2958                                     orig_entries, oinfo,
2959                                     log.get_can_rollback_to(),
2960                                     missing, &rollbacker,
2961                                     this);
2962     // No core dump
2963   }
2964   {
2965     // skip leading error entries
2966     clear();
2967     hobject_t hoid(object_t(/*name*/"notify.7"),
2968                    /*key*/string(""),
2969                    /*snap*/7,
2970                    /*hash*/77,
2971                    /*pool*/5,
2972                    /*nspace*/string(""));
2973     mempool::osd_pglog::list<pg_log_entry_t> orig_entries;
2974     orig_entries.push_back(mk_ple_err(hoid, eversion_t(8336, 956)));
2975     orig_entries.push_back(mk_ple_mod(hoid, eversion_t(8336, 957), eversion_t(8336, 952)));
2976     log.add(mk_ple_mod(hoid, eversion_t(8973, 1075), eversion_t(8971, 1070)));
2977     missing.add(hoid,
2978                 /*need*/eversion_t(8971, 1070),
2979                 /*have*/eversion_t(8336, 952),
2980                 false);
2981     pg_info_t oinfo;
2982     LogHandler rollbacker;
2983     _merge_object_divergent_entries(log, hoid,
2984                                     orig_entries, oinfo,
2985                                     log.get_can_rollback_to(),
2986                                     missing, &rollbacker,
2987                                     this);
2988     // No core dump
2989   }
2990 }
2991
2992
2993 // Local Variables:
2994 // compile-command: "cd ../.. ; make unittest_pglog ; ./unittest_pglog --log-to-stderr=true  --debug-osd=20 # --gtest_filter=*.* "
2995 // End: