X-Git-Url: https://gerrit.opnfv.org/gerrit/gitweb?a=blobdiff_plain;f=src%2Fceph%2Fsrc%2Ftest%2Fcompressor%2Ftest_compression.cc;fp=src%2Fceph%2Fsrc%2Ftest%2Fcompressor%2Ftest_compression.cc;h=179d1acbf54a30ce0b6d8bd1bd998144113eb198;hb=812ff6ca9fcd3e629e49d4328905f33eee8ca3f5;hp=0000000000000000000000000000000000000000;hpb=15280273faafb77777eab341909a3f495cf248d9;p=stor4nfv.git

diff --git a/src/ceph/src/test/compressor/test_compression.cc b/src/ceph/src/test/compressor/test_compression.cc
new file mode 100644
index 0000000..179d1ac
--- /dev/null
+++ b/src/ceph/src/test/compressor/test_compression.cc
@@ -0,0 +1,478 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph distributed storage system
+ *
+ * Copyright (C) 2015 Mirantis, Inc.
+ *
+ * Author: Alyona Kiseleva <akiselyova@mirantis.com>
+ *
+ *  This library is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU Lesser General Public
+ *  License as published by the Free Software Foundation; either
+ *  version 2.1 of the License, or (at your option) any later version.
+ *
+ */
+
+#include <errno.h>
+#include <signal.h>
+#include <stdlib.h>
+#include "gtest/gtest.h"
+#include "common/config.h"
+#include "compressor/Compressor.h"
+#include "compressor/CompressionPlugin.h"
+#include "global/global_context.h"
+
+class CompressorTest : public ::testing::Test,
+			public ::testing::WithParamInterface<const char*> {
+public:
+  std::string plugin;
+  CompressorRef compressor;
+  bool old_zlib_isal;
+
+  CompressorTest() {
+    // note for later
+    old_zlib_isal = g_conf->compressor_zlib_isal;
+
+    plugin = GetParam();
+    size_t pos = plugin.find('/');
+    if (pos != std::string::npos) {
+      string isal = plugin.substr(pos + 1);
+      plugin = plugin.substr(0, pos);
+      if (isal == "isal") {
+	g_conf->set_val("compressor_zlib_isal", "true");
+	g_ceph_context->_conf->apply_changes(NULL);
+      } else if (isal == "noisal") {
+	g_conf->set_val("compressor_zlib_isal", "false");
+	g_ceph_context->_conf->apply_changes(NULL);
+      } else {
+	assert(0 == "bad option");
+      }
+    }
+    cout << "[plugin " << plugin << " (" << GetParam() << ")]" << std::endl;
+  }
+  ~CompressorTest() override {
+    g_conf->set_val("compressor_zlib_isal", old_zlib_isal ? "true" : "false");
+    g_ceph_context->_conf->apply_changes(NULL);
+  }
+
+  void SetUp() override {
+    compressor = Compressor::create(g_ceph_context, plugin);
+    ASSERT_TRUE(compressor);
+  }
+  void TearDown() override {
+    compressor.reset();
+  }
+};
+
+TEST_P(CompressorTest, load_plugin)
+{
+}
+
+TEST_P(CompressorTest, small_round_trip)
+{
+  bufferlist orig;
+  orig.append("This is a short string.  There are many strings like it but this one is mine.");
+  bufferlist compressed;
+  int r = compressor->compress(orig, compressed);
+  ASSERT_EQ(0, r);
+  bufferlist decompressed;
+  r = compressor->decompress(compressed, decompressed);
+  ASSERT_EQ(0, r);
+  ASSERT_EQ(decompressed.length(), orig.length());
+  ASSERT_TRUE(decompressed.contents_equal(orig));
+  cout << "orig " << orig.length() << " compressed " << compressed.length()
+       << " with " << GetParam() << std::endl;
+}
+
+TEST_P(CompressorTest, big_round_trip_repeated)
+{
+  unsigned len = 1048576 * 4;
+  bufferlist orig;
+  while (orig.length() < len) {
+    orig.append("This is a short string.  There are many strings like it but this one is mine.");
+  }
+  bufferlist compressed;
+  int r = compressor->compress(orig, compressed);
+  ASSERT_EQ(0, r);
+  bufferlist decompressed;
+  r = compressor->decompress(compressed, decompressed);
+  ASSERT_EQ(0, r);
+  ASSERT_EQ(decompressed.length(), orig.length());
+  ASSERT_TRUE(decompressed.contents_equal(orig));
+  cout << "orig " << orig.length() << " compressed " << compressed.length()
+       << " with " << GetParam() << std::endl;
+}
+
+TEST_P(CompressorTest, big_round_trip_randomish)
+{
+  unsigned len = 1048576 * 10;//269;
+  bufferlist orig;
+  const char *alphabet = "abcdefghijklmnopqrstuvwxyz";
+  if (false) {
+    while (orig.length() < len) {
+      orig.append(alphabet[rand() % 10]);
+    }
+  } else {
+    bufferptr bp(len);
+    char *p = bp.c_str();
+    for (unsigned i=0; i<len; ++i) {
+      p[i] = alphabet[rand() % 10];
+    }
+    orig.append(bp);
+  }
+  bufferlist compressed;
+  int r = compressor->compress(orig, compressed);
+  ASSERT_EQ(0, r);
+  bufferlist decompressed;
+  r = compressor->decompress(compressed, decompressed);
+  ASSERT_EQ(0, r);
+  ASSERT_EQ(decompressed.length(), orig.length());
+  ASSERT_TRUE(decompressed.contents_equal(orig));
+  cout << "orig " << orig.length() << " compressed " << compressed.length()
+       << " with " << GetParam() << std::endl;
+}
+
+#if 0
+TEST_P(CompressorTest, big_round_trip_file)
+{
+  bufferlist orig;
+  int fd = ::open("bin/ceph-osd", O_RDONLY);
+  struct stat st;
+  ::fstat(fd, &st);
+  orig.read_fd(fd, st.st_size);
+
+  bufferlist compressed;
+  int r = compressor->compress(orig, compressed);
+  ASSERT_EQ(0, r);
+  bufferlist decompressed;
+  r = compressor->decompress(compressed, decompressed);
+  ASSERT_EQ(0, r);
+  ASSERT_EQ(decompressed.length(), orig.length());
+  ASSERT_TRUE(decompressed.contents_equal(orig));
+  cout << "orig " << orig.length() << " compressed " << compressed.length()
+       << " with " << GetParam() << std::endl;
+}
+#endif
+
+
+TEST_P(CompressorTest, compress_decompress)
+{
+  const char* test = "This is test text";
+  int res;
+  int len = strlen(test);
+  bufferlist in, out;
+  bufferlist after;
+  bufferlist exp;
+  in.append(test, len);
+  res = compressor->compress(in, out);
+  EXPECT_EQ(res, 0);
+  res = compressor->decompress(out, after);
+  EXPECT_EQ(res, 0);
+  exp.append(test);
+  EXPECT_TRUE(exp.contents_equal(after));
+  after.clear();
+  size_t compressed_len = out.length();
+  out.append_zero(12);
+  auto it = out.begin();
+  res = compressor->decompress(it, compressed_len, after);
+  EXPECT_EQ(res, 0);
+  EXPECT_TRUE(exp.contents_equal(after));
+
+  //large block and non-begin iterator for continuous block
+  std::string data;
+  data.resize(0x10000 * 1);
+  for(size_t i = 0; i < data.size(); i++)
+    data[i] = i / 256;
+  in.clear();
+  out.clear();
+  in.append(data);
+  exp = in;
+  res = compressor->compress(in, out);
+  EXPECT_EQ(res, 0);
+  compressed_len = out.length();
+  out.append_zero(0x10000 - out.length());
+  after.clear();
+  out.c_str();
+  bufferlist prefix;
+  prefix.append(string("some prefix"));
+  size_t prefix_len = prefix.length();
+  out.claim_prepend(prefix);
+  it = out.begin();
+  it.advance(prefix_len);
+  res = compressor->decompress(it, compressed_len, after);
+  EXPECT_EQ(res, 0);
+  EXPECT_TRUE(exp.contents_equal(after));
+}
+
+TEST_P(CompressorTest, sharded_input_decompress)
+{
+  const size_t small_prefix_size=3;
+
+  string test(128*1024,0);
+  int len = test.size();
+  bufferlist in, out;
+  in.append(test.c_str(), len);
+  int res = compressor->compress(in, out);
+  EXPECT_EQ(res, 0);
+  EXPECT_GT(out.length(), small_prefix_size);
+
+  bufferlist out2, tmp;
+  tmp.substr_of(out, 0, small_prefix_size );
+  out2.append( tmp );
+  size_t left = out.length()-small_prefix_size;
+  size_t offs = small_prefix_size;
+  while( left > 0 ){
+    size_t shard_size = MIN( 2048, left );
+    tmp.substr_of(out, offs, shard_size );
+    out2.append( tmp );
+    left -= shard_size;
+    offs += shard_size;
+  }
+
+  bufferlist after;
+  res = compressor->decompress(out2, after);
+  EXPECT_EQ(res, 0);
+}
+
+void test_compress(CompressorRef compressor, size_t size)
+{
+  char* data = (char*) malloc(size);
+  for (size_t t = 0; t < size; t++) {
+    data[t] = (t & 0xff) | (t >> 8);
+  }
+  bufferlist in;
+  in.append(data, size);
+  for (size_t t = 0; t < 10000; t++) {
+    bufferlist out;
+    int res = compressor->compress(in, out);
+    EXPECT_EQ(res, 0);
+  }
+}
+
+void test_decompress(CompressorRef compressor, size_t size)
+{
+  char* data = (char*) malloc(size);
+  for (size_t t = 0; t < size; t++) {
+    data[t] = (t & 0xff) | (t >> 8);
+  }
+  bufferlist in, out;
+  in.append(data, size);
+  int res = compressor->compress(in, out);
+  EXPECT_EQ(res, 0);
+  for (size_t t = 0; t < 10000; t++) {
+    bufferlist out_dec;
+    int res = compressor->decompress(out, out_dec);
+    EXPECT_EQ(res, 0);
+  }
+}
+
+TEST_P(CompressorTest, compress_1024)
+{
+  test_compress(compressor, 1024);
+}
+
+TEST_P(CompressorTest, compress_2048)
+{
+  test_compress(compressor, 2048);
+}
+
+TEST_P(CompressorTest, compress_4096)
+{
+  test_compress(compressor, 4096);
+}
+
+TEST_P(CompressorTest, compress_8192)
+{
+  test_compress(compressor, 8192);
+}
+
+TEST_P(CompressorTest, compress_16384)
+{
+  test_compress(compressor, 16384);
+}
+
+TEST_P(CompressorTest, decompress_1024)
+{
+  test_decompress(compressor, 1024);
+}
+
+TEST_P(CompressorTest, decompress_2048)
+{
+  test_decompress(compressor, 2048);
+}
+
+TEST_P(CompressorTest, decompress_4096)
+{
+  test_decompress(compressor, 4096);
+}
+
+TEST_P(CompressorTest, decompress_8192)
+{
+  test_decompress(compressor, 8192);
+}
+
+TEST_P(CompressorTest, decompress_16384)
+{
+  test_decompress(compressor, 16384);
+}
+
+
+INSTANTIATE_TEST_CASE_P(
+  Compressor,
+  CompressorTest,
+  ::testing::Values(
+#ifdef HAVE_LZ4
+    "lz4",
+#endif
+#ifdef __x86_64__
+    "zlib/isal",
+#endif
+    "zlib/noisal",
+    "snappy",
+    "zstd"));
+
+#ifdef __x86_64__
+
+TEST(ZlibCompressor, zlib_isal_compatibility)
+{
+  g_conf->set_val("compressor_zlib_isal", "true");
+  g_ceph_context->_conf->apply_changes(NULL);
+  CompressorRef isal = Compressor::create(g_ceph_context, "zlib");
+  if (!isal) {
+    // skip the test if the plugin is not ready
+    return;
+  }
+  g_conf->set_val("compressor_zlib_isal", "false");
+  g_ceph_context->_conf->apply_changes(NULL);
+  CompressorRef zlib = Compressor::create(g_ceph_context, "zlib");
+  char test[101];
+  srand(time(0));
+  for (int i=0; i<100; ++i)
+    test[i] = 'a' + rand()%26;
+  test[100] = '\0';
+  int len = strlen(test);
+  bufferlist in, out;
+  in.append(test, len);
+  // isal -> zlib
+  int res = isal->compress(in, out);
+  EXPECT_EQ(res, 0);
+  bufferlist after;
+  res = zlib->decompress(out, after);
+  EXPECT_EQ(res, 0);
+  bufferlist exp;
+  exp.append(test);
+  EXPECT_TRUE(exp.contents_equal(after));
+  after.clear();
+  out.clear();
+  exp.clear();
+  // zlib -> isal
+  res = zlib->compress(in, out);
+  EXPECT_EQ(res, 0);
+  res = isal->decompress(out, after);
+  EXPECT_EQ(res, 0);
+  exp.append(test);
+  EXPECT_TRUE(exp.contents_equal(after));
+}
+#endif
+
+TEST(CompressionPlugin, all)
+{
+  const char* env = getenv("CEPH_LIB");
+  std::string directory(env ? env : ".libs");
+  CompressorRef compressor;
+  PluginRegistry *reg = g_ceph_context->get_plugin_registry();
+  EXPECT_TRUE(reg);
+  CompressionPlugin *factory = dynamic_cast<CompressionPlugin*>(reg->get_with_load("compressor", "invalid"));
+  EXPECT_FALSE(factory);
+  factory = dynamic_cast<CompressionPlugin*>(reg->get_with_load("compressor", "example"));
+  EXPECT_TRUE(factory);
+  stringstream ss;
+  EXPECT_EQ(0, factory->factory(&compressor, &ss));
+  EXPECT_TRUE(compressor.get());
+  {
+    Mutex::Locker l(reg->lock);
+    EXPECT_EQ(-ENOENT, reg->remove("compressor", "does not exist"));
+    EXPECT_EQ(0, reg->remove("compressor", "example"));
+    EXPECT_EQ(0, reg->load("compressor", "example"));
+  }
+}
+
+#ifdef __x86_64__
+
+TEST(ZlibCompressor, isal_compress_zlib_decompress_random)
+{
+  g_conf->set_val("compressor_zlib_isal", "true");
+  g_ceph_context->_conf->apply_changes(NULL);
+  CompressorRef isal = Compressor::create(g_ceph_context, "zlib");
+  if (!isal) {
+    // skip the test if the plugin is not ready
+    return;
+  }
+  g_conf->set_val("compressor_zlib_isal", "false");
+  g_ceph_context->_conf->apply_changes(NULL);
+  CompressorRef zlib = Compressor::create(g_ceph_context, "zlib");
+
+  for (int cnt=0; cnt<100; cnt++)
+  {
+    srand(cnt + 1000);
+    int log2 = (rand()%18) + 1;
+    int size = (rand() % (1 << log2)) + 1;
+
+    char test[size];
+    for (int i=0; i<size; ++i)
+      test[i] = rand()%256;
+    bufferlist in, out;
+    in.append(test, size);
+
+    int res = isal->compress(in, out);
+    EXPECT_EQ(res, 0);
+    bufferlist after;
+    res = zlib->decompress(out, after);
+    EXPECT_EQ(res, 0);
+    bufferlist exp;
+    exp.append(test, size);
+    EXPECT_TRUE(exp.contents_equal(after));
+  }
+}
+
+TEST(ZlibCompressor, isal_compress_zlib_decompress_walk)
+{
+  g_conf->set_val("compressor_zlib_isal", "true");
+  g_ceph_context->_conf->apply_changes(NULL);
+  CompressorRef isal = Compressor::create(g_ceph_context, "zlib");
+  if (!isal) {
+    // skip the test if the plugin is not ready
+    return;
+  }
+  g_conf->set_val("compressor_zlib_isal", "false");
+  g_ceph_context->_conf->apply_changes(NULL);
+  CompressorRef zlib = Compressor::create(g_ceph_context, "zlib");
+
+  for (int cnt=0; cnt<100; cnt++)
+  {
+    srand(cnt + 1000);
+    int log2 = (rand()%18) + 1;
+    int size = (rand() % (1 << log2)) + 1;
+
+    int range = 1;
+
+    char test[size];
+    test[0] = rand()%256;
+    for (int i=1; i<size; ++i)
+      test[i] = test[i-1] + rand()%(range*2+1) - range;
+    bufferlist in, out;
+    in.append(test, size);
+
+    int res = isal->compress(in, out);
+    EXPECT_EQ(res, 0);
+    bufferlist after;
+    res = zlib->decompress(out, after);
+    EXPECT_EQ(res, 0);
+    bufferlist exp;
+    exp.append(test, size);
+    EXPECT_TRUE(exp.contents_equal(after));
+  }
+}
+
+#endif	// __x86_64__