Merge pull request #251 from ceph-dovecot/jrse_#249

Jrse #249
ceph-dovecot · Feb 21, 2019 · 56d6c90 · 56d6c90
2 parents a633d3a + 5968679
commit 56d6c90
Show file tree

Hide file tree

Showing 29 changed files with 207 additions and 123 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -53,6 +53,7 @@ script:
 
 - docker exec build sh -c 'rm -r /usr/local/var/mail/rbox'
 - docker exec build sh -c 'cd /usr/local/bin; ./imaptest user=t%d pass=t port=10143 error_quit secs=60 copybox=INBOX.Drafts output=/var/log/imaptest.log'
+- docker exec build sh -c 'doveadm -D force-resync -u t1 INBOX'
 - docker exec build bash -c '/usr/local/bin/exec.sh "cat /var/log/dovecot.log | grep \"Error:\""'
 - docker exec build bash -c '/usr/local/bin/exec.sh "cat /var/log/dovecot.log | grep \"failed:\""'
 - docker exec build bash -c '/usr/local/bin/exec.sh "cat /var/log/dovecot.log | grep \"Internal error\""'

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,7 +1,18 @@
 # Change Log
 
-## [0.0.20](https://github.com/ceph-dovecot/dovecot-ceph-plugin/tree/0.0.20)
+## [0.0.21](https://github.com/ceph-dovecot/dovecot-ceph-plugin/tree/0.0.21)
 
+[Full Changelog](https://github.com/ceph-dovecot/dovecot-ceph-plugin/compare/0.0.20...0.0.21)
+
+**Implemented enhancements:**
+
+- pass ceph client configuration via 90-plugin to [\#250](https://github.com/ceph-dovecot/dovecot-ceph-plugin/issues/250)
+
+**Fixed bugs:**
+
+- doveadm force-resync & virtual Namespace [\#249](https://github.com/ceph-dovecot/dovecot-ceph-plugin/issues/249)
+
+## [0.0.20](https://github.com/ceph-dovecot/dovecot-ceph-plugin/tree/0.0.20) (2019-02-11)
 [Full Changelog](https://github.com/ceph-dovecot/dovecot-ceph-plugin/compare/0.0.19...0.0.20)
 
 **Implemented enhancements:**
@@ -14,6 +25,10 @@
 - Performance: rbox backup [\#246](https://github.com/ceph-dovecot/dovecot-ceph-plugin/issues/246)
 - doveadm rmb check indices shows duplicate folder in output. [\#224](https://github.com/ceph-dovecot/dovecot-ceph-plugin/issues/224)
 
+**Merged pull requests:**
+
+- Doveadm rmb update jrse & performance [\#248](https://github.com/ceph-dovecot/dovecot-ceph-plugin/pull/248) ([jrse](https://github.com/jrse))
+
 ## [0.0.19](https://github.com/ceph-dovecot/dovecot-ceph-plugin/tree/0.0.19) (2019-02-06)
 [Full Changelog](https://github.com/ceph-dovecot/dovecot-ceph-plugin/compare/0.0.18...0.0.19)
 

diff --git a/configure.ac b/configure.ac
@@ -8,7 +8,7 @@
 
 AC_PREREQ([2.59])
 
-AC_INIT([dovecot-ceph-plugin], [0.0.20], [https://github.com/ceph-dovecot/dovecot-ceph-plugin/issues/new], ,[https://github.com/ceph-dovecot/dovecot-ceph-plugin])
+AC_INIT([dovecot-ceph-plugin], [0.0.21], [https://github.com/ceph-dovecot/dovecot-ceph-plugin/issues/new], ,[https://github.com/ceph-dovecot/dovecot-ceph-plugin])
 AC_CONFIG_AUX_DIR([.])
 AC_CONFIG_SRCDIR([src])
 AC_CONFIG_MACRO_DIR([m4])

diff --git a/rpm/dovecot-ceph-plugin.spec b/rpm/dovecot-ceph-plugin.spec
@@ -13,7 +13,7 @@
 
 Name:		dovecot-ceph-plugin
 Summary:	Dovecot Ceph RADOS plugins
-Version:	0.0.20
+Version:	0.0.21
 Release:	0%{?dist}
 URL:		https://github.com/ceph-dovecot/dovecot-ceph-plugin
 Group:		Productivity/Networking/Email/Servers

diff --git a/src/librmb/rados-cluster-impl.cpp b/src/librmb/rados-cluster-impl.cpp
@@ -60,6 +60,7 @@ int RadosClusterImpl::init(const std::string &clustername, const std::string &ra
   int ret = 0;
   if (RadosClusterImpl::cluster_ref_count == 0) {
     RadosClusterImpl::cluster = new librados::Rados();
+
     ret = RadosClusterImpl::cluster->init2(rados_username.c_str(), clustername.c_str(), 0);
     if (ret == 0) {
       ret = initialize();
@@ -94,6 +95,9 @@ int RadosClusterImpl::initialize() {
     RadosClusterImpl::cluster->conf_set(RADOS_OSD_OP_TIMEOUT, RADOS_OSD_OP_TIMEOUT_DEFAULT);
   }
 
+  for (std::map<const char *, const char *>::iterator it = client_options.begin(); it != client_options.end(); ++it) {
+    RadosClusterImpl::cluster->conf_set(it->first, it->second);
+  }
   return ret;
 }
 
@@ -115,6 +119,7 @@ void RadosClusterImpl::deinit() {
         RadosClusterImpl::cluster->shutdown();
         RadosClusterImpl::connected = false;
         delete RadosClusterImpl::cluster;
+        RadosClusterImpl::cluster = nullptr;
       }
     }
   }
@@ -175,3 +180,5 @@ int RadosClusterImpl::io_ctx_create(const string &pool, librados::IoCtx *io_ctx)
 int RadosClusterImpl::get_config_option(const char *option, string *value) {
   return RadosClusterImpl::cluster->conf_get(option, *value);
 }
+
+void RadosClusterImpl::set_config_option(const char *option, const char *value) { client_options[option] = value; }
diff --git a/src/librmb/rados-cluster-impl.h b/src/librmb/rados-cluster-impl.h
@@ -16,7 +16,7 @@
 #include <string>
 
 #include <rados/librados.hpp>
-
+#include <map>
 #include "rados-cluster.h"
 namespace librmb {
 
@@ -38,6 +38,7 @@ class RadosClusterImpl : public RadosCluster {
                         RadosDictionary **dictionary);
   bool is_connected() override;
   librados::Rados &get_cluster() { return *cluster; }
+  void set_config_option(const char *option, const char *value);
 
  private:
   int initialize();
@@ -46,6 +47,7 @@ class RadosClusterImpl : public RadosCluster {
   static librados::Rados *cluster;
   static int cluster_ref_count;
   static bool connected;
+  std::map<const char *, const char *> client_options;
 
   static const char *CLIENT_MOUNT_TIMEOUT;
   static const char *RADOS_MON_OP_TIMEOUT;

diff --git a/src/librmb/rados-cluster.h b/src/librmb/rados-cluster.h
@@ -61,6 +61,16 @@ class RadosCluster {
    *
    */
   virtual int get_config_option(const char *option, std::string *value) = 0;
+
+  /*!
+     * set ceph configuration
+     * @param[in] option option name as described in the ceph documentation
+     * @param[out] value valid ptr to a string buffer.
+     * @return linux error code or 0 if successful
+     *
+     */
+  virtual void set_config_option(const char *option, const char *value) = 0;
+
   /*!
    * Check if cluster connection does exist and is working-
    * @return true if connected

diff --git a/src/librmb/rados-mail.cpp b/src/librmb/rados-mail.cpp
@@ -15,7 +15,6 @@
 
 #include <cstring>
 #include <sstream>
-#include <vector>
 #include "rados-util.h"
 
 using std::endl;
@@ -28,10 +27,10 @@ RadosMail::RadosMail()
       completion(nullptr),
       write_operation(nullptr),
       active_op(0),
+      mail_buffer(nullptr),
       save_date_rados(-1),
       valid(true),
-      index_ref(false),
-      mail_buffer(nullptr) {}
+      index_ref(false) {}
 
 RadosMail::~RadosMail() {}
 

diff --git a/src/librmb/rados-storage-impl.cpp b/src/librmb/rados-storage-impl.cpp
@@ -16,7 +16,6 @@
 #include <set>
 #include <string>
 #include <utility>
-#include <vector>
 
 #include <rados/librados.hpp>
 #include "encoding.h"
@@ -235,13 +234,13 @@ bool RadosStorageImpl::wait_for_write_operations_complete(librados::AioCompletio
   return failed;
 }
 
-bool RadosStorageImpl::wait_for_rados_operations(const std::vector<librmb::RadosMail *> &object_list) {
+bool RadosStorageImpl::wait_for_rados_operations(const std::list<librmb::RadosMail *> &object_list) {
   bool ctx_failed = false;
   // wait for all writes to finish!
   // imaptest shows it's possible that begin -> continue -> finish cycle is invoked several times before
   // rbox_transaction_save_commit_pre is called.
-  for (std::vector<librmb::RadosMail *>::const_iterator it_cur_obj = object_list.begin();
-       it_cur_obj != object_list.end(); ++it_cur_obj) {
+  for (std::list<librmb::RadosMail *>::const_iterator it_cur_obj = object_list.begin(); it_cur_obj != object_list.end();
+       ++it_cur_obj) {
     // if we come from copy mail, there is no operation to wait for.
     if ((*it_cur_obj)->has_active_op()) {
       bool op_failed =
@@ -380,7 +379,7 @@ bool RadosStorageImpl::save_mail(librados::ObjectWriteOperation *write_op_xattr,
     delete write_op_xattr;
     mail->set_active_op(0);
   } else if (!save_async) {
-    std::vector<librmb::RadosMail *> objects;
+    std::list<librmb::RadosMail *> objects;
     objects.push_back(mail);
     return wait_for_rados_operations(objects);
   }

diff --git a/src/librmb/rados-storage-impl.h b/src/librmb/rados-storage-impl.h
@@ -17,7 +17,6 @@
 #include <map>
 #include <string>
 #include <cstdint>
-#include <vector>
 #include <list>
 #include <algorithm>
 #include <rados/librados.hpp>
@@ -57,7 +56,7 @@ class RadosStorageImpl : public RadosStorage {
   bool wait_for_write_operations_complete(librados::AioCompletion *completion,
                                           librados::ObjectWriteOperation *write_operation) override;
 
-  bool wait_for_rados_operations(const std::vector<librmb::RadosMail *> &object_list) override;
+  bool wait_for_rados_operations(const std::list<librmb::RadosMail *> &object_list) override;
 
   int read_mail(const std::string &oid, librados::bufferlist *buffer) override;
   int move(std::string &src_oid, const char *src_ns, std::string &dest_oid, const char *dest_ns,

diff --git a/src/librmb/rados-storage.h b/src/librmb/rados-storage.h
@@ -14,7 +14,6 @@
 
 #include <string>
 #include <map>
-#include <vector>
 #include <list>
 
 #include <rados/librados.hpp>
@@ -133,7 +132,7 @@ class RadosStorage {
    *
    * @return true if successful
    */
-  virtual bool wait_for_rados_operations(const std::vector<librmb::RadosMail *> &object_list) = 0;
+  virtual bool wait_for_rados_operations(const std::list<librmb::RadosMail *> &object_list) = 0;
 
   /*! save the mail object
    *

diff --git a/src/librmb/rados-util.cpp b/src/librmb/rados-util.cpp
@@ -16,7 +16,7 @@
 #include "rados-util.h"
 #include <limits.h>
 #include <string>
-#include <vector>
+#include <list>
 #include <iostream>
 #include <sstream>
 #include <set>
@@ -289,7 +289,7 @@ int RadosUtils::copy_to_alt(std::string &src_oid, std::string &dest_oid, RadosSt
     return 0;
   }
 
-  std::vector<librmb::RadosMail *> objects;
+  std::list<librmb::RadosMail *> objects;
   objects.push_back(&mail);
   if (inverse) {
     success = primary->wait_for_rados_operations(objects);

diff --git a/src/librmb/tools/rmb/ls_cmd_parser.h b/src/librmb/tools/rmb/ls_cmd_parser.h
@@ -13,7 +13,7 @@
 #define SRC_LIBRMB_TOOLS_RMB_LS_CMD_PARSER_H_
 
 #include <string>
-#include <vector>
+#include <list>
 #include <iostream>
 #include <ctime>
 #include <map>
@@ -46,7 +46,8 @@ class Predicate {
       try {
         val2 = std::stol(_p_value);
       } catch (std::exception &e) {
-        std::cerr << "eval: search criteria: RBOX_METADATA_RECEIVED_TIME or RBOX_METADATA_OLDV1_SAVE_TIME '" << _p_value << "' is not a number " << std::endl;
+        std::cerr << "eval: search criteria: RBOX_METADATA_RECEIVED_TIME or RBOX_METADATA_OLDV1_SAVE_TIME '" << _p_value
+                  << "' is not a number " << std::endl;
       }
       time_t obj_date = static_cast<time_t>(val2);
 
@@ -69,7 +70,9 @@ class Predicate {
         val = std::stol(_p_value);
         val2 = std::stol(this->value);
       } catch (std::exception &e) {
-        std::cerr << "eval: search criteria: RBOX_METADATA_VIRTUAL_SIZE or RBOX_METADATA_PHYSICAL_SIZE or RBOX_METADATA_MAIL_UID: _p_value " << _p_value << " or " << this->value << " is not a number" << std::endl;
+        std::cerr << "eval: search criteria: RBOX_METADATA_VIRTUAL_SIZE or RBOX_METADATA_PHYSICAL_SIZE or "
+                     "RBOX_METADATA_MAIL_UID: _p_value "
+                  << _p_value << " or " << this->value << " is not a number" << std::endl;
       }
 
       if (this->op.compare("=") == 0) {
@@ -114,7 +117,7 @@ class CmdLineParser {
   Predicate *get_predicate(const std::string &key) { return predicates[key]; }
   Predicate *create_predicate(const std::string &ls_value);
 
-  void set_output_dir(const std::string& out);
+  void set_output_dir(const std::string &out);
   std::string &get_output_dir() { return this->out_dir; }
 
  private:

diff --git a/src/librmb/tools/rmb/rados-mail-box.h b/src/librmb/tools/rmb/rados-mail-box.h
@@ -14,7 +14,7 @@
 
 #include <string>
 #include <sstream>
-#include <vector>
+#include <list>
 #include <map>
 
 #include "../../rados-mail.h"
@@ -74,17 +74,17 @@ class RadosMailBox {
        << "         mailbox_size=" << mailbox_size << " bytes " << std::endl;
 
     std::string padding("         ");
-    for (std::vector<RadosMail *>::iterator it = mails.begin(); it != mails.end(); ++it) {
+    for (std::list<RadosMail *>::iterator it = mails.begin(); it != mails.end(); ++it) {
       ss << (*it)->to_string(padding);
     }
     return ss.str();
   }
   inline void add_to_mailbox_size(const uint64_t &_mailbox_size) { this->mailbox_size += _mailbox_size; }
-  void set_mails(const std::vector<RadosMail *> &_mails) { this->mails = _mails; }
+  void set_mails(const std::list<RadosMail *> &_mails) { this->mails = _mails; }
 
   CmdLineParser *get_xattr_filter() { return this->parser; }
   void set_xattr_filter(CmdLineParser *_parser) { this->parser = _parser; }
-  std::vector<RadosMail *> &get_mails() { return this->mails; }
+  std::list<RadosMail *> &get_mails() { return this->mails; }
 
   std::string &get_mailbox_guid() { return this->mailbox_guid; }
   void set_mailbox_guid(const std::string &_mailbox_guid) { this->mailbox_guid = _mailbox_guid; }
@@ -97,7 +97,7 @@ class RadosMailBox {
   std::string mailbox_guid;
   int mail_count;
   uint64_t mailbox_size;
-  std::vector<RadosMail *> mails;
+  std::list<RadosMail *> mails;
   uint64_t total_mails;
   std::string mbox_orig_name;
 };