Merge pull request ceph#5102 from ceph/wip-mds-refactor-small

Minor MDS:: refactor Reviewed-by: Greg Farnum <gfarnum@redhat.com>
weiqian · Jul 13, 2015 · d5dbf02 · d5dbf02
2 parents 9d2efcc + 840011b
commit d5dbf02
Show file tree

Hide file tree

Showing 10 changed files with 87 additions and 29 deletions.
diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
@@ -168,6 +168,9 @@ class MDCacheIOContext : public virtual MDSIOContextBase {
 
 MDCache::MDCache(MDS *m) :
   logger(0),
+  filer(m->objecter, &m->finisher),
+  rejoin_done(NULL),
+  resolve_done(NULL),
   recovery_queue(m),
   stray_manager(m)
 {
@@ -222,7 +225,9 @@ MDCache::~MDCache()
     delete logger;
     logger = 0;
   }
-  //delete renamer;
+
+  delete rejoin_done; rejoin_done = NULL;
+  delete resolve_done; resolve_done = NULL;
 }
 
 
@@ -2644,9 +2649,11 @@ ESubtreeMap *MDCache::create_subtree_map()
 }
 
 
-void MDCache::resolve_start()
+void MDCache::resolve_start(MDSInternalContext *resolve_done_)
 {
   dout(10) << "resolve_start" << dendl;
+  assert(resolve_done == NULL);
+  resolve_done = resolve_done_;
 
   if (mds->mdsmap->get_root() != mds->whoami) {
     // if we don't have the root dir, adjust it to UNKNOWN.  during
@@ -3266,7 +3273,9 @@ void MDCache::maybe_resolve_finish()
   if (mds->is_resolve()) {
     trim_unlinked_inodes();
     recalc_auth_bits(false);
-    mds->resolve_done();
+    assert(resolve_done != NULL);
+    resolve_done->complete(0);
+    resolve_done = NULL;
   } else {
     maybe_send_pending_rejoins();
   }
@@ -3836,9 +3845,11 @@ void MDCache::recalc_auth_bits(bool replay)
  *   after recovery.
  */
 
-void MDCache::rejoin_start()
+void MDCache::rejoin_start(MDSInternalContext *rejoin_done_)
 {
   dout(10) << "rejoin_start" << dendl;
+  assert(rejoin_done == NULL);
+  rejoin_done = rejoin_done_;
 
   rejoin_gather = recovery_set;
   // need finish opening cap inodes before sending cache rejoins
@@ -5656,7 +5667,9 @@ void MDCache::open_snap_parents()
     do_delayed_cap_imports();
 
     start_files_to_recover(rejoin_recover_q, rejoin_check_q);
-    mds->rejoin_done();
+    assert(rejoin_done != NULL);
+    rejoin_done->complete(0);
+    rejoin_done = NULL;
   }
 }
 
@@ -6086,10 +6099,10 @@ void MDCache::_truncate_inode(CInode *in, LogSegment *ls)
     assert(in->last == CEPH_NOSNAP);
   }
   dout(10) << "_truncate_inode  snapc " << snapc << " on " << *in << dendl;
-  mds->filer->truncate(in->inode.ino, &in->inode.layout, *snapc,
-		       pi->truncate_size, pi->truncate_from-pi->truncate_size,
-		       pi->truncate_seq, utime_t(), 0,
-		       0, new C_OnFinisher(new C_IO_MDC_TruncateFinish(this, in,
+  filer.truncate(in->inode.ino, &in->inode.layout, *snapc,
+		 pi->truncate_size, pi->truncate_from-pi->truncate_size,
+		 pi->truncate_seq, utime_t(), 0,
+		 0, new C_OnFinisher(new C_IO_MDC_TruncateFinish(this, in,
 								       ls),
 					   &mds->finisher));
 }

diff --git a/src/mds/MDCache.h b/src/mds/MDCache.h
@@ -21,6 +21,7 @@
 #include "include/filepath.h"
 #include "include/elist.h"
 
+#include "osdc/Filer.h"
 #include "CInode.h"
 #include "CDentry.h"
 #include "CDir.h"
@@ -139,6 +140,8 @@ class MDCache {
 
   PerfCounters *logger;
 
+  Filer filer;
+
 public:
   void advance_stray() {
     stray_index = (stray_index+1)%NUM_STRAY;
@@ -458,7 +461,7 @@ class MDCache {
   }
   void cancel_ambiguous_import(CDir *);
   void finish_ambiguous_import(dirfrag_t dirino);
-  void resolve_start();
+  void resolve_start(MDSInternalContext *resolve_done_);
   void send_resolves();
   void send_slave_resolves();
   void send_subtree_resolves();
@@ -517,8 +520,10 @@ class MDCache {
     if (rejoins_pending)
       rejoin_send_rejoins();
   }
+  MDSInternalContext *rejoin_done;
+  MDSInternalContext *resolve_done;
 public:
-  void rejoin_start();
+  void rejoin_start(MDSInternalContext *rejoin_done_);
   void rejoin_gather_finish();
   void rejoin_send_rejoins();
   void rejoin_export_caps(inodeno_t ino, client_t client, ceph_mds_cap_reconnect& capinfo,

diff --git a/src/mds/MDS.cc b/src/mds/MDS.cc
@@ -132,8 +132,6 @@ MDS::MDS(const std::string &n, Messenger *m, MonClient *mc) :
   objecter = new Objecter(m->cct, messenger, monc, NULL, 0, 0);
   objecter->unset_honor_osdmap_full();
 
-  filer = new Filer(objecter, &finisher);
-
   mdcache = new MDCache(this);
   mdlog = new MDLog(this);
   balancer = new MDBalancer(this);
@@ -182,7 +180,6 @@ MDS::~MDS() {
   if (server) { delete server; server = 0; }
   if (locker) { delete locker; locker = 0; }
 
-  if (filer) { delete filer; filer = 0; }
   if (objecter) { delete objecter; objecter = 0; }
 
   if (logger) {
@@ -2421,6 +2418,29 @@ void MDS::replay_done()
   }
 }
 
+/**
+ * Helper for simple callbacks that call a void fn with no args.
+ */
+class C_VoidFn : public MDSInternalContext
+{
+  typedef void (MDS::*fn_ptr)();
+  protected:
+   MDS *mds_daemon;
+   fn_ptr fn; 
+  public:
+  C_VoidFn(MDS *mds_, fn_ptr fn_)
+    : MDSInternalContext(mds_), mds_daemon(mds_), fn(fn_)
+  {
+    assert(mds_);
+    assert(fn_);
+  }
+
+  void finish(int r)
+  {
+    (mds_daemon->*fn)();
+  }
+};
+
 void MDS::reopen_log()
 {
   dout(1) << "reopen_log" << dendl;
@@ -2434,7 +2454,7 @@ void MDS::resolve_start()
 
   reopen_log();
 
-  mdcache->resolve_start();
+  mdcache->resolve_start(new C_VoidFn(this, &MDS::resolve_done));
   finish_contexts(g_ceph_context, waiting_for_resolve);
 }
 void MDS::resolve_done()
@@ -2450,7 +2470,7 @@ void MDS::reconnect_start()
   if (last_state == MDSMap::STATE_REPLAY)
     reopen_log();
 
-  server->reconnect_clients();
+  server->reconnect_clients(new C_VoidFn(this, &MDS::reconnect_done));
   finish_contexts(g_ceph_context, waiting_for_reconnect);
 }
 void MDS::reconnect_done()
@@ -2467,7 +2487,7 @@ void MDS::rejoin_joint_start()
 void MDS::rejoin_start()
 {
   dout(1) << "rejoin_start" << dendl;
-  mdcache->rejoin_start();
+  mdcache->rejoin_start(new C_VoidFn(this, &MDS::rejoin_done));
 }
 void MDS::rejoin_done()
 {

diff --git a/src/mds/MDS.h b/src/mds/MDS.h
@@ -174,7 +174,6 @@ class MDS : public Dispatcher, public md_config_obs_t {
   MonClient    *monc;
   MDSMap       *mdsmap;
   Objecter     *objecter;
-  Filer        *filer;       // for reading/writing to/from osds
   LogClient    log_client;
   LogChannelRef clog;
 
@@ -445,6 +444,7 @@ class MDS : public Dispatcher, public md_config_obs_t {
 
   void reopen_log();
 
+ protected:
   void resolve_start();
   void resolve_done();
   void reconnect_start();
@@ -458,6 +458,7 @@ class MDS : public Dispatcher, public md_config_obs_t {
   void active_start();
   void stopping_start();
   void stopping_done();
+ public:
 
   void handle_mds_recovery(mds_rank_t who);
   void handle_mds_failure(mds_rank_t who);

diff --git a/src/mds/RecoveryQueue.cc b/src/mds/RecoveryQueue.cc
@@ -47,6 +47,11 @@ class C_MDC_Recover : public MDSIOContextBase {
 };
 
 
+RecoveryQueue::RecoveryQueue(MDS *mds_)
+  : mds(mds_), logger(NULL), filer(mds_->objecter, &mds_->finisher)
+{}
+
+
 /**
  * Progress the queue.  Call this after enqueuing something or on
  * completion of something.
@@ -93,7 +98,7 @@ void RecoveryQueue::_start(CInode *in)
     file_recovering.insert(in);
 
     C_MDC_Recover *fin = new C_MDC_Recover(this, in);
-    mds->filer->probe(in->inode.ino, &in->inode.layout, in->last,
+    filer.probe(in->inode.ino, &in->inode.layout, in->last,
 		      pi->get_max_size(), &fin->size, &fin->mtime, false,
 		      0, fin);
   } else {

diff --git a/src/mds/RecoveryQueue.h b/src/mds/RecoveryQueue.h
@@ -19,6 +19,8 @@
 
 #include <set>
 
+#include "osdc/Filer.h"
+
 class CInode;
 class MDS;
 class PerfCounters;
@@ -28,7 +30,7 @@ class RecoveryQueue {
   void enqueue(CInode *in);
   void advance();
   void prioritize(CInode *in);   ///< do this inode now/soon
-  RecoveryQueue(MDS *mds_) : mds(mds_), logger(NULL) {}
+  RecoveryQueue(MDS *mds_);
 
   void set_logger(PerfCounters *p) {logger=p;}
 
@@ -41,6 +43,7 @@ class RecoveryQueue {
   void _recovered(CInode *in, int r, uint64_t size, utime_t mtime);
   MDS *mds;
   PerfCounters *logger;
+  Filer filer;
 
   friend class C_MDC_Recover;
 };

diff --git a/src/mds/Server.cc b/src/mds/Server.cc
@@ -674,8 +674,9 @@ void Server::journal_close_session(Session *session, int state, Context *on_safe
   finish_flush_session(session, session->get_push_seq());
 }
 
-void Server::reconnect_clients()
+void Server::reconnect_clients(MDSInternalContext *reconnect_done_)
 {
+  reconnect_done = reconnect_done_;
   mds->sessionmap.get_client_set(client_reconnect_gather);
 
   if (client_reconnect_gather.empty()) {
@@ -810,7 +811,9 @@ void Server::handle_client_reconnect(MClientReconnect *m)
 void Server::reconnect_gather_finish()
 {
   dout(7) << "reconnect_gather_finish.  failed on " << failed_reconnects << " clients" << dendl;
-  mds->reconnect_done();
+  assert(reconnect_done);
+  reconnect_done->complete(0);
+  reconnect_done = NULL;
 }
 
 void Server::reconnect_tick()

diff --git a/src/mds/Server.h b/src/mds/Server.h
@@ -53,9 +53,11 @@ class Server {
   // OSDMap full status, used to generate ENOSPC on some operations
   bool is_full;
 
-public:
+  // State for while in reconnect
+  MDSInternalContext *reconnect_done;
   int failed_reconnects;
 
+public:
   bool terminating_sessions;
 
   Server(MDS *m) : 
@@ -64,12 +66,14 @@ class Server {
     messenger(mds->messenger),
     logger(0),
     is_full(false),
+    reconnect_done(NULL),
     failed_reconnects(0),
     terminating_sessions(false) {
   }
   ~Server() {
     g_ceph_context->get_perfcounters_collection()->remove(logger);
     delete logger;
+    delete reconnect_done;
   }
 
   void create_logger();
@@ -99,7 +103,7 @@ class Server {
   void find_idle_sessions();
   void kill_session(Session *session, Context *on_safe);
   void journal_close_session(Session *session, int state, Context *on_safe);
-  void reconnect_clients();
+  void reconnect_clients(MDSInternalContext *reconnect_done_);
   void handle_client_reconnect(class MClientReconnect *m);
   //void process_reconnect_cap(CInode *in, int from, ceph_mds_cap_reconnect& capinfo);
   void reconnect_gather_finish();

diff --git a/src/mds/StrayManager.cc b/src/mds/StrayManager.cc
@@ -139,7 +139,7 @@ void StrayManager::purge(CDentry *dn, uint32_t op_allowance)
       uint64_t num = (to + period - 1) / period;
       dout(10) << __func__ << " 0~" << to << " objects 0~" << num
 	       << " snapc " << snapc << " on " << *in << dendl;
-      mds->filer->purge_range(in->inode.ino, &in->inode.layout, *snapc,
+      filer.purge_range(in->inode.ino, &in->inode.layout, *snapc,
 			      0, num, ceph_clock_now(g_ceph_context), 0,
 			      gather.new_sub());
     }
@@ -747,11 +747,12 @@ void StrayManager::migrate_stray(CDentry *dn, mds_rank_t to)
   mds->send_message_mds(req, to);
 }
 
-  StrayManager::StrayManager(MDS *mds)
+StrayManager::StrayManager(MDS *mds)
   : delayed_eval_stray(member_offset(CDentry, item_stray)),
     mds(mds), logger(NULL),
     ops_in_flight(0), files_purging(0),
-    num_strays(0), num_strays_purging(0), num_strays_delayed(0)
+    num_strays(0), num_strays_purging(0), num_strays_delayed(0),
+    filer(mds->objecter, &mds->finisher)
 {
   assert(mds != NULL);
   update_op_limit();
@@ -812,14 +813,14 @@ void StrayManager::truncate(CDentry *dn, uint32_t op_allowance)
     uint64_t num = (to - 1) / period;
     dout(10) << __func__ << " 0~" << to << " objects 0~" << num
       << " snapc " << snapc << " on " << *in << dendl;
-    mds->filer->purge_range(in->ino(), &in->inode.layout, *snapc,
+    filer.purge_range(in->ino(), &in->inode.layout, *snapc,
 			    1, num, ceph_clock_now(g_ceph_context),
 			    0, gather.new_sub());
   }
 
   // keep backtrace object
   if (period && to > 0) {
-    mds->filer->zero(in->ino(), &in->inode.layout, *snapc,
+    filer.zero(in->ino(), &in->inode.layout, *snapc,
 		     0, period, ceph_clock_now(g_ceph_context),
 		     0, true, NULL, gather.new_sub());
   }

diff --git a/src/mds/StrayManager.h b/src/mds/StrayManager.h
@@ -16,6 +16,7 @@
 
 #include "include/elist.h"
 #include <list>
+#include "osdc/Filer.h"
 
 class MDS;
 class PerfCounters;
@@ -56,6 +57,8 @@ class StrayManager : public md_config_obs_t
   uint64_t num_strays_purging;
   uint64_t num_strays_delayed;
 
+  Filer filer;
+
   void truncate(CDentry *dn, uint32_t op_allowance);
 
   /**