kclient: trim expired rdcaps; ignored issued but timed out caps

Introduce new CEPH_CAP_EXPIREABLE mask to control which caps (if
alone) time out.

src/TODO

@@ -46,13 +46,11 @@ timer
 
 
 caps
-- issue excl leases to loner if open for write...
-
-
 - kclient
   - only pin caps with dirty metadata? and/or wanted != 0?
   - and/or, put unwanted caps on an lru list, and expire?
 - mds
+  - trim expired rdcaps
   - segregate wanted/unwanted caps?  rd/wr caps?
 
 

src/config.cc

@@ -343,7 +343,7 @@ md_config_t g_conf = {
   mds_client_prealloc_inos: 20,
   mds_early_reply: true,
 
-  mds_rdcap_ttl_ms: 30 * 10000,
+  mds_rdcap_ttl_ms: 60*1000,
 
   mds_log: true,
   mds_log_max_events: -1,

src/include/ceph_fs.h

@@ -952,6 +952,12 @@ static inline int ceph_flags_to_mode(int flags)
 #define CEPH_CAP_ANY_FILE_WR ((CEPH_CAP_GWR|CEPH_CAP_GWRBUFFER) << CEPH_CAP_SFILE)
 #define CEPH_CAP_ANY_WR   (CEPH_CAP_ANY_EXCL | CEPH_CAP_ANY_FILE_WR)
 
+/*
+ * these cap bits time out, if no others are held and nothing is
+ * registered as 'wanted' by the client.
+ */
+#define CEPH_CAP_EXPIREABLE (CEPH_CAP_PIN|CEPH_CAP_ANY_RDCACHE)
+
 
 static inline int ceph_caps_for_mode(int mode)
 {

src/kernel/caps.c

@@ -226,7 +226,7 @@ retry:
 		session->s_nr_caps++;
 		INIT_LIST_HEAD(&cap->session_rdcaps);
 	}
-	if ((cap->issued & ~(CEPH_CAP_ANY_RDCACHE|CEPH_CAP_PIN)) == 0) {
+	if ((cap->issued & ~CEPH_CAP_EXPIREABLE) == 0) {
 		/* move to tail of session rdcaps lru */
 		if (!list_empty(&cap->session_rdcaps))
 		    list_del(&cap->session_rdcaps);
@@ -284,6 +284,15 @@ int __ceph_caps_issued(struct ceph_inode_info *ci, int *implemented)
 			     cap, ceph_cap_string(cap->issued), cap->gen, gen);
 			continue;
 		}
+
+		if (time_after_eq(jiffies, cap->expires) &&
+		    (cap->issued & ~CEPH_CAP_EXPIREABLE) == 0) {
+			dout(30, "__ceph_caps_issued %p cap %p issued %s "
+			     "but readonly and expired\n", &ci->vfs_inode,
+			     cap, ceph_cap_string(cap->issued));
+			continue;
+		}
+
 		dout(30, "__ceph_caps_issued %p cap %p issued %s\n",
 		     &ci->vfs_inode, cap, ceph_cap_string(cap->issued));
 		have |= cap->issued;
@@ -1665,3 +1674,43 @@ void ceph_check_delayed_caps(struct ceph_mds_client *mdsc)
 	spin_unlock(&mdsc->cap_delay_lock);
 }
 
+
+/*
+ * caller must hold session s_mutex
+ */
+void ceph_trim_session_rdcaps(struct ceph_mds_session *session)
+{
+	struct inode *inode;
+	struct ceph_cap *cap;
+	struct list_head *p, *n;
+	int wanted;
+
+	dout(10, "trim_rdcaps for mds%d\n", session->s_mds);
+	list_for_each_safe(p, n, &session->s_rdcaps) {
+		cap = list_entry(p, struct ceph_cap, session_rdcaps);
+
+		inode = &cap->ci->vfs_inode;
+		igrab(inode);
+		spin_lock(&inode->i_lock);
+
+		if (time_before(jiffies, cap->expires)) {
+			dout(20, " stopping at %p cap %p expires %lu > %lu\n",
+			     inode, cap, cap->expires, jiffies);
+			spin_unlock(&inode->i_lock);
+			iput(inode);
+			break;
+		}
+
+		/* wanted? */
+		wanted = __ceph_caps_wanted(cap->ci);
+		if (wanted == 0) {
+			dout(20, " dropping %p cap %p\n", inode, cap);
+			__ceph_remove_cap(cap);
+		} else {
+			dout(20, " keeping %p cap %p (wanted %s)\n", inode, cap,
+			     ceph_cap_string(wanted));
+		}
+		spin_unlock(&inode->i_lock);
+		iput(inode);
+	}
+}

src/kernel/mds_client.c

@@ -1930,6 +1930,7 @@ static void delayed_work(struct work_struct *work)
 		if (renew_caps)
 			send_renew_caps(mdsc, s);
 		trim_session_leases(s);
+		ceph_trim_session_rdcaps(s);
 		mutex_unlock(&s->s_mutex);
 		ceph_put_mds_session(s);
 

src/kernel/super.h

@@ -669,7 +669,9 @@ extern void __ceph_flush_snaps(struct ceph_inode_info *ci,
 			       struct ceph_mds_session **psession);
 extern void ceph_check_caps(struct ceph_inode_info *ci, int delayed, int drop);
 extern void ceph_check_delayed_caps(struct ceph_mds_client *mdsc);
-extern inline void ceph_release_caps(struct inode *inode, int mask)
+void ceph_trim_session_rdcaps(struct ceph_mds_session *session);
+
+static inline void ceph_release_caps(struct inode *inode, int mask)
 {
 	ceph_check_caps(ceph_inode(inode), 1, mask);
 }