rgw: more quota fixes

Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>

src/rgw/rgw_op.cc

@@ -442,28 +442,32 @@ int RGWOp::init_quota()
     return 0;
   }
 
-  if (s->bucket_info.quota.enabled) {
-    bucket_quota = s->bucket_info.quota;
-    return 0;
-  }
+  RGWUserInfo owner_info;
+  RGWUserInfo *uinfo;
+
   if (s->user.user_id == s->bucket_owner.get_id()) {
-    if (s->user.bucket_quota.enabled) {
-      bucket_quota = s->user.bucket_quota;
-      return 0;
-    }
+    uinfo = &s->user;
   } else {
-    RGWUserInfo owner_info;
     int r = rgw_get_user_info_by_uid(store, s->bucket_info.owner, owner_info);
     if (r < 0)
       return r;
-
-    if (owner_info.bucket_quota.enabled) {
-      bucket_quota = owner_info.bucket_quota;
-      return 0;
-    }
+    uinfo = &owner_info;
+  }
+
+  if (s->bucket_info.quota.enabled) {
+    bucket_quota = s->bucket_info.quota;
+  } else if (uinfo->bucket_quota.enabled) {
+    bucket_quota = uinfo->bucket_quota;
+  } else {
+    bucket_quota = store->region_map.bucket_quota;
+  }
+
+  if (uinfo->user_quota.enabled) {
+    user_quota = uinfo->user_quota;
+  } else {
+    user_quota = store->region_map.user_quota;
   }
 
-  bucket_quota = store->region_map.bucket_quota;
   return 0;
 }
 
@@ -1416,7 +1420,7 @@ void RGWPutObj::execute()
   if (!chunked_upload) { /* with chunked upload we don't know how big is the upload.
                             we also check sizes at the end anyway */
     ret = store->check_quota(s->bucket_owner.get_id(), s->bucket,
-                             s->user.user_quota, bucket_quota, s->content_length);
+                             user_quota, bucket_quota, s->content_length);
     if (ret < 0) {
       goto done;
     }
@@ -1467,7 +1471,7 @@ void RGWPutObj::execute()
   perfcounter->inc(l_rgw_put_b, s->obj_size);
 
   ret = store->check_quota(s->bucket_owner.get_id(), s->bucket,
-                           s->user.user_quota, bucket_quota, s->obj_size);
+                           user_quota, bucket_quota, s->obj_size);
   if (ret < 0) {
     goto done;
   }

src/rgw/rgw_op.h

@@ -38,6 +38,7 @@ protected:
   RGWCORSConfiguration bucket_cors;
   bool cors_exist;
   RGWQuotaInfo bucket_quota;
+  RGWQuotaInfo user_quota;
 
   virtual int init_quota();
 public:

src/rgw/rgw_quota.cc

@@ -59,6 +59,8 @@ protected:
 
   virtual bool map_find_and_update(const string& user, rgw_bucket& bucket, typename lru_map<T, RGWQuotaCacheStats>::UpdateContext *ctx) = 0;
   virtual void map_add(const string& user, rgw_bucket& bucket, RGWQuotaCacheStats& qs) = 0;
+
+  virtual int handle_set_stats(const string& user, rgw_bucket& bucket, RGWStorageStats& stats) { return 0; }
 public:
   RGWQuotaCache(RGWRados *_store, int size) : store(_store), stats_map(size) {
     async_refcount = new RefCountedWaitObject;
@@ -170,6 +172,11 @@ void RGWQuotaCache<T>::set_stats(const string& user, rgw_bucket& bucket, RGWQuot
   qs.async_refresh_time += store->ctx()->_conf->rgw_bucket_quota_ttl / 2;
 
   map_add(user, bucket, qs);
+
+  int ret = handle_set_stats(user, bucket, stats);
+  if (ret < 0) {
+    /* can't really do much about it, user stats are going to be off a bit for now */
+  }
 }
 
 template<class T>
@@ -295,6 +302,14 @@ protected:
   }
 
   int fetch_stats_from_storage(const string& user, rgw_bucket& bucket, RGWStorageStats& stats);
+  int handle_set_stats(const string& user, rgw_bucket& bucket, RGWStorageStats& stats) {
+    int ret = store->update_user_bucket_stats(user, bucket, stats);
+    if (ret < 0) {
+      derr << "ERROR: store->update_bucket_stats() returned " << ret << dendl;
+      return ret;
+    }
+    return 0;
+  }
 
 public:
   RGWBucketStatsCache(RGWRados *_store) : RGWQuotaCache(_store, _store->ctx()->_conf->rgw_bucket_quota_cache_size) {
@@ -451,6 +466,17 @@ public:
       ret = check_quota("bucket", bucket_quota, bucket_stats, num_objs, size_kb);
       if (ret < 0)
         return ret;
+    } else if (user_quota.enabled) {
+      /*
+       * we need to fetch bucket stats if the user quota is enabled, because the whole system relies
+       * on us periodically updating the user's bucket stats in the user's header, this happens in
+       * get_stats() if we actually fetch that info and not rely on cached data
+       */
+      RGWStorageStats bucket_stats;
+
+      int ret = bucket_stats_cache.get_stats(user, bucket, bucket_stats, bucket_quota);
+      if (ret < 0)
+        return ret;
     }
 
     if (user_quota.enabled) {

src/rgw/rgw_rados.cc

@@ -4720,8 +4720,8 @@ public:
     if (r >= 0) {
       RGWStorageStats stats;
 
-      stats.num_kb = header.total_bytes;
-      stats.num_kb_rounded = header.total_bytes_rounded;
+      stats.num_kb = (header.total_bytes + 1023) / 1024;
+      stats.num_kb_rounded = (header.total_bytes_rounded + 1023) / 1024;
       stats.num_objects = header.total_entries;
 
       cb->set_response(stats);
@@ -4740,8 +4740,8 @@ int RGWRados::get_user_stats(const string& user, RGWStorageStats& stats)
   if (r < 0)
     return r;
 
-  stats.num_kb = header.total_bytes;
-  stats.num_kb_rounded = header.total_bytes_rounded;
+  stats.num_kb = (header.total_bytes + 1023) / 1024;
+  stats.num_kb_rounded = (header.total_bytes_rounded + 1023) / 1024;
   stats.num_objects = header.total_entries;
 
   return 0;
@@ -5735,6 +5735,30 @@ int RGWRados::cls_user_sync_bucket_stats(rgw_obj& user_obj, rgw_bucket& bucket)
   return 0;
 }
 
+int RGWRados::update_user_bucket_stats(const string& user_id, rgw_bucket& bucket, RGWStorageStats& stats)
+{
+  cls_user_bucket_entry entry;
+
+  entry.size = stats.num_kb * 1024;
+  entry.size_rounded = stats.num_kb_rounded * 1024;
+  entry.count += stats.num_objects;
+
+  list<cls_user_bucket_entry> entries;
+  entries.push_back(entry);
+
+  string buckets_obj_id;
+  rgw_get_buckets_obj(user_id, buckets_obj_id);
+  rgw_obj obj(zone.user_uid_pool, buckets_obj_id);
+
+  int r = cls_user_update_buckets(obj, entries);
+  if (r < 0) {
+    ldout(cct, 20) << "cls_user_update_buckets() returned " << r << dendl;
+    return r;
+  }
+
+  return 0;
+}
+
 int RGWRados::cls_user_list_buckets(rgw_obj& obj,
                                     const string& in_marker, int max_entries,
                                     list<cls_user_bucket_entry>& entries,

src/rgw/rgw_rados.h

@@ -644,6 +644,7 @@ struct RGWRegionMap {
   string master_region;
 
   RGWQuotaInfo bucket_quota;
+  RGWQuotaInfo user_quota;
 
   RGWRegionMap() : lock("RGWRegionMap") {}
 
@@ -1438,6 +1439,7 @@ public:
   int cls_user_get_header(const string& user_id, cls_user_header *header);
   int cls_user_get_header_async(const string& user_id, RGWGetUserHeader_CB *ctx);
   int cls_user_sync_bucket_stats(rgw_obj& user_obj, rgw_bucket& bucket);
+  int update_user_bucket_stats(const string& user_id, rgw_bucket& bucket, RGWStorageStats& stats);
   int cls_user_list_buckets(rgw_obj& obj,
                             const string& in_marker, int max_entries,
                             list<cls_user_bucket_entry>& entries,