RepoMirrors/ceph
1
0
Fork 0
mirror of https://github.com/ceph/ceph synced 2025-02-19 08:57:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #15807 from zhangsw/feature-lifecycle-date

Browse Source 
rgw: S3 lifecycle now supports expiration date

Reviewed-by: Daniel Gryniewicz <dang@fprintf.net>
This commit is contained in:
Yuri Weinstein 2017-07-17 07:59:39 -07:00 committed by GitHub
commit eb26360b45
4 changed files with 125 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

77
src/rgw/rgw_lc.cc
View File

@ -27,7 +27,7 @@ const char* LC_STATUS[] = {
using namespace std;
using namespace librados;
bool LCRule::validate()
bool LCRule::valid()
{
if (id.length() > MAX_ID_LEN) {
return false;
@ -35,13 +35,7 @@ bool LCRule::validate()
else if(expiration.empty() && noncur_expiration.empty() && mp_expiration.empty() && !dm_expiration) {
return false;
}
else if (!expiration.empty() && expiration.get_days() <= 0) {
return false;
}
else if (!noncur_expiration.empty() && noncur_expiration.get_days() <=0) {
return false;
}
else if (!mp_expiration.empty() && mp_expiration.get_days() <= 0) {
else if (!expiration.valid() || !noncur_expiration.valid() || !mp_expiration.valid()) {
return false;
}
return true;
@ -60,13 +54,16 @@ bool RGWLifecycleConfiguration::_add_rule(LCRule *rule)
if (rule->get_status().compare("Enabled") == 0) {
op.status = true;
}
if (!rule->get_expiration().empty()) {
if (rule->get_expiration().has_days()) {
op.expiration = rule->get_expiration().get_days();
}
if (!rule->get_noncur_expiration().empty()) {
if (rule->get_expiration().has_date()) {
op.expiration_date = ceph::from_iso_8601(rule->get_expiration().get_date());
}
if (rule->get_noncur_expiration().has_days()) {
op.noncur_expiration = rule->get_noncur_expiration().get_days();
}
if (!rule->get_mp_expiration().empty()) {
if (rule->get_mp_expiration().has_days()) {
op.mp_expiration = rule->get_mp_expiration().get_days();
}
op.dm_expiration = rule->get_dm_expiration();
@ -76,7 +73,7 @@ bool RGWLifecycleConfiguration::_add_rule(LCRule *rule)
int RGWLifecycleConfiguration::check_and_add_rule(LCRule *rule)
{
if (!rule->validate()) {
if (!rule->valid()) {
return -EINVAL;
}
string id;
@ -92,9 +89,22 @@ int RGWLifecycleConfiguration::check_and_add_rule(LCRule *rule)
return 0;
}
bool RGWLifecycleConfiguration::has_same_action(const lc_op& first, const lc_op& second) {
if ((first.expiration > 0 || first.expiration_date != boost::none) &&
(second.expiration > 0 || second.expiration_date != boost::none)) {
return true;
} else if (first.noncur_expiration > 0 && second.noncur_expiration > 0) {
return true;
} else if (first.mp_expiration > 0 && second.mp_expiration > 0) {
return true;
} else {
return false;
}
}
//Rules are conflicted: if one rule's prefix starts with other rule's prefix, and these two rules
//define same action.
bool RGWLifecycleConfiguration::validate()
bool RGWLifecycleConfiguration::valid()
{
if (prefix_map.size() < 2) {
return true;
@ -107,9 +117,7 @@ bool RGWLifecycleConfiguration::validate()
string c_pre = cur_iter->first;
string n_pre = next_iter->first;
if (n_pre.compare(0, c_pre.length(), c_pre) == 0) {
if ((cur_iter->second.expiration > 0 && next_iter->second.expiration > 0) ||
(cur_iter->second.noncur_expiration > 0 && next_iter->second.noncur_expiration > 0) ||
(cur_iter->second.mp_expiration > 0 && next_iter->second.mp_expiration > 0)) {
if (has_same_action(cur_iter->second, next_iter->second)) {
return false;
} else {
++next_iter;
@ -346,7 +354,12 @@ int RGWLC::bucket_lc_process(string& shard_id)
list_op.params.list_versions = bucket_info.versioned();
if (!bucket_info.versioned()) {
for(auto prefix_iter = prefix_map.begin(); prefix_iter != prefix_map.end(); ++prefix_iter) {
if (!prefix_iter->second.status || prefix_iter->second.expiration <=0) {
if (!prefix_iter->second.status ||
(prefix_iter->second.expiration <=0 && prefix_iter->second.expiration_date == boost::none)) {
continue;
}
if (prefix_iter->second.expiration_date != boost::none &&
ceph_clock_now() < ceph::real_clock::to_time_t(*prefix_iter->second.expiration_date)) {
continue;
}
list_op.params.prefix = prefix_iter->first;
@ -361,17 +374,22 @@ int RGWLC::bucket_lc_process(string& shard_id)
ldout(cct, 0) << "ERROR: store->list_objects():" <<dendl;
return ret;
}
utime_t now = ceph_clock_now();
bool is_expired;
for (auto obj_iter = objs.begin(); obj_iter != objs.end(); ++obj_iter) {
rgw_obj_key key(obj_iter->key);
if (!key.ns.empty()) {
continue;
}
if (obj_has_expired(now - ceph::real_clock::to_time_t(obj_iter->meta.mtime), prefix_iter->second.expiration)) {
if (prefix_iter->second.expiration_date != boost::none) {
//we have checked it before
is_expired = true;
} else {
is_expired = obj_has_expired(now - ceph::real_clock::to_time_t(obj_iter->meta.mtime), prefix_iter->second.expiration);
}
if (is_expired) {
RGWObjectCtx rctx(store);
rgw_obj obj(bucket_info.bucket, key);
RGWObjState *state;
@ -396,6 +414,7 @@ int RGWLC::bucket_lc_process(string& shard_id)
rgw_obj_key pre_marker;
for(auto prefix_iter = prefix_map.begin(); prefix_iter != prefix_map.end(); ++prefix_iter) {
if (!prefix_iter->second.status || (prefix_iter->second.expiration <= 0
&& prefix_iter->second.expiration_date == boost::none
&& prefix_iter->second.noncur_expiration <= 0 && !prefix_iter->second.dm_expiration)) {
continue;
}
@ -427,10 +446,13 @@ int RGWLC::bucket_lc_process(string& shard_id)
bool remove_indeed = true;
int expiration;
bool skip_expiration;
bool is_expired;
for (auto obj_iter = objs.begin(); obj_iter != objs.end(); ++obj_iter) {
skip_expiration = false;
is_expired = false;
if (obj_iter->is_current()) {
if (prefix_iter->second.expiration <= 0 && !prefix_iter->second.dm_expiration) {
if (prefix_iter->second.expiration <= 0 && prefix_iter->second.expiration_date == boost::none
&& !prefix_iter->second.dm_expiration) {
continue;
}
if (obj_iter->is_delete_marker()) {
@ -450,8 +472,14 @@ int RGWLC::bucket_lc_process(string& shard_id)
}
mtime = obj_iter->meta.mtime;
expiration = prefix_iter->second.expiration;
if (!skip_expiration && expiration <= 0) {
if (!skip_expiration && expiration <= 0 && prefix_iter->second.expiration_date == boost::none) {
continue;
} else if (!skip_expiration) {
if (expiration > 0) {
is_expired = obj_has_expired(now - ceph::real_clock::to_time_t(mtime), expiration);
} else {
is_expired = now >= ceph::real_clock::to_time_t(*prefix_iter->second.expiration_date);
}
}
} else {
if (prefix_iter->second.noncur_expiration <=0) {
@ -460,8 +488,9 @@ int RGWLC::bucket_lc_process(string& shard_id)
remove_indeed = true;
mtime = (obj_iter == objs.begin())?pre_obj.meta.mtime:(obj_iter - 1)->meta.mtime;
expiration = prefix_iter->second.noncur_expiration;
is_expired = obj_has_expired(now - ceph::real_clock::to_time_t(mtime), expiration);
}
if (skip_expiration || obj_has_expired(now - ceph::real_clock::to_time_t(mtime), expiration)) {
if (skip_expiration || is_expired) {
if (obj_iter->is_visible()) {
RGWObjectCtx rctx(store);
rgw_obj obj(bucket_info.bucket, obj_iter->key);

44
src/rgw/rgw_lc.h
View File

@ -12,6 +12,7 @@
#include "include/rados/librados.hpp"
#include "common/Mutex.h"
#include "common/Cond.h"
#include "common/iso_8601.h"
#include "common/Thread.h"
#include "rgw_common.h"
#include "rgw_rados.h"
@ -38,29 +39,54 @@ class LCExpiration
{
protected:
string days;
//At present only current object has expiration date
string date;
public:
LCExpiration() {}
~LCExpiration() {}
void encode(bufferlist& bl) const {
ENCODE_START(2, 2, bl);
ENCODE_START(3, 2, bl);
::encode(days, bl);
::encode(date, bl);
ENCODE_FINISH(bl);
}
void decode(bufferlist::iterator& bl) {
DECODE_START_LEGACY_COMPAT_LEN(2, 2, 2, bl);
DECODE_START_LEGACY_COMPAT_LEN(3, 2, 2, bl);
::decode(days, bl);
if (struct_v >= 3) {
::decode(date, bl);
}
DECODE_FINISH(bl);
}
void dump(Formatter *f) const;
// static void generate_test_instances(list<ACLOwner*>& o);
void set_days(const string& _days) { days = _days; }
string get_days_str() const{
string get_days_str() const {
return days;
}
int get_days() {return atoi(days.c_str()); }
bool empty() const{
return days.empty();
int get_days() const {return atoi(days.c_str()); }
bool has_days() const {
return !days.empty();
}
void set_date(const string& _date) { date = _date; }
string get_date() const {
return date;
}
bool has_date() const {
return !date.empty();
}
bool empty() const {
return days.empty() && date.empty();
}
bool valid() const {
if (!days.empty() && !date.empty()) {
return false;
} else if (!days.empty() && get_days() <= 0) {
return false;
}
//We've checked date in xml parsing
return true;
}
};
WRITE_CLASS_ENCODER(LCExpiration)
@ -138,7 +164,7 @@ public:
dm_expiration = _dm_expiration;
}
bool validate();
bool valid();
void encode(bufferlist& bl) const {
ENCODE_START(4, 1, bl);
@ -179,6 +205,7 @@ struct lc_op
int expiration;
int noncur_expiration;
int mp_expiration;
boost::optional<ceph::real_time> expiration_date;
lc_op() : status(false), dm_expiration(false), expiration(0), noncur_expiration(0), mp_expiration(0) {}
@ -191,6 +218,7 @@ protected:
map<string, lc_op> prefix_map;
multimap<string, LCRule> rule_map;
bool _add_rule(LCRule *rule);
bool has_same_action(const lc_op& first, const lc_op& second);
public:
RGWLifecycleConfiguration(CephContext *_cct) : cct(_cct) {}
RGWLifecycleConfiguration() : cct(NULL) {}
@ -225,7 +253,7 @@ public:
int check_and_add_rule(LCRule* rule);
bool validate();
bool valid();
multimap<string, LCRule>& get_rule_map() { return rule_map; }
map<string, lc_op>& get_prefix_map() { return prefix_map; }

20
src/rgw/rgw_lc_s3.cc
View File

@ -16,8 +16,10 @@ using namespace std;
bool LCExpiration_S3::xml_end(const char * el) {
LCDays_S3 *lc_days = static_cast<LCDays_S3 *>(find_first("Days"));
LCDeleteMarker_S3 *lc_dm = static_cast<LCDeleteMarker_S3 *>(find_first("ExpiredObjectDeleteMarker"));
LCDate_S3 *lc_date = static_cast<LCDate_S3 *>(find_first("Date"));
if ((!lc_days && !lc_dm) || (lc_days && lc_dm)) {
if ((!lc_days && !lc_dm && !lc_date) || (lc_days && lc_dm)
|| (lc_days && lc_date) || (lc_dm && lc_date)) {
return false;
}
if (lc_days) {
@ -27,6 +29,12 @@ bool LCExpiration_S3::xml_end(const char * el) {
if (!dm_expiration) {
return false;
}
} else {
date = lc_date->get_data();
//We need return xml error according to S3
if (boost::none == ceph::from_iso_8601(date)) {
return false;
}
}
return true;
}
@ -96,8 +104,10 @@ bool LCRule_S3::xml_end(const char *el) {
return false;
} else {
if (lc_expiration) {
if (!lc_expiration->empty()) {
if (lc_expiration->has_days()) {
expiration.set_days(lc_expiration->get_days_str());
} else if (lc_expiration->has_date()) {
expiration.set_date(lc_expiration->get_date());
} else {
dm_expiration = lc_expiration->get_dm_expiration();
}
@ -119,7 +129,7 @@ void LCRule_S3::to_xml(CephContext *cct, ostream& out) {
out << "<Prefix>" << prefix << "</Prefix>";
out << "<Status>" << status << "</Status>";
if (!expiration.empty() || dm_expiration) {
LCExpiration_S3 expir(expiration.get_days_str(), dm_expiration);
LCExpiration_S3 expir(expiration.get_days_str(), expiration.get_date(), dm_expiration);
expir.to_xml(out);
}
if (!noncur_expiration.empty()) {
@ -143,7 +153,7 @@ int RGWLifecycleConfiguration_S3::rebuild(RGWRados *store, RGWLifecycleConfigura
if (ret < 0)
return ret;
}
if (!dest.validate()) {
if (!dest.valid()) {
ret = -ERR_INVALID_REQUEST;
}
return ret;
@ -178,6 +188,8 @@ XMLObj *RGWLCXMLParser_S3::alloc_obj(const char *el)
obj = new LCExpiration_S3();
} else if (strcmp(el, "Days") == 0) {
obj = new LCDays_S3();
} else if (strcmp(el, "Date") == 0) {
obj = new LCDate_S3();
} else if (strcmp(el, "ExpiredObjectDeleteMarker") == 0) {
obj = new LCDeleteMarker_S3();
} else if (strcmp(el, "NoncurrentVersionExpiration") == 0) {

25
src/rgw/rgw_lc_s3.h
View File

@ -42,6 +42,14 @@ public:
string& to_str() { return data; }
};
class LCDate_S3 : public XMLObj
{
public:
LCDate_S3() {}
~LCDate_S3() override {}
string& to_str() { return data; }
};
class LCDeleteMarker_S3 : public XMLObj
{
public:
@ -56,26 +64,33 @@ private:
bool dm_expiration;
public:
LCExpiration_S3(): dm_expiration(false) {}
LCExpiration_S3(string _days, bool _dm_expiration) {
LCExpiration_S3(string _days, string _date, bool _dm_expiration) {
days = _days;
date = _date;
dm_expiration = _dm_expiration;
}
~LCExpiration_S3() override {}
bool xml_end(const char *el) override;
void to_xml(ostream& out) {
out << "<Expiration>";
if (dm_expiration) {
out << "<Expiration>" << "<ExpiredObjectDeleteMarker>" << "true" << "</ExpiredObjectDeleteMarker>" << "</Expiration>";
out << "<ExpiredObjectDeleteMarker>" << "true" << "</ExpiredObjectDeleteMarker>";
} else if (!days.empty()){
out << "<Days>" << days << "</Days>";
} else {
out << "<Expiration>" << "<Days>" << days << "</Days>"<< "</Expiration>";
out << "<Date>" << date << "</Date>";
}
out << "</Expiration>";
}
void dump_xml(Formatter *f) const {
f->open_object_section("Expiration");
if (dm_expiration) {
encode_xml("ExpiredObjectDeleteMarker", "true", f);
} else {
} else if (!days.empty()) {
encode_xml("Days", days, f);
} else {
encode_xml("Date", date, f);
}
f->close_section(); // Expiration
}
@ -138,7 +153,7 @@ public:
encode_xml("Prefix", prefix, f);
encode_xml("Status", status, f);
if (!expiration.empty() || dm_expiration) {
LCExpiration_S3 expir(expiration.get_days_str(), dm_expiration);
LCExpiration_S3 expir(expiration.get_days_str(), expiration.get_date(), dm_expiration);
expir.dump_xml(f);
}
if (!noncur_expiration.empty()) {