RepoMirrors/go-ceph
1
0
Fork 0
mirror of https://github.com/ceph/go-ceph synced 2025-02-27 16:10:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

rgw/admin: add quota support

Browse Source 
This commit introduces support for user and bucket quota on the rgw
admin ops API.

Co-authored-by: Irek Fasikhov malmyzh@gmail.com
Co-authored-by: Quentin Perez qperez42@gmail.com
Signed-off-by: Sébastien Han <seb@redhat.com>
This commit is contained in:
Sébastien Han 2021-06-04 13:22:52 +02:00 committed by John Mulligan
parent 7b585cc9b4
commit a2c50e28a5
8 changed files with 245 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
rgw/admin/bucket.go
View File

@ -44,15 +44,9 @@ type Bucket struct {
NumObjects *uint64 `json:"num_objects"`
} `json:"rgw.multimeta"`
} `json:"usage"`
BucketQuota struct {
Enabled *bool `json:"enabled"`
CheckOnRaw *bool `json:"check_on_raw"`
MaxSize *uint64 `json:"max_size"`
MaxSizeKb *uint64 `json:"max_size_kb"`
MaxObjects *uint64 `json:"max_objects"`
} `json:"bucket_quota"`
Policy *bool `url:"policy"`
PurgeObject *bool `url:"purge-objects"`
BucketQuota QuotaSpec `json:"bucket_quota"`
Policy *bool `url:"policy"`
PurgeObject *bool `url:"purge-objects"`
}
// Policy describes a bucket policy

24
rgw/admin/bucket_test.go
View File

@ -14,10 +14,16 @@ func (suite *RadosGWTestSuite) TestBucket() {
co.Debug = true
assert.NoError(suite.T(), err)
s3, err := newS3Agent(suite.accessKey, suite.secretKey, suite.endpoint, true)
assert.NoError(suite.T(), err)
err = s3.createBucket(suite.bucketTestName)
assert.NoError(suite.T(), err)
suite.T().Run("list buckets", func(t *testing.T) {
buckets, err := co.ListBuckets(context.Background())
assert.NoError(suite.T(), err)
assert.Equal(suite.T(), 0, len(buckets))
assert.Equal(suite.T(), 1, len(buckets))
})
suite.T().Run("info non-existing bucket", func(t *testing.T) {
@ -26,6 +32,22 @@ func (suite *RadosGWTestSuite) TestBucket() {
assert.True(suite.T(), errors.Is(err, ErrNoSuchBucket), err)
})
suite.T().Run("info existing bucket", func(t *testing.T) {
_, err := co.GetBucketInfo(context.Background(), Bucket{Bucket: suite.bucketTestName})
assert.NoError(suite.T(), err)
})
suite.T().Run("remove bucket", func(t *testing.T) {
err := co.RemoveBucket(context.Background(), Bucket{Bucket: suite.bucketTestName})
assert.NoError(suite.T(), err)
})
suite.T().Run("list bucket is now zero", func(t *testing.T) {
buckets, err := co.ListBuckets(context.Background())
assert.NoError(suite.T(), err)
assert.Equal(suite.T(), 0, len(buckets))
})
suite.T().Run("remove non-existing bucket", func(t *testing.T) {
err := co.RemoveBucket(context.Background(), Bucket{Bucket: "foo"})
assert.Error(suite.T(), err)

61
rgw/admin/quota.go Normal file
View File

@ -0,0 +1,61 @@
package admin
import (
"context"
"encoding/json"
"fmt"
)
// QuotaSpec describes an object store quota for a user or a bucket
// Only user's quota are supported
type QuotaSpec struct {
UID string `json:"user_id" url:"uid"`
QuotaType string `url:"quota-type"`
Enabled *bool `json:"enabled" url:"enabled"`
CheckOnRaw bool `json:"check_on_raw"`
MaxSize *int64 `json:"max_size" url:"max-size"`
MaxSizeKb *int `json:"max_size_kb" url:"max-size-kb"`
MaxObjects *int64 `json:"max_objects" url:"max-objects"`
}
// GetUserQuota will return the quota for a user
func (api *API) GetUserQuota(ctx context.Context, quota QuotaSpec) (QuotaSpec, error) {
// Always for quota type to user
quota.QuotaType = "user"
if quota.UID == "" {
return QuotaSpec{}, errMissingUserID
}
body, err := api.call(ctx, get, "/user?quota", valueToURLParams(quota))
if err != nil {
return QuotaSpec{}, err
}
ref := QuotaSpec{}
err = json.Unmarshal(body, &ref)
if err != nil {
return QuotaSpec{}, fmt.Errorf("%s. %s. %w", unmarshalError, string(body), err)
}
return ref, nil
}
// SetUserQuota sets quota to a user
// Global quotas (https://docs.ceph.com/en/latest/radosgw/admin/#reading-writing-global-quotas) are not surfaced in the Admin Ops API
// So this library cannot expose it yet
func (api *API) SetUserQuota(ctx context.Context, quota QuotaSpec) error {
// Always for quota type to user
quota.QuotaType = "user"
if quota.UID == "" {
return errMissingUserID
}
_, err := api.call(ctx, put, "/user?quota", valueToURLParams(quota))
if err != nil {
return err
}
return nil
}

28
rgw/admin/quota_test.go Normal file
View File

@ -0,0 +1,28 @@
package admin
import (
"context"
"testing"
"github.com/stretchr/testify/assert"
)
func (suite *RadosGWTestSuite) TestQuota() {
suite.SetupConnection()
co, err := New(suite.endpoint, suite.accessKey, suite.secretKey, nil)
co.Debug = true
assert.NoError(suite.T(), err)
suite.T().Run("set quota to user but user ID is empty", func(t *testing.T) {
err := co.SetUserQuota(context.Background(), QuotaSpec{})
assert.Error(suite.T(), err)
assert.EqualError(suite.T(), err, errMissingUserID.Error())
})
suite.T().Run("get user quota but no user is specified", func(t *testing.T) {
_, err := co.GetUserQuota(context.Background(), QuotaSpec{})
assert.Error(suite.T(), err)
assert.EqualError(suite.T(), err, errMissingUserID.Error())
})
}

70
rgw/admin/radosgw_test.go
View File

@ -1,24 +1,87 @@
package admin
import (
"fmt"
"net/http"
"os"
"reflect"
"testing"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/stretchr/testify/suite"
)
type RadosGWTestSuite struct {
suite.Suite
endpoint string
accessKey string
secretKey string
endpoint string
accessKey string
secretKey string
bucketTestName string
}
func TestRadosGWTestSuite(t *testing.T) {
suite.Run(t, new(RadosGWTestSuite))
}
// S3Agent wraps the s3.S3 structure to allow for wrapper methods
type S3Agent struct {
Client *s3.S3
}
func newS3Agent(accessKey, secretKey, endpoint string, debug bool) (*S3Agent, error) {
const cephRegion = "us-east-1"
logLevel := aws.LogOff
if debug {
logLevel = aws.LogDebug
}
client := http.Client{
Timeout: time.Second * 15,
}
sess, err := session.NewSession(
aws.NewConfig().
WithRegion(cephRegion).
WithCredentials(credentials.NewStaticCredentials(accessKey, secretKey, "")).
WithEndpoint(endpoint).
WithS3ForcePathStyle(true).
WithMaxRetries(5).
WithDisableSSL(true).
WithHTTPClient(&client).
WithLogLevel(logLevel),
)
if err != nil {
return nil, err
}
svc := s3.New(sess)
return &S3Agent{
Client: svc,
}, nil
}
func (s *S3Agent) createBucket(name string) error {
bucketInput := &s3.CreateBucketInput{
Bucket: &name,
}
_, err := s.Client.CreateBucket(bucketInput)
if err != nil {
if aerr, ok := err.(awserr.Error); ok {
switch aerr.Code() {
case s3.ErrCodeBucketAlreadyExists:
return nil
case s3.ErrCodeBucketAlreadyOwnedByYou:
return nil
}
}
return fmt.Errorf("failed to create bucket %q. %w", name, err)
}
return nil
}
func (suite *RadosGWTestSuite) SetupConnection() {
suite.accessKey = "2262XNX11FZRR44XWIRD"
suite.secretKey = "rmtuS1Uj1bIC08QFYGW18GfSHAbkPqdsuYynNudw"
@ -28,6 +91,7 @@ func (suite *RadosGWTestSuite) SetupConnection() {
endpoint = "test_ceph_a"
}
suite.endpoint = "http://" + endpoint
suite.bucketTestName = "test"
}
func TestNew(t *testing.T) {

68
rgw/admin/user.go
View File

@ -8,47 +8,41 @@ import (
// User is GO representation of the json output of a user creation
type User struct {
ID string `json:"user_id" url:"uid"`
DisplayName string `json:"display_name" url:"display-name"`
Email string `json:"email" url:"email"`
Suspended *int `json:"suspended" url:"suspended"`
MaxBuckets *int `json:"max_buckets" url:"max-buckets"`
Subusers []interface{} `json:"subusers"`
Keys []struct {
User string `json:"user"`
AccessKey string `json:"access_key" url:"access-key"`
SecretKey string `json:"secret_key" url:"secret-key"`
} `json:"keys"`
SwiftKeys []interface{} `json:"swift_keys"`
Caps []struct {
Type string `json:"type"`
Perm string `json:"perm"`
} `json:"caps"`
ID string `json:"user_id" url:"uid"`
DisplayName string `json:"display_name" url:"display-name"`
Email string `json:"email" url:"email"`
Suspended *int `json:"suspended" url:"suspended"`
MaxBuckets *int `json:"max_buckets" url:"max-buckets"`
Subusers []interface{} `json:"subusers"`
Keys []UserKeySpec `json:"keys"`
SwiftKeys []interface{} `json:"swift_keys"`
Caps []UserCapSpec `json:"caps"`
OpMask string `json:"op_mask"`
DefaultPlacement string `json:"default_placement"`
DefaultStorageClass string `json:"default_storage_class"`
PlacementTags []interface{} `json:"placement_tags"`
BucketQuota struct {
Enabled *bool `json:"enabled"`
CheckOnRaw *bool `json:"check_on_raw"`
MaxSize *int `json:"max_size"`
MaxSizeKb *int `json:"max_size_kb"`
MaxObjects *int `json:"max_objects"`
} `json:"bucket_quota"`
UserQuota struct {
Enabled *bool `json:"enabled"`
CheckOnRaw *bool `json:"check_on_raw"`
MaxSize *int `json:"max_size"`
MaxSizeKb *int `json:"max_size_kb"`
MaxObjects *int `json:"max_objects"`
} `json:"user_quota"`
TempURLKeys []interface{} `json:"temp_url_keys"`
Type string `json:"type"`
MfaIds []interface{} `json:"mfa_ids"`
KeyType string `url:"key-type"`
Tenant string `url:"tenant"`
GenerateKey *bool `url:"generate-key"`
PurgeData *int `url:"purge-data"`
BucketQuota QuotaSpec `json:"bucket_quota"`
UserQuota QuotaSpec `json:"user_quota"`
TempURLKeys []interface{} `json:"temp_url_keys"`
Type string `json:"type"`
MfaIds []interface{} `json:"mfa_ids"`
KeyType string `url:"key-type"`
Tenant string `url:"tenant"`
GenerateKey *bool `url:"generate-key"`
PurgeData *int `url:"purge-data"`
}
// UserCapSpec represents a user capability which gives access to certain ressources
type UserCapSpec struct {
Type string `json:"type"`
Perm string `json:"perm"`
}
// UserKeySpec is the user credential configuration
type UserKeySpec struct {
User string `json:"user"`
AccessKey string `json:"access_key" url:"access-key"`
SecretKey string `json:"secret_key" url:"secret-key"`
}
// GetUser retrieves a given object store user

21
rgw/admin/user_test.go
View File

@ -97,6 +97,14 @@ func (suite *RadosGWTestSuite) TestUser() {
assert.Equal(suite.T(), "leseb@leseb.com", user.Email)
})
suite.T().Run("modify user max bucket", func(t *testing.T) {
maxBuckets := -1
user, err := co.ModifyUser(context.Background(), User{ID: "leseb", MaxBuckets: &maxBuckets})
assert.NoError(suite.T(), err)
assert.Equal(suite.T(), "leseb@leseb.com", user.Email)
assert.Equal(suite.T(), -1, *user.MaxBuckets)
})
suite.T().Run("user already exists", func(t *testing.T) {
_, err := co.CreateUser(context.Background(), User{ID: "admin", DisplayName: "Admin user"})
assert.Error(suite.T(), err)
@ -109,6 +117,19 @@ func (suite *RadosGWTestSuite) TestUser() {
assert.Equal(suite.T(), 2, len(*users))
})
suite.T().Run("set user quota", func(t *testing.T) {
quotaEnable := true
maxObjects := int64(100)
err := co.SetUserQuota(context.Background(), QuotaSpec{QuotaType: "user", UID: "leseb", MaxObjects: &maxObjects, Enabled: &quotaEnable})
assert.NoError(suite.T(), err)
})
suite.T().Run("get user quota", func(t *testing.T) {
q, err := co.GetUserQuota(context.Background(), QuotaSpec{QuotaType: "user", UID: "leseb"})
assert.NoError(suite.T(), err)
assert.Equal(suite.T(), int64(100), *q.MaxObjects)
})
suite.T().Run("remove user", func(t *testing.T) {
err = co.RemoveUser(context.Background(), User{ID: "leseb"})
assert.NoError(suite.T(), err)

11
rgw/admin/utils.go
View File

@ -12,6 +12,17 @@ const (
)
func buildQueryPath(endpoint, path, args string) string {
// Sometimes the API requires single URL key with no values
// For instance, the Quota code uses the admin API path to "/user?quota"
// This is done this way since url.Values does not support adding keys without values.
//
// So Quota code passes the begining of the query (indicated with a marker "?") in its path already, so we need to escape it
// and add a separator key instead
// So we can get something like "/admin/user?quota&" instead of passing two beginning query markers ("?")
if strings.Contains(path, "?") {
return fmt.Sprintf("%s%s%s&%s", endpoint, queryAdminPath, path, args)
}
return fmt.Sprintf("%s%s%s?%s", endpoint, queryAdminPath, path, args)
}