From 373a3043d765d2e9039f820cb1670ad760d2aa01 Mon Sep 17 00:00:00 2001 From: Livio Soares Date: Thu, 24 Mar 2016 23:18:07 -0400 Subject: [PATCH 1/2] rados: Implement Iter() interface to allow incremental listing of Pools. The current IOContext struct has a ListObjects() method that works well for pools with a reasonable number of objects. However, for very large pools, having finer control over the iteration may be desirable. This commit adds a new Iter() interface for pools. It allows callers to stop iterating at any time by issuing a iter.Close(). It also allows callers to "Seek" into the pool list using a token object. --- rados/ioctx.go | 74 +++++++++++++++++++++++++++++++++++++++++++++ rados/rados_test.go | 49 +++++++++++++++++++++++++++++- 2 files changed, 122 insertions(+), 1 deletion(-) diff --git a/rados/ioctx.go b/rados/ioctx.go index 27b687d..82f8d79 100644 --- a/rados/ioctx.go +++ b/rados/ioctx.go @@ -536,3 +536,77 @@ func (ioctx *IOContext) CleanOmap(oid string) error { return GetRadosError(ret) } + +type Iter struct { + ctx C.rados_list_ctx_t + err error + entry string +} + +type IterToken uint32 + +// Return a Iterator object that can be used to list the object names in the current pool +func (ioctx *IOContext) Iter() (*Iter, error) { + iter := Iter{} + if cerr := C.rados_objects_list_open(ioctx.ioctx, &iter.ctx); cerr < 0 { + return nil, GetRadosError(cerr) + } + return &iter, nil +} + +// Returns a token marking the current position of the iterator. To be used in combination with Iter.Seek() +func (iter *Iter) Token() IterToken { + return IterToken(C.rados_objects_list_get_pg_hash_position(iter.ctx)) +} + +func (iter *Iter) Seek(token IterToken) { + C.rados_objects_list_seek(iter.ctx, C.uint32_t(token)) +} + +// Next retrieves the next object name in the pool/namespace iterator. +// Upon a successful invocation (return value of true), the Value method should +// be used to obtain the name of the retrieved object name. When the iterator is +// exhausted, Next returns false. The Err method should used to verify whether the +// end of the iterator was reached, or the iterator received an error. +// +// Example: +// iter := pool.Iter() +// defer iter.Close() +// for iter.Next() { +// fmt.Printf("%v\n", iter.Value()) +// } +// return iter.Err() +// +func (iter *Iter) Next() bool { + centry := (*C.char)(C.calloc(1, 1024)) + defer C.free(unsafe.Pointer(centry)) + + if cerr := C.rados_objects_list_next(iter.ctx, ¢ry, nil); cerr < 0 { + iter.err = GetRadosError(cerr) + return false + } + iter.entry = C.GoString(centry) + return true +} + +// Returns the current value of the iterator (object name), after a successful call to Next. +func (iter *Iter) Value() string { + if iter.err != nil { + return "" + } + return iter.entry +} + +// Checks whether the iterator has encountered an error. +func (iter *Iter) Err() error { + if iter.err == RadosErrorNotFound { + return nil + } + return iter.err +} + +// Closes the iterator cursor on the server. Be aware that iterators are not closed automatically +// at the end of iteration. +func (iter *Iter) Close() { + C.rados_objects_list_close(iter.ctx) +} diff --git a/rados/rados_test.go b/rados/rados_test.go index b2b7c70..804ccfe 100644 --- a/rados/rados_test.go +++ b/rados/rados_test.go @@ -488,7 +488,7 @@ func TestMonCommand(t *testing.T) { conn.Shutdown() } -func TestObjectIterator(t *testing.T) { +func TestObjectListObjects(t *testing.T) { conn, _ := rados.NewConn() conn.ReadDefaultConfigFile() conn.Connect() @@ -529,6 +529,53 @@ func TestObjectIterator(t *testing.T) { assert.Equal(t, objectList, createdList) } +func TestObjectIterator(t *testing.T) { + conn, _ := rados.NewConn() + conn.ReadDefaultConfigFile() + conn.Connect() + + poolname := GetUUID() + err := conn.MakePool(poolname) + assert.NoError(t, err) + + ioctx, err := conn.OpenIOContext(poolname) + assert.NoError(t, err) + + objectList := []string{} + iter, err := ioctx.Iter() + assert.NoError(t, err) + for iter.Next() { + objectList = append(objectList, iter.Value()) + } + iter.Close() + assert.NoError(t, iter.Err()) + assert.True(t, len(objectList) == 0) + + createdList := []string{} + for i := 0; i < 200; i++ { + oid := GetUUID() + bytes_in := []byte("input data") + err = ioctx.Write(oid, bytes_in, 0) + assert.NoError(t, err) + createdList = append(createdList, oid) + } + assert.True(t, len(createdList) == 200) + + iter, err = ioctx.Iter() + assert.NoError(t, err) + for iter.Next() { + objectList = append(objectList, iter.Value()) + } + iter.Close() + assert.NoError(t, iter.Err()) + assert.Equal(t, len(objectList), len(createdList)) + + sort.Strings(objectList) + sort.Strings(createdList) + + assert.Equal(t, objectList, createdList) +} + func TestNewConnWithUser(t *testing.T) { _, err := rados.NewConnWithUser("admin") assert.Equal(t, err, nil) From 2d33c55c6bfa8903c3318e75cd03af51088c179a Mon Sep 17 00:00:00 2001 From: Livio Soares Date: Fri, 25 Mar 2016 00:08:28 -0400 Subject: [PATCH 2/2] rados: Fix iterator memory allocation; allocating a c_entry is not necessary and performed directly by the library. --- rados/ioctx.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/rados/ioctx.go b/rados/ioctx.go index 82f8d79..585f1a7 100644 --- a/rados/ioctx.go +++ b/rados/ioctx.go @@ -578,14 +578,12 @@ func (iter *Iter) Seek(token IterToken) { // return iter.Err() // func (iter *Iter) Next() bool { - centry := (*C.char)(C.calloc(1, 1024)) - defer C.free(unsafe.Pointer(centry)) - - if cerr := C.rados_objects_list_next(iter.ctx, ¢ry, nil); cerr < 0 { + var c_entry *C.char + if cerr := C.rados_objects_list_next(iter.ctx, &c_entry, nil); cerr < 0 { iter.err = GetRadosError(cerr) return false } - iter.entry = C.GoString(centry) + iter.entry = C.GoString(c_entry) return true }