cephfs: add unmount, release, chmod, chown, etc

This commit adds the following cephfs functions:

* Unmount // Unmounting is necessary to cleanup mounts
* Release // Release destroys the cmount ~ end of transaction
* RemoveDir // inverse of MakeDir
* Chown // change ownership of file or directory
* Chmod // change permissions of file or directory

Tests are included for each function.

In addition to these changes modifications to:

.travis.yml, Dockerfile, and Makefile

were made to accomodate tests to mount the ceph volume.  Tests use
fuse to mount the volume which requires adding:

--device /dev/fuse --cap-add SYS_ADMIN --security-opt \
apparmor:unconfined

to the docker container (alternatively --privileged works but adds
additional permissions).

Changes to README add the above docker changes as well as point
users to the necessary ceph development libraries.
This commit is contained in:
Lincoln Thurlow 2018-10-10 15:06:47 -07:00
parent f3df337fab
commit ecf9a99249
7 changed files with 171 additions and 92 deletions

View File

@ -21,5 +21,6 @@ before_install: |
exit 1
fi
# cephfs (fuse) requires: --device /dev/fuse --cap-add SYS_ADMIN --security-opt apparmor:unconfined
script:
- docker run --rm -it -v ${PWD}:/go/src/github.com/ceph/go-ceph:z ceph-golang-ci
- docker run --device /dev/fuse --cap-add SYS_ADMIN --security-opt apparmor:unconfined --rm -it -v ${PWD}:/go/src/github.com/ceph/go-ceph:z ceph-golang-ci

View File

@ -20,6 +20,10 @@ RUN apt-get update && apt-get install -y \
librbd-dev \
golang-1.10-go
# add user account to test permissions
RUN groupadd -g 1010 bob
RUN useradd -u 1010 -g bob -M bob
ENV GOPATH /go
WORKDIR /go/src/github.com/ceph/go-ceph
VOLUME /go/src/github.com/ceph/go-ceph

View File

@ -7,7 +7,7 @@ test:
go test -v ./...
test-docker: .build-docker
docker run --rm -it -v $(CURDIR):/go/src/github.com/ceph/go-ceph $(DOCKER_CI_IMAGE)
docker run --device /dev/fuse --cap-add SYS_ADMIN --security-opt apparmor:unconfined --rm -it -v $(CURDIR):/go/src/github.com/ceph/go-ceph $(DOCKER_CI_IMAGE)
.build-docker:
docker build -t $(DOCKER_CI_IMAGE) .

View File

@ -8,6 +8,16 @@
The native RADOS library and development headers are expected to be installed.
On debian systems (apt):
```sh
libcephfs-dev librbd-dev librados-dev
```
On rpm based systems (dnf, yum, etc):
```sh
libcephfs-devel librbd-devel librados-devel
```
## Documentation
Detailed documentation is available at
@ -92,10 +102,11 @@ conn.DeletePool("new_pool")
# Development
```
docker run --rm -it --net=host
-v ${PWD}:/go/src/github.com/ceph/go-ceph:z
-v /home/nwatkins/src/ceph/build:/home/nwatkins/src/ceph/build:z
-e CEPH_CONF=/home/nwatkins/src/ceph/build/ceph.conf
docker run --rm -it --net=host \
--device /dev/fuse --cap-add SYS_ADMIN --security-opt apparmor:unconfined \
-v ${PWD}:/go/src/github.com/ceph/go-ceph:z \
-v /home/nwatkins/src/ceph/build:/home/nwatkins/src/ceph/build:z \
-e CEPH_CONF=/home/nwatkins/src/ceph/build/ceph.conf \
ceph-golang
```
@ -122,4 +133,3 @@ Contributions are welcome & greatly appreciated, every little bit helps. Make co
```
make test-docker
```

View File

@ -20,11 +20,10 @@ type CephError int
func (e CephError) Error() string {
if e == 0 {
return fmt.Sprintf("")
} else {
err := syscall.Errno(uint(math.Abs(float64(e))))
return fmt.Sprintf("cephfs: ret=(%d) %v", e, err)
return fmt.Sprintf("cephfs: no error given")
}
err := syscall.Errno(uint(math.Abs(float64(e))))
return fmt.Sprintf("cephfs: ret=(%d) %v", e, err)
}
type MountInfo struct {
@ -36,33 +35,36 @@ func CreateMount() (*MountInfo, error) {
ret := C.ceph_create(&mount.mount, nil)
if ret == 0 {
return mount, nil
} else {
log.Errorf("CreateMount: Failed to create mount")
return nil, CephError(ret)
}
log.Errorf("CreateMount: Failed to create mount")
return nil, CephError(ret)
}
func (mount *MountInfo) RemoveDir(path string) error {
c_path := C.CString(path)
defer C.free(unsafe.Pointer(c_path))
ret := C.ceph_rmdir(mount.mount, c_path)
func (mount *MountInfo) ReadDefaultConfigFile() error {
ret := C.ceph_conf_read_file(mount.mount, nil)
if ret == 0 {
return nil
} else {
log.Errorf("RemoveDir: Failed to remove directory")
return CephError(ret)
}
log.Errorf("ReadDefaultConfigFile: Failed to read ceph config")
return CephError(ret)
}
func (mount *MountInfo) Mount() error {
ret := C.ceph_mount(mount.mount, nil)
if ret == 0 {
return nil
}
log.Errorf("Mount: Failed to mount")
return CephError(ret)
}
func (mount *MountInfo) Unmount() error {
ret := C.ceph_unmount(mount.mount)
if ret == 0 {
return nil
} else {
log.Errorf("Unmount: Failed to unmount")
return CephError(ret)
}
log.Errorf("Unmount: Failed to unmount")
return CephError(ret)
}
func (mount *MountInfo) Release() error {
@ -75,34 +77,13 @@ func (mount *MountInfo) Release() error {
}
}
func (mount *MountInfo) ReadDefaultConfigFile() error {
ret := C.ceph_conf_read_file(mount.mount, nil)
if ret == 0 {
return nil
} else {
log.Errorf("ReadDefaultConfigFile: Failed to read ceph config")
return CephError(ret)
}
}
func (mount *MountInfo) Mount() error {
ret := C.ceph_mount(mount.mount, nil)
if ret == 0 {
return nil
} else {
log.Errorf("Mount: Failed to mount")
return CephError(ret)
}
}
func (mount *MountInfo) SyncFs() error {
ret := C.ceph_sync_fs(mount.mount)
if ret == 0 {
return nil
} else {
log.Errorf("Mount: Failed to sync filesystem")
return CephError(ret)
}
log.Errorf("Mount: Failed to sync filesystem")
return CephError(ret)
}
func (mount *MountInfo) CurrentDir() string {
@ -117,10 +98,9 @@ func (mount *MountInfo) ChangeDir(path string) error {
ret := C.ceph_chdir(mount.mount, c_path)
if ret == 0 {
return nil
} else {
log.Errorf("ChangeDir: Failed to change directory")
return CephError(ret)
}
log.Errorf("ChangeDir: Failed to change directory")
return CephError(ret)
}
func (mount *MountInfo) MakeDir(path string, mode uint32) error {
@ -130,10 +110,21 @@ func (mount *MountInfo) MakeDir(path string, mode uint32) error {
ret := C.ceph_mkdir(mount.mount, c_path, C.mode_t(mode))
if ret == 0 {
return nil
} else {
log.Errorf("MakeDir: Failed to make directory %s", path)
return CephError(ret)
}
log.Errorf("MakeDir: Failed to make directory %s", path)
return CephError(ret)
}
func (mount *MountInfo) RemoveDir(path string) error {
c_path := C.CString(path)
defer C.free(unsafe.Pointer(c_path))
ret := C.ceph_rmdir(mount.mount, c_path)
if ret == 0 {
return nil
}
log.Errorf("RemoveDir: Failed to remove directory")
return CephError(ret)
}
func (mount *MountInfo) Chmod(path string, mode uint32) error {
@ -143,10 +134,9 @@ func (mount *MountInfo) Chmod(path string, mode uint32) error {
ret := C.ceph_chmod(mount.mount, c_path, C.mode_t(mode))
if ret == 0 {
return nil
} else {
log.Errorf("Chmod: Failed to chmod :%s", path)
return CephError(ret)
}
log.Errorf("Chmod: Failed to chmod :%s", path)
return CephError(ret)
}
func (mount *MountInfo) Chown(path string, user uint32, group uint32) error {
@ -156,10 +146,9 @@ func (mount *MountInfo) Chown(path string, user uint32, group uint32) error {
ret := C.ceph_chown(mount.mount, c_path, C.int(user), C.int(group))
if ret == 0 {
return nil
} else {
log.Errorf("Chown: Failed to chown :%s", path)
return CephError(ret)
}
log.Errorf("Chown: Failed to chown :%s", path)
return CephError(ret)
}
/*
@ -168,9 +157,5 @@ func (mount *MountInfo) Chown(path string, user uint32, group uint32) error {
func (mount *MountInfo) IsMounted() bool {
ret := C.ceph_is_mounted(mount.mount)
return ret == 0
}
func (mount *MountInfo) GetMount() *C.struct_ceph_mount_info {
return mount.mount
return ret == 1
}

View File

@ -4,23 +4,25 @@ import (
"fmt"
"github.com/ceph/go-ceph/cephfs"
"github.com/stretchr/testify/assert"
"io/ioutil"
"os"
"syscall"
"testing"
)
var (
CephMountTest string = "/tmp/ceph/mds/mnt/"
)
func TestCreateMount(t *testing.T) {
mount, err := cephfs.CreateMount()
assert.NoError(t, err)
assert.NotNil(t, mount)
assert.True(t, mount.IsMounted())
}
func TestMountRoot(t *testing.T) {
mount, err := cephfs.CreateMount()
assert.NoError(t, err)
assert.NotNil(t, mount)
assert.True(t, mount.IsMounted())
err = mount.ReadDefaultConfigFile()
assert.NoError(t, err)
@ -33,7 +35,6 @@ func TestSyncFs(t *testing.T) {
mount, err := cephfs.CreateMount()
assert.NoError(t, err)
assert.NotNil(t, mount)
assert.True(t, mount.IsMounted())
err = mount.ReadDefaultConfigFile()
assert.NoError(t, err)
@ -49,7 +50,6 @@ func TestChangeDir(t *testing.T) {
mount, err := cephfs.CreateMount()
assert.NoError(t, err)
assert.NotNil(t, mount)
assert.True(t, mount.IsMounted())
err = mount.ReadDefaultConfigFile()
assert.NoError(t, err)
@ -75,11 +75,10 @@ func TestChangeDir(t *testing.T) {
}
func TestRemoveDir(t *testing.T) {
dirname := "/one"
dirname := "one"
mount, err := cephfs.CreateMount()
assert.NoError(t, err)
assert.NotNil(t, mount)
assert.True(t, mount.IsMounted())
err = mount.ReadDefaultConfigFile()
assert.NoError(t, err)
@ -87,35 +86,35 @@ func TestRemoveDir(t *testing.T) {
err = mount.Mount()
assert.NoError(t, err)
dir1 := mount.CurrentDir()
assert.NotNil(t, dir1)
fmt.Printf("path: %v\n", dir1)
err = mount.MakeDir(dirname, 0755)
assert.NoError(t, err)
err = mount.SyncFs()
assert.NoError(t, err)
files, _ := ioutil.ReadDir("./")
for _, f := range files {
fmt.Println(f.Name())
}
_, err = os.Stat(dirname)
// os.Stat the actual mounted location to verify Makedir/RemoveDir
_, err = os.Stat(CephMountTest + dirname)
assert.NoError(t, err)
err = mount.RemoveDir(dirname)
assert.NoError(t, err)
_, err = os.Stat(dirname)
assert.EqualError(t, err, fmt.Sprintf("stat %s: no such file or directory", dirname))
_, err = os.Stat(CephMountTest + dirname)
assert.EqualError(t, err,
fmt.Sprintf("stat %s: no such file or directory", CephMountTest+dirname))
}
func TestUnmountMount(t *testing.T) {
mount, err := cephfs.CreateMount()
assert.NoError(t, err)
assert.NotNil(t, mount)
fmt.Printf("%#v\n", mount.IsMounted())
err = mount.ReadDefaultConfigFile()
assert.NoError(t, err)
err = mount.Mount()
assert.NoError(t, err)
assert.True(t, mount.IsMounted())
err = mount.Unmount()
@ -127,9 +126,80 @@ func TestReleaseMount(t *testing.T) {
mount, err := cephfs.CreateMount()
assert.NoError(t, err)
assert.NotNil(t, mount)
assert.True(t, mount.IsMounted())
err = mount.Release()
assert.NoError(t, err)
assert.Nil(t, mount.GetMount())
}
func TestChmodDir(t *testing.T) {
dirname := "two"
var stats_before uint32 = 0755
var stats_after uint32 = 0700
mount, err := cephfs.CreateMount()
assert.NoError(t, err)
assert.NotNil(t, mount)
err = mount.ReadDefaultConfigFile()
assert.NoError(t, err)
err = mount.Mount()
assert.NoError(t, err)
err = mount.MakeDir(dirname, stats_before)
assert.NoError(t, err)
err = mount.SyncFs()
assert.NoError(t, err)
// os.Stat the actual mounted location to verify Makedir/RemoveDir
stats, err := os.Stat(CephMountTest + dirname)
assert.NoError(t, err)
assert.Equal(t, uint32(stats.Mode().Perm()), stats_before)
err = mount.Chmod(dirname, stats_after)
assert.NoError(t, err)
stats, err = os.Stat(CephMountTest + dirname)
assert.Equal(t, uint32(stats.Mode().Perm()), stats_after)
}
// Not cross-platform, go's os does not specifiy Sys return type
func TestChown(t *testing.T) {
dirname := "three"
// dockerfile creates bob user account
var bob uint32 = 1010
var root uint32 = 0
mount, err := cephfs.CreateMount()
assert.NoError(t, err)
assert.NotNil(t, mount)
err = mount.ReadDefaultConfigFile()
assert.NoError(t, err)
err = mount.Mount()
assert.NoError(t, err)
err = mount.MakeDir(dirname, 0755)
assert.NoError(t, err)
err = mount.SyncFs()
assert.NoError(t, err)
// os.Stat the actual mounted location to verify Makedir/RemoveDir
stats, err := os.Stat(CephMountTest + dirname)
assert.NoError(t, err)
assert.Equal(t, uint32(stats.Sys().(*syscall.Stat_t).Uid), root)
assert.Equal(t, uint32(stats.Sys().(*syscall.Stat_t).Gid), root)
err = mount.Chown(dirname, bob, bob)
assert.NoError(t, err)
stats, err = os.Stat(CephMountTest + dirname)
assert.NoError(t, err)
assert.Equal(t, uint32(stats.Sys().(*syscall.Stat_t).Uid), bob)
assert.Equal(t, uint32(stats.Sys().(*syscall.Stat_t).Gid), bob)
}

View File

@ -26,9 +26,12 @@ rm -rf ${DIR}/*
LOG_DIR=${DIR}/log
MON_DATA=${DIR}/mon
MDS_DATA=${DIR}/mds
MOUNTPT=${MDS_DATA}/mnt
OSD_DATA=${DIR}/osd
mkdir ${LOG_DIR} ${MON_DATA} ${OSD_DATA} ${MDS_DATA}
mkdir ${LOG_DIR} ${MON_DATA} ${OSD_DATA} ${MDS_DATA} ${MOUNTPT}
MDS_NAME="Z"
MON_NAME="a"
MGR_NAME="x"
# cluster wide parameters
cat >> ${DIR}/ceph.conf <<EOF
@ -44,7 +47,7 @@ osd pool default size = 1
[mds.${MDS_NAME}]
host = localhost
[mon.a]
[mon.${MON_NAME}]
log file = ${LOG_DIR}/mon.log
chdir = ""
mon cluster log file = ${LOG_DIR}/mon-cluster.log
@ -66,9 +69,9 @@ EOF
export CEPH_CONF=${DIR}/ceph.conf
# start an osd
ceph-mon --id a --mkfs --keyring /dev/null
ceph-mon --id ${MON_NAME} --mkfs --keyring /dev/null
touch ${MON_DATA}/keyring
ceph-mon --id a
ceph-mon --id ${MON_NAME}
# start an osd
OSD_ID=$(ceph osd create)
@ -83,9 +86,15 @@ ceph osd pool create cephfs_metadata 8
ceph fs new cephfs cephfs_metadata cephfs_data
ceph fs ls
ceph-mds -i ${MDS_NAME}
ceph status
while [[ ! $(ceph mds stat | grep "up:active") ]]; do sleep 1; done
# fuse: device not found, try 'modprobe fuse' first
# Make sure to run with --privileged or --cap-add SYS_ADMIN --device /dev/fuse --security apparmor:unconfined for docker
ceph-fuse ${MOUNTPT}
# start a manager
ceph-mgr --id x
ceph-mgr --id ${MGR_NAME}
# test the setup
ceph --version