cephfs: add unmount, release, chmod, chown, etc

This commit adds the following cephfs functions:

* Unmount // Unmounting is necessary to cleanup mounts
* Release // Release destroys the cmount ~ end of transaction
* RemoveDir // inverse of MakeDir
* Chown // change ownership of file or directory
* Chmod // change permissions of file or directory

Tests are included for each function.

In addition to these changes modifications to:

.travis.yml, Dockerfile, and Makefile

were made to accomodate tests to mount the ceph volume.  Tests use
fuse to mount the volume which requires adding:

--device /dev/fuse --cap-add SYS_ADMIN --security-opt \
apparmor:unconfined

to the docker container (alternatively --privileged works but adds
additional permissions).

Changes to README add the above docker changes as well as point
users to the necessary ceph development libraries.

.travis.yml

@@ -21,5 +21,6 @@ before_install: |
     exit 1
   fi
 
+# cephfs (fuse) requires: --device /dev/fuse --cap-add SYS_ADMIN --security-opt apparmor:unconfined
 script:
-  - docker run --rm -it -v ${PWD}:/go/src/github.com/ceph/go-ceph:z ceph-golang-ci
+  - docker run --device /dev/fuse --cap-add SYS_ADMIN --security-opt apparmor:unconfined --rm -it -v ${PWD}:/go/src/github.com/ceph/go-ceph:z ceph-golang-ci

Dockerfile

@@ -20,6 +20,10 @@ RUN apt-get update && apt-get install -y \
   librbd-dev \
   golang-1.10-go
 
+# add user account to test permissions
+RUN groupadd -g 1010 bob
+RUN useradd -u 1010 -g bob -M bob
+
 ENV GOPATH /go
 WORKDIR /go/src/github.com/ceph/go-ceph
 VOLUME /go/src/github.com/ceph/go-ceph

Makefile

@@ -7,7 +7,7 @@ test:
 	go test -v ./...
 
 test-docker: .build-docker
-	docker run --rm -it -v $(CURDIR):/go/src/github.com/ceph/go-ceph $(DOCKER_CI_IMAGE)
+	docker run --device /dev/fuse --cap-add SYS_ADMIN --security-opt apparmor:unconfined --rm -it -v $(CURDIR):/go/src/github.com/ceph/go-ceph $(DOCKER_CI_IMAGE)
 
 .build-docker:
 	docker build -t $(DOCKER_CI_IMAGE) .

README.md

@@ -8,6 +8,16 @@
 
 The native RADOS library and development headers are expected to be installed.
 
+On debian systems (apt):
+```sh
+libcephfs-dev librbd-dev librados-dev
+```
+
+On rpm based systems (dnf, yum, etc):
+```sh
+libcephfs-devel librbd-devel librados-devel
+```
+
 ## Documentation
 
 Detailed documentation is available at
@@ -92,10 +102,11 @@ conn.DeletePool("new_pool")
 # Development
 
 ```
-docker run --rm -it --net=host
+docker run --rm -it --net=host \
-  -v ${PWD}:/go/src/github.com/ceph/go-ceph:z
+  --device /dev/fuse --cap-add SYS_ADMIN --security-opt apparmor:unconfined \
-  -v /home/nwatkins/src/ceph/build:/home/nwatkins/src/ceph/build:z
+  -v ${PWD}:/go/src/github.com/ceph/go-ceph:z \
-  -e CEPH_CONF=/home/nwatkins/src/ceph/build/ceph.conf
+  -v /home/nwatkins/src/ceph/build:/home/nwatkins/src/ceph/build:z \
+  -e CEPH_CONF=/home/nwatkins/src/ceph/build/ceph.conf \
   ceph-golang
 ```
 
@@ -122,4 +133,3 @@ Contributions are welcome & greatly appreciated, every little bit helps. Make co
 ```
 make test-docker
 ```
-

cephfs/cephfs.go

@@ -20,11 +20,10 @@ type CephError int
 
 func (e CephError) Error() string {
 	if e == 0 {
-		return fmt.Sprintf("")
+		return fmt.Sprintf("cephfs: no error given")
-	} else {
-		err := syscall.Errno(uint(math.Abs(float64(e))))
-		return fmt.Sprintf("cephfs: ret=(%d) %v", e, err)
 	}
+	err := syscall.Errno(uint(math.Abs(float64(e))))
+	return fmt.Sprintf("cephfs: ret=(%d) %v", e, err)
 }
 
 type MountInfo struct {
@@ -36,33 +35,36 @@ func CreateMount() (*MountInfo, error) {
 	ret := C.ceph_create(&mount.mount, nil)
 	if ret == 0 {
 		return mount, nil
-	} else {
-		log.Errorf("CreateMount: Failed to create mount")
-		return nil, CephError(ret)
 	}
+	log.Errorf("CreateMount: Failed to create mount")
+	return nil, CephError(ret)
 }
 
-func (mount *MountInfo) RemoveDir(path string) error {
+func (mount *MountInfo) ReadDefaultConfigFile() error {
-	c_path := C.CString(path)
+	ret := C.ceph_conf_read_file(mount.mount, nil)
-	defer C.free(unsafe.Pointer(c_path))
-
-	ret := C.ceph_rmdir(mount.mount, c_path)
 	if ret == 0 {
 		return nil
-	} else {
-		log.Errorf("RemoveDir: Failed to remove directory")
-		return CephError(ret)
 	}
+	log.Errorf("ReadDefaultConfigFile: Failed to read ceph config")
+	return CephError(ret)
+}
+
+func (mount *MountInfo) Mount() error {
+	ret := C.ceph_mount(mount.mount, nil)
+	if ret == 0 {
+		return nil
+	}
+	log.Errorf("Mount: Failed to mount")
+	return CephError(ret)
 }
 
 func (mount *MountInfo) Unmount() error {
 	ret := C.ceph_unmount(mount.mount)
 	if ret == 0 {
 		return nil
-	} else {
-		log.Errorf("Unmount: Failed to unmount")
-		return CephError(ret)
 	}
+	log.Errorf("Unmount: Failed to unmount")
+	return CephError(ret)
 }
 
 func (mount *MountInfo) Release() error {
@@ -75,34 +77,13 @@ func (mount *MountInfo) Release() error {
 	}
 }
 
-func (mount *MountInfo) ReadDefaultConfigFile() error {
-	ret := C.ceph_conf_read_file(mount.mount, nil)
-	if ret == 0 {
-		return nil
-	} else {
-		log.Errorf("ReadDefaultConfigFile: Failed to read ceph config")
-		return CephError(ret)
-	}
-}
-
-func (mount *MountInfo) Mount() error {
-	ret := C.ceph_mount(mount.mount, nil)
-	if ret == 0 {
-		return nil
-	} else {
-		log.Errorf("Mount: Failed to mount")
-		return CephError(ret)
-	}
-}
-
 func (mount *MountInfo) SyncFs() error {
 	ret := C.ceph_sync_fs(mount.mount)
 	if ret == 0 {
 		return nil
-	} else {
-		log.Errorf("Mount: Failed to sync filesystem")
-		return CephError(ret)
 	}
+	log.Errorf("Mount: Failed to sync filesystem")
+	return CephError(ret)
 }
 
 func (mount *MountInfo) CurrentDir() string {
@@ -117,10 +98,9 @@ func (mount *MountInfo) ChangeDir(path string) error {
 	ret := C.ceph_chdir(mount.mount, c_path)
 	if ret == 0 {
 		return nil
-	} else {
-		log.Errorf("ChangeDir: Failed to change directory")
-		return CephError(ret)
 	}
+	log.Errorf("ChangeDir: Failed to change directory")
+	return CephError(ret)
 }
 
 func (mount *MountInfo) MakeDir(path string, mode uint32) error {
@@ -130,10 +110,21 @@ func (mount *MountInfo) MakeDir(path string, mode uint32) error {
 	ret := C.ceph_mkdir(mount.mount, c_path, C.mode_t(mode))
 	if ret == 0 {
 		return nil
-	} else {
-		log.Errorf("MakeDir: Failed to make directory %s", path)
-		return CephError(ret)
 	}
+	log.Errorf("MakeDir: Failed to make directory %s", path)
+	return CephError(ret)
+}
+
+func (mount *MountInfo) RemoveDir(path string) error {
+	c_path := C.CString(path)
+	defer C.free(unsafe.Pointer(c_path))
+
+	ret := C.ceph_rmdir(mount.mount, c_path)
+	if ret == 0 {
+		return nil
+	}
+	log.Errorf("RemoveDir: Failed to remove directory")
+	return CephError(ret)
 }
 
 func (mount *MountInfo) Chmod(path string, mode uint32) error {
@@ -143,10 +134,9 @@ func (mount *MountInfo) Chmod(path string, mode uint32) error {
 	ret := C.ceph_chmod(mount.mount, c_path, C.mode_t(mode))
 	if ret == 0 {
 		return nil
-	} else {
-		log.Errorf("Chmod: Failed to chmod :%s", path)
-		return CephError(ret)
 	}
+	log.Errorf("Chmod: Failed to chmod :%s", path)
+	return CephError(ret)
 }
 
 func (mount *MountInfo) Chown(path string, user uint32, group uint32) error {
@@ -156,10 +146,9 @@ func (mount *MountInfo) Chown(path string, user uint32, group uint32) error {
 	ret := C.ceph_chown(mount.mount, c_path, C.int(user), C.int(group))
 	if ret == 0 {
 		return nil
-	} else {
-		log.Errorf("Chown: Failed to chown :%s", path)
-		return CephError(ret)
 	}
+	log.Errorf("Chown: Failed to chown :%s", path)
+	return CephError(ret)
 }
 
 /*
@@ -168,9 +157,5 @@ func (mount *MountInfo) Chown(path string, user uint32, group uint32) error {
 
 func (mount *MountInfo) IsMounted() bool {
 	ret := C.ceph_is_mounted(mount.mount)
-	return ret == 0
+	return ret == 1
-}
-
-func (mount *MountInfo) GetMount() *C.struct_ceph_mount_info {
-	return mount.mount
 }

cephfs/cephfs_test.go

@@ -4,23 +4,25 @@ import (
 	"fmt"
 	"github.com/ceph/go-ceph/cephfs"
 	"github.com/stretchr/testify/assert"
-	"io/ioutil"
 	"os"
+	"syscall"
 	"testing"
 )
 
+var (
+	CephMountTest string = "/tmp/ceph/mds/mnt/"
+)
+
 func TestCreateMount(t *testing.T) {
 	mount, err := cephfs.CreateMount()
 	assert.NoError(t, err)
 	assert.NotNil(t, mount)
-	assert.True(t, mount.IsMounted())
 }
 
 func TestMountRoot(t *testing.T) {
 	mount, err := cephfs.CreateMount()
 	assert.NoError(t, err)
 	assert.NotNil(t, mount)
-	assert.True(t, mount.IsMounted())
 
 	err = mount.ReadDefaultConfigFile()
 	assert.NoError(t, err)
@@ -33,7 +35,6 @@ func TestSyncFs(t *testing.T) {
 	mount, err := cephfs.CreateMount()
 	assert.NoError(t, err)
 	assert.NotNil(t, mount)
-	assert.True(t, mount.IsMounted())
 
 	err = mount.ReadDefaultConfigFile()
 	assert.NoError(t, err)
@@ -49,7 +50,6 @@ func TestChangeDir(t *testing.T) {
 	mount, err := cephfs.CreateMount()
 	assert.NoError(t, err)
 	assert.NotNil(t, mount)
-	assert.True(t, mount.IsMounted())
 
 	err = mount.ReadDefaultConfigFile()
 	assert.NoError(t, err)
@@ -75,11 +75,10 @@ func TestChangeDir(t *testing.T) {
 }
 
 func TestRemoveDir(t *testing.T) {
-	dirname := "/one"
+	dirname := "one"
 	mount, err := cephfs.CreateMount()
 	assert.NoError(t, err)
 	assert.NotNil(t, mount)
-	assert.True(t, mount.IsMounted())
 
 	err = mount.ReadDefaultConfigFile()
 	assert.NoError(t, err)
@@ -87,35 +86,35 @@ func TestRemoveDir(t *testing.T) {
 	err = mount.Mount()
 	assert.NoError(t, err)
 
-	dir1 := mount.CurrentDir()
-	assert.NotNil(t, dir1)
-
-	fmt.Printf("path: %v\n", dir1)
-
 	err = mount.MakeDir(dirname, 0755)
 	assert.NoError(t, err)
 
 	err = mount.SyncFs()
 	assert.NoError(t, err)
 
-	files, _ := ioutil.ReadDir("./")
+	// os.Stat the actual mounted location to verify Makedir/RemoveDir
-	for _, f := range files {
+	_, err = os.Stat(CephMountTest + dirname)
-		fmt.Println(f.Name())
-	}
-
-	_, err = os.Stat(dirname)
 	assert.NoError(t, err)
 
 	err = mount.RemoveDir(dirname)
 	assert.NoError(t, err)
-	_, err = os.Stat(dirname)
+
-	assert.EqualError(t, err, fmt.Sprintf("stat %s: no such file or directory", dirname))
+	_, err = os.Stat(CephMountTest + dirname)
+	assert.EqualError(t, err,
+		fmt.Sprintf("stat %s: no such file or directory", CephMountTest+dirname))
 }
 
 func TestUnmountMount(t *testing.T) {
 	mount, err := cephfs.CreateMount()
 	assert.NoError(t, err)
 	assert.NotNil(t, mount)
+	fmt.Printf("%#v\n", mount.IsMounted())
+
+	err = mount.ReadDefaultConfigFile()
+	assert.NoError(t, err)
+
+	err = mount.Mount()
+	assert.NoError(t, err)
 	assert.True(t, mount.IsMounted())
 
 	err = mount.Unmount()
@@ -127,9 +126,80 @@ func TestReleaseMount(t *testing.T) {
 	mount, err := cephfs.CreateMount()
 	assert.NoError(t, err)
 	assert.NotNil(t, mount)
-	assert.True(t, mount.IsMounted())
 
 	err = mount.Release()
 	assert.NoError(t, err)
-	assert.Nil(t, mount.GetMount())
+}
+
+func TestChmodDir(t *testing.T) {
+	dirname := "two"
+	var stats_before uint32 = 0755
+	var stats_after uint32 = 0700
+	mount, err := cephfs.CreateMount()
+	assert.NoError(t, err)
+	assert.NotNil(t, mount)
+
+	err = mount.ReadDefaultConfigFile()
+	assert.NoError(t, err)
+
+	err = mount.Mount()
+	assert.NoError(t, err)
+
+	err = mount.MakeDir(dirname, stats_before)
+	assert.NoError(t, err)
+
+	err = mount.SyncFs()
+	assert.NoError(t, err)
+
+	// os.Stat the actual mounted location to verify Makedir/RemoveDir
+	stats, err := os.Stat(CephMountTest + dirname)
+	assert.NoError(t, err)
+
+	assert.Equal(t, uint32(stats.Mode().Perm()), stats_before)
+
+	err = mount.Chmod(dirname, stats_after)
+	assert.NoError(t, err)
+
+	stats, err = os.Stat(CephMountTest + dirname)
+	assert.Equal(t, uint32(stats.Mode().Perm()), stats_after)
+}
+
+// Not cross-platform, go's os does not specifiy Sys return type
+func TestChown(t *testing.T) {
+	dirname := "three"
+	// dockerfile creates bob user account
+	var bob uint32 = 1010
+	var root uint32 = 0
+
+	mount, err := cephfs.CreateMount()
+	assert.NoError(t, err)
+	assert.NotNil(t, mount)
+
+	err = mount.ReadDefaultConfigFile()
+	assert.NoError(t, err)
+
+	err = mount.Mount()
+	assert.NoError(t, err)
+
+	err = mount.MakeDir(dirname, 0755)
+	assert.NoError(t, err)
+
+	err = mount.SyncFs()
+	assert.NoError(t, err)
+
+	// os.Stat the actual mounted location to verify Makedir/RemoveDir
+	stats, err := os.Stat(CephMountTest + dirname)
+	assert.NoError(t, err)
+
+	assert.Equal(t, uint32(stats.Sys().(*syscall.Stat_t).Uid), root)
+	assert.Equal(t, uint32(stats.Sys().(*syscall.Stat_t).Gid), root)
+
+	err = mount.Chown(dirname, bob, bob)
+	assert.NoError(t, err)
+
+	stats, err = os.Stat(CephMountTest + dirname)
+	assert.NoError(t, err)
+	assert.Equal(t, uint32(stats.Sys().(*syscall.Stat_t).Uid), bob)
+	assert.Equal(t, uint32(stats.Sys().(*syscall.Stat_t).Gid), bob)
+
 }

micro-osd.sh

@@ -26,9 +26,12 @@ rm -rf ${DIR}/*
 LOG_DIR=${DIR}/log
 MON_DATA=${DIR}/mon
 MDS_DATA=${DIR}/mds
+MOUNTPT=${MDS_DATA}/mnt
 OSD_DATA=${DIR}/osd
-mkdir ${LOG_DIR} ${MON_DATA} ${OSD_DATA} ${MDS_DATA}
+mkdir ${LOG_DIR} ${MON_DATA} ${OSD_DATA} ${MDS_DATA} ${MOUNTPT}
 MDS_NAME="Z"
+MON_NAME="a"
+MGR_NAME="x"
 
 # cluster wide parameters
 cat >> ${DIR}/ceph.conf <<EOF
@@ -44,7 +47,7 @@ osd pool default size = 1
 [mds.${MDS_NAME}]
 host = localhost
 
-[mon.a]
+[mon.${MON_NAME}]
 log file = ${LOG_DIR}/mon.log
 chdir = ""
 mon cluster log file = ${LOG_DIR}/mon-cluster.log
@@ -66,9 +69,9 @@ EOF
 export CEPH_CONF=${DIR}/ceph.conf
 
 # start an osd
-ceph-mon --id a --mkfs --keyring /dev/null
+ceph-mon --id ${MON_NAME} --mkfs --keyring /dev/null
 touch ${MON_DATA}/keyring
-ceph-mon --id a
+ceph-mon --id ${MON_NAME}
 
 # start an osd
 OSD_ID=$(ceph osd create)
@@ -83,9 +86,15 @@ ceph osd pool create cephfs_metadata 8
 ceph fs new cephfs cephfs_metadata cephfs_data
 ceph fs ls
 ceph-mds -i ${MDS_NAME}
+ceph status
+while [[ ! $(ceph mds stat | grep "up:active") ]]; do sleep 1; done
+# fuse: device not found, try 'modprobe fuse' first
+# Make sure to run with --privileged or --cap-add SYS_ADMIN --device /dev/fuse --security apparmor:unconfined for docker
+ceph-fuse ${MOUNTPT}
+
 
 # start a manager
-ceph-mgr --id x
+ceph-mgr --id ${MGR_NAME}
 
 # test the setup
 ceph --version