btrfs-progs: switch open helper functions to return negative errno

It's commonly used elsewhere in the code to return the -errno values if
possible, do that for the open helpers too.

Signed-off-by: David Sterba <dsterba@suse.com>

cmds/filesystem-du.c

@@ -458,7 +458,7 @@ static int du_add_file(const char *filename, int dirfd,
 
 	fd = btrfs_open_fd2(path, false, false, false);
 	if (fd < 0) {
-		ret = -errno;
+		ret = fd;
 		goto out;
 	}
 

cmds/filesystem.c

@@ -1152,8 +1152,9 @@ static int cmd_filesystem_defrag(const struct cmd_struct *cmd,
 
 		fd = btrfs_open_fd2(argv[i], false, defrag_open_mode == O_RDWR, false);
 		if (fd < 0) {
+			errno = -fd;
 			error("cannot open %s: %m", argv[i]);
-			ret = -errno;
+			ret = fd;
 			goto next;
 		}
 
@@ -1408,7 +1409,7 @@ static int cmd_filesystem_resize(const struct cmd_struct *cmd,
 	fd = btrfs_open_dir_fd(path);
 	if (fd < 0) {
 		/* The path is a directory */
-		if (fd == -3) {
+		if (fd == -ENOTDIR) {
 			error(
 		"resize works on mounted filesystems and accepts only\n"
 		"directories as argument. Passing file containing a btrfs image\n"

cmds/inspect.c

@@ -1084,6 +1084,7 @@ static int cmd_inspect_list_chunks(const struct cmd_struct *cmd,
 
 	fd = btrfs_open_fd2(path, false, true, false);
 	if (fd < 0) {
+		errno = -fd;
 	        error("cannot access '%s': %m", path);
 		return 1;
 	}

cmds/property.c

@@ -180,9 +180,10 @@ static int prop_compression(enum prop_object_type type,
 	int open_flags = value ? O_RDWR : O_RDONLY;
 
 	fd = btrfs_open_fd2(object, false, open_flags == O_RDWR, false);
-	if (fd == -1) {
-		ret = -errno;
+	if (fd < 0) {
+		errno = -fd;
 		error("failed to open %s: %m", object);
+		ret = fd;
 		goto out;
 	}
 

cmds/subvolume.c

@@ -1625,7 +1625,9 @@ static int cmd_subvolume_show(const struct cmd_struct *cmd, int argc, char **arg
 
 	fd = btrfs_open_fd2(fullpath, false, true, false);
 	if (fd < 0) {
-		error("can't access '%s'", fullpath);
+		errno = -fd;
+		error("can't access '%s': %m", fullpath);
+		ret = fd;
 		goto out;
 	}
 

common/open-utils.c

@@ -185,6 +185,8 @@ out:
 
 /*
  * Open the given path and check if it's a btrfs filesystem.
+ *
+ * Return the file descriptor or -errno.
  */
 int btrfs_open_fd2(const char *path, bool verbose, bool read_write, bool dir_only)
 {
@@ -192,24 +194,24 @@ int btrfs_open_fd2(const char *path, bool verbose, bool read_write, bool dir_onl
 	struct stat st;
 	int ret;
 
-	if (stat(path, &st) != 0) {
+	if (stat(path, &st) < 0) {
 		error_on(verbose, "cannot access '%s': %m", path);
-		return -1;
+		return -errno;
 	}
 
 	if (dir_only && !S_ISDIR(st.st_mode)) {
 		error_on(verbose, "not a directory: %s", path);
-		return -3;
+		return -ENOTDIR;
 	}
 
-	if (statfs(path, &stfs) != 0) {
+	if (statfs(path, &stfs) < 0) {
 		error_on(verbose, "cannot access '%s': %m", path);
-		return -1;
+		return -errno;
 	}
 
 	if (stfs.f_type != BTRFS_SUPER_MAGIC) {
 		error_on(verbose, "not a btrfs filesystem: %s", path);
-		return -2;
+		return -EINVAL;
 	}
 
 	if (S_ISDIR(st.st_mode) || !read_write)
@@ -219,6 +221,7 @@ int btrfs_open_fd2(const char *path, bool verbose, bool read_write, bool dir_onl
 
 	if (ret < 0) {
 		error_on(verbose, "cannot access '%s': %m", path);
+		ret = -errno;
 	}
 
 	return ret;
@@ -238,7 +241,7 @@ int btrfs_open_dir_fd(const char *path)
  * Given a path, return a file descriptor to the original path name or, if the
  * pathname is a mounted btrfs device, to its mountpoint.
  *
- * On error, return -1, errno should be set.
+ * Return the file descriptor or -errno.
  */
 int btrfs_open_mnt_fd(const char *path, bool verbose)
 {
@@ -249,8 +252,7 @@ int btrfs_open_mnt_fd(const char *path, bool verbose)
 		ret = get_btrfs_mount(path, mp, sizeof(mp));
 		if (ret < 0) {
 			error_on(verbose, "'%s' is not a mounted btrfs device", path);
-			errno = EINVAL;
-			return -1;
+			return -EINVAL;
 		}
 		ret = btrfs_open_fd2(mp, verbose, true, true);
 	} else {

common/utils.c

@@ -252,7 +252,7 @@ int get_fs_info(const char *path, struct btrfs_ioctl_fs_info_args *fi_args,
 	/* at this point path must not be for a block device */
 	fd = btrfs_open_fd2(path, false, true, false);
 	if (fd < 0) {
-		ret = -errno;
+		ret = -fd;
 		goto out;
 	}