RepoMirrors/ceph
1
0
Fork 0
mirror of https://github.com/ceph/ceph synced 2025-02-20 17:37:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #27902 from mikechristie/rbd-nbd-netlink-support

Browse Source 
rbd-nbd: add netlink map/unmap support

Reviewed-by: Jason Dillaman <dillaman@redhat.com>
This commit is contained in:
Jason Dillaman 2019-06-06 10:36:48 -04:00 committed by GitHub
commit 6cd6f5e607
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 600 additions and 168 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
CMakeLists.txt
View File

@ -210,20 +210,30 @@ endif()
find_package(Backtrace)
# remote block storage
option(WITH_RBD "Remote block storage is here" ON)
if(LINUX)
find_package(udev REQUIRED)
set(HAVE_UDEV ${UDEV_FOUND})
find_package(blkid REQUIRED)
set(HAVE_BLKID ${BLKID_FOUND})
if(WITH_RBD)
find_package(genl REQUIRED)
set(HAVE_GENL $GENL_FOUND)
endif()
elseif(FREEBSD)
set(HAVE_UDEV OFF)
set(HAVE_LIBAIO OFF)
set(HAVE_BLKID OFF)
set(HAVE_GENL OFF)
else()
set(HAVE_UDEV OFF)
message(STATUS "Not using udev")
set(HAVE_BLKID OFF)
message(STATUS "Not using BLKID")
set(HAVE_GENL OFF)
message(STATUS "Not using GENL")
endif(LINUX)
option(WITH_OPENLDAP "OPENLDAP is here" ON)
@ -311,9 +321,6 @@ option(WITH_LIBCEPHFS "libcephfs client library" ON)
# key-value store
option(WITH_KVS "Key value store is here" ON)
# remote block storage
option(WITH_RBD "Remote block storage is here" ON)
# KERNEL remote block storage
option(WITH_KRBD "Kernel Remote block storage is here" ON)

1
ceph.spec.in
View File

@ -169,6 +169,7 @@ BuildRequires: libaio-devel
BuildRequires: libblkid-devel >= 2.17
BuildRequires: libcurl-devel
BuildRequires: libudev-devel
BuildRequires: libnl3-devel
BuildRequires: liboath-devel
BuildRequires: libtool
BuildRequires: libxml2-devel

23
cmake/modules/Findgenl.cmake Normal file
View File

@ -0,0 +1,23 @@
# - Find libnl-genl3
# Find the genl library and includes
#
# GENL_INCLUDE_DIR - where to find netlink.h, etc.
# GENL_LIBRARIES - List of libraries when using genl.
# GENL_FOUND - True if genl found.
find_path(GENL_INCLUDE_DIR NAMES netlink/netlink.h PATH_SUFFIXES libnl3)
find_library(LIBNL_LIB nl-3)
find_library(LIBNL_GENL_LIB nl-genl-3)
set(GENL_LIBRARIES
${LIBNL_LIB}
${LIBNL_GENL_LIB}
)
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(nl-genl-3
DEFAULT_MSG GENL_LIBRARIES GENL_INCLUDE_DIR)
mark_as_advanced(
GENL_LIBRARIES
GENL_INCLUDE_DIR)

2
debian/control vendored
View File

@ -45,6 +45,8 @@ Build-Depends: cmake (>= 3.5),
libssl-dev,
libtool,
libudev-dev,
libnl-3-dev,
libnl-genl-3-dev,
libxml2-dev,
librabbitmq-dev,
lsb-release,

3
src/tools/rbd_nbd/CMakeLists.txt
View File

@ -1,3 +1,4 @@
add_executable(rbd-nbd rbd-nbd.cc)
target_link_libraries(rbd-nbd librbd librados global)
target_include_directories(rbd-nbd PUBLIC ${GENL_INCLUDE_DIR})
target_link_libraries(rbd-nbd librbd librados global ${GENL_LIBRARIES})
install(TARGETS rbd-nbd DESTINATION bin)

69
src/tools/rbd_nbd/nbd-netlink.h Normal file
View File

@ -0,0 +1,69 @@
/*
* Copyright (C) 2017 Facebook. All rights reserved.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public
* License v2 as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* You should have received a copy of the GNU General Public
* License along with this program; if not, write to the
* Free Software Foundation, Inc., 59 Temple Place - Suite 330,
* Boston, MA 021110-1307, USA.
*/
#ifndef _UAPILINUX_NBD_NETLINK_H
#define _UAPILINUX_NBD_NETLINK_H
#define NBD_GENL_FAMILY_NAME "nbd"
#define NBD_GENL_VERSION 0x1
/* Configuration policy attributes, used for CONNECT */
enum {
NBD_ATTR_UNSPEC,
NBD_ATTR_INDEX,
NBD_ATTR_SIZE_BYTES,
NBD_ATTR_BLOCK_SIZE_BYTES,
NBD_ATTR_TIMEOUT,
NBD_ATTR_SERVER_FLAGS,
NBD_ATTR_CLIENT_FLAGS,
NBD_ATTR_SOCKETS,
__NBD_ATTR_MAX,
};
#define NBD_ATTR_MAX (__NBD_ATTR_MAX - 1)
/*
* This is the format for multiple sockets with NBD_ATTR_SOCKETS
*
* [NBD_ATTR_SOCKETS]
* [NBD_SOCK_ITEM]
* [NBD_SOCK_FD]
* [NBD_SOCK_ITEM]
* [NBD_SOCK_FD]
*/
enum {
NBD_SOCK_ITEM_UNSPEC,
NBD_SOCK_ITEM,
__NBD_SOCK_ITEM_MAX,
};
#define NBD_SOCK_ITEM_MAX (__NBD_SOCK_ITEM_MAX - 1)
enum {
NBD_SOCK_UNSPEC,
NBD_SOCK_FD,
__NBD_SOCK_MAX,
};
#define NBD_SOCK_MAX (__NBD_SOCK_MAX - 1)
enum {
NBD_CMD_UNSPEC,
NBD_CMD_CONNECT,
NBD_CMD_DISCONNECT,
__NBD_CMD_MAX,
};
#define NBD_CMD_MAX (__NBD_CMD_MAX - 1)
#endif /* _UAPILINUX_NBD_NETLINK_H */

657
src/tools/rbd_nbd/rbd-nbd.cc
View File

@ -32,6 +32,11 @@
#include <sys/ioctl.h>
#include <sys/socket.h>
#include "nbd-netlink.h"
#include <libnl3/netlink/genl/genl.h>
#include <libnl3/netlink/genl/ctrl.h>
#include <libnl3/netlink/genl/mngt.h>
#include <fstream>
#include <iostream>
#include <memory>
@ -72,6 +77,7 @@ struct Config {
bool exclusive = false;
bool readonly = false;
bool set_max_part = false;
bool try_netlink = false;
std::string poolname;
std::string nsname;
@ -95,6 +101,7 @@ static void usage()
<< " --max_part <limit> Override for module param max_part\n"
<< " --exclusive Forbid writes by other clients\n"
<< " --timeout <seconds> Set nbd request timeout\n"
<< " --try-netlink Use the nbd netlink interface\n"
<< "\n"
<< "List options:\n"
<< " --format plain|json|xml Output format (default: plain)\n"
@ -104,6 +111,7 @@ static void usage()
}
static int nbd = -1;
static int nbd_index = -1;
enum Command {
None,
@ -131,18 +139,6 @@ static Command cmd = None;
static int parse_args(vector<const char*>& args, std::ostream *err_msg,
Command *command, Config *cfg);
static void handle_signal(int signum)
{
ceph_assert(signum == SIGINT || signum == SIGTERM);
derr << "*** Got signal " << sig_str(signum) << " ***" << dendl;
dout(20) << __func__ << ": " << "sending NBD_DISCONNECT" << dendl;
if (ioctl(nbd, NBD_DISCONNECT) < 0) {
derr << "rbd-nbd: disconnect failed: " << cpp_strerror(errno) << dendl;
} else {
dout(20) << __func__ << ": " << "disconnected" << dendl;
}
}
class NBDServer
{
private:
@ -153,6 +149,7 @@ public:
NBDServer(int _fd, librbd::Image& _image)
: fd(_fd)
, image(_image)
, disconnect_lock("NBDServer::DisconnectLocker")
, lock("NBDServer::Locker")
, reader_thread(*this, &NBDServer::reader_entry)
, writer_thread(*this, &NBDServer::writer_entry)
@ -160,6 +157,8 @@ public:
{}
private:
Mutex disconnect_lock;
Cond disconnect_cond;
std::atomic<bool> terminated = { false };
void shutdown()
@ -284,12 +283,12 @@ private:
if (r < 0) {
derr << "failed to read nbd request header: " << cpp_strerror(r)
<< dendl;
return;
goto signal;
}
if (ctx->request.magic != htonl(NBD_REQUEST_MAGIC)) {
derr << "invalid nbd request header" << dendl;
return;
goto signal;
}
ctx->request.from = ntohll(ctx->request.from);
@ -308,14 +307,14 @@ private:
case NBD_CMD_DISC:
// NBD_DO_IT will return when pipe is closed
dout(0) << "disconnect request received" << dendl;
return;
goto signal;
case NBD_CMD_WRITE:
bufferptr ptr(ctx->request.len);
r = safe_read_exact(fd, ptr.c_str(), ctx->request.len);
if (r < 0) {
derr << *ctx << ": failed to read nbd request data: "
<< cpp_strerror(r) << dendl;
return;
goto signal;
}
ctx->data.push_back(ptr);
break;
@ -341,10 +340,14 @@ private:
default:
derr << *pctx << ": invalid request command" << dendl;
c->release();
return;
goto signal;
}
}
dout(20) << __func__ << ": terminated" << dendl;
signal:
Mutex::Locker l(disconnect_lock);
disconnect_cond.Signal();
}
void writer_entry()
@ -413,6 +416,15 @@ public:
}
}
void wait_for_disconnect()
{
if (!started)
return;
Mutex::Locker l(disconnect_lock);
disconnect_cond.Wait(disconnect_lock);
}
~NBDServer()
{
if (started) {
@ -579,37 +591,30 @@ private:
}
};
static int open_device(const char* path, Config *cfg = nullptr, bool try_load_module = false)
static int load_module(Config *cfg)
{
int nbd = open(path, O_RDWR);
bool loaded_module = false;
ostringstream param;
int ret;
if (nbd < 0 && try_load_module && access("/sys/module/nbd", F_OK) != 0) {
ostringstream param;
int r;
if (cfg->nbds_max) {
param << "nbds_max=" << cfg->nbds_max;
}
if (cfg->max_part) {
param << " max_part=" << cfg->max_part;
}
r = module_load("nbd", param.str().c_str());
if (r < 0) {
cerr << "rbd-nbd: failed to load nbd kernel module: " << cpp_strerror(-r) << std::endl;
return r;
} else {
loaded_module = true;
}
nbd = open(path, O_RDWR);
if (cfg->nbds_max)
param << "nbds_max=" << cfg->nbds_max;
if (cfg->max_part)
param << " max_part=" << cfg->max_part;
if (!access("/sys/module/nbd", F_OK)) {
if (cfg->nbds_max || cfg->set_max_part)
cerr << "rbd-nbd: ignoring kernel module parameter options: nbd module already loaded"
<< std::endl;
return 0;
}
if (try_load_module && !loaded_module &&
(cfg->nbds_max || cfg->set_max_part)) {
cerr << "rbd-nbd: ignoring kernel module parameter options: nbd module already loaded"
ret = module_load("nbd", param.str().c_str());
if (ret < 0)
cerr << "rbd-nbd: failed to load nbd kernel module: " << cpp_strerror(-ret)
<< std::endl;
}
return nbd;
return ret;
}
static int check_device_size(int nbd_index, unsigned long expected_size)
@ -644,6 +649,402 @@ static int check_device_size(int nbd_index, unsigned long expected_size)
return 0;
}
static int parse_nbd_index(const std::string& devpath)
{
int index, ret;
ret = sscanf(devpath.c_str(), "/dev/nbd%d", &index);
if (ret <= 0) {
// mean an early matching failure. But some cases need a negative value.
if (ret == 0)
ret = -EINVAL;
cerr << "rbd-nbd: invalid device path: " << devpath
<< " (expected /dev/nbd{num})" << std::endl;
return ret;
}
return index;
}
static int try_ioctl_setup(Config *cfg, int fd, uint64_t size, uint64_t flags)
{
int index = 0, r;
if (cfg->devpath.empty()) {
char dev[64];
const char *path = "/sys/module/nbd/parameters/nbds_max";
int nbds_max = -1;
if (access(path, F_OK) == 0) {
std::ifstream ifs;
ifs.open(path, std::ifstream::in);
if (ifs.is_open()) {
ifs >> nbds_max;
ifs.close();
}
}
while (true) {
snprintf(dev, sizeof(dev), "/dev/nbd%d", index);
nbd = open(dev, O_RDWR);
if (nbd < 0) {
if (nbd == -EPERM && nbds_max != -1 && index < (nbds_max-1)) {
++index;
continue;
}
r = nbd;
cerr << "rbd-nbd: failed to find unused device" << std::endl;
goto done;
}
r = ioctl(nbd, NBD_SET_SOCK, fd);
if (r < 0) {
close(nbd);
++index;
continue;
}
cfg->devpath = dev;
break;
}
} else {
r = parse_nbd_index(cfg->devpath);
if (r < 0)
goto done;
index = r;
nbd = open(cfg->devpath.c_str(), O_RDWR);
if (nbd < 0) {
r = nbd;
cerr << "rbd-nbd: failed to open device: " << cfg->devpath << std::endl;
goto done;
}
r = ioctl(nbd, NBD_SET_SOCK, fd);
if (r < 0) {
r = -errno;
cerr << "rbd-nbd: the device " << cfg->devpath << " is busy" << std::endl;
close(nbd);
goto done;
}
}
r = ioctl(nbd, NBD_SET_BLKSIZE, RBD_NBD_BLKSIZE);
if (r < 0) {
r = -errno;
goto close_nbd;
}
r = ioctl(nbd, NBD_SET_SIZE, size);
if (r < 0) {
r = -errno;
goto close_nbd;
}
ioctl(nbd, NBD_SET_FLAGS, flags);
if (cfg->timeout >= 0) {
r = ioctl(nbd, NBD_SET_TIMEOUT, (unsigned long)cfg->timeout);
if (r < 0) {
r = -errno;
cerr << "rbd-nbd: failed to set timeout: " << cpp_strerror(r)
<< std::endl;
goto close_nbd;
}
}
dout(10) << "ioctl setup complete for " << cfg->devpath << dendl;
nbd_index = index;
return 0;
close_nbd:
if (r < 0) {
ioctl(nbd, NBD_CLEAR_SOCK);
cerr << "rbd-nbd: failed to map, status: " << cpp_strerror(-r) << std::endl;
}
close(nbd);
done:
return r;
}
static void netlink_cleanup(struct nl_sock *sock)
{
if (!sock)
return;
nl_close(sock);
nl_socket_free(sock);
}
static struct nl_sock *netlink_init(int *id)
{
struct nl_sock *sock;
int ret;
sock = nl_socket_alloc();
if (!sock) {
cerr << "rbd-nbd: Could not allocate netlink socket." << std::endl;
return NULL;
}
ret = genl_connect(sock);
if (ret < 0) {
cerr << "rbd-nbd: Could not connect netlink socket. Error " << ret
<< std::endl;
goto free_sock;
}
*id = genl_ctrl_resolve(sock, "nbd");
if (*id < 0)
// nbd netlink interface not supported.
goto close_sock;
return sock;
close_sock:
nl_close(sock);
free_sock:
nl_socket_free(sock);
return NULL;
}
static int netlink_disconnect(int index)
{
struct nl_sock *sock;
struct nl_msg *msg;
int ret, nl_id;
sock = netlink_init(&nl_id);
if (!sock)
// Try ioctl
return 1;
nl_socket_modify_cb(sock, NL_CB_VALID, NL_CB_CUSTOM, genl_handle_msg, NULL);
msg = nlmsg_alloc();
if (!msg) {
cerr << "rbd-nbd: Could not allocate netlink message." << std::endl;
goto free_sock;
}
if (!genlmsg_put(msg, NL_AUTO_PORT, NL_AUTO_SEQ, nl_id, 0, 0,
NBD_CMD_DISCONNECT, 0)) {
cerr << "rbd-nbd: Could not setup message." << std::endl;
goto nla_put_failure;
}
NLA_PUT_U32(msg, NBD_ATTR_INDEX, index);
ret = nl_send_sync(sock, msg);
netlink_cleanup(sock);
if (ret < 0) {
cerr << "rbd-nbd: netlink disconnect failed: " << nl_geterror(-ret)
<< std::endl;
return -EIO;
}
return 0;
nla_put_failure:
nlmsg_free(msg);
free_sock:
netlink_cleanup(sock);
return -EIO;
}
static int netlink_disconnect_by_path(const std::string& devpath)
{
int index;
index = parse_nbd_index(devpath);
if (index < 0)
return index;
return netlink_disconnect(index);
}
static int netlink_connect_cb(struct nl_msg *msg, void *arg)
{
struct genlmsghdr *gnlh = (struct genlmsghdr *)nlmsg_data(nlmsg_hdr(msg));
Config *cfg = (Config *)arg;
struct nlattr *msg_attr[NBD_ATTR_MAX + 1];
uint32_t index;
int ret;
ret = nla_parse(msg_attr, NBD_ATTR_MAX, genlmsg_attrdata(gnlh, 0),
genlmsg_attrlen(gnlh, 0), NULL);
if (ret) {
cerr << "rbd-nbd: Unsupported netlink reply" << std::endl;
return -NLE_MSGTYPE_NOSUPPORT;
}
if (!msg_attr[NBD_ATTR_INDEX]) {
cerr << "rbd-nbd: netlink connect reply missing device index." << std::endl;
return -NLE_MSGTYPE_NOSUPPORT;
}
index = nla_get_u32(msg_attr[NBD_ATTR_INDEX]);
cfg->devpath = "/dev/nbd" + stringify(index);
nbd_index = index;
return NL_OK;
}
static int netlink_connect(Config *cfg, struct nl_sock *sock, int nl_id, int fd,
uint64_t size, uint64_t flags)
{
struct nlattr *sock_attr;
struct nlattr *sock_opt;
struct nl_msg *msg;
int ret;
nl_socket_modify_cb(sock, NL_CB_VALID, NL_CB_CUSTOM, netlink_connect_cb, cfg);
msg = nlmsg_alloc();
if (!msg) {
cerr << "rbd-nbd: Could not allocate netlink message." << std::endl;
return -ENOMEM;
}
if (!genlmsg_put(msg, NL_AUTO_PORT, NL_AUTO_SEQ, nl_id, 0, 0, NBD_CMD_CONNECT,
0)) {
cerr << "rbd-nbd: Could not setup message." << std::endl;
goto free_msg;
}
if (!cfg->devpath.empty()) {
ret = parse_nbd_index(cfg->devpath);
if (ret < 0)
goto free_msg;
NLA_PUT_U32(msg, NBD_ATTR_INDEX, ret);
}
if (cfg->timeout >= 0)
NLA_PUT_U64(msg, NBD_ATTR_TIMEOUT, cfg->timeout);
NLA_PUT_U64(msg, NBD_ATTR_SIZE_BYTES, size);
NLA_PUT_U64(msg, NBD_ATTR_BLOCK_SIZE_BYTES, RBD_NBD_BLKSIZE);
NLA_PUT_U64(msg, NBD_ATTR_SERVER_FLAGS, flags);
sock_attr = nla_nest_start(msg, NBD_ATTR_SOCKETS);
if (!sock_attr) {
cerr << "rbd-nbd: Could not init sockets in netlink message." << std::endl;
goto free_msg;
}
sock_opt = nla_nest_start(msg, NBD_SOCK_ITEM);
if (!sock_opt) {
cerr << "rbd-nbd: Could not init sock in netlink message." << std::endl;
goto free_msg;
}
NLA_PUT_U32(msg, NBD_SOCK_FD, fd);
nla_nest_end(msg, sock_opt);
nla_nest_end(msg, sock_attr);
ret = nl_send_sync(sock, msg);
if (ret < 0) {
cerr << "rbd-nbd: netlink connect failed: " << nl_geterror(ret)
<< std::endl;
return -EIO;
}
dout(10) << "netlink connect complete for " << cfg->devpath << dendl;
return 0;
nla_put_failure:
free_msg:
nlmsg_free(msg);
return -EIO;
}
static int try_netlink_setup(Config *cfg, int fd, uint64_t size, uint64_t flags)
{
struct nl_sock *sock;
int nl_id, ret;
sock = netlink_init(&nl_id);
if (!sock) {
cerr << "rbd-nbd: Netlink interface not supported. Using ioctl interface."
<< std::endl;
return 1;
}
dout(10) << "netlink interface supported." << dendl;
ret = netlink_connect(cfg, sock, nl_id, fd, size, flags);
netlink_cleanup(sock);
if (ret != 0)
return ret;
nbd = open(cfg->devpath.c_str(), O_RDWR);
if (nbd < 0) {
cerr << "rbd-nbd: failed to open device: " << cfg->devpath << std::endl;
return nbd;
}
return 0;
}
static void handle_signal(int signum)
{
int ret;
ceph_assert(signum == SIGINT || signum == SIGTERM);
derr << "*** Got signal " << sig_str(signum) << " ***" << dendl;
if (nbd < 0 || nbd_index < 0) {
dout(20) << __func__ << ": " << "disconnect not needed." << dendl;
return;
}
dout(20) << __func__ << ": " << "sending NBD_DISCONNECT" << dendl;
ret = netlink_disconnect(nbd_index);
if (ret == 1)
ret = ioctl(nbd, NBD_DISCONNECT);
if (ret != 0) {
derr << "rbd-nbd: disconnect failed. Error: " << ret << dendl;
} else {
dout(20) << __func__ << ": " << "disconnected" << dendl;
}
}
static NBDServer *start_server(int fd, librbd::Image& image)
{
NBDServer *server;
server = new NBDServer(fd, image);
server->start();
init_async_signal_handler();
register_async_signal_handler(SIGHUP, sighup_handler);
register_async_signal_handler_oneshot(SIGINT, handle_signal);
register_async_signal_handler_oneshot(SIGTERM, handle_signal);
return server;
}
static void run_server(Preforker& forker, NBDServer *server, bool netlink_used)
{
if (g_conf()->daemonize) {
global_init_postfork_finish(g_ceph_context);
forker.daemonize();
}
if (netlink_used)
server->wait_for_disconnect();
else
ioctl(nbd, NBD_DO_IT);
unregister_async_signal_handler(SIGHUP, sighup_handler);
unregister_async_signal_handler(SIGINT, handle_signal);
unregister_async_signal_handler(SIGTERM, handle_signal);
shutdown_async_signal_handler();
}
static int do_map(int argc, const char *argv[], Config *cfg)
{
int r;
@ -656,13 +1057,14 @@ static int do_map(int argc, const char *argv[], Config *cfg)
int read_only = 0;
unsigned long flags;
unsigned long size;
bool use_netlink;
int index = 0;
int fd[2];
librbd::image_info_t info;
Preforker forker;
NBDServer *server;
vector<const char*> args;
argv_to_vec(argc, argv, args);
@ -741,104 +1143,46 @@ static int do_map(int argc, const char *argv[], Config *cfg)
if (r < 0)
goto close_fd;
if (cfg->devpath.empty()) {
char dev[64];
bool try_load_module = true;
const char *path = "/sys/module/nbd/parameters/nbds_max";
int nbds_max = -1;
if (access(path, F_OK) == 0) {
std::ifstream ifs;
ifs.open(path, std::ifstream::in);
if (ifs.is_open()) {
ifs >> nbds_max;
ifs.close();
}
}
while (true) {
snprintf(dev, sizeof(dev), "/dev/nbd%d", index);
nbd = open_device(dev, cfg, try_load_module);
try_load_module = false;
if (nbd < 0) {
if (nbd == -EPERM && nbds_max != -1 && index < (nbds_max-1)) {
++index;
continue;
}
r = nbd;
cerr << "rbd-nbd: failed to find unused device" << std::endl;
goto close_fd;
}
r = ioctl(nbd, NBD_SET_SOCK, fd[0]);
if (r < 0) {
close(nbd);
++index;
continue;
}
cfg->devpath = dev;
break;
}
} else {
r = sscanf(cfg->devpath.c_str(), "/dev/nbd%d", &index);
if (r <= 0) {
// mean an early matching failure. But some cases need a negative value.
if (r == 0)
r = -EINVAL;
cerr << "rbd-nbd: invalid device path: " << cfg->devpath
<< " (expected /dev/nbd{num})" << std::endl;
goto close_fd;
}
nbd = open_device(cfg->devpath.c_str(), cfg, true);
if (nbd < 0) {
r = nbd;
cerr << "rbd-nbd: failed to open device: " << cfg->devpath << std::endl;
goto close_fd;
}
r = ioctl(nbd, NBD_SET_SOCK, fd[0]);
if (r < 0) {
r = -errno;
cerr << "rbd-nbd: the device " << cfg->devpath << " is busy" << std::endl;
close(nbd);
goto close_fd;
}
}
flags = NBD_FLAG_SEND_FLUSH | NBD_FLAG_SEND_TRIM | NBD_FLAG_HAS_FLAGS;
if (!cfg->snapname.empty() || cfg->readonly) {
flags |= NBD_FLAG_READ_ONLY;
read_only = 1;
}
r = ioctl(nbd, NBD_SET_BLKSIZE, RBD_NBD_BLKSIZE);
if (r < 0) {
r = -errno;
goto close_nbd;
}
if (info.size > ULONG_MAX) {
r = -EFBIG;
cerr << "rbd-nbd: image is too large (" << byte_u_t(info.size)
<< ", max is " << byte_u_t(ULONG_MAX) << ")" << std::endl;
goto close_nbd;
goto close_fd;
}
size = info.size;
r = ioctl(nbd, NBD_SET_SIZE, size);
if (r < 0) {
r = -errno;
goto close_nbd;
r = load_module(cfg);
if (r < 0)
goto close_fd;
server = start_server(fd[1], image);
use_netlink = cfg->try_netlink;
if (use_netlink) {
r = try_netlink_setup(cfg, fd[0], size, flags);
if (r < 0) {
goto free_server;
} else if (r == 1) {
use_netlink = false;
}
}
r = check_device_size(index, size);
if (r < 0) {
goto close_nbd;
if (!use_netlink) {
r = try_ioctl_setup(cfg, fd[0], size, flags);
if (r < 0)
goto free_server;
}
ioctl(nbd, NBD_SET_FLAGS, flags);
r = check_device_size(nbd_index, size);
if (r < 0)
goto close_nbd;
r = ioctl(nbd, BLKROSET, (unsigned long) &read_only);
if (r < 0) {
@ -846,16 +1190,6 @@ static int do_map(int argc, const char *argv[], Config *cfg)
goto close_nbd;
}
if (cfg->timeout >= 0) {
r = ioctl(nbd, NBD_SET_TIMEOUT, (unsigned long)cfg->timeout);
if (r < 0) {
r = -errno;
cerr << "rbd-nbd: failed to set timeout: " << cpp_strerror(r)
<< std::endl;
goto close_nbd;
}
}
{
uint64_t handle;
@ -866,28 +1200,7 @@ static int do_map(int argc, const char *argv[], Config *cfg)
cout << cfg->devpath << std::endl;
if (g_conf()->daemonize) {
global_init_postfork_finish(g_ceph_context);
forker.daemonize();
}
{
NBDServer server(fd[1], image);
server.start();
init_async_signal_handler();
register_async_signal_handler(SIGHUP, sighup_handler);
register_async_signal_handler_oneshot(SIGINT, handle_signal);
register_async_signal_handler_oneshot(SIGTERM, handle_signal);
ioctl(nbd, NBD_DO_IT);
unregister_async_signal_handler(SIGHUP, sighup_handler);
unregister_async_signal_handler(SIGINT, handle_signal);
unregister_async_signal_handler(SIGTERM, handle_signal);
shutdown_async_signal_handler();
}
run_server(forker, server, use_netlink);
r = image.update_unwatch(handle);
ceph_assert(r == 0);
@ -895,10 +1208,17 @@ static int do_map(int argc, const char *argv[], Config *cfg)
close_nbd:
if (r < 0) {
ioctl(nbd, NBD_CLEAR_SOCK);
cerr << "rbd-nbd: failed to map, status: " << cpp_strerror(-r) << std::endl;
if (use_netlink) {
netlink_disconnect(nbd_index);
} else {
ioctl(nbd, NBD_CLEAR_SOCK);
cerr << "rbd-nbd: failed to map, status: " << cpp_strerror(-r)
<< std::endl;
}
}
close(nbd);
free_server:
delete server;
close_fd:
close(fd[0]);
close(fd[1]);
@ -912,23 +1232,30 @@ close_ret:
return r;
}
static int do_unmap(const std::string &devpath)
static int do_unmap(Config *cfg)
{
int r = 0;
int r, nbd;
int nbd = open_device(devpath.c_str());
/*
* The netlink disconnect call supports devices setup with netlink or ioctl,
* so we always try that first.
*/
r = netlink_disconnect_by_path(cfg->devpath);
if (r != 1)
return r;
nbd = open(cfg->devpath.c_str(), O_RDWR);
if (nbd < 0) {
cerr << "rbd-nbd: failed to open device: " << devpath << std::endl;
cerr << "rbd-nbd: failed to open device: " << cfg->devpath << std::endl;
return nbd;
}
r = ioctl(nbd, NBD_DISCONNECT);
if (r < 0) {
cerr << "rbd-nbd: the device is not used" << std::endl;
cerr << "rbd-nbd: the device is not used" << std::endl;
}
close(nbd);
return r;
}
@ -1097,6 +1424,8 @@ static int parse_args(vector<const char*>& args, std::ostream *err_msg,
(char *)NULL)) {
} else if (ceph_argparse_flag(args, i, "--pretty-format", (char *)NULL)) {
cfg->pretty_format = true;
} else if (ceph_argparse_flag(args, i, "--try-netlink", (char *)NULL)) {
cfg->try_netlink = true;
} else {
++i;
}
@ -1197,7 +1526,7 @@ static int rbd_nbd(int argc, const char *argv[])
return -EINVAL;
break;
case Disconnect:
r = do_unmap(cfg.devpath);
r = do_unmap(&cfg);
if (r < 0)
return -EINVAL;
break;