RepoMirrors/ceph
1
0
Fork 0
mirror of https://github.com/ceph/ceph synced 2025-01-18 09:02:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

pick_address: Warn and continue when you find at least 1 IPv4 or IPv6 address

Browse Source 
Currently if specify a single public or cluster network, yet have both
`ms bind ipv4` and `ms bind ipv6` set daemons crash when they can't find
both IPs from the same network:

    unable to find any IPv4 address in networks '2001:db8:11d::/120' interfaces ''

And rightly so, of course it can't find an IPv4 network in an IPv6
network.
This patch, adds a new helper method, networks_address_family_coverage,
that takes the list of networks and returns a bitmap of address families
supported.
We then check to see if we have enough networks defined and if you don't
it'll warn and then continue.

Also update the network-config-ref to mention having to define both
address family addresses for cluster and or public networks.

As well as a warning about `ms bind ipv4` being enabled by default which
is easy to miss, there by enabling dual stack when you may only be
expect single stack IPv6.

Thee is also a drive by to fix a `note` that wan't being displayed due
to missing RST syntax.

Signed-off-by: Matthew Oliver <moliver@suse.com>
Fixes: https://tracker.ceph.com/issues/46845
Fixes: https://tracker.ceph.com/issues/39711
This commit is contained in:
Matthew Oliver 2020-08-10 04:46:21 +00:00
parent 6447e20f78
commit 9f75dfbf36
5 changed files with 202 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
doc/rados/configuration/msgr2.rst
View File

@ -88,7 +88,7 @@ Similarly, two options control whether IPv4 and IPv6 addresses are used:
* ``ms_bind_ipv6`` [default: false] controls whether a daemon binds
to an IPv6 address
.. note: The ability to bind to multiple ports has paved the way for
.. note:: The ability to bind to multiple ports has paved the way for
dual-stack IPv4 and IPv6 support. That said, dual-stack support is
not yet tested as of Nautilus v14.2.0 and likely needs some
additional code changes to work correctly.

30
doc/rados/configuration/network-config-ref.rst
View File

@ -201,6 +201,27 @@ following option to the ``[global]`` section of your Ceph configuration file.
We prefer that the cluster network is **NOT** reachable from the public network
or the Internet for added security.
IPv4/IPv6 Dual Stack Mode
-------------------------
If you want to run in an IPv4/IPv6 dual stack mode and want to define your public and/or
cluster networks, then you need to specify both your IPv4 and IPv6 networks for each:
.. code-block:: ini
[global]
# ... elided configuration
public network = {IPv4 public-network/netmask}, {IPv6 public-network/netmask}
This is so ceph can find a valid IP address for both address families.
If you want just an IPv4 or an IPv6 stack environment, then make sure you set the `ms bind`
options correctly.
.. note::
Binding to IPv4 is enabled by default, so if you just add the option to bind to IPv6
you'll actually put yourself into dual stack mode. If you want just IPv6, then disable IPv4 and
enable IPv6. See `Bind`_ below.
Ceph Daemons
============
@ -336,11 +357,16 @@ addresses.
:Default: ``7300``
:Required: No.
``ms bind ipv4``
:Description: Enables Ceph daemons to bind to IPv4 addresses.
:Type: Boolean
:Default: ``true``
:Required: No
``ms bind ipv6``
:Description: Enables Ceph daemons to bind to IPv6 addresses. Currently the
messenger *either* uses IPv4 or IPv6, but it cannot do both.
:Description: Enables Ceph daemons to bind to IPv6 addresses.
:Type: Boolean
:Default: ``false``
:Required: No

32
src/common/pick_address.cc
View File

@ -285,6 +285,32 @@ static int fill_in_one_address(
return 0;
}
unsigned networks_address_family_coverage(CephContext *cct, const std::string &networks) {
std::list<string> nets;
get_str_list(networks, nets);
unsigned found_ipv = 0;
for (auto& s : nets) {
struct sockaddr_storage net;
unsigned prefix_len;
if (!parse_network(s.c_str(), &net, &prefix_len)) {
lderr(cct) << "unable to parse network: " << s << dendl;
exit(1);
}
switch (net.ss_family) {
case AF_INET:
found_ipv |= CEPH_PICK_ADDRESS_IPV4;
break;
case AF_INET6:
found_ipv |= CEPH_PICK_ADDRESS_IPV6;
break;
}
}
return found_ipv;
}
int pick_addresses(
CephContext *cct,
unsigned flags,
@ -358,6 +384,7 @@ int pick_addresses(
!networks.empty()) {
int ipv4_r = !(ipv & CEPH_PICK_ADDRESS_IPV4) ? 0 : -1;
int ipv6_r = !(ipv & CEPH_PICK_ADDRESS_IPV6) ? 0 : -1;
unsigned found_ipv = networks_address_family_coverage(cct, networks);
// first try on preferred numa node (if >= 0), then anywhere.
while (true) {
// note: pass in ipv to filter the matching addresses
@ -378,6 +405,11 @@ int pick_addresses(
networks, interfaces, addrs,
preferred_numa_node);
}
if (found_ipv != 0 && (found_ipv & ipv != ipv)) {
lderr(cct) << "An IP address was found, but not enough networks to cover both address families. "
<< "An IPv4 and IPv6 network is required for dual stack. Continuing with one stack" << dendl;
break;
}
if (ipv4_r >= 0 && ipv6_r >= 0) {
break;
}

7
src/common/pick_address.h
View File

@ -80,4 +80,11 @@ int get_iface_numa_node(
const std::string& iface,
int *node);
/**
* Return a bitmap of address families that are covered by networks
*
* @param cct context (used for logging)
* @param string of networks
*/
unsigned networks_address_family_coverage(CephContext *cct, const std::string &networks);
#endif

163
src/test/test_ipaddr.cc
View File

@ -863,6 +863,119 @@ TEST(pick_address, filtering)
}
TEST(pick_address, ipv4_ipv6_enabled)
{
struct ifaddrs one, two;
struct sockaddr_in a_one;
struct sockaddr_in6 a_two;
one.ifa_next = &two;
one.ifa_addr = (struct sockaddr*)&a_one;
one.ifa_name = eth0;
two.ifa_next = NULL;
two.ifa_addr = (struct sockaddr*)&a_two;
two.ifa_name = eth0;
ipv4(&a_one, "10.1.1.2");
ipv6(&a_two, "2001:1234:5678:90ab::cdef");
CephContext *cct = new CephContext(CEPH_ENTITY_TYPE_OSD);
cct->_conf._clear_safe_to_start_threads(); // so we can set configs
cct->_conf.set_val("public_addr", "");
cct->_conf.set_val("public_network", "10.1.1.0/24, 2001::/16");
cct->_conf.set_val("public_network_interface", "");
cct->_conf.set_val("cluster_addr", "");
cct->_conf.set_val("cluster_network", "");
cct->_conf.set_val("cluster_network_interface", "");
cct->_conf.set_val("ms_bind_ipv6", "true");
entity_addrvec_t av;
{
int r = pick_addresses(cct,
CEPH_PICK_ADDRESS_PUBLIC |
CEPH_PICK_ADDRESS_MSGR1,
&one, &av);
cout << av << std::endl;
ASSERT_EQ(0, r);
// Got 2 address
ASSERT_EQ(2u, av.v.size());
ASSERT_EQ(string("v1:[2001:1234:5678:90ab::cdef]:0/0"), stringify(av.v[0]));
ASSERT_EQ(string("v1:10.1.1.2:0/0"), stringify(av.v[1]));
}
}
TEST(pick_address, only_ipv6_enabled)
{
struct ifaddrs one;
struct sockaddr_in6 a_one;
one.ifa_next = NULL;
one.ifa_addr = (struct sockaddr*)&a_one;
one.ifa_name = eth0;
ipv6(&a_one, "2001:1234:5678:90ab::cdef");
CephContext *cct = new CephContext(CEPH_ENTITY_TYPE_OSD);
cct->_conf._clear_safe_to_start_threads(); // so we can set configs
cct->_conf.set_val("public_addr", "");
cct->_conf.set_val("public_network", "2001::/16");
cct->_conf.set_val("public_network_interface", "");
cct->_conf.set_val("cluster_addr", "");
cct->_conf.set_val("cluster_network", "");
cct->_conf.set_val("cluster_network_interface", "");
cct->_conf.set_val("ms_bind_ipv6", "true");
cct->_conf.set_val("ms_bind_ipv4", "false");
entity_addrvec_t av;
{
int r = pick_addresses(cct,
CEPH_PICK_ADDRESS_PUBLIC |
CEPH_PICK_ADDRESS_MSGR1,
&one, &av);
cout << av << std::endl;
ASSERT_EQ(0, r);
ASSERT_EQ(1u, av.v.size());
ASSERT_EQ(string("v1:[2001:1234:5678:90ab::cdef]:0/0"), stringify(av.v[0]));
}
}
TEST(pick_address, only_ipv4_enabled)
{
struct ifaddrs one;
struct sockaddr_in a_one;
one.ifa_next = NULL;
one.ifa_addr = (struct sockaddr*)&a_one;
one.ifa_name = eth0;
ipv4(&a_one, "10.1.1.2");
CephContext *cct = new CephContext(CEPH_ENTITY_TYPE_OSD);
cct->_conf._clear_safe_to_start_threads(); // so we can set configs
cct->_conf.set_val("public_addr", "");
cct->_conf.set_val("public_network", "10.1.1.0/24");
cct->_conf.set_val("public_network_interface", "");
cct->_conf.set_val("cluster_addr", "");
cct->_conf.set_val("cluster_network", "");
cct->_conf.set_val("cluster_network_interface", "");
entity_addrvec_t av;
{
int r = pick_addresses(cct,
CEPH_PICK_ADDRESS_PUBLIC |
CEPH_PICK_ADDRESS_MSGR1,
&one, &av);
cout << av << std::endl;
ASSERT_EQ(0, r);
ASSERT_EQ(1u, av.v.size());
ASSERT_EQ(string("v1:10.1.1.2:0/0"), stringify(av.v[0]));
}
}
TEST(pick_address, ipv4_ipv6_enabled_not_enough_networks)
{
struct ifaddrs one;
struct sockaddr_in a_one;
@ -892,39 +1005,31 @@ TEST(pick_address, ipv4_ipv6_enabled)
&one, &av);
cout << av << std::endl;
ASSERT_EQ(-1, r);
ASSERT_EQ(1u, av.v.size());
ASSERT_EQ(string("v2:10.1.1.2:0/0"), stringify(av.v[0]));
}
}
TEST(pick_address, ipv4_ipv6_enabled2)
TEST(networks_address_family_coverage, just_ipv4)
{
struct ifaddrs one;
struct sockaddr_in6 a_one;
one.ifa_next = NULL;
one.ifa_addr = (struct sockaddr*)&a_one;
one.ifa_name = eth0;
ipv6(&a_one, "2001:1234:5678:90ab::cdef");
CephContext *cct = new CephContext(CEPH_ENTITY_TYPE_OSD);
cct->_conf._clear_safe_to_start_threads(); // so we can set configs
cct->_conf.set_val("public_addr", "");
cct->_conf.set_val("public_network", "2001::/16");
cct->_conf.set_val("public_network_interface", "");
cct->_conf.set_val("cluster_addr", "");
cct->_conf.set_val("cluster_network", "");
cct->_conf.set_val("cluster_network_interface", "");
cct->_conf.set_val("ms_bind_ipv6", "true");
entity_addrvec_t av;
{
int r = pick_addresses(cct,
CEPH_PICK_ADDRESS_PUBLIC |
CEPH_PICK_ADDRESS_MSGR1,
&one, &av);
cout << av << std::endl;
ASSERT_EQ(-1, r);
}
std::string networks = "10.0.0.0/24";
unsigned r = networks_address_family_coverage(cct, networks);
ASSERT_EQ(CEPH_PICK_ADDRESS_IPV4, r);
}
TEST(networks_address_family_coverage, just_ipv6)
{
CephContext *cct = new CephContext(CEPH_ENTITY_TYPE_OSD);
std::string networks = "2001::/16";
unsigned r = networks_address_family_coverage(cct, networks);
ASSERT_EQ(CEPH_PICK_ADDRESS_IPV6, r);
}
TEST(networks_address_family_coverage, ipv6_and_ipv4)
{
CephContext *cct = new CephContext(CEPH_ENTITY_TYPE_OSD);
std::string networks = "2001::/16, 10.0.0.0/16";
unsigned r = networks_address_family_coverage(cct, networks);
ASSERT_EQ(CEPH_PICK_ADDRESS_IPV4 | CEPH_PICK_ADDRESS_IPV6, r);
}