fix #5794: cluster join: check that the interfaces of the links are up

Before joining a cluster, the IPs of the corosync links are checked to
be configured on the node. An IP on an interface without carrier, for
example with an unplugged cable, still counts as configured, so the
join is started even though corosync cannot reach the other nodes.
The join then fails halfway, and the node stays without quorum until
the link comes up, which looks like a split brain.

Additionally check the operational state of the interface the IP is
configured on and refuse to join if it is not up. Interfaces that
report their state as unknown, like dummy interfaces, are treated as
up. Like the other checks, this can be overridden with 'force'.

Signed-off-by: Michal Fox <me@dualfroz.com>

src/PVE/Cluster/Setup.pm

@@ -6,6 +6,7 @@ use warnings;
 use Digest::HMAC_SHA1;
 use Digest::SHA;
 use IO::File;
+use JSON;
 use MIME::Base64;
 use Net::IP;
 use UUID;
@@ -661,8 +662,23 @@ sub assert_joinable {
         my $cidr = (Net::IP::ip_is_ipv6($ip)) ? "$ip/128" : "$ip/32";
         my $configured_ips = PVE::Network::get_local_ip_from_cidr($cidr);
 
-        $error->("$logid: cannot use IP '$ip', not found on local node!\n")
-            if scalar(@$configured_ips) < 1;
+        if (scalar(@$configured_ips) < 1) {
+            $error->("$logid: cannot use IP '$ip', not found on local node!\n");
+            return;
+        }
+
+        # the IP is also listed if the interface has no carrier, but corosync cannot use it then
+        my $addresses = '';
+        PVE::Tools::run_command(
+            ['/sbin/ip', '-json', 'address', 'show', 'to', $cidr, 'up'],
+            outfunc => sub { $addresses .= shift },
+        );
+        for my $iface (decode_json($addresses)->@*) {
+            my $name = $iface->{ifname} or next;
+            my $state = $iface->{operstate} // 'UNKNOWN';
+            next if $state eq 'UP' || $state eq 'UNKNOWN';
+            $error->("$logid: cannot use IP '$ip', interface '$name' is not up ($state)\n");
+        }
     };
 
     $check_ip->($local_addr, 'local node address');