Upstream proxmox-git/pve-cluster state of the touched files before 8cdf8f53fe
This commit is contained in:
upstream committed
1 parent
4298a304e1
commit
39dd92ebb6
1 file changed
+856
@@ -0,0 +1,856 @@
|
||||
package PVE::Cluster::Setup;
|
||||
|
||||
use strict;
|
||||
use warnings;
|
||||
|
||||
use Digest::HMAC_SHA1;
|
||||
use Digest::SHA;
|
||||
use IO::File;
|
||||
use MIME::Base64;
|
||||
use Net::IP;
|
||||
use UUID;
|
||||
use POSIX qw(EEXIST);
|
||||
|
||||
use PVE::APIClient::LWP;
|
||||
use PVE::Cluster;
|
||||
use PVE::Corosync;
|
||||
use PVE::INotify;
|
||||
use PVE::JSONSchema;
|
||||
use PVE::Network;
|
||||
use PVE::Tools;
|
||||
use PVE::Certificate;
|
||||
|
||||
# relevant for joining or getting joined checks, after that versions in cluster can differ
|
||||
use constant JOIN_API_VERSION => 1;
|
||||
# (APIVER-this) is oldest version a new node in addnode can have and still be accepted
|
||||
use constant JOIN_API_AGE_AS_CLUSTER => 1;
|
||||
# (APIVER-this) is oldest version a cluster node can have to still try joining
|
||||
use constant JOIN_API_AGE_AS_JOINEE => 1;
|
||||
|
||||
sub assert_we_can_join_cluster_version {
|
||||
my ($version) = @_;
|
||||
my $min_version = JOIN_API_VERSION - JOIN_API_AGE_AS_JOINEE;
|
||||
return if $version >= $min_version;
|
||||
die "error: incompatible join API version on cluster ($version), local node" . " has "
|
||||
. JOIN_API_VERSION
|
||||
. " and supports >= $min_version. Make sure"
|
||||
. "all cluster nodes are up-to-date.\n";
|
||||
}
|
||||
|
||||
sub assert_node_can_join_our_version {
|
||||
my ($version) = @_;
|
||||
my $min_version = JOIN_API_VERSION - JOIN_API_AGE_AS_CLUSTER;
|
||||
return if $version >= $min_version;
|
||||
die "error: unsupported old API version on joining node ($version), cluster"
|
||||
. " node has "
|
||||
. JOIN_API_VERSION
|
||||
. " and supports >= $min_version. Please"
|
||||
. " upgrade node before joining\n";
|
||||
}
|
||||
|
||||
my $pmxcfs_base_dir = PVE::Cluster::base_dir();
|
||||
my $pmxcfs_auth_dir = PVE::Cluster::auth_dir();
|
||||
|
||||
# only write output if something fails
|
||||
sub run_silent_cmd {
|
||||
my ($cmd) = @_;
|
||||
|
||||
my $outbuf = '';
|
||||
my $record = sub { $outbuf .= shift . "\n"; };
|
||||
|
||||
eval { PVE::Tools::run_command($cmd, outfunc => $record, errfunc => $record) };
|
||||
|
||||
if (my $err = $@) {
|
||||
print STDERR $outbuf;
|
||||
die $err;
|
||||
}
|
||||
}
|
||||
|
||||
# Corosync related files
|
||||
my $localclusterdir = "/etc/corosync";
|
||||
my $localclusterconf = "$localclusterdir/corosync.conf";
|
||||
my $authfile = "$localclusterdir/authkey";
|
||||
my $clusterconf = "$pmxcfs_base_dir/corosync.conf";
|
||||
|
||||
# CA/certificate related files
|
||||
my $pveca_key_fn = "$pmxcfs_auth_dir/pve-root-ca.key";
|
||||
my $pveca_srl_fn = "$pmxcfs_auth_dir/pve-root-ca.srl";
|
||||
my $pveca_cert_fn = "$pmxcfs_base_dir/pve-root-ca.pem";
|
||||
# this is just a secret accessable by the web browser
|
||||
# and is used for CSRF prevention
|
||||
my $pvewww_key_fn = "$pmxcfs_base_dir/pve-www.key";
|
||||
|
||||
# ssh related files
|
||||
my $ssh_host_rsa_id = "/etc/ssh/ssh_host_rsa_key.pub";
|
||||
my $ssh_cluster_known_hosts = "$pmxcfs_auth_dir/known_hosts";
|
||||
my $ssh_cluster_authorized_keys = "$pmxcfs_auth_dir/authorized_keys";
|
||||
my $ssh_system_known_hosts = "/etc/ssh/ssh_known_hosts";
|
||||
my $ssh_system_server_config = "/etc/ssh/sshd_config";
|
||||
|
||||
my $ssh_root_rsa_key_private = "/root/.ssh/id_rsa";
|
||||
my $ssh_root_rsa_key_public = "/root/.ssh/id_rsa.pub";
|
||||
my $ssh_root_authorized_keys = "/root/.ssh/authorized_keys";
|
||||
my $ssh_root_authorized_keys_backup = "${ssh_root_authorized_keys}.org";
|
||||
my $ssh_root_client_config = "/root/.ssh/config";
|
||||
|
||||
# ssh related utility functions
|
||||
|
||||
sub ssh_merge_keys {
|
||||
# remove duplicate keys in $ssh_cluster_authorized_keys
|
||||
# ssh-copy-id simply add keys, so the file can grow to large
|
||||
|
||||
my $data = '';
|
||||
if (-f $ssh_cluster_authorized_keys) {
|
||||
$data = PVE::Tools::file_get_contents($ssh_cluster_authorized_keys);
|
||||
chomp($data);
|
||||
}
|
||||
|
||||
my $found_backup;
|
||||
if (-f $ssh_root_authorized_keys_backup) {
|
||||
$data .= "\n";
|
||||
$data .= PVE::Tools::file_get_contents($ssh_root_authorized_keys_backup);
|
||||
chomp($data);
|
||||
$found_backup = 1;
|
||||
}
|
||||
|
||||
# always add ourself
|
||||
if (-f $ssh_root_rsa_key_public) {
|
||||
my $pub = PVE::Tools::file_get_contents($ssh_root_rsa_key_public);
|
||||
chomp($pub);
|
||||
$data .= "\n$pub\n";
|
||||
}
|
||||
|
||||
my $newdata = "";
|
||||
my $vhash = {};
|
||||
my @lines = split(/\n/, $data);
|
||||
foreach my $line (@lines) {
|
||||
if ($line !~ /^#/ && $line =~ m/(^|\s)ssh-(rsa|dsa)\s+(\S+)\s+\S+$/) {
|
||||
next if $vhash->{$3}++;
|
||||
}
|
||||
$newdata .= "$line\n";
|
||||
}
|
||||
|
||||
PVE::Tools::file_set_contents($ssh_cluster_authorized_keys, $newdata, 0600);
|
||||
|
||||
if ($found_backup && -l $ssh_root_authorized_keys) {
|
||||
# everything went well, so we can remove the backup
|
||||
unlink $ssh_root_authorized_keys_backup;
|
||||
}
|
||||
}
|
||||
|
||||
sub setup_sshd_config {
|
||||
my () = @_;
|
||||
|
||||
my $conf = PVE::Tools::file_get_contents($ssh_system_server_config);
|
||||
|
||||
return if $conf =~ m/^PermitRootLogin\s+yes\s*$/m;
|
||||
|
||||
if ($conf !~ s/^#?PermitRootLogin.*$/PermitRootLogin yes/m) {
|
||||
chomp $conf;
|
||||
$conf .= "\nPermitRootLogin yes\n";
|
||||
}
|
||||
|
||||
PVE::Tools::file_set_contents($ssh_system_server_config, $conf);
|
||||
|
||||
PVE::Tools::run_command(['systemctl', 'reload-or-restart', 'sshd']);
|
||||
}
|
||||
|
||||
sub setup_rootsshconfig {
|
||||
|
||||
# create ssh key if it does not exist
|
||||
if (!-f $ssh_root_rsa_key_public) {
|
||||
mkdir '/root/.ssh/';
|
||||
system("echo|ssh-keygen -t rsa -N '' -b 4096 -f ${ssh_root_rsa_key_private}");
|
||||
}
|
||||
|
||||
# create ssh config if it does not exist
|
||||
if (!-f $ssh_root_client_config) {
|
||||
mkdir '/root/.ssh';
|
||||
if (my $fh = IO::File->new($ssh_root_client_config, O_CREAT | O_WRONLY | O_EXCL, 0640)) {
|
||||
# this is the default ciphers list from Debian's OpenSSH package (OpenSSH_7.4p1 Debian-10, OpenSSL 1.0.2k 26 Jan 2017)
|
||||
# changed order to put AES before Chacha20 (most hardware has AESNI)
|
||||
print $fh
|
||||
"Ciphers aes128-ctr,aes192-ctr,aes256-ctr,aes128-gcm\@openssh.com,aes256-gcm\@openssh.com,chacha20-poly1305\@openssh.com\n";
|
||||
close($fh);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sub setup_ssh_keys {
|
||||
|
||||
mkdir $pmxcfs_auth_dir;
|
||||
|
||||
my $import_ok;
|
||||
|
||||
if (!-f $ssh_cluster_authorized_keys) {
|
||||
my $old;
|
||||
if (-f $ssh_root_authorized_keys) {
|
||||
$old = PVE::Tools::file_get_contents($ssh_root_authorized_keys);
|
||||
}
|
||||
if (
|
||||
my $fh = IO::File->new($ssh_cluster_authorized_keys, O_CREAT | O_WRONLY | O_EXCL, 0400)
|
||||
) {
|
||||
PVE::Tools::safe_print($ssh_cluster_authorized_keys, $fh, $old) if $old;
|
||||
close($fh);
|
||||
$import_ok = 1;
|
||||
}
|
||||
}
|
||||
|
||||
warn "can't create shared ssh key database '$ssh_cluster_authorized_keys'\n"
|
||||
if !-f $ssh_cluster_authorized_keys;
|
||||
|
||||
if (-f $ssh_root_authorized_keys && !-l $ssh_root_authorized_keys) {
|
||||
if (!rename($ssh_root_authorized_keys, $ssh_root_authorized_keys_backup)) {
|
||||
warn "rename $ssh_root_authorized_keys failed - $!\n";
|
||||
}
|
||||
}
|
||||
|
||||
if (!-l $ssh_root_authorized_keys) {
|
||||
symlink $ssh_cluster_authorized_keys, $ssh_root_authorized_keys;
|
||||
}
|
||||
|
||||
if (!-l $ssh_root_authorized_keys) {
|
||||
warn
|
||||
"can't create symlink for ssh keys '$ssh_root_authorized_keys' -> '$ssh_cluster_authorized_keys'\n";
|
||||
} else {
|
||||
unlink $ssh_root_authorized_keys_backup if $import_ok;
|
||||
}
|
||||
}
|
||||
|
||||
sub ssh_unmerge_known_hosts {
|
||||
return if !-l $ssh_system_known_hosts;
|
||||
|
||||
my $old = '';
|
||||
$old = PVE::Tools::file_get_contents($ssh_cluster_known_hosts)
|
||||
if -f $ssh_cluster_known_hosts;
|
||||
|
||||
PVE::Tools::file_set_contents($ssh_system_known_hosts, $old);
|
||||
}
|
||||
|
||||
sub ssh_create_node_known_hosts {
|
||||
my ($nodename) = @_;
|
||||
|
||||
my $hostkey = PVE::Tools::file_get_contents($ssh_host_rsa_id);
|
||||
# Note: file sometimes containe empty lines at start, so we use multiline match
|
||||
die "can't parse $ssh_host_rsa_id" if $hostkey !~ m/^(ssh-rsa\s\S+)(\s.*)?$/m;
|
||||
$hostkey = $1;
|
||||
|
||||
my $raw = "$nodename $hostkey";
|
||||
PVE::Tools::file_set_contents("/etc/pve/nodes/$nodename/ssh_known_hosts", $raw);
|
||||
|
||||
# TODO: also setup custom keypair and client config here to disentangle entirely from /root/.ssh?
|
||||
}
|
||||
|
||||
sub ssh_merge_known_hosts {
|
||||
my ($nodename, $ip_address, $createLink) = @_;
|
||||
|
||||
die "no node name specified" if !$nodename;
|
||||
die "no ip address specified" if !$ip_address;
|
||||
|
||||
# ssh lowercases hostnames (aliases) before comparision, so we need too
|
||||
$nodename = lc($nodename);
|
||||
$ip_address = lc($ip_address);
|
||||
|
||||
mkdir $pmxcfs_auth_dir;
|
||||
|
||||
if (!-f $ssh_cluster_known_hosts) {
|
||||
if (my $fh = IO::File->new($ssh_cluster_known_hosts, O_CREAT | O_WRONLY | O_EXCL, 0600)) {
|
||||
close($fh);
|
||||
}
|
||||
}
|
||||
|
||||
my $old = PVE::Tools::file_get_contents($ssh_cluster_known_hosts);
|
||||
|
||||
my $new = '';
|
||||
|
||||
if ((!-l $ssh_system_known_hosts) && (-f $ssh_system_known_hosts)) {
|
||||
$new = PVE::Tools::file_get_contents($ssh_system_known_hosts);
|
||||
}
|
||||
|
||||
my $hostkey = PVE::Tools::file_get_contents($ssh_host_rsa_id);
|
||||
# Note: file sometimes containe emty lines at start, so we use multiline match
|
||||
die "can't parse $ssh_host_rsa_id" if $hostkey !~ m/^(ssh-rsa\s\S+)(\s.*)?$/m;
|
||||
$hostkey = $1;
|
||||
|
||||
my $data = '';
|
||||
my $vhash = {};
|
||||
|
||||
my $found_nodename;
|
||||
my $found_local_ip;
|
||||
|
||||
my $merge_line = sub {
|
||||
my ($line, $all) = @_;
|
||||
|
||||
return if $line =~ m/^\s*$/; # skip empty lines
|
||||
return if $line =~ m/^#/; # skip comments
|
||||
|
||||
if ($line =~ m/^(\S+)\s(ssh-rsa\s\S+)(\s.*)?$/) {
|
||||
my $key = $1;
|
||||
my $rsakey = $2;
|
||||
if (!$vhash->{$key}) {
|
||||
$vhash->{$key} = 1;
|
||||
if ($key =~ m/\|1\|([^\|\s]+)\|([^\|\s]+)$/) {
|
||||
my $salt = decode_base64($1);
|
||||
my $digest = $2;
|
||||
my $hmac = Digest::HMAC_SHA1->new($salt);
|
||||
$hmac->add($nodename);
|
||||
my $hd = $hmac->b64digest . '=';
|
||||
if ($digest eq $hd) {
|
||||
if ($rsakey eq $hostkey) {
|
||||
$found_nodename = 1;
|
||||
$data .= $line;
|
||||
}
|
||||
return;
|
||||
}
|
||||
$hmac = Digest::HMAC_SHA1->new($salt);
|
||||
$hmac->add($ip_address);
|
||||
$hd = $hmac->b64digest . '=';
|
||||
if ($digest eq $hd) {
|
||||
if ($rsakey eq $hostkey) {
|
||||
$found_local_ip = 1;
|
||||
$data .= $line;
|
||||
}
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
$key = lc($key); # avoid duplicate entries, ssh compares lowercased
|
||||
if ($key eq $ip_address) {
|
||||
$found_local_ip = 1 if $rsakey eq $hostkey;
|
||||
} elsif ($key eq $nodename) {
|
||||
$found_nodename = 1 if $rsakey eq $hostkey;
|
||||
}
|
||||
}
|
||||
$data .= $line;
|
||||
}
|
||||
} elsif ($all) {
|
||||
$data .= $line;
|
||||
}
|
||||
};
|
||||
|
||||
while ($old && $old =~ s/^((.*?)(\n|$))//) {
|
||||
my $line = "$2\n";
|
||||
&$merge_line($line, 1);
|
||||
}
|
||||
|
||||
while ($new && $new =~ s/^((.*?)(\n|$))//) {
|
||||
my $line = "$2\n";
|
||||
&$merge_line($line);
|
||||
}
|
||||
|
||||
# add our own key if not already there
|
||||
$data .= "$nodename $hostkey\n" if !$found_nodename;
|
||||
$data .= "$ip_address $hostkey\n" if !$found_local_ip;
|
||||
|
||||
PVE::Tools::file_set_contents($ssh_cluster_known_hosts, $data);
|
||||
|
||||
return if !$createLink;
|
||||
|
||||
unlink $ssh_system_known_hosts;
|
||||
symlink $ssh_cluster_known_hosts, $ssh_system_known_hosts;
|
||||
|
||||
warn
|
||||
"can't create symlink for ssh known hosts '$ssh_system_known_hosts' -> '$ssh_cluster_known_hosts'\n"
|
||||
if !-l $ssh_system_known_hosts;
|
||||
|
||||
}
|
||||
|
||||
# directory and file creation
|
||||
|
||||
sub gen_local_dirs {
|
||||
my ($nodename) = @_;
|
||||
|
||||
PVE::Cluster::check_cfs_is_mounted();
|
||||
|
||||
my @required_dirs = (
|
||||
"$pmxcfs_base_dir/priv",
|
||||
"$pmxcfs_base_dir/nodes",
|
||||
"$pmxcfs_base_dir/nodes/$nodename",
|
||||
"$pmxcfs_base_dir/nodes/$nodename/lxc",
|
||||
"$pmxcfs_base_dir/nodes/$nodename/qemu-server",
|
||||
"$pmxcfs_base_dir/nodes/$nodename/openvz",
|
||||
"$pmxcfs_base_dir/nodes/$nodename/priv",
|
||||
);
|
||||
|
||||
foreach my $dir (@required_dirs) {
|
||||
if (!-d $dir) {
|
||||
mkdir($dir) || $! == EEXIST || die "unable to create directory '$dir' - $!\n";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sub gen_auth_key {
|
||||
my $authprivkeyfn = "$pmxcfs_auth_dir/authkey.key";
|
||||
my $authpubkeyfn = "$pmxcfs_base_dir/authkey.pub";
|
||||
|
||||
return if -f "$authprivkeyfn";
|
||||
|
||||
PVE::Cluster::check_cfs_is_mounted();
|
||||
|
||||
PVE::Cluster::cfs_lock_authkey(
|
||||
undef,
|
||||
sub {
|
||||
mkdir $pmxcfs_auth_dir
|
||||
|| $! == EEXIST
|
||||
|| die "unable to create dir '$pmxcfs_auth_dir' - $!\n";
|
||||
|
||||
run_silent_cmd(['openssl', 'genrsa', '-out', $authprivkeyfn, '2048']);
|
||||
|
||||
run_silent_cmd(
|
||||
['openssl', 'rsa', '-in', $authprivkeyfn, '-pubout', '-out', $authpubkeyfn]);
|
||||
},
|
||||
);
|
||||
|
||||
die "$@\n" if $@;
|
||||
}
|
||||
|
||||
sub gen_pveca_key {
|
||||
|
||||
return if -f $pveca_key_fn;
|
||||
|
||||
eval { run_silent_cmd(['openssl', 'genrsa', '-out', $pveca_key_fn, '4096']); };
|
||||
|
||||
die "unable to generate pve ca key:\n$@" if $@;
|
||||
}
|
||||
|
||||
sub gen_pveca_cert {
|
||||
|
||||
if (-f $pveca_key_fn && -f $pveca_cert_fn) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
gen_pveca_key();
|
||||
|
||||
# we try to generate an unique 'subject' to avoid browser problems (reused serial numbers, ..)
|
||||
my $uuid;
|
||||
UUID::generate($uuid);
|
||||
my $uuid_str;
|
||||
UUID::unparse($uuid, $uuid_str);
|
||||
|
||||
eval {
|
||||
# wrap openssl with faketime to prevent bug #904
|
||||
run_silent_cmd([
|
||||
'faketime',
|
||||
'yesterday',
|
||||
'openssl',
|
||||
'req',
|
||||
'-batch',
|
||||
'-days',
|
||||
'3650',
|
||||
'-new',
|
||||
'-x509',
|
||||
'-nodes',
|
||||
'-addext',
|
||||
'keyUsage=critical,keyCertSign,cRLSign',
|
||||
'-key',
|
||||
$pveca_key_fn,
|
||||
'-out',
|
||||
$pveca_cert_fn,
|
||||
'-subj',
|
||||
"/CN=Proxmox Virtual Environment/OU=$uuid_str/O=PVE Cluster Manager CA/",
|
||||
]);
|
||||
};
|
||||
|
||||
die "generating pve root certificate failed:\n$@" if $@;
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
sub gen_pve_ssl_key {
|
||||
my ($nodename) = @_;
|
||||
|
||||
die "no node name specified" if !$nodename;
|
||||
|
||||
my $pvessl_key_fn = "$pmxcfs_base_dir/nodes/$nodename/pve-ssl.key";
|
||||
|
||||
return if -f $pvessl_key_fn;
|
||||
|
||||
eval { run_silent_cmd(['openssl', 'genrsa', '-out', $pvessl_key_fn, '2048']); };
|
||||
|
||||
die "unable to generate pve ssl key for node '$nodename':\n$@" if $@;
|
||||
}
|
||||
|
||||
sub gen_pve_www_key {
|
||||
|
||||
return if -f $pvewww_key_fn;
|
||||
|
||||
eval { run_silent_cmd(['openssl', 'genrsa', '-out', $pvewww_key_fn, '2048']); };
|
||||
|
||||
die "unable to generate pve www key:\n$@" if $@;
|
||||
}
|
||||
|
||||
sub update_serial {
|
||||
my ($serial) = @_;
|
||||
|
||||
PVE::Tools::file_set_contents($pveca_srl_fn, $serial);
|
||||
}
|
||||
|
||||
sub gen_pve_ssl_cert {
|
||||
my ($force, $nodename, $ip) = @_;
|
||||
|
||||
die "no node name specified" if !$nodename;
|
||||
die "no IP specified" if !$ip;
|
||||
|
||||
my $pvessl_cert_fn = "$pmxcfs_base_dir/nodes/$nodename/pve-ssl.pem";
|
||||
|
||||
return if !$force && -f $pvessl_cert_fn;
|
||||
|
||||
my $names = "IP:127.0.0.1,IP:::1,DNS:localhost,IP:${ip},DNS:${nodename}";
|
||||
|
||||
my $fqdn = $nodename;
|
||||
|
||||
my $rc = PVE::INotify::read_file('resolvconf');
|
||||
if ($rc && $rc->{search}) {
|
||||
$fqdn .= ".$rc->{search}";
|
||||
$names .= ",DNS:$fqdn";
|
||||
}
|
||||
|
||||
my $reqfn = "/run/pve-cluster/pvecertreq-$$.tmp";
|
||||
unlink $reqfn;
|
||||
|
||||
my $pvessl_key_fn = "$pmxcfs_base_dir/nodes/$nodename/pve-ssl.key";
|
||||
eval {
|
||||
run_silent_cmd([
|
||||
'openssl',
|
||||
'req',
|
||||
'-batch',
|
||||
'-new',
|
||||
'-key',
|
||||
$pvessl_key_fn,
|
||||
'-out',
|
||||
$reqfn,
|
||||
'-subj',
|
||||
"/OU=PVE Cluster Node/O=Proxmox Virtual Environment/CN=$fqdn",
|
||||
'-addext',
|
||||
'basicConstraints=CA:FALSE',
|
||||
'-addext',
|
||||
'extendedKeyUsage=serverAuth',
|
||||
'-addext',
|
||||
"subjectAltName=$names",
|
||||
]);
|
||||
};
|
||||
|
||||
if (my $err = $@) {
|
||||
unlink $reqfn;
|
||||
die "unable to generate pve certificate request:\n$err";
|
||||
}
|
||||
|
||||
update_serial("0000000000000000") if !-f $pveca_srl_fn;
|
||||
|
||||
# get ca expiry
|
||||
my $cainfo = PVE::Certificate::get_certificate_info($pveca_cert_fn);
|
||||
my $daysleft = int(($cainfo->{notafter} - time()) / (24 * 60 * 60));
|
||||
|
||||
if ($daysleft < 14) {
|
||||
die "CA expires in less than 2 weeks, unable to generate certificate.\n";
|
||||
}
|
||||
|
||||
# let the certificate expire a little sooner that the ca, so subtract 2 days
|
||||
$daysleft -= 2;
|
||||
|
||||
# we want the certificates to only last 2 years, since some browsers
|
||||
# do not accept certificates with very long expiry time
|
||||
if ($daysleft >= 2 * 365) {
|
||||
$daysleft = 2 * 365;
|
||||
}
|
||||
|
||||
eval {
|
||||
run_silent_cmd([
|
||||
'faketime', 'yesterday', # NOTE: wrap openssl with faketime to prevent bug #904
|
||||
'openssl', 'x509', '-req', '-in', $reqfn, '-days', $daysleft, '-out',
|
||||
$pvessl_cert_fn,
|
||||
'-CAkey', $pveca_key_fn, '-CA', $pveca_cert_fn, '-CAserial', $pveca_srl_fn,
|
||||
'-copy_extensions', 'copyall',
|
||||
]);
|
||||
};
|
||||
my $err = $@;
|
||||
|
||||
unlink $reqfn or $!{ENOENT} or warn "failed to clean up '$reqfn' - $!";
|
||||
|
||||
die "unable to generate pve ssl certificate:\n$err" if $err;
|
||||
}
|
||||
|
||||
sub gen_pve_node_files {
|
||||
my ($nodename, $ip, $opt_force) = @_;
|
||||
|
||||
gen_local_dirs($nodename);
|
||||
|
||||
gen_auth_key();
|
||||
|
||||
# make sure we have a (cluster-wide) secret for CSRF prevention
|
||||
gen_pve_www_key();
|
||||
|
||||
# make sure we have a (per node) private key
|
||||
gen_pve_ssl_key($nodename);
|
||||
|
||||
# make sure we have a CA
|
||||
my $force = gen_pveca_cert();
|
||||
|
||||
$force = 1 if $opt_force;
|
||||
|
||||
gen_pve_ssl_cert($force, $nodename, $ip);
|
||||
}
|
||||
|
||||
my $vzdump_cron_dummy = <<__EOD;
|
||||
# cluster wide vzdump cron schedule
|
||||
# Atomatically generated file - do not edit
|
||||
|
||||
PATH="/usr/sbin:/usr/bin:/sbin:/bin"
|
||||
|
||||
__EOD
|
||||
|
||||
sub gen_pve_vzdump_symlink {
|
||||
|
||||
my $filename = "/etc/pve/vzdump.cron";
|
||||
|
||||
my $link_fn = "/etc/cron.d/vzdump";
|
||||
|
||||
if ((-f $filename) && (!-l $link_fn)) {
|
||||
rename($link_fn, "/root/etc_cron_vzdump.org"); # make backup if file exists
|
||||
symlink($filename, $link_fn);
|
||||
}
|
||||
}
|
||||
|
||||
sub gen_pve_vzdump_files {
|
||||
|
||||
my $filename = "/etc/pve/vzdump.cron";
|
||||
|
||||
PVE::Tools::file_set_contents($filename, $vzdump_cron_dummy)
|
||||
if !-f $filename;
|
||||
|
||||
gen_pve_vzdump_symlink();
|
||||
}
|
||||
|
||||
# join helpers
|
||||
|
||||
sub assert_joinable {
|
||||
my ($local_addr, $links, $force) = @_;
|
||||
|
||||
my $errors = '';
|
||||
my $error = sub { $errors .= "* $_[0]\n"; };
|
||||
|
||||
if (-f $authfile) {
|
||||
$error->("authentication key '$authfile' already exists");
|
||||
}
|
||||
|
||||
if (-f $clusterconf) {
|
||||
$error->("cluster config '$clusterconf' already exists");
|
||||
}
|
||||
|
||||
my $vmlist = PVE::Cluster::get_vmlist();
|
||||
if ($vmlist && $vmlist->{ids} && scalar(keys %{ $vmlist->{ids} })) {
|
||||
$error->("this host already contains virtual guests");
|
||||
}
|
||||
|
||||
if (PVE::Tools::run_command(['corosync-quorumtool', '-l'], noerr => 1, quiet => 1) == 0) {
|
||||
$error->("corosync is already running, is this node already in a cluster?!");
|
||||
}
|
||||
|
||||
# check if corosync ring IPs are configured on the current nodes interfaces
|
||||
my $check_ip = sub {
|
||||
my $ip = shift // return;
|
||||
my $logid = shift;
|
||||
if (!PVE::JSONSchema::pve_verify_ip($ip, 1)) {
|
||||
my $host = $ip;
|
||||
eval { $ip = PVE::Network::get_ip_from_hostname($host); };
|
||||
if ($@) {
|
||||
$error->("$logid: cannot use '$host': $@\n");
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
my $cidr = (Net::IP::ip_is_ipv6($ip)) ? "$ip/128" : "$ip/32";
|
||||
my $configured_ips = PVE::Network::get_local_ip_from_cidr($cidr);
|
||||
|
||||
$error->("$logid: cannot use IP '$ip', not found on local node!\n")
|
||||
if scalar(@$configured_ips) < 1;
|
||||
};
|
||||
|
||||
$check_ip->($local_addr, 'local node address');
|
||||
|
||||
foreach my $link (keys %$links) {
|
||||
$check_ip->($links->{$link}->{address}, "link$link");
|
||||
}
|
||||
|
||||
if ($errors) {
|
||||
warn "detected the following error(s):\n$errors";
|
||||
die "Check if node may join a cluster failed!\n" if !$force;
|
||||
warn "\nWARNING : detected error but forced to continue!\n\n";
|
||||
}
|
||||
}
|
||||
|
||||
sub join {
|
||||
my ($param) = @_;
|
||||
|
||||
my $nodename = PVE::INotify::nodename();
|
||||
my $local_ip_address = PVE::Cluster::remote_node_ip($nodename);
|
||||
|
||||
my $links = PVE::Corosync::extract_corosync_link_args($param);
|
||||
|
||||
# check if we can join with the given parameters and current node state
|
||||
assert_joinable($local_ip_address, $links, $param->{force});
|
||||
|
||||
setup_sshd_config();
|
||||
setup_rootsshconfig();
|
||||
setup_ssh_keys();
|
||||
|
||||
# make sure known_hosts is on local filesystem
|
||||
ssh_unmerge_known_hosts();
|
||||
|
||||
my $host = $param->{hostname};
|
||||
my $conn_args = {
|
||||
username => 'root@pam',
|
||||
password => $param->{password},
|
||||
cookie_name => 'PVEAuthCookie',
|
||||
protocol => 'https',
|
||||
host => $host,
|
||||
port => 8006,
|
||||
};
|
||||
|
||||
if (my $fp = $param->{fingerprint}) {
|
||||
$conn_args->{cached_fingerprints} = { uc($fp) => 1 };
|
||||
} else {
|
||||
# API schema ensures that we can only get here from CLI handler
|
||||
$conn_args->{manual_verification} = 1;
|
||||
}
|
||||
|
||||
print "Establishing API connection with host '$host'\n";
|
||||
|
||||
my $conn = PVE::APIClient::LWP->new(%$conn_args);
|
||||
$conn->login();
|
||||
|
||||
# login raises an exception on failure, so if we get here we're good
|
||||
print "Login succeeded.\n";
|
||||
|
||||
print "check cluster join API version\n";
|
||||
my $apiver = eval { $conn->get("/cluster/config/apiversion") } // 0;
|
||||
assert_we_can_join_cluster_version($apiver);
|
||||
|
||||
my $args = {};
|
||||
$args->{force} = $param->{force} if defined($param->{force});
|
||||
$args->{nodeid} = $param->{nodeid} if $param->{nodeid};
|
||||
$args->{votes} = $param->{votes} if defined($param->{votes});
|
||||
foreach my $link (keys %$links) {
|
||||
$args->{"link$link"} = PVE::Corosync::print_corosync_link($links->{$link});
|
||||
}
|
||||
|
||||
# this will be used as fallback if no links are specified
|
||||
if (!%$links) {
|
||||
$args->{link0} = $local_ip_address if $apiver == 0;
|
||||
$args->{new_node_ip} = $local_ip_address if $apiver >= 1;
|
||||
|
||||
print "No cluster network links passed explicitly, fallback to local node"
|
||||
. " IP '$local_ip_address'\n";
|
||||
}
|
||||
|
||||
if ($apiver >= 1) {
|
||||
$args->{apiversion} = JOIN_API_VERSION;
|
||||
}
|
||||
|
||||
print "Request addition of this node\n";
|
||||
my $res = eval { $conn->post("/cluster/config/nodes/$nodename", $args); };
|
||||
if (my $err = $@) {
|
||||
if (ref($err) && $err->isa('PVE::APIClient::Exception')) {
|
||||
# we received additional info about the error, show the user
|
||||
chomp $err->{msg};
|
||||
warn "An error occurred on the cluster node: $err->{msg}\n";
|
||||
foreach my $key (sort keys %{ $err->{errors} }) {
|
||||
my $symbol = ($key =~ m/^warning/) ? '*' : '!';
|
||||
warn "$symbol $err->{errors}->{$key}\n";
|
||||
}
|
||||
|
||||
die "Cluster join aborted!\n";
|
||||
}
|
||||
|
||||
die $@;
|
||||
}
|
||||
|
||||
if (defined($res->{warnings})) {
|
||||
foreach my $warn (@{ $res->{warnings} }) {
|
||||
warn "cluster: $warn\n";
|
||||
}
|
||||
}
|
||||
|
||||
print "Join request OK, finishing setup locally\n";
|
||||
|
||||
# added successfuly - now prepare local node
|
||||
finish_join($nodename, $res->{corosync_conf}, $res->{corosync_authkey});
|
||||
}
|
||||
|
||||
sub finish_join {
|
||||
my ($nodename, $corosync_conf, $corosync_authkey) = @_;
|
||||
|
||||
mkdir "$localclusterdir";
|
||||
PVE::Tools::file_set_contents($authfile, $corosync_authkey);
|
||||
PVE::Tools::file_set_contents($localclusterconf, $corosync_conf);
|
||||
|
||||
print "stopping pve-cluster service\n";
|
||||
my $cmd = ['systemctl', 'stop', 'pve-cluster'];
|
||||
PVE::Tools::run_command($cmd, errmsg => "can't stop pve-cluster service");
|
||||
|
||||
eval { PVE::Cluster::cfs_backup_database() };
|
||||
if (my $err = $@) {
|
||||
warn "failed to create backup of pre-join config database - $err";
|
||||
PVE::Cluster::cfs_rename_db_unsafe();
|
||||
} else {
|
||||
PVE::Cluster::cfs_unlink_db_unsafe();
|
||||
}
|
||||
|
||||
$cmd = ['systemctl', 'start', 'corosync', 'pve-cluster'];
|
||||
PVE::Tools::run_command($cmd, errmsg => "starting pve-cluster failed");
|
||||
|
||||
# wait for quorum
|
||||
my $printqmsg = 1;
|
||||
while (!PVE::Cluster::check_cfs_quorum(1)) {
|
||||
if ($printqmsg) {
|
||||
print "waiting for quorum...";
|
||||
STDOUT->flush();
|
||||
$printqmsg = 0;
|
||||
}
|
||||
sleep(1);
|
||||
}
|
||||
print "OK\n" if !$printqmsg;
|
||||
|
||||
generate_local_files();
|
||||
updatecerts_and_ssh(1);
|
||||
|
||||
print "generated new node certificate, restart pveproxy and pvedaemon services\n";
|
||||
PVE::Tools::run_command(['systemctl', 'reload-or-restart', 'pvedaemon', 'pveproxy']);
|
||||
|
||||
print "successfully added node '$nodename' to cluster.\n";
|
||||
}
|
||||
|
||||
sub generate_local_files {
|
||||
setup_rootsshconfig();
|
||||
gen_pve_vzdump_symlink();
|
||||
}
|
||||
|
||||
sub updatecerts_and_ssh {
|
||||
my ($force_new_cert, $silent, $unmerge_ssh) = @_;
|
||||
|
||||
my $p = sub { print "$_[0]\n" if !$silent };
|
||||
|
||||
if (!PVE::Cluster::check_cfs_quorum(1)) {
|
||||
return undef if $silent;
|
||||
die "no quorum - unable to update files\n";
|
||||
}
|
||||
|
||||
setup_ssh_keys();
|
||||
|
||||
my $nodename = PVE::INotify::nodename();
|
||||
my $local_ip_address = PVE::Cluster::remote_node_ip($nodename);
|
||||
|
||||
$p->("(re)generate node files");
|
||||
$p->("generate new node certificate") if $force_new_cert;
|
||||
gen_pve_node_files($nodename, $local_ip_address, $force_new_cert);
|
||||
|
||||
$p->("merge authorized SSH keys");
|
||||
ssh_merge_keys();
|
||||
if ($unmerge_ssh) {
|
||||
$p->("unmerge SSH known hosts");
|
||||
ssh_unmerge_known_hosts();
|
||||
}
|
||||
ssh_create_node_known_hosts($nodename);
|
||||
gen_pve_vzdump_files();
|
||||
}
|
||||
|
||||
1;
|
||||
Reference in new issue
Block a user