fix #4986: jobs: do not get stuck if the task log is missing

When the scheduler checks whether the last run of a job has finished,
it dies if the task log of that run does not exist. The job is then
skipped with 'could not update job state, skipping - no such task' on
every iteration, so it never runs again until its state file is
cleaned up manually.

The task log can be gone, for example after a crash. If the task is
not running anymore, treat it as stopped with an error status instead,
so that the job gets scheduled again, like Proxmox Backup Server does
since commit 48fbce07 ("fix #4895: scheduled jobs: ignore task-log not
found error").

The check now comes after the one whether the task is still running,
so a running task does not get marked as stopped.

Signed-off-by: Michal Fox <me@dualfroz.com>
Link: https://lore.proxmox.com/20261003153820.7-1-me@dualfroz.com
[FE: use file_exists()
     issue warning]
Signed-off-by: Fiona Ebner <f.ebner@proxmox.com>

PVE/Jobs.pm

@@ -5,6 +5,7 @@ use warnings;
 use JSON;
 
 use PVE::Cluster qw(cfs_lock_file cfs_read_file cfs_register_file);
+use PVE::File;
 use PVE::Job::Registry;
 use PVE::Jobs::VZDump;
 use PVE::Jobs::RealmSync;
@@ -109,13 +110,18 @@ my $get_job_task_status = sub {
 
     my ($task, $filename) = PVE::Tools::upid_decode($state->{upid}, 1);
     die "unable to parse worker upid - $state->{upid}\n" if !$task;
-    die "no such task\n" if !-f $filename;
 
     my $pstart = PVE::ProcFSTools::read_proc_starttime($task->{pid});
     if ($pstart && $pstart == $task->{pstart}) {
         return; # still running
     }
 
+    # the task log might be gone, e.g. after a crash, do not let that block the job forever
+    if (!PVE::File::file_exists($filename)) {
+        warn "missing task log for $state->{upid}\n";
+        return "task log not found";
+    }
+
     return PVE::Tools::upid_read_status($state->{upid});
 };