#!/usr/bin/env perl
# Copyright 2014-2020 SUSE LLC
# SPDX-License-Identifier: GPL-2.0-or-later

use Test::Most;

use FindBin;
use lib "$FindBin::Bin/lib", "$FindBin::Bin/../external/os-autoinst-common/lib";
use OpenQA::Jobs::Constants;
use OpenQA::WebAPI::Controller::API::V1::Worker;
use OpenQA::WebSockets::Client;
use OpenQA::Constants 'WEBSOCKET_API_VERSION';
use OpenQA::Test::Case;
use OpenQA::Test::Client 'client';
require OpenQA::Test::Database;
use OpenQA::Test::Utils 'embed_server_for_testing';
use Test::MockModule;
use Test::MockObject;
use Test::Mojo;
use DBIx::Class::Timestamps 'now';
use Test::Warnings ':report_warnings';
use OpenQA::Test::TimeLimit '10';

OpenQA::Test::Case->new->init_data(fixtures_glob => '01-jobs.pl 06-job_dependencies.pl');
my $t = client(Test::Mojo->new('OpenQA::WebAPI'));
my $schema = $t->app->schema;

embed_server_for_testing(
    server_name => 'OpenQA::WebSockets',
    client => OpenQA::WebSockets::Client->singleton,
);

sub job_get {
    my ($id) = @_;

    my $job = $schema->resultset('Jobs')->find({id => $id});
    return $job;
}

my $job = job_get(99963);
is_deeply
  $job->cluster_jobs,
  {
    99961 => {
        is_parent_or_initial_job => 1,
        ok => 0,
        state => RUNNING,
        chained_children => [],
        chained_parents => [],
        directly_chained_children => [],
        directly_chained_parents => [],
        parallel_children => [99963],
        parallel_parents => [],
    },
    99963 => {
        is_parent_or_initial_job => 1,
        ok => 0,
        state => RUNNING,
        chained_children => [],
        chained_parents => [],
        directly_chained_children => [],
        directly_chained_parents => [],
        parallel_children => [],
        parallel_parents => [99961],
    },
  },
  '99963 is part of a duett';
my $new_job = $job->auto_duplicate;
ok $new_job, 'got new job id ' . $new_job->id;

is $new_job->state, 'scheduled', 'new job is scheduled';
is_deeply
  $new_job->cluster_jobs,
  {
    99982 => {
        is_parent_or_initial_job => 1,
        ok => 0,
        state => SCHEDULED,
        chained_children => [],
        chained_parents => [],
        directly_chained_children => [],
        directly_chained_parents => [],
        parallel_children => [99983],
        parallel_parents => [],
    },
    99983 => {
        is_parent_or_initial_job => 1,
        ok => 0,
        state => SCHEDULED,
        chained_children => [],
        chained_parents => [],
        directly_chained_children => [],
        directly_chained_parents => [],
        parallel_children => [],
        parallel_parents => [99982],
    },
  },
  'new job is part of a new duett';

$job = job_get(99963);
is $job->state, 'running', 'old job is running';
is $job->t_finished, undef, 'There is a no finish time yet';

sub lj {
    # check the call succeeds every time, only output if verbose
    my @jobs = $schema->resultset('Jobs')->all;
    return unless $ENV{HARNESS_IS_VERBOSE};
    printf "%d %-10s %s\n", $_->id, $_->state, $_->name for @jobs;    # uncoverable statement
}

lj;

my $form = {DISTRI => 'opensuse', VERSION => '13.1', FLAVOR => 'DVD', ARCH => 'x86_64'};
my $jobs = $schema->resultset('Jobs');
$schema->txn_begin;
my $ret = $jobs->cancel_by_settings($form);
is $ret, 3, 'three jobs cancelled by settings (99963 and the new cluster of 2)';
$job = job_get(99963);
is $job->reason, 'cancelled based on job settings', 'jobs reason points to settings';
my $event = OpenQA::Test::Case::find_most_recent_event($schema, 'job_cancel_by_settings');
is_deeply $event, $form, 'recent event about cancelled job by settings created';
$schema->txn_rollback;
my $scheduled_product = Test::MockObject->new->set_always(id => 42);
$ret = $jobs->cancel_by_settings($form, undef, undef, undef, $scheduled_product);
$job = job_get(99963);
is $ret, 3, 'three jobs cancelled again';
is $job->reason, 'cancelled by scheduled product 42', 'jobs cancellation reason points to product';

lj;

$job = $new_job->discard_changes;
is $job->state, 'cancelled', 'new job is cancelled';
ok $job->t_finished, 'There is a finish time';

$job = job_get(99963);
is $job->state, 'cancelled', 'old job cancelled as well';

$job = job_get(99982);
is $job->state, 'cancelled', 'new job 99982 cancelled';

$job = job_get(99983);
is $job->state, 'cancelled', 'new job 99983 cancelled';

$job = job_get(99928);
is $job->state, 'scheduled', 'unrelated job 99928 still scheduled';
$job = job_get(99927);
is $job->state, 'scheduled', 'unrelated job 99927 still scheduled';

$job = job_get(99928);
$ret = $job->cancel(USER_CANCELLED);
is $ret, 1, 'one job cancelled by id';

$job = job_get(99928);
is $job->state, 'cancelled', 'job 99928 cancelled';
$job = job_get(99927);
is $job->state, 'scheduled', 'unrelated job 99927 still scheduled';


$new_job = job_get(99981)->auto_duplicate;
ok $new_job, 'duplicate new job for iso test';

$job = $new_job;
is $job->state, 'scheduled', 'new job is scheduled';

lj;

$form = {ISO => 'openSUSE-13.1-GNOME-Live-i686-Build0091-Media.iso'};
$ret = $schema->resultset('Jobs')->cancel_by_settings($form);
is $ret, 1, 'one job cancelled by iso';
is_deeply OpenQA::Test::Case::find_most_recent_event($schema, 'job_cancel_by_settings'),
  $form, 'Cancellation was logged with settings';

$job = job_get(99927);
is $job->state, 'scheduled', 'unrelated job 99927 still scheduled';

my %settings = (
    DISTRI => 'Unicorn',
    FLAVOR => 'pink',
    VERSION => '42',
    BUILD => '666',
    ISO => 'whatever.iso',
    MACHINE => 'RainbowPC',
    ARCH => 'x86_64',
);

sub _job_create {
    my $job = $schema->resultset('Jobs')->create_from_settings(@_);
    # reload all values from database so we can check against default values
    $job->discard_changes;
    return $job;
}

subtest 'chained or directly chained parent fails -> children are canceled (skipped)' => sub {
    my %settingsA = (%settings, TEST => 'A');
    my %settingsB = (%settings, TEST => 'B');
    my %settingsC = (%settings, TEST => 'C');
    my %settingsD = (%settings, TEST => 'D');

    my $jobA = _job_create(\%settingsA);
    $settingsB{_START_AFTER_JOBS} = [$jobA->id];
    my $jobB = _job_create(\%settingsB);
    $settingsC{_START_AFTER_JOBS} = [$jobB->id];
    my $jobC = _job_create(\%settingsC);
    $settingsD{_START_DIRECTLY_AFTER_JOBS} = [$jobB->id];
    my $jobD = _job_create(\%settingsD);

    $jobA->state(OpenQA::Jobs::Constants::RUNNING);
    $jobA->update;

    # set A as failed and reload B, C from database
    $jobA->done(result => OpenQA::Jobs::Constants::FAILED);
    $jobB->discard_changes;
    $jobC->discard_changes;
    $jobD->discard_changes;

    is $jobB->state, OpenQA::Jobs::Constants::CANCELLED, 'B state is cancelled';
    is $jobC->state, OpenQA::Jobs::Constants::CANCELLED, 'C state is cancelled';
    is $jobB->result, OpenQA::Jobs::Constants::SKIPPED, 'B result is skipped';
    is $jobC->result, OpenQA::Jobs::Constants::SKIPPED, 'C (regularly chained) result is skipped';
    is $jobD->result, OpenQA::Jobs::Constants::SKIPPED, 'D (directly chained) result is skipped';

    # note: A feasible alternative would be making it the worker's responsibility to set
    #       *directly* chained jobs to SKIPPED. However, it is likely safer to let the web UI handle
    #       this. Of course we still need to take care that the worker really skips those jobs.
};

subtest 'cancelling directly chained jobs' => sub {
    my $parent_job = _job_create({%settings, TEST => 'parent'});
    my $to_cancel_job = _job_create({%settings, TEST => 'to-cancel', _START_DIRECTLY_AFTER_JOBS => [$parent_job->id]});
    my $child_job = _job_create({%settings, TEST => 'child', _START_DIRECTLY_AFTER_JOBS => [$to_cancel_job->id]});
    my $sibling_job = _job_create({%settings, TEST => 'sibling', _START_DIRECTLY_AFTER_JOBS => [$parent_job->id]});
    my $multiple_parents = [$to_cancel_job->id, $sibling_job->id];
    my $child_job2 = _job_create({%settings, TEST => 'multip-parent', _START_AFTER_JOBS => $multiple_parents});
    my @jobs = ($parent_job, $to_cancel_job, $child_job, $sibling_job, $child_job2);

    $to_cancel_job->cancel(USER_CANCELLED);
    $_->discard_changes for @jobs;

    is $parent_job->state, OpenQA::Jobs::Constants::SCHEDULED, 'parent not cancelled';
    is $to_cancel_job->state, OpenQA::Jobs::Constants::CANCELLED, 'cancelled job is cancelled';
    is $sibling_job->state, OpenQA::Jobs::Constants::SCHEDULED, 'sibling not cancelled';
    is $child_job->state, OpenQA::Jobs::Constants::CANCELLED, 'child job is cancelled';
    is $child_job2->state, OpenQA::Jobs::Constants::CANCELLED, 'child job with multiple parents is cancelled';
};

subtest 'parallel parent fails -> children are cancelled (parallel_failed)' => sub {
    # monkey patch ws_send of OpenQA::WebSockets to store received command
    my $mock_server = Test::MockModule->new('OpenQA::WebSockets');
    my $server_called;
    my @sent_commands;
    $mock_server->redefine(
        ws_send => sub {
            my ($workerid, $command, $jobid) = @_;
            $server_called++;
            push @sent_commands, $command;
        });

    my %settingsA = (%settings, TEST => 'A');
    my %settingsB = (%settings, TEST => 'B');
    my %settingsC = (%settings, TEST => 'C');

    my $jobA = _job_create(\%settingsA);
    $settingsB{_PARALLEL_JOBS} = [$jobA->id];
    my $jobB = _job_create(\%settingsB);
    $settingsC{_PARALLEL_JOBS} = [$jobA->id];
    my $jobC = _job_create(\%settingsC);

    # create 3 workers for command issue test
    my %workercaps = (
        cpu_modelname => 'Rainbow CPU',
        cpu_arch => 'x86_64',
        cpu_opmode => '32-bit, 64-bit',
        mem_max => '4096',
        websocket_api_version => WEBSOCKET_API_VERSION,
        isotovideo_interface_version => WEBSOCKET_API_VERSION,
    );
    my $c = OpenQA::WebAPI::Controller::API::V1::Worker->new;
    my $workers = $schema->resultset('Workers');
    my $w1 = $workers->find($c->_register($schema, 'host', '1', \%workercaps));
    my $w2 = $workers->find($c->_register($schema, 'host', '2', \%workercaps));
    my $w3 = $workers->find($c->_register($schema, 'host', '3', \%workercaps));
    for my $job_and_worker ([$jobA, $w1], [$jobB, $w2], [$jobC, $w3]) {
        $job_and_worker->[0]
          ->update({state => OpenQA::Jobs::Constants::RUNNING, assigned_worker_id => $job_and_worker->[1]->id});
        $job_and_worker->[1]->update({job_id => $job_and_worker->[0]->id});
    }

    # set A as failed and reload B, C from database
    @sent_commands = ();
    my $now = now();
    $jobA->done(result => OpenQA::Jobs::Constants::FAILED);
    $jobB->discard_changes;
    $jobC->discard_changes;

    is $jobB->result, OpenQA::Jobs::Constants::PARALLEL_FAILED, 'B result is parallel failed';
    is $jobB->state, OpenQA::Jobs::Constants::RUNNING, 'B is still running';
    is $jobB->t_finished, undef, 'B does not has t_finished set since it is still running';
    is $jobC->result, OpenQA::Jobs::Constants::PARALLEL_FAILED, 'C result is parallel failed';
    is_deeply \@sent_commands, [qw(cancel cancel)], 'both cancel commands issued';

    # assume B has actually been cancelled
    $jobB->update({state => OpenQA::Jobs::Constants::DONE});
    ok $jobB->t_finished, 'B has t_finished set after being actually cancelled'
      and ok $jobB->t_finished ge $now, 'B has t_finished set to a sane value';

    ok $server_called, 'Mocked ws_send function has been called';
};

subtest 'chained or directly chained parent fails -> parallel parents of children are cancelled (skipped)' => sub {
    # https://progress.opensuse.org/issues/36565 - A is install, B is support server,
    # C and D are children to both and parallel to each other
    my %settingsA = (%settings, TEST => 'A');
    my %settingsB = (%settings, TEST => 'B');
    my %settingsC = (%settings, TEST => 'C');
    my %settingsD = (%settings, TEST => 'D');

    my $jobA = _job_create(\%settingsA);
    my $jobB = _job_create(\%settingsB);
    $settingsC{_START_AFTER_JOBS} = [$jobA->id];
    $settingsD{_START_DIRECTLY_AFTER_JOBS} = [$jobA->id];
    $settingsC{_PARALLEL_JOBS} = [$jobB->id];
    my $jobC = _job_create(\%settingsC);
    $settingsC{_PARALLEL_JOBS} = [$jobB->id, $jobC->id];
    my $jobD = _job_create(\%settingsD);

    $jobA->state(OpenQA::Jobs::Constants::RUNNING);
    $jobA->update;

    # set A as failed and reload B, C and D from database
    $jobA->done(result => OpenQA::Jobs::Constants::FAILED);
    $jobB->discard_changes;
    $jobC->discard_changes;
    $jobD->discard_changes;

    is $jobB->state, OpenQA::Jobs::Constants::CANCELLED, 'B state is cancelled';
    is $jobC->state, OpenQA::Jobs::Constants::CANCELLED, 'C state is cancelled';
    is $jobD->state, OpenQA::Jobs::Constants::CANCELLED, 'D state is cancelled';
    is $jobB->result, OpenQA::Jobs::Constants::SKIPPED, 'B result is skipped';
    is $jobC->result, OpenQA::Jobs::Constants::SKIPPED, 'C result is skipped';
    is $jobD->result, OpenQA::Jobs::Constants::SKIPPED, 'D result is skipped';
};

subtest 'parallel child with one parent fails -> parent is cancelled' => sub {
    my %settingsA = %settings;
    my %settingsB = %settings;
    $settingsA{TEST} = 'A';
    $settingsB{TEST} = 'B';
    my $jobA = _job_create(\%settingsA);
    $settingsB{_PARALLEL_JOBS} = [$jobA->id];
    my $jobB = _job_create(\%settingsB);

    # set B as failed and reload A from database
    $jobB->done(result => OpenQA::Jobs::Constants::FAILED);
    $jobA->discard_changes;

    is $jobA->state, OpenQA::Jobs::Constants::CANCELLED, 'A state is cancelled';
};

subtest 'cancellation behaviour of parallel cluster when one job is skipped or fails' => sub {
    my %settingsA = (%settings, TEST => 'A');
    my %settingsB = (%settings, TEST => 'B');
    my %settingsC = (%settings, TEST => 'C');
    my $jobA = _job_create(\%settingsA);
    $settingsB{_PARALLEL_JOBS} = [$jobA->id];
    $settingsC{_PARALLEL_JOBS} = [$jobA->id];
    my $jobB = _job_create(\%settingsB);
    my $jobC = _job_create(\%settingsC);

    # set B as failed and reload jobs from database
    $jobB->done(result => FAILED);
    $_->discard_changes for $jobA, $jobB, $jobC;

    # A and C should be cancelled
    is $jobA->state, CANCELLED, 'A state is cancelled';
    is $jobB->state, DONE, 'B state is done';
    is $jobC->state, CANCELLED, 'C state is cancelled';

    # skip the parallel parent
    $jobA = _job_create(\%settingsA);
    $settingsB{_PARALLEL_JOBS} = [$jobA->id];
    $settingsC{_PARALLEL_JOBS} = [$jobA->id];
    $jobB = _job_create(\%settingsB);
    $jobC = _job_create(\%settingsC);

    # skip A and reload jobs from database
    $jobA->done(result => SKIPPED);
    $_->discard_changes for $jobA, $jobB, $jobC;

    # B and C should be cancelled
    is $jobA->state, DONE, 'A state is done as we skipped it';
    is $jobB->state, CANCELLED, 'B state is cancelled after parallel parent was skipped';
    is $jobC->state, CANCELLED, 'C state is cancelled after parallel parent was skipped';

    # now test in 'do not cancel parent and other children' mode
    $settingsA{PARALLEL_CANCEL_WHOLE_CLUSTER} = '0';
    $jobA = _job_create(\%settingsA);
    $settingsB{_PARALLEL_JOBS} = [$jobA->id];
    $settingsC{_PARALLEL_JOBS} = [$jobA->id];
    $jobB = _job_create(\%settingsB);
    $jobC = _job_create(\%settingsC);

    # set B as failed and reload jobs from database
    $jobB->done(result => FAILED);
    $_->discard_changes for $jobA, $jobB, $jobC;

    # this time A and C should still be scheduled (*not* cancelled)
    is $jobA->state, SCHEDULED, 'new A state is scheduled';
    is $jobB->state, DONE, 'B state is done';
    is $jobC->state, SCHEDULED, 'new C state is scheduled';

    # now set C as failed and reload A
    $jobC->done(result => FAILED);
    $jobA->discard_changes;

    # now A *should* be cancelled
    is $jobA->state, CANCELLED, 'new A state is cancelled';
};

done_testing();
