deleting failed task will trigger a re-run of the task

current not supported operation will succeed immeditely.
https://github.com/pacoxu/kubeadm-operator/blob/9721e44e8e2f00b767972cbd69f857eb3ce153c5/operations/upgrade.go#L51
func planUpgrade(operation *operatorv1.Operation, spec *operatorv1.UpgradeOperationSpec, c client.Client) *operatorv1.RuntimeTaskGroupList {
    log := ctrl.Log.WithName("operations").WithName("Upgrade").WithValues("task", operation.Name)
    var items []operatorv1.RuntimeTaskGroup

    serverVersion, err := getServerVersion()
    if err != nil {
        log.Error(err, "get server version failed")
        return nil
    }
    isServerSupported, isServerCrossVersion, isServerCanSkip := upgradeCheck(serverVersion, spec.KubernetesVersion)
    if !isServerSupported {
        log.Info("Upgrade is not supported", "serverVersion", serverVersion, "kubernetesVersion", spec.KubernetesVersion)
        // TODO current not supported operation will succeed immeditely.
        return nil
    }
    log.Info("Upgrade is supported", "serverVersion", serverVersion, "kubernetesVersion", spec.KubernetesVersion)
    nodes, err := listNodesBySelector(c, getAllSelector())
    if err != nil {
        log.Error(err, "list node failed")
        return nil
    }
    var isClientSupported, isClientCrossVersion, isClientCanSkip bool = true, false, true
    var clientServerMatch bool = true
    log.Info("nodes list", "nodes", len(nodes.Items))
    for _, n := range nodes.Items {
        supported, cross, skip := upgradeCheck(n.Status.NodeInfo.KubeletVersion, spec.KubernetesVersion)
        isClientSupported = isClientSupported && supported
        isClientCrossVersion = isClientCrossVersion || cross
        isClientCanSkip = isClientCanSkip && skip
        clientServerMatch = clientServerMatch && n.Status.NodeInfo.KubeletVersion == serverVersion
        if n.Status.NodeInfo.KubeletVersion != serverVersion {
            log.Info("node is not match server version", "node", n.Name, "serverVersion", serverVersion, "kubeletVersion", n.Status.NodeInfo.KubeletVersion)
        }
    }
    if !isClientSupported {
        log.Info("Upgrade is not supported", "clientVersion", spec.KubernetesVersion)
        return nil
    }
    log.Info("show all client version check results", "isClientSupported", isClientSupported, "isClientCrossVersion", isClientCrossVersion, "isClientCanSkip", isClientCanSkip, "clientServerMatch", clientServerMatch)
    log.Info("show all server version check results", "isServerSupported", isServerSupported, "isServerCrossVersion", isServerCrossVersion, "isServerCanSkip", isServerCanSkip)
    if isClientCanSkip && isServerCanSkip {
        // skip upgrade directly
        return &operatorv1.RuntimeTaskGroupList{
            Items: items,
        }
    } else if isClientCrossVersion || isServerCrossVersion {
        // support upgrade to v1.n-1~v1.n of current kubernetes server version.
        // If the current kubernetes server version is v1.n-2 which is below the target version, we need to generate a further upgrade plan
        log.Info("Upgrade is not supported, need cross version for client or server", "targetVersion", spec.KubernetesVersion, "serverVersion", serverVersion)
        if !clientServerMatch {
            // upgrade nodes to the target version
            log.Info("[cross-upgrade] add items to make server client match", "serverVersion", serverVersion)
            items = append(items, planNextUpgrade(operation, serverVersion, c, true)...)
        }
        crossVersions := getCrossVersions(serverVersion, spec.KubernetesVersion)
        for _, v := range crossVersions {
            log.Info("[cross-upgrade] add items to upgrade to a middle version", "version", v)
            items = append(items, planNextUpgrade(operation, v, c, false)...)
        }
        log.Info("[cross-upgrade] add items to upgrade to the target version", "version", spec.KubernetesVersion)
        items = append(items, planNextUpgrade(operation, operation.Spec.Upgrade.KubernetesVersion, c, false)...)

    } else {
        log.Info("add items to upgrade to the target version", "version", spec.KubernetesVersion)
        items = append(items, planNextUpgrade(operation, operation.Spec.Upgrade.KubernetesVersion, c, isServerCanSkip)...)
    }

    return &operatorv1.RuntimeTaskGroupList{
        Items: items,
    }
}

// the version may not be operation.Spec.Upgrade.KubernetesVersion for cross upgrade
func planNextUpgrade(operation *operatorv1.Operation, version string, c client.Client, isServerCanSkip bool) []operatorv1.RuntimeTaskGroup {
    log := ctrl.Log.WithName("operations").WithName("Upgrade").WithValues("task", operation.Name)
    log.Info("add task for upgrading", "version", version, "isServerCanSkip", isServerCanSkip)

    var items []operatorv1.RuntimeTaskGroup
    dryRun := operation.Spec.GetTypedOperationExecutionMode() == operatorv1.OperationExecutionModeDryRun

    if !isServerCanSkip {
        t1 := createUpgradeApplyTaskGroup(operation, fmt.Sprintf("%s-01", version), "upgrade-apply")
        setCP1Selector(&t1)
        // run `upgrade apply`` on the first node of all control plane
        t1.Spec.NodeFilter = string(operatorv1.RuntimeTaskGroupNodeFilterHead)
        t1.Spec.Template.Spec.Commands = append(t1.Spec.Template.Spec.Commands,
            operatorv1.CommandDescriptor{
                UpgradeKubeadm: &operatorv1.UpgradeKubeadmCommandSpec{
                    KubernetesVersion: version,
                    Local:             operation.Spec.Upgrade.Local,
                },
            },
            operatorv1.CommandDescriptor{
                KubeadmUpgradeApply: &operatorv1.KubeadmUpgradeApplyCommandSpec{
                    DryRun:            dryRun,
                    KubernetesVersion: version,
                    SkipKubeProxy:     operation.Spec.Upgrade.UpgradeKubeProxyAtLast,
                },
            },
            // as it depends on kubelet-reloader, we need to run it after upgrade-kubeadm apply
            operatorv1.CommandDescriptor{
                UpgradeKubeletAndKubeactl: &operatorv1.UpgradeKubeletAndKubeactlCommandSpec{
                    KubernetesVersion: version,
                    Local:             operation.Spec.Upgrade.Local,
                },
            },
        )
        log.Info("add upgrade-apply task group", "task", t1.Name)
        items = append(items, t1)
cac394b1e91a4f1a2486af0baa58fdadf7683262
pacoxu / kubeadm-operator

deleting failed task will trigger a re-run of the task #76
