search

acorn-io / runtime

A simple application deployment framework built on Kubernetes
https://docs.acorn.io/
Apache License 2.0
1.13k stars 100 forks source link

Changes to JobSpec on update causes controller to error loop #178

Open cloudnautique opened 2 years ago

cloudnautique commented 2 years ago

If a job contains a file with dynamic content, the controller will continue to error trying to update the job spec.

If you define something like:

jobs: {
  test: {
     ....
     files: {
       "/acorn/test": "\(serverCount)"
     }
     ... 
  }
}

Where serverCount is tied to the number of containers running, if you run: acorn run <image> --server-count 1

And later scale it up: acorn update ... MY-APP --server-count 2

The controller will output:

ERRO[0132] error syncing 'acorn-redis/misty-dew': handler acorn acorn.io/v1, Kind=AppInstance: failed to update misty-dew-515e8f97/redis-init-cluster batch/v1, Kind=Job for acorn acorn-redis/misty-dew: Job.batch "redis-init-cluster" is invalid: spec.template: Invalid value: core.PodTemplateSpec ...: field is immutable, requeuing

Workaround: Place the template in a secret.

cloudnautique commented 2 years ago

Still seems to be an issue.

given:

jobs: testing: {
  image: "alpine"
  command: [
    "sleep",
    "1",
  ]
    files: {
      "/tmp/servers": "1345"                                                          
   }
 }

If you change the contents of file /tmp/servers by editing the string the controller throws an error.

STATUS: ENDPOINTS[] HEALTHY[2] UPTODATE[2] [controller: failed to update restless-feather-b6f957b2-a80/testing batch/v1, Kind=Job for  acorn/restless-feather: Job.batch "testing" is invalid: spec.template: Invalid value: core.PodTemplateSpec{ObjectMeta:v1.ObjectMeta{Name:"", GenerateName:"", Namespace:"", SelfLink:"", UID:"", ResourceVersion:"", Generation:0, CreationTimestamp:time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC), DeletionTimestamp:<nil>, DeletionGracePeriodSeconds:(*int64)(nil), Labels:map[string]string{"acorn.io/app-name":"restless-feather", "acorn.io/app-namespace":"acorn", "acorn.io/job-name":"testing", "acorn.io/managed":"true", "acorn.io/root-namespace":"acorn", "acorn.io/root-prefix":"restless-feather", "controller-uid":"6cc94afc-cbe3-4090-94a9-670eda23edc9", "job-name":"testing"}, Annotations:map[string]string{"acorn.io/container-spec":"{\"build\":{\"baseImage\":\"alpine\"},\"command\":[\"sleep\",\"1\"],\"files\":{\"/tmp/servers\":{\"content\":\"MTM0NQ==\",\"mode\":\"0644\",\"secret\":{}}},\"image\":\"sha256:ebf22eccf26294e1b277dc2a52e8ddf789f721039c635ec24703f0293034ed51\",\"permissions\":{}}", "acorn.io/image-mapping":"{\"sha256:ebf22eccf26294e1b277dc2a52e8ddf789f721039c635ec24703f0293034ed51\":\"alpine\"}"}, OwnerReferences:[]v1.OwnerReference(nil), Finalizers:[]string(nil), ClusterName:"", ManagedFields:[]v1.ManagedFieldsEntry(nil)}, Spec:core.PodSpec{Volumes:[]core.Volume{core.Volume{Name:"files", VolumeSource:core.VolumeSource{HostPath:(*core.HostPathVolumeSource)(nil), EmptyDir:(*core.EmptyDirVolumeSource)(nil), GCEPersistentDisk:(*core.GCEPersistentDiskVolumeSource)(nil), AWSElasticBlockStore:(*core.AWSElasticBlockStoreVolumeSource)(nil), GitRepo:(*core.GitRepoVolumeSource)(nil), Secret:(*core.SecretVolumeSource)(nil), NFS:(*core.NFSVolumeSource)(nil), ISCSI:(*core.ISCSIVolumeSource)(nil), Glusterfs:(*core.GlusterfsVolumeSource)(nil), PersistentVolumeClaim:(*core.PersistentVolumeClaimVolumeSource)(nil), RBD:(*core.RBDVolumeSource)(nil), Quobyte:(*core.QuobyteVolumeSource)(nil), FlexVolume:(*core.FlexVolumeSource)(nil), Cinder:(*core.CinderVolumeSource)(nil), CephFS:(*core.CephFSVolumeSource)(nil), Flocker:(*core.FlockerVolumeSource)(nil), DownwardAPI:(*core.DownwardAPIVolumeSource)(nil), FC:(*core.FCVolumeSource)(nil), AzureFile:(*core.AzureFileVolumeSource)(nil), ConfigMap:(*core.ConfigMapVolumeSource)(0x4015465a80), VsphereVolume:(*core.VsphereVirtualDiskVolumeSource)(nil), AzureDisk:(*core.AzureDiskVolumeSource)(nil), PhotonPersistentDisk:(*core.PhotonPersistentDiskVolumeSource)(nil), Projected:(*core.ProjectedVolumeSource)(nil), PortworxVolume:(*core.PortworxVolumeSource)(nil), ScaleIO:(*core.ScaleIOVolumeSource)(nil), StorageOS:(*core.StorageOSVolumeSource)(nil), CSI:(*core.CSIVolumeSource)(nil), Ephemeral:(*core.EphemeralVolumeSource)(nil)}}}, InitContainers:[]core.Container(nil), Containers:[]core.Container{core.Container{Name:"testing", Image:"127.0.0.1:30887/acorn/acorn@sha256:ebf22eccf26294e1b277dc2a52e8ddf789f721039c635ec24703f0293034ed51", Command:[]string(nil), Args:[]string{"sleep", "1"}, WorkingDir:"", Ports:[]core.ContainerPort(nil), EnvFrom:[]core.EnvFromSource(nil), Env:[]core.EnvVar(nil), Resources:core.ResourceRequirements{Limits:core.ResourceList(nil), Requests:core.ResourceList(nil)}, VolumeMounts:[]core.VolumeMount{core.VolumeMount{Name:"files", ReadOnly:false, MountPath:"/tmp/servers", SubPath:"0987bd917543", MountPropagation:(*core.MountPropagationMode)(nil), SubPathExpr:""}}, VolumeDevices:[]core.VolumeDevice(nil), LivenessProbe:(*core.Probe)(nil), ReadinessProbe:(*core.Probe)(nil), StartupProbe:(*core.Probe)(nil), Lifecycle:(*core.Lifecycle)(nil), TerminationMessagePath:"/run/secrets/output", TerminationMessagePolicy:"File", ImagePullPolicy:"IfNotPresent", SecurityContext:(*core.SecurityContext)(nil), Stdin:false, StdinOnce:false, TTY:false}}, EphemeralContainers:[]core.EphemeralContainer(nil), RestartPolicy:"Never", TerminationGracePeriodSeconds:(*int64)(0x401e4664a0), ActiveDeadlineSeconds:(*int64)(nil), DNSPolicy:"ClusterFirst", NodeSelector:map[string]string(nil), ServiceAccountName:"", AutomountServiceAccountToken:(*bool)(0x401e4664a8), NodeName:"", SecurityContext:(*core.PodSecurityContext)(0x401b59de80), ImagePullSecrets:[]core.LocalObjectReference{core.LocalObjectReference{Name:"testing-pull-b6f957b2-a80"}}, Hostname:"", Subdomain:"", SetHostnameAsFQDN:(*bool)(nil), Affinity:(*core.Affinity)(nil), SchedulerName:"default-scheduler", Tolerations:[]core.Toleration(nil), HostAliases:[]core.HostAlias(nil), PriorityClassName:"", Priority:(*int32)(nil), PreemptionPolicy:(*core.PreemptionPolicy)(nil), DNSConfig:(*core.PodDNSConfig)(nil), ReadinessGates:[]core.PodReadinessGate(nil), RuntimeClassName:(*string)(nil), Overhead:core.ResourceList(nil), EnableServiceLinks:(*bool)(0x401e4664aa), TopologySpreadConstraints:[]core.TopologySpreadConstraint(nil), OS:(*core.PodOS)(nil)}}: field is immutable]