incus remote list shows the new image, but incus image copy tries to fetch the old image

tl;dr :

# incus image list yunohost:yunohost/trixie-unstable/appci
+-----------------------------------------+--------------+--------+-----------------------------------------------+--------------+-----------+-----------+----------------------+
|                  ALIAS                  | FINGERPRINT  | PUBLIC |                  DESCRIPTION                  | ARCHITECTURE |   TYPE    |   SIZE    |     UPLOAD DATE      |
+-----------------------------------------+--------------+--------+-----------------------------------------------+--------------+-----------+-----------+----------------------+
| yunohost/trixie-unstable/appci (1 more) | 500dedd061a0 | yes    | yunohost trixie-unstable amd64 (202411120627) | x86_64       | CONTAINER | 369.71MiB | 2024/11/12 01:00 CET |
+-----------------------------------------+--------------+--------+-----------------------------------------------+--------------+-----------+-----------+----------------------+

but :

# incus image copy yunohost:yunohost/trixie-unstable/appci local: --copy-aliases --auto-update --debug
DEBUG  [2024-11-14T00:12:23+01:00] Connecting to a remote simplestreams server   URL="https://repo.yunohost.org/incus/"
DEBUG  [2024-11-14T00:12:23+01:00] Connecting to a local Incus over a Unix socket 
DEBUG  [2024-11-14T00:12:23+01:00] Sending request to Incus                      etag= method=GET url="http://unix.socket/1.0"
DEBUG  [2024-11-14T00:12:23+01:00] Got response struct from Incus               
DEBUG  [2024-11-14T00:12:23+01:00] 
    {
        "config": {},
        "api_extensions": [
            "storage_zfs_remove_snapshots",
            "container_host_shutdown_timeout",
            "container_stop_priority",
            "container_syscall_filtering",
            "auth_pki",
            "container_last_used_at",
            "etag",
            "patch",
            "usb_devices",
            "https_allowed_credentials",
            "image_compression_algorithm",
            "directory_manipulation",
            "container_cpu_time",
            "storage_zfs_use_refquota",
            "storage_lvm_mount_options",
            "network",
            "profile_usedby",
            "container_push",
            "container_exec_recording",
            "certificate_update",
            "container_exec_signal_handling",
            "gpu_devices",
            "container_image_properties",
            "migration_progress",
            "id_map",
            "network_firewall_filtering",
            "network_routes",
            "storage",
            "file_delete",
            "file_append",
            "network_dhcp_expiry",
            "storage_lvm_vg_rename",
            "storage_lvm_thinpool_rename",
            "network_vlan",
            "image_create_aliases",
            "container_stateless_copy",
            "container_only_migration",
            "storage_zfs_clone_copy",
            "unix_device_rename",
            "storage_lvm_use_thinpool",
            "storage_rsync_bwlimit",
            "network_vxlan_interface",
            "storage_btrfs_mount_options",
            "entity_description",
            "image_force_refresh",
            "storage_lvm_lv_resizing",
            "id_map_base",
            "file_symlinks",
            "container_push_target",
            "network_vlan_physical",
            "storage_images_delete",
            "container_edit_metadata",
            "container_snapshot_stateful_migration",
            "storage_driver_ceph",
            "storage_ceph_user_name",
            "resource_limits",
            "storage_volatile_initial_source",
            "storage_ceph_force_osd_reuse",
            "storage_block_filesystem_btrfs",
            "resources",
            "kernel_limits",
            "storage_api_volume_rename",
            "network_sriov",
            "console",
            "restrict_dev_incus",
            "migration_pre_copy",
            "infiniband",
            "dev_incus_events",
            "proxy",
            "network_dhcp_gateway",
            "file_get_symlink",
            "network_leases",
            "unix_device_hotplug",
            "storage_api_local_volume_handling",
            "operation_description",
            "clustering",
            "event_lifecycle",
            "storage_api_remote_volume_handling",
            "nvidia_runtime",
            "container_mount_propagation",
            "container_backup",
            "dev_incus_images",
            "container_local_cross_pool_handling",
            "proxy_unix",
            "proxy_udp",
            "clustering_join",
            "proxy_tcp_udp_multi_port_handling",
            "network_state",
            "proxy_unix_dac_properties",
            "container_protection_delete",
            "unix_priv_drop",
            "pprof_http",
            "proxy_haproxy_protocol",
            "network_hwaddr",
            "proxy_nat",
            "network_nat_order",
            "container_full",
            "backup_compression",
            "nvidia_runtime_config",
            "storage_api_volume_snapshots",
            "storage_unmapped",
            "projects",
            "network_vxlan_ttl",
            "container_incremental_copy",
            "usb_optional_vendorid",
            "snapshot_scheduling",
            "snapshot_schedule_aliases",
            "container_copy_project",
            "clustering_server_address",
            "clustering_image_replication",
            "container_protection_shift",
            "snapshot_expiry",
            "container_backup_override_pool",
            "snapshot_expiry_creation",
            "network_leases_location",
            "resources_cpu_socket",
            "resources_gpu",
            "resources_numa",
            "kernel_features",
            "id_map_current",
            "event_location",
            "storage_api_remote_volume_snapshots",
            "network_nat_address",
            "container_nic_routes",
            "cluster_internal_copy",
            "seccomp_notify",
            "lxc_features",
            "container_nic_ipvlan",
            "network_vlan_sriov",
            "storage_cephfs",
            "container_nic_ipfilter",
            "resources_v2",
            "container_exec_user_group_cwd",
            "container_syscall_intercept",
            "container_disk_shift",
            "storage_shifted",
            "resources_infiniband",
            "daemon_storage",
            "instances",
            "image_types",
            "resources_disk_sata",
            "clustering_roles",
            "images_expiry",
            "resources_network_firmware",
            "backup_compression_algorithm",
            "ceph_data_pool_name",
            "container_syscall_intercept_mount",
            "compression_squashfs",
            "container_raw_mount",
            "container_nic_routed",
            "container_syscall_intercept_mount_fuse",
            "container_disk_ceph",
            "virtual-machines",
            "image_profiles",
            "clustering_architecture",
            "resources_disk_id",
            "storage_lvm_stripes",
            "vm_boot_priority",
            "unix_hotplug_devices",
            "api_filtering",
            "instance_nic_network",
            "clustering_sizing",
            "firewall_driver",
            "projects_limits",
            "container_syscall_intercept_hugetlbfs",
            "limits_hugepages",
            "container_nic_routed_gateway",
            "projects_restrictions",
            "custom_volume_snapshot_expiry",
            "volume_snapshot_scheduling",
            "trust_ca_certificates",
            "snapshot_disk_usage",
            "clustering_edit_roles",
            "container_nic_routed_host_address",
            "container_nic_ipvlan_gateway",
            "resources_usb_pci",
            "resources_cpu_threads_numa",
            "resources_cpu_core_die",
            "api_os",
            "container_nic_routed_host_table",
            "container_nic_ipvlan_host_table",
            "container_nic_ipvlan_mode",
            "resources_system",
            "images_push_relay",
            "network_dns_search",
            "container_nic_routed_limits",
            "instance_nic_bridged_vlan",
            "network_state_bond_bridge",
            "usedby_consistency",
            "custom_block_volumes",
            "clustering_failure_domains",
            "resources_gpu_mdev",
            "console_vga_type",
            "projects_limits_disk",
            "network_type_macvlan",
            "network_type_sriov",
            "container_syscall_intercept_bpf_devices",
            "network_type_ovn",
            "projects_networks",
            "projects_networks_restricted_uplinks",
            "custom_volume_backup",
            "backup_override_name",
            "storage_rsync_compression",
            "network_type_physical",
            "network_ovn_external_subnets",
            "network_ovn_nat",
            "network_ovn_external_routes_remove",
            "tpm_device_type",
            "storage_zfs_clone_copy_rebase",
            "gpu_mdev",
            "resources_pci_iommu",
            "resources_network_usb",
            "resources_disk_address",
            "network_physical_ovn_ingress_mode",
            "network_ovn_dhcp",
            "network_physical_routes_anycast",
            "projects_limits_instances",
            "network_state_vlan",
            "instance_nic_bridged_port_isolation",
            "instance_bulk_state_change",
            "network_gvrp",
            "instance_pool_move",
            "gpu_sriov",
            "pci_device_type",
            "storage_volume_state",
            "network_acl",
            "migration_stateful",
            "disk_state_quota",
            "storage_ceph_features",
            "projects_compression",
            "projects_images_remote_cache_expiry",
            "certificate_project",
            "network_ovn_acl",
            "projects_images_auto_update",
            "projects_restricted_cluster_target",
            "images_default_architecture",
            "network_ovn_acl_defaults",
            "gpu_mig",
            "project_usage",
            "network_bridge_acl",
            "warnings",
            "projects_restricted_backups_and_snapshots",
            "clustering_join_token",
            "clustering_description",
            "server_trusted_proxy",
            "clustering_update_cert",
            "storage_api_project",
            "server_instance_driver_operational",
            "server_supported_storage_drivers",
            "event_lifecycle_requestor_address",
            "resources_gpu_usb",
            "clustering_evacuation",
            "network_ovn_nat_address",
            "network_bgp",
            "network_forward",
            "custom_volume_refresh",
            "network_counters_errors_dropped",
            "metrics",
            "image_source_project",
            "clustering_config",
            "network_peer",
            "linux_sysctl",
            "network_dns",
            "ovn_nic_acceleration",
            "certificate_self_renewal",
            "instance_project_move",
            "storage_volume_project_move",
            "cloud_init",
            "network_dns_nat",
            "database_leader",
            "instance_all_projects",
            "clustering_groups",
            "ceph_rbd_du",
            "instance_get_full",
            "qemu_metrics",
            "gpu_mig_uuid",
            "event_project",
            "clustering_evacuation_live",
            "instance_allow_inconsistent_copy",
            "network_state_ovn",
            "storage_volume_api_filtering",
            "image_restrictions",
            "storage_zfs_export",
            "network_dns_records",
            "storage_zfs_reserve_space",
            "network_acl_log",
            "storage_zfs_blocksize",
            "metrics_cpu_seconds",
            "instance_snapshot_never",
            "certificate_token",
            "instance_nic_routed_neighbor_probe",
            "event_hub",
            "agent_nic_config",
            "projects_restricted_intercept",
            "metrics_authentication",
            "images_target_project",
            "images_all_projects",
            "cluster_migration_inconsistent_copy",
            "cluster_ovn_chassis",
            "container_syscall_intercept_sched_setscheduler",
            "storage_lvm_thinpool_metadata_size",
            "storage_volume_state_total",
            "instance_file_head",
            "instances_nic_host_name",
            "image_copy_profile",
            "container_syscall_intercept_sysinfo",
            "clustering_evacuation_mode",
            "resources_pci_vpd",
            "qemu_raw_conf",
            "storage_cephfs_fscache",
            "network_load_balancer",
            "vsock_api",
            "instance_ready_state",
            "network_bgp_holdtime",
            "storage_volumes_all_projects",
            "metrics_memory_oom_total",
            "storage_buckets",
            "storage_buckets_create_credentials",
            "metrics_cpu_effective_total",
            "projects_networks_restricted_access",
            "storage_buckets_local",
            "loki",
            "acme",
            "internal_metrics",
            "cluster_join_token_expiry",
            "remote_token_expiry",
            "init_preseed",
            "storage_volumes_created_at",
            "cpu_hotplug",
            "projects_networks_zones",
            "network_txqueuelen",
            "cluster_member_state",
            "instances_placement_scriptlet",
            "storage_pool_source_wipe",
            "zfs_block_mode",
            "instance_generation_id",
            "disk_io_cache",
            "amd_sev",
            "storage_pool_loop_resize",
            "migration_vm_live",
            "ovn_nic_nesting",
            "oidc",
            "network_ovn_l3only",
            "ovn_nic_acceleration_vdpa",
            "cluster_healing",
            "instances_state_total",
            "auth_user",
            "security_csm",
            "instances_rebuild",
            "numa_cpu_placement",
            "custom_volume_iso",
            "network_allocations",
            "zfs_delegate",
            "storage_api_remote_volume_snapshot_copy",
            "operations_get_query_all_projects",
            "metadata_configuration",
            "syslog_socket",
            "event_lifecycle_name_and_project",
            "instances_nic_limits_priority",
            "disk_initial_volume_configuration",
            "operation_wait",
            "image_restriction_privileged",
            "cluster_internal_custom_volume_copy",
            "disk_io_bus",
            "storage_cephfs_create_missing",
            "instance_move_config",
            "ovn_ssl_config",
            "certificate_description",
            "disk_io_bus_virtio_blk",
            "loki_config_instance",
            "instance_create_start",
            "clustering_evacuation_stop_options",
            "boot_host_shutdown_action",
            "agent_config_drive",
            "network_state_ovn_lr",
            "image_template_permissions",
            "storage_bucket_backup",
            "storage_lvm_cluster",
            "shared_custom_block_volumes",
            "auth_tls_jwt",
            "oidc_claim",
            "device_usb_serial",
            "numa_cpu_balanced",
            "image_restriction_nesting",
            "network_integrations",
            "instance_memory_swap_bytes",
            "network_bridge_external_create",
            "network_zones_all_projects",
            "storage_zfs_vdev",
            "container_migration_stateful",
            "profiles_all_projects",
            "instances_scriptlet_get_instances",
            "instances_scriptlet_get_cluster_members",
            "instances_scriptlet_get_project",
            "network_acl_stateless",
            "instance_state_started_at",
            "networks_all_projects",
            "network_acls_all_projects",
            "storage_buckets_all_projects",
            "resources_load",
            "instance_access",
            "project_access",
            "projects_force_delete",
            "resources_cpu_flags",
            "disk_io_bus_cache_filesystem",
            "instance_oci",
            "clustering_groups_config",
            "instances_lxcfs_per_instance",
            "clustering_groups_vm_cpu_definition",
            "disk_volume_subpath",
            "projects_limits_disk_pool",
            "network_ovn_isolated",
            "qemu_raw_qmp",
            "network_load_balancer_health_check",
            "oidc_scopes",
            "network_integrations_peer_name",
            "qemu_scriptlet",
            "instance_auto_restart",
            "storage_lvm_metadatasize",
            "ovn_nic_promiscuous",
            "ovn_nic_ip_address_none",
            "instances_state_os_info",
            "network_load_balancer_state",
            "instance_nic_macvlan_mode",
            "storage_lvm_cluster_create",
            "network_ovn_external_interfaces"
        ],
        "api_status": "stable",
        "api_version": "1.0",
        "auth": "trusted",
        "public": false,
        "auth_methods": [
            "tls"
        ],
        "auth_user_name": "root",
        "auth_user_method": "unix",
        "environment": {
            "addresses": [],
            "architectures": [
                "x86_64",
                "i686"
            ],
            "certificate": "-----BEGIN CERTIFICATE-----\nMIICQjCCAcmgAwIBAgIRAPNK9A1pSQhAfNQz+CJrxOkwCgYIKoZIzj0EAwMwSDEZ\nMBcGA1UEChMQTGludXggQ29udGFpbmVyczErMCkGA1UEAwwicm9vdEBjaS1hcHBz\nLXRlbXBsYXRlLnl1bm9ob3N0Lm9yZzAeFw0yNDExMDYxNjQxMTJaFw0zNDExMDQx\nNjQxMTJaMEgxGTAXBgNVBAoTEExpbnV4IENvbnRhaW5lcnMxKzApBgNVBAMMInJv\nb3RAY2ktYXBwcy10ZW1wbGF0ZS55dW5vaG9zdC5vcmcwdjAQBgcqhkjOPQIBBgUr\ngQQAIgNiAAQyJAW4NyvK9l2J6ViVxk48+W6f2H8McLJ911vvBHRfJQIYiggIqoCM\n/qK2LqEa+37wBl2DHKs2mgwscNcCc3AEbWFzHByJ6Ug1+Cyo8/ta1e10QUBlu8af\nxQDsfrO+T3GjdzB1MA4GA1UdDwEB/wQEAwIFoDATBgNVHSUEDDAKBggrBgEFBQcD\nATAMBgNVHRMBAf8EAjAAMEAGA1UdEQQ5MDeCHWNpLWFwcHMtdGVtcGxhdGUueXVu\nb2hvc3Qub3JnhwR/AAABhxAAAAAAAAAAAAAAAAAAAAABMAoGCCqGSM49BAMDA2cA\nMGQCMHUxSRquKEDmpeEREtg39IzlIrW8CLypKaf+m21Jagf06MnUw3xumEExDPY7\nWoDQTAIwb4/t8Je5FtecnnmsOwEbB8JFuOks2OrBNNasi4jKnPV/DLvYK1VO+JF7\nYoIsC9p8\n-----END CERTIFICATE-----\n",
            "certificate_fingerprint": "b5ecab9e9501758aed58618d0926744ff9870d0c44b8cc892c64dbbc378885b9",
            "driver": "lxc",
            "driver_version": "6.0.2",
            "firewall": "nftables",
            "kernel": "Linux",
            "kernel_architecture": "x86_64",
            "kernel_features": {
                "idmapped_mounts": "true",
                "netnsid_getifaddrs": "true",
                "seccomp_listener": "true",
                "seccomp_listener_continue": "true",
                "uevent_injection": "true",
                "unpriv_binfmt": "false",
                "unpriv_fscaps": "true"
            },
            "kernel_version": "6.1.0-26-amd64",
            "lxc_features": {
                "cgroup2": "true",
                "core_scheduling": "true",
                "devpts_fd": "true",
                "idmapped_mounts_v2": "true",
                "mount_injection_file": "true",
                "network_gateway_device_route": "true",
                "network_ipvlan": "true",
                "network_l2proxy": "true",
                "network_phys_macvlan_mtu": "true",
                "network_veth_router": "true",
                "pidfd": "true",
                "seccomp_allow_deny_syntax": "true",
                "seccomp_notify": "true",
                "seccomp_proxy_send_notify_fd": "true"
            },
            "os_name": "Debian GNU/Linux",
            "os_version": "12",
            "project": "default",
            "server": "incus",
            "server_clustered": false,
            "server_event_mode": "full-mesh",
            "server_name": "ci-apps-trixie.yunohost.org",
            "server_pid": 1939726,
            "server_version": "6.6",
            "storage": "btrfs",
            "storage_version": "6.2",
            "storage_supported_drivers": [
                {
                    "Name": "dir",
                    "Version": "1",
                    "Remote": false
                },
                {
                    "Name": "lvm",
                    "Version": "2.03.16(2) (2022-05-18) / 1.02.185 (2022-05-18) / 4.47.0",
                    "Remote": false
                },
                {
                    "Name": "lvmcluster",
                    "Version": "2.03.16(2) (2022-05-18) / 1.02.185 (2022-05-18) / 4.47.0",
                    "Remote": true
                },
                {
                    "Name": "btrfs",
                    "Version": "6.2",
                    "Remote": false
                }
            ]
        }
    } 
DEBUG  [2024-11-14T00:12:23+01:00] Connected to the websocket: ws://unix.socket/1.0/events 
DEBUG  [2024-11-14T00:12:23+01:00] Sending request to Incus                      etag= method=POST url="http://unix.socket/1.0/images"
DEBUG  [2024-11-14T00:12:23+01:00] 
    {
        "auto_update": true,
        "properties": null,
        "public": false,
        "expires_at": "0001-01-01T00:00:00Z",
        "profiles": null,
        "filename": "",
        "source": {
            "alias": "",
            "certificate": "",
            "protocol": "simplestreams",
            "server": "https://repo.yunohost.org/incus",
            "image_type": "",
            "mode": "pull",
            "type": "image",
            "url": "",
            "name": "",
            "fingerprint": "yunohost/trixie-unstable/appci",
            "secret": "",
            "project": ""
        },
        "compression_algorithm": "",
        "aliases": null
    } 
DEBUG  [2024-11-14T00:12:23+01:00] Got operation from Incus                     
DEBUG  [2024-11-14T00:12:23+01:00] 
    {
        "id": "8a8365f8-e3eb-46aa-bee4-9475b245a558",
        "class": "task",
        "description": "Downloading image",
        "created_at": "2024-11-14T00:12:23.453309982+01:00",
        "updated_at": "2024-11-14T00:12:23.453309982+01:00",
        "status": "Running",
        "status_code": 103,
        "resources": {},
        "metadata": null,
        "may_cancel": false,
        "err": "",
        "location": "none"
    } 
DEBUG  [2024-11-14T00:12:23+01:00] Sending request to Incus                      etag= method=GET url="http://unix.socket/1.0/operations/8a8365f8-e3eb-46aa-bee4-9475b245a558"
DEBUG  [2024-11-14T00:12:23+01:00] Got response struct from Incus               
DEBUG  [2024-11-14T00:12:23+01:00] 
    {
        "id": "8a8365f8-e3eb-46aa-bee4-9475b245a558",
        "class": "task",
        "description": "Downloading image",
        "created_at": "2024-11-14T00:12:23.453309982+01:00",
        "updated_at": "2024-11-14T00:12:23.453309982+01:00",
        "status": "Running",
        "status_code": 103,
        "resources": {},
        "metadata": null,
        "may_cancel": false,
        "err": "",
        "location": "none"
    } 
Error: Failed remote image download: Unable to fetch https://repo.yunohost.org/incus/images/1b46a0dfacf7eedb0bfef3f9a3877b54ed9185c8960de67e1b53d2e0e8d52f91.incus_combined.tar.gz: 404 Not Found

It looks like the incus daemon caches the index of the repository but doesn't know when to refresh it. When reading the logs of the nginx server serving the simplestreams repository, i can see the requests for the image, but not for the images.json index. You can see the sha1 mismatch (500dedd061a0 when listing, 1b46a0dfacf7eedb0bfef3f9a3877b54ed9185c8960de67e1b53d2e0e8d52f91 when fetching)

(I'm going to sleep, hence the very short issue, but feel free to request more info, i'll reply tomorrow :D )

lxc / incus

incus remote list shows the new image, but incus image copy tries to fetch the old image #1373