/var/lib/incus dataset of an encrypted, delegated guest mounted on top of host mount

[almereyda]

Is there an existing issue for this?

• There is no existing issue for this bug

Is this happening on an up to date version of Incus?

• This is happening on a supported version of Incus

Incus system details

config:
  images.auto_update_interval: "0"
api_extensions:
- storage_zfs_remove_snapshots
- container_host_shutdown_timeout
- container_stop_priority
- container_syscall_filtering
- auth_pki
- container_last_used_at
- etag
- patch
- usb_devices
- https_allowed_credentials
- image_compression_algorithm
- directory_manipulation
- container_cpu_time
- storage_zfs_use_refquota
- storage_lvm_mount_options
- network
- profile_usedby
- container_push
- container_exec_recording
- certificate_update
- container_exec_signal_handling
- gpu_devices
- container_image_properties
- migration_progress
- id_map
- network_firewall_filtering
- network_routes
- storage
- file_delete
- file_append
- network_dhcp_expiry
- storage_lvm_vg_rename
- storage_lvm_thinpool_rename
- network_vlan
- image_create_aliases
- container_stateless_copy
- container_only_migration
- storage_zfs_clone_copy
- unix_device_rename
- storage_lvm_use_thinpool
- storage_rsync_bwlimit
- network_vxlan_interface
- storage_btrfs_mount_options
- entity_description
- image_force_refresh
- storage_lvm_lv_resizing
- id_map_base
- file_symlinks
- container_push_target
- network_vlan_physical
- storage_images_delete
- container_edit_metadata
- container_snapshot_stateful_migration
- storage_driver_ceph
- storage_ceph_user_name
- resource_limits
- storage_volatile_initial_source
- storage_ceph_force_osd_reuse
- storage_block_filesystem_btrfs
- resources
- kernel_limits
- storage_api_volume_rename
- network_sriov
- console
- restrict_dev_incus
- migration_pre_copy
- infiniband
- dev_incus_events
- proxy
- network_dhcp_gateway
- file_get_symlink
- network_leases
- unix_device_hotplug
- storage_api_local_volume_handling
- operation_description
- clustering
- event_lifecycle
- storage_api_remote_volume_handling
- nvidia_runtime
- container_mount_propagation
- container_backup
- dev_incus_images
- container_local_cross_pool_handling
- proxy_unix
- proxy_udp
- clustering_join
- proxy_tcp_udp_multi_port_handling
- network_state
- proxy_unix_dac_properties
- container_protection_delete
- unix_priv_drop
- pprof_http
- proxy_haproxy_protocol
- network_hwaddr
- proxy_nat
- network_nat_order
- container_full
- backup_compression
- nvidia_runtime_config
- storage_api_volume_snapshots
- storage_unmapped
- projects
- network_vxlan_ttl
- container_incremental_copy
- usb_optional_vendorid
- snapshot_scheduling
- snapshot_schedule_aliases
- container_copy_project
- clustering_server_address
- clustering_image_replication
- container_protection_shift
- snapshot_expiry
- container_backup_override_pool
- snapshot_expiry_creation
- network_leases_location
- resources_cpu_socket
- resources_gpu
- resources_numa
- kernel_features
- id_map_current
- event_location
- storage_api_remote_volume_snapshots
- network_nat_address
- container_nic_routes
- cluster_internal_copy
- seccomp_notify
- lxc_features
- container_nic_ipvlan
- network_vlan_sriov
- storage_cephfs
- container_nic_ipfilter
- resources_v2
- container_exec_user_group_cwd
- container_syscall_intercept
- container_disk_shift
- storage_shifted
- resources_infiniband
- daemon_storage
- instances
- image_types
- resources_disk_sata
- clustering_roles
- images_expiry
- resources_network_firmware
- backup_compression_algorithm
- ceph_data_pool_name
- container_syscall_intercept_mount
- compression_squashfs
- container_raw_mount
- container_nic_routed
- container_syscall_intercept_mount_fuse
- container_disk_ceph
- virtual-machines
- image_profiles
- clustering_architecture
- resources_disk_id
- storage_lvm_stripes
- vm_boot_priority
- unix_hotplug_devices
- api_filtering
- instance_nic_network
- clustering_sizing
- firewall_driver
- projects_limits
- container_syscall_intercept_hugetlbfs
- limits_hugepages
- container_nic_routed_gateway
- projects_restrictions
- custom_volume_snapshot_expiry
- volume_snapshot_scheduling
- trust_ca_certificates
- snapshot_disk_usage
- clustering_edit_roles
- container_nic_routed_host_address
- container_nic_ipvlan_gateway
- resources_usb_pci
- resources_cpu_threads_numa
- resources_cpu_core_die
- api_os
- container_nic_routed_host_table
- container_nic_ipvlan_host_table
- container_nic_ipvlan_mode
- resources_system
- images_push_relay
- network_dns_search
- container_nic_routed_limits
- instance_nic_bridged_vlan
- network_state_bond_bridge
- usedby_consistency
- custom_block_volumes
- clustering_failure_domains
- resources_gpu_mdev
- console_vga_type
- projects_limits_disk
- network_type_macvlan
- network_type_sriov
- container_syscall_intercept_bpf_devices
- network_type_ovn
- projects_networks
- projects_networks_restricted_uplinks
- custom_volume_backup
- backup_override_name
- storage_rsync_compression
- network_type_physical
- network_ovn_external_subnets
- network_ovn_nat
- network_ovn_external_routes_remove
- tpm_device_type
- storage_zfs_clone_copy_rebase
- gpu_mdev
- resources_pci_iommu
- resources_network_usb
- resources_disk_address
- network_physical_ovn_ingress_mode
- network_ovn_dhcp
- network_physical_routes_anycast
- projects_limits_instances
- network_state_vlan
- instance_nic_bridged_port_isolation
- instance_bulk_state_change
- network_gvrp
- instance_pool_move
- gpu_sriov
- pci_device_type
- storage_volume_state
- network_acl
- migration_stateful
- disk_state_quota
- storage_ceph_features
- projects_compression
- projects_images_remote_cache_expiry
- certificate_project
- network_ovn_acl
- projects_images_auto_update
- projects_restricted_cluster_target
- images_default_architecture
- network_ovn_acl_defaults
- gpu_mig
- project_usage
- network_bridge_acl
- warnings
- projects_restricted_backups_and_snapshots
- clustering_join_token
- clustering_description
- server_trusted_proxy
- clustering_update_cert
- storage_api_project
- server_instance_driver_operational
- server_supported_storage_drivers
- event_lifecycle_requestor_address
- resources_gpu_usb
- clustering_evacuation
- network_ovn_nat_address
- network_bgp
- network_forward
- custom_volume_refresh
- network_counters_errors_dropped
- metrics
- image_source_project
- clustering_config
- network_peer
- linux_sysctl
- network_dns
- ovn_nic_acceleration
- certificate_self_renewal
- instance_project_move
- storage_volume_project_move
- cloud_init
- network_dns_nat
- database_leader
- instance_all_projects
- clustering_groups
- ceph_rbd_du
- instance_get_full
- qemu_metrics
- gpu_mig_uuid
- event_project
- clustering_evacuation_live
- instance_allow_inconsistent_copy
- network_state_ovn
- storage_volume_api_filtering
- image_restrictions
- storage_zfs_export
- network_dns_records
- storage_zfs_reserve_space
- network_acl_log
- storage_zfs_blocksize
- metrics_cpu_seconds
- instance_snapshot_never
- certificate_token
- instance_nic_routed_neighbor_probe
- event_hub
- agent_nic_config
- projects_restricted_intercept
- metrics_authentication
- images_target_project
- images_all_projects
- cluster_migration_inconsistent_copy
- cluster_ovn_chassis
- container_syscall_intercept_sched_setscheduler
- storage_lvm_thinpool_metadata_size
- storage_volume_state_total
- instance_file_head
- instances_nic_host_name
- image_copy_profile
- container_syscall_intercept_sysinfo
- clustering_evacuation_mode
- resources_pci_vpd
- qemu_raw_conf
- storage_cephfs_fscache
- network_load_balancer
- vsock_api
- instance_ready_state
- network_bgp_holdtime
- storage_volumes_all_projects
- metrics_memory_oom_total
- storage_buckets
- storage_buckets_create_credentials
- metrics_cpu_effective_total
- projects_networks_restricted_access
- storage_buckets_local
- loki
- acme
- internal_metrics
- cluster_join_token_expiry
- remote_token_expiry
- init_preseed
- storage_volumes_created_at
- cpu_hotplug
- projects_networks_zones
- network_txqueuelen
- cluster_member_state
- instances_placement_scriptlet
- storage_pool_source_wipe
- zfs_block_mode
- instance_generation_id
- disk_io_cache
- amd_sev
- storage_pool_loop_resize
- migration_vm_live
- ovn_nic_nesting
- oidc
- network_ovn_l3only
- ovn_nic_acceleration_vdpa
- cluster_healing
- instances_state_total
- auth_user
- security_csm
- instances_rebuild
- numa_cpu_placement
- custom_volume_iso
- network_allocations
- zfs_delegate
- storage_api_remote_volume_snapshot_copy
- operations_get_query_all_projects
- metadata_configuration
- syslog_socket
- event_lifecycle_name_and_project
- instances_nic_limits_priority
- disk_initial_volume_configuration
- operation_wait
- image_restriction_privileged
- cluster_internal_custom_volume_copy
- disk_io_bus
- storage_cephfs_create_missing
- instance_move_config
- ovn_ssl_config
- certificate_description
- disk_io_bus_virtio_blk
- loki_config_instance
- instance_create_start
- clustering_evacuation_stop_options
- boot_host_shutdown_action
- agent_config_drive
- network_state_ovn_lr
- image_template_permissions
- storage_bucket_backup
- storage_lvm_cluster
- shared_custom_block_volumes
- auth_tls_jwt
- oidc_claim
- device_usb_serial
- numa_cpu_balanced
- image_restriction_nesting
- network_integrations
- instance_memory_swap_bytes
- network_bridge_external_create
- network_zones_all_projects
- storage_zfs_vdev
- container_migration_stateful
- profiles_all_projects
- instances_scriptlet_get_instances
- instances_scriptlet_get_cluster_members
- instances_scriptlet_get_project
- network_acl_stateless
- instance_state_started_at
- networks_all_projects
- network_acls_all_projects
- storage_buckets_all_projects
- resources_load
- instance_access
- project_access
- projects_force_delete
- resources_cpu_flags
- disk_io_bus_cache_filesystem
- instance_oci
- clustering_groups_config
- instances_lxcfs_per_instance
- clustering_groups_vm_cpu_definition
- disk_volume_subpath
- projects_limits_disk_pool
- network_ovn_isolated
- qemu_raw_qmp
- network_load_balancer_health_check
- oidc_scopes
- network_integrations_peer_name
- qemu_scriptlet
- instance_auto_restart
- storage_lvm_metadatasize
- ovn_nic_promiscuous
- ovn_nic_ip_address_none
- instances_state_os_info
- network_load_balancer_state
- instance_nic_macvlan_mode
- storage_lvm_cluster_create
- network_ovn_external_interfaces
- instances_scriptlet_get_instances_count
- cluster_rebalance
- custom_volume_refresh_exclude_older_snapshots
- storage_initial_owner
- storage_live_migration
- instance_console_screenshot
- image_import_alias
- authorization_scriptlet
- console_force
- network_ovn_state_addresses
- network_bridge_acl_devices
- instance_debug_memory
- init_preseed_storage_volumes
- init_preseed_profile_project
- instance_nic_routed_host_address
- instance_smbios11
- api_filtering_extended
- acme_dns01
- security_iommu
- network_ipv4_dhcp_routes
- network_state_ovn_ls
- network_dns_nameservers
- acme_http01_port
- network_ovn_ipv4_dhcp_expiry
- instance_state_cpu_time
- network_io_bus
- disk_io_bus_usb
- storage_driver_linstor
- instance_oci_entrypoint
- network_address_set
- server_logging
- network_forward_snat
- memory_hotplug
- instance_nic_routed_host_tables
- instance_publish_split
- init_preseed_certificates
- custom_volume_sftp
- network_ovn_external_nic_address
- network_physical_gateway_hwaddr
- backup_s3_upload
- snapshot_manual_expiry
- resources_cpu_address_sizes
- disk_attached
- limits_memory_hotplug
- disk_wwn
- server_logging_webhook
- storage_driver_truenas
- container_disk_tmpfs
- instance_limits_oom
- backup_override_config
- network_ovn_tunnels
- init_preseed_cluster_groups
- usb_attached
- backup_iso
- instance_systemd_credentials
- cluster_group_usedby
- bpf_token_delegation
- file_storage_volume
- network_hwaddr_pattern
api_status: stable
api_version: "1.0"
auth: trusted
public: false
auth_methods:
- tls
auth_user_name: yala
auth_user_method: unix
environment:
  addresses: []
  architectures:
  - x86_64
  - i686
  certificate: |
    -----BEGIN CERTIFICATE-----
    …
    -----END CERTIFICATE-----
  certificate_fingerprint: …
  driver: lxc | qemu
  driver_version: 6.0.5 | 10.1.2
  firewall: nftables
  kernel: Linux
  kernel_architecture: x86_64
  kernel_features:
    idmapped_mounts: "true"
    netnsid_getifaddrs: "true"
    seccomp_listener: "true"
    seccomp_listener_continue: "true"
    uevent_injection: "true"
    unpriv_binfmt: "true"
    unpriv_fscaps: "true"
  kernel_version: 6.17.0-6-generic
  lxc_features:
    cgroup2: "true"
    core_scheduling: "true"
    devpts_fd: "true"
    idmapped_mounts_v2: "true"
    mount_injection_file: "true"
    network_gateway_device_route: "true"
    network_ipvlan: "true"
    network_l2proxy: "true"
    network_phys_macvlan_mtu: "true"
    network_veth_router: "true"
    pidfd: "true"
    seccomp_allow_deny_syntax: "true"
    seccomp_notify: "true"
    seccomp_proxy_send_notify_fd: "true"
  os_name: Ubuntu
  os_version: "25.10"
  project: default
  server: incus
  server_clustered: false
  server_event_mode: full-mesh
  server_name: ganglion
  server_pid: 15328
  server_version: "6.18"
  storage: dir | zfs
  storage_version: 1 | 2.3.4-1ubuntu2
  storage_supported_drivers:
  - name: dir
    version: "1"
    remote: false
  - name: lvm
    version: 2.03.31(2) (2025-02-27) / 1.02.205 (2025-02-27) / 4.50.0
    remote: false
  - name: lvmcluster
    version: 2.03.31(2) (2025-02-27) / 1.02.205 (2025-02-27) / 4.50.0
    remote: true
  - name: truenas
    version: 0.7.3
    remote: true
  - name: zfs
    version: 2.3.4-1ubuntu2
    remote: false
  - name: btrfs
    version: "6.16"
    remote: false

Instance details

N/A

Instance log

N/A

Current behavior

When launching Incus on a Ubuntu 25.10 host that has Ubuntu guests with Incus which use

1. ZFS delegation
2. provide a custom, during boot still locked encryptionroot in a canmount=off dataset, e.g. rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA
3. and within that have a canmount=noauto dataset for an Incus configuration state at /var/lib/incus, rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus

then Incus will not start during boot. Starting it manually forces the decryption of the encryptionroot, else that is aborted. In case one proceeds supplying the key material, still during launch, it will recursively mount that dataset on the host system, including the Incus configuration store /var/lib/incus.

After launch, Incus will run the Incus instance from inside the container on the host. All storage pool configuration remains intact, due to the absolute paths of datasets being the same outside and within guests.

While stopping Incus works, unmounting the dataset does not. The Incus instance from the host remains inaccessible.

Expected behavior

1. Incus starts without forcing the decryption of encryptionroots in storage volumes from instances that are provided with ZFS delegation.
2. Datasets below the storage volume of an instance are not mounted on the host system. canmount=noauto is respected.
3. Decryption of the encrypted dataset below the dataset of the storage volume of the instance can happen from within the guest and is not enforced beforehand.

Steps to reproduce

This is a very rare edge case, where a serias of conditions have to be present.

• An Ubuntu Incus root host using ZFS
  • using a separate dataset for /var/lib/incus.
• A ZFS-delegated Ubuntu instance with an Incus host, that
  • brings its own encryptionroot within the instance's storage volume
  • below that uses a separate dataset for /var/lib/incus

NAME                                                                                                                                                       USED  AVAIL  REFER  MOUNTPOINT
rpool/ROOT/ubuntu_d4psvq/var/incus                                                                                                                        2.93G  97.2G    96K  legacy
rpool/ROOT/ubuntu_d4psvq/var/incus/buckets                                                                                                                 288K  97.2G    96K  legacy
rpool/ROOT/ubuntu_d4psvq/var/incus/buckets/default_yala                                                                                                    192K  97.2G   192K  legacy
rpool/ROOT/ubuntu_d4psvq/var/incus/containers                                                                                                             1.94G  97.2G    96K  legacy
rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u0                                                                                                          61.8M  97.2G   355M  legacy
rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1                                                                                                          1.88G  97.2G  1.21G  legacy
rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/ROOT                                                                                                      372K  97.2G   228K  /srv/ROOT
rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA                                                                                                  976M  97.2G   192K  legacy
rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus                                                                                            975M  97.2G   243M  /var/lib/incus
rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus/storage-pools                                                                              732M  97.2G   496K  /var/lib/incus/storage-pools
rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus/storage-pools/u1                                                                           732M  97.2G   192K  legacy
rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus/storage-pools/u1/buckets                                                                   192K  97.2G   192K  legacy
rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus/storage-pools/u1/containers                                                                192K  97.2G   192K  legacy
rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus/storage-pools/u1/custom                                                                    192K  97.2G   192K  legacy
rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus/storage-pools/u1/deleted                                                                  1.12M  97.2G   192K  legacy
rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus/storage-pools/u1/deleted/buckets                                                           192K  97.2G   192K  legacy
rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus/storage-pools/u1/deleted/containers                                                        192K  97.2G   192K  legacy
rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus/storage-pools/u1/deleted/custom                                                            192K  97.2G   192K  legacy
rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus/storage-pools/u1/deleted/images                                                            192K  97.2G   192K  legacy
rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus/storage-pools/u1/deleted/virtual-machines                                                  192K  97.2G   192K  legacy
rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus/storage-pools/u1/images                                                                    730M  97.2G   192K  legacy
rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus/storage-pools/u1/images/8c0bedc41b796d32d0f80d0b7b40be8592e1389bd1a92d4fa0c4acee9d907854   373M  97.2G   373M  legacy
rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus/storage-pools/u1/images/d753ba4a12fb3f773e41e4964973964ea2c7441add6b3155582ca2786fb40cc8   357M  97.2G   357M  legacy
rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus/storage-pools/u1/virtual-machines                                                          192K  97.2G   192K  legacy
rpool/ROOT/ubuntu_d4psvq/var/incus/custom                                                                                                                  192K  97.2G    96K  legacy
rpool/ROOT/ubuntu_d4psvq/var/incus/custom/default_u0-data                                                                                                   96K  97.2G    96K  legacy
rpool/ROOT/ubuntu_d4psvq/var/incus/deleted                                                                                                                 301M  97.2G    96K  legacy
rpool/ROOT/ubuntu_d4psvq/var/incus/deleted/buckets                                                                                                          96K  97.2G    96K  legacy
rpool/ROOT/ubuntu_d4psvq/var/incus/deleted/containers                                                                                                       96K  97.2G    96K  legacy
rpool/ROOT/ubuntu_d4psvq/var/incus/deleted/custom                                                                                                           96K  97.2G    96K  legacy
rpool/ROOT/ubuntu_d4psvq/var/incus/deleted/images                                                                                                          300M  97.2G    96K  legacy
rpool/ROOT/ubuntu_d4psvq/var/incus/deleted/images/73fe6edfe11e491e208df76f01cbc7303c8ad203a40d0bae2c9a7ad1cfe4e750                                         300M  97.2G   300M  legacy
rpool/ROOT/ubuntu_d4psvq/var/incus/deleted/virtual-machines                                                                                                 96K  97.2G    96K  legacy
rpool/ROOT/ubuntu_d4psvq/var/incus/images                                                                                                                  710M  97.2G    96K  legacy
rpool/ROOT/ubuntu_d4psvq/var/incus/images/058f298087312e0b7bd685520fb66b9f520bf2123bec237da64b79b04a5363ea                                                51.4M  97.2G  51.4M  legacy
rpool/ROOT/ubuntu_d4psvq/var/incus/images/5f69e85b1d16b6f0b5b4ba87a7020ae56463a32138345656b0ca78d74ac96a70                                                 329M  97.2G   329M  legacy
rpool/ROOT/ubuntu_d4psvq/var/incus/images/8c0bedc41b796d32d0f80d0b7b40be8592e1389bd1a92d4fa0c4acee9d907854                                                 329M  97.2G   329M  legacy
rpool/ROOT/ubuntu_d4psvq/var/incus/virtual-machines                                                                                                         96K  97.2G    96K  legacy

$ sudo zfs list -o name,canmount,mountpoint,mounted,encryptionroot rpool/ROOT/ubuntu_d4psvq/var/incus{,/containers/u1,/containers/u1/USERDATA,/containers/u1/USERDATA/incus}
NAME                                                             CANMOUNT  MOUNTPOINT      MOUNTED  ENCROOT
rpool/ROOT/ubuntu_d4psvq/var/incus                               on        legacy          no       -
rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1                 noauto    legacy          no       -
rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA        off       legacy          no       rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA
rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus  noauto    /var/lib/incus  yes      rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA

Before the reboot, these were regular ZFS mounts, where allowed.

In this scenario, starting Incus on the host system will look like:

$ incus version    
Client Version: 6.18
Server version: nicht erreichbar
$ sudo systemctl start incus.service incus.socket
🔐 Enter passphrase for rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA: ••••••••

This is zfs-load-key@rpool-ROOT-ubuntu_d4psvq-var-incus-containers-u1-USERDATA.service speaking. While initially being failed after boot, it then switches to active. It is not directly referenced from the incus.service unit. No respective mount units are created, but it could explain the side-effects on the host system.

• zfs-mount-generator 'ignore' option only covers mount unit, not key-loading unit openzfs/zfs#17404

1. Which process enforced the start of the zfs-load-key@ unit?
2. Which process mounted the /var/lib/incus dataset from within the container, in presence of canmount=noauto and in absence of a mount unit?

• zfs-mount-generator should use WantsMountsFor= instead of RequiresMountsFor= openzfs/zfs#15884

The startup continues as expected, but surprisingly incus list does not show the u1 instance, but incus storage list shows the u1 storage pool from the Incus host inside the u1 instance.

$ incus ls     
+------+-------+------+------+------+-----------+
| NAME | STATE | IPV4 | IPV6 | TYPE | SNAPSHOTS |
+------+-------+------+------+------+-----------+

$ incus storage ls
+---------+--------+--------------+---------+---------+
|  NAME   | DRIVER | BESCHREIBUNG | USED BY |  STATE  |
+---------+--------+--------------+---------+---------+
| default | dir    |              | 1       | CREATED |
+---------+--------+--------------+---------+---------+
| u1      | zfs    |              | 2       | CREATED |
+---------+--------+--------------+---------+---------+

$ incus storage show u1 | yq '@json' | jq '{config, driver}' | yq -P
config:
  source: rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus/storage-pools/u1
  volatile.initial_source: rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus/storage-pools/u1
  zfs.pool_name: rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus/storage-pools/u1
driver: zfs

It is not possible to unmount the unintended /var/lib/incus mount.

$ sudo systemctl stop incus.service incus.socket
$ df -h /var/lib/incus/
Dateisystem                                                     Größe Benutzt Verf. Verw% Eingehängt auf
rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus   98G    244M   98G    1% /var/lib/incus

$ sudo zfs umount rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus         
cannot unmount '/var/lib/incus': pool or dataset is busy

$ sudo umount /var/lib/incus
umount: /var/lib/incus: das Ziel wird gerade benutzt.

A reboot is required to remove the unintended mount. It is not possible to start the Incus instance from the host, as unlocking and mounting the canmount=noauto dataset on the host is forced. Masking the zfs-load-key@ unit appears as a chance of remediation that will be tested.

[almereyda]

After reboot, the situation presents itself as follows:

$ sudo systemctl mask zfs-load-key@rpool-ROOT-ubuntu_d4psvq-var-incus-containers-u1-ROOT.service zfs-load-key@rpool-ROOT-ubuntu_d4psvq-var-incus-containers-u1-USERDATA.service
[sudo: authenticate] Password: 
Created symlink '/etc/systemd/system/zfs-load-key@rpool-ROOT-ubuntu_d4psvq-var-incus-containers-u1-ROOT.service' → '/dev/null'.
Created symlink '/etc/systemd/system/zfs-load-key@rpool-ROOT-ubuntu_d4psvq-var-incus-containers-u1-USERDATA.service' → '/dev/null'.

$ df -h /var/lib/incus/
Dateisystem                      Größe Benutzt Verf. Verw% Eingehängt auf
rpool/ROOT/ubuntu_d4psvq/var/lib  129G     32G   98G   25% /var/lib

$ sudo ls -lan /var/lib/incus        
total 68
drwx--x--x  16 0 0    20 Oct 25 14:58 .
drwxr-xr-x 108 0 0   110 Oct 23 19:39 ..
drwx------   4 0 0     4 Nov  6  2023 backups
drwx--x--x   2 0 0     4 Nov  1 00:15 containers
drwx------   2 0 0     2 Nov  6  2023 containers-snapshots
drwx------   4 0 0     5 Oct 25 15:00 database
drwx--x--x   4 0 0     4 Nov  1 00:15 devices
drwx------   2 0 0     2 Nov  6  2023 disks
drwxr-xr-x   2 0 0     2 Nov  6  2023 guestapi
drwx------   2 0 0     4 Oct 30 15:40 images
drwx--x--x   4 0 0     4 Mar 27  2024 networks
drwx------   4 0 0     4 Nov  6  2023 security
-rw-r--r--   1 0 0   761 Nov  6  2023 server.crt
-rw-------   1 0 0   288 Nov  6  2023 server.key
drwx--x--x   2 0 0     2 Nov  6  2023 shmounts
drwx--x--x   3 0 0     3 Nov  6  2023 storage-pools
srw-rw----   1 0 152   0 Oct 25 14:58 unix.socket
srw-rw----   1 0 151   0 Oct 25 14:58 unix.socket.user
drwx--x--x   2 0 0     2 Nov  6  2023 virtual-machines
drwx------   2 0 0     2 Nov  6  2023 virtual-machines-snapshots

$ df /var/lib/incus/storage-pools
Dateisystem                      1K-Blöcke  Benutzt Verfügbar Verw% Eingehängt auf
rpool/ROOT/ubuntu_d4psvq/var/lib 134618624 32691328 101927296   25% /var/lib

$ sudo zfs list rpool/ROOT/ubuntu_d4psvq/var/incus
NAME                                 USED  AVAIL  REFER  MOUNTPOINT
rpool/ROOT/ubuntu_d4psvq/var/incus  2.93G  97.2G    96K  legacy

$ sudo zfs set mountpoint=/var/lib/incus rpool/ROOT/ubuntu_d4psvq/var/incus

$ zfs get mounted rpool/ROOT/ubuntu_d4psvq/var/incus
NAME                                PROPERTY  VALUE    SOURCE
rpool/ROOT/ubuntu_d4psvq/var/incus  mounted   no       -

$ sudo zfs mount rpool/ROOT/ubuntu_d4psvq/var/incus                        

$ zfs get mounted rpool/ROOT/ubuntu_d4psvq/var/incus
NAME                                PROPERTY  VALUE    SOURCE
rpool/ROOT/ubuntu_d4psvq/var/incus  mounted   no       -

$ zfs get mountpoint rpool/ROOT/ubuntu_d4psvq/var/incus                             
NAME                                PROPERTY    VALUE           SOURCE
rpool/ROOT/ubuntu_d4psvq/var/incus  mountpoint  /var/lib/incus  local

$ zfs get canmount rpool/ROOT/ubuntu_d4psvq/var/incus  
NAME                                PROPERTY  VALUE     SOURCE
rpool/ROOT/ubuntu_d4psvq/var/incus  canmount  on        default

$ sudo systemctl start incus.service incus.socket
[sudo: authenticate] Password: 
Failed to start incus.service: Unit zfs-load-key@rpool-ROOT-ubuntu_d4psvq-var-incus-containers-u1-USERDATA.service is masked.
Failed to start incus.socket: Unit zfs-load-key@rpool-ROOT-ubuntu_d4psvq-var-incus-containers-u1-USERDATA.service is masked.

$ incus ls                                                          
Error: The incus daemon doesn't appear to be started (socket path: /var/lib/incus/unix.socket)

Further discussion of ZFS migrations with delegation and encryption can be found in the forum:

• Thoughts on stabilising ZFS delegation and migration together with encryption - Incus - Linux Containers Forum

[almereyda]

Setting org.openzfs.systemd:ignore=on on the affected datasets took part in remediating the issue.

It was not necessary to combine it with mask.

This still doesn't tell us why the datasets have to be unlocked when starting Incus, as in why there is a hard dependency, when those units are present. And even then, why is the dataset for which a key is loaded and canmount=noauto mounted automatically?

Also the Incus configuration state has disappeared, despite its configuration is loaded from the correct path:

$ zfs list -r -o canmount,mounted,mountpoint,org.openzfs.systemd:ignore,name /var/lib/incus
CANMOUNT  MOUNTED  MOUNTPOINT                    ORG.OPENZFS.SYSTEMD:IGNORE  NAME
on        yes      /var/lib/incus                -                           rpool/ROOT/ubuntu_d4psvq/var/incus
on        no       legacy                        -                           rpool/ROOT/ubuntu_d4psvq/var/incus/buckets
noauto    no       legacy                        -                           rpool/ROOT/ubuntu_d4psvq/var/incus/buckets/default_yala
on        no       legacy                        -                           rpool/ROOT/ubuntu_d4psvq/var/incus/containers
noauto    no       legacy                        -                           rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u0
noauto    no       legacy                        -                           rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1
on        no       /srv/ROOT                     on                          rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/ROOT
off       no       legacy                        on                          rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA
noauto    no       /var/lib/incus                on                          rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus
on        no       /var/lib/incus/storage-pools  on                          rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus/storage-pools
on        no       legacy                        on                          rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus/storage-pools/u1
on        no       legacy                        on                          rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus/storage-pools/u1/buckets
on        no       legacy                        on                          rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus/storage-pools/u1/containers
on        no       legacy                        on                          rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus/storage-pools/u1/custom
on        no       legacy                        on                          rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus/storage-pools/u1/deleted
on        no       legacy                        on                          rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus/storage-pools/u1/deleted/buckets
on        no       legacy                        on                          rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus/storage-pools/u1/deleted/containers
on        no       legacy                        on                          rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus/storage-pools/u1/deleted/custom
on        no       legacy                        on                          rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus/storage-pools/u1/deleted/images
on        no       legacy                        on                          rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus/storage-pools/u1/deleted/virtual-machines
on        no       legacy                        on                          rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus/storage-pools/u1/images
noauto    no       legacy                        on                          rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus/storage-pools/u1/images/8c0bedc41b796d32d0f80d0b7b40be8592e1389bd1a92d4fa0c4acee9d907854
noauto    no       legacy                        on                          rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus/storage-pools/u1/images/d753ba4a12fb3f773e41e4964973964ea2c7441add6b3155582ca2786fb40cc8
on        no       legacy                        on                          rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus/storage-pools/u1/virtual-machines
on        no       legacy                        -                           rpool/ROOT/ubuntu_d4psvq/var/incus/custom
noauto    no       legacy                        -                           rpool/ROOT/ubuntu_d4psvq/var/incus/custom/default_u0-data
on        no       legacy                        -                           rpool/ROOT/ubuntu_d4psvq/var/incus/deleted
on        no       legacy                        -                           rpool/ROOT/ubuntu_d4psvq/var/incus/deleted/buckets
on        no       legacy                        -                           rpool/ROOT/ubuntu_d4psvq/var/incus/deleted/containers
on        no       legacy                        -                           rpool/ROOT/ubuntu_d4psvq/var/incus/deleted/custom
on        no       legacy                        -                           rpool/ROOT/ubuntu_d4psvq/var/incus/deleted/images
noauto    no       legacy                        -                           rpool/ROOT/ubuntu_d4psvq/var/incus/deleted/images/73fe6edfe11e491e208df76f01cbc7303c8ad203a40d0bae2c9a7ad1cfe4e750
on        no       legacy                        -                           rpool/ROOT/ubuntu_d4psvq/var/incus/deleted/virtual-machines
on        no       legacy                        -                           rpool/ROOT/ubuntu_d4psvq/var/incus/images
noauto    no       legacy                        -                           rpool/ROOT/ubuntu_d4psvq/var/incus/images/058f298087312e0b7bd685520fb66b9f520bf2123bec237da64b79b04a5363ea
noauto    no       legacy                        -                           rpool/ROOT/ubuntu_d4psvq/var/incus/images/5f69e85b1d16b6f0b5b4ba87a7020ae56463a32138345656b0ca78d74ac96a70
noauto    no       legacy                        -                           rpool/ROOT/ubuntu_d4psvq/var/incus/images/8c0bedc41b796d32d0f80d0b7b40be8592e1389bd1a92d4fa0c4acee9d907854
on        no       legacy                        -                           rpool/ROOT/ubuntu_d4psvq/var/incus/virtual-machines

$ incus ls
+------+-------+------+------+------+-----------+
| NAME | STATE | IPV4 | IPV6 | TYPE | SNAPSHOTS |
+------+-------+------+------+------+-----------+

$ incus storage ls
+------+--------+--------------+---------+-------+
| NAME | DRIVER | BESCHREIBUNG | USED BY | STATE |
+------+--------+--------------+---------+-------+

Now trying to find out where it went. Maybe the dataset was never mounted at all? Then why is the Incus dataset hierarchy placed beneath it, if it never worked? Which leaves me wondering about what the common recovery procedure is in this situation.

$ incus admin recover
This server currently has the following storage pools:
Would you like to recover another storage pool? (yes/no) [default=no]:

The storage pool is not found. Incidentally the dataset mounted to /var/lib/incus originally didn't have that mount point. It's parent contains the Incus directory. It is the storage pool. Validating with stopping and disabling incus, rebooting to free the dataset from an Incus-related IO lock and then unmounting it, probably setting to canmount=off.

[almereyda]

This was a side-effect from setting the wrong mountpoint for the storage pool dataset in place of the Incus configuration directory, combined with during recovery not knowing that incus-user.socket exists.

Stopping the Incus service and both sockets then allowed to set the mountpoint back to /var/lib/incus/storage-pools/default and restarting all three. The instances are accessible again.

Am I correct in assumming, that the dataset holding the dataset hierarchy for an Incus storage pool does not necessarily need a mountpoint at all and is considered legacy like the others, because I just specified the dataset name during initialisation, or will the mountpoint in any case be set to the basepath of /var/lib/incus/storage-pools/ plus the name of the Incus storage pool? (It's easy to mix up with the ZFS pools here, why I'm overly specific.)

I would like to leave this open as a reminder to add minimal notes to the documentation about what kind of side-effects can be expected when dealing with nested datasets for nested Incus configurations and (nested) encryption.

I would also like to invite to reflect about the assumption that unlocking nested encryptionroots below an instance's dataset within an Incus storage pool is required for Incus to start. The workaround works well enough, marking nested encryptionroots with org.openzfs.systemd:ignore=on to avoid their automatic handling. Why does the incus.service unit depend on the zfs-load-key@.service units, when this is not done? Incus itself has no understanding of ZFS encryption. Why should it then involve those units? Or are we seeing the beginnings of tighter integration of encryption with Incus here?

---

Jumping back and forth and ultimately being able to getting close to starting Incus again shows, that with switching the namespace also some UIDs and GIDs have changed, since the host slice uses no subuid map and no subgid maps. That's a bit interesting:

root@u1:~# systemctl start incus.service
Job for incus.service failed because the control process exited with error code.
See "systemctl status incus.service" and "journalctl -xeu incus.service" for details.

root@u1:~# journalctl -n -u incus.service
Nov 01 20:31:20 u1 systemd[1]: incus.service: Control process exited, code=killed, status=15/TERM
Nov 01 20:31:20 u1 systemd[1]: incus.service: Failed with result 'exit-code'.
Nov 01 20:31:20 u1 systemd[1]: Failed to start incus.service - Incus - Daemon.
Nov 01 20:31:20 u1 systemd[1]: incus.service: Scheduled restart job, restart counter is at 1.
Nov 01 20:31:20 u1 systemd[1]: Starting incus.service - Incus - Daemon...
Nov 01 20:31:20 u1 incusd[1800]: time="2025-11-01T20:31:20Z" level=warning msg="Instance type not operational" driver=qemu err="KVM support is missing (no /dev/kvm)" type=vi>
Nov 01 20:31:20 u1 incusd[1800]: time="2025-11-01T20:31:20Z" level=warning msg="Unable to access device nodes, likely running on a nodev mount"
Nov 01 20:31:20 u1 incusd[1800]: time="2025-11-01T20:31:20Z" level=error msg="Failed to start the daemon" err="Failed to start dqlite server: raft_start(): io: load closed s>
Nov 01 20:31:20 u1 incusd[1800]: Error: Failed to start dqlite server: raft_start(): io: load closed segment 0000000000000172-0000000000000180: read file: open: permission d>
Nov 01 20:31:20 u1 systemd[1]: incus.service: Main process exited, code=exited, status=1/FAILURE

It is the storage pool and the database that are affected. We can clearly see how the UID of the segment in question mismatches from that counter on.

root@u1:~# ls -lan /var/lib/incus/{storage-pools,database/global}
/var/lib/incus/database/global:
total 315
drwxr-x--- 2     0     0      15 Nov  1 20:31 .
drwx------ 3     0     0       4 Nov  1 11:59 ..
-rw------- 1     0     0      64 Oct 31 21:17 0000000000000001-0000000000000001
-rw------- 1     0     0 2412512 Oct 31 23:19 0000000000000002-0000000000000106
-rw------- 1     0     0  874736 Nov  1 02:09 0000000000000107-0000000000000171
-rw------- 1 65534 65534   99152 Nov  1 02:28 0000000000000172-0000000000000180
-rw------- 1 65534 65534  148832 Nov  1 02:31 0000000000000181-0000000000000195
-rw------- 1 65534 65534  148832 Nov  1 02:55 0000000000000196-0000000000000210
-rw------- 1 65534 65534  148832 Nov  1 03:07 0000000000000211-0000000000000225
-rw------- 1 65534 65534  148832 Nov  1 11:14 0000000000000226-0000000000000240
-rw------- 1 65534 65534  148880 Nov  1 11:29 0000000000000241-0000000000000256
-rw------- 1 65534 65534  148832 Nov  1 12:10 0000000000000257-0000000000000271
-rw------- 1 65534 65534  655360 Nov  1 12:10 db.bin
-rw------- 1 65534 65534  156592 Nov  1 12:10 db.bin-wal
-rw------- 1     0     0      32 Oct 31 21:17 metadata1

/var/lib/incus/storage-pools:
total 36
drwx--x--x  4     0     0  4 Nov  1 02:53 .
drwx--x--x 16     0     0 20 Nov  1 20:21 ..
drwx--x--x 10 65534 65534 10 Nov  1 02:53 default
drwx--x--x 10 65534 65534 10 Nov  1 02:53 u1

It is not possible to rectify this from inside the container.

root@u1:~# chown 0:0 /var/lib/incus/{storage-pools/*,database/global/*}
chown: changing ownership of '/var/lib/incus/storage-pools/default': Operation not permitted
chown: changing ownership of '/var/lib/incus/storage-pools/u1': Operation not permitted
chown: changing ownership of '/var/lib/incus/database/global/0000000000000172-0000000000000180': Operation not permitted
chown: changing ownership of '/var/lib/incus/database/global/0000000000000181-0000000000000195': Operation not permitted
chown: changing ownership of '/var/lib/incus/database/global/0000000000000196-0000000000000210': Operation not permitted
chown: changing ownership of '/var/lib/incus/database/global/0000000000000211-0000000000000225': Operation not permitted
chown: changing ownership of '/var/lib/incus/database/global/0000000000000226-0000000000000240': Operation not permitted
chown: changing ownership of '/var/lib/incus/database/global/0000000000000241-0000000000000256': Operation not permitted
chown: changing ownership of '/var/lib/incus/database/global/0000000000000257-0000000000000271': Operation not permitted
chown: changing ownership of '/var/lib/incus/database/global/db.bin': Operation not permitted
chown: changing ownership of '/var/lib/incus/database/global/db.bin-wal': Operation not permitted

Seeing root: Operation not permitted is also quite a new'ish experience. All of the affected files live on the expected dataset:

root@u1:~# df -h /var/lib/incus/{storage-pools/*,database/global/*}
Filesystem                                                       Size  Used Avail Use% Mounted on
rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus   97G  244M   97G   1% /var/lib/incus
rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus   97G  244M   97G   1% /var/lib/incus
rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus   97G  244M   97G   1% /var/lib/incus
rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus   97G  244M   97G   1% /var/lib/incus
rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus   97G  244M   97G   1% /var/lib/incus
rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus   97G  244M   97G   1% /var/lib/incus
rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus   97G  244M   97G   1% /var/lib/incus
rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus   97G  244M   97G   1% /var/lib/incus
rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus   97G  244M   97G   1% /var/lib/incus
rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus   97G  244M   97G   1% /var/lib/incus
rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus   97G  244M   97G   1% /var/lib/incus
rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus   97G  244M   97G   1% /var/lib/incus
rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus   97G  244M   97G   1% /var/lib/incus
rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus   97G  244M   97G   1% /var/lib/incus
rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus   97G  244M   97G   1% /var/lib/incus

root@u1:~# zfs list -o mounted,keystatus rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus
MOUNTED  KEYSTATUS
yes      available

For the host, this looks differently.

$ sudo zfs list -o mounted,keystatus rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus 
MOUNTED  KEYSTATUS
no       available

To mount it on the host for forcefully re-matching the UIDs/GIDs to those inside the container, we need to stop the instance.

$ incus stop u1     

$ sudo zfs get mountpoint -Ho value rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus 
/var/lib/incus

$ sudo zfs list -o mounted,keystatus rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus 
MOUNTED  KEYSTATUS
no       available

Mounting the dataset as is on the host would have the effect of mounting on top of the current mount for that path. As Incus is still running, we don't want that. The key of the dataset remains loaded, also after shutdown. Repeating this here to emphasise it, as this is something to take note of.

Yet the zfs command lacks a temporary relocation of the mountpoint during invocation, other than with zpool import -R /mnt, which means we need to specifically provide a new mountpoint to switch back to later on, if this ought to succeed. A way to set different options per namespace for at least mountpoints can be something that appears at least considerable.

$ sudo zfs set mountpoint=/mnt rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus      

$ sudo zfs get mounted -Ho value rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus 
no

$ sudo zfs mount rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus                

# yala @ ganglion in ~ [21:50:09] 
$ df -h /mnt          
Dateisystem                                                     Größe Benutzt Verf. Verw% Eingehängt auf
rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus   97G    244M   97G    1% /mnt

We can now inspect and rectify the other side from the host perspective on the subuid/subgid maps.

$ sudo ls -lan /mnt/{storage-pools,database/global}
/mnt/database/global:
total 315
drwxr-x--- 2 1000000 1000000      15 Nov  1 21:41 .
drwx------ 3 1000000 1000000       4 Nov  1 12:59 ..
-rw------- 1 1000000 1000000      64 Oct 31 22:17 0000000000000001-0000000000000001
-rw------- 1 1000000 1000000 2412512 Nov  1 00:19 0000000000000002-0000000000000106
-rw------- 1 1000000 1000000  874736 Nov  1 03:09 0000000000000107-0000000000000171
-rw------- 1 0       0         99152 Nov  1 03:28 0000000000000172-0000000000000180
-rw------- 1 0       0        148832 Nov  1 03:31 0000000000000181-0000000000000195
-rw------- 1 0       0        148832 Nov  1 03:55 0000000000000196-0000000000000210
-rw------- 1 0       0        148832 Nov  1 04:07 0000000000000211-0000000000000225
-rw------- 1 0       0        148832 Nov  1 12:14 0000000000000226-0000000000000240
-rw------- 1 0       0        148880 Nov  1 12:29 0000000000000241-0000000000000256
-rw------- 1 0       0        148832 Nov  1 13:10 0000000000000257-0000000000000271
-rw------- 1 0       0        655360 Nov  1 13:10 db.bin
-rw------- 1 0       0        156592 Nov  1 13:10 db.bin-wal
-rw------- 1 1000000 1000000      32 Oct 31 22:17 metadata1

/mnt/storage-pools:
total 36
drwx--x--x  4 1000000 1000000  4 Nov  1 03:53 .
drwx--x--x 16 1000000 1000000 20 Nov  1 21:21 ..
drwx--x--x 10 0       0       10 Nov  1 03:53 default
drwx--x--x 10 0       0       10 Nov  1 03:53 u1

$ sudo chown -R 1000000:1000000 /mnt/{storage-pools,database/global}

$ sudo ls -lan /mnt/{storage-pools,database/global}
/mnt/database/global:
total 315
drwxr-x--- 2 1000000 1000000      15 Nov  1 21:41 .
drwx------ 3 1000000 1000000       4 Nov  1 12:59 ..
-rw------- 1 1000000 1000000      64 Oct 31 22:17 0000000000000001-0000000000000001
-rw------- 1 1000000 1000000 2412512 Nov  1 00:19 0000000000000002-0000000000000106
-rw------- 1 1000000 1000000  874736 Nov  1 03:09 0000000000000107-0000000000000171
-rw------- 1 1000000 1000000   99152 Nov  1 03:28 0000000000000172-0000000000000180
-rw------- 1 1000000 1000000  148832 Nov  1 03:31 0000000000000181-0000000000000195
-rw------- 1 1000000 1000000  148832 Nov  1 03:55 0000000000000196-0000000000000210
-rw------- 1 1000000 1000000  148832 Nov  1 04:07 0000000000000211-0000000000000225
-rw------- 1 1000000 1000000  148832 Nov  1 12:14 0000000000000226-0000000000000240
-rw------- 1 1000000 1000000  148880 Nov  1 12:29 0000000000000241-0000000000000256
-rw------- 1 1000000 1000000  148832 Nov  1 13:10 0000000000000257-0000000000000271
-rw------- 1 1000000 1000000  655360 Nov  1 13:10 db.bin
-rw------- 1 1000000 1000000  156592 Nov  1 13:10 db.bin-wal
-rw------- 1 1000000 1000000      32 Oct 31 22:17 metadata1

/mnt/storage-pools:
total 36
drwx--x--x  4 1000000 1000000  4 Nov  1 03:53 .
drwx--x--x 16 1000000 1000000 20 Nov  1 21:21 ..
drwx--x--x 10 1000000 1000000 10 Nov  1 03:53 default
drwx--x--x 10 1000000 1000000 10 Nov  1 03:53 u1

We can now reverse the operation again.

$ sudo zfs get canmount -Ho value rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus
noauto

$ sudo zfs umount rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus

$ sudo zfs set mountpoint=/var/lib/incus rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus

$ sudo zfs get mounted -Ho value rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus 
no

It seems safe to start the container again. (Can a dataset be mounted twice, once on the host and once in the namespace? That could be interesting to find out for a curious reader. This is especially interesting with the synced mountpoint. We don't deviate here; for now.)

$ incus start u1

$ incus exec u1 bash
root@u1:~# df -h  /var/lib/incus
Filesystem                                        Size  Used Avail Use% Mounted on
rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1   98G  1.3G   97G   2% /
root@u1:~# systemctl status incus.service incus.socket incus-user.socket
○ incus.service - Incus - Daemon
     Loaded: loaded (/usr/lib/systemd/system/incus.service; indirect; preset: enabled)
    Drop-In: /run/systemd/system/service.d
             └─zzz-lxc-service.conf
     Active: inactive (dead)
TriggeredBy: ○ incus.socket

○ incus.socket - Incus - Daemon (unix socket)
     Loaded: loaded (/usr/lib/systemd/system/incus.socket; disabled; preset: enabled)
     Active: inactive (dead)
   Triggers: ● incus.service
     Listen: /var/lib/incus/unix.socket (Stream)

○ incus-user.socket - Incus - Daemon (user unix socket)
     Loaded: loaded (/usr/lib/systemd/system/incus-user.socket; disabled; preset: enabled)
     Active: inactive (dead)
   Triggers: ● incus-user.service
     Listen: /var/lib/incus/unix.socket.user (Stream)
root@u1:~# zfs mount rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus
root@u1:~# df -h  /var/lib/incus
Filesystem                                                       Size  Used Avail Use% Mounted on
rpool/ROOT/ubuntu_d4psvq/var/incus/containers/u1/USERDATA/incus   97G  244M   97G   1% /var/lib/incus
root@u1:~# systemctl start incus.service incus.socket incus-user.socket
root@u1:~# incus ls
+------+-------+------+------+------+-----------+
| NAME | STATE | IPV4 | IPV6 | TYPE | SNAPSHOTS |
+------+-------+------+------+------+-----------+
root@u1:~# incus storage ls
+---------+--------+-------------+---------+---------+
|  NAME   | DRIVER | DESCRIPTION | USED BY |  STATE  |
+---------+--------+-------------+---------+---------+
| default | dir    |             | 1       | CREATED |
+---------+--------+-------------+---------+---------+
| u1      | zfs    |             | 2       | CREATED |
+---------+--------+-------------+---------+---------+

The nested Incus host works again as expected:

root@u1:~# incus launch -s u1 images:fedora/43 f1 -c security.nesting=true -c security.syscalls.intercept.mknod=true -c security.syscalls.intercept.setxattr=true
Launching f1
root@u1:~# incus exec f1 sh -- -c 'dnf install --assumeyes podman > /dev/null 2>&1; podman run --rm hello-world 2>/dev/null | head -n 1'
!... Hello Podman World ...!

The poor -s u1 flag can be remediated with a more careful preparation of the preseed during initialisation of the incus host.