When the overlay volatile feature is enabled, creating a pod with an image configured with anonymous volumes will fail #10228
Description
Description
When the overlay volatile feature is enabled, creating a pod with an image configured with anonymous volumes will fail. error logs:
containerd[2347130]: time="2024-05-14T14:57:15.539317523+08:00" level=error msg="StartContainer for \"01a7b8fe53ec1626010723c109f154e9f2aedfae68a4b7cbc44e5b8edfea18d6\" failed" error="failed to create containerd task: failed to create shim task: failed to mount rootfs component &{overlay overlay [volatile index=off workdir=/media/disk1/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/22257/work upperdir=/media/disk1/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/22257/fs lowerdir=/media/disk1/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/7417/fs:/media/disk1/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/7416/fs:/media/disk1/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/7415/fs:/media/disk1/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/7414/fs:/media/disk1/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/7413/fs:/media/disk1/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/7412/fs:/media/disk1/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/7411/fs:/media/disk1/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/7410/fs:/media/disk1/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/7409/fs]}: invalid argument: unknown"
the key info is:
failed to create containerd task: failed to create shim task: failed to mount rootfs component ... invalid argument: unknown
Steps to reproduce the issue
- enable the overlay volatile feature:
[plugins."io.containerd.snapshotter.v1.overlayfs"]
root_path = ""
upperdir_label = false
mount_options = ["volatile"]- Build a image with anonymous volume:
# The problem persists when using other base images as well.
FROM centos:centos7.9.2009
# The problem persists when switching other directories as well.
VOLUME ["/home/volatile"]- Create pod using the image, following is pod yaml:
apiVersion: v1
kind: Pod
metadata:
name: volatile-test
labels:
app: volatile-test
spec:
hostNetwork: true
nodeName: nodename
containers:
- name: volatile-test
args:
- /bin/sh
- -c
- sleep 600
image: volatile:testDescribe the results you received and expected
When the overlay volatile feature is enabled, creating a pod with an image configured with anonymous volumes will success.
What version of containerd are you using?
v1.6.24 and 1.7.13
Any other relevant information
# runc --version:
runc version 1.1.2
commit: dc2d880
spec: 1.0.2-dev
go: go1.18.10
libseccomp: 2.3.1
# crictl info
{
"status": {
"conditions": [
{
"type": "RuntimeReady",
"status": true,
"reason": "",
"message": ""
},
{
"type": "NetworkReady",
"status": true,
"reason": "",
"message": ""
}
]
},
"cniconfig": {
"PluginDirs": [
"/opt/cni/bin"
],
"PluginConfDir": "/etc/cni/net.d",
"PluginMaxConfNum": 1,
"Prefix": "eth",
"Networks": [
{
"Config": {
"Name": "cni-loopback",
"CNIVersion": "0.3.1",
"Plugins": [
{
"Network": {
"type": "loopback",
"ipam": {},
"dns": {}
},
"Source": "{\"type\":\"loopback\"}"
}
],
"Source": "{\n\"cniVersion\": \"0.3.1\",\n\"name\": \"cni-loopback\",\n\"plugins\": [{\n \"type\": \"loopback\"\n}]\n}"
},
"IFName": "lo"
},
{
"Config": {
"Name": "kflax",
"CNIVersion": "0.3.1",
"Plugins": [
{
"Network": {
"type": "kflax",
"capabilities": {
"io.kubernetes.cri.pod-annotations": true,
"portMappings": true
},
"ipam": {},
"dns": {}
},
"Source": "{\"args\":{\"cloud\":\"kflax_non_cloud\"},\"capabilities\":{\"io.kubernetes.cri.pod-annotations\":true,\"portMappings\":true},\"sysctl\":{\"net/core/somaxconn\":\"auto\",\"net/ipv4/tcp_fin_timeout\":\"auto\",\"net/ipv4/tcp_keepalive_time\":\"auto\",\"net/ipv4/tcp_max_syn_backlog\":\"auto\",\"net/ipv4/tcp_max_tw_buckets\":\"auto\",\"net/ipv4/tcp_no_metrics_save\":\"auto\",\"net/ipv4/tcp_slow_start_after_idle\":\"auto\",\"net/ipv4/tcp_syn_retries\":\"auto\",\"net/ipv4/tcp_synack_retries\":\"auto\",\"net/ipv4/tcp_timestamps\":\"auto\",\"net/ipv4/tcp_tw_reuse\":\"auto\"},\"type\":\"kflax\"}"
}
],
"Source": "{\n \"name\": \"kflax\",\n \"cniVersion\": \"0.3.1\",\n \"plugins\": [\n {\n \"type\": \"kflax\",\n \"capabilities\": {\n \"portMappings\": true,\n \"io.kubernetes.cri.pod-annotations\": true\n },\n \"sysctl\": {\n \"net/ipv4/tcp_syn_retries\": \"auto\",\n \"net/ipv4/tcp_synack_retries\": \"auto\",\n \"net/ipv4/tcp_max_syn_backlog\": \"auto\",\n \"net/ipv4/tcp_keepalive_time\": \"auto\",\n \"net/ipv4/tcp_fin_timeout\": \"auto\",\n \"net/ipv4/tcp_slow_start_after_idle\": \"auto\",\n \"net/ipv4/tcp_max_tw_buckets\": \"auto\",\n \"net/ipv4/tcp_no_metrics_save\": \"auto\",\n \"net/ipv4/tcp_timestamps\": \"auto\",\n \"net/ipv4/tcp_tw_reuse\": \"auto\",\n \"net/core/somaxconn\": \"auto\"\n },\n \"args\": {\n \"cloud\": \"kflax_non_cloud\"\n }\n }\n ]\n}\n"
},
"IFName": "eth0"
}
]
},
"config": {
"containerd": {
"snapshotter": "overlayfs",
"defaultRuntimeName": "runc",
"defaultRuntime": {
"runtimeType": "",
"runtimePath": "",
"runtimeEngine": "",
"PodAnnotations": [],
"ContainerAnnotations": [],
"runtimeRoot": "",
"options": {},
"privileged_without_host_devices": false,
"baseRuntimeSpec": "",
"cniConfDir": "",
"cniMaxConfNum": 0
},
"untrustedWorkloadRuntime": {
"runtimeType": "io.containerd.runtime.v1.linux",
"runtimePath": "",
"runtimeEngine": "/opt/kata/bin/kata-runtime",
"PodAnnotations": [],
"ContainerAnnotations": [],
"runtimeRoot": "",
"options": null,
"privileged_without_host_devices": true,
"baseRuntimeSpec": "",
"cniConfDir": "",
"cniMaxConfNum": 0
},
"runtimes": {
"runc": {
"runtimeType": "io.containerd.runc.v2",
"runtimePath": "",
"runtimeEngine": "",
"PodAnnotations": [],
"ContainerAnnotations": [],
"runtimeRoot": "",
"options": {
"BinaryName": "",
"CriuImagePath": "",
"CriuPath": "",
"CriuWorkPath": "",
"IoGid": 0,
"IoUid": 0,
"NoNewKeyring": false,
"NoPivotRoot": false,
"Root": "",
"ShimCgroup": "",
"SystemdCgroup": true
},
"privileged_without_host_devices": false,
"baseRuntimeSpec": "",
"cniConfDir": "",
"cniMaxConfNum": 0
},
"untrusted": {
"runtimeType": "io.containerd.runtime.v1.linux",
"runtimePath": "",
"runtimeEngine": "/opt/kata/bin/kata-runtime",
"PodAnnotations": [],
"ContainerAnnotations": [],
"runtimeRoot": "",
"options": null,
"privileged_without_host_devices": true,
"baseRuntimeSpec": "",
"cniConfDir": "",
"cniMaxConfNum": 0
}
},
"noPivot": false,
"disableSnapshotAnnotations": true,
"discardUnpackedLayers": false,
"ignoreRdtNotEnabledErrors": false
},
"cni": {
"binDir": "/opt/cni/bin",
"confDir": "/etc/cni/net.d",
"maxConfNum": 1,
"confTemplate": "",
"ipPref": ""
},
"registry": {
"configPath": "/etc/containerd/certs.d",
"mirrors": {},
"configs": {},
"auths": {},
"headers": {}
},
"imageDecryption": {
"keyModel": "node"
},
"disableTCPService": true,
"streamServerAddress": "127.0.0.1",
"streamServerPort": "0",
"streamIdleTimeout": "4h0m0s",
"enableSelinux": false,
"selinuxCategoryRange": 1024,
"sandboxImage": "pause:3.1",
"statsCollectPeriod": 10,
"systemdCgroup": false,
"enableTLSStreaming": false,
"x509KeyPairStreaming": {
"tlsCertFile": "",
"tlsKeyFile": ""
},
"maxContainerLogSize": 16384,
"disableCgroup": false,
"disableApparmor": false,
"restrictOOMScoreAdj": false,
"maxConcurrentDownloads": 10,
"disableProcMount": false,
"unsetSeccompProfile": "",
"tolerateMissingHugetlbController": true,
"disableHugetlbController": true,
"device_ownership_from_security_context": false,
"ignoreImageDefinedVolumes": false,
"netnsMountsUnderStateDir": false,
"enableUnprivilegedPorts": false,
"enableUnprivilegedICMP": false,
"containerdRootDir": "/media/disk1/containerd",
"containerdEndpoint": "/run/containerd/containerd.sock",
"rootDir": "/media/disk1/containerd/io.containerd.grpc.v1.cri",
"stateDir": "/run/containerd/io.containerd.grpc.v1.cri"
},
"golang": "go1.20.8",
"lastCNILoadStatus": "OK",
"lastCNILoadStatus.default": "OK"
}
# kubernetes version: 1.17/1.22Show configuration if it is related to CRI plugin.
disabled_plugins = []
imports = []
oom_score = 0
plugin_dir = ""
required_plugins = []
root = "/media/disk1/containerd"
state = "/run/containerd"
temp = ""
version = 2
[cgroup]
path = ""
[debug]
address = ""
format = ""
gid = 0
level = "debug"
uid = 0
[grpc]
address = "/run/containerd/containerd.sock"
gid = 0
max_recv_message_size = 16777216
max_send_message_size = 16777216
tcp_address = ""
tcp_tls_ca = ""
tcp_tls_cert = ""
tcp_tls_key = ""
uid = 0
[metrics]
address = ""
grpc_histogram = false
[plugins]
[plugins."io.containerd.gc.v1.scheduler"]
deletion_threshold = 0
mutation_threshold = 100
pause_threshold = 0.02
schedule_delay = "0s"
startup_delay = "100ms"
[plugins."io.containerd.grpc.v1.cri"]
device_ownership_from_security_context = false
disable_apparmor = false
disable_cgroup = false
disable_hugetlb_controller = true
disable_proc_mount = false
disable_tcp_service = true
enable_selinux = false
enable_tls_streaming = false
enable_unprivileged_icmp = false
enable_unprivileged_ports = false
ignore_image_defined_volumes = false
max_concurrent_downloads = 10
max_container_log_line_size = 16384
netns_mounts_under_state_dir = false
restrict_oom_score_adj = false
sandbox_image = "pause:3.1"
selinux_category_range = 1024
stats_collect_period = 10
stream_idle_timeout = "4h0m0s"
stream_server_address = "127.0.0.1"
stream_server_port = "0"
systemd_cgroup = false
tolerate_missing_hugetlb_controller = true
unset_seccomp_profile = ""
[plugins."io.containerd.grpc.v1.cri".cni]
bin_dir = "/opt/cni/bin"
conf_dir = "/etc/cni/net.d"
conf_template = ""
ip_pref = ""
max_conf_num = 1
[plugins."io.containerd.grpc.v1.cri".containerd]
default_runtime_name = "runc"
disable_snapshot_annotations = true
discard_unpacked_layers = false
ignore_rdt_not_enabled_errors = false
no_pivot = false
snapshotter = "overlayfs"
[plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime]
base_runtime_spec = ""
cni_conf_dir = ""
cni_max_conf_num = 0
container_annotations = []
pod_annotations = []
privileged_without_host_devices = true
runtime_engine = "/opt/kata/bin/kata-runtime"
runtime_path = ""
runtime_root = ""
runtime_type = "io.containerd.runtime.v1.linux"
[plugins."io.containerd.grpc.v1.cri".containerd.default_runtime]
base_runtime_spec = ""
cni_conf_dir = ""
cni_max_conf_num = 0
container_annotations = []
pod_annotations = []
privileged_without_host_devices = false
runtime_engine = ""
runtime_path = ""
runtime_root = ""
runtime_type = ""
[plugins."io.containerd.grpc.v1.cri".containerd.default_runtime.options]
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
base_runtime_spec = ""
cni_conf_dir = ""
cni_max_conf_num = 0
container_annotations = []
pod_annotations = []
privileged_without_host_devices = false
runtime_engine = ""
runtime_path = ""
runtime_root = ""
runtime_type = "io.containerd.runc.v2"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
BinaryName = ""
CriuImagePath = ""
CriuPath = ""
CriuWorkPath = ""
IoGid = 0
IoUid = 0
NoNewKeyring = false
NoPivotRoot = false
Root = ""
ShimCgroup = ""
SystemdCgroup = true
[plugins."io.containerd.grpc.v1.cri".image_decryption]
key_model = "node"
[plugins."io.containerd.grpc.v1.cri".registry]
config_path = "/etc/containerd/certs.d"
[plugins."io.containerd.grpc.v1.cri".registry.auths]
[plugins."io.containerd.grpc.v1.cri".registry.configs]
[plugins."io.containerd.grpc.v1.cri".registry.headers]
[plugins."io.containerd.grpc.v1.cri".registry.mirrors]
[plugins."io.containerd.grpc.v1.cri".x509_key_pair_streaming]
tls_cert_file = ""
tls_key_file = ""
[plugins."io.containerd.internal.v1.opt"]
path = "/opt/containerd"
[plugins."io.containerd.internal.v1.restart"]
interval = "10s"
[plugins."io.containerd.internal.v1.tracing"]
sampling_ratio = 1.0
service_name = "containerd"
[plugins."io.containerd.metadata.v1.bolt"]
content_sharing_policy = "shared"
[plugins."io.containerd.monitor.v1.cgroups"]
no_prometheus = false
[plugins."io.containerd.runtime.v2.task"]
platforms = ["linux/amd64"]
sched_core = false
[plugins."io.containerd.service.v1.diff-service"]
default = ["walking"]
[plugins."io.containerd.service.v1.tasks-service"]
rdt_config_file = ""
[plugins."io.containerd.snapshotter.v1.aufs"]
root_path = ""
[plugins."io.containerd.snapshotter.v1.btrfs"]
root_path = ""
[plugins."io.containerd.snapshotter.v1.devmapper"]
async_remove = false
base_image_size = ""
discard_blocks = false
fs_options = ""
fs_type = ""
pool_name = ""
root_path = ""
[plugins."io.containerd.snapshotter.v1.native"]
root_path = ""
[plugins."io.containerd.snapshotter.v1.overlayfs"]
root_path = ""
upperdir_label = false
mount_options = ["volatile"]
[plugins."io.containerd.snapshotter.v1.zfs"]
root_path = ""
[plugins."io.containerd.tracing.processor.v1.otlp"]
endpoint = ""
insecure = false
protocol = ""
[proxy_plugins]
[stream_processors]
[stream_processors."io.containerd.ocicrypt.decoder.v1.tar"]
accepts = ["application/vnd.oci.image.layer.v1.tar+encrypted"]
args = ["--decryption-keys-path", "/etc/containerd/ocicrypt/keys"]
env = ["OCICRYPT_KEYPROVIDER_CONFIG=/etc/containerd/ocicrypt/ocicrypt_keyprovider.conf"]
path = "ctd-decoder"
returns = "application/vnd.oci.image.layer.v1.tar"
[stream_processors."io.containerd.ocicrypt.decoder.v1.tar.gzip"]
accepts = ["application/vnd.oci.image.layer.v1.tar+gzip+encrypted"]
args = ["--decryption-keys-path", "/etc/containerd/ocicrypt/keys"]
env = ["OCICRYPT_KEYPROVIDER_CONFIG=/etc/containerd/ocicrypt/ocicrypt_keyprovider.conf"]
path = "ctd-decoder"
returns = "application/vnd.oci.image.layer.v1.tar+gzip"
[timeouts]
"io.containerd.timeout.bolt.open" = "0s"
"io.containerd.timeout.shim.cleanup" = "5s"
"io.containerd.timeout.shim.load" = "5s"
"io.containerd.timeout.shim.shutdown" = "3s"
"io.containerd.timeout.task.state" = "2s"
[ttrpc]
address = ""
gid = 0
uid = 0