Description
Following a change introduced in 2.3.4, most likely #9261, we're hitting a scenario where a container is recreated when it needn't be, and then will fail to start following that
Steps to reproduce the issue:
- Create a
docker-compose.yml similar to the below:
version: '3.7'
services:
db:
image: postgres:11
hostname: postgres
container_name: engage-db
ports:
- "5437:5432"
environment:
- POSTGRES_DB=somedb
- POSTGRES_USER=someuser
- POSTGRES_PASSWORD=somepassword
wait-for-db:
image: postgres:11
command: [ sh, -c, "until pg_isready --username=someuser --host=db --port=5432; do sleep 5; done" ]
links:
- db
- Ensure
postgres:11 is not already in the docker image cache
- Run
docker-compose up -d db to start the db in the background, this will also pull the postgres:11 image.
- Run
docker-compose up wait-for-db to start the wait in the foreground
Describe the results you received:
When the last step is run to start the wait-for-db container, it will result in the db container being recreated, despite the image etc. not having since been rebuilt or changed in any way:
$ docker-compose up wait-for-db
[+] Running 2/2
⠿ Container engage-db Recreated 0.3s
⠿ Container engage-wait-for-db-1 Created 0.1s
Attaching to engage-wait-for-db-1
Following this, the recreated db container will also not be running, it will be sat in a Created state:
125fee896799 postgres:11 "docker-entrypoint.s…" About a minute ago Created engage-db
$docker inspect 125fee896799
```
[
{
"Id": "125fee896799d4367c6ccae891065e120ded3ad6ccc8c525a45253abd5c88c27",
"Created": "2022-04-06T10:05:51.95171076Z",
"Path": "docker-entrypoint.sh",
"Args": [
"postgres"
],
"State": {
"Status": "created",
"Running": false,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 0,
"ExitCode": 0,
"Error": "",
"StartedAt": "0001-01-01T00:00:00Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Image": "sha256:04a660188f48f78bc42b2a03597123f9b0504f2db039daf8f2d98f93b04c2711",
"ResolvConfPath": "",
"HostnamePath": "",
"HostsPath": "",
"LogPath": "",
"Name": "/engage-db",
"RestartCount": 0,
"Driver": "overlay2",
"Platform": "linux",
"MountLabel": "",
"ProcessLabel": "",
"AppArmorProfile": "",
"ExecIDs": null,
"HostConfig": {
"Binds": [],
"ContainerIDFile": "",
"LogConfig": {
"Type": "json-file",
"Config": {}
},
"NetworkMode": "engage_default",
"PortBindings": {
"5432/tcp": [
{
"HostIp": "",
"HostPort": "5437"
}
]
},
"RestartPolicy": {
"Name": "",
"MaximumRetryCount": 0
},
"AutoRemove": false,
"VolumeDriver": "",
"VolumesFrom": null,
"CapAdd": null,
"CapDrop": null,
"CgroupnsMode": "private",
"Dns": null,
"DnsOptions": null,
"DnsSearch": null,
"ExtraHosts": null,
"GroupAdd": null,
"IpcMode": "private",
"Cgroup": "",
"Links": null,
"OomScoreAdj": 0,
"PidMode": "",
"Privileged": false,
"PublishAllPorts": false,
"ReadonlyRootfs": false,
"SecurityOpt": null,
"UTSMode": "",
"UsernsMode": "",
"ShmSize": 67108864,
"Runtime": "runc",
"ConsoleSize": [
0,
0
],
"Isolation": "",
"CpuShares": 0,
"Memory": 0,
"NanoCpus": 0,
"CgroupParent": "",
"BlkioWeight": 0,
"BlkioWeightDevice": null,
"BlkioDeviceReadBps": null,
"BlkioDeviceWriteBps": null,
"BlkioDeviceReadIOps": null,
"BlkioDeviceWriteIOps": null,
"CpuPeriod": 0,
"CpuQuota": 0,
"CpuRealtimePeriod": 0,
"CpuRealtimeRuntime": 0,
"CpusetCpus": "",
"CpusetMems": "",
"Devices": null,
"DeviceCgroupRules": null,
"DeviceRequests": null,
"KernelMemory": 0,
"KernelMemoryTCP": 0,
"MemoryReservation": 0,
"MemorySwap": 0,
"MemorySwappiness": null,
"OomKillDisable": false,
"PidsLimit": null,
"Ulimits": null,
"CpuCount": 0,
"CpuPercent": 0,
"IOMaximumIOps": 0,
"IOMaximumBandwidth": 0,
"Mounts": [
{
"Type": "volume",
"Source": "26b0d26dc817cc36f1ad1290dfc27f1bc9984a9c5796a948eb3d6c9f27b2abe6",
"Target": "/var/lib/postgresql/data"
}
],
"MaskedPaths": [
"/proc/asound",
"/proc/acpi",
"/proc/kcore",
"/proc/keys",
"/proc/latency_stats",
"/proc/timer_list",
"/proc/timer_stats",
"/proc/sched_debug",
"/proc/scsi",
"/sys/firmware"
],
"ReadonlyPaths": [
"/proc/bus",
"/proc/fs",
"/proc/irq",
"/proc/sys",
"/proc/sysrq-trigger"
]
},
"GraphDriver": {
"Data": {
"LowerDir": "/var/lib/docker/overlay2/10d96b1e87acb2e1617cc0c714fda6dfc727806d697033bd961637a3e372758d-init/diff:/var/lib/docker/overlay2/0d6e1a46381ea25e8cf96a9106f039946dd3e4427627e6cb1083005c718675e9/diff:/var/lib/docker/overlay2/4477f40fb0f241080e56a5450f965526ef4d7db71d62d81027cb1910b2e00f9d/diff:/var/lib/docker/overlay2/8bb3fbfc8a6bad7774fcac5117745a279a3d93a40cd8b1c4d577c34e2e9e5963/diff:/var/lib/docker/overlay2/112cd7cff09b5078cad860b6ae71dc9fb37e351656a229ac79f5ca68d0a2344b/diff:/var/lib/docker/overlay2/f6eb995351d9cbf225916217b20936bff5d4206931cddb1b9046cfeb22624ac7/diff:/var/lib/docker/overlay2/f5816c70b0e48705ce9827561cb91b99d67bb1671a341eb4101e329f780a13ba/diff:/var/lib/docker/overlay2/12453a6429f68bfe21e3b4f21b9febfb8117a896210dd2e29492001e5dc0bf1b/diff:/var/lib/docker/overlay2/42badba2cfaf487b90abdfc0417f3f814c0909f1b35242f0163a3535ccc23f8b/diff:/var/lib/docker/overlay2/ae6ff9f97ba34f0b9964492930f1e800e78ea65883e1b53d26e1136762895fde/diff:/var/lib/docker/overlay2/0f426746f21a5c0ddc48638a6b341190b522fce06b5b8f446760f33180556818/diff:/var/lib/docker/overlay2/fa6edca9880603046c4c5e5b65bbe59577cb7f6ec14264c81cf50c70600108bf/diff:/var/lib/docker/overlay2/9d45d8aedf1e0fcbe07c0c87822f8568c91d35f0dbb65e6cade336b000273725/diff:/var/lib/docker/overlay2/74b44bb155159c41af7b7ec4fad504f6753c0666d191bf2f4f234485e37d82b1/diff",
"MergedDir": "/var/lib/docker/overlay2/10d96b1e87acb2e1617cc0c714fda6dfc727806d697033bd961637a3e372758d/merged",
"UpperDir": "/var/lib/docker/overlay2/10d96b1e87acb2e1617cc0c714fda6dfc727806d697033bd961637a3e372758d/diff",
"WorkDir": "/var/lib/docker/overlay2/10d96b1e87acb2e1617cc0c714fda6dfc727806d697033bd961637a3e372758d/work"
},
"Name": "overlay2"
},
"Mounts": [
{
"Type": "volume",
"Name": "26b0d26dc817cc36f1ad1290dfc27f1bc9984a9c5796a948eb3d6c9f27b2abe6",
"Source": "/var/lib/docker/volumes/26b0d26dc817cc36f1ad1290dfc27f1bc9984a9c5796a948eb3d6c9f27b2abe6/_data",
"Destination": "/var/lib/postgresql/data",
"Driver": "local",
"Mode": "z",
"RW": true,
"Propagation": ""
}
],
"Config": {
"Hostname": "postgres",
"Domainname": "",
"User": "",
"AttachStdin": false,
"AttachStdout": true,
"AttachStderr": true,
"ExposedPorts": {
"5432/tcp": {}
},
"Tty": false,
"OpenStdin": false,
"StdinOnce": false,
"Env": [
"POSTGRES_PASSWORD=somepassword",
"POSTGRES_DB=somedb",
"POSTGRES_USER=someuser",
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/postgresql/11/bin",
"GOSU_VERSION=1.14",
"LANG=en_US.utf8",
"PG_MAJOR=11",
"PG_VERSION=11.15-1.pgdg90+1",
"PGDATA=/var/lib/postgresql/data"
],
"Cmd": [
"postgres"
],
"Image": "postgres:11",
"Volumes": {
"/var/lib/postgresql/data": {}
},
"WorkingDir": "",
"Entrypoint": [
"docker-entrypoint.sh"
],
"OnBuild": null,
"Labels": {
"com.docker.compose.config-hash": "2cfb5e6a3de7283e5cf2ef6ff09ada28fbdc8c9447ae459da4f7fa2c00c1ec6c",
"com.docker.compose.container-number": "1",
"com.docker.compose.depends_on": "",
"com.docker.compose.image": "sha256:04a660188f48f78bc42b2a03597123f9b0504f2db039daf8f2d98f93b04c2711",
"com.docker.compose.oneoff": "False",
"com.docker.compose.project": "engage",
"com.docker.compose.project.config_files": "/Users/Michael.Eves/Workspace/Docker/engage/docker-compose.yml",
"com.docker.compose.project.working_dir": "/Users/Michael.Eves/Workspace/Docker/engage",
"com.docker.compose.service": "db",
"com.docker.compose.version": "2.3.4"
},
"StopSignal": "SIGINT"
},
"NetworkSettings": {
"Bridge": "",
"SandboxID": "",
"HairpinMode": false,
"LinkLocalIPv6Address": "",
"LinkLocalIPv6PrefixLen": 0,
"Ports": {},
"SandboxKey": "",
"SecondaryIPAddresses": null,
"SecondaryIPv6Addresses": null,
"EndpointID": "",
"Gateway": "",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"IPAddress": "",
"IPPrefixLen": 0,
"IPv6Gateway": "",
"MacAddress": "",
"Networks": {
"engage_default": {
"IPAMConfig": null,
"Links": null,
"Aliases": [
"engage-db",
"db",
"125fee896799",
"postgres"
],
"NetworkID": "",
"EndpointID": "",
"Gateway": "",
"IPAddress": "",
"IPPrefixLen": 0,
"IPv6Gateway": "",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"MacAddress": "",
"DriverOpts": null
}
}
}
}
]
```
If you stop everything docker-compose down -v and try again now the image postgres:11 is already pulled, the container is not recreated and everything works as expected.
Describe the results you expected:
The container should not be recreated as nothing has changed about the image. Adding --no-recreate on the second compose command also works around the issue, but I don't believe the behaviour is correct currently on two points:
- The container shouldn't be recreated
- If a container is recreated, it should start back up
Output of docker compose version: Happens from vesion 2.3.4 inc. current release 2.4.1.
Output of docker info: 20.10.11/20.10.8
Description
Following a change introduced in
2.3.4, most likely #9261, we're hitting a scenario where a container is recreated when it needn't be, and then will fail to start following thatSteps to reproduce the issue:
docker-compose.ymlsimilar to the below:postgres:11is not already in the docker image cachedocker-compose up -d dbto start the db in the background, this will also pull thepostgres:11image.docker-compose up wait-for-dbto start the wait in the foregroundDescribe the results you received:
When the last step is run to start the
wait-for-dbcontainer, it will result in thedbcontainer being recreated, despite the image etc. not having since been rebuilt or changed in any way:Following this, the recreated
dbcontainer will also not be running, it will be sat in aCreatedstate:$docker inspect 125fee896799
``` [ { "Id": "125fee896799d4367c6ccae891065e120ded3ad6ccc8c525a45253abd5c88c27", "Created": "2022-04-06T10:05:51.95171076Z", "Path": "docker-entrypoint.sh", "Args": [ "postgres" ], "State": { "Status": "created", "Running": false, "Paused": false, "Restarting": false, "OOMKilled": false, "Dead": false, "Pid": 0, "ExitCode": 0, "Error": "", "StartedAt": "0001-01-01T00:00:00Z", "FinishedAt": "0001-01-01T00:00:00Z" }, "Image": "sha256:04a660188f48f78bc42b2a03597123f9b0504f2db039daf8f2d98f93b04c2711", "ResolvConfPath": "", "HostnamePath": "", "HostsPath": "", "LogPath": "", "Name": "/engage-db", "RestartCount": 0, "Driver": "overlay2", "Platform": "linux", "MountLabel": "", "ProcessLabel": "", "AppArmorProfile": "", "ExecIDs": null, "HostConfig": { "Binds": [], "ContainerIDFile": "", "LogConfig": { "Type": "json-file", "Config": {} }, "NetworkMode": "engage_default", "PortBindings": { "5432/tcp": [ { "HostIp": "", "HostPort": "5437" } ] }, "RestartPolicy": { "Name": "", "MaximumRetryCount": 0 }, "AutoRemove": false, "VolumeDriver": "", "VolumesFrom": null, "CapAdd": null, "CapDrop": null, "CgroupnsMode": "private", "Dns": null, "DnsOptions": null, "DnsSearch": null, "ExtraHosts": null, "GroupAdd": null, "IpcMode": "private", "Cgroup": "", "Links": null, "OomScoreAdj": 0, "PidMode": "", "Privileged": false, "PublishAllPorts": false, "ReadonlyRootfs": false, "SecurityOpt": null, "UTSMode": "", "UsernsMode": "", "ShmSize": 67108864, "Runtime": "runc", "ConsoleSize": [ 0, 0 ], "Isolation": "", "CpuShares": 0, "Memory": 0, "NanoCpus": 0, "CgroupParent": "", "BlkioWeight": 0, "BlkioWeightDevice": null, "BlkioDeviceReadBps": null, "BlkioDeviceWriteBps": null, "BlkioDeviceReadIOps": null, "BlkioDeviceWriteIOps": null, "CpuPeriod": 0, "CpuQuota": 0, "CpuRealtimePeriod": 0, "CpuRealtimeRuntime": 0, "CpusetCpus": "", "CpusetMems": "", "Devices": null, "DeviceCgroupRules": null, "DeviceRequests": null, "KernelMemory": 0, "KernelMemoryTCP": 0, "MemoryReservation": 0, "MemorySwap": 0, "MemorySwappiness": null, "OomKillDisable": false, "PidsLimit": null, "Ulimits": null, "CpuCount": 0, "CpuPercent": 0, "IOMaximumIOps": 0, "IOMaximumBandwidth": 0, "Mounts": [ { "Type": "volume", "Source": "26b0d26dc817cc36f1ad1290dfc27f1bc9984a9c5796a948eb3d6c9f27b2abe6", "Target": "/var/lib/postgresql/data" } ], "MaskedPaths": [ "/proc/asound", "/proc/acpi", "/proc/kcore", "/proc/keys", "/proc/latency_stats", "/proc/timer_list", "/proc/timer_stats", "/proc/sched_debug", "/proc/scsi", "/sys/firmware" ], "ReadonlyPaths": [ "/proc/bus", "/proc/fs", "/proc/irq", "/proc/sys", "/proc/sysrq-trigger" ] }, "GraphDriver": { "Data": { "LowerDir": "/var/lib/docker/overlay2/10d96b1e87acb2e1617cc0c714fda6dfc727806d697033bd961637a3e372758d-init/diff:/var/lib/docker/overlay2/0d6e1a46381ea25e8cf96a9106f039946dd3e4427627e6cb1083005c718675e9/diff:/var/lib/docker/overlay2/4477f40fb0f241080e56a5450f965526ef4d7db71d62d81027cb1910b2e00f9d/diff:/var/lib/docker/overlay2/8bb3fbfc8a6bad7774fcac5117745a279a3d93a40cd8b1c4d577c34e2e9e5963/diff:/var/lib/docker/overlay2/112cd7cff09b5078cad860b6ae71dc9fb37e351656a229ac79f5ca68d0a2344b/diff:/var/lib/docker/overlay2/f6eb995351d9cbf225916217b20936bff5d4206931cddb1b9046cfeb22624ac7/diff:/var/lib/docker/overlay2/f5816c70b0e48705ce9827561cb91b99d67bb1671a341eb4101e329f780a13ba/diff:/var/lib/docker/overlay2/12453a6429f68bfe21e3b4f21b9febfb8117a896210dd2e29492001e5dc0bf1b/diff:/var/lib/docker/overlay2/42badba2cfaf487b90abdfc0417f3f814c0909f1b35242f0163a3535ccc23f8b/diff:/var/lib/docker/overlay2/ae6ff9f97ba34f0b9964492930f1e800e78ea65883e1b53d26e1136762895fde/diff:/var/lib/docker/overlay2/0f426746f21a5c0ddc48638a6b341190b522fce06b5b8f446760f33180556818/diff:/var/lib/docker/overlay2/fa6edca9880603046c4c5e5b65bbe59577cb7f6ec14264c81cf50c70600108bf/diff:/var/lib/docker/overlay2/9d45d8aedf1e0fcbe07c0c87822f8568c91d35f0dbb65e6cade336b000273725/diff:/var/lib/docker/overlay2/74b44bb155159c41af7b7ec4fad504f6753c0666d191bf2f4f234485e37d82b1/diff", "MergedDir": "/var/lib/docker/overlay2/10d96b1e87acb2e1617cc0c714fda6dfc727806d697033bd961637a3e372758d/merged", "UpperDir": "/var/lib/docker/overlay2/10d96b1e87acb2e1617cc0c714fda6dfc727806d697033bd961637a3e372758d/diff", "WorkDir": "/var/lib/docker/overlay2/10d96b1e87acb2e1617cc0c714fda6dfc727806d697033bd961637a3e372758d/work" }, "Name": "overlay2" }, "Mounts": [ { "Type": "volume", "Name": "26b0d26dc817cc36f1ad1290dfc27f1bc9984a9c5796a948eb3d6c9f27b2abe6", "Source": "/var/lib/docker/volumes/26b0d26dc817cc36f1ad1290dfc27f1bc9984a9c5796a948eb3d6c9f27b2abe6/_data", "Destination": "/var/lib/postgresql/data", "Driver": "local", "Mode": "z", "RW": true, "Propagation": "" } ], "Config": { "Hostname": "postgres", "Domainname": "", "User": "", "AttachStdin": false, "AttachStdout": true, "AttachStderr": true, "ExposedPorts": { "5432/tcp": {} }, "Tty": false, "OpenStdin": false, "StdinOnce": false, "Env": [ "POSTGRES_PASSWORD=somepassword", "POSTGRES_DB=somedb", "POSTGRES_USER=someuser", "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/postgresql/11/bin", "GOSU_VERSION=1.14", "LANG=en_US.utf8", "PG_MAJOR=11", "PG_VERSION=11.15-1.pgdg90+1", "PGDATA=/var/lib/postgresql/data" ], "Cmd": [ "postgres" ], "Image": "postgres:11", "Volumes": { "/var/lib/postgresql/data": {} }, "WorkingDir": "", "Entrypoint": [ "docker-entrypoint.sh" ], "OnBuild": null, "Labels": { "com.docker.compose.config-hash": "2cfb5e6a3de7283e5cf2ef6ff09ada28fbdc8c9447ae459da4f7fa2c00c1ec6c", "com.docker.compose.container-number": "1", "com.docker.compose.depends_on": "", "com.docker.compose.image": "sha256:04a660188f48f78bc42b2a03597123f9b0504f2db039daf8f2d98f93b04c2711", "com.docker.compose.oneoff": "False", "com.docker.compose.project": "engage", "com.docker.compose.project.config_files": "/Users/Michael.Eves/Workspace/Docker/engage/docker-compose.yml", "com.docker.compose.project.working_dir": "/Users/Michael.Eves/Workspace/Docker/engage", "com.docker.compose.service": "db", "com.docker.compose.version": "2.3.4" }, "StopSignal": "SIGINT" }, "NetworkSettings": { "Bridge": "", "SandboxID": "", "HairpinMode": false, "LinkLocalIPv6Address": "", "LinkLocalIPv6PrefixLen": 0, "Ports": {}, "SandboxKey": "", "SecondaryIPAddresses": null, "SecondaryIPv6Addresses": null, "EndpointID": "", "Gateway": "", "GlobalIPv6Address": "", "GlobalIPv6PrefixLen": 0, "IPAddress": "", "IPPrefixLen": 0, "IPv6Gateway": "", "MacAddress": "", "Networks": { "engage_default": { "IPAMConfig": null, "Links": null, "Aliases": [ "engage-db", "db", "125fee896799", "postgres" ], "NetworkID": "", "EndpointID": "", "Gateway": "", "IPAddress": "", "IPPrefixLen": 0, "IPv6Gateway": "", "GlobalIPv6Address": "", "GlobalIPv6PrefixLen": 0, "MacAddress": "", "DriverOpts": null } } } } ] ```
If you stop everything
docker-compose down -vand try again now the imagepostgres:11is already pulled, the container is not recreated and everything works as expected.Describe the results you expected:
The container should not be recreated as nothing has changed about the image. Adding
--no-recreateon the second compose command also works around the issue, but I don't believe the behaviour is correct currently on two points:Output of
docker compose version: Happens from vesion2.3.4inc. current release2.4.1.Output of
docker info:20.10.11/20.10.8