Skip to content

Panic on log rotation when running docker logs -f xxxx #20327

@mountkin

Description

@mountkin

Reproduce:

docker run -d --name=test --log-opt max-size=500 --log-opt max-file=5 busybox sh -c "sleep 10;yes X|head -c 200"
docker logs -f test

Daemon log:

DEBU[23526] GET /v1.22/containers/logs/logs?follow=1&stderr=1&stdout=1&tail=all 
DEBU[23526] logs: begin stream                           
DEBU[23526] waiting for events                            logger=json-file
DEBU[23527] waiting for events                            logger=json-file
WARN[23527] falling back to file poller                   logger=json-file
ERRO[23527] error watching log file for modifications: stat /data1/docker/containers/f09b6bb01578d11ee96393cf9d99fb5ba32d54a941abf1520c1afea953980646/f09b6bb01578d11ee96393cf9d99fb5ba32d54a941abf1520c1afea953980646-json.log: no such file or directory 
DEBU[23527] waiting for events                            logger=json-file
WARN[23527] falling back to file poller                   logger=json-file
DEBU[23527] waiting for events                            logger=json-file
WARN[23527] falling back to file poller                   logger=json-file
DEBU[23527] waiting for events                            logger=json-file
ERRO[23527] Error streaming logs: stat /data1/docker/containers/f09b6bb01578d11ee96393cf9d99fb5ba32d54a941abf1520c1afea953980646/f09b6bb01578d11ee96393cf9d99fb5ba32d54a941abf1520c1afea953980646-json.log: no such file or directory 
WARN[23527] falling back to file poller                   logger=json-file
DEBU[23527] waiting for events                            logger=json-file
WARN[23527] falling back to file poller                   logger=json-file
DEBU[23527] waiting for events                            logger=json-file
WARN[23527] falling back to file poller                   logger=json-file
DEBU[23527] waiting for events                            logger=json-file
WARN[23527] falling back to file poller                   logger=json-file
DEBU[23527] waiting for events                            logger=json-file
WARN[23527] falling back to file poller                   logger=json-file
DEBU[23527] waiting for events                            logger=json-file
WARN[23527] falling back to file poller                   logger=json-file
ERRO[23527] error watching log file for modifications: open /data1/docker/containers/f09b6bb01578d11ee96393cf9d99fb5ba32d54a941abf1520c1afea953980646/f09b6bb01578d11ee96393cf9d99fb5ba32d54a941abf1520c1afea953980646-json.log: no such file or directory 
DEBU[23527] waiting for events                            logger=json-file
WARN[23527] falling back to file poller                   logger=json-file
ERRO[23527] error watching log file for modifications: open /data1/docker/containers/f09b6bb01578d11ee96393cf9d99fb5ba32d54a941abf1520c1afea953980646/f09b6bb01578d11ee96393cf9d99fb5ba32d54a941abf1520c1afea953980646-json.log: no such file or directory 
DEBU[23527] watch for /data1/docker/containers/f09b6bb01578d11ee96393cf9d99fb5ba32d54a941abf1520c1afea953980646/f09b6bb01578d11ee96393cf9d99fb5ba32d54a941abf1520c1afea953980646-json.log closed 
panic: send on closed channel

goroutine 980 [running]:
github.com/docker/docker/pkg/filenotify.(*filePoller).sendEvent(0xc82145a000, 0xc82179652a, 0x49, 0x2, 0xc8215780c0, 0x0, 0x0)
    /usr/src/docker/.gopath/src/github.com/docker/docker/pkg/filenotify/poller.go:128 +0x164
github.com/docker/docker/pkg/filenotify.(*filePoller).watch(0xc82145a000, 0xc8207f54a0, 0x7fd26a65b648, 0xc8214d0000, 0xc8215780c0)
    /usr/src/docker/.gopath/src/github.com/docker/docker/pkg/filenotify/poller.go:198 +0x676
created by github.com/docker/docker/pkg/filenotify.(*filePoller).Add
    /usr/src/docker/.gopath/src/github.com/docker/docker/pkg/filenotify/poller.go:69 +0x37e

goroutine 1 [chan receive, 392 minutes]:
main.(*DaemonCli).CmdDaemon(0xc8203ad760, 0xc82000a0c0, 0x8, 0x8, 0x0, 0x0)
    /usr/src/docker/docker/daemon.go:305 +0x20a7
reflect.callMethod(0xc8206e0de0, 0xc82093fc78)
    /usr/local/go/src/reflect/value.go:628 +0x1fc
reflect.methodValueCall(0xc82000a0c0, 0x8, 0x8, 0x1, 0xc8206e0de0, 0x0, 0x0, 0xc8206e0de0, 0x0, 0x479f34, ...)
    /usr/local/go/src/reflect/asm_amd64.s:29 +0x36
github.com/docker/docker/cli.(*Cli).Run(0xc8206e0d50, 0xc82000a0b0, 0x9, 0x9, 0x0, 0x0)
    /usr/src/docker/.gopath/src/github.com/docker/docker/cli/cli.go:89 +0x383
main.main()
    /usr/src/docker/docker/docker.go:63 +0x43c

goroutine 17 [syscall, 392 minutes, locked to thread]:
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:1721 +0x1

docker version:

Client:
 Version:      1.10.1
 API version:  1.22
 Go version:   go1.5.3
 Git commit:   9e83765
 Built:        Thu Feb 11 19:27:08 2016
 OS/Arch:      linux/amd64

Server:
 Version:      1.10.1
 API version:  1.22
 Go version:   go1.5.3
 Git commit:   9e83765
 Built:        Thu Feb 11 19:27:08 2016
 OS/Arch:      linux/amd64

docker info:

Containers: 1
 Running: 1
 Paused: 0
 Stopped: 0
Images: 232
Server Version: 1.10.1
Storage Driver: aufs
 Root Dir: /data1/docker/aufs
 Backing Filesystem: extfs
 Dirs: 242
 Dirperm1 Supported: false
Execution Driver: native-0.2
Logging Driver: json-file
Plugins: 
 Volume: local
 Network: null host bridge
Kernel Version: 3.13.0-76-generic
Operating System: Ubuntu 14.04.3 LTS
OSType: linux
Architecture: x86_64
CPUs: 8
Total Memory: 7.703 GiB
Name: sky
ID: HJRN:G4UP:MRFS:DPGA:UETY:4533:2KPV:EIXV:R5T7:N5IU:YQWU:TKK2
Debug mode (server): true
 File Descriptors: 23
 Goroutines: 59
 System Time: 2016-02-15T19:32:01.93920522+08:00
 EventsListeners: 0
 Init SHA1: e1042dbb0bcf49bb9da188176d9a5063cdb92a01
 Init Path: /usr/lib/docker/dockerinit
 Docker Root Dir: /data1/docker
WARNING: No swap limit support

Metadata

Metadata

Assignees

Labels

area/loggingkind/bugBugs are bugs. The cause may or may not be known at triage time so debugging may be needed.priority/P1Important: P1 issues are a top priority and a must-have for the next release.

Type

No type

Projects

No projects

Milestone

Relationships

None yet

Development

No branches or pull requests

Issue actions