Skip to content

Spool to disk causes process to hang at start #9874

@andrewkroh

Description

@andrewkroh

When using the disk backed queue feature with Filebeat 6.6.0 the Beat is hanging at startup. I tried Filebeat 7.0.0 (filebeat version 7.0.0 (arm), libbeat 7.0.0 [e5afe89 built 2018-12-28 23:39:49 +0000 UTC]) and it started up fine.

For confirmed bugs, please report:

queue:
  spool:
    size: 250 MiB
  • ./filebeat test config -e -d "*" or ./filebeat -e -d "*"
  • It will hang.
  • Dump stack. Ctrl+\
{"level":"warn","timestamp":"2019-01-03T17:32:01.509Z","logger":"cfgwarn","caller":"spool/module.go:43","message":"BETA: Spooling to disk is beta"}
^\SIGQUIT: quit
PC=0x5649b0 m=0 sigcode=128
 
goroutine 0 [idle]:
runtime.futex(0x177c1b8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x177c1b8, 0x0, 0x0, 0x53c594, ...)
	/usr/local/go/src/runtime/sys_linux_arm.s:266 +0x1c
runtime.futexsleep(0x177c1b8, 0x0, 0xffffffff, 0xffffffff)
	/usr/local/go/src/runtime/os_linux.go:45 +0x74
runtime.notesleep(0x177c1b8)
	/usr/local/go/src/runtime/lock_futex.go:151 +0xac
runtime.stopm()
	/usr/local/go/src/runtime/proc.go:1952 +0xac
runtime.findrunnable(0x12344600, 0x0)
	/usr/local/go/src/runtime/proc.go:2415 +0x378
runtime.schedule()
	/usr/local/go/src/runtime/proc.go:2541 +0x110
runtime.goexit0(0x124a1dc0)
	/usr/local/go/src/runtime/proc.go:2713 +0x1e0
runtime.mcall(0x0)
	/usr/local/go/src/runtime/asm_arm.s:285 +0x5c
 
goroutine 1 [semacquire]:
sync.runtime_SemacquireMutex(0x12690030, 0x97100)
	/usr/local/go/src/runtime/sema.go:71 +0x2c
sync.(*Mutex).Lock(0x1269002c)
	/usr/local/go/src/sync/mutex.go:134 +0x168
github.com/elastic/beats/vendor/github.com/elastic/go-txfile.(*reservedLock).Lock(0x1269001c)
	/go/src/github.com/elastic/beats/vendor/github.com/elastic/go-txfile/lock.go:104 +0x24
github.com/elastic/beats/vendor/github.com/elastic/go-txfile.(*File).Close(0x12690000, 0x0, 0x0)
	/go/src/github.com/elastic/beats/vendor/github.com/elastic/go-txfile/file.go:239 +0xf8
github.com/elastic/beats/vendor/github.com/elastic/go-txfile.(*File).Close-fm(0x0, 0x5098ac)
	/go/src/github.com/elastic/beats/libbeat/publisher/queue/spool/spool.go:111 +0x1c
github.com/elastic/beats/libbeat/publisher/queue/spool.ignoreErr.func1()
	/go/src/github.com/elastic/beats/libbeat/publisher/queue/spool/spool.go:248 +0x1c
github.com/elastic/beats/libbeat/publisher/queue/spool.ifNotOK(0x127a775c, 0x12781998)
	/go/src/github.com/elastic/beats/libbeat/publisher/queue/spool/spool.go:243 +0x30
panic(0xf5c1f0, 0x168cbd0)
	/usr/local/go/src/runtime/panic.go:502 +0x204
sync/atomic.addUint64(0x12690864, 0x1, 0x0, 0x1, 0x1)
	/usr/local/go/src/sync/atomic/64bit_arm.go:31 +0x4c
github.com/elastic/beats/vendor/github.com/elastic/go-txfile.(*File).beginTx(0x12690000, 0x0, 0x0, 0x0, 0x0, 0x8, 0x1)
	/go/src/github.com/elastic/beats/vendor/github.com/elastic/go-txfile/file.go:303 +0x114
github.com/elastic/beats/vendor/github.com/elastic/go-txfile.(*File).BeginWith(0x12690000, 0x12350000, 0x0, 0x0, 0x1, 0x12350a00, 0x76d3d364)
	/go/src/github.com/elastic/beats/vendor/github.com/elastic/go-txfile/file.go:287 +0x3c
github.com/elastic/beats/vendor/github.com/elastic/go-txfile.(*File).Begin(0x12690000, 0x1276e474, 0xd6ec80, 0x1276e46c)
	/go/src/github.com/elastic/beats/vendor/github.com/elastic/go-txfile/file.go:275 +0x38
github.com/elastic/beats/vendor/github.com/elastic/go-txfile/pq.NewStandaloneDelegate(0x12690000, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/elastic/beats/vendor/github.com/elastic/go-txfile/pq/delegate.go:59 +0x30
github.com/elastic/beats/libbeat/publisher/queue/spool.NewSpool(0x1147548, 0x12780750, 0x123aa900, 0x1b, 0x180, 0x0, 0x0, 0x6400000, 0x0, 0x1000, ...)
	/go/src/github.com/elastic/beats/libbeat/publisher/queue/spool/spool.go:113 +0x2ac
github.com/elastic/beats/libbeat/publisher/queue/spool.create(0x113c940, 0x124b8118, 0x12780750, 0x12785620, 0xf47c30, 0x101e210, 0x12657e00, 0x743c99a8)
	/go/src/github.com/elastic/beats/libbeat/publisher/queue/spool/module.go:65 +0x1b0
github.com/elastic/beats/libbeat/publisher/pipeline.createQueueBuilder.func1(0x113c940, 0x124b8118, 0x1276efc0, 0x743c99a8, 0x1276efc0, 0x127af440)
	/go/src/github.com/elastic/beats/libbeat/publisher/pipeline/module.go:192 +0x40
github.com/elastic/beats/libbeat/publisher/pipeline.New(0x105515d, 0x8, 0x105515d, 0x8, 0x1050a08, 0x5, 0x12656d10, 0xb, 0x12656d10, 0xb, ...)
	/go/src/github.com/elastic/beats/libbeat/publisher/pipeline/pipeline.go:191 +0x230
github.com/elastic/beats/libbeat/publisher/pipeline.Load(0x105515d, 0x8, 0x105515d, 0x8, 0x1050a08, 0x5, 0x12656d10, 0xb, 0x12656d10, 0xb, ...)
	/go/src/github.com/elastic/beats/libbeat/publisher/pipeline/module.go:105 +0x4b0
github.com/elastic/beats/libbeat/cmd/instance.(*Beat).createBeater(0x1267ea20, 0x10b1c8c, 0x0, 0x105515d, 0x8, 0x0)
	/go/src/github.com/elastic/beats/libbeat/cmd/instance/beat.go:315 +0x2f4
github.com/elastic/beats/libbeat/cmd/instance.(*Beat).launch(0x1267ea20, 0x105515d, 0x8, 0x105515d, 0x8, 0x0, 0x0, 0x0, 0x0, 0x12448900, ...)
	/go/src/github.com/elastic/beats/libbeat/cmd/instance/beat.go:358 +0x12c
github.com/elastic/beats/libbeat/cmd/instance.Run.func1(0x105515d, 0x8, 0x105515d, 0x8, 0x0, 0x0, 0x105515d, 0x8, 0x105515d, 0x8, ...)
	/go/src/github.com/elastic/beats/libbeat/cmd/instance/beat.go:184 +0x4e8
github.com/elastic/beats/libbeat/cmd/instance.Run(0x105515d, 0x8, 0x105515d, 0x8, 0x0, 0x0, 0x0, 0x0, 0x12448900, 0x0, ...)
	/go/src/github.com/elastic/beats/libbeat/cmd/instance/beat.go:185 +0x58
github.com/elastic/beats/libbeat/cmd.genRunCmd.func1(0x12612a00, 0x124af810, 0x0, 0xd)
	/go/src/github.com/elastic/beats/libbeat/cmd/run.go:37 +0x30
github.com/elastic/beats/vendor/github.com/spf13/cobra.(*Command).execute(0x12612a00, 0x123ae008, 0xd, 0xd, 0x12612a00, 0x123ae008)
	/go/src/github.com/elastic/beats/vendor/github.com/spf13/cobra/command.go:704 +0x210
github.com/elastic/beats/vendor/github.com/spf13/cobra.(*Command).ExecuteC(0x12612a00, 0x12612a00, 0x105515d, 0x0)
	/go/src/github.com/elastic/beats/vendor/github.com/spf13/cobra/command.go:785 +0x1f8
github.com/elastic/beats/vendor/github.com/spf13/cobra.(*Command).Execute(0x12612a00, 0x0, 0x12396030)
	/go/src/github.com/elastic/beats/vendor/github.com/spf13/cobra/command.go:738 +0x1c
main.main()
	/go/src/github.com/elastic/beats/x-pack/filebeat/main.go:22 +0x24
 
goroutine 36 [syscall]:
os/signal.signal_recv(0x563e2c)
	/usr/local/go/src/runtime/sigqueue.go:139 +0x130
os/signal.loop()
	/usr/local/go/src/os/signal/signal_unix.go:22 +0x14
created by os/signal.init.0
	/usr/local/go/src/os/signal/signal_unix.go:28 +0x30
 
goroutine 93 [semacquire]:
sync.runtime_notifyListWait(0x124e4128, 0x0)
	/usr/local/go/src/runtime/sema.go:510 +0x13c
sync.(*Cond).Wait(0x124e4120)
	/usr/local/go/src/sync/cond.go:56 +0x6c
github.com/elastic/beats/vendor/github.com/elastic/go-txfile.(*writer).nextCommand(0x1269004c, 0x12570000, 0x400, 0x400, 0x0, 0x0, 0x509800, 0x518200)
	/go/src/github.com/elastic/beats/vendor/github.com/elastic/go-txfile/write.go:217 +0x100
github.com/elastic/beats/vendor/github.com/elastic/go-txfile.(*writer).Run(0x1269004c, 0x10b2f78, 0x1269003c, 0x126701e0)
	/go/src/github.com/elastic/beats/vendor/github.com/elastic/go-txfile/write.go:149 +0x21c
github.com/elastic/beats/vendor/github.com/elastic/go-txfile.newFile.func1(0x12690000)
	/go/src/github.com/elastic/beats/vendor/github.com/elastic/go-txfile/file.go:203 +0x4c
created by github.com/elastic/beats/vendor/github.com/elastic/go-txfile.newFile
	/go/src/github.com/elastic/beats/vendor/github.com/elastic/go-txfile/file.go:201 +0x310
 
trap    0x0
error   0x0
oldmask 0x0
r0      0x177c1b8
r1      0x0
r2      0x0
r3      0x0
r4      0x0
r5      0x0
r6      0x3
r7      0xf0
r8      0x53f0a17f
r9      0x124a1e88
r10     0x177bcc8
fp      0x0
ip      0x126617d3
sp      0x7eb92434
lr      0x53395c
pc      0x5649b0
cpsr    0x20000010
fault   0x0

Metadata

Metadata

Assignees

No one assigned

    Labels

    Type

    No type
    No fields configured for issues without a type.

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions