Skip to content

Upgrading from 1.21.3 to 1.22.3 causes telegraf to crash  #11308

@shaffahmed

Description

@shaffahmed

Relevant telegraf.conf

[[inputs.file]]
  files =[ "/data/users/sas_metrics/sasviya/viya_usage_summary_users.csv" ]

  data_format = "csv"
  csv_header_row_count = 1
  csv_column_names = [ "usage_user_env","usage_user_server","usage_user_start_dt","usage_user_count" ]
  csv_tag_columns = [ "usage_user_env" ]
#  csv_measurement_column = "usage_user_count"
  csv_timestamp_column = "usage_user_start_dt"
  csv_timestamp_format = "Monday, 02-01-06 15:04:05 MST"

Logs from Telegraf

2022-06-15T15:24:04Z I! Starting Telegraf 1.22.3
2022-06-15T15:24:04Z I! Loaded inputs: cpu disk diskio exec file (4x) mem net
2022-06-15T15:24:04Z I! Loaded aggregators: 
2022-06-15T15:24:04Z I! Loaded processors: 
2022-06-15T15:24:04Z I! Loaded outputs: cloudwatch
2022-06-15T15:24:04Z I! Tags enabled: environment=sandpit host=ip-10-199-69-173.corp.xxxxxxx.uk role=sasviya target=spre0 zone=a
2022-06-15T15:24:04Z I! [agent] Config: Interval:30s, Quiet:false, Hostname:"ip-10-199-69-173.corp.xxxxxxxxxx.uk", Flush Interval:1m0s
2022-06-15T15:24:04Z D! [agent] Initializing plugins
2022-06-15T15:24:04Z D! [agent] Connecting outputs
2022-06-15T15:24:04Z D! [agent] Attempting connection to [outputs.cloudwatch]
2022-06-15T15:24:04Z D! [agent] Successfully connected to outputs.cloudwatch
2022-06-15T15:24:04Z D! [agent] Starting service inputs
2022-06-15T15:25:04Z D! [outputs.cloudwatch] Wrote batch of 71 metrics in 139.533084ms
2022-06-15T15:25:04Z D! [outputs.cloudwatch] Buffer fullness: 0 / 10000 metrics



2022-06-15T15:26:04Z D! [outputs.cloudwatch] Wrote batch of 72 metrics in 141.483312ms
2022-06-15T15:26:04Z D! [outputs.cloudwatch] Buffer fullness: 0 / 10000 metrics
2022-06-15T15:27:04Z D! [outputs.cloudwatch] Wrote batch of 72 metrics in 167.004316ms
2022-06-15T15:27:04Z D! [outputs.cloudwatch] Buffer fullness: 0 / 10000 metrics
2022-06-15T15:28:04Z D! [outputs.cloudwatch] Wrote batch of 72 metrics in 104.09637ms
2022-06-15T15:28:04Z D! [outputs.cloudwatch] Buffer fullness: 0 / 10000 metrics
2022-06-15T15:29:04Z D! [outputs.cloudwatch] Wrote batch of 72 metrics in 100.152659ms
2022-06-15T15:29:04Z D! [outputs.cloudwatch] Buffer fullness: 0 / 10000 metrics
2022-06-15T15:30:04Z D! [outputs.cloudwatch] Wrote batch of 72 metrics in 156.04586ms
2022-06-15T15:30:04Z D! [outputs.cloudwatch] Buffer fullness: 0 / 10000 metrics
fatal error: concurrent map writes
fatal error: concurrent map writes

goroutine 709 [running]:
runtime.throw({0x5334040?, 0x198?})
	/usr/local/go/src/runtime/panic.go:992 +0x71 fp=0xc0010f5890 sp=0xc0010f5860 pc=0x43a731
runtime.mapassign_faststr(0x5d95e28?, 0xc0005cfc80?, {0xc0005f63a8, 0x5})
	/usr/local/go/src/runtime/map_faststr.go:212 +0x39c fp=0xc0010f58f8 sp=0xc0010f5890 pc=0x41569c
github.com/influxdata/telegraf/config.(*Config).missingTomlField(0xc000455320, {0x1?, 0x0?}, {0xc0005f63a8, 0x5})
	/go/src/github.com/influxdata/telegraf/config/config.go:1860 +0xec9 fp=0xc0010f5928 sp=0xc0010f58f8 pc=0xaee829
github.com/influxdata/telegraf/config.(*Config).missingTomlField-fm({0x5dcbe70?, 0x51e1740?}, {0xc0005f63a8?, 0x5?})
	<autogenerated>:1 +0x45 fp=0xc0010f5960 sp=0xc0010f5928 pc=0xaf4345
github.com/influxdata/toml.fieldCache.findField({0xc0016345d0?, 0xc001634600?}, 0xc00000e0a8, {0x51e1740?, 0xc00164c480?, 0xc0005fc8a0?}, {0xc0005f63a8, 0x5})
	/go/pkg/mod/github.com/influxdata/toml@v0.0.0-20190415235208-270119a8ce65/util.go:74 +0x3da fp=0xc0010f5a50 sp=0xc0010f5960 pc=0xadd67a
github.com/influxdata/toml.unmarshalTable(0x7f1ee0691908?, {0x4f60c00?, 0xc00164c480?, 0x40?}, 0xc0005c6d70, 0x0)
	/go/pkg/mod/github.com/influxdata/toml@v0.0.0-20190415235208-270119a8ce65/decode.go:157 +0x525 fp=0xc0010f5c80 sp=0xc0010f5a50 pc=0xacdd45
github.com/influxdata/toml.(*Config).UnmarshalTable(0x48e73e0?, 0x4?, {0x4f60c00?, 0xc00164c480?})
	/go/pkg/mod/github.com/influxdata/toml@v0.0.0-20190415235208-270119a8ce65/decode.go:121 +0x1c5 fp=0xc0010f5cf8 sp=0xc0010f5c80 pc=0xacd3c5
github.com/influxdata/telegraf/config.(*Config).addParser(0xc000455320, {0xc0005f6370, 0x4}, 0xc0005c6d70)
	/go/src/github.com/influxdata/telegraf/config/config.go:1127 +0x216 fp=0xc0010f5d98 sp=0xc0010f5cf8 pc=0xae62b6
github.com/influxdata/telegraf/config.(*Config).addInput.func1()
	/go/src/github.com/influxdata/telegraf/config/config.go:1325 +0x29 fp=0xc0010f5dd0 sp=0xc0010f5d98 pc=0xae81e9
github.com/influxdata/telegraf/plugins/inputs/file.(*File).readMetric(0xc0000711a0, {0xc001374a40, 0x40})
	/go/src/github.com/influxdata/telegraf/plugins/inputs/file/file.go:125 +0x32f fp=0xc0010f5ec8 sp=0xc0010f5dd0 pc=0x16792ef
github.com/influxdata/telegraf/plugins/inputs/file.(*File).Gather(0xc0000711a0, {0x5dbdc10, 0xc0004c8640})
	/go/src/github.com/influxdata/telegraf/plugins/inputs/file/file.go:76 +0xaa fp=0xc0010f5f50 sp=0xc0010f5ec8 pc=0x1678b2a
github.com/influxdata/telegraf/models.(*RunningInput).Gather(0xc0005c6dc0, {0x5dbdc10, 0xc0004c8640})
	/go/src/github.com/influxdata/telegraf/models/running_input.go:118 +0x5a fp=0xc0010f5fa0 sp=0xc0010f5f50 pc=0x7acbda
github.com/influxdata/telegraf/agent.(*Agent).gatherOnce.func1()
	/go/src/github.com/influxdata/telegraf/agent/agent.go:487 +0x2e fp=0xc0010f5fe0 sp=0xc0010f5fa0 pc=0xb589ae
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1571 +0x1 fp=0xc0010f5fe8 sp=0xc0010f5fe0 pc=0x46dec1
created by github.com/influxdata/telegraf/agent.(*Agent).gatherOnce
	/go/src/github.com/influxdata/telegraf/agent/agent.go:486 +0x12a

goroutine 1 [semacquire, 6 minutes]:
sync.runtime_Semacquire(0xc0005c6e60?)
	/usr/local/go/src/runtime/sema.go:56 +0x25
sync.(*WaitGroup).Wait(0x5371fb4?)
	/usr/local/go/src/sync/waitgroup.go:136 +0x52
github.com/influxdata/telegraf/agent.(*Agent).Run(0xc000a041a8, {0x5da6bf0?, 0xc0010da9c0})
	/go/src/github.com/influxdata/telegraf/agent/agent.go:181 +0x865
main.runAgent({0x5da6bf0, 0xc0010da9c0}, {0x88c31a8, 0x0, 0x0}, {0x88c31a8, 0x0, 0x0})
	/go/src/github.com/influxdata/telegraf/cmd/telegraf/telegraf.go:328 +0x118d
main.reloadLoop({0x88c31a8, 0x0, 0x0}, {0x88c31a8, 0x0, 0x0})
	/go/src/github.com/influxdata/telegraf/cmd/telegraf/telegraf.go:158 +0x272
main.run(...)
	/go/src/github.com/influxdata/telegraf/cmd/telegraf/telegraf_posix.go:8
main.main()
	/go/src/github.com/influxdata/telegraf/cmd/telegraf/telegraf.go:501 +0xa9a

goroutine 71 [select]:
go.opencensus.io/stats/view.(*worker).start(0xc0002d2380)
	/go/pkg/mod/go.opencensus.io@v0.23.0/stats/view/worker.go:276 +0xad
created by go.opencensus.io/stats/view.init.0
	/go/pkg/mod/go.opencensus.io@v0.23.0/stats/view/worker.go:34 +0x8d

goroutine 72 [chan receive]:
k8s.io/klog/v2.(*loggingT).flushDaemon(0x0?)
	/go/pkg/mod/k8s.io/klog/v2@v2.30.0/klog.go:1181 +0x6a
created by k8s.io/klog/v2.init.0
	/go/pkg/mod/k8s.io/klog/v2@v2.30.0/klog.go:420 +0xf6

goroutine 73 [chan receive]:
github.com/ClickHouse/clickhouse-go.init.0.func1()
	/go/pkg/mod/github.com/!click!house/clickhouse-go@v1.5.4/bootstrap.go:48 +0x2d
created by github.com/ClickHouse/clickhouse-go.init.0
	/go/pkg/mod/github.com/!click!house/clickhouse-go@v1.5.4/bootstrap.go:45 +0x3f

goroutine 38 [syscall, 6 minutes]:
os/signal.signal_recv()
	/usr/local/go/src/runtime/sigqueue.go:151 +0x2f
os/signal.loop()
	/usr/local/go/src/os/signal/signal_unix.go:23 +0x19
created by os/signal.Notify.func1.1
	/usr/local/go/src/os/signal/signal.go:151 +0x2a

goroutine 55 [select, 6 minutes]:
main.reloadLoop.func1()
	/go/src/github.com/influxdata/telegraf/cmd/telegraf/telegraf.go:145 +0x79
created by main.reloadLoop
	/go/src/github.com/influxdata/telegraf/cmd/telegraf/telegraf.go:144 +0x22d

goroutine 56 [runnable]:
github.com/influxdata/telegraf/agent.(*Agent).runOutputs(0xc000a041a8, 0xc0004c82c0)
	/go/src/github.com/influxdata/telegraf/agent/agent.go:773 +0x2c5
github.com/influxdata/telegraf/agent.(*Agent).Run.func1()
	/go/src/github.com/influxdata/telegraf/agent/agent.go:150 +0x5c
created by github.com/influxdata/telegraf/agent.(*Agent).Run
	/go/src/github.com/influxdata/telegraf/agent/agent.go:148 +0x50a

goroutine 57 [semacquire, 6 minutes]:
sync.runtime_Semacquire(0xc00000fc68?)
	/usr/local/go/src/runtime/sema.go:56 +0x25
sync.(*WaitGroup).Wait(0xc00000fc50?)
	/usr/local/go/src/sync/waitgroup.go:136 +0x52
github.com/influxdata/telegraf/agent.(*Agent).runInputs(0xc000a041a8, {0x5da6bf0?, 0xc0010da9c0}, {0x0?, 0x0?, 0x8880b20?}, 0xc0004c83e0)
	/go/src/github.com/influxdata/telegraf/agent/agent.go:330 +0x4a6
github.com/influxdata/telegraf/agent.(*Agent).Run.func5()
	/go/src/github.com/influxdata/telegraf/agent/agent.go:178 +0x74
created by github.com/influxdata/telegraf/agent.(*Agent).Run
	/go/src/github.com/influxdata/telegraf/agent/agent.go:176 +0x855

goroutine 28 [select]:
github.com/influxdata/telegraf/agent.(*Agent).flushLoop(0xc0010e03c0?, {0x5da6bf0, 0xc0010daa00}, 0xc0004d4100, {0x5d82da0, 0xc0010e03c0})
	/go/src/github.com/influxdata/telegraf/agent/agent.go:818 +0x1d4
github.com/influxdata/telegraf/agent.(*Agent).runOutputs.func1(0x0?)
	/go/src/github.com/influxdata/telegraf/agent/agent.go:769 +0x125
created by github.com/influxdata/telegraf/agent.(*Agent).runOutputs
	/go/src/github.com/influxdata/telegraf/agent/agent.go:763 +0xc5

goroutine 29 [select]:
github.com/influxdata/telegraf/agent.(*RollingTicker).run(0xc0010e03c0, {0x5da6bf0, 0xc0010daa80}, 0xc000f0f220)
	/go/src/github.com/influxdata/telegraf/agent/tick.go:261 +0xe6
github.com/influxdata/telegraf/agent.(*RollingTicker).start.func1()
	/go/src/github.com/influxdata/telegraf/agent/tick.go:249 +0x6a
created by github.com/influxdata/telegraf/agent.(*RollingTicker).start
	/go/src/github.com/influxdata/telegraf/agent/tick.go:247 +0x15c

goroutine 58 [select]:
github.com/influxdata/telegraf/agent.(*AlignedTicker).run(0xc0005cff40, {0x5da6bf0, 0xc0005cff80}, 0xc0005c6f00)
	/go/src/github.com/influxdata/telegraf/agent/tick.go:86 +0xf8
github.com/influxdata/telegraf/agent.(*AlignedTicker).start.func1()
	/go/src/github.com/influxdata/telegraf/agent/tick.go:64 +0x6a
created by github.com/influxdata/telegraf/agent.(*AlignedTicker).start
	/go/src/github.com/influxdata/telegraf/agent/tick.go:62 +0x17c

goroutine 59 [select]:
github.com/influxdata/telegraf/agent.(*Agent).gatherLoop(0x0?, {0x5da6bf0, 0xc0010da9c0}, {0x5dbdc10, 0xc0004c8460}, 0xc0005c63c0, {0x5d82d78, 0xc0005cff40}, 0x0?)
	/go/src/github.com/influxdata/telegraf/agent/agent.go:465 +0x11c
github.com/influxdata/telegraf/agent.(*Agent).runInputs.func1(0x0?)
	/go/src/github.com/influxdata/telegraf/agent/agent.go:326 +0x7e
created by github.com/influxdata/telegraf/agent.(*Agent).runInputs
	/go/src/github.com/influxdata/telegraf/agent/agent.go:324 +0xa5

goroutine 60 [select]:
github.com/influxdata/telegraf/agent.(*AlignedTicker).run(0xc0005cffc0, {0x5da6bf0, 0xc001380000}, 0xc0005c7040)
	/go/src/github.com/influxdata/telegraf/agent/tick.go:86 +0xf8
github.com/influxdata/telegraf/agent.(*AlignedTicker).start.func1()
	/go/src/github.com/influxdata/telegraf/agent/tick.go:64 +0x6a
created by github.com/influxdata/telegraf/agent.(*AlignedTicker).start
	/go/src/github.com/influxdata/telegraf/agent/tick.go:62 +0x17c

goroutine 61 [select]:
github.com/influxdata/telegraf/agent.(*Agent).gatherLoop(0x0?, {0x5da6bf0, 0xc0010da9c0}, {0x5dbdc10, 0xc0004c84a0}, 0xc0005c6460, {0x5d82d78, 0xc0005cffc0}, 0x0?)
	/go/src/github.com/influxdata/telegraf/agent/agent.go:465 +0x11c
github.com/influxdata/telegraf/agent.(*Agent).runInputs.func1(0x0?)
	/go/src/github.com/influxdata/telegraf/agent/agent.go:326 +0x7e
created by github.com/influxdata/telegraf/agent.(*Agent).runInputs
	/go/src/github.com/influxdata/telegraf/agent/agent.go:324 +0xa5

goroutine 62 [select]:
github.com/influxdata/telegraf/agent.(*AlignedTicker).run(0xc001380040, {0x5da6bf0, 0xc001380080}, 0xc0005c7180)
	/go/src/github.com/influxdata/telegraf/agent/tick.go:86 +0xf8
github.com/influxdata/telegraf/agent.(*AlignedTicker).start.func1()
	/go/src/github.com/influxdata/telegraf/agent/tick.go:64 +0x6a
created by github.com/influxdata/telegraf/agent.(*AlignedTicker).start
	/go/src/github.com/influxdata/telegraf/agent/tick.go:62 +0x17c

goroutine 63 [select]:
github.com/influxdata/telegraf/agent.(*Agent).gatherLoop(0x0?, {0x5da6bf0, 0xc0010da9c0}, {0x5dbdc10, 0xc0004c8500}, 0xc0005c6500, {0x5d82d78, 0xc001380040}, 0x0?)
	/go/src/github.com/influxdata/telegraf/agent/agent.go:465 +0x11c
github.com/influxdata/telegraf/agent.(*Agent).runInputs.func1(0x0?)
	/go/src/github.com/influxdata/telegraf/agent/agent.go:326 +0x7e
created by github.com/influxdata/telegraf/agent.(*Agent).runInputs
	/go/src/github.com/influxdata/telegraf/agent/agent.go:324 +0xa5

goroutine 64 [select]:
github.com/influxdata/telegraf/agent.(*AlignedTicker).run(0xc0013800c0, {0x5da6bf0, 0xc001380100}, 0xc0005c72c0)
	/go/src/github.com/influxdata/telegraf/agent/tick.go:86 +0xf8
github.com/influxdata/telegraf/agent.(*AlignedTicker).start.func1()
	/go/src/github.com/influxdata/telegraf/agent/tick.go:64 +0x6a
created by github.com/influxdata/telegraf/agent.(*AlignedTicker).start
	/go/src/github.com/influxdata/telegraf/agent/tick.go:62 +0x17c

goroutine 65 [select]:
github.com/influxdata/telegraf/agent.(*Agent).gatherLoop(0x0?, {0x5da6bf0, 0xc0010da9c0}, {0x5dbdc10, 0xc0004c8520}, 0xc0005c65a0, {0x5d82d78, 0xc0013800c0}, 0x0?)
	/go/src/github.com/influxdata/telegraf/agent/agent.go:465 +0x11c
github.com/influxdata/telegraf/agent.(*Agent).runInputs.func1(0x0?)
	/go/src/github.com/influxdata/telegraf/agent/agent.go:326 +0x7e
created by github.com/influxdata/telegraf/agent.(*Agent).runInputs
	/go/src/github.com/influxdata/telegraf/agent/agent.go:324 +0xa5

goroutine 82 [select]:
github.com/influxdata/telegraf/agent.(*AlignedTicker).run(0xc001380140, {0x5da6bf0, 0xc001380180}, 0xc0005c7400)
	/go/src/github.com/influxdata/telegraf/agent/tick.go:86 +0xf8
github.com/influxdata/telegraf/agent.(*AlignedTicker).start.func1()
	/go/src/github.com/influxdata/telegraf/agent/tick.go:64 +0x6a
created by github.com/influxdata/telegraf/agent.(*AlignedTicker).start
	/go/src/github.com/influxdata/telegraf/agent/tick.go:62 +0x17c

goroutine 83 [runnable]:
github.com/influxdata/telegraf/agent.(*Agent).gatherOnce(0xc001504f18?, {0x5dbdc10?, 0xc0004c8560}, 0xc0005c6640, {0x5d82d78, 0xc001380140}, 0x0?)
	/go/src/github.com/influxdata/telegraf/agent/agent.go:497 +0x213
github.com/influxdata/telegraf/agent.(*Agent).gatherLoop(0x0?, {0x5da6bf0, 0xc0010da9c0}, {0x5dbdc10, 0xc0004c8560}, 0xc0005c6640, {0x5d82d78, 0xc001380140}, 0x0?)
	/go/src/github.com/influxdata/telegraf/agent/agent.go:467 +0x166
github.com/influxdata/telegraf/agent.(*Agent).runInputs.func1(0x0?)
	/go/src/github.com/influxdata/telegraf/agent/agent.go:326 +0x7e
created by github.com/influxdata/telegraf/agent.(*Agent).runInputs
	/go/src/github.com/influxdata/telegraf/agent/agent.go:324 +0xa5

goroutine 84 [select]:
github.com/influxdata/telegraf/agent.(*AlignedTicker).run(0xc001380200, {0x5da6bf0, 0xc001380240}, 0xc0005c7540)
	/go/src/github.com/influxdata/telegraf/agent/tick.go:86 +0xf8
github.com/influxdata/telegraf/agent.(*AlignedTicker).start.func1()
	/go/src/github.com/influxdata/telegraf/agent/tick.go:64 +0x6a
created by github.com/influxdata/telegraf/agent.(*AlignedTicker).start
	/go/src/github.com/influxdata/telegraf/agent/tick.go:62 +0x17c

goroutine 85 [select]:
github.com/influxdata/telegraf/agent.(*Agent).gatherOnce(0xc0000f1f18?, {0x5dbdc10?, 0xc0004c8580}, 0xc0005c6820, {0x5d82d78, 0xc001380200}, 0x0?)
	/go/src/github.com/influxdata/telegraf/agent/agent.go:497 +0x213
github.com/influxdata/telegraf/agent.(*Agent).gatherLoop(0x0?, {0x5da6bf0, 0xc0010da9c0}, {0x5dbdc10, 0xc0004c8580}, 0xc0005c6820, {0x5d82d78, 0xc001380200}, 0x0?)
	/go/src/github.com/influxdata/telegraf/agent/agent.go:467 +0x166
github.com/influxdata/telegraf/agent.(*Agent).runInputs.func1(0x0?)
	/go/src/github.com/influxdata/telegraf/agent/agent.go:326 +0x7e
created by github.com/influxdata/telegraf/agent.(*Agent).runInputs
	/go/src/github.com/influxdata/telegraf/agent/agent.go:324 +0xa5

goroutine 86 [select]:
github.com/influxdata/telegraf/agent.(*AlignedTicker).run(0xc001380280, {0x5da6bf0, 0xc0013802c0}, 0xc0005c7680)
	/go/src/github.com/influxdata/telegraf/agent/tick.go:86 +0xf8
github.com/influxdata/telegraf/agent.(*AlignedTicker).start.func1()
	/go/src/github.com/influxdata/telegraf/agent/tick.go:64 +0x6a
created by github.com/influxdata/telegraf/agent.(*AlignedTicker).start
	/go/src/github.com/influxdata/telegraf/agent/tick.go:62 +0x17c

goroutine 87 [runnable]:
github.com/influxdata/telegraf/agent.(*Agent).gatherOnce(0xc001031f18?, {0x5dbdc10?, 0xc0004c85c0}, 0xc0005c6960, {0x5d82d78, 0xc001380280}, 0x0?)
	/go/src/github.com/influxdata/telegraf/agent/agent.go:497 +0x213
github.com/influxdata/telegraf/agent.(*Agent).gatherLoop(0x0?, {0x5da6bf0, 0xc0010da9c0}, {0x5dbdc10, 0xc0004c85c0}, 0xc0005c6960, {0x5d82d78, 0xc001380280}, 0x0?)
	/go/src/github.com/influxdata/telegraf/agent/agent.go:467 +0x166
github.com/influxdata/telegraf/agent.(*Agent).runInputs.func1(0x0?)
	/go/src/github.com/influxdata/telegraf/agent/agent.go:326 +0x7e
created by github.com/influxdata/telegraf/agent.(*Agent).runInputs
	/go/src/github.com/influxdata/telegraf/agent/agent.go:324 +0xa5

goroutine 88 [select]:
github.com/influxdata/telegraf/agent.(*AlignedTicker).run(0xc001380300, {0x5da6bf0, 0xc001380340}, 0xc0005c77c0)
	/go/src/github.com/influxdata/telegraf/agent/tick.go:86 +0xf8
github.com/influxdata/telegraf/agent.(*AlignedTicker).start.func1()
	/go/src/github.com/influxdata/telegraf/agent/tick.go:64 +0x6a
created by github.com/influxdata/telegraf/agent.(*AlignedTicker).start
	/go/src/github.com/influxdata/telegraf/agent/tick.go:62 +0x17c

goroutine 89 [select]:
github.com/influxdata/telegraf/agent.(*Agent).gatherLoop(0x0?, {0x5da6bf0, 0xc0010da9c0}, {0x5dbdc10, 0xc0004c85e0}, 0xc0005c6b40, {0x5d82d78, 0xc001380300}, 0x0?)
	/go/src/github.com/influxdata/telegraf/agent/agent.go:465 +0x11c
github.com/influxdata/telegraf/agent.(*Agent).runInputs.func1(0x0?)
	/go/src/github.com/influxdata/telegraf/agent/agent.go:326 +0x7e
created by github.com/influxdata/telegraf/agent.(*Agent).runInputs
	/go/src/github.com/influxdata/telegraf/agent/agent.go:324 +0xa5

goroutine 90 [select]:
github.com/influxdata/telegraf/agent.(*AlignedTicker).run(0xc001380380, {0x5da6bf0, 0xc0013803c0}, 0xc0005c7900)
	/go/src/github.com/influxdata/telegraf/agent/tick.go:86 +0xf8
github.com/influxdata/telegraf/agent.(*AlignedTicker).start.func1()
	/go/src/github.com/influxdata/telegraf/agent/tick.go:64 +0x6a
created by github.com/influxdata/telegraf/agent.(*AlignedTicker).start
	/go/src/github.com/influxdata/telegraf/agent/tick.go:62 +0x17c

goroutine 91 [select]:
github.com/influxdata/telegraf/agent.(*Agent).gatherOnce(0xc0000eef18?, {0x5dbdc10?, 0xc0004c8620}, 0xc0005c6c80, {0x5d82d78, 0xc001380380}, 0x0?)
	/go/src/github.com/influxdata/telegraf/agent/agent.go:497 +0x213
github.com/influxdata/telegraf/agent.(*Agent).gatherLoop(0x0?, {0x5da6bf0, 0xc0010da9c0}, {0x5dbdc10, 0xc0004c8620}, 0xc0005c6c80, {0x5d82d78, 0xc001380380}, 0x0?)
	/go/src/github.com/influxdata/telegraf/agent/agent.go:467 +0x166
github.com/influxdata/telegraf/agent.(*Agent).runInputs.func1(0x0?)
	/go/src/github.com/influxdata/telegraf/agent/agent.go:326 +0x7e
created by github.com/influxdata/telegraf/agent.(*Agent).runInputs
	/go/src/github.com/influxdata/telegraf/agent/agent.go:324 +0xa5

goroutine 92 [select]:
github.com/influxdata/telegraf/agent.(*AlignedTicker).run(0xc001380400, {0x5da6bf0, 0xc001380440}, 0xc0005c7a40)
	/go/src/github.com/influxdata/telegraf/agent/tick.go:86 +0xf8
github.com/influxdata/telegraf/agent.(*AlignedTicker).start.func1()
	/go/src/github.com/influxdata/telegraf/agent/tick.go:64 +0x6a
created by github.com/influxdata/telegraf/agent.(*AlignedTicker).start
	/go/src/github.com/influxdata/telegraf/agent/tick.go:62 +0x17c

goroutine 93 [select]:
github.com/influxdata/telegraf/agent.(*Agent).gatherOnce(0xc0000ecf18?, {0x5dbdc10?, 0xc0004c8640}, 0xc0005c6dc0, {0x5d82d78, 0xc001380400}, 0x0?)
	/go/src/github.com/influxdata/telegraf/agent/agent.go:497 +0x213
github.com/influxdata/telegraf/agent.(*Agent).gatherLoop(0x0?, {0x5da6bf0, 0xc0010da9c0}, {0x5dbdc10, 0xc0004c8640}, 0xc0005c6dc0, {0x5d82d78, 0xc001380400}, 0x0?)
	/go/src/github.com/influxdata/telegraf/agent/agent.go:467 +0x166
github.com/influxdata/telegraf/agent.(*Agent).runInputs.func1(0x0?)
	/go/src/github.com/influxdata/telegraf/agent/agent.go:326 +0x7e
created by github.com/influxdata/telegraf/agent.(*Agent).runInputs
	/go/src/github.com/influxdata/telegraf/agent/agent.go:324 +0xa5

goroutine 770 [runnable]:
encoding/csv.(*Reader).readRecord(0xc00036f680, {0x0, 0xc001a550e0?, 0x0})
	/usr/local/go/src/encoding/csv/reader.go:361 +0xb9c
encoding/csv.(*Reader).ReadAll(0xc001434180?)
	/usr/local/go/src/encoding/csv/reader.go:225 +0x65
github.com/influxdata/telegraf/plugins/parsers/csv.parseCSV(0xc001434180, {0x5d59760?, 0xc001354930?})
	/go/src/github.com/influxdata/telegraf/plugins/parsers/csv/parser.go:253 +0x39c
github.com/influxdata/telegraf/plugins/parsers/csv.(*Parser).Parse(0xc001434180?, {0xc0013ac000, 0x321, 0x380})
	/go/src/github.com/influxdata/telegraf/plugins/parsers/csv/parser.go:168 +0x8e
github.com/influxdata/telegraf/models.(*RunningParser).Parse(0xc000930140, {0xc0013ac000, 0x321, 0x380})
	/go/src/github.com/influxdata/telegraf/models/running_parsers.go:73 +0x70
github.com/influxdata/telegraf/plugins/inputs/file.(*File).readMetric(0xc000070ba0, {0xc0013942c0, 0x36})
	/go/src/github.com/influxdata/telegraf/plugins/inputs/file/file.go:129 +0x3fa
github.com/influxdata/telegraf/plugins/inputs/file.(*File).Gather(0xc000070ba0, {0x5dbdc10, 0xc0004c8580})
	/go/src/github.com/influxdata/telegraf/plugins/inputs/file/file.go:76 +0xaa
github.com/influxdata/telegraf/models.(*RunningInput).Gather(0xc0005c6820, {0x5dbdc10, 0xc0004c8580})
	/go/src/github.com/influxdata/telegraf/models/running_input.go:118 +0x5a
github.com/influxdata/telegraf/agent.(*Agent).gatherOnce.func1()
	/go/src/github.com/influxdata/telegraf/agent/agent.go:487 +0x2e
created by github.com/influxdata/telegraf/agent.(*Agent).gatherOnce
	/go/src/github.com/influxdata/telegraf/agent/agent.go:486 +0x12a

goroutine 583 [running]:
	goroutine running on other thread; stack unavailable
created by github.com/influxdata/telegraf/agent.(*Agent).gatherOnce
	/go/src/github.com/influxdata/telegraf/agent/agent.go:486 +0x12a

goroutine 583 [running]:
runtime.throw({0x5334040?, 0x198?})
	/usr/local/go/src/runtime/panic.go:992 +0x71 fp=0xc00102b890 sp=0xc00102b860 pc=0x43a731
runtime.mapassign_faststr(0x5d95e28?, 0xc0005cf900?, {0xc0005f6198, 0x5})
	/usr/local/go/src/runtime/map_faststr.go:212 +0x39c fp=0xc00102b8f8 sp=0xc00102b890 pc=0x41569c
github.com/influxdata/telegraf/config.(*Config).missingTomlField(0xc000455320, {0x1?, 0x0?}, {0xc0005f6198, 0x5})
	/go/src/github.com/influxdata/telegraf/config/config.go:1860 +0xec9 fp=0xc00102b928 sp=0xc00102b8f8 pc=0xaee829
github.com/influxdata/telegraf/config.(*Config).missingTomlField-fm({0x5dcbe70?, 0x51e1740?}, {0xc0005f6198?, 0x5?})
	<autogenerated>:1 +0x45 fp=0xc00102b960 sp=0xc00102b928 pc=0xaf4345
github.com/influxdata/toml.fieldCache.findField({0xc0010e0690?, 0xc0010e06c0?}, 0xc00000e0a8, {0x51e1740?, 0xc0016fec00?, 0xc0005fc030?}, {0xc0005f6198, 0x5})
	/go/pkg/mod/github.com/influxdata/toml@v0.0.0-20190415235208-270119a8ce65/util.go:74 +0x3da fp=0xc00102ba50 sp=0xc00102b960 pc=0xadd67a
github.com/influxdata/toml.unmarshalTable(0x7f1ee0468d80?, {0x4f60c00?, 0xc0016fec00?, 0x40?}, 0xc0005c6c30, 0x0)
	/go/pkg/mod/github.com/influxdata/toml@v0.0.0-20190415235208-270119a8ce65/decode.go:157 +0x525 fp=0xc00102bc80 sp=0xc00102ba50 pc=0xacdd45
github.com/influxdata/toml.(*Config).UnmarshalTable(0x48e73e0?, 0x4?, {0x4f60c00?, 0xc0016fec00?})
	/go/pkg/mod/github.com/influxdata/toml@v0.0.0-20190415235208-270119a8ce65/decode.go:121 +0x1c5 fp=0xc00102bcf8 sp=0xc00102bc80 pc=0xacd3c5
github.com/influxdata/telegraf/config.(*Config).addParser(0xc000455320, {0xc0005f6160, 0x4}, 0xc0005c6c30)
	/go/src/github.com/influxdata/telegraf/config/config.go:1127 +0x216 fp=0xc00102bd98 sp=0xc00102bcf8 pc=0xae62b6
github.com/influxdata/telegraf/config.(*Config).addInput.func1()
	/go/src/github.com/influxdata/telegraf/config/config.go:1325 +0x29 fp=0xc00102bdd0 sp=0xc00102bd98 pc=0xae81e9
github.com/influxdata/telegraf/plugins/inputs/file.(*File).readMetric(0xc0000710e0, {0xc0013913c0, 0x3c})
	/go/src/github.com/influxdata/telegraf/plugins/inputs/file/file.go:125 +0x32f fp=0xc00102bec8 sp=0xc00102bdd0 pc=0x16792ef
github.com/influxdata/telegraf/plugins/inputs/file.(*File).Gather(0xc0000710e0, {0x5dbdc10, 0xc0004c8620})
	/go/src/github.com/influxdata/telegraf/plugins/inputs/file/file.go:76 +0xaa fp=0xc00102bf50 sp=0xc00102bec8 pc=0x1678b2a
github.com/influxdata/telegraf/models.(*RunningInput).Gather(0xc0005c6c80, {0x5dbdc10, 0xc0004c8620})
	/go/src/github.com/influxdata/telegraf/models/running_input.go:118 +0x5a fp=0xc00102bfa0 sp=0xc00102bf50 pc=0x7acbda
github.com/influxdata/telegraf/agent.(*Agent).gatherOnce.func1()
	/go/src/github.com/influxdata/telegraf/agent/agent.go:487 +0x2e fp=0xc00102bfe0 sp=0xc00102bfa0 pc=0xb589ae
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1571 +0x1 fp=0xc00102bfe8 sp=0xc00102bfe0 pc=0x46dec1
created by github.com/influxdata/telegraf/agent.(*Agent).gatherOnce
	/go/src/github.com/influxdata/telegraf/agent/agent.go:486 +0x12a

System info

Telegraf 1.22.3 Rhel 7.9

Docker

No response

Steps to reproduce

1.Load exact same config that was installed on 1.21.3
2. Start telegraf - within 20 minutes a crash occurs

Expected behavior

Telegraf should run without crashing

Actual behavior

After 5 - 20mins. Telegraf crashes and restarts

Additional info

No response

Metadata

Metadata

Assignees

Labels

area/configurationbugunexpected problem or unintended behaviorpanicissue that results in panics from Telegraf

Type

No type

Projects

No projects

Milestone

No milestone

Relationships

None yet

Development

No branches or pull requests

Issue actions