Relevant telegraf.conf
[agent]
collection_jitter = "0s"
debug = true
flush_interval = "10s"
flush_jitter = "0s"
hostname = "$HOSTNAME"
interval = "20s"
logfile = ""
metric_batch_size = 1000
metric_buffer_limit = 100000
omit_hostname = false
precision = "1ms"
quiet = false
round_interval = true
[[processors.enum]]
[[processors.enum.mapping]]
dest = "status_code"
field = "status"
[processors.enum.mapping.value_mappings]
critical = 3
healthy = 1
problem = 2
[[outputs.file]]
files = ["temp.carbon"]
data_format="carbon2"
[[inputs.kafka_consumer]]
brokers = [
"xxx"
]
client_id = "local-telegraf-test2"
consumer_group = "local-telegraf-test2"
data_format = "json_v2"
enable_tls = true
insecure_skip_verify = true
max_message_len = 1000000
sasl_mechanism = "PLAIN"
sasl_password = ""
sasl_username = ""
topics = [
"test"
]
topic_tag = "topic"
max_processing_time = "1000ms"
offset = "oldest"
[[inputs.kafka_consumer.json_v2]]
measurement_name = "weight"
timestamp_format = "unix_ms"
timestamp_path = "weight_createdAt"
[[inputs.kafka_consumer.json_v2.field]]
path = "weight_weight"
rename = "weight"
type = "float"
[[inputs.kafka_consumer.json_v2.tag]]
path = "weight_serialNumber"
rename = "serial_number"
[[inputs.kafka_consumer.json_v2.tag]]
path = "weight_imei"
rename = "imei"
[[inputs.kafka_consumer.json_v2.tag]]
path = "sensor_customer_name"
rename = "customer_name"
[[inputs.kafka_consumer.json_v2.tag]]
path = "sensor_distributor_name"
rename = "distributor_name"
[[inputs.kafka_consumer.json_v2.tag]]
path = "sensor_dist_name"Logs from Telegraf
No relevant logs
System info
Telegraf 1.21.3 tested on MacOs and Kubernetes(Docker image)
Docker
No response
Steps to reproduce
- Add data to kafka topic (seems large volume creates more "wrong" data)
...
This is tested with data from kafka to both InfluxDB Cloud and Carbon output file. Both outputs have the same error, but different results.
Expected behavior
Expect carbon output to mirror kafka topic
Actual behavior
$ cat temp.carbon| grep "164482" | grep "0004G"
metric=weight field=weight customer_name=Shakey's distributor_name=xx host=$HOSTNAME imei=356611075967160 sensor_dist_name=null serial_number=0004G topic=telegraf-test2 64.4540023803711 1644824511
metric=weight field=weight customer_name=Shakey's distributor_name=xx host=$HOSTNAME imei=356611075967160 sensor_dist_name=null serial_number=0004G topic=telegraf-test2 52.34299850463867 1644825774
metric=weight field=weight customer_name=Shakey's distributor_name=xx host=$HOSTNAME imei=356611075967160 sensor_dist_name=null serial_number=0004G topic=telegraf-test2 69.56500244140625 1644820026
metric=weight field=weight customer_name=Shakey's distributor_name=xx host=$HOSTNAME imei=356611075967160 sensor_dist_name=null serial_number=0004G topic=telegraf-test2 69.45500183105469 1644820627
metric=weight field=weight customer_name=Shakey's distributor_name=xxhost=$HOSTNAME imei=356611075967160 sensor_dist_name=null serial_number=0004G topic=telegraf-test2 69.24700164794922 1644821228
metric=weight field=weight customer_name=Shakey's distributor_name=xx host=$HOSTNAME imei=356611075967160 sensor_dist_name=null serial_number=0004G topic=telegraf-test2 69.16100311279297 1644821829
metric=weight field=weight customer_name=Shakey's distributor_name=xx host=$HOSTNAME imei=356611075967160 sensor_dist_name=null serial_number=0004G topic=telegraf-test2 68.97699737548828 1644822430
metric=weight field=weight customer_name=Shakey's distributor_name=xx host=$HOSTNAME imei=356611075967160 sensor_dist_name=null serial_number=0004G topic=telegraf-test2 68.78099822998047 1644823031
metric=weight field=weight customer_name=Shakey's distributor_name=xx host=$HOSTNAME imei=356611075967160 sensor_dist_name=null serial_number=0004G topic=telegraf-test2 68.65799713134766 1644823632
metric=weight field=weight customer_name=Shakey's distributor_name=xx host=$HOSTNAME imei=356611075967160 sensor_dist_name=null serial_number=0004G topic=telegraf-test2 68.55999755859375 1644824233
metric=weight field=weight customer_name=Shakey's distributor_name=xx host=$HOSTNAME imei=356611075967160 sensor_dist_name=null serial_number=0004G topic=telegraf-test2 68.44999694824219 1644824834
metric=weight field=weight customer_name=Shakey's distributor_name=xx host=$HOSTNAME imei=356611075967160 sensor_dist_name=null serial_number=0004G topic=telegraf-test2 68.3759994506836 1644825435
metric=weight field=weight customer_name=Shakey's distributor_name=xx host=$HOSTNAME imei=356611075967160 sensor_dist_name=null serial_number=0004G topic=telegraf-test2 68.25399780273438 1644826036
ksql> select FROM_UNIXTIME(weight_createdat), weight_serialnumber, weight_weight, weight_createdat from telegraf_test where weight_serialnumber = '0004G' and weight_createdat > 1644820000000 emit changes;
+---------------------------+---------------------------+---------------------------+---------------------------+
|KSQL_COL_0 |WEIGHT_SERIALNUMBER |WEIGHT_WEIGHT |WEIGHT_CREATEDAT |
+---------------------------+---------------------------+---------------------------+---------------------------+
|2022-02-14T06:27:06.857 |0004G |69.56500244140625 |1644820026857 |
|2022-02-14T06:37:07.857 |0004G |69.45500183105469 |1644820627857 |
|2022-02-14T06:47:08.857 |0004G |69.24700164794922 |1644821228857 |
|2022-02-14T06:57:09.857 |0004G |69.16100311279297 |1644821829857 |
|2022-02-14T07:07:10.857 |0004G |68.97699737548828 |1644822430857 |
|2022-02-14T07:17:11.857 |0004G |68.78099822998047 |1644823031857 |
|2022-02-14T07:27:12.857 |0004G |68.65799713134766 |1644823632857 |
|2022-02-14T07:37:13.857 |0004G |68.55999755859375 |1644824233857 |
|2022-02-14T07:47:14.857 |0004G |68.44999694824219 |1644824834857 |
|2022-02-14T07:57:15.857 |0004G |68.3759994506836 |1644825435857 |
|2022-02-14T08:07:16.857 |0004G |68.25399780273438 |1644826036857 |
Additional info
No response
Relevant telegraf.conf
Logs from Telegraf
No relevant logs
System info
Telegraf 1.21.3 tested on MacOs and Kubernetes(Docker image)
Docker
No response
Steps to reproduce
...
This is tested with data from kafka to both InfluxDB Cloud and Carbon output file. Both outputs have the same error, but different results.
Expected behavior
Expect carbon output to mirror kafka topic
Actual behavior
Additional info
No response