azure: Add event.kind pipeline_error to on_failure handlers

andrewkroh · andrewkroh · commit 46a32cc152f1 · 2026-01-30T11:32:25.000-05:00
Update ingest pipeline on_failure handlers to set event.kind to pipeline_error per best practices. This change updates the 23 pipelines that were missing this processor. The graphactivitylogs and signinlogs default.yml pipelines already had it and were not modified. The error.message format was also updated to follow the guidance from https://github.com/elastic/integrations/wiki/Fleet-Package-Code-Review-Comments#pipeline-on_failure-handler-must-set-errormessage
diff --git a/packages/azure/changelog.yml b/packages/azure/changelog.yml
@@ -1,3 +1,8 @@
+- version: "1.35.2"
+  changes:
+    - description: Add missing event.kind pipeline_error handling to ingest pipeline on_failure handlers.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/17176
 - version: "1.35.1"
   changes:
     - description: Fix pipeline error when the `category` field is missing.
diff --git a/packages/azure/data_stream/activitylogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml b/packages/azure/data_stream/activitylogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml
@@ -87,5 +87,11 @@ processors:
       ignore_missing: true
 on_failure:
   - set:
+      field: event.kind
+      value: pipeline_error
+  - append:
       field: error.message
-      value: '{{ _ingest.on_failure_message }}'
+      value: >-
+        Processor '{{{ _ingest.on_failure_processor_type }}}'
+        {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+        {{/_ingest.on_failure_processor_tag}}failed with message '{{{ _ingest.on_failure_message }}}'
diff --git a/packages/azure/data_stream/activitylogs/elasticsearch/ingest_pipeline/default.yml b/packages/azure/data_stream/activitylogs/elasticsearch/ingest_pipeline/default.yml
@@ -350,7 +350,12 @@ processors:
       name: '{{ IngestPipeline "azure-shared-pipeline" }}'
 on_failure:
   - set:
+      field: event.kind
+      value: pipeline_error
+  - append:
       field: error.message
-      value: |-
-        Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}"
+      value: >-
+        Processor '{{{ _ingest.on_failure_processor_type }}}'
+        {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+        {{/_ingest.on_failure_processor_tag}}failed with message '{{{ _ingest.on_failure_message }}}'
 
diff --git a/packages/azure/data_stream/application_gateway/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml b/packages/azure/data_stream/application_gateway/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml
@@ -84,5 +84,11 @@ processors:
       ignore_empty_value: true
 on_failure:
   - set:
+      field: event.kind
+      value: pipeline_error
+  - append:
       field: error.message
-      value: '{{ _ingest.on_failure_message }}'
+      value: >-
+        Processor '{{{ _ingest.on_failure_processor_type }}}'
+        {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+        {{/_ingest.on_failure_processor_tag}}failed with message '{{{ _ingest.on_failure_message }}}'
diff --git a/packages/azure/data_stream/application_gateway/elasticsearch/ingest_pipeline/default.yml b/packages/azure/data_stream/application_gateway/elasticsearch/ingest_pipeline/default.yml
@@ -327,9 +327,15 @@ processors:
         }
         dropEmptyFields(ctx);
 on_failure:
+  - set:
+      field: event.kind
+      value: pipeline_error
   - append:
-      field: "error.message"
-      value: "{{ _ingest.on_failure_message }} {{ _ingest.on_failure_processor_type }}"
+      field: error.message
+      value: >-
+        Processor '{{{ _ingest.on_failure_processor_type }}}'
+        {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+        {{/_ingest.on_failure_processor_tag}}failed with message '{{{ _ingest.on_failure_message }}}'
   - remove:
       field:
         - json
diff --git a/packages/azure/data_stream/auditlogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml b/packages/azure/data_stream/auditlogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml
@@ -87,5 +87,11 @@ processors:
       ignore_missing: true
 on_failure:
   - set:
+      field: event.kind
+      value: pipeline_error
+  - append:
       field: error.message
-      value: '{{ _ingest.on_failure_message }}'
+      value: >-
+        Processor '{{{ _ingest.on_failure_processor_type }}}'
+        {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+        {{/_ingest.on_failure_processor_tag}}failed with message '{{{ _ingest.on_failure_message }}}'
diff --git a/packages/azure/data_stream/auditlogs/elasticsearch/ingest_pipeline/default.yml b/packages/azure/data_stream/auditlogs/elasticsearch/ingest_pipeline/default.yml
@@ -291,5 +291,11 @@ processors:
       name: '{{ IngestPipeline "azure-shared-pipeline" }}'
 on_failure:
   - set:
+      field: event.kind
+      value: pipeline_error
+  - append:
       field: error.message
-      value: '{{ _ingest.on_failure_message }}'
+      value: >-
+        Processor '{{{ _ingest.on_failure_processor_type }}}'
+        {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+        {{/_ingest.on_failure_processor_tag}}failed with message '{{{ _ingest.on_failure_message }}}'
diff --git a/packages/azure/data_stream/eventhub/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml b/packages/azure/data_stream/eventhub/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml
@@ -87,5 +87,11 @@ processors:
       ignore_missing: true
 on_failure:
   - set:
+      field: event.kind
+      value: pipeline_error
+  - append:
       field: error.message
-      value: '{{ _ingest.on_failure_message }}'
+      value: >-
+        Processor '{{{ _ingest.on_failure_processor_type }}}'
+        {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+        {{/_ingest.on_failure_processor_tag}}failed with message '{{{ _ingest.on_failure_message }}}'
diff --git a/packages/azure/data_stream/eventhub/elasticsearch/ingest_pipeline/default.yml b/packages/azure/data_stream/eventhub/elasticsearch/ingest_pipeline/default.yml
@@ -16,5 +16,11 @@ processors:
       name: '{{ IngestPipeline "parsed-message" }}'
 on_failure:
   - set:
+      field: event.kind
+      value: pipeline_error
+  - append:
       field: error.message
-      value: '{{ _ingest.on_failure_message }}'
+      value: >-
+        Processor '{{{ _ingest.on_failure_processor_type }}}'
+        {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+        {{/_ingest.on_failure_processor_tag}}failed with message '{{{ _ingest.on_failure_message }}}'
diff --git a/packages/azure/data_stream/eventhub/elasticsearch/ingest_pipeline/parsed-message.yml b/packages/azure/data_stream/eventhub/elasticsearch/ingest_pipeline/parsed-message.yml
@@ -76,5 +76,11 @@ processors:
       ignore_failure: true
 on_failure:
   - set:
+      field: event.kind
+      value: pipeline_error
+  - append:
       field: error.message
-      value: '{{ _ingest.on_failure_message }}'
+      value: >-
+        Processor '{{{ _ingest.on_failure_processor_type }}}'
+        {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+        {{/_ingest.on_failure_processor_tag}}failed with message '{{{ _ingest.on_failure_message }}}'
diff --git a/packages/azure/data_stream/events/elasticsearch/ingest_pipeline/default.yml b/packages/azure/data_stream/events/elasticsearch/ingest_pipeline/default.yml
@@ -130,5 +130,11 @@ processors:
 
 on_failure:
   - set:
+      field: event.kind
+      value: pipeline_error
+  - append:
       field: error.message
-      value: "{{ _ingest.on_failure_message }}"
+      value: >-
+        Processor '{{{ _ingest.on_failure_processor_type }}}'
+        {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+        {{/_ingest.on_failure_processor_tag}}failed with message '{{{ _ingest.on_failure_message }}}'
diff --git a/packages/azure/data_stream/firewall_logs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml b/packages/azure/data_stream/firewall_logs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml
@@ -84,5 +84,11 @@ processors:
       ignore_empty_value: true
 on_failure:
   - set:
+      field: event.kind
+      value: pipeline_error
+  - append:
       field: error.message
-      value: '{{ _ingest.on_failure_message }}'
+      value: >-
+        Processor '{{{ _ingest.on_failure_processor_type }}}'
+        {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+        {{/_ingest.on_failure_processor_tag}}failed with message '{{{ _ingest.on_failure_message }}}'
diff --git a/packages/azure/data_stream/firewall_logs/elasticsearch/ingest_pipeline/default.yml b/packages/azure/data_stream/firewall_logs/elasticsearch/ingest_pipeline/default.yml
@@ -458,12 +458,12 @@ processors:
         dropEmptyFields(ctx);
 
 on_failure:
+  - set:
+      field: event.kind
+      value: pipeline_error
   - append:
-      field: "error.message"
-      value: "{{ _ingest.on_failure_message }} {{ _ingest.on_failure_processor_type }}"
-  - remove:
-      field:
-        - json
-        - _conf
-        - message
-      ignore_missing: true
+      field: error.message
+      value: >-
+        Processor '{{{ _ingest.on_failure_processor_type }}}'
+        {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+        {{/_ingest.on_failure_processor_tag}}failed with message '{{{ _ingest.on_failure_message }}}'
diff --git a/packages/azure/data_stream/graphactivitylogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml b/packages/azure/data_stream/graphactivitylogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml
@@ -87,5 +87,11 @@ processors:
       ignore_missing: true
 on_failure:
   - set:
+      field: event.kind
+      value: pipeline_error
+  - append:
       field: error.message
-      value: '{{ _ingest.on_failure_message }}'
+      value: >-
+        Processor '{{{ _ingest.on_failure_processor_type }}}'
+        {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+        {{/_ingest.on_failure_processor_tag}}failed with message '{{{ _ingest.on_failure_message }}}'
diff --git a/packages/azure/data_stream/identity_protection/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml b/packages/azure/data_stream/identity_protection/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml
@@ -22,5 +22,11 @@ processors:
       ignore_missing: true
 on_failure:
   - set:
+      field: event.kind
+      value: pipeline_error
+  - append:
       field: error.message
-      value: '{{ _ingest.on_failure_message }}'
+      value: >-
+        Processor '{{{ _ingest.on_failure_processor_type }}}'
+        {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+        {{/_ingest.on_failure_processor_tag}}failed with message '{{{ _ingest.on_failure_message }}}'
diff --git a/packages/azure/data_stream/identity_protection/elasticsearch/ingest_pipeline/default.yml b/packages/azure/data_stream/identity_protection/elasticsearch/ingest_pipeline/default.yml
@@ -378,6 +378,11 @@ processors:
       name: '{{ IngestPipeline "azure-shared-pipeline" }}'
 on_failure:
   - set:
+      field: event.kind
+      value: pipeline_error
+  - append:
       field: error.message
-      value: |-
-        Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}"
+      value: >-
+        Processor '{{{ _ingest.on_failure_processor_type }}}'
+        {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+        {{/_ingest.on_failure_processor_tag}}failed with message '{{{ _ingest.on_failure_message }}}'
diff --git a/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml b/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml
@@ -81,5 +81,11 @@ processors:
       ignore_missing: true
 on_failure:
   - set:
+      field: event.kind
+      value: pipeline_error
+  - append:
       field: error.message
-      value: '{{ _ingest.on_failure_message }}'
+      value: >-
+        Processor '{{{ _ingest.on_failure_processor_type }}}'
+        {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+        {{/_ingest.on_failure_processor_tag}}failed with message '{{{ _ingest.on_failure_message }}}'
diff --git a/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/default.yml b/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/default.yml
@@ -401,7 +401,12 @@ processors:
       target_field: azure.platformlogs.result_description
       ignore_missing: true
 on_failure:
-- set:
-    field: error.message
-    value: |-
-      Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}"
+  - set:
+      field: event.kind
+      value: pipeline_error
+  - append:
+      field: error.message
+      value: >-
+        Processor '{{{ _ingest.on_failure_processor_type }}}'
+        {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+        {{/_ingest.on_failure_processor_tag}}failed with message '{{{ _ingest.on_failure_message }}}'
diff --git a/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/springcloudlogs-inner-pipeline.yml b/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/springcloudlogs-inner-pipeline.yml
@@ -82,5 +82,11 @@ processors:
       ignore_missing: true
 on_failure:
   - set:
+      field: event.kind
+      value: pipeline_error
+  - append:
       field: error.message
-      value: '{{ _ingest.on_failure_message }}'
+      value: >-
+        Processor '{{{ _ingest.on_failure_processor_type }}}'
+        {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+        {{/_ingest.on_failure_processor_tag}}failed with message '{{{ _ingest.on_failure_message }}}'
diff --git a/packages/azure/data_stream/provisioning/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml b/packages/azure/data_stream/provisioning/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml
@@ -22,5 +22,11 @@ processors:
       ignore_missing: true
 on_failure:
   - set:
+      field: event.kind
+      value: pipeline_error
+  - append:
       field: error.message
-      value: '{{ _ingest.on_failure_message }}'
+      value: >-
+        Processor '{{{ _ingest.on_failure_processor_type }}}'
+        {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+        {{/_ingest.on_failure_processor_tag}}failed with message '{{{ _ingest.on_failure_message }}}'
diff --git a/packages/azure/data_stream/provisioning/elasticsearch/ingest_pipeline/default.yml b/packages/azure/data_stream/provisioning/elasticsearch/ingest_pipeline/default.yml
@@ -382,6 +382,11 @@ processors:
       name: '{{ IngestPipeline "azure-shared-pipeline" }}'
 on_failure:
   - set:
+      field: event.kind
+      value: pipeline_error
+  - append:
       field: error.message
-      value: |-
-        Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}"
+      value: >-
+        Processor '{{{ _ingest.on_failure_processor_type }}}'
+        {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+        {{/_ingest.on_failure_processor_tag}}failed with message '{{{ _ingest.on_failure_message }}}'
diff --git a/packages/azure/data_stream/signinlogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml b/packages/azure/data_stream/signinlogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml
@@ -87,5 +87,11 @@ processors:
       ignore_missing: true
 on_failure:
   - set:
+      field: event.kind
+      value: pipeline_error
+  - append:
       field: error.message
-      value: '{{ _ingest.on_failure_message }}'
+      value: >-
+        Processor '{{{ _ingest.on_failure_processor_type }}}'
+        {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+        {{/_ingest.on_failure_processor_tag}}failed with message '{{{ _ingest.on_failure_message }}}'
diff --git a/packages/azure/data_stream/springcloudlogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml b/packages/azure/data_stream/springcloudlogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml
@@ -87,5 +87,11 @@ processors:
       ignore_missing: true
 on_failure:
   - set:
+      field: event.kind
+      value: pipeline_error
+  - append:
       field: error.message
-      value: '{{ _ingest.on_failure_message }}'
+      value: >-
+        Processor '{{{ _ingest.on_failure_processor_type }}}'
+        {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+        {{/_ingest.on_failure_processor_tag}}failed with message '{{{ _ingest.on_failure_message }}}'
diff --git a/packages/azure/data_stream/springcloudlogs/elasticsearch/ingest_pipeline/default.yml b/packages/azure/data_stream/springcloudlogs/elasticsearch/ingest_pipeline/default.yml
@@ -324,6 +324,11 @@ processors:
       name: '{{ IngestPipeline "azure-shared-pipeline" }}'
 on_failure:
   - set:
+      field: event.kind
+      value: pipeline_error
+  - append:
       field: error.message
-      value: |-
-        Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}"
+      value: >-
+        Processor '{{{ _ingest.on_failure_processor_type }}}'
+        {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+        {{/_ingest.on_failure_processor_tag}}failed with message '{{{ _ingest.on_failure_message }}}'
