updated pipelines per comments

legoguy1000 · legoguy1000 · commit f02b12a7edf9 · 2021-06-29T12:20:09.000Z
diff --git a/x-pack/filebeat/module/threatintel/abuseurl/ingest/pipeline.yml b/x-pack/filebeat/module/threatintel/abuseurl/ingest/pipeline.yml
@@ -42,11 +42,11 @@ processors:
     field: threatintel.abuseurl.url
     target_field: threatintel.indicator.url
     keep_original: true
-- rename:
-    field: threatintel.abuseurl.url
-    target_field: threatintel.indicator.url.full
-    ignore_missing: true
-    if: ctx?.threatintel?.abuseurl?.url != null
+    remove_if_successful: true
+- set:
+    field: threatintel.indicator.url.full
+    copy_from: threatintel.indicator.url.original
+    ignore_empty_value: true
 - rename:
     field: threatintel.abuseurl.host
     target_field: threatintel.indicator.domain
diff --git a/x-pack/filebeat/module/threatintel/anomali/ingest/pipeline.yml b/x-pack/filebeat/module/threatintel/anomali/ingest/pipeline.yml
@@ -80,12 +80,12 @@ processors:
     field: _tmp.threatvalue
     target_field: threatintel.indicator.url
     keep_original: true
+    remove_if_successful: true
     if: ctx?.threatintel?.indicator?.type == 'url'
-- rename:
-    field: _tmp.threatvalue
-    target_field: threatintel.indicator.url.full
-    ignore_missing: true
-    if: ctx?.threatintel?.indicator?.type == 'url'
+- set:
+    field: threatintel.indicator.url.full
+    copy_from: threatintel.indicator.url.original
+    ignore_empty_value: true
 - rename:
     field: _tmp.threatvalue
     target_field: threatintel.indicator.email.address
diff --git a/x-pack/filebeat/module/threatintel/anomalithreatstream/ingest/pipeline.yml b/x-pack/filebeat/module/threatintel/anomalithreatstream/ingest/pipeline.yml
@@ -256,16 +256,17 @@ processors:
     field: json.url
     target_field: threatintel.indicator.url
     keep_original: true
+    remove_if_successful: true
     if: 'ctx.json.url != null'
     on_failure:
       - append:
           field: error.message
           value: 'Cannot parse url field `{{{ json.url }}}`: {{{ _ingest.on_failure_message }}}'
-- rename:
-    field: json.url
-    target_field: threatintel.indicator.url.full
-    ignore_missing: true
-    if: 'ctx.json.url != null'
+
+- set:
+    field: threatintel.indicator.url.full
+    copy_from: threatintel.indicator.url.original
+    ignore_empty_value: true
 
 - rename:
     field: json.country
diff --git a/x-pack/filebeat/module/threatintel/misp/ingest/pipeline.yml b/x-pack/filebeat/module/threatintel/misp/ingest/pipeline.yml
@@ -134,11 +134,13 @@ processors:
     field: threatintel.misp.attribute.value
     target_field: threatintel.indicator.url
     keep_original: true
+    remove_if_successful: true
     if: ctx?.threatintel?.indicator?.type == 'url' && ctx?.threatintel?.misp?.attribute?.type != 'uri'
-- rename:
-    field: threatintel.misp.attribute.value
-    target_field: threatintel.indicator.url.full
-    ignore_missing: true
+
+- set:
+    field: threatintel.indicator.url.full
+    copy_from: threatintel.indicator.url.original
+    ignore_empty_value: true
     if: "ctx?.threatintel?.indicator?.type == 'url' && ctx?.threatintel?.misp?.attribute?.type != 'uri'"
 
 ## Regkey indicator operations
diff --git a/x-pack/filebeat/module/threatintel/otx/ingest/pipeline.yml b/x-pack/filebeat/module/threatintel/otx/ingest/pipeline.yml
@@ -83,11 +83,12 @@ processors:
     field: threatintel.otx.indicator
     target_field: threatintel.indicator.url
     keep_original: true
+    remove_if_successful: true
     if: ctx?.threatintel?.indicator?.type == 'url'
-- rename:
-    field: threatintel.otx.indicator
-    target_field: threatintel.indicator.url.full
-    ignore_missing: true
+- set:
+    field: threatintel.indicator.url.full
+    copy_from: threatintel.indicator.url.original
+    ignore_empty_value: true
     if: "ctx?.threatintel?.otx?.type == 'URL'"
 
 ## Email indicator operations
