{
  "filebeat-7.15.0-apache-error-pipeline" : {
    "description" : "Pipeline for parsing apache error logs",
    "processors" : [
      {
        "set" : {
          "value" : "{{_ingest.timestamp}}",
          "field" : "event.ingested"
        }
      },
      {
        "rename" : {
          "field" : "message",
          "target_field" : "event.original"
        }
      },
      {
        "grok" : {
          "field" : "event.original",
          "patterns" : [
            """\[%{APACHE_TIME:apache.error.timestamp}\] \[%{APACHE_LOG_LEVEL:log.level}\]( \[client %{IPORHOST:source.address}(:%{POSINT:source.port})?\])? %{GREEDYDATA:message}""",
            """\[%{APACHE_TIME:apache.error.timestamp}\] \[%{DATA:apache.error.module}:%{APACHE_LOG_LEVEL:log.level}\] \[pid %{NUMBER:process.pid:long}(:tid %{NUMBER:process.thread.id:long})?\]( \[client %{IPORHOST:source.address}(:%{POSINT:source.port})?\])? %{GREEDYDATA:message}"""
          ],
          "pattern_definitions" : {
            "APACHE_TIME" : "%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}",
            "APACHE_LOG_LEVEL" : "(emerg|alert|crit|error|warn|notice|info|debug|trace[1-8])"
          },
          "ignore_missing" : true
        }
      },
      {
        "grok" : {
          "ignore_failure" : true,
          "field" : "message",
          "patterns" : [
            "File does not exist: %{URIPATH:file.path}, referer: %{URI:http.request.referrer}",
            "File does not exist: %{URIPATH:file.path}"
          ],
          "ignore_missing" : true
        }
      },
      {
        "date" : {
          "formats" : [
            "EEE MMM dd H:m:s yyyy",
            "EEE MMM dd H:m:s.SSSSSS yyyy"
          ],
          "if" : "ctx.event.timezone == null",
          "field" : "apache.error.timestamp",
          "target_field" : "@timestamp",
          "on_failure" : [
            {
              "append" : {
                "field" : "error.message",
                "value" : "{{ _ingest.on_failure_message }}"
              }
            }
          ]
        }
      },
      {
        "date" : {
          "field" : "apache.error.timestamp",
          "target_field" : "@timestamp",
          "formats" : [
            "EEE MMM dd H:m:s yyyy",
            "EEE MMM dd H:m:s.SSSSSS yyyy"
          ],
          "timezone" : "{{ event.timezone }}",
          "if" : "ctx.event.timezone != null",
          "on_failure" : [
            {
              "append" : {
                "field" : "error.message",
                "value" : "{{ _ingest.on_failure_message }}"
              }
            }
          ]
        }
      },
      {
        "remove" : {
          "ignore_failure" : true,
          "field" : [
            "apache.error.timestamp",
            "_tmp.url_orig"
          ]
        }
      },
      {
        "set" : {
          "field" : "event.kind",
          "value" : "event"
        }
      },
      {
        "set" : {
          "field" : "event.category",
          "value" : "web"
        }
      },
      {
        "script" : {
          "if" : "ctx?.log?.level != null",
          "lang" : "painless",
          "source" : """def err_levels = ["emerg", "alert", "crit", "error", "warn"]; if (err_levels.contains(ctx.log.level)) {
  ctx.event.type = "error";
} else {
  ctx.event.type = "info";
}"""
        }
      },
      {
        "grok" : {
          "ignore_missing" : true,
          "patterns" : [
            "^(%{IP:source.ip}|%{HOSTNAME:source.domain})$"
          ],
          "field" : "source.address"
        }
      },
      {
        "geoip" : {
          "field" : "source.ip",
          "target_field" : "source.geo",
          "ignore_missing" : true
        }
      },
      {
        "geoip" : {
          "target_field" : "source.as",
          "properties" : [
            "asn",
            "organization_name"
          ],
          "ignore_missing" : true,
          "database_file" : "GeoLite2-ASN.mmdb",
          "field" : "source.ip"
        }
      },
      {
        "rename" : {
          "field" : "source.as.asn",
          "target_field" : "source.as.number",
          "ignore_missing" : true
        }
      },
      {
        "rename" : {
          "target_field" : "source.as.organization.name",
          "ignore_missing" : true,
          "field" : "source.as.organization_name"
        }
      },
      {
        "script" : {
          "description" : "This script processor iterates over the whole document to remove fields with null values.",
          "source" : """void handleMap(Map map) {
  for (def x : map.values()) {
    if (x instanceof Map) {
        handleMap(x);
    } else if (x instanceof List) {
        handleList(x);
    }
  }
  map.values().removeIf(v -> v == null);
}
void handleList(List list) {
  for (def x : list) {
      if (x instanceof Map) {
          handleMap(x);
      } else if (x instanceof List) {
          handleList(x);
      }
  }
}
handleMap(ctx);
""",
          "lang" : "painless"
        }
      }
    ],
    "on_failure" : [
      {
        "set" : {
          "field" : "error.message",
          "value" : "{{ _ingest.on_failure_message }}"
        }
      }
    ]
  }
}