[system][syslog] add a pattern in filebeat.system module to capture greedy multiline logs with ISO timestamps

Author: stefans-elastic

packages/system/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.67.4"
+  changes:
+    - description: Add new grok pattern to system (syslog) module to capture multiline logs with ISO 8601 timestamps.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/1
 - version: "1.67.3"
   changes:
     - description: Remove dynamic template to avoid ECS overrides.

packages/system/data_stream/syslog/_dev/test/pipeline/test-iso8601_timestamp_multiline_syslog.log

@@ -0,0 +1,3 @@
+2020-12-25T23:59:59+02:00 a-mac-with-esc-key GoogleSoftwareUpdateAgent[21412]: 2016-12-13 11:35:28.421
+		GoogleSoftwareUpdateAgent[21412/0x700007399000] [lvl=2] -[KSUpdateEngine updateAllExceptProduct:] KSUpdateEngine updating all
+		installed products, except:'com.google.Keystone'.

packages/system/data_stream/syslog/_dev/test/pipeline/test-iso8601_timestamp_multiline_syslog.log-config.yml

@@ -0,0 +1,9 @@
+dynamic_fields:
+  "@timestamp": "^[0-9]{4}(-[0-9]{2}){2}T[0-9]{2}(:[0-9]{2}){2}\\.[0-9]{3}-[0-9]{2}:[0-9]{2}$"
+multiline:
+  first_line_pattern: "^\\w+ \\d+ "
+fields:
+  event:
+    kind: "event"
+    timezone: "GMT-0200"
+  input.type: log

packages/system/data_stream/syslog/_dev/test/pipeline/test-iso8601_timestamp_multiline_syslog.log-expected.json

@@ -0,0 +1,27 @@
+{
+  "expected": [
+    {
+      "ecs": {
+        "version": "8.11.0"
+      },
+      "event": {
+        "kind": "event",
+        "timezone": "GMT-0200"
+      },
+      "host": {
+        "hostname": "a-mac-with-esc-key"
+      },
+      "input": {
+        "type": "log"
+      },
+      "message": "2016-12-13 11:35:28.421\n\t\tGoogleSoftwareUpdateAgent[21412/0x700007399000] [lvl=2] -[KSUpdateEngine updateAllExceptProduct:] KSUpdateEngine updating all\n\t\tinstalled products, except:'com.google.Keystone'.",
+      "process": {
+        "name": "GoogleSoftwareUpdateAgent",
+        "pid": 21412
+      },
+      "system": {
+        "syslog": {}
+      }
+    }
+  ]
+}

packages/system/data_stream/syslog/elasticsearch/ingest_pipeline/default.yml

@@ -8,6 +8,7 @@ processors:
         - '%{SYSLOGTIMESTAMP:system.syslog.timestamp} %{SYSLOGHOST:host.hostname} %{DATA:process.name}(?:\[%{POSINT:process.pid:long}\])?: %{GREEDYMULTILINE:system.syslog.message}'
         - '%{SYSLOGTIMESTAMP:system.syslog.timestamp} %{GREEDYMULTILINE:system.syslog.message}'
         - '%{TIMESTAMP_ISO8601:system.syslog.timestamp} %{SYSLOGHOST:host.hostname} %{DATA:process.name}(?:\[%{POSINT:process.pid:long}\])?: %{GREEDYMULTILINE:system.syslog.message}'
+        - '%{TIMESTAMP_ISO8601:system.syslog.timestamp} %{GREEDYMULTILINE:system.syslog.message}'
       pattern_definitions:
         GREEDYMULTILINE: |-
           (.|