[Cloudflare] Cloudflare audit logs (#2294)

* Add Cloudflare audit logs

* update pipeline, agent

* update changelog

* version typo

* fix tests

Author: legoguy1000

packages/cloudflare/_dev/build/docs/README.md

@@ -1,9 +1,18 @@
 # Cloudflare Integration
 
-The Cloudflare integration collects events from the Cloudflare API, specifically reading from the Cloudflare Logpull API.
+The Cloudflare integration collects events from the Cloudflare API.
 
 ## Logs
 
+### Audit
+
+The Cloudflare Audit records all events related to your Cloudflare account. 
+To use this integration, you must have the `Account.Access: Audit Logs: Read` permission and you must use your email and your Global API Key (not an API Token).
+
+{{fields "audit"}}
+
+{{event "audit"}}
+
 ### Logpull
 
 The Cloudflare Logpull records network events related to your organization in order to provide an audit trail that can be used to understand platform activity and to diagnose problems. This module is implemented using the httpjson input.

packages/cloudflare/_dev/deploy/docker/files/config.yml

@@ -14,3 +14,43 @@ rules:
           {"CacheCacheStatus":"unknown","CacheResponseBytes":0,"CacheResponseStatus":0,"CacheTieredFill":false,"ClientASN":15169,"ClientCountry":"us","ClientDeviceType":"desktop","ClientIP":"35.232.161.245","ClientIPClass":"noRecord","ClientRequestBytes":2577,"ClientRequestHost":"cf-analytics.com","ClientRequestMethod":"POST","ClientRequestPath":"/wp-cron.php","ClientRequestProtocol":"HTTP/1.1","ClientRequestReferer":"https://cf-analytics.com/wp-cron.php?doing_wp_cron=1564759748.3962020874023437500000","ClientRequestURI":"/wp-cron.php?doing_wp_cron=1564759748.3962020874023437500000","ClientRequestUserAgent":"WordPress/5.2.2;https://cf-analytics.com","ClientSSLCipher":"ECDHE-ECDSA-AES128-GCM-SHA256","ClientSSLProtocol":"TLSv1.2","ClientSrcPort":55028,"EdgeColoID":14,"EdgeEndTimestamp":"2019-08-02T15:29:08Z","EdgePathingOp":"chl","EdgePathingSrc":"filterBasedFirewall","EdgePathingStatus":"captchaNew","EdgeRateLimitAction":"","EdgeRateLimitID":0,"EdgeRequestHost":"","EdgeResponseBytes":2848,"EdgeResponseCompressionRatio":2.64,"EdgeResponseContentType":"text/html","EdgeResponseStatus":403,"EdgeServerIP":"","EdgeStartTimestamp":"2019-08-02T15:29:08Z","FirewallMatchesActions":["simulate","challenge"],"FirewallMatchesSources":["firewallRules","firewallRules"],"FirewallMatchesRuleIDs":["094b71fea25d4860a61fa0c6fbbd8d8b","e454fd4a0ce546b3a9a462536613692c"],"OriginIP":"","OriginResponseBytes":0,"OriginResponseHTTPExpires":"","OriginResponseHTTPLastModified":"","OriginResponseStatus":0,"OriginResponseTime":0,"OriginSSLProtocol":"unknown","ParentRayID":"00","RayID":"500115ec386354d8","SecurityLevel":"med","WAFAction":"unknown","WAFFlags":"0","WAFMatchedVar":"","WAFProfile":"unknown","WAFRuleID":"","WAFRuleMessage":"","WorkerCPUTime":0,"WorkerStatus":"unknown","WorkerSubrequest":false,"WorkerSubrequestCount":0,"ZoneID":155978002}
           {"CacheCacheStatus":"hit","CacheResponseBytes":26888,"CacheResponseStatus":200,"CacheTieredFill":true,"ClientASN":1136,"ClientCountry":"nl","ClientDeviceType":"desktop","ClientIP":"222.97.65.242","ClientIPClass":"noRecord","ClientRequestBytes":5324,"ClientRequestHost":"eqlplayground.io","ClientRequestMethod":"GET","ClientRequestPath":"/40865/bundles/plugin/securitySolution/8.0.0/securitySolution.chunk.9.js","ClientRequestProtocol":"HTTP/1.1","ClientRequestReferer":"https://eqlplayground.io/s/eqldemo/app/security/timelines/default?sourcerer=(default:!(.siem-signals-eqldemo))&timerange=(global:(linkTo:!(),timerange:(from:%272021-03-03T19:55:15.519Z%27,fromStr:now-24h,kind:relative,to:%272021-03-04T19:55:15.519Z%27,toStr:now)),timeline:(linkTo:!(),timerange:(from:%272020-03-04T19:55:28.684Z%27,fromStr:now-1y,kind:relative,to:%272021-03-04T19:55:28.692Z%27,toStr:now)))&timeline=(activeTab:eql,graphEventId:%27%27,id:%2769f93840-7d23-11eb-866c-79a0609409ba%27,isOpen:!t)","ClientRequestURI":"/40865/bundles/plugin/securitySolution/8.0.0/securitySolution.chunk.9.js","ClientRequestUserAgent":"Mozilla/5.0(WindowsNT10.0;Win64;x64)AppleWebKit/537.36(KHTML,likeGecko)Chrome/91.0.4472.124Safari/537.36","ClientSSLCipher":"NONE","ClientSSLProtocol":"none","ClientSrcPort":0,"ClientXRequestedWith":"","EdgeColoCode":"33.147.138.217","EdgeColoID":20,"EdgeEndTimestamp":1625752958875000000,"EdgePathingOp":"wl","EdgePathingSrc":"macro","EdgePathingStatus":"nr","EdgeRateLimitAction":"","EdgeRateLimitID":0,"EdgeRequestHost":"eqlplayground.io","EdgeResponseBytes":24743,"EdgeResponseCompressionRatio":0,"EdgeResponseContentType":"application/javascript","EdgeResponseStatus":200,"EdgeServerIP":"","EdgeStartTimestamp":1625752958812000000,"FirewallMatchesActions":[],"FirewallMatchesRuleIDs":[],"FirewallMatchesSources":[],"OriginIP":"","OriginResponseBytes":0,"OriginResponseHTTPExpires":"","OriginResponseHTTPLastModified":"","OriginResponseStatus":0,"OriginResponseTime":0,"OriginSSLProtocol":"unknown","ParentRayID":"66b9d9f88b5b4c4f","RayID":"66b9d9f890ae4c4f","SecurityLevel":"off","WAFAction":"unknown","WAFFlags":"0","WAFMatchedVar":"","WAFProfile":"unknown","WAFRuleID":"","WAFRuleMessage":"","WorkerCPUTime":0,"WorkerStatus":"unknown","WorkerSubrequest":true,"WorkerSubrequestCount":0,"ZoneID":393347122}
           {"CacheCacheStatus":"unknown","CacheResponseBytes":0,"CacheResponseStatus":0,"CacheTieredFill":false,"ClientASN":1136,"ClientCountry":"nl","ClientDeviceType":"desktop","ClientIP":"149.175.108.201","ClientIPClass":"noRecord","ClientRequestBytes":2520,"ClientRequestHost":"eqlplayground.io","ClientRequestMethod":"GET","ClientRequestPath":"/s/eqldemo/security/account","ClientRequestProtocol":"HTTP/2","ClientRequestReferer":"","ClientRequestURI":"/s/eqldemo/security/account","ClientRequestUserAgent":"Mozilla/5.0(WindowsNT10.0;Win64;x64)AppleWebKit/537.36(KHTML,likeGecko)Chrome/91.0.4472.124Safari/537.36","ClientSSLCipher":"AEAD-AES128-GCM-SHA256","ClientSSLProtocol":"TLSv1.3","ClientSrcPort":61593,"ClientXRequestedWith":"","EdgeColoCode":"AMS","EdgeColoID":20,"EdgeEndTimestamp":1625754264684000000,"EdgePathingOp":"ban","EdgePathingSrc":"filterBasedFirewall","EdgePathingStatus":"nr","EdgeRateLimitAction":"","EdgeRateLimitID":0,"EdgeRequestHost":"183.53.30.34","EdgeResponseBytes":2066,"EdgeResponseCompressionRatio":2.45,"EdgeResponseContentType":"text/html","EdgeResponseStatus":403,"EdgeServerIP":"","EdgeStartTimestamp":1625754264676000000,"FirewallMatchesActions":["block"],"FirewallMatchesRuleIDs":["391eb601201e4f2a81038910f2b63f6d"],"FirewallMatchesSources":["firewallRules"],"OriginIP":"","OriginResponseBytes":0,"OriginResponseHTTPExpires":"","OriginResponseHTTPLastModified":"","OriginResponseStatus":0,"OriginResponseTime":0,"OriginSSLProtocol":"unknown","ParentRayID":"00","RayID":"66b9f9da396e4c01","SecurityLevel":"unk","WAFAction":"unknown","WAFFlags":"0","WAFMatchedVar":"","WAFProfile":"unknown","WAFRuleID":"","WAFRuleMessage":"","WorkerCPUTime":0,"WorkerStatus":"unknown","WorkerSubrequest":false,"WorkerSubrequestCount":0,"ZoneID":393347122}
+  - path: /client/v4/accounts/aaabbbccc/audit_logs
+    methods: ["GET"]
+    request_headers:
+      x-auth-email: [email protected]
+      x-auth-key: xxxxxxxxxx
+    query_params:
+      since: "{since:.*}"
+      page: "1"
+    responses:
+      - status_code: 200
+        body: |
+          {
+            "result": [
+              {"action":{"result":true,"type":"token_create"},"actor":{"email":"[email protected]","id":"enl3j9du8rnx2swwd9l32qots7l54t9s","ip":"52.91.36.10","type":"user"},"id":"73fd39ed-5aab-4a2a-b93c-c9a4abf0c425","interface":"","metadata":{"token_name":"test","token_tag":"b7261c49a793a82678d12285f0bc1401"},"newValue":"","oldValue":"","owner":{"id":"enl3j9du8rnx2swwd9l32qots7l54t9s"},"resource":{"id":"enl3j9du8rnx2swwd9l32qots7l54t9s","type":"account"},"when":"2021-11-30T20:19:48Z"},
+              {"action":{"result":true,"type":"token_revoke"},"actor":{"email":"[email protected]","id":"enl3j9du8rnx2swwd9l32qots7l54t9s","ip":"52.91.36.10","type":"user"},"id":"9929d149-1c4e-4524-87b5-bb81e83b5c84","interface":"","metadata":{"new_token_status":"deleted","old_token_status":"active","token_name":"test","token_tag":"70b6abc4efe977131126486cdd1c00c5"},"newValue":"","oldValue":"","owner":{"id":"enl3j9du8rnx2swwd9l32qots7l54t9s"},"resource":{"id":"enl3j9du8rnx2swwd9l32qots7l54t9s","type":"account"},"when":"2021-11-30T20:19:27Z"}
+            ],
+            "success": true,
+            "errors": [],
+            "messages": []
+          }
+  - path: /client/v4/accounts/aaabbbccc/audit_logs
+    methods: ["GET"]
+    request_headers:
+      x-auth-email: [email protected]
+      x-auth-key: xxxxxxxxxx
+    query_params:
+      since: "{since:.*}"
+      page: "2"
+    responses:
+      - status_code: 200
+        body: |-
+          {
+              "result": [
+                {"action":{"result":true,"type":"API_key_view"},"actor":{"email":"[email protected]","id":"enl3j9du8rnx2swwd9l32qots7l54t9s","ip":"52.91.36.10","type":"user"},"id":"dc0b470f-17b0-4bff-9113-a4fba3bf052c","interface":"","metadata":{},"newValue":"","oldValue":"","owner":{"id":"enl3j9du8rnx2swwd9l32qots7l54t9s"},"resource":{"id":"enl3j9du8rnx2swwd9l32qots7l54t9s","type":"user"},"when":"2021-11-30T13:42:17Z"},
+                {"action":{"info":"key digest: c6b5d100d7ce492d24c5b13160fce1cc0092ce7e8d8430e9f5cf5468868be6f6","result":true,"type":"rotate_API_key"},"actor":{"email":"[email protected]","id":"enl3j9du8rnx2swwd9l32qots7l54t9s","ip":"52.91.36.10","type":"user"},"id":"8d3396e8-c903-5a66-9421-00fc34570550","interface":"","metadata":{},"newValue":"","oldValue":"","owner":{"id":"enl3j9du8rnx2swwd9l32qots7l54t9s"},"resource":{"id":"enl3j9du8rnx2swwd9l32qots7l54t9s","type":"account"},"when":"2021-11-30T13:42:04Z"}
+              ],
+              "success": true,
+              "errors": [],
+              "messages": []
+          }

packages/cloudflare/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.2.0"
+  changes:
+    - description: Add audit logs
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/2294
 - version: "1.1.1"
   changes:
     - description: Change test public IPs to the supported subset