Merge branch 'main' into fsstat-ignore

Author: VihasMakwana

.buildkite/scripts/backport_branch.sh

@@ -96,10 +96,15 @@ createLocalBackportBranch() {
 
 removeOtherPackages() {
   local sourceFolder=$1
+  local currentPackage=""
   for dir in "$sourceFolder"/*; do
     if [[ -d "$dir" ]] && [[ "$(basename "$dir")" != "$PACKAGE_NAME" ]]; then
       echo "Removing directory: $dir"
       rm -rf "$dir"
+
+      currentPackage=$(basename "${dir}")
+      echo "Removing ${currentPackage} from .github/CODEOWNERS"
+      sed -i "/^\/packages\/${currentPackage}\//d" .github/CODEOWNERS
     fi
   done
 }
@@ -116,70 +121,90 @@ updateBackportBranchContents() {
   local BUILDKITE_FOLDER_PATH=".buildkite"
   local JENKINS_FOLDER_PATH=".ci"
   local files_cached_num=""
+
+  git checkout "$BACKPORT_BRANCH_NAME"
+  echo "Copying $BUILDKITE_FOLDER_PATH from $SOURCE_BRANCH..."
+  git checkout $SOURCE_BRANCH -- $BUILDKITE_FOLDER_PATH
+  git add $BUILDKITE_FOLDER_PATH
+
   if git ls-tree -d --name-only main:.ci >/dev/null 2>&1; then
-    git checkout $BACKPORT_BRANCH_NAME
-    echo "Copying $BUILDKITE_FOLDER_PATH from $SOURCE_BRANCH..."
-    git checkout $SOURCE_BRANCH -- $BUILDKITE_FOLDER_PATH
     echo "Copying $JENKINS_FOLDER_PATH from $SOURCE_BRANCH..."
     git checkout $SOURCE_BRANCH -- $JENKINS_FOLDER_PATH
+    git add $JENKINS_FOLDER_PATH
   else
-    git checkout $BACKPORT_BRANCH_NAME
-    echo "Copying $BUILDKITE_FOLDER_PATH from $SOURCE_BRANCH..."
-    git checkout $SOURCE_BRANCH -- $BUILDKITE_FOLDER_PATH
-    echo "Removing $JENKINS_FOLDER_PATH from $BACKPORT_BRANCH_NAME..."
-    rm -rf "$JENKINS_FOLDER_PATH"
+    if [ -d "${JENKINS_FOLDER_PATH}" ]; then
+      echo "Removing $JENKINS_FOLDER_PATH from $BACKPORT_BRANCH_NAME..."
+      rm -rf "$JENKINS_FOLDER_PATH"
+      git add "$JENKINS_FOLDER_PATH"
+    fi
   fi
 
   # Update scripts used by mage
   local MAGEFILE_SCRIPTS_FOLDER="dev/citools"
   local TESTSREPORTER_SCRIPTS_FOLDER="dev/testsreporter"
   local COVERAGE_SCRIPTS_FOLDER="dev/coverage"
+  local CODEOWNERS_SCRIPTS_FOLDER="dev/codeowners"
+
   if git ls-tree -d --name-only main:${MAGEFILE_SCRIPTS_FOLDER} > /dev/null 2>&1 ; then
     echo "Copying $MAGEFILE_SCRIPTS_FOLDER from $SOURCE_BRANCH..."
     git checkout "$SOURCE_BRANCH" -- "${MAGEFILE_SCRIPTS_FOLDER}"
+    git add ${MAGEFILE_SCRIPTS_FOLDER}
+
     echo "Copying $TESTSREPORTER_SCRIPTS_FOLDER from $SOURCE_BRANCH..."
     git checkout "$SOURCE_BRANCH" -- "${TESTSREPORTER_SCRIPTS_FOLDER}"
+    git add ${TESTSREPORTER_SCRIPTS_FOLDER}
+
     echo "Copying $COVERAGE_SCRIPTS_FOLDER from $SOURCE_BRANCH..."
     git checkout "$SOURCE_BRANCH" -- "${COVERAGE_SCRIPTS_FOLDER}"
+    git add ${COVERAGE_SCRIPTS_FOLDER}
+
+    echo "Copying $CODEOWNERS_SCRIPTS_FOLDER from $SOURCE_BRANCH..."
+    git checkout "$SOURCE_BRANCH" -- "${CODEOWNERS_SCRIPTS_FOLDER}"
+    git add ${CODEOWNERS_SCRIPTS_FOLDER}
+
     echo "Copying magefile.go from $SOURCE_BRANCH..."
     git checkout "$SOURCE_BRANCH" -- "magefile.go"
+    git add magefile.go
+
+    # As this script runs in the context of the main branch (mainly go mod tidy), we need to copy
+    # the .go-version file from the main branch to the backport branch. This avoids failures
+    # installing dependencies in the backport Pull Request.
+    echo "Copying .go-version from $SOURCE_BRANCH..."
+    git checkout "$SOURCE_BRANCH" -- ".go-version"
+    git add .go-version
+
     # Run go mod tidy to update just the dependencies related to magefile and dev scripts
     go mod tidy
+
+    git add go.mod go.sum
   fi
 
   if [ "${REMOVE_OTHER_PACKAGES}" == "true" ]; then
     echo "Removing all packages from $PACKAGES_FOLDER_PATH folder"
     removeOtherPackages "${PACKAGES_FOLDER_PATH}"
-    ls -la $PACKAGES_FOLDER_PATH
+    ls -la "${PACKAGES_FOLDER_PATH}"
+
+    git add "${PACKAGES_FOLDER_PATH}/"
+    git add .github/CODEOWNERS
   fi
 
+  git status
+
   echo "Setting up git environment..."
   update_git_config
 
-  echo "Commiting"
-  git add $BUILDKITE_FOLDER_PATH
-  if [ -d "${JENKINS_FOLDER_PATH}" ]; then
-    git add "${JENKINS_FOLDER_PATH}"
-  fi
-  if [ -d "${MAGEFILE_SCRIPTS_FOLDER}" ] ; then
-    git add ${MAGEFILE_SCRIPTS_FOLDER}
-    git add ${TESTSREPORTER_SCRIPTS_FOLDER}
-    git add go.mod go.sum
-  fi
-  git add $PACKAGES_FOLDER_PATH/
-  git status
-
   files_cached_num=$(git diff --name-only --cached | wc -l)
   if [ "${files_cached_num}" -gt 0 ]; then
+    echo "Committing changes..."
     git commit -m "Add $BUILDKITE_FOLDER_PATH and $JENKINS_FOLDER_PATH to backport branch: $BACKPORT_BRANCH_NAME from the $SOURCE_BRANCH branch"
   else
     echo "Nothing to commit, skip."
   fi
 
   if [ "$DRY_RUN" == "true" ];then
     echo "DRY_RUN mode, nothing will be pushed."
-    # Show just the relevant files diff (go.mod, go.sum, .buildkite, dev and package to be backported)
-    git --no-pager diff $SOURCE_BRANCH...$BACKPORT_BRANCH_NAME go.mod go.sum .buildkite/ dev/ "packages/${PACKAGE_NAME}"
+    # Show just the relevant files diff (go.mod, go.sum, .buildkite, dev, .go-version, .github/CODEOWNERS and package to be backported)
+    git --no-pager diff $SOURCE_BRANCH...$BACKPORT_BRANCH_NAME .buildkite/ dev/ go.sum go.mod .go-version .github/CODEOWNERS "packages/${PACKAGE_NAME}"
   else
     echo "Pushing..."
     git push origin $BACKPORT_BRANCH_NAME

.github/CODEOWNERS

@@ -126,6 +126,7 @@
 /packages/bbot @elastic/security-service-integrations
 /packages/beaconing @elastic/ml-ui @elastic/sec-applied-ml
 /packages/beat @elastic/stack-monitoring
+/packages/beelzebub @elastic/security-service-integrations
 /packages/beyondinsight_password_safe @elastic/security-service-integrations
 /packages/beyondtrust_pra @elastic/security-service-integrations
 /packages/bitdefender @elastic/security-service-integrations

packages/azure/changelog.yml

@@ -1,3 +1,8 @@
+- version: "1.24.0"
+  changes:
+    - description: Set `service.id`, `device.id`, `user.id`, `session.id`, and `token.id` in graphactivitylogs dataset.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/13931
 - version: "1.23.3"
   changes:
     - description: Improve Azure logs documentation with more details on log categories routing rules.

packages/azure/data_stream/graphactivitylogs/_dev/test/pipeline/test-activitylogs-raw.log

@@ -2,4 +2,5 @@
 {"Level":4,"callerIpAddress":"81.2.69.143","category":"MicrosoftGraphActivityLogs","correlationId":"f7739jk0-e6d1-4e3f-985a-64937fbge367","durationMs":453011,"location":"Germany West Central","operationName":"Microsoft Graph Activity","operationVersion":"v1.0","properties":{"apiVersion":"v1.0","appId":"a5a68e12-268a-3c91-a5e2-b9254e67hb29","atContent":"","clientAuthMethod":"2","clientRequestId":"2fe56790-a848-4c83-9d2c-5675972aejk9","durationMs":453011,"identityProvider":"https://sts.windows.net/aa30985b-427d-4434-b4dc-8f9040719adb/","ipAddress":"81.2.69.143","location":"Germany West Central","operationId":"f7739jk0-e6d1-4e3f-985a-64937fbge367","requestId":"f7739jk0-e6d1-4e3f-985a-64937fbge367","requestMethod":"GET","requestUri":"https://graph.microsoft.com/v1.0/me/photos/96x96/$value","responseSizeBytes":294,"responseStatusCode":404,"roles":null,"scopes":"email openid Organization.Read.All Policy.ReadWrite.ApplicationConfiguration profile User.Read","servicePrincipalId":null,"signInActivityId":"sign-in_ActivityId","tenantId":"aa30985b-427d-4434-b4dc-8f9040719adb","timeGenerated":"2024-03-07T10:35:31.9597832Z","tokenIssuedAt":"2024-03-07T10:30:30Z","userAgent":"","userId":"b37ec517-0a34-4266-b627-f7bb0d679d70","wids":"1997b4d3-0g8d-90cb-bhj5-d80n3122e98 1997b4d3-0f8d-76cb-bhj5-d80n3122e98"},"resourceId":"/TENANTS/AA30985B-427D-4434-B4DC-8F9040719ADB/PROVIDERS/MICROSOFT.AADIAM","resultSignature":"404","tenantId":"aa30985b-427d-4434-b4dc-8f9040719adb","time":"2024-03-07T10:35:31.9597832Z"}
 {"Level":4,"callerIpAddress":"81.2.69.144","category":"MicrosoftGraphActivityLogs","correlationId":"f7739da0-e6d1-4e3f-875a-64934fbge347","durationMs":16688471,"location":"UK South","operationName":"Microsoft Graph Activity","operationVersion":"beta","properties":{"apiVersion":"beta","appId":"a5a68e12-268a-3c91-a5f2-b9254e67hb28","atContent":"","clientAuthMethod":"0","clientRequestId":"2fe56789-a848-4c93-9s2c-5675972aghk9","durationMs":16688471,"identityProvider":null,"ipAddress":"81.2.69.144","location":"UK South","operationId":"f7739da0-e6d1-4e3f-875a-64934fbge347","requestId":"f7739da0-e6d1-4e3f-875a-64934fbge347","requestMethod":"GET","requestUri":"https://graph.microsoft.com/beta//users/7ef3c2ad-d52l-4a89-8cf9-c30178181027/photos/48x48/$value","responseSizeBytes":0,"responseStatusCode":404,"roles":null,"scopes":"AdministrativeUnit.ReadWrite.All AuditLog.Read.All Directory.AccessAsUser.All Directory.Write.Restricted email openid Organization.Read.All Policy.ReadWrite.Authorization profile User.EnableDisableAccount.All User.ReadWrite.All","servicePrincipalId":null,"signInActivityId":"signin_ActivityId","tenantId":"ab47545b-420e-46fg-c4dc-8f7697k1aadb","timeGenerated":"2024-03-07T16:42:22.84914Z","tokenIssuedAt":"2024-03-07T16:37:20Z","userAgent":"","userId":"285e0849-a706-4a9a-9eb1-f4e21cc78793","wids":"1997b4d3-0g8d-90cb-bhj5-d80n3122e98 1997b4d3-0g8d-90cb-bhj5-d80n3122e99 1997b4d3-0g8d-90cb-bhj5-d80n3122e80 1997b4d3-0g8d-90cb-bhj5-d80n3122e83"},"resourceId":"/TENANTS/AB47545B-420E-46FG-C4DC-8F7697K1AADB/PROVIDERS/MICROSOFT.AADIAM","resultSignature":"404","tenantId":"ab47545b-420e-46fg-c4dc-8f7697k1aadb","time":"2024-03-07T16:42:22.8491400Z"}
 {"Level":4,"callerIpAddress":"2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6","category":"MicrosoftGraphActivityLogs","correlationId":"f7749da0-e6g1-4f3f-975a-64937fbge347","durationMs":846544,"location":"UK South","operationName":"Microsoft Graph Activity","operationVersion":"beta","properties":{"apiVersion":"beta","appId":"a5a68e32-269a-3c91-a5e2-b9254e67hb29","atContent":"","clientAuthMethod":"0","clientRequestId":"2fe56789-a848-4c93-9d2d-5675972ardk9","durationMs":846544,"identityProvider":null,"ipAddress":"2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6","location":"UK South","operationId":"f7749da0-e6g1-4f3f-975a-64937fbge347","requestId":"f7749da0-e6g1-4f3f-975a-64937fbge347","requestMethod":"GET","requestUri":"https://graph.microsoft.com/beta/devices?$select=displayName,id,deviceId\\u0026$search=%22displayName:a%22\\u0026$top=30","responseSizeBytes":0,"responseStatusCode":200,"roles":null,"scopes":"AccessReview.ReadWrite.All AuditLog.Read.All ChangeManagement.Read.All ConsentRequest.Create ConsentRequest.Read ConsentRequest.ReadApprove.All ConsentRequest.ReadWrite.All CustomSecAttributeAuditLogs.Read.All Directory.AccessAsUser.All Directory.Read.All Directory.ReadWrite.All Directory.Write.Restricted DirectoryRecommendations.Read.All DirectoryRecommendations.ReadWrite.All email EntitlementManagement.Read.All Group.ReadWrite.All IdentityProvider.ReadWrite.All IdentityRiskEvent.ReadWrite.All IdentityRiskyServicePrincipal.ReadWrite.All IdentityRiskyUser.ReadWrite.All IdentityUserFlow.Read.All LifecycleWorkflows.ReadWrite.All openid Policy.Read.All Policy.Read.IdentityProtection Policy.ReadWrite.AuthenticationFlows Policy.ReadWrite.AuthenticationMethod Policy.ReadWrite.ConditionalAccess Policy.ReadWrite.ExternalIdentities Policy.ReadWrite.IdentityProtection Policy.ReadWrite.MobilityManagement profile Reports.Read.All RoleManagement.ReadWrite.Directory RoleManagement.ReadWrite.Exchange SecurityEvents.ReadWrite.All TrustFrameworkKeySet.Read.All User.Export.All User.ReadWrite.All UserAuthenticationMethod.ReadWrite.All","servicePrincipalId":null,"signInActivityId":"signin_Activity-Id","tenantId":"ab47545b-420e-46fg-c4dc-8f7697k1aadb","timeGenerated":"2024-03-07T16:42:12.0485843Z","tokenIssuedAt":"2024-03-07T16:36:51Z","userAgent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36 Edg/122.0.0.0","userId":"285e0849-a706-4a9a-9eb1-f4e21cc78793","wids":"a207b4d3-0g8d-90cb-bhj5-d80n3122e67 a207b4d3-0g8d-90cb-bhj5-d80n3122e69 a207b4d3-0g8d-90cb-bhj5-d80n3122e89"},"resourceId":"/TENANTS/AB47545B-420E-46FG-C4DC-8F7697K1AADB/PROVIDERS/MICROSOFT.AADIAM","resultSignature":"200","tenantId":"ab47545b-420e-46fg-c4dc-8f7697k1aadb","time":"2024-03-07T16:42:12.0485843Z"}
-{"Level":4,"callerIpAddress":"81.2.69.143","category":"MicrosoftGraphActivityLogs","correlationId":"f7839da0-e7d1-4e4f-985a-64937fbge347","durationMs":1100725,"location":"France Central","operationName":"Microsoft Graph Activity","operationVersion":"v1.0","properties":{"apiVersion":"v1.0","appId":"a5a68e32-269a-3c91-a5e2-b9254e67hb29","atContent":"","clientAuthMethod":"2","clientRequestId":"2fe58790-a848-4a93-9d2c-5645972aejk9","durationMs":1100725,"identityProvider":"https://sts.windows.net/ab30785b-417f-42a4-b5dc-8f9051718acb/","ipAddress":"81.2.69.143","location":"France Central","operationId":"f7839da0-e7d1-4e4f-985a-64937fbge347","requestId":"f7839da0-e7d1-4e4f-985a-64937fbge347","requestMethod":"GET","requestUri":"https://graph.microsoft.com/v1.0/directoryRoles","responseSizeBytes":4300,"responseStatusCode":200,"roles":"Application.Read.All Domain.Read.All GroupMember.Read.All LicenseAssignment.ReadWrite.All Organization.Read.All Policy.Read.ConditionalAccess RoleManagement.Read.Directory Team.ReadBasic.All TeamsTab.Create TeamsTab.Read.All TeamsTab.ReadWrite.All User.Read.All","scopes":null,"servicePrincipalId":"f2aq4c71-31e3-5065-91g3-4b2dfbsv50fg","signInActivityId":"sign-in_ActivityId","tenantId":"ab30785b-417f-42a4-b5dc-8f9051718acb","timeGenerated":"2024-03-07T10:24:44.7939418Z","tokenIssuedAt":"2024-03-07T10:19:44Z","userAgent":"","userId":null,"wids":"a207b4d3-0g8d-90cb-bhj5-d80n3121e69"},"resourceId":"/TENANTS/AB30785B-417F-42A4-B5DC-8F9051718ACB/PROVIDERS/MICROSOFT.AADIAM","resultSignature":"200","tenantId":"ab30785b-417f-42a4-b5dc-8f9051718acb","time":"2024-03-07T10:24:44.7939418Z"}
+{"Level":4,"callerIpAddress":"81.2.69.143","category":"MicrosoftGraphActivityLogs","correlationId":"f7839da0-e7d1-4e4f-985a-64937fbge347","durationMs":1100725,"location":"France Central","operationName":"Microsoft Graph Activity","operationVersion":"v1.0","properties":{"apiVersion":"v1.0","appId":"a5a68e32-269a-3c91-a5e2-b9254e67hb29","atContent":"","clientAuthMethod":"2","clientRequestId":"2fe58790-a848-4a93-9d2c-5645972aejk9","durationMs":1100725,"identityProvider":"https://sts.windows.net/ab30785b-417f-42a4-b5dc-8f9051718acb/","ipAddress":"81.2.69.143","location":"France Central","operationId":"f7839da0-e7d1-4e4f-985a-64937fbge347","requestId":"f7839da0-e7d1-4e4f-985a-64937fbge347","requestMethod":"GET","requestUri":"https://graph.microsoft.com/v1.0/directoryRoles","responseSizeBytes":4300,"responseStatusCode":200,"roles":"Application.Read.All Domain.Read.All GroupMember.Read.All LicenseAssignment.ReadWrite.All Organization.Read.All Policy.Read.ConditionalAccess RoleManagement.Read.Directory Team.ReadBasic.All TeamsTab.Create TeamsTab.Read.All TeamsTab.ReadWrite.All User.Read.All","scopes":null,"servicePrincipalId":"f2aq4c71-31e3-5065-91g3-4b2dfbsv50fg","signInActivityId":"sign-in_ActivityId","tenantId":"ab30785b-417f-42a4-b5dc-8f9051718acb","timeGenerated":"2024-03-07T10:24:44.7939418Z","tokenIssuedAt":"2024-03-07T10:19:44Z","userAgent":"","userId":null,"wids":"a207b4d3-0g8d-90cb-bhj5-d80n3121e69"},"resourceId":"/TENANTS/AB30785B-417F-42A4-B5DC-8F9051718ACB/PROVIDERS/MICROSOFT.AADIAM","resultSignature":"200","tenantId":"ab30785b-417f-42a4-b5dc-8f9051718acb","time":"2024-03-07T10:24:44.7939418Z"}
+{"Level":4,"callerIpAddress":"81.2.69.143","category":"MicrosoftGraphActivityLogs","correlationId":"f7839da0-e7d1-4e4f-985a-64937fbge347","durationMs":1100725,"location":"France Central","operationName":"Microsoft Graph Activity","operationVersion":"v1.0","properties":{"apiVersion":"v1.0","appId":"a5a68e32-269a-3c91-a5e2-b9254e67hb29","atContent":"","clientAuthMethod":"2","C_DeviceId": "abc123","c_Sid":"xyz000","clientRequestId":"2fe58790-a848-4a93-9d2c-5645972aejk9","durationMs":1100725,"identityProvider":"https://sts.windows.net/ab30785b-417f-42a4-b5dc-8f9051718acb/","ipAddress":"81.2.69.143","location":"France Central","operationId":"f7839da0-e7d1-4e4f-985a-64937fbge347","requestId":"f7839da0-e7d1-4e4f-985a-64937fbge347","requestMethod":"GET","requestUri":"https://graph.microsoft.com/v1.0/directoryRoles","responseSizeBytes":4300,"responseStatusCode":200,"roles":"Application.Read.All Domain.Read.All GroupMember.Read.All LicenseAssignment.ReadWrite.All Organization.Read.All Policy.Read.ConditionalAccess RoleManagement.Read.Directory Team.ReadBasic.All TeamsTab.Create TeamsTab.Read.All TeamsTab.ReadWrite.All User.Read.All","scopes":null,"servicePrincipalId":"f2aq4c71-31e3-5065-91g3-4b2dfbsv50fg","signInActivityId":"sign-in_ActivityId","tenantId":"ab30785b-417f-42a4-b5dc-8f9051718acb","timeGenerated":"2024-03-07T10:24:44.7939418Z","tokenIssuedAt":"2024-03-07T10:19:44Z","userAgent":"","userId":null,"wids":"a207b4d3-0g8d-90cb-bhj5-d80n3121e69"},"resourceId":"/TENANTS/AB30785B-417F-42A4-B5DC-8F9051718ACB/PROVIDERS/MICROSOFT.AADIAM","resultSignature":"200","tenantId":"ab30785b-417f-42a4-b5dc-8f9051718acb","time":"2024-03-07T10:24:44.7939418Z"}