|
| 1 | +# Licensed to the Apache Software Foundation (ASF) under one or more |
| 2 | +# contributor license agreements. See the NOTICE file distributed with |
| 3 | +# this work for additional information regarding copyright ownership. |
| 4 | +# The ASF licenses this file to You under the Apache License, Version 2.0 |
| 5 | +# (the "License"); you may not use this file except in compliance with |
| 6 | +# the License. You may obtain a copy of the License at |
| 7 | +# |
| 8 | +# http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | +# |
| 10 | +# Unless required by applicable law or agreed to in writing, software |
| 11 | +# distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | +# See the License for the specific language governing permissions and |
| 14 | +# limitations under the License. |
| 15 | + |
| 16 | +# This reusable workflow executes a single check from `hadoop-ozone/dev-support/checks/`. |
| 17 | +# Before and after the check, it performs various steps based on workflow inputs. |
| 18 | + |
| 19 | +name: ci-check |
| 20 | + |
| 21 | +on: |
| 22 | + workflow_call: |
| 23 | + inputs: |
| 24 | + # REQUIRED |
| 25 | + script: |
| 26 | + type: string |
| 27 | + description: "Test script to run from hadoop-ozone/dev-support/checks, without .sh extension" |
| 28 | + required: true |
| 29 | + |
| 30 | + sha: |
| 31 | + type: string |
| 32 | + description: "Commit SHA to test" |
| 33 | + required: true |
| 34 | + |
| 35 | + # OPTIONAL (ordered alphabetically) |
| 36 | + checkout-fetch-depth: |
| 37 | + type: number |
| 38 | + description: "Fetch depth for checking out the repo (default: no history)" |
| 39 | + default: 1 |
| 40 | + required: false |
| 41 | + |
| 42 | + java-version: |
| 43 | + type: string |
| 44 | + description: "Java version to set up (default: none)" |
| 45 | + default: '' |
| 46 | + required: false |
| 47 | + |
| 48 | + needs-maven-cache: |
| 49 | + type: boolean |
| 50 | + description: "Whether to restore Maven cache before run (default: yes)" |
| 51 | + default: true |
| 52 | + required: false |
| 53 | + |
| 54 | + needs-npm-cache: |
| 55 | + type: boolean |
| 56 | + description: "Whether to restore NPM cache before run (default: no)" |
| 57 | + default: false |
| 58 | + required: false |
| 59 | + |
| 60 | + needs-ozone-binary-tarball: |
| 61 | + type: boolean |
| 62 | + description: "Whether to download Ozone binary tarball created by build (default: no)" |
| 63 | + default: false |
| 64 | + required: false |
| 65 | + |
| 66 | + needs-ozone-repo: |
| 67 | + type: boolean |
| 68 | + description: "Whether to download Ozone jars created by build (default: no)" |
| 69 | + default: false |
| 70 | + required: false |
| 71 | + |
| 72 | + needs-ozone-source-tarball: |
| 73 | + type: boolean |
| 74 | + description: "Whether to download Ozone source tarball created by build (default: no)" |
| 75 | + default: false |
| 76 | + required: false |
| 77 | + |
| 78 | + pre-script: |
| 79 | + type: string |
| 80 | + description: "Command to execute before the test script (default: none)" |
| 81 | + default: '' |
| 82 | + required: false |
| 83 | + |
| 84 | + post-failure: |
| 85 | + type: string |
| 86 | + description: "Command to execute after the test script, if it failed (default: none)" |
| 87 | + default: '' |
| 88 | + required: false |
| 89 | + |
| 90 | + post-success: |
| 91 | + type: string |
| 92 | + description: "Command to execute after the test script, if it succeeded (default: none)" |
| 93 | + default: '' |
| 94 | + required: false |
| 95 | + |
| 96 | + ratis-args: |
| 97 | + type: string |
| 98 | + description: "Version overrides from custom Ratis build (default: none)" |
| 99 | + default: '' |
| 100 | + required: false |
| 101 | + |
| 102 | + runner: |
| 103 | + type: string |
| 104 | + description: "GitHub Actions runner to use" |
| 105 | + default: 'ubuntu-24.04' |
| 106 | + required: false |
| 107 | + |
| 108 | + script-args: |
| 109 | + type: string |
| 110 | + description: "Arguments for the test script, ratis-args are appended" |
| 111 | + default: '' |
| 112 | + required: false |
| 113 | + |
| 114 | + split: |
| 115 | + type: string |
| 116 | + description: "Name of split for matrix jobs, only used in display name" |
| 117 | + default: '' |
| 118 | + required: false |
| 119 | + |
| 120 | + timeout-minutes: |
| 121 | + type: number |
| 122 | + description: "Job timeout in minutes (default: 30)" |
| 123 | + default: 30 |
| 124 | + required: false |
| 125 | + |
| 126 | + with-coverage: |
| 127 | + type: boolean |
| 128 | + description: "The value of OZONE_WITH_COVERAGE to set" |
| 129 | + default: true |
| 130 | + required: false |
| 131 | + |
| 132 | +env: |
| 133 | + HADOOP_IMAGE: ghcr.io/apache/hadoop |
| 134 | + MAVEN_ARGS: --batch-mode --settings ${{ github.workspace }}/dev-support/ci/maven-settings.xml |
| 135 | + MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 |
| 136 | + OZONE_IMAGE: ghcr.io/apache/ozone |
| 137 | + OZONE_RUNNER_IMAGE: ghcr.io/apache/ozone-runner |
| 138 | + OZONE_VOLUME_OWNER: 1000 |
| 139 | + |
| 140 | +jobs: |
| 141 | + check: |
| 142 | + name: ${{ (inputs.split && format('{0} ({1})', inputs.script, inputs.split)) || inputs.script }} |
| 143 | + runs-on: ${{ inputs.runner }} |
| 144 | + timeout-minutes: ${{ inputs.timeout-minutes }} |
| 145 | + steps: |
| 146 | + - name: Checkout project |
| 147 | + if: ${{ !inputs.needs-ozone-source-tarball }} |
| 148 | + uses: actions/checkout@v4 |
| 149 | + with: |
| 150 | + ref: ${{ inputs.sha }} |
| 151 | + fetch-depth: ${{ inputs.checkout-fetch-depth }} |
| 152 | + |
| 153 | + - name: Download Ozone source tarball |
| 154 | + if: ${{ inputs.needs-ozone-source-tarball }} |
| 155 | + uses: actions/download-artifact@v4 |
| 156 | + with: |
| 157 | + name: ozone-src |
| 158 | + |
| 159 | + - name: Extract source tarball |
| 160 | + if: ${{ inputs.needs-ozone-source-tarball }} |
| 161 | + run: | |
| 162 | + tar --strip-components 1 -xzvf ozone*-src.tar.gz |
| 163 | +
|
| 164 | + - name: Cache for NPM dependencies |
| 165 | + if: ${{ inputs.needs-npm-cache }} |
| 166 | + uses: actions/cache@v4 |
| 167 | + with: |
| 168 | + path: | |
| 169 | + ~/.pnpm-store |
| 170 | + **/node_modules |
| 171 | + key: ${{ runner.os }}-pnpm-${{ hashFiles('**/pnpm-lock.yaml') }} |
| 172 | + restore-keys: | |
| 173 | + ${{ runner.os }}-pnpm- |
| 174 | +
|
| 175 | + - name: Cache for Maven dependencies |
| 176 | + if: ${{ inputs.needs-maven-cache }} |
| 177 | + uses: actions/cache/restore@v4 |
| 178 | + with: |
| 179 | + path: | |
| 180 | + ~/.m2/repository/*/*/* |
| 181 | + !~/.m2/repository/org/apache/ozone |
| 182 | + key: maven-repo-${{ hashFiles('**/pom.xml') }} |
| 183 | + restore-keys: | |
| 184 | + maven-repo- |
| 185 | +
|
| 186 | + - name: Download Ozone repo |
| 187 | + id: download-ozone-repo |
| 188 | + if: ${{ inputs.needs-ozone-repo }} |
| 189 | + uses: actions/download-artifact@v4 |
| 190 | + with: |
| 191 | + name: ozone-repo |
| 192 | + path: | |
| 193 | + ~/.m2/repository/org/apache/ozone |
| 194 | +
|
| 195 | + - name: Download Ratis repo |
| 196 | + if: ${{ inputs.ratis-args != '' }} |
| 197 | + uses: actions/download-artifact@v4 |
| 198 | + with: |
| 199 | + name: ratis-jars |
| 200 | + path: | |
| 201 | + ~/.m2/repository/org/apache/ratis |
| 202 | +
|
| 203 | + - name: Download Ozone binary tarball |
| 204 | + if: ${{ inputs.needs-ozone-binary-tarball }} |
| 205 | + uses: actions/download-artifact@v4 |
| 206 | + with: |
| 207 | + name: ozone-bin |
| 208 | + |
| 209 | + - name: Extract binary tarball |
| 210 | + if: ${{ inputs.needs-ozone-binary-tarball }} |
| 211 | + run: | |
| 212 | + mkdir -p hadoop-ozone/dist/target |
| 213 | + tar xzvf ozone*.tar.gz -C hadoop-ozone/dist/target |
| 214 | + rm ozone*.tar.gz |
| 215 | +
|
| 216 | + - name: Setup java ${{ inputs.java-version }} |
| 217 | + if: ${{ inputs.java-version }} |
| 218 | + uses: actions/setup-java@v4 |
| 219 | + with: |
| 220 | + distribution: 'temurin' |
| 221 | + java-version: ${{ inputs.java-version }} |
| 222 | + |
| 223 | + - name: Execute pre-test steps |
| 224 | + if: ${{ inputs.pre-script }} |
| 225 | + run: | |
| 226 | + ${{ inputs.pre-script }} |
| 227 | +
|
| 228 | + - name: Execute tests |
| 229 | + run: | |
| 230 | + hadoop-ozone/dev-support/checks/${{ inputs.script }}.sh ${{ inputs.script-args }} ${{ inputs.ratis-args }} |
| 231 | + env: |
| 232 | + DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} |
| 233 | + OZONE_WITH_COVERAGE: ${{ inputs.with-coverage }} |
| 234 | + |
| 235 | + - name: Execute post-failure steps |
| 236 | + if: ${{ failure() && inputs.post-failure }} |
| 237 | + run: | |
| 238 | + ${{ inputs.post-failure }} |
| 239 | +
|
| 240 | + - name: Execute post-success steps |
| 241 | + if: ${{ !failure() && inputs.post-success }} |
| 242 | + run: | |
| 243 | + ${{ inputs.post-success }} |
| 244 | +
|
| 245 | + - name: Summary of failures |
| 246 | + if: ${{ failure() }} |
| 247 | + run: | |
| 248 | + if [[ -s "target/${{ inputs.script }}/summary.md" ]]; then |
| 249 | + cat target/${{ inputs.script }}/summary.md >> $GITHUB_STEP_SUMMARY |
| 250 | + fi |
| 251 | + hadoop-ozone/dev-support/checks/_summary.sh target/${{ inputs.script }}/summary.txt |
| 252 | +
|
| 253 | + - name: Archive build results |
| 254 | + if: ${{ !cancelled() }} |
| 255 | + uses: actions/upload-artifact@v4 |
| 256 | + with: |
| 257 | + name: ${{ (inputs.split && format('{0}-{1}', inputs.script, inputs.split)) || inputs.script }} |
| 258 | + path: target/${{ inputs.script }} |
| 259 | + continue-on-error: true |
| 260 | + |
| 261 | + # The following steps are hard-coded to be run only for 'build' check, |
| 262 | + # to avoid the need for 3 more inputs. |
| 263 | + - name: Store binaries for tests |
| 264 | + if: ${{ inputs.script == 'build' && !cancelled() }} |
| 265 | + uses: actions/upload-artifact@v4 |
| 266 | + with: |
| 267 | + name: ozone-bin |
| 268 | + path: | |
| 269 | + hadoop-ozone/dist/target/ozone-*.tar.gz |
| 270 | + !hadoop-ozone/dist/target/ozone-*-src.tar.gz |
| 271 | + retention-days: 1 |
| 272 | + |
| 273 | + - name: Store source tarball for compilation |
| 274 | + if: ${{ inputs.script == 'build' && !cancelled() }} |
| 275 | + uses: actions/upload-artifact@v4 |
| 276 | + with: |
| 277 | + name: ozone-src |
| 278 | + path: | |
| 279 | + hadoop-ozone/dist/target/ozone-*-src.tar.gz |
| 280 | + retention-days: 1 |
| 281 | + |
| 282 | + - name: Store Maven repo for tests |
| 283 | + if: ${{ inputs.script == 'build' && !cancelled() }} |
| 284 | + uses: actions/upload-artifact@v4 |
| 285 | + with: |
| 286 | + name: ozone-repo |
| 287 | + path: | |
| 288 | + ~/.m2/repository/org/apache/ozone |
| 289 | + retention-days: 1 |
0 commit comments