From d3aec635a863243b3c972597ae5d9fea51db5bc7 Mon Sep 17 00:00:00 2001 From: Colin Davidson Date: Thu, 12 Dec 2024 14:04:02 +0000 Subject: [PATCH] Support downloading of previous workflow's artefacts Support download inputs such as host_x86_64_linux=242234434 to download from a previous run's artefacts. This allows us to try different parts of the pipeline known to fail. --- .github/actions/do_build_dpcpp/action.yml | 39 +++++++++++++------ .../actions/do_build_ock_artefact/action.yml | 23 +++++++++-- .github/actions/do_build_sycl_cts/action.yml | 28 ++++++++++++- .github/actions/setup_gh/action.yml | 17 ++++++++ .github/workflows/planned_testing.yml | 32 ++++++++++++++- .github/workflows/planned_testing_caller.yml | 5 +++ 6 files changed, 128 insertions(+), 16 deletions(-) create mode 100644 .github/actions/setup_gh/action.yml diff --git a/.github/actions/do_build_dpcpp/action.yml b/.github/actions/do_build_dpcpp/action.yml index 6bac0624e..0672d54f8 100644 --- a/.github/actions/do_build_dpcpp/action.yml +++ b/.github/actions/do_build_dpcpp/action.yml @@ -4,8 +4,8 @@ description: build dpc++ inputs: target: description: 'target architecture' - create_dpcpp_artefact_method: - description: 'build | download_release' # TODO: add support for 'previous workflow' download + download_dpcpp_artefact: + description: 'download ock artefact rather than building, of form =id;. Special value of download_release applies to all targets.' type: string default: "build" @@ -22,14 +22,14 @@ runs: uses: llvm/actions/install-ninja@main - name: clone dpc++ - if: inputs.create_dpcpp_artefact_method == 'build' + if: contains(inputs.download_dpcpp_artefact, inputs.target) != true uses: actions/checkout@v4 with: repository: intel/llvm path: llvm - name: dpcpp configure - if: inputs.create_dpcpp_artefact_method == 'build' + if: contains(inputs.download_dpcpp_artefact, inputs.target) != true shell: bash run: cd llvm; python3 buildbot/configure.py @@ -42,19 +42,19 @@ runs: --cmake-opt=-DLLVM_ENABLE_ZSTD=OFF - name: build sycl-headers - if: inputs.create_dpcpp_artefact_method == 'build' + if: contains(inputs.download_dpcpp_artefact, inputs.target) != true shell: bash run: cmake --build llvm/build -- sycl-headers - name: build dpc plus plus - if: inputs.create_dpcpp_artefact_method == 'build' + if: contains(inputs.download_dpcpp_artefact, inputs.target) != true shell: bash run: python3 llvm/buildbot/compile.py -o llvm/build -v -j 8 - name: build extra utilties - if: inputs.create_dpcpp_artefact_method == 'build' + if: contains(inputs.download_dpcpp_artefact, inputs.target) != true # Build various utilities, since those aren't proper dependencies. # FileCheck and not are needed for tests. The rest are needed for # cross builds. They are enabled on all targets for consistency. @@ -65,7 +65,7 @@ runs: opt prepare_builtins -j8 - name: copy utilities - if: inputs.create_dpcpp_artefact_method == 'build' + if: contains(inputs.download_dpcpp_artefact, inputs.target) != true shell: bash run: cd llvm/build/bin; @@ -75,7 +75,7 @@ runs: # TODO: For now just linux x86_64 # Review for location of components (paths) and any archive package/unpackage reqs. #- name: install config files to pick up libraries for cross compilation. - # if: inputs.create_dpcpp_artefact_method == 'build' + # if: contains(inputs.download_dpcpp_artefact, inputs.target) != true # shell: bash # run: | # echo Installing configuration files @@ -89,8 +89,9 @@ runs: # " >../install/bin/$arch-unknown-linux-gnu.cfg; # done - - name: download dpc plus plus - if: inputs.create_dpcpp_artefact_method == 'download_release' + - name: download dpc plus plus from official releases + # TODO: This is a bit imperfect as it should parse it properly + if: contains(inputs.download_dpcpp_artefact, inputs.target) && contains(inputs.download_dpcpp_artefact, 'download_release') shell: bash run: | mkdir -p llvm/build/install @@ -103,8 +104,24 @@ runs: tar xf sycl_linux.tar.gz rm sycl_linux.tar.gz + - name: download previous dpcpp if needed ${{ inputs.target }} ${{ matrix.download_dpcpp_artefact}} + shell: bash + if: contains(inputs.download_dpcpp_artefact, inputs.target) && contains(inputs.download_dpcpp_artefact, 'download_release') != true + run: | + download_id=`echo "${{inputs.download_dpcpp_artefact}}"` + echo "echo $download_id | sed 's/.*${{inputs.target}}=//' | sed 's/;.*//'" + download_id=`echo $download_id | sed 's/.*${{inputs.target}}=//' | sed 's/;.*//'` + echo download id is "'$download_id'" + # TODO : make this work on windows + mkdir -p llvm/build/install + git config --global --add safe.directory $PWD + gh run download $download_id -n dpcpp_${{ inputs.target }} -D llvm/build/install + ls llvm/build/install + + - name: package artefacts # package/unpackage avoids known 'permissions loss' issue shell: bash + if: contains(inputs.download_dpcpp_artefact, inputs.target) != true run: | cd llvm/build/install tar cf dpcpp.tar * diff --git a/.github/actions/do_build_ock_artefact/action.yml b/.github/actions/do_build_ock_artefact/action.yml index 9c4b9d024..01a89b745 100644 --- a/.github/actions/do_build_ock_artefact/action.yml +++ b/.github/actions/do_build_ock_artefact/action.yml @@ -12,6 +12,9 @@ inputs: vulkan_sdk_install: description: 'vulkan install flag' default: true + download_ock_artefact: + description: 'download ock artefact rather than building, of form =id;.' + default: '' # TODO: This has not been tested yet on windows so would likely need some updating. runs: @@ -32,6 +35,7 @@ runs: # installs tools, ninja, installs llvm and sets up ccache - name: setup uses: ./.github/actions/setup_build + if: contains(inputs.download_ock_artefact, inputs.target) != true with: llvm_version: ${{ inputs.llvm_version }} llvm_build_type: RelAssert @@ -39,7 +43,7 @@ runs: os: ${{ contains(inputs.target, 'windows') && 'windows' || 'ubuntu' }} - name: build ock x86 - if: steps.calc_vars.outputs.arch == 'x86_64' + if: steps.calc_vars.outputs.arch == 'x86_64' && contains(inputs.download_ock_artefact, inputs.target) != true uses: ./.github/actions/do_build_ock with: build_targets: install @@ -47,8 +51,8 @@ runs: extra_flags: -DCA_ENABLE_TESTS=OFF -DCA_ENABLE_EXAMPLES=OFF -DCA_ENABLE_DOCUMENTATION=OFF shell_to_use: ${{ contains(inputs.target, 'windows') && 'pwsh' || 'bash' }} - - name: build ock other ${{ matrix.target }} - if: steps.calc_vars.outputs.arch != 'x86_64' + - name: build ock other ${{ inputs.target }} + if: steps.calc_vars.outputs.arch != 'x86_64' && contains(inputs.download_ock_artefact, inputs.target) != true uses: ./.github/actions/do_build_ock with: build_targets: install @@ -58,6 +62,19 @@ runs: offline_kernel_tests: OFF host_fp16: ON + - name: download previous ock if needed ${{ inputs.target }} ${{ matrix.download_dpcpp_artefact}} + shell: bash + if: contains(inputs.download_ock_artefact, inputs.target) + run: | + download_id=`echo "${{inputs.download_ock_artefact}}"` + echo "echo $download_id | sed 's/.*${{inputs.target}}=//' | sed 's/;.*//'" + download_id=`echo $download_id | sed 's/.*${{inputs.target}}=//' | sed 's/;.*//'` + echo download id is "'$download_id'" + git config --global --add safe.directory $PWD + # TODO : make this work on windows + gh run download $download_id -n ock_${{ inputs.target }} -D install + ls install + # Prune it as there is too much things in there we don't want to use # Todo: move this logic to cmake settings so that we build only what we # want to install. As time goes on we may want to install more. diff --git a/.github/actions/do_build_sycl_cts/action.yml b/.github/actions/do_build_sycl_cts/action.yml index f190d2208..b6c8fb69b 100644 --- a/.github/actions/do_build_sycl_cts/action.yml +++ b/.github/actions/do_build_sycl_cts/action.yml @@ -4,6 +4,9 @@ description: build sycl cts inputs: target: description: 'target architecture' + download_sycl_cts_artefact: + description: 'download ock artefact rather than building, of form =id;.' + default: '' runs: using: "composite" @@ -15,27 +18,32 @@ runs: target: ${{ inputs.target }} - name: Install Ninja + if: contains(inputs.download_sycl_cts_artefact, inputs.target) != true uses: llvm/actions/install-ninja@main - name: download icd artifact + if: contains(inputs.download_sycl_cts_artefact, inputs.target) != true uses: actions/download-artifact@v4 with: name: icd_${{inputs.target}} path: install_icd - name: download headers artifact + if: contains(inputs.download_sycl_cts_artefact, inputs.target) != true uses: actions/download-artifact@v4 with: name: header_${{inputs.target}} path: install_headers - name: download dpc++ artifact + if: contains(inputs.download_sycl_cts_artefact, inputs.target) != true uses: actions/download-artifact@v4 with: name: dpcpp_${{inputs.target}} path: install_dpcpp - name: unpackage dpc++ artifacts # package/unpackage avoids known 'permissions loss' issue + if: contains(inputs.download_sycl_cts_artefact, inputs.target) != true shell: bash run: | cd install_dpcpp @@ -43,6 +51,7 @@ runs: rm dpcpp.tar - name: checkout sycl cts + if: contains(inputs.download_sycl_cts_artefact, inputs.target) != true uses: actions/checkout@v4 with: repository: KhronosGroup/SYCL-CTS @@ -50,6 +59,7 @@ runs: submodules: true - name: build SYCL CTS + if: contains(inputs.download_sycl_cts_artefact, inputs.target) != true shell: bash run: | echo calling cmake and ninja on SYCL CTS @@ -70,16 +80,32 @@ runs: ninja -C SYCL-CTS -v -j4 -k 0 || : - name: package artefacts # package/unpackage avoids known 'permissions loss' issue + if: contains(inputs.download_sycl_cts_artefact, inputs.target) != true shell: bash run: | cd SYCL-CTS # only bin tar cf sycl-cts.tar bin + - name: download sycl cts artefact # package/unpackage avoids known 'permissions loss' issue + if: contains(inputs.download_sycl_cts_artefact, inputs.target) + shell: bash + run: | + download_id=`echo "${{inputs.download_sycl_cts_artefact}}"` + echo "echo $download_id | sed 's/.*${{inputs.target}}=//' | sed 's/;.*//'" + download_id=`echo $download_id | sed 's/.*${{inputs.target}}=//' | sed 's/;.*//'` + echo download id is "'$download_id'" + # TODO : make this work on windows + mkdir -p SYCL-CTS + git config --global --add safe.directory $PWD + gh run download $download_id -n sycl_cts_${{ inputs.target }} -D SYCL-CTS + ls SYCL-CTS + + - name: upload artefact uses: actions/upload-artifact@v4 with: name: sycl_cts_${{inputs.target}} path: SYCL-CTS/sycl-cts.tar - retention-days: 1 + retention-days: 2 diff --git a/.github/actions/setup_gh/action.yml b/.github/actions/setup_gh/action.yml new file mode 100644 index 000000000..2ae46ff6d --- /dev/null +++ b/.github/actions/setup_gh/action.yml @@ -0,0 +1,17 @@ +name: setup gh +description: Set up github's cli gh + +inputs: + os: + description: 'os to use (contains windows or ubuntu)' + default: 'ubuntu' + token: + description: 'token for gh' + +runs: + using: "composite" + steps: + - name: set up secret ubuntu + shell: bash + run: + echo "GH_TOKEN=${{ inputs.token }}" >> $GITHUB_ENV diff --git a/.github/workflows/planned_testing.yml b/.github/workflows/planned_testing.yml index 7571a496d..d970ab690 100644 --- a/.github/workflows/planned_testing.yml +++ b/.github/workflows/planned_testing.yml @@ -30,6 +30,18 @@ on: required: false type: boolean default: false + download_ock_artefact: + required: false + type: string + default: '' + download_dpcpp_artefact: + required: false + type: string + default: '' + download_sycl_cts_artefact: + required: false + type: string + default: '' permissions: packages: read @@ -92,12 +104,18 @@ jobs: steps: - name: Checkout repo uses: actions/checkout@v4 + - name: set up gh + uses: ./.github/actions/setup_gh + with: + os: ${{ contains( matrix.target, 'windows') && 'windows' || 'ubuntu' }} + token: ${{ secrets.GITHUB_TOKEN }} - name: build ock artefact uses: ./.github/actions/do_build_ock_artefact with: target: ${{ matrix.target }} llvm_version: ${{ inputs.llvm_version }} vulkan_sdk_install: false + download_ock_artefact: ${{ inputs.download_ock_artefact }} build_icd: if: inputs.test_tornado || inputs.test_opencl_cts || inputs.test_sycl_cts @@ -209,10 +227,16 @@ jobs: steps: - name: Checkout repo uses: actions/checkout@v4 + - name: set up gh + uses: ./.github/actions/setup_gh + with: + os: ${{ contains( matrix.target, 'windows') && 'windows' || 'ubuntu' }} + token: ${{ secrets.GITHUB_TOKEN }} - name: build dpc++ artefact uses: ./.github/actions/do_build_dpcpp with: target: ${{ matrix.target }} + download_dpcpp_artefact: ${{ inputs.download_dpcpp_artefact }} build_sycl_cts: needs: [workflow_vars, build_icd, build_dpcpp_native_host] @@ -233,10 +257,16 @@ jobs: steps: - name: Checkout repo uses: actions/checkout@v4 - - name: build dpc++ artefact + - name: set up gh + uses: ./.github/actions/setup_gh + with: + os: ${{ contains( matrix.target, 'windows') && 'windows' || 'ubuntu' }} + token: ${{ secrets.GITHUB_TOKEN }} + - name: build sycl cts artefact uses: ./.github/actions/do_build_sycl_cts with: target: ${{ matrix.target }} + download_sycl_cts_artefact: ${{ inputs.download_sycl_cts_artefact }} run_sycl_cts: needs: [workflow_vars, create_ock_artefacts_ubuntu, build_dpcpp_native_host, build_sycl_cts] diff --git a/.github/workflows/planned_testing_caller.yml b/.github/workflows/planned_testing_caller.yml index 56bad6742..f6015a8a4 100644 --- a/.github/workflows/planned_testing_caller.yml +++ b/.github/workflows/planned_testing_caller.yml @@ -30,4 +30,9 @@ jobs: test_opencl_cts: true # Have a pull request setting which can be used to test the flow as best as possible # in a reasonable time + + # The following can be used to download from a previous workflow run (change id) + # download_ock_artefact: host_x86_64_linux=12915462445;host_aarch64_linux=12915462445 + # download_dpcpp_artefact: host_x86_64_linux=12915462445;host_aarch64_linux=12915462445 + # download_sycl_cts_artefact: host_x86_64_linux=12915462445;host_aarch64_linux=12915462445 pull_request: ${{ github.event_name == 'pull_request' }}