Merge pull request #629 from coldav/colin/add_reusing_artefacts

Reuse previous workflow run's artefacts

.github/actions/do_build_dpcpp/action.yml

@@ -4,8 +4,8 @@ description: build dpc++
 inputs:
   target:
     description: 'target architecture'
-  create_dpcpp_artefact_method:
-    description: 'build | download_release' # TODO: add support for 'previous workflow' download
+  download_dpcpp_artefact:
+    description: 'download ock artefact rather than building, of form <target>=id;<target2=id2>. Special value of download_release applies to all targets.'
     type: string
     default: "build"
 
@@ -22,14 +22,14 @@ runs:
       uses: llvm/actions/install-ninja@main
 
     - name: clone dpc++
-      if: inputs.create_dpcpp_artefact_method == 'build'
+      if: contains(inputs.download_dpcpp_artefact, inputs.target) != true
       uses: actions/checkout@v4
       with:
         repository: intel/llvm
         path: llvm
 
     - name: dpcpp configure
-      if: inputs.create_dpcpp_artefact_method == 'build'
+      if: contains(inputs.download_dpcpp_artefact, inputs.target) != true
       shell: bash
       run:
         cd llvm; python3 buildbot/configure.py
@@ -42,19 +42,19 @@ runs:
         --cmake-opt=-DLLVM_ENABLE_ZSTD=OFF
 
     - name: build sycl-headers
-      if: inputs.create_dpcpp_artefact_method == 'build'
+      if: contains(inputs.download_dpcpp_artefact, inputs.target) != true
       shell: bash
       run:
         cmake --build llvm/build -- sycl-headers
 
     - name: build dpc plus plus
-      if: inputs.create_dpcpp_artefact_method == 'build'
+      if: contains(inputs.download_dpcpp_artefact, inputs.target) != true
       shell: bash
       run:
        python3 llvm/buildbot/compile.py -o llvm/build -v -j 8
 
     - name: build extra utilties
-      if: inputs.create_dpcpp_artefact_method == 'build'
+      if: contains(inputs.download_dpcpp_artefact, inputs.target) != true
       # Build various utilities, since those aren't proper dependencies.
       # FileCheck and not are needed for tests. The rest are needed for
       # cross builds. They are enabled on all targets for consistency.
@@ -65,7 +65,7 @@ runs:
         opt prepare_builtins -j8
 
     - name: copy utilities
-      if: inputs.create_dpcpp_artefact_method == 'build'
+      if: contains(inputs.download_dpcpp_artefact, inputs.target) != true
       shell: bash
       run:
         cd llvm/build/bin;
@@ -75,7 +75,7 @@ runs:
     # TODO: For now just linux x86_64
     #       Review for location of components (paths) and any archive package/unpackage reqs.
     #- name: install config files to pick up libraries for cross compilation.
-    #  if: inputs.create_dpcpp_artefact_method == 'build'
+    #  if: contains(inputs.download_dpcpp_artefact, inputs.target) != true
     #  shell: bash
     #  run: |
     #    echo Installing configuration files
@@ -89,8 +89,9 @@ runs:
     #      " >../install/bin/$arch-unknown-linux-gnu.cfg;
     #    done
 
-    - name: download dpc plus plus
-      if: inputs.create_dpcpp_artefact_method == 'download_release'
+    - name: download dpc plus plus from official releases
+      # TODO: This is a bit imperfect as it should parse it properly
+      if: contains(inputs.download_dpcpp_artefact, inputs.target) && contains(inputs.download_dpcpp_artefact, 'download_release')
       shell: bash
       run: |
         mkdir -p llvm/build/install
@@ -103,8 +104,24 @@ runs:
         tar xf sycl_linux.tar.gz
         rm sycl_linux.tar.gz
 
+    - name: download previous dpcpp if needed ${{ inputs.target }} ${{ matrix.download_dpcpp_artefact}}
+      shell: bash      
+      if: contains(inputs.download_dpcpp_artefact, inputs.target) && contains(inputs.download_dpcpp_artefact, 'download_release') != true
+      run: |
+        download_id=`echo "${{inputs.download_dpcpp_artefact}}"`
+        echo "echo $download_id | sed 's/.*${{inputs.target}}=//' | sed 's/;.*//'"
+        download_id=`echo $download_id | sed 's/.*${{inputs.target}}=//' | sed 's/;.*//'`
+        echo download id is "'$download_id'"
+        # TODO : make this work on windows
+        mkdir -p llvm/build/install
+        git config --global --add safe.directory $PWD        
+        gh run download $download_id -n dpcpp_${{ inputs.target }} -D llvm/build/install
+        ls llvm/build/install
+
+
     - name: package artefacts  # package/unpackage avoids known 'permissions loss' issue
       shell: bash
+      if: contains(inputs.download_dpcpp_artefact, inputs.target) != true
       run: |
         cd llvm/build/install
         tar cf dpcpp.tar *

.github/actions/do_build_ock_artefact/action.yml

@@ -12,6 +12,9 @@ inputs:
   vulkan_sdk_install:
     description: 'vulkan install flag'
     default: true
+  download_ock_artefact:
+    description: 'download ock artefact rather than building, of form <target>=id;<target2=id2>.'
+    default: ''
 
 # TODO: This has not been tested yet on windows so would likely need some updating.
 runs:
@@ -32,23 +35,24 @@ runs:
       # installs tools, ninja, installs llvm and sets up ccache
       - name: setup
         uses:  ./.github/actions/setup_build
+        if: contains(inputs.download_ock_artefact, inputs.target) != true
         with:
           llvm_version: ${{ inputs.llvm_version }}
           llvm_build_type: RelAssert
           cross_arch: ${{ steps.calc_vars.outputs.arch == 'x86_64' && 'none' || steps.calc_vars.outputs.arch }}
           os: ${{ contains(inputs.target, 'windows') && 'windows' || 'ubuntu' }}
 
       - name: build ock x86
-        if: steps.calc_vars.outputs.arch  == 'x86_64'
+        if: steps.calc_vars.outputs.arch  == 'x86_64' && contains(inputs.download_ock_artefact, inputs.target) != true
         uses: ./.github/actions/do_build_ock
         with:
           build_targets: install
           offline_kernel_tests: OFF
           extra_flags: -DCA_ENABLE_TESTS=OFF -DCA_ENABLE_EXAMPLES=OFF -DCA_ENABLE_DOCUMENTATION=OFF
           shell_to_use: ${{ contains(inputs.target, 'windows') && 'pwsh' || 'bash' }}
 
-      - name: build ock other ${{ matrix.target }}
-        if: steps.calc_vars.outputs.arch  != 'x86_64'
+      - name: build ock other ${{ inputs.target }}
+        if: steps.calc_vars.outputs.arch  != 'x86_64' && contains(inputs.download_ock_artefact, inputs.target) != true
         uses: ./.github/actions/do_build_ock
         with:
           build_targets: install
@@ -58,6 +62,19 @@ runs:
           offline_kernel_tests: OFF
           host_fp16: ON
 
+      - name: download previous ock if needed ${{ inputs.target }} ${{ matrix.download_dpcpp_artefact}}
+        shell: bash      
+        if: contains(inputs.download_ock_artefact, inputs.target)
+        run: |
+          download_id=`echo "${{inputs.download_ock_artefact}}"`
+          echo "echo $download_id | sed 's/.*${{inputs.target}}=//' | sed 's/;.*//'"
+          download_id=`echo $download_id | sed 's/.*${{inputs.target}}=//' | sed 's/;.*//'`
+          echo download id is "'$download_id'"
+          git config --global --add safe.directory $PWD
+          # TODO : make this work on windows
+          gh run download $download_id -n ock_${{ inputs.target }} -D install
+          ls install
+
       # Prune it as there is too much things in there we don't want to use
       # Todo: move this logic to cmake settings so that we build only what we
       # want to install. As time goes on we may want to install more.

.github/actions/do_build_sycl_cts/action.yml

@@ -4,6 +4,9 @@ description: build sycl cts
 inputs:
   target:
     description: 'target architecture'
+  download_sycl_cts_artefact:
+    description: 'download ock artefact rather than building, of form <target>=id;<target2=id2>.'
+    default: ''    
 
 runs:
   using: "composite"
@@ -15,41 +18,48 @@ runs:
         target: ${{ inputs.target }}  
 
     - name: Install Ninja
+      if: contains(inputs.download_sycl_cts_artefact, inputs.target) != true
       uses: llvm/actions/install-ninja@main
 
     - name: download icd artifact
+      if: contains(inputs.download_sycl_cts_artefact, inputs.target) != true
       uses: actions/download-artifact@v4
       with:
         name: icd_${{inputs.target}}
         path: install_icd
 
     - name: download headers artifact
+      if: contains(inputs.download_sycl_cts_artefact, inputs.target) != true
       uses: actions/download-artifact@v4
       with:
         name: header_${{inputs.target}}
         path: install_headers
 
     - name: download dpc++ artifact
+      if: contains(inputs.download_sycl_cts_artefact, inputs.target) != true
       uses: actions/download-artifact@v4
       with:
         name: dpcpp_${{inputs.target}}
         path: install_dpcpp
 
     - name: unpackage dpc++ artifacts # package/unpackage avoids known 'permissions loss' issue
+      if: contains(inputs.download_sycl_cts_artefact, inputs.target) != true
       shell: bash
       run: |
         cd install_dpcpp
         tar xf dpcpp.tar
         rm dpcpp.tar
 
     - name: checkout sycl cts
+      if: contains(inputs.download_sycl_cts_artefact, inputs.target) != true
       uses: actions/checkout@v4
       with:
         repository: KhronosGroup/SYCL-CTS
         path: SYCL-CTS.src
         submodules: true
 
     - name: build SYCL CTS
+      if: contains(inputs.download_sycl_cts_artefact, inputs.target) != true    
       shell: bash
       run: |
         echo calling cmake and ninja on SYCL CTS
@@ -70,16 +80,32 @@ runs:
         ninja -C SYCL-CTS -v -j4 -k 0 || :
 
     - name: package artefacts  # package/unpackage avoids known 'permissions loss' issue
+      if: contains(inputs.download_sycl_cts_artefact, inputs.target) != true    
       shell: bash
       run: |
         cd SYCL-CTS
         # only bin
         tar cf sycl-cts.tar bin
 
+    - name: download sycl cts artefact  # package/unpackage avoids known 'permissions loss' issue
+      if: contains(inputs.download_sycl_cts_artefact, inputs.target)
+      shell: bash
+      run: |
+        download_id=`echo "${{inputs.download_sycl_cts_artefact}}"`
+        echo "echo $download_id | sed 's/.*${{inputs.target}}=//' | sed 's/;.*//'"
+        download_id=`echo $download_id | sed 's/.*${{inputs.target}}=//' | sed 's/;.*//'`
+        echo download id is "'$download_id'"
+        # TODO : make this work on windows
+        mkdir -p SYCL-CTS
+        git config --global --add safe.directory $PWD        
+        gh run download $download_id -n sycl_cts_${{ inputs.target }} -D SYCL-CTS
+        ls SYCL-CTS
+
+
     - name: upload artefact
       uses: actions/upload-artifact@v4
       with:
         name: sycl_cts_${{inputs.target}}
         path: SYCL-CTS/sycl-cts.tar
-        retention-days: 1
+        retention-days: 2
 

.github/actions/setup_gh/action.yml

@@ -0,0 +1,17 @@
+name: setup gh
+description: Set up github's cli gh
+
+inputs:
+  os:
+    description: 'os to use (contains windows or ubuntu)'
+    default: 'ubuntu'
+  token:
+    description: 'token for gh'
+
+runs:
+  using: "composite"
+  steps:
+      - name: set up secret ubuntu
+        shell: bash
+        run:
+          echo "GH_TOKEN=${{ inputs.token }}" >> $GITHUB_ENV

.github/workflows/planned_testing.yml

@@ -30,6 +30,18 @@ on:
         required: false
         type: boolean
         default: false
+      download_ock_artefact:
+        required: false
+        type: string
+        default: ''
+      download_dpcpp_artefact:
+        required: false
+        type: string
+        default: ''
+      download_sycl_cts_artefact:
+        required: false
+        type: string
+        default: ''
 
 permissions:
   packages: read
@@ -92,12 +104,18 @@ jobs:
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
+      - name: set up gh
+        uses: ./.github/actions/setup_gh
+        with:
+          os: ${{ contains( matrix.target, 'windows') && 'windows' || 'ubuntu' }}
+          token: ${{ secrets.GITHUB_TOKEN }}
       - name: build ock artefact
         uses: ./.github/actions/do_build_ock_artefact
         with:
           target: ${{ matrix.target }}
           llvm_version: ${{ inputs.llvm_version }}
           vulkan_sdk_install: false
+          download_ock_artefact: ${{ inputs.download_ock_artefact }}
 
   build_icd:
     if: inputs.test_tornado || inputs.test_opencl_cts || inputs.test_sycl_cts
@@ -209,10 +227,16 @@ jobs:
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
+      - name: set up gh
+        uses: ./.github/actions/setup_gh
+        with:
+          os: ${{ contains( matrix.target, 'windows') && 'windows' || 'ubuntu' }}
+          token: ${{ secrets.GITHUB_TOKEN }}
       - name: build dpc++ artefact
         uses: ./.github/actions/do_build_dpcpp
         with:
           target: ${{ matrix.target }}
+          download_dpcpp_artefact: ${{ inputs.download_dpcpp_artefact }}
 
   build_sycl_cts:
     needs: [workflow_vars, build_icd, build_dpcpp_native_host]
@@ -233,10 +257,16 @@ jobs:
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
-      - name: build dpc++ artefact
+      - name: set up gh
+        uses: ./.github/actions/setup_gh
+        with:
+          os: ${{ contains( matrix.target, 'windows') && 'windows' || 'ubuntu' }}
+          token: ${{ secrets.GITHUB_TOKEN }}        
+      - name: build sycl cts artefact
         uses: ./.github/actions/do_build_sycl_cts
         with:
           target: ${{ matrix.target }}
+          download_sycl_cts_artefact: ${{ inputs.download_sycl_cts_artefact }}
 
   run_sycl_cts:
     needs: [workflow_vars, create_ock_artefacts_ubuntu, build_dpcpp_native_host, build_sycl_cts]

.github/workflows/planned_testing_caller.yml

@@ -30,4 +30,9 @@ jobs:
       test_opencl_cts: true
       # Have a pull request setting which can be used to test the flow as best as possible
       # in a reasonable time
+
+      # The following can be used to download from a previous workflow run (change id)
+      # download_ock_artefact: host_x86_64_linux=12915462445;host_aarch64_linux=12915462445
+      # download_dpcpp_artefact: host_x86_64_linux=12915462445;host_aarch64_linux=12915462445
+      # download_sycl_cts_artefact: host_x86_64_linux=12915462445;host_aarch64_linux=12915462445      
       pull_request: ${{ github.event_name == 'pull_request' }}