feat: wait for artifact to become available

This commit is contained in:
Ivan Dlugos 2024-08-15 12:02:25 +02:00
parent fa0a91b85d
commit e7141b6a94
6 changed files with 31303 additions and 37483 deletions

View file

@ -128,3 +128,81 @@ jobs:
Write-Error "File contents of downloaded artifacts are incorrect"
}
shell: pwsh
# Test "wait-until-available" functionality by running two jobs, one requiring artifacts of another
test-wait-producer:
name: 'Test: wait (producer)'
strategy:
matrix:
runs-on: [ubuntu-latest, macos-latest, windows-latest]
fail-fast: false
runs-on: ${{ matrix.runs-on }}
steps:
# TODO find a better way to ensure the "consumer" job started before the artifact is produced.
# Maybe run the download in the background process in the same job and checking it's result after upload-artifact.
- run: sleep 300
# Test "wait until available"end-to-end by uploading two artifacts and then downloading them
- name: Create artifacts
run: echo "Lorem ipsum dolor sit amet" > file-A.txt
- uses: actions/upload-artifact@v4
with:
name: 'Artifact-wait-${{ matrix.runs-on }}'
path: file-A.txt
test-wait-consumer:
name: 'Test: wait (consumer)'
strategy:
matrix:
runs-on: [ubuntu-latest, macos-latest, windows-latest]
fail-fast: false
runs-on: ${{ matrix.runs-on }}
steps:
- uses: actions/checkout@v4
- name: Setup Node 20
uses: actions/setup-node@v4
with:
node-version: 20.x
cache: 'npm'
- run: npm install
- run: npm run build
# Test downloading a single artifact
- name: Download artifact A
uses: ./
with:
name: 'Artifact-wait-${{ matrix.runs-on }}'
path: some/new/path
wait-timeout: 600
# Test downloading an artifact using tilde expansion
- name: Download artifact A
uses: ./
with:
name: 'Artifact-wait-${{ matrix.runs-on }}'
path: ~/some/path/with/a/tilde
# no need for a timeout here
- name: Verify successful download
run: |
$file1 = "some/new/path/file-A.txt"
$file2 = "~/some/path/with/a/tilde/file-A.txt"
if(!(Test-Path -path $file1) -or !(Test-Path -path $file2))
{
Write-Error "Expected files do not exist"
}
if(!((Get-Content $file1) -ceq "Lorem ipsum dolor sit amet") -or !((Get-Content $file2) -ceq "Lorem ipsum dolor sit amet"))
{
Write-Error "File contents of downloaded artifacts are incorrect"
}
shell: pwsh

View file

@ -233,6 +233,43 @@ steps:
run-id: 1234
```
## Waiting for the artifact to be available
You can specify `wait-timeout` (seconds) to instruct the download-artifact action to retry until the artifact is available.
This is useful if you want to launch the job before its dependency job has finished, e.g. if the dependant requires some time-consuming steps.
You can do this by removing the `needs` dependency and relying on the retry logic of download-artifact to fetch the artifact after it's uploaded.
Beware that GitHub actions come with a limited number of runners available, so if your workflow uses up the limt on the "dependant" jobs,
your artifact-source jobs may never be scheduled.
```yaml
jobs:
producer-job:
name: This job produces an artifact
steps:
# ... do something
# ... then upload an artifact as usual
- uses: actions/upload-artifact@v4
with:
name: artifact-name
path: path/to/artifact
dependant-job:
name: This job has some long running preparation that can run before the artifact is necessary
steps:
# your long-running steps come first - they're run in parallel with the `producer-job` above (given you have enouth GH actions runners available)
run: # e.g. install some large SDK
# then when you finally need the artifact to be downloaded
uses: actions/download-artifact@v4
with:
name: artifact-name
path: output-path
# wait for 300 seconds
wait-timeout: 300
```
## Limitations
### Permission Loss

View file

@ -32,6 +32,9 @@ inputs:
If github-token is specified, this is the run that artifacts will be downloaded from.'
required: false
default: ${{ github.run_id }}
wait-timeout:
description: 'Wait for the artifact to become available (timeout in seconds)'
required: false
outputs:
download-path:
description: 'Path of artifact download'

68149
dist/index.js vendored

File diff suppressed because one or more lines are too long

View file

@ -5,7 +5,8 @@ export enum Inputs {
Repository = 'repository',
RunID = 'run-id',
Pattern = 'pattern',
MergeMultiple = 'merge-multiple'
MergeMultiple = 'merge-multiple',
WaitTimeout = 'wait-timeout'
}
export enum Outputs {

View file

@ -23,6 +23,7 @@ async function run(): Promise<void> {
repository: core.getInput(Inputs.Repository, {required: false}),
runID: parseInt(core.getInput(Inputs.RunID, {required: false})),
pattern: core.getInput(Inputs.Pattern, {required: false}),
waitTimeout: core.getInput(Inputs.WaitTimeout, {required: false}),
mergeMultiple: core.getBooleanInput(Inputs.MergeMultiple, {required: false})
}
@ -60,10 +61,35 @@ async function run(): Promise<void> {
if (isSingleArtifactDownload) {
core.info(`Downloading single artifact`)
const {artifact: targetArtifact} = await artifactClient.getArtifact(
inputs.name,
options
const downloadFn = () => artifactClient.getArtifact(inputs.name, options)
const waitAndDownload = async <T>(action: () => T) => {
const waitUntil = Date.now() + parseInt(inputs.waitTimeout) * 1000
let lastError
do {
try {
return await action()
} catch (e) {
lastError = e
core.info(
'Waiting for the artifact to become available... ' +
`Remaining time until timeout: ${Math.max(
0,
Math.floor((waitUntil - Date.now()) / 1000)
)} seconds`
)
await new Promise(f => setTimeout(f, 10000))
}
} while (Date.now() < waitUntil)
throw Error(
'Waiting for the artifact has timed out. Latest error was: ' + lastError
)
}
const {artifact: targetArtifact} =
inputs.waitTimeout === ''
? await downloadFn()
: await waitAndDownload(() => downloadFn())
if (!targetArtifact) {
throw new Error(`Artifact '${inputs.name}' not found`)
@ -75,6 +101,12 @@ async function run(): Promise<void> {
artifacts = [targetArtifact]
} else {
if (inputs.waitTimeout !== '') {
core.warning(
`Waiting for multiple artifact (i.e. specifying non-zero wait-timeout: '${inputs.waitTimeout}') is not supported.`
)
}
const listArtifactResponse = await artifactClient.listArtifacts({
latest: true,
...options