diff --git a/README.md b/README.md index 12e0545..ab8062a 100644 --- a/README.md +++ b/README.md @@ -135,11 +135,35 @@ steps: > Note: The `id` defined in the `download/artifact` step must match the `id` defined in the `echo` step (i.e `steps.[ID].outputs.download-path`) +# Limitations + +### Permission Loss + +:exclamation: File permissions are not maintained during artifact upload :exclamation: For example, if you make a file executable using `chmod` and then upload that file, post-download the file is no longer guaranteed to be set as an executable. + +### Case Insensitive Uploads + +:exclamation: File uploads are case insensitive :exclamation: If you upload `A.txt` and `a.txt` with the same root path, only a single file will be saved and available during download. + +### Maintaining file permissions and case sensitive files + +If file permissions and case sensitivity are required, you can `tar` all of your files together before artifact upload. Post download, the `tar` file will maintain file permissions and case sensitivity. + +```yaml + - name: 'Tar files' + run: tar -cvf my_files.tar /path/to/my/directory + + - name: 'Upload Artifact' + uses: actions/upload-artifact@v2 + with: + name: my-artifact + path: my_files.tar +``` + # @actions/artifact package Internally the [@actions/artifact](https://github.com/actions/toolkit/tree/main/packages/artifact) NPM package is used to interact with artifacts. You can find additional documentation there along with all the source code related to artifact download. - # License The scripts and documentation in this project are released under the [MIT License](LICENSE) diff --git a/dist/index.js b/dist/index.js index 429e9ad..1cbb4d5 100644 --- a/dist/index.js +++ b/dist/index.js @@ -3509,7 +3509,6 @@ class DefaultArtifactClient { }); } downloadArtifact(name, path, options) { - var _a; return __awaiter(this, void 0, void 0, function* () { const downloadHttpClient = new download_http_client_1.DownloadHttpClient(); const artifacts = yield downloadHttpClient.listArtifacts(); @@ -3529,7 +3528,7 @@ class DefaultArtifactClient { path = path_1.normalize(path); path = path_1.resolve(path); // During upload, empty directories are rejected by the remote server so there should be no artifacts that consist of only empty directories - const downloadSpecification = download_specification_1.getDownloadSpecification(name, items.value, path, ((_a = options) === null || _a === void 0 ? void 0 : _a.createArtifactFolder) || false); + const downloadSpecification = download_specification_1.getDownloadSpecification(name, items.value, path, (options === null || options === void 0 ? void 0 : options.createArtifactFolder) || false); if (downloadSpecification.filesToDownload.length === 0) { core.info(`No downloadable files were found for the artifact: ${artifactToDownload.name}`); } @@ -4557,11 +4556,12 @@ const utils_1 = __webpack_require__(870); * Used for managing http clients during either upload or download */ class HttpManager { - constructor(clientCount) { + constructor(clientCount, userAgent) { if (clientCount < 1) { throw new Error('There must be at least one client'); } - this.clients = new Array(clientCount).fill(utils_1.createHttpClient()); + this.userAgent = userAgent; + this.clients = new Array(clientCount).fill(utils_1.createHttpClient(userAgent)); } getClient(index) { return this.clients[index]; @@ -4570,7 +4570,7 @@ class HttpManager { // for more information see: https://github.com/actions/http-client/blob/04e5ad73cd3fd1f5610a32116b0759eddf6570d2/index.ts#L292 disposeAndReplaceClient(index) { this.clients[index].dispose(); - this.clients[index] = utils_1.createHttpClient(); + this.clients[index] = utils_1.createHttpClient(this.userAgent); } disposeAndReplaceAllClients() { for (const [index] of this.clients.entries()) { @@ -5929,7 +5929,7 @@ const upload_gzip_1 = __webpack_require__(647); const stat = util_1.promisify(fs.stat); class UploadHttpClient { constructor() { - this.uploadHttpManager = new http_manager_1.HttpManager(config_variables_1.getUploadFileConcurrency()); + this.uploadHttpManager = new http_manager_1.HttpManager(config_variables_1.getUploadFileConcurrency(), 'actions/upload-artifact'); this.statusReporter = new status_reporter_1.StatusReporter(10000); } /** @@ -5947,8 +5947,8 @@ class UploadHttpClient { const artifactUrl = utils_1.getArtifactUrl(); // use the first client from the httpManager, `keep-alive` is not used so the connection will close immediately const client = this.uploadHttpManager.getClient(0); - const requestOptions = utils_1.getUploadRequestOptions('application/json', false); - const rawResponse = yield client.post(artifactUrl, data, requestOptions); + const headers = utils_1.getUploadHeaders('application/json', false); + const rawResponse = yield client.post(artifactUrl, data, headers); const body = yield rawResponse.readBody(); if (utils_1.isSuccessStatusCode(rawResponse.message.statusCode) && body) { return JSON.parse(body); @@ -6060,21 +6060,25 @@ class UploadHttpClient { // for creating a new GZip file, an in-memory buffer is used for compression if (totalFileSize < 65536) { const buffer = yield upload_gzip_1.createGZipFileInBuffer(parameters.file); - let uploadStream; + //An open stream is needed in the event of a failure and we need to retry. If a NodeJS.ReadableStream is directly passed in, + // it will not properly get reset to the start of the stream if a chunk upload needs to be retried + let openUploadStream; if (totalFileSize < buffer.byteLength) { // compression did not help with reducing the size, use a readable stream from the original file for upload - uploadStream = fs.createReadStream(parameters.file); + openUploadStream = () => fs.createReadStream(parameters.file); isGzip = false; uploadFileSize = totalFileSize; } else { // create a readable stream using a PassThrough stream that is both readable and writable - const passThrough = new stream.PassThrough(); - passThrough.end(buffer); - uploadStream = passThrough; + openUploadStream = () => { + const passThrough = new stream.PassThrough(); + passThrough.end(buffer); + return passThrough; + }; uploadFileSize = buffer.byteLength; } - const result = yield this.uploadChunk(httpClientIndex, parameters.resourceUrl, uploadStream, 0, uploadFileSize - 1, uploadFileSize, isGzip, totalFileSize); + const result = yield this.uploadChunk(httpClientIndex, parameters.resourceUrl, openUploadStream, 0, uploadFileSize - 1, uploadFileSize, isGzip, totalFileSize); if (!result) { // chunk failed to upload isUploadSuccessful = false; @@ -6116,7 +6120,7 @@ class UploadHttpClient { failedChunkSizes += chunkSize; continue; } - const result = yield this.uploadChunk(httpClientIndex, parameters.resourceUrl, fs.createReadStream(uploadFilePath, { + const result = yield this.uploadChunk(httpClientIndex, parameters.resourceUrl, () => fs.createReadStream(uploadFilePath, { start, end, autoClose: false @@ -6146,7 +6150,7 @@ class UploadHttpClient { * indicates a retryable status, we try to upload the chunk as well * @param {number} httpClientIndex The index of the httpClient being used to make all the necessary calls * @param {string} resourceUrl Url of the resource that the chunk will be uploaded to - * @param {NodeJS.ReadableStream} data Stream of the file that will be uploaded + * @param {NodeJS.ReadableStream} openStream Stream of the file that will be uploaded * @param {number} start Starting byte index of file that the chunk belongs to * @param {number} end Ending byte index of file that the chunk belongs to * @param {number} uploadFileSize Total size of the file in bytes that is being uploaded @@ -6154,13 +6158,13 @@ class UploadHttpClient { * @param {number} totalFileSize Original total size of the file that is being uploaded * @returns if the chunk was successfully uploaded */ - uploadChunk(httpClientIndex, resourceUrl, data, start, end, uploadFileSize, isGzip, totalFileSize) { + uploadChunk(httpClientIndex, resourceUrl, openStream, start, end, uploadFileSize, isGzip, totalFileSize) { return __awaiter(this, void 0, void 0, function* () { // prepare all the necessary headers before making any http call - const requestOptions = utils_1.getUploadRequestOptions('application/octet-stream', true, isGzip, totalFileSize, end - start + 1, utils_1.getContentRange(start, end, uploadFileSize)); + const headers = utils_1.getUploadHeaders('application/octet-stream', true, isGzip, totalFileSize, end - start + 1, utils_1.getContentRange(start, end, uploadFileSize)); const uploadChunkRequest = () => __awaiter(this, void 0, void 0, function* () { const client = this.uploadHttpManager.getClient(httpClientIndex); - return yield client.sendStream('PUT', resourceUrl, data, requestOptions); + return yield client.sendStream('PUT', resourceUrl, openStream(), headers); }); let retryCount = 0; const retryLimit = config_variables_1.getRetryLimit(); @@ -6238,7 +6242,7 @@ class UploadHttpClient { */ patchArtifactSize(size, artifactName) { return __awaiter(this, void 0, void 0, function* () { - const requestOptions = utils_1.getUploadRequestOptions('application/json', false); + const headers = utils_1.getUploadHeaders('application/json', false); const resourceUrl = new url_1.URL(utils_1.getArtifactUrl()); resourceUrl.searchParams.append('artifactName', artifactName); const parameters = { Size: size }; @@ -6246,7 +6250,7 @@ class UploadHttpClient { core.debug(`URL is ${resourceUrl.toString()}`); // use the first client from the httpManager, `keep-alive` is not used so the connection will close immediately const client = this.uploadHttpManager.getClient(0); - const response = yield client.patch(resourceUrl.toString(), data, requestOptions); + const response = yield client.patch(resourceUrl.toString(), data, headers); const body = yield response.readBody(); if (utils_1.isSuccessStatusCode(response.message.statusCode)) { core.debug(`Artifact ${artifactName} has been successfully uploaded, total size in bytes: ${size}`); @@ -6726,7 +6730,7 @@ const http_manager_1 = __webpack_require__(452); const config_variables_1 = __webpack_require__(401); class DownloadHttpClient { constructor() { - this.downloadHttpManager = new http_manager_1.HttpManager(config_variables_1.getDownloadFileConcurrency()); + this.downloadHttpManager = new http_manager_1.HttpManager(config_variables_1.getDownloadFileConcurrency(), 'actions/download-artifact'); // downloads are usually significantly faster than uploads so display status information every second this.statusReporter = new status_reporter_1.StatusReporter(1000); } @@ -6738,8 +6742,8 @@ class DownloadHttpClient { const artifactUrl = utils_1.getArtifactUrl(); // use the first client from the httpManager, `keep-alive` is not used so the connection will close immediately const client = this.downloadHttpManager.getClient(0); - const requestOptions = utils_1.getDownloadRequestOptions('application/json'); - const response = yield client.get(artifactUrl, requestOptions); + const headers = utils_1.getDownloadHeaders('application/json'); + const response = yield client.get(artifactUrl, headers); const body = yield response.readBody(); if (utils_1.isSuccessStatusCode(response.message.statusCode) && body) { return JSON.parse(body); @@ -6760,8 +6764,8 @@ class DownloadHttpClient { resourceUrl.searchParams.append('itemPath', artifactName); // use the first client from the httpManager, `keep-alive` is not used so the connection will close immediately const client = this.downloadHttpManager.getClient(0); - const requestOptions = utils_1.getDownloadRequestOptions('application/json'); - const response = yield client.get(resourceUrl.toString(), requestOptions); + const headers = utils_1.getDownloadHeaders('application/json'); + const response = yield client.get(resourceUrl.toString(), headers); const body = yield response.readBody(); if (utils_1.isSuccessStatusCode(response.message.statusCode) && body) { return JSON.parse(body); @@ -6818,15 +6822,16 @@ class DownloadHttpClient { let retryCount = 0; const retryLimit = config_variables_1.getRetryLimit(); const destinationStream = fs.createWriteStream(downloadPath); - const requestOptions = utils_1.getDownloadRequestOptions('application/json', true, true); + const headers = utils_1.getDownloadHeaders('application/json', true, true); // a single GET request is used to download a file const makeDownloadRequest = () => __awaiter(this, void 0, void 0, function* () { const client = this.downloadHttpManager.getClient(httpClientIndex); - return yield client.get(artifactLocation, requestOptions); + return yield client.get(artifactLocation, headers); }); // check the response headers to determine if the file was compressed using gzip - const isGzip = (headers) => { - return ('content-encoding' in headers && headers['content-encoding'] === 'gzip'); + const isGzip = (incomingHeaders) => { + return ('content-encoding' in incomingHeaders && + incomingHeaders['content-encoding'] === 'gzip'); }; // Increments the current retry count and then checks if the retry limit has been reached // If there have been too many retries, fail so the download stops. If there is a retryAfterValue value provided, @@ -7302,9 +7307,9 @@ exports.getContentRange = getContentRange; * @param {boolean} isKeepAlive is the same connection being used to make multiple calls * @param {boolean} acceptGzip can we accept a gzip encoded response * @param {string} acceptType the type of content that we can accept - * @returns appropriate request options to make a specific http call during artifact download + * @returns appropriate headers to make a specific http call during artifact download */ -function getDownloadRequestOptions(contentType, isKeepAlive, acceptGzip) { +function getDownloadHeaders(contentType, isKeepAlive, acceptGzip) { const requestOptions = {}; if (contentType) { requestOptions['Content-Type'] = contentType; @@ -7325,7 +7330,7 @@ function getDownloadRequestOptions(contentType, isKeepAlive, acceptGzip) { } return requestOptions; } -exports.getDownloadRequestOptions = getDownloadRequestOptions; +exports.getDownloadHeaders = getDownloadHeaders; /** * Sets all the necessary headers when uploading an artifact * @param {string} contentType the type of content being uploaded @@ -7334,9 +7339,9 @@ exports.getDownloadRequestOptions = getDownloadRequestOptions; * @param {number} uncompressedLength the original size of the content if something is being uploaded that has been compressed * @param {number} contentLength the length of the content that is being uploaded * @param {string} contentRange the range of the content that is being uploaded - * @returns appropriate request options to make a specific http call during artifact upload + * @returns appropriate headers to make a specific http call during artifact upload */ -function getUploadRequestOptions(contentType, isKeepAlive, isGzip, uncompressedLength, contentLength, contentRange) { +function getUploadHeaders(contentType, isKeepAlive, isGzip, uncompressedLength, contentLength, contentRange) { const requestOptions = {}; requestOptions['Accept'] = `application/json;api-version=${getApiVersion()}`; if (contentType) { @@ -7359,9 +7364,9 @@ function getUploadRequestOptions(contentType, isKeepAlive, isGzip, uncompressedL } return requestOptions; } -exports.getUploadRequestOptions = getUploadRequestOptions; -function createHttpClient() { - return new http_client_1.HttpClient('action/artifact', [ +exports.getUploadHeaders = getUploadHeaders; +function createHttpClient(userAgent) { + return new http_client_1.HttpClient(userAgent, [ new auth_1.BearerCredentialHandler(config_variables_1.getRuntimeToken()) ]); } diff --git a/package-lock.json b/package-lock.json index eedb0d7..82ced9e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5,9 +5,9 @@ "requires": true, "dependencies": { "@actions/artifact": { - "version": "0.3.1", - "resolved": "https://registry.npmjs.org/@actions/artifact/-/artifact-0.3.1.tgz", - "integrity": "sha512-czRvOioOpuvmF/qDevfVVpZeBt7pjYlrnmM1+tRuCpKJxjWFYgi5MIW7TfscyupXPvtJz9jIxMjvxy9Eug1QEA==", + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/@actions/artifact/-/artifact-0.3.3.tgz", + "integrity": "sha512-sKC1uA5p6064C6Qypmmt6O8iKlpDyMTfqqDlS4/zfJX1Hs8NbbzPLLN81RpewuJPWQNnroeF52w4VCWypbSNaA==", "dev": true, "requires": { "@actions/core": "^1.2.1", @@ -2627,9 +2627,9 @@ } }, "tmp-promise": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/tmp-promise/-/tmp-promise-2.0.2.tgz", - "integrity": "sha512-zl71nFWjPKW2KXs+73gEk8RmqvtAeXPxhWDkTUoa3MSMkjq3I+9OeknjF178MQoMYsdqL730hfzvNfEkePxq9Q==", + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/tmp-promise/-/tmp-promise-2.1.1.tgz", + "integrity": "sha512-Z048AOz/w9b6lCbJUpevIJpRpUztENl8zdv1bmAKVHimfqRFl92ROkmT9rp7TVBnrEw2gtMTol/2Cp2S2kJa4Q==", "dev": true, "requires": { "tmp": "0.1.0" diff --git a/package.json b/package.json index 15d5b82..2887ca0 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,7 @@ }, "homepage": "https://github.com/actions/download-artifact#readme", "devDependencies": { - "@actions/artifact": "^0.3.1", + "@actions/artifact": "^0.3.3", "@actions/core": "^1.2.4", "@types/node": "^12.12.6", "@typescript-eslint/parser": "^2.30.0",