Zos download attributes

packages/cli/CHANGELOG.md

@@ -2,6 +2,10 @@
 
 All notable changes to the Zowe CLI package will be documented in this file.
 
+## Recent Changes
+
+- Enhancement: Added optional `--attributes` flag to `zowe zos-files download uss-file` to allow passing a .zosattributes file path for download encoding format. [#2322](https://github.com/zowe/zowe-cli/issues/2322)
+
 ## `8.7.0`
 
 - Enhancement: Added --wait-for-active and --wait-for-output to download options on zosjobs. [#2328](https://github.com/zowe/zowe-cli/pull/2328)
@@ -18,6 +22,7 @@ All notable changes to the Zowe CLI package will be documented in this file.
 
 - Enhancement: Added support for running applications on TSO/E address spaces. Start applications and receive/transmit messages using the new `tso start`, `tso receive` and `tso send` commands. [#2280](https://github.com/zowe/zowe-cli/pull/2280)
 
+
 ## `8.4.0`
 
 - Enhancement: Added optional `--attributes` flag to `zowe zos-files upload file-to-uss` to allow passing a .zosattributes file path for upload encoding format. [#2319](https://github.com/zowe/zowe-cli/pull/2319)

packages/cli/__tests__/zosfiles/__unit__/download/uss/__snapshots__/UssFile.definition.unit.test.ts.snap

@@ -2,6 +2,18 @@
 
 exports[`zos-files download uss-file command definition should not have changed 1`] = `
 Array [
+  Object {
+    "aliases": Array [
+      "attrs",
+    ],
+    "conflictsWith": Array [
+      "binary",
+      "record",
+    ],
+    "description": "Path of an attributes file to control how files are downloaded.",
+    "name": "attributes",
+    "type": "existingLocalFile",
+  },
   Object {
     "aliases": Array [
       "b",

packages/cli/__tests__/zosjobs/__system__/download/cli.zos-jobs.download.output.system.test.ts

@@ -14,8 +14,6 @@ import { ITestEnvironment } from "../../../../../../__tests__/__src__/environmen
 import { runCliScript } from "@zowe/cli-test-utils";
 import { ITestPropertiesSchema } from "../../../../../../__tests__/__src__/properties/ITestPropertiesSchema";
 import * as fs from "fs";
-import { Session } from "@zowe/imperative";
-import { GetJobs } from "@zowe/zos-jobs-for-zowe-sdk";
 
 // Test Environment populated in the beforeAll();
 let TEST_ENVIRONMENT: ITestEnvironment<ITestPropertiesSchema>;
@@ -49,14 +47,14 @@ describe("zos-jobs download output command", () => {
         it("should download a job and wait for it to reach output status", async () => {
             const response = runCliScript(__dirname + "/__scripts__/download-output/download_job_wait_for_output.sh",
                 TEST_ENVIRONMENT, [IEFBR14_JCL]);
-                expect(response.stderr.toString()).toBe("");
-                expect(response.status).toBe(0);
+            expect(response.stderr.toString()).toBe("");
+            expect(response.status).toBe(0);
         });
         it("should download a job and wait for it to reach active status", async () => {
             const response = runCliScript(__dirname + "/__scripts__/download-output/download_job_wait_for_active.sh",
                 TEST_ENVIRONMENT, [IEFBR14_JCL]);
-                expect(response.stderr.toString()).toBe("");
-                expect(response.status).toBe(0);
+            expect(response.stderr.toString()).toBe("");
+            expect(response.status).toBe(0);
         });
     });
     describe("output", () => {

packages/cli/src/zosfiles/download/uss/UssFile.definition.ts

@@ -39,6 +39,7 @@ export const UssFileDefinition: ICommandDefinition = {
         }
     ],
     options: [
+        DownloadOptions.attributes,
         DownloadOptions.file,
         DownloadOptions.binary,
         DownloadOptions.encoding

packages/cli/src/zosfiles/download/uss/UssFile.handler.ts

@@ -11,7 +11,7 @@
 
 import { ZosFilesBaseHandler } from "../../ZosFilesBase.handler";
 import { AbstractSession, IHandlerParameters, ITaskWithStatus, TaskStage } from "@zowe/imperative";
-import { Download, IZosFilesResponse } from "@zowe/zos-files-for-zowe-sdk";
+import { Download, IDownloadOptions, IZosFilesResponse, ZosFilesAttributes } from "@zowe/zos-files-for-zowe-sdk";
 
 /**
  * Handler to download an uss file
@@ -25,13 +25,20 @@ export default class UssFileHandler extends ZosFilesBaseHandler {
             stageName: TaskStage.IN_PROGRESS
         };
         commandParameters.response.progress.startBar({task});
-        return Download.ussFile(session, commandParameters.arguments.ussFileName, {
+
+        const attributes = ZosFilesAttributes.loadFromFile(
+            commandParameters.arguments.attributes,
+        );
+        const downloadOptions: IDownloadOptions = {
             binary: commandParameters.arguments.binary,
             encoding: commandParameters.arguments.encoding,
             file: commandParameters.arguments.file,
             task,
             responseTimeout: commandParameters.arguments.responseTimeout,
-            overwrite: commandParameters.arguments.overwrite
-        });
+            overwrite: commandParameters.arguments.overwrite,
+            ...attributes != null && { attributes }
+        };
+
+        return Download.ussFile(session, commandParameters.arguments.ussFileName, downloadOptions);
     }
 }

packages/zosfiles/CHANGELOG.md

@@ -2,14 +2,18 @@
 
 All notable changes to the Zowe z/OS files SDK package will be documented in this file.
 
+## Recent Changes
+
+- Enhancement: Added optional `--attributes` flag to `zowe zos-files download uss-file` to allow passing a .zosattributes file path for download encoding format. [#2322](https://github.com/zowe/zowe-cli/pull/2322)
+
 ## `8.6.2`
 
 - BugFix: Resolved issue where encoding argument was missing from `FileToUss.handler.ts` options object. [#2234](https://github.com/zowe/zowe-cli/pull/2334)
 - BugFix: Resolved issue where `FileToUss.handler.ts` options object was not properly passed through subsequent command calls. [#2234](https://github.com/zowe/zowe-cli/pull/2334)
 
 ## `8.4.0`
 
-- Enhancement: Added optional `--attributes` flag to `zowe zos-files upload file-to-uss` to allow passing a .zosattributes file path for upload encoding format. [#2319] (https://github.com/zowe/zowe-cli/pull/2319)
+- Enhancement: Added optional `--attributes` flag to `zowe zos-files upload file-to-uss` to allow passing a .zosattributes file path for upload encoding format. [#2319](https://github.com/zowe/zowe-cli/pull/2319)
 
 ## `8.2.0`
 

packages/zosfiles/__tests__/__system__/methods/download/Download.system.test.ts

@@ -24,7 +24,7 @@ import {
     IMountFsOptions,
     Mount,
     Unmount,
-    IUSSListOptions
+    IUSSListOptions,
 } from "../../../../src";
 import { Imperative, IO, Session } from "@zowe/imperative";
 import { inspect } from "util";
@@ -37,6 +37,7 @@ import { posix } from "path";
 import { Shell } from "@zowe/zos-uss-for-zowe-sdk";
 import { PassThrough } from "stream";
 import { text } from "stream/consumers";
+import { runCliScript } from "@zowe/cli-test-utils";
 
 const rimraf = require("rimraf").sync;
 const delayTime = 2000;
@@ -1081,6 +1082,52 @@ describe.each([false, true])("Download Data Set - Encoded: %s", (encoded: boolea
 
             });
 
+            it("should upload and download file with correct encoding", async () => {
+                let response: any;
+
+                // Upload binary or encoded file
+                const uploadFile: string = encoded ? "downloadEncodingCheck.txt" : "downloadEncodingCheckBinary.txt";
+                let downloadResponse: any;
+                let error: any;
+
+                // Use text file as to get proper response from getRemoteEncoding()
+                const ussnameAsTxt = ussname + ".txt";
+                try {
+                    response = runCliScript(__dirname +
+                        `/__resources__/${encoded ? "upload_file_to_uss.sh" : "upload_file_to_uss_binary.sh"}`, testEnvironment, [
+                        defaultSystem.tso.account,
+                        defaultSystem.zosmf.host,
+                        defaultSystem.zosmf.port,
+                        defaultSystem.zosmf.user,
+                        defaultSystem.zosmf.password,
+                        defaultSystem.zosmf.rejectUnauthorized,
+                        __dirname + "/__resources__/testfiles/" + uploadFile,
+                        ussnameAsTxt,
+                        encoded ? "1047" : true
+                    ]);
+                    downloadResponse = runCliScript(__dirname + "/__resources__/download_file.sh", testEnvironment, [
+                        defaultSystem.tso.account,
+                        defaultSystem.zosmf.host,
+                        defaultSystem.zosmf.port,
+                        defaultSystem.zosmf.user,
+                        defaultSystem.zosmf.password,
+                        defaultSystem.zosmf.rejectUnauthorized,
+                        ussnameAsTxt,
+                        __dirname + `/__resources__/${encoded ? ".zosattributes" : ".zosattributes-binary"}`
+                    ]);
+                } catch (err) {
+                    error = err;
+                }
+
+                expect(error).toBeFalsy();
+                expect(response).toBeTruthy();
+                expect(downloadResponse).toBeTruthy();
+
+                // Compare the downloaded contents to those uploaded
+                const fileContents = fs.readFileSync(`${testEnvironment.workingDir}/${posix.basename(ussnameAsTxt)}`).toString();
+                expect(fileContents).toEqual(fs.readFileSync(__dirname + "/__resources__/testfiles/" + uploadFile).toString());
+            });
+
             // When requesting etag, z/OSMF has a limit on file size when it stops to return etag by default (>8mb)
             // We are passing X-IBM-Return-Etag to force z/OSMF to always return etag, but testing here for case where it would be optional
             it("should download a 10mb uss file and return Etag", async () => {

packages/zosfiles/__tests__/__system__/methods/download/__resources__/.zosattributes

@@ -0,0 +1,5 @@
+*.json -
+*.bin binary binary
+*.jcl IBM-1047 IBM-1047
+*.md UTF-8 UTF-8
+*.txt UTF-8 IBM-1047

packages/zosfiles/__tests__/__system__/methods/download/__resources__/.zosattributes-binary

@@ -0,0 +1,5 @@
+*.json -
+*.bin binary binary
+*.jcl IBM-1047 IBM-1047
+*.md UTF-8 UTF-8
+*.txt binary binary

packages/zosfiles/__tests__/__system__/methods/download/__resources__/download_file.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+account=$1
+host=$2
+port=$3
+user=$4
+password=$5
+ru=$6
+fileToDownload=$7
+attributes=$8
+
+zowe zos-files download uss $fileToDownload --attributes $attributes --host $host --port $port --user $user --password $password --ru $ru
+exit $?

packages/zosfiles/__tests__/__system__/methods/download/__resources__/testfiles/downloadEncodingCheck.txt

@@ -0,0 +1 @@
+abcdefghijklmnopqrstuvwxyz

packages/zosfiles/__tests__/__system__/methods/download/__resources__/testfiles/downloadEncodingCheckBinary.txt

@@ -0,0 +1 @@
+[][][]

packages/zosfiles/__tests__/__system__/methods/download/__resources__/upload_file_to_uss.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+account=$1
+host=$2
+port=$3
+user=$4
+password=$5
+ru=$6
+inputFile=$7
+ussName=$8
+encoding=$9
+
+zowe zos-files upload file-to-uss $inputFile $ussName --encoding $encoding --host $host --port $port --user $user --password $password --ru $ru
+exit $?

packages/zosfiles/__tests__/__system__/methods/download/__resources__/upload_file_to_uss_binary.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+account=$1
+host=$2
+port=$3
+user=$4
+password=$5
+ru=$6
+inputFile=$7
+ussName=$8
+binary=$9
+
+zowe zos-files upload file-to-uss $inputFile $ussName --binary $binary --host $host --port $port --user $user --password $password --ru $ru
+exit $?

packages/zosfiles/__tests__/__unit__/methods/download/Download.unit.test.ts

@@ -2184,6 +2184,90 @@ describe("z/OS Files - Download", () => {
             expect(ioCreateDirSpy).not.toHaveBeenCalled();
             expect(ioWriteStreamSpy).not.toHaveBeenCalled();
         });
+        it("should download uss file using .zosattributes file", async () => {
+            let response;
+            let caughtError;
+            const destination = localFileName;
+            const zosAttributes = Object.create(ZosFilesAttributes.prototype);
+            zosAttributes.attributes = new Map([
+                ['*.json', { ignore: true }],
+                ['*.bin', { ignore: false, localEncoding: 'binary', remoteEncoding: 'binary' }],
+                ['*.jcl', { ignore: false, localEncoding: 'IBM-1047', remoteEncoding: 'IBM-1047' }],
+                ['*.md', { ignore: false, localEncoding: 'UTF-8', remoteEncoding: 'UTF-8' }],
+                ['*.txt', { ignore: false, localEncoding: 'UTF-8', remoteEncoding: 'IBM-1047' }]
+            ]);
+            try {
+                response = await Download.ussFile(dummySession, ussname, { attributes: zosAttributes });
+            } catch (e) {
+                caughtError = e;
+            }
+
+            const endpoint = path.posix.join(ZosFilesConstants.RESOURCE, ZosFilesConstants.RES_USS_FILES, encodeURIComponent(ussname.substring(1)));
+
+            expect(caughtError).toBeUndefined();
+            expect(response).toEqual({
+                success: true,
+                commandResponse: util.format(ZosFilesMessages.ussFileDownloadedWithDestination.message, destination),
+                apiResponse: {}
+            });
+
+            expect(zosmfGetFullSpy).toHaveBeenCalledTimes(1);
+            expect(zosmfGetFullSpy).toHaveBeenCalledWith(dummySession, {
+                resource: endpoint,
+                reqHeaders: [{ "X-IBM-Data-Type": "text;fileEncoding=IBM-1047" }, ZosmfHeaders.ACCEPT_ENCODING, ZosmfHeaders.TEXT_PLAIN],
+                responseStream: fakeStream,
+                normalizeResponseNewLines: true,
+                task: undefined /* no progress task */
+            });
+
+            expect(ioCreateDirSpy).toHaveBeenCalledTimes(1);
+            expect(ioCreateDirSpy).toHaveBeenCalledWith(destination);
+
+            expect(ioWriteStreamSpy).toHaveBeenCalledTimes(1);
+            expect(ioWriteStreamSpy).toHaveBeenCalledWith(destination);
+        });
+        it("should download uss file using .zosattributes file - binary", async () => {
+            let response;
+            let caughtError;
+            const destination = localFileName;
+            const zosAttributes = Object.create(ZosFilesAttributes.prototype);
+            zosAttributes.attributes = new Map([
+                ['*.json', { ignore: true }],
+                ['*.bin', { ignore: false, localEncoding: 'binary', remoteEncoding: 'binary' }],
+                ['*.jcl', { ignore: false, localEncoding: 'IBM-1047', remoteEncoding: 'IBM-1047' }],
+                ['*.md', { ignore: false, localEncoding: 'UTF-8', remoteEncoding: 'UTF-8' }],
+                ['*.txt', { ignore: false, localEncoding: 'binary', remoteEncoding: 'binary' }]
+            ]);
+            try {
+                response = await Download.ussFile(dummySession, ussname, { attributes: zosAttributes });
+            } catch (e) {
+                caughtError = e;
+            }
+
+            const endpoint = path.posix.join(ZosFilesConstants.RESOURCE, ZosFilesConstants.RES_USS_FILES, encodeURIComponent(ussname.substring(1)));
+
+            expect(caughtError).toBeUndefined();
+            expect(response).toEqual({
+                success: true,
+                commandResponse: util.format(ZosFilesMessages.ussFileDownloadedWithDestination.message, destination),
+                apiResponse: {}
+            });
+
+            expect(zosmfGetFullSpy).toHaveBeenCalledTimes(1);
+            expect(zosmfGetFullSpy).toHaveBeenCalledWith(dummySession, {
+                resource: endpoint,
+                reqHeaders: [{ "X-IBM-Data-Type": "binary" }],
+                responseStream: fakeStream,
+                normalizeResponseNewLines: false,
+                task: undefined /* no progress task */
+            });
+
+            expect(ioCreateDirSpy).toHaveBeenCalledTimes(1);
+            expect(ioCreateDirSpy).toHaveBeenCalledWith(destination);
+
+            expect(ioWriteStreamSpy).toHaveBeenCalledTimes(1);
+            expect(ioWriteStreamSpy).toHaveBeenCalledWith(destination);
+        });
     });
 
     describe("USS Directory", () => {

packages/zosfiles/src/doc/IOptions.ts

@@ -11,6 +11,7 @@
 
 import { ITaskWithStatus } from "@zowe/imperative";
 import { IZosFilesOptions } from "./IZosFilesOptions";
+import { ZosFilesAttributes } from "../utils/ZosFilesAttributes";
 
 /**
  * This interface defines the options that can be sent to get a data set or USS file function
@@ -57,4 +58,8 @@ export interface IOptions extends IZosFilesOptions {
      * @type {ITaskWithStatus}
      */
     task?: ITaskWithStatus;
+    /**
+     * The path to a .zosattributes file used to control file conversion and tagging.
+     */
+    attributes?: ZosFilesAttributes;
 }

packages/zosfiles/src/methods/download/Download.ts

@@ -24,7 +24,7 @@ import { ZosFilesUtils } from "../../utils/ZosFilesUtils";
 import { List } from "../list/List";
 import { IDownloadOptions, IDownloadSingleOptions } from "./doc/IDownloadOptions";
 import { CLIENT_PROPERTY } from "../../doc/types/ZosmfRestClientProperties";
-import { Utilities } from "../utilities";
+import { Tag, Utilities } from "../utilities";
 import { IZosmfListResponse } from "../list/doc/IZosmfListResponse";
 import { IDownloadDsmResult } from "./doc/IDownloadDsmResult";
 import { IDownloadUssDirResult } from "./doc/IDownloadUssDirResult";
@@ -521,7 +521,15 @@ export class Download {
 
             const writeStream = options.stream ?? IO.createWriteStream(destination);
 
-            // If data type is not defined by user, check for USS tags
+            if(options.attributes)
+            {
+                options.binary = options.attributes.getFileTransferMode(ussFileName, options.binary) === TransferMode.BINARY;
+                const remoteEncoding = options.attributes.getRemoteEncoding(ussFileName);
+                if(remoteEncoding === Tag.BINARY) options.encoding = undefined;
+                else if(remoteEncoding !== null) options.encoding = remoteEncoding;
+            }
+
+            // If data type is not defined by user via encoding flag or attributes file, check for USS tags
             if (options.binary == null && options.encoding == null) {
                 await Utilities.applyTaggedEncoding(session, ussFileName, options);
             }