Zos download attributes

packages/cli/CHANGELOG.md

@@ -2,6 +2,10 @@
 
 All notable changes to the Zowe CLI package will be documented in this file.
 
+## Recent Changes
+
+- Enhancement: Pass a `.zosattributes` file path for the download encoding format by adding the new --attributes flag to the `zowe zos-files upload` command.[#2322](https://github.com/zowe/zowe-cli/issues/2322)
+
 ## `8.7.0`
 
 - Enhancement: Added --wait-for-active and --wait-for-output to download options on zosjobs. [#2328](https://github.com/zowe/zowe-cli/pull/2328)
@@ -18,6 +22,7 @@ All notable changes to the Zowe CLI package will be documented in this file.
 
 - Enhancement: Added support for running applications on TSO/E address spaces. Start applications and receive/transmit messages using the new `tso start`, `tso receive` and `tso send` commands. [#2280](https://github.com/zowe/zowe-cli/pull/2280)
 
+
 ## `8.4.0`
 
 - Enhancement: Added optional `--attributes` flag to `zowe zos-files upload file-to-uss` to allow passing a .zosattributes file path for upload encoding format. [#2319](https://github.com/zowe/zowe-cli/pull/2319)

packages/cli/__tests__/zosfiles/__unit__/download/uss/__snapshots__/UssFile.definition.unit.test.ts.snap

@@ -2,6 +2,18 @@
 
 exports[`zos-files download uss-file command definition should not have changed 1`] = `
 Array [
+  Object {
+    "aliases": Array [
+      "attrs",
+    ],
+    "conflictsWith": Array [
+      "binary",
+      "record",
+    ],
+    "description": "Path of an attributes file to control how files are downloaded.",
+    "name": "attributes",
+    "type": "existingLocalFile",
+  },
   Object {
     "aliases": Array [
       "b",

packages/cli/__tests__/zosjobs/__system__/download/cli.zos-jobs.download.output.system.test.ts

@@ -14,8 +14,6 @@ import { ITestEnvironment } from "../../../../../../__tests__/__src__/environmen
 import { runCliScript } from "@zowe/cli-test-utils";
 import { ITestPropertiesSchema } from "../../../../../../__tests__/__src__/properties/ITestPropertiesSchema";
 import * as fs from "fs";
-import { Session } from "@zowe/imperative";
-import { GetJobs } from "@zowe/zos-jobs-for-zowe-sdk";
 
 // Test Environment populated in the beforeAll();
 let TEST_ENVIRONMENT: ITestEnvironment<ITestPropertiesSchema>;
@@ -49,14 +47,14 @@ describe("zos-jobs download output command", () => {
         it("should download a job and wait for it to reach output status", async () => {
             const response = runCliScript(__dirname + "/__scripts__/download-output/download_job_wait_for_output.sh",
                 TEST_ENVIRONMENT, [IEFBR14_JCL]);
-                expect(response.stderr.toString()).toBe("");
-                expect(response.status).toBe(0);
+            expect(response.stderr.toString()).toBe("");
+            expect(response.status).toBe(0);
         });
         it("should download a job and wait for it to reach active status", async () => {
             const response = runCliScript(__dirname + "/__scripts__/download-output/download_job_wait_for_active.sh",
                 TEST_ENVIRONMENT, [IEFBR14_JCL]);
-                expect(response.stderr.toString()).toBe("");
-                expect(response.status).toBe(0);
+            expect(response.stderr.toString()).toBe("");
+            expect(response.status).toBe(0);
         });
     });
     describe("output", () => {

packages/cli/src/zosfiles/download/uss/UssFile.definition.ts

@@ -39,6 +39,7 @@ export const UssFileDefinition: ICommandDefinition = {
         }
     ],
     options: [
+        DownloadOptions.attributes,
         DownloadOptions.file,
         DownloadOptions.binary,
         DownloadOptions.encoding

packages/cli/src/zosfiles/download/uss/UssFile.handler.ts

@@ -11,7 +11,7 @@
 
 import { ZosFilesBaseHandler } from "../../ZosFilesBase.handler";
 import { AbstractSession, IHandlerParameters, ITaskWithStatus, TaskStage } from "@zowe/imperative";
-import { Download, IZosFilesResponse } from "@zowe/zos-files-for-zowe-sdk";
+import { Download, IDownloadOptions, IZosFilesResponse, ZosFilesAttributes } from "@zowe/zos-files-for-zowe-sdk";
 
 /**
  * Handler to download an uss file
@@ -25,13 +25,20 @@ export default class UssFileHandler extends ZosFilesBaseHandler {
             stageName: TaskStage.IN_PROGRESS
         };
         commandParameters.response.progress.startBar({task});
-        return Download.ussFile(session, commandParameters.arguments.ussFileName, {
+
+        const attributes = ZosFilesAttributes.loadFromFile(
+            commandParameters.arguments.attributes,
+        );
+        const downloadOptions: IDownloadOptions = {
             binary: commandParameters.arguments.binary,
             encoding: commandParameters.arguments.encoding,
             file: commandParameters.arguments.file,
             task,
             responseTimeout: commandParameters.arguments.responseTimeout,
-            overwrite: commandParameters.arguments.overwrite
-        });
+            overwrite: commandParameters.arguments.overwrite,
+            attributes
+        };
+
+        return Download.ussFile(session, commandParameters.arguments.ussFileName, downloadOptions);
     }
 }

packages/zosfiles/CHANGELOG.md

@@ -2,14 +2,18 @@
 
 All notable changes to the Zowe z/OS files SDK package will be documented in this file.
 
+## Recent Changes
+
+- Enhancement: Allows for passing a `.zosattributues` file path for the download encoding format via the `--attributes` flag on the `zowe zos-files download` command [#2322](https://github.com/zowe/zowe-cli/issues/2322)
+
 ## `8.6.2`
 
 - BugFix: Resolved issue where encoding argument was missing from `FileToUss.handler.ts` options object. [#2234](https://github.com/zowe/zowe-cli/pull/2334)
 - BugFix: Resolved issue where `FileToUss.handler.ts` options object was not properly passed through subsequent command calls. [#2234](https://github.com/zowe/zowe-cli/pull/2334)
 
 ## `8.4.0`
 
-- Enhancement: Added optional `--attributes` flag to `zowe zos-files upload file-to-uss` to allow passing a .zosattributes file path for upload encoding format. [#2319] (https://github.com/zowe/zowe-cli/pull/2319)
+- Enhancement: Added optional `--attributes` flag to `zowe zos-files upload file-to-uss` to allow passing a .zosattributes file path for upload encoding format. [#2319](https://github.com/zowe/zowe-cli/pull/2319)
 
 ## `8.2.0`
 

packages/zosfiles/__tests__/__system__/methods/download/Download.system.test.ts

@@ -24,7 +24,7 @@ import {
     IMountFsOptions,
     Mount,
     Unmount,
-    IUSSListOptions
+    IUSSListOptions,
 } from "../../../../src";
 import { Imperative, IO, Session } from "@zowe/imperative";
 import { inspect } from "util";
@@ -37,6 +37,7 @@ import { posix } from "path";
 import { Shell } from "@zowe/zos-uss-for-zowe-sdk";
 import { PassThrough } from "stream";
 import { text } from "stream/consumers";
+import { runCliScript } from "@zowe/cli-test-utils";
 
 const rimraf = require("rimraf").sync;
 const delayTime = 2000;
@@ -1080,7 +1081,91 @@ describe.each([false, true])("Download Data Set - Encoded: %s", (encoded: boolea
                 expect(fileContents).toEqual(testData);
 
             });
-
+            describe.each([false, true])(
+                "Download Data Set - Binary: %s",
+                (binary: boolean) => {
+                    it("should upload and download file with correct encoding", async () => {
+                        let response: any;
+
+                        // Upload binary or encoded file
+                        const uploadFile: string = binary
+                            ? "downloadEncodingCheck.txt"
+                            : "downloadEncodingCheckBinary.txt";
+                        let downloadResponse: any;
+                        let error: any;
+
+                        // Use text file as to get proper response from getRemoteEncoding()
+                        const ussnameAsTxt = ussname + ".txt";
+                        try {
+                            response = runCliScript(
+                                __dirname +
+                                    `/__resources__/${
+                                        binary
+                                            ? "upload_file_to_uss.sh"
+                                            : "upload_file_to_uss_binary.sh"
+                                    }`,
+                                testEnvironment,
+                                [
+                                    defaultSystem.tso.account,
+                                    defaultSystem.zosmf.host,
+                                    defaultSystem.zosmf.port,
+                                    defaultSystem.zosmf.user,
+                                    defaultSystem.zosmf.password,
+                                    defaultSystem.zosmf.rejectUnauthorized,
+                                    __dirname +
+                                        "/__resources__/testfiles/" +
+                                        uploadFile,
+                                    ussnameAsTxt,
+                                    binary ? "1047" : true,
+                                ]
+                            );
+                            downloadResponse = runCliScript(
+                                __dirname + "/__resources__/download_file.sh",
+                                testEnvironment,
+                                [
+                                    defaultSystem.tso.account,
+                                    defaultSystem.zosmf.host,
+                                    defaultSystem.zosmf.port,
+                                    defaultSystem.zosmf.user,
+                                    defaultSystem.zosmf.password,
+                                    defaultSystem.zosmf.rejectUnauthorized,
+                                    ussnameAsTxt,
+                                    __dirname +
+                                        `/__resources__/${
+                                            binary
+                                                ? ".zosattributes"
+                                                : ".zosattributes-binary"
+                                        }`,
+                                ]
+                            );
+                        } catch (err) {
+                            error = err;
+                        }
+
+                        expect(error).toBeFalsy();
+                        expect(response).toBeTruthy();
+                        expect(downloadResponse).toBeTruthy();
+
+                        // Compare the downloaded contents to those uploaded
+                        const fileContents = fs
+                            .readFileSync(
+                                `${testEnvironment.workingDir}/${posix.basename(
+                                    ussnameAsTxt
+                                )}`
+                            )
+                            .toString();
+                        expect(fileContents).toEqual(
+                            fs
+                                .readFileSync(
+                                    __dirname +
+                                        "/__resources__/testfiles/" +
+                                        uploadFile
+                                )
+                                .toString()
+                        );
+                    });
+                }
+            );
             // When requesting etag, z/OSMF has a limit on file size when it stops to return etag by default (>8mb)
             // We are passing X-IBM-Return-Etag to force z/OSMF to always return etag, but testing here for case where it would be optional
             it("should download a 10mb uss file and return Etag", async () => {

packages/zosfiles/__tests__/__system__/methods/download/__resources__/.zosattributes

@@ -0,0 +1,5 @@
+*.json -
+*.bin binary binary
+*.jcl IBM-1047 IBM-1047
+*.md UTF-8 UTF-8
+*.txt UTF-8 IBM-1047

packages/zosfiles/__tests__/__system__/methods/download/__resources__/.zosattributes-binary

@@ -0,0 +1,5 @@
+*.json -
+*.bin binary binary
+*.jcl IBM-1047 IBM-1047
+*.md UTF-8 UTF-8
+*.txt binary binary

packages/zosfiles/__tests__/__system__/methods/download/__resources__/download_file.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+account=$1
+host=$2
+port=$3
+user=$4
+password=$5
+ru=$6
+fileToDownload=$7
+attributes=$8
+
+zowe zos-files download uss "$fileToDownload" --attributes "$attributes" --host $host --port $port --user $user --password $password --ru $ru
+exit $?

packages/zosfiles/__tests__/__system__/methods/download/__resources__/testfiles/downloadEncodingCheck.txt

@@ -0,0 +1 @@
+abcdefghijklmnopqrstuvwxyz

packages/zosfiles/__tests__/__system__/methods/download/__resources__/testfiles/downloadEncodingCheckBinary.txt

@@ -0,0 +1 @@
+[][][]

packages/zosfiles/__tests__/__system__/methods/download/__resources__/upload_file_to_uss.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+account=$1
+host=$2
+port=$3
+user=$4
+password=$5
+ru=$6
+inputFile=$7
+ussName=$8
+encoding=$9
+
+zowe zos-files upload file-to-uss "$inputFile" "$ussName" --encoding $encoding --host $host --port $port --user $user --password $password --ru $ru
+exit $?

packages/zosfiles/__tests__/__system__/methods/download/__resources__/upload_file_to_uss_binary.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+account=$1
+host=$2
+port=$3
+user=$4
+password=$5
+ru=$6
+inputFile=$7
+ussName=$8
+binary=$9
+
+zowe zos-files upload file-to-uss "$inputFile" "$ussName" --binary $binary --host $host --port $port --user $user --password $password --ru $ru
+exit $?

packages/zosfiles/__tests__/__unit__/methods/download/Download.unit.test.ts

@@ -2184,6 +2184,90 @@ describe("z/OS Files - Download", () => {
             expect(ioCreateDirSpy).not.toHaveBeenCalled();
             expect(ioWriteStreamSpy).not.toHaveBeenCalled();
         });
+        it("should download uss file using .zosattributes file", async () => {
+            let response;
+            let caughtError;
+            const destination = localFileName;
+            const zosAttributes = Object.create(ZosFilesAttributes.prototype);
+            zosAttributes.attributes = new Map([
+                ['*.json', { ignore: true }],
+                ['*.bin', { ignore: false, localEncoding: 'binary', remoteEncoding: 'binary' }],
+                ['*.jcl', { ignore: false, localEncoding: 'IBM-1047', remoteEncoding: 'IBM-1047' }],
+                ['*.md', { ignore: false, localEncoding: 'UTF-8', remoteEncoding: 'UTF-8' }],
+                ['*.txt', { ignore: false, localEncoding: 'UTF-8', remoteEncoding: 'IBM-1047' }]
+            ]);
+            try {
+                response = await Download.ussFile(dummySession, ussname, { attributes: zosAttributes });
+            } catch (e) {
+                caughtError = e;
+            }
+
+            const endpoint = path.posix.join(ZosFilesConstants.RESOURCE, ZosFilesConstants.RES_USS_FILES, encodeURIComponent(ussname.substring(1)));
+
+            expect(caughtError).toBeUndefined();
+            expect(response).toEqual({
+                success: true,
+                commandResponse: util.format(ZosFilesMessages.ussFileDownloadedWithDestination.message, destination),
+                apiResponse: {}
+            });
+
+            expect(zosmfGetFullSpy).toHaveBeenCalledTimes(1);
+            expect(zosmfGetFullSpy).toHaveBeenCalledWith(dummySession, {
+                resource: endpoint,
+                reqHeaders: [{ "X-IBM-Data-Type": "text;fileEncoding=IBM-1047" }, ZosmfHeaders.ACCEPT_ENCODING, {"Content-Type": "UTF-8"}],
+                responseStream: fakeStream,
+                normalizeResponseNewLines: true,
+                task: undefined /* no progress task */
+            });
+
+            expect(ioCreateDirSpy).toHaveBeenCalledTimes(1);
+            expect(ioCreateDirSpy).toHaveBeenCalledWith(destination);
+
+            expect(ioWriteStreamSpy).toHaveBeenCalledTimes(1);
+            expect(ioWriteStreamSpy).toHaveBeenCalledWith(destination);
+        });
+        it("should download uss file using .zosattributes file - binary", async () => {
+            let response;
+            let caughtError;
+            const destination = localFileName;
+            const zosAttributes = Object.create(ZosFilesAttributes.prototype);
+            zosAttributes.attributes = new Map([
+                ['*.json', { ignore: true }],
+                ['*.bin', { ignore: false, localEncoding: 'binary', remoteEncoding: 'binary' }],
+                ['*.jcl', { ignore: false, localEncoding: 'IBM-1047', remoteEncoding: 'IBM-1047' }],
+                ['*.md', { ignore: false, localEncoding: 'UTF-8', remoteEncoding: 'UTF-8' }],
+                ['*.txt', { ignore: false, localEncoding: 'binary', remoteEncoding: 'binary' }]
+            ]);
+            try {
+                response = await Download.ussFile(dummySession, ussname, { attributes: zosAttributes });
+            } catch (e) {
+                caughtError = e;
+            }
+
+            const endpoint = path.posix.join(ZosFilesConstants.RESOURCE, ZosFilesConstants.RES_USS_FILES, encodeURIComponent(ussname.substring(1)));
+
+            expect(caughtError).toBeUndefined();
+            expect(response).toEqual({
+                success: true,
+                commandResponse: util.format(ZosFilesMessages.ussFileDownloadedWithDestination.message, destination),
+                apiResponse: {}
+            });
+
+            expect(zosmfGetFullSpy).toHaveBeenCalledTimes(1);
+            expect(zosmfGetFullSpy).toHaveBeenCalledWith(dummySession, {
+                resource: endpoint,
+                reqHeaders: [{ "X-IBM-Data-Type": "binary" }],
+                responseStream: fakeStream,
+                normalizeResponseNewLines: false,
+                task: undefined /* no progress task */
+            });
+
+            expect(ioCreateDirSpy).toHaveBeenCalledTimes(1);
+            expect(ioCreateDirSpy).toHaveBeenCalledWith(destination);
+
+            expect(ioWriteStreamSpy).toHaveBeenCalledTimes(1);
+            expect(ioWriteStreamSpy).toHaveBeenCalledWith(destination);
+        });
     });
 
     describe("USS Directory", () => {