Zos download attributes

packages/cli/__tests__/zosfiles/__system__/download/ussdir/__resources__/.zosattributes

@@ -2,4 +2,4 @@
 *.bin binary binary
 *.jcl IBM-1047 IBM-1047
 *.md UTF-8 UTF-8
-*.txt UTF-8 IBM-1047
+*.txt binary binary

packages/cli/__tests__/zosfiles/__unit__/download/uss/__snapshots__/UssFile.definition.unit.test.ts.snap

@@ -2,6 +2,18 @@
 
 exports[`zos-files download uss-file command definition should not have changed 1`] = `
 Array [
+  Object {
+    "aliases": Array [
+      "attrs",
+    ],
+    "conflictsWith": Array [
+      "binary",
+      "record",
+    ],
+    "description": "Path of an attributes file to control how files are downloaded.",
+    "name": "attributes",
+    "type": "existingLocalFile",
+  },
   Object {
     "aliases": Array [
       "b",

packages/cli/src/zosfiles/download/uss/UssFile.definition.ts

@@ -39,6 +39,7 @@ export const UssFileDefinition: ICommandDefinition = {
         }
     ],
     options: [
+        DownloadOptions.attributes,
         DownloadOptions.file,
         DownloadOptions.binary,
         DownloadOptions.encoding

packages/cli/src/zosfiles/download/uss/UssFile.handler.ts

@@ -11,7 +11,7 @@
 
 import { ZosFilesBaseHandler } from "../../ZosFilesBase.handler";
 import { AbstractSession, IHandlerParameters, ITaskWithStatus, TaskStage } from "@zowe/imperative";
-import { Download, IZosFilesResponse } from "@zowe/zos-files-for-zowe-sdk";
+import { Download, IDownloadOptions, IZosFilesResponse, ZosFilesAttributes } from "@zowe/zos-files-for-zowe-sdk";
 
 /**
  * Handler to download an uss file
@@ -25,13 +25,21 @@ export default class UssFileHandler extends ZosFilesBaseHandler {
             stageName: TaskStage.IN_PROGRESS
         };
         commandParameters.response.progress.startBar({task});
-        return Download.ussFile(session, commandParameters.arguments.ussFileName, {
+
+        const downloadOptions: IDownloadOptions = {
             binary: commandParameters.arguments.binary,
             encoding: commandParameters.arguments.encoding,
             file: commandParameters.arguments.file,
             task,
             responseTimeout: commandParameters.arguments.responseTimeout,
-            overwrite: commandParameters.arguments.overwrite
-        });
+            overwrite: commandParameters.arguments.overwrite,
+        };
+        const attributes = ZosFilesAttributes.loadFromFile(
+            commandParameters.arguments.attributes,
+        );
+        if (attributes != null) {
+            downloadOptions.attributes = attributes;
+        }
+        return Download.ussFile(session, commandParameters.arguments.ussFileName, downloadOptions);
     }
 }

packages/zosfiles/__tests__/__system__/methods/download/Download.system.test.ts

@@ -24,23 +24,26 @@
     IMountFsOptions,
     Mount,
     Unmount,
-    IUSSListOptions
+    IUSSListOptions,
+    ZosFilesAttributes
 } from "../../../../src";
 import { Imperative, IO, Session } from "@zowe/imperative";
 import { inspect } from "util";
 import { ITestEnvironment } from "../../../../../../__tests__/__src__/environment/ITestEnvironment";
 import { TestEnvironment } from "../../../../../../__tests__/__src__/environment/TestEnvironment";
 import { ITestPropertiesSchema } from "../../../../../../__tests__/__src__/properties/ITestPropertiesSchema";
-import { deleteFiles, getUniqueDatasetName, stripNewLines, wait } from "../../../../../../__tests__/__src__/TestUtils";
+import { deleteFiles, getUniqueDatasetName, stripNewLines, wait, } from "../../../../../../__tests__/__src__/TestUtils";
 import * as fs from "fs";
 import { posix } from "path";
 import { Shell } from "@zowe/zos-uss-for-zowe-sdk";
 import { PassThrough } from "stream";
 import { text } from "stream/consumers";
+import { runCliScript } from "@zowe/cli-test-utils";
 
 const rimraf = require("rimraf").sync;
 const delayTime = 2000;
 const testData = "abcdefghijklmnopqrstuvwxyz";
+const testDataBinary = "[][][]";
 
 let REAL_SESSION: Session;
 let testEnvironment: ITestEnvironment<ITestPropertiesSchema>;
@@ -1018,7 +1021,7 @@
                 let error;
                 let response: IZosFilesResponse;
 
-                await Upload.bufferToUssFile(REAL_SESSION, ussname, Buffer.from(testData));
+                await Upload.bufferToUssFile(REAL_SESSION, ussname, Buffer.from(testDataBinary));
                 await wait(delayTime);
 
                 try {
@@ -1081,6 +1084,84 @@
 
             });
 
+            it("should download uss file with --attributes flag", async () => {
+                let error;
+                let response: any;
+                let downloadResponse: any;
+                let zosAttributes = Object.create(ZosFilesAttributes.prototype);
+                zosAttributes.attributes = new Map([
+                    ['*.json', { ignore: true }],
+                    ['*.bin', { ignore: false, localEncoding: 'binary', remoteEncoding: 'binary' }],
+                    ['*.jcl', { ignore: false, localEncoding: 'IBM-1047', remoteEncoding: 'IBM-1047' }],
+                    ['*.md', { ignore: false, localEncoding: 'UTF-8', remoteEncoding: 'UTF-8' }],
+                    ['*.txt', { ignore: false, localEncoding: 'UTF-8', remoteEncoding: 'IBM-1047' }]
+                ]);
+                try {
+                    response = await Upload.bufferToUssFile(REAL_SESSION, ussname, Buffer.from(testDataBinary));
+                    downloadResponse = await Download.ussFile(REAL_SESSION, ussname, { attributes: zosAttributes});
+                } catch (err) {
+                    error = err;
+                }
+                expect(error).toBeFalsy();
+                expect(response).toBeTruthy();
+                expect(downloadResponse).toBeTruthy();
+
+                // Compare the downloaded contents to those uploaded
+                const fileContents = stripNewLines(fs.readFileSync(`./${posix.basename(ussname)}`).toString());
+                expect(fileContents).toEqual(testDataBinary);
+            });
+
+            it("should download uss file with --attributes flag - binary", async () => {
+                let error;
+                let response: any;
+                let uploadResponse: any;
+                let downloadResponse: any;
+                let zosAttributes = Object.create(ZosFilesAttributes.prototype);
+                zosAttributes.attributes = new Map([
+                    ['*.json', { ignore: true }],
+                    ['*.bin', { ignore: false, localEncoding: 'binary', remoteEncoding: 'binary' }],
+                    ['*.jcl', { ignore: false, localEncoding: 'IBM-1047', remoteEncoding: 'IBM-1047' }],
+                    ['*.md', { ignore: false, localEncoding: 'UTF-8', remoteEncoding: 'UTF-8' }],
+                    ['*.txt', { ignore: false, localEncoding: 'binary', remoteEncoding: 'binary' }]
+                ]);
+                try {
+                    response = runCliScript(__dirname + "/__resources__/upload_file_to_uss.sh", testEnvironment,[
+                        defaultSystem.tso.account,
+                        defaultSystem.zosmf.host,
+                        defaultSystem.zosmf.port,
+                        defaultSystem.zosmf.user,
+                        defaultSystem.zosmf.password,
+                        defaultSystem.zosmf.rejectUnauthorized,
+                        __dirname + "/__resources__/testfiles/downloadEncodingCheck.txt",
+                        // "/u/users/jr897694/example/example.txt",
+                        ussname,
+                        true
+                    ]);
+                    downloadResponse = runCliScript(__dirname + "/__resources__/download_file.sh", testEnvironment,[
+                        defaultSystem.tso.account,
+                        defaultSystem.zosmf.host,
+                        defaultSystem.zosmf.port,
+                        defaultSystem.zosmf.user,
+                        defaultSystem.zosmf.password,
+                        defaultSystem.zosmf.rejectUnauthorized,
+                        ussname,
+                        __dirname+"/__resources__/.zosattributes-binary"
+                    ]);
+
+                    // downloadResponse = await Download.ussFile(REAL_SESSION, /*"/u/users/jr897694/example/example.txt"*/ ussname, { attributes: zosAttributes});
+                } catch (err) {
+                    error = err;
+                }
+                expect(error).toBeFalsy();
+                expect(response).toBeTruthy();
+                expect(downloadResponse).toBeTruthy();
+
+                // Compare the downloaded contents to those uploaded
+                const fileContents = stripNewLines(fs.readFileSync(`./${posix.basename(ussname)}`).toString());
+
+                expect(fileContents).toEqual(fs.readFileSync(__dirname+"/__resources__/testfiles/downloadEncodingCheck.txt").toString());
+            });
+
             // When requesting etag, z/OSMF has a limit on file size when it stops to return etag by default (>8mb)
             // We are passing X-IBM-Return-Etag to force z/OSMF to always return etag, but testing here for case where it would be optional
             it("should download a 10mb uss file and return Etag", async () => {

packages/zosfiles/__tests__/__system__/methods/download/__resources__/.zosattributes

@@ -0,0 +1,5 @@
+*.json -
+*.bin binary binary
+*.jcl IBM-1047 IBM-1047
+*.md UTF-8 UTF-8
+*.txt UTF-8 IBM-1047

packages/zosfiles/__tests__/__system__/methods/download/__resources__/.zosattributes-binary

@@ -0,0 +1,5 @@
+*.json -
+*.bin binary binary
+*.jcl IBM-1047 IBM-1047
+*.md UTF-8 UTF-8
+*.txt binary binary

packages/zosfiles/__tests__/__system__/methods/download/__resources__/download_file.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+account=$1
+host=$2
+port=$3
+user=$4
+password=$5
+ru=$6
+fileToDownload=$7
+attributes=$8
+
+zowe zos-files download uss $fileToDownload --attributes $attributes --host $host --port $port --user $user --password $password --ru $ru
+exit $?

packages/zosfiles/__tests__/__system__/methods/download/__resources__/testfiles/downloadEncodingCheck.txt

@@ -0,0 +1 @@
+[][][]

packages/zosfiles/__tests__/__system__/methods/download/__resources__/upload_file_to_uss.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+account=$1
+host=$2
+port=$3
+user=$4
+password=$5
+ru=$6
+inputFile=$7
+ussName=$8
+binary=$9
+
+zowe zos-files upload file-to-uss $inputFile $ussName --binary $binary --host $host --port $port --user $user --password $password --ru $ru
+exit $?

packages/zosfiles/src/doc/IOptions.ts

@@ -11,6 +11,7 @@
 
 import { ITaskWithStatus } from "@zowe/imperative";
 import { IZosFilesOptions } from "./IZosFilesOptions";
+import { ZosFilesAttributes } from "../utils/ZosFilesAttributes";
 
 /**
  * This interface defines the options that can be sent to get a data set or USS file function
@@ -57,4 +58,8 @@ export interface IOptions extends IZosFilesOptions {
      * @type {ITaskWithStatus}
      */
     task?: ITaskWithStatus;
+    /**
+     * The path to a .zosattributes file used to control file conversion and tagging.
+     */
+    attributes?: ZosFilesAttributes;
 }

packages/zosfiles/src/methods/download/Download.ts

@@ -24,7 +24,7 @@ import { ZosFilesUtils } from "../../utils/ZosFilesUtils";
 import { List } from "../list/List";
 import { IDownloadOptions, IDownloadSingleOptions } from "./doc/IDownloadOptions";
 import { CLIENT_PROPERTY } from "../../doc/types/ZosmfRestClientProperties";
-import { Utilities } from "../utilities";
+import { Tag, Utilities } from "../utilities";
 import { IZosmfListResponse } from "../list/doc/IZosmfListResponse";
 import { IDownloadDsmResult } from "./doc/IDownloadDsmResult";
 import { IDownloadUssDirResult } from "./doc/IDownloadUssDirResult";
@@ -521,7 +521,16 @@ export class Download {
 
             const writeStream = options.stream ?? IO.createWriteStream(destination);
 
-            // If data type is not defined by user, check for USS tags
+            //const remoteEncoding = options.attributes.getRemoteEncoding(ussFileName);
+            if(options.attributes)
+            {
+                options.binary = options.attributes.getFileTransferMode(ussFileName, options.binary) === TransferMode.BINARY;
+                const remoteEncoding = options.attributes.getRemoteEncoding(ussFileName);
+                if(remoteEncoding === Tag.BINARY) options.encoding = undefined;
+                else if(remoteEncoding !== null) options.encoding = remoteEncoding;
+            }
+
+            // If data type is not defined by user via encoding flag or attributes file, check for USS tags
             if (options.binary == null && options.encoding == null) {
                 await Utilities.applyTaggedEncoding(session, ussFileName, options);
             }

packages/zosfiles/src/methods/download/doc/IDownloadOptions.ts

@@ -58,9 +58,13 @@ export interface IDownloadSingleOptions extends IGetOptions {
     overwrite?: boolean;
 
     /**
-     * Optional stream to read the file contents
+     * Optional stream to write the file contents
      */
     stream?: Writable;
+    /**
+     * The ZosFilesAttributes instance describe upload attributes for the files and directories
+     */
+    attributes?: ZosFilesAttributes;
 }
 
 /**