feat: add selectObject method

README.md

@@ -163,6 +163,7 @@ All operation use es7 async/await to implement. All api is async function.
   - [.getObjectTagging(name, [, options])](#getObjectTaggingname-options)
   - [.putObjectTagging(name, tag[, options])](#putObjectTaggingname-tag-options)
   - [.deleteObjectTagging(name, [, options])](#deleteObjectTaggingname-options)
+  - [.selectObject(name, query, grammar[, options])](#selectObjectname-query-grammar-options)
 - [RTMP Operations](#rtmp-operations)
   - [.putChannel(id, conf[, options])](#putchannelid-conf-options)
   - [.getChannel(id[, options])](#getchannelid-options)
@@ -3619,6 +3620,62 @@ object:
 - status {Number} response status
 - res {Object} response info
 
+### .selectObject(name, query, grammar[, options])
+
+SelectObject is used to execute SQL statements on the target file and return the execution result.
+
+parameters:
+
+- name {String} the object name
+- query {Object} query parameters eg. `'select * from my_table'`
+- grammar {String} The request syntax is divided into two formats, CSV and JSON.
+- [options] {Object} optional args
+  - [InputSerialization] {Object} InputSerialization
+    - [CompressionType] {String} The type of compression used to compress the object.
+    - [CSV] {Object} CSV
+      - [FileHeaderInfo] {String} The file header information, which is optional.
+      - [RecordDelimiter] {String} The record delimiter, which is optional.
+      - [FieldDelimiter] {String} The field delimiter, which is optional.
+      - [QuoteCharacter] {String} The quote character, which is optional.
+      - [CommentCharacter] {String} The comment character, which is optional.
+      - [Range] {String} The range, which is optional.
+      - [AllowQuotedRecordDelimiter] {Boolean} Whether to allow quoted record delimiter, which is optional.
+    - [JSON] {Object} JSON
+      - [Type] {String} The type, which is optional.
+      - [Range] {String} The range, which is optional.
+      - [ParseJsonNumberAsString] {Boolean} The parse json number as string, which is optional.
+  - [OutputSerialization] {Object} OutputSerialization
+    - [CSV] {Object} CSV
+      - [RecordDelimiter] {String} The record delimiter, which is optional.
+      - [FieldDelimiter] {String} The field delimiter, which is optional.
+    - [JSON] {Object} JSON
+        - [RecordDelimiter] {String} The record delimiter, which is optional.
+    - [KeepAllColumns] {Boolean} Whether to keep all columns, which is optional.
+    - [OutputRawData] {Boolean} Whether to output raw data, which is optional.
+    - [EnablePayloadCrc] {Boolean} Whether to enable payload crc, which is optional.
+    - [OutputHeader] {Boolean} Whether to output header, which is optional.
+  - [Other] {Object} Other option
+    - [SkipPartialDataRecord] {Boolean} Whether to skip partial data record, which is optional.
+    - [MaxSkippedRecordsAllowed] {String} The max skipped records allowed, which is optional.
+
+[More code info](https://help.aliyun.com/document_detail/74054.html#section-v2y-dzp-o5k)
+
+Succeed will return the channel information.
+
+object:
+
+- res {Object} response info
+- data {Object} data info
+
+```javascript
+// example CSV
+store.selectObject('example.csv', 'select * from ossobject limit 10', 'csv').then(r => console.log(r));
+
+// example JSON
+store.selectObject('example.json', 'select address from ossobject.contacts[*]', 'json')
+  .then(r => console.log(r));
+```
+
 ### .processObjectSave(sourceObject, targetObject, process[, targetBucket])
 
 Persistency indicates that images are asynchronously stored in the specified Bucket

lib/common/object/index.d.ts

@@ -23,6 +23,7 @@ import { putObjectTagging } from './putObjectTagging';
 import { putSymlink } from './putSymlink';
 import { restore } from './restore';
 import { signatureUrl } from './signatureUrl';
+import { selectObject } from './selectObject';
 declare const _default: {
     append: typeof append;
     calculatePostSignature: typeof calculatePostSignature;
@@ -51,5 +52,6 @@ declare const _default: {
     putSymlink: typeof putSymlink;
     restore: typeof restore;
     signatureUrl: typeof signatureUrl;
+    selectObject: typeof selectObject;
 };
 export default _default;

lib/common/object/index.js

@@ -25,6 +25,7 @@ const putObjectTagging_1 = require("./putObjectTagging");
 const putSymlink_1 = require("./putSymlink");
 const restore_1 = require("./restore");
 const signatureUrl_1 = require("./signatureUrl");
+const selectObject_1 = require("./selectObject");
 exports.default = {
     append: append_1.append,
     calculatePostSignature: calculatePostSignature_1.calculatePostSignature,
@@ -55,4 +56,5 @@ exports.default = {
     putSymlink: putSymlink_1.putSymlink,
     restore: restore_1.restore,
     signatureUrl: signatureUrl_1.signatureUrl,
+    selectObject: selectObject_1.selectObject
 };

lib/common/object/selectObject.d.ts

@@ -0,0 +1,4 @@
+export declare function selectObject(this: any, name: string, expression: string, grammar: string, options: any): Promise<{
+    res: any;
+    data: any;
+}>;

lib/common/object/selectObject.js

@@ -0,0 +1,95 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.selectObject = void 0;
+const obj2xml_1 = require("../utils/obj2xml");
+const js_base64_1 = require("js-base64");
+const unpackFrame_1 = __importDefault(require("../utils/unpackFrame"));
+const needToEncode = ['RecordDelimiter', 'FieldDelimiter', 'QuoteCharacter', 'CommentCharacter'];
+async function selectObject(name, expression, grammar, options) {
+    if (!['json', 'csv'].includes(grammar.toLocaleLowerCase())) {
+        throw new Error('grammar must be json or csv');
+    }
+    const opts = Object.assign({}, options);
+    opts.subres = Object.assign({ 'x-oss-process': `${grammar}/select` });
+    let InputSerialization;
+    let OutputSerialization;
+    let Options;
+    const paramsXMLObj = {
+        SelectRequest: {
+            Expression: js_base64_1.Base64.encode(expression)
+        }
+    };
+    // CompressionType
+    if (opts.InputSerialization) {
+        opts.InputSerialization.CompressionType = opts.InputSerialization.CompressionType
+            ? opts.InputSerialization.CompressionType
+            : 'None';
+    }
+    // CSV
+    if (grammar.toLocaleLowerCase() === 'csv') {
+        // inputSerialization
+        if (opts.InputSerialization && opts.InputSerialization.CSV) {
+            Object.keys(opts.InputSerialization.CSV).forEach(i => {
+                if (needToEncode.includes(i)) {
+                    opts.InputSerialization.CSV[i] = js_base64_1.Base64.encode(opts.InputSerialization.CSV[i]);
+                }
+            });
+        }
+        InputSerialization = Object.assign({}, opts.InputSerialization);
+        paramsXMLObj.SelectRequest.InputSerialization = InputSerialization;
+        // OutputSerialization
+        if (opts.OutputSerialization && opts.OutputSerialization.CSV) {
+            Object.keys(opts.OutputSerialization.CSV).forEach(i => {
+                if (needToEncode.includes(i)) {
+                    opts.OutputSerialization.CSV[i] = js_base64_1.Base64.encode(opts.OutputSerialization.CSV[i]);
+                }
+            });
+        }
+        OutputSerialization = Object.assign({}, opts.OutputSerialization);
+        paramsXMLObj.SelectRequest.OutputSerialization = OutputSerialization;
+    }
+    // JSON
+    if (grammar.toLowerCase() === 'json') {
+        // InputSerialization
+        if (opts.InputSerialization && opts.InputSerialization.JSON) {
+            opts.InputSerialization.JSON.Type = opts.InputSerialization.JSON.Type
+                ? opts.InputSerialization.JSON.Type.toUpperCase()
+                : 'DOCUMENT';
+            opts.InputSerialization.JSON = Object.assign({}, opts.InputSerialization.JSON);
+        }
+        InputSerialization = Object.assign({}, opts.InputSerialization);
+        paramsXMLObj.SelectRequest.InputSerialization = InputSerialization;
+        // OutputSerialization
+        if (opts.OutputSerialization && opts.OutputSerialization.JSON) {
+            if (opts.OutputSerialization.JSON.RecordDelimiter) {
+                opts.OutputSerialization.JSON.RecordDelimiter = js_base64_1.Base64.encode(opts.OutputSerialization.JSON.RecordDelimiter);
+            }
+        }
+        OutputSerialization = Object.assign({}, opts.OutputSerialization);
+        paramsXMLObj.SelectRequest.OutputSerialization = OutputSerialization;
+    }
+    // Options
+    if (opts.Other) {
+        Options = Object.assign({}, opts.Other);
+        paramsXMLObj.SelectRequest.Options = Options;
+    }
+    const params = this._objectRequestParams('POST', name, opts);
+    params.content = obj2xml_1.obj2xml(paramsXMLObj);
+    params.mime = 'xml';
+    params.successStatuses = [206];
+    const result = await this.request(params);
+    if (result.res.headers['x-oss-select-output-raw'] !== 'true') {
+        result.data = unpackFrame_1.default(result.data);
+    }
+    else {
+        result.data = result.data.toString();
+    }
+    return {
+        res: result.res,
+        data: result.data
+    };
+}
+exports.selectObject = selectObject;

lib/common/utils/unpackFrame.d.ts

@@ -0,0 +1,19 @@
+/// <reference types="node" />
+declare type frameData = {
+    payload: payloadType;
+    version: number;
+    frameType: number;
+    payloadLength: number;
+    headerCheckSum: number;
+    payloadCheckSum: number;
+};
+declare type payloadType = {
+    offset: number;
+    frameType: string;
+    data?: string;
+    totalScannedBytes?: number;
+    httpStatusCode?: number;
+    errorMessage?: string;
+};
+declare const _default: (data: Buffer) => frameData;
+export default _default;

lib/common/utils/unpackFrame.js

@@ -0,0 +1,40 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+// all integers in Frame are encoded in big endian and Version is currently 1.
+exports.default = (data) => {
+    const payload = { frameType: '', offset: 0 };
+    const version = data.readUIntBE(0, 1);
+    const frameType = data.readUIntBE(1, 3);
+    const payloadLength = data.readUIntBE(4, 4);
+    const headerCheckSum = data.readUIntBE(8, 4);
+    const payloadCheckSum = data.readUIntBE(data.length - 4, 4);
+    // payload offset
+    payload.offset = parseInt(data.slice(12, 20).toString('hex'), 16);
+    /**
+     *  Frame Type
+     *  8388609 - Data Frame
+     *  8388612 - Continuous Frame
+     *  8388613 - End Frame
+     */
+    if (frameType === 8388609) {
+        payload.frameType = 'DataFrame';
+        payload.data = data.slice(20, 12 + payloadLength).toString();
+    }
+    else if (frameType === 8388612) {
+        payload.frameType = 'ContinuousFrame';
+    }
+    else if (frameType === 8388613) {
+        payload.frameType = 'End Frame';
+        payload.totalScannedBytes = data.readUIntBE(20, 8);
+        payload.httpStatusCode = data.readUIntBE(28, 4);
+        payload.errorMessage = data.slice(32, data.length - 4).toString();
+    }
+    return {
+        payload,
+        version,
+        frameType,
+        payloadCheckSum,
+        payloadLength,
+        headerCheckSum
+    };
+};

package.json

@@ -152,7 +152,6 @@
   "lint-staged": {
     "**/!(dist)/*": [
       "npm run detect-secrets --"
-    ],
-    "*.js": "eslint --cache --fix"
+    ]
   }
-}
+}

src/common/object/index.ts

@@ -23,6 +23,7 @@ import { putObjectTagging } from './putObjectTagging';
 import { putSymlink } from './putSymlink';
 import { restore } from './restore';
 import { signatureUrl } from './signatureUrl';
+import { selectObject } from './selectObject';
 
 export default {
   append,
@@ -52,4 +53,5 @@ export default {
   putSymlink,
   restore,
   signatureUrl,
+  selectObject
 };

src/common/object/putSymlink.ts

@@ -1,4 +1,3 @@
-
 import { objectName } from '../utils/objectName';
 import { convertMetaToHeaders } from '../utils/convertMetaToHeaders';
 import { escapeName } from '../utils/escapeName';
@@ -12,7 +11,12 @@ import { putSymlinkOptions } from '../../types/params';
  * @param {{res}}
  */
 
-export async function putSymlink(this: any, name: string, targetName: string, options: putSymlinkOptions = {}) {
+export async function putSymlink(
+  this: any,
+  name: string,
+  targetName: string,
+  options: putSymlinkOptions = {}
+) {
   options.headers = options.headers || {};
   targetName = escapeName(objectName(targetName));
   convertMetaToHeaders(options.meta, options.headers);