Skip to content

Commit

Permalink
Merge pull request #313 from ardriveapp/dev
Browse files Browse the repository at this point in the history
Release 1.20.0
  • Loading branch information
javdhu authored Aug 11, 2022
2 parents fa28ca8 + 653da6e commit 8156fc9
Show file tree
Hide file tree
Showing 9 changed files with 307 additions and 12 deletions.
17 changes: 14 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -134,9 +134,10 @@ ardrive upload-file --wallet-file /path/to/my/wallet.json --parent-folder-id "f0
15. [Uploading Manifests](#uploading-manifests)
16. [Hosting a Webpage with Manifest](#hosting-a-webpage-with-manifest)
17. [Uploading With a Custom Content Type](#custom-content-type)
18. [Uploading a Custom Manifest](#custom-manifest)
19. [Uploading Files with Custom MetaData](#uploading-files-with-custom-metadata)
20. [Applying Unique Custom MetaData During Bulk Workflows](#applying-unique-custom-metadata-during-bulk-workflows)
18. [Uploading From a Remote URL](#remote-path)
19. [Uploading a Custom Manifest](#custom-manifest)
20. [Uploading Files with Custom MetaData](#uploading-files-with-custom-metadata)
21. [Applying Unique Custom MetaData During Bulk Workflows](#applying-unique-custom-metadata-during-bulk-workflows)
8. [Other Utility Operations](#other-utility-operations)
1. [Monitoring Transactions](#monitoring-transactions)
2. [Dealing With Network Congestion](#dealing-with-network-congestion)
Expand Down Expand Up @@ -1175,6 +1176,16 @@ It is currently possible to set this value to any given string, but the gateway
Note: In the case of multi-file uploads or recursive folder uploads, setting this `--content-type` flag will set the provided custom content type on EVERY file entity within a given upload.
### Uploading From a Remote URL<a id="remote-path"></a>
You can upload a file from an existing url using the `--remote-path` flag. This must be used in conjunction with `--dest-file-name`.
You can use a custom content type using the `--content-type` flag, but if this isn't used the app will use the content type from the response header of the request for the remote data.
```shell
ardrive upload-file --remote-path "https://url/to/file" --parent-folder-id "9af694f6-4cfc-4eee-88a8-1b02704760c0" -d "example.jpg" -w /path/to/wallet.json
```
### Uploading a Custom Manifest<a id="custom-manifest"></a>
Using the custom content type feature, it is possible for users to upload their own custom manifests. The Arweave gateways use this special content type in order to identify an uploaded file as a manifest:
Expand Down
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "ardrive-cli",
"version": "1.19.0",
"version": "1.20.0",
"description": "The ArDrive Command Line Interface (CLI is a Node.js application for terminal-based ArDrive workflows. It also offers utility operations for securely interacting with Arweave wallets and inspecting various Arweave blockchain conditions.",
"main": "./lib/index.js",
"bin": {
Expand Down
53 changes: 52 additions & 1 deletion src/commands/upload_file.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import {
LocalCSVParameter,
GatewayParameter,
CustomContentTypeParameter,
RemotePathParameter,
CustomMetaDataParameters,
IPFSParameter
} from '../parameter_declarations';
Expand All @@ -33,6 +34,9 @@ import {
import { cliArDriveFactory } from '..';
import * as fs from 'fs';
import { getArweaveFromURL } from '../utils/get_arweave_for_url';
import { cleanUpTempFolder, getTempFolder } from '../utils/temp_folder';
import { downloadFile } from '../utils/download_file';
import { showProgressLog } from '../utils/show_progress_log';

interface UploadPathParameter {
parentFolderId: FolderID;
Expand All @@ -50,7 +54,6 @@ function getFilesFromCSV(parameters: ParametersHelper): UploadPathParameter[] |
if (!localCSVFile) {
return undefined;
}

const localCSVFileData = fs.readFileSync(localCSVFile).toString().trim();
const COLUMN_SEPARATOR = ',';
const ROW_SEPARATOR = '\n';
Expand Down Expand Up @@ -135,6 +138,41 @@ async function getSingleFile(parameters: ParametersHelper, parentFolderId: Folde
return [singleParameter];
}

async function getRemoteFile(
parameters: ParametersHelper,
parentFolderId: FolderID
): Promise<UploadPathParameter[] | undefined> {
const remoteFilePath = parameters.getParameterValue(RemotePathParameter);
if (!remoteFilePath) {
return undefined;
}

const tempFolder = getTempFolder();
const destinationFileName = parameters.getRequiredParameterValue(DestinationFileNameParameter);

const { pathToFile, contentType } = await downloadFile(
remoteFilePath,
tempFolder,
destinationFileName,
(downloadProgress: number) => {
if (showProgressLog) {
process.stderr.write(`Downloading file... ${downloadProgress.toFixed(1)}% \r`);
}
}
);
process.stderr.clearLine(0);
const customContentType = parameters.getParameterValue(CustomContentTypeParameter);

const wrappedEntity = wrapFileOrFolder(pathToFile, customContentType ?? contentType);
const singleParameter = {
parentFolderId: parentFolderId,
wrappedEntity,
destinationFileName: parameters.getParameterValue(DestinationFileNameParameter)
};

return [singleParameter];
}

new CLICommand({
name: 'upload-file',
parameters: [
Expand All @@ -155,6 +193,7 @@ new CLICommand({
LocalFilesParameter_DEPRECATED,
BoostParameter,
GatewayParameter,
RemotePathParameter,
IPFSParameter
],
action: new CLIAction(async function action(options) {
Expand All @@ -174,6 +213,10 @@ new CLICommand({
if (fileList) {
return fileList;
}
const filesFromRemote = await getRemoteFile(parameters, parentFolderId);
if (filesFromRemote) {
return filesFromRemote;
}

// If neither the multi-file input case or csv case produced files, try the single file case (deprecated)
return getSingleFile(parameters, parentFolderId);
Expand All @@ -183,6 +226,7 @@ new CLICommand({

const conflictResolution = parameters.getFileNameConflictResolution();
const shouldBundle = !!parameters.getParameterValue(ShouldBundleParameter);
const remoteFilePath = parameters.getParameterValue(RemotePathParameter);

const arweave = getArweaveFromURL(parameters.getGateway());

Expand Down Expand Up @@ -228,7 +272,14 @@ new CLICommand({
prompts: fileAndFolderUploadConflictPrompts
});

if (remoteFilePath && results.created[0].type === 'file') {
// TODO: Include ArFSRemoteFileToUpload functionality in ArDrive Core
// TODO: Account for bulk remote path uploads in the future
results.created[0].sourceUri = remoteFilePath;
}

console.log(JSON.stringify(results, null, 4));
cleanUpTempFolder();
return SUCCESS_EXIT_CODE;
}

Expand Down
40 changes: 33 additions & 7 deletions src/parameter_declarations.ts
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ export const LocalCSVParameter = 'localCsv';
export const WithKeysParameter = 'withKeys';
export const GatewayParameter = 'gateway';
export const CustomContentTypeParameter = 'contentType';
export const RemotePathParameter = 'remotePath';
export const IPFSParameter = 'addIpfsTag';
export const DataGqlTagsParameter = 'dataGqlTags';
export const MetaDataFileParameter = 'metadataFile';
Expand Down Expand Up @@ -98,6 +99,7 @@ export const AllParameters = [
UnsafeDrivePasswordParameter,
WalletFileParameter,
WithKeysParameter,
RemotePathParameter,
IPFSParameter
] as const;
export type ParameterName = typeof AllParameters[number];
Expand Down Expand Up @@ -276,15 +278,17 @@ Parameter.declare({
\t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-file-path
\t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-files
\t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-paths
\t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-csv`,
\t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-csv
\t\t\t\t\t\t\t• Can NOT be used in conjunction with --remote-path`,
forbiddenConjunctionParameters: [LocalFilePathParameter_DEPRECATED, LocalPathsParameter, LocalCSVParameter]
});

Parameter.declare({
name: DestinationFileNameParameter,
aliases: ['-d', '--dest-file-name'],
description: `(OPTIONAL) a destination file name to use when uploaded to ArDrive
\t\t\t\t\t\t\t• Only valid for use with --local-path or --local-file-path`
description: `a destination file name to use when uploaded to ArDrive
\t\t\t\t\t\t\t• Required for use with --remote-path
\t\t\t\t\t\t\t• Optional when using with --local-path or --local-file-path`
});

Parameter.declare({
Expand All @@ -309,7 +313,8 @@ Parameter.declare({
\t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-path
\t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-paths
\t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-csv
\t\t\t\t\t\t\t• Can NOT be used in conjunction with --dest-file-name`,
\t\t\t\t\t\t\t• Can NOT be used in conjunction with --dest-file-name
\t\t\t\t\t\t\t• Can NOT be used in conjunction with --remote-path`,
forbiddenConjunctionParameters: [
LocalFilePathParameter_DEPRECATED,
LocalPathParameter,
Expand Down Expand Up @@ -410,7 +415,8 @@ Parameter.declare({
\t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-file-path
\t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-files
\t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-paths
\t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-csv`,
\t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-csv
\t\t\t\t\t\t\t• Can NOT be used in conjunction with --remote-path`,
forbiddenConjunctionParameters: [
LocalFilePathParameter_DEPRECATED,
LocalFilesParameter_DEPRECATED,
Expand All @@ -428,7 +434,8 @@ Parameter.declare({
\t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-files
\t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-path
\t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-csv
\t\t\t\t\t\t\t• Can NOT be used in conjunction with --dest-file-name`,
\t\t\t\t\t\t\t• Can NOT be used in conjunction with --dest-file-name
\t\t\t\t\t\t\t• Can NOT be used in conjunction with --remote-path`,
forbiddenConjunctionParameters: [
LocalFilePathParameter_DEPRECATED,
LocalFilesParameter_DEPRECATED,
Expand All @@ -454,7 +461,8 @@ Parameter.declare({
\t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-files
\t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-path
\t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-paths
\t\t\t\t\t\t\t• Can NOT be used in conjunction with --dest-file-name`,
\t\t\t\t\t\t\t• Can NOT be used in conjunction with --dest-file-name
\t\t\t\t\t\t\t• Can NOT be used in conjunction with --remote-path`,
forbiddenConjunctionParameters: [
LocalFilePathParameter_DEPRECATED,
LocalFilesParameter_DEPRECATED,
Expand Down Expand Up @@ -484,6 +492,24 @@ Parameter.declare({
'(OPTIONAL) Provide a custom content type to all files within the upload to be used by the gateway to display the content'
});

Parameter.declare({
name: RemotePathParameter,
aliases: ['--remote-path'],
description: `the remote path for the file that will be uploaded
\t\t\t\t\t\t\t• MUST be used in conjunction with --dest-file-name
\t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-file-path
\t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-files
\t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-paths
\t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-path
\t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-csv`,
requiredConjunctionParameters: [DestinationFileNameParameter],
forbiddenConjunctionParameters: [
LocalFilePathParameter_DEPRECATED,
LocalPathsParameter,
LocalCSVParameter,
LocalPathParameter
]
});
Parameter.declare({
name: IPFSParameter,
aliases: ['--add-ipfs-tag'],
Expand Down
33 changes: 33 additions & 0 deletions src/utils/download_file.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
import { expect } from 'chai';
import { cleanUpTempFolder, getTempFolder } from './temp_folder';
import { downloadFile } from './download_file';
import * as fs from 'fs';

describe('downloadFile function', () => {
const validDownloadLink = 'https://arweave.net/pVoSqZgJUCiNw7oS6CtlVEd8gREQlpRbccrsMLkeIuQ';
const invalidDownloadLink = 'https://arweave.net/pVoSqZgJUCiNw7oS6CtlVEV8gREQlpRbccrsMLkeIuQ';
const destinationFileName = 'cat.jpg';
const tempFolderPath = getTempFolder();
it('downloads a file into the provided folder when given a valid link', async () => {
const { pathToFile, contentType } = await downloadFile(validDownloadLink, tempFolderPath, destinationFileName);
expect(fs.existsSync(pathToFile)).to.equal(true);
expect(contentType).to.equal('image/jpeg');
});

it('download throws when given an invalid link', async () => {
let error;
try {
await downloadFile(invalidDownloadLink, tempFolderPath, destinationFileName);
} catch (err) {
error = err;
}
expect(error?.name).to.equal('Error');
expect(error?.message).to.equal(
'Failed to download file from remote path https://arweave.net/pVoSqZgJUCiNw7oS6CtlVEV8gREQlpRbccrsMLkeIuQ: Request failed with status code 404'
);
});

after(() => {
cleanUpTempFolder();
});
});
50 changes: 50 additions & 0 deletions src/utils/download_file.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
import * as fs from 'fs';
import path from 'path';
import axios from 'axios';
import util from 'util';
import stream from 'stream';

const pipeline = util.promisify(stream.pipeline);

type DownloadProgressCallback = (downloadProgress: number) => void;
type DownloadResult = { pathToFile: string; contentType: string };

/**
* Downloads file from remote HTTP[S] host and puts its contents to the
* specified location.
* @param url URL of the file to download.
* @param destinationPath Path to the destination file.
* @param destinationFileName The file name.
*/

export async function downloadFile(
url: string,
destinationPath: string,
destinationFileName: string,
downloadProgressCallback?: DownloadProgressCallback
): Promise<DownloadResult> {
const pathToFile = path.join(destinationPath, destinationFileName);
const writer = fs.createWriteStream(pathToFile);

try {
const { data, headers } = await axios({
method: 'get',
url: url,
responseType: 'stream'
});
const totalLength = headers['content-length'];
const contentType = headers['content-type'];
let downloadedLength = 0;
data.on('data', (chunk: string | unknown[]) => {
downloadedLength += chunk.length;
const downloadProgressPct = totalLength > 0 ? (downloadedLength / totalLength) * 100 : 0;

downloadProgressCallback && downloadProgressCallback(downloadProgressPct);
});
await pipeline(data, writer);
return { pathToFile, contentType };
} catch (error) {
writer.close();
throw new Error(`Failed to download file from remote path ${url}: ${error.message}`);
}
}
3 changes: 3 additions & 0 deletions src/utils/show_progress_log.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
const ARDRIVE_PROGRESS_LOG = 'ARDRIVE_PROGRESS_LOG';

export const showProgressLog = process.env[ARDRIVE_PROGRESS_LOG] && process.env[ARDRIVE_PROGRESS_LOG] === '1';
44 changes: 44 additions & 0 deletions src/utils/temp_folder.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
import { expect } from 'chai';
import { cleanUpTempFolder, getTempFolder } from './temp_folder';
import * as fs from 'fs';
import * as os from 'os';

describe('temp folder functions', () => {
describe('getTempFolder function', () => {
it('returns a folder that exists', () => {
const tempFolderPath = getTempFolder();
expect(fs.existsSync(tempFolderPath)).to.equal(true);
});

it('getTempFolder can be called twice in a row', () => {
const tempFolderPath = getTempFolder();
const tempFolderPath2 = getTempFolder();
expect(fs.existsSync(tempFolderPath)).to.equal(true);
expect(fs.existsSync(tempFolderPath2)).to.equal(true);
});

it('returns a folder that contains the correct subfolders', () => {
const tempFolderPath = getTempFolder();
const expectedPathComponent =
os.platform() === 'win32' ? '\\ardrive-downloads' : '/.ardrive/ardrive-downloads';
expect(tempFolderPath).to.contains(expectedPathComponent);
});
});

describe('cleanUpTempFolder function', () => {
it('cleanUpTempFolder removes the temporary folder from the local system', () => {
const tempFolderPath = getTempFolder();
expect(fs.existsSync(tempFolderPath)).to.equal(true);
cleanUpTempFolder();
expect(fs.existsSync(tempFolderPath)).to.equal(false);
});
it('cleanUpTempFolder can be called twice in a row', () => {
const tempFolderPath = getTempFolder();
expect(fs.existsSync(tempFolderPath)).to.equal(true);
cleanUpTempFolder();
expect(fs.existsSync(tempFolderPath)).to.equal(false);
cleanUpTempFolder();
expect(fs.existsSync(tempFolderPath)).to.equal(false);
});
});
});
Loading

0 comments on commit 8156fc9

Please sign in to comment.