diff --git a/README.md b/README.md index d968cda6..62492482 100644 --- a/README.md +++ b/README.md @@ -134,9 +134,10 @@ ardrive upload-file --wallet-file /path/to/my/wallet.json --parent-folder-id "f0 15. [Uploading Manifests](#uploading-manifests) 16. [Hosting a Webpage with Manifest](#hosting-a-webpage-with-manifest) 17. [Uploading With a Custom Content Type](#custom-content-type) - 18. [Uploading a Custom Manifest](#custom-manifest) - 19. [Uploading Files with Custom MetaData](#uploading-files-with-custom-metadata) - 20. [Applying Unique Custom MetaData During Bulk Workflows](#applying-unique-custom-metadata-during-bulk-workflows) + 18. [Uploading From a Remote URL](#remote-path) + 19. [Uploading a Custom Manifest](#custom-manifest) + 20. [Uploading Files with Custom MetaData](#uploading-files-with-custom-metadata) + 21. [Applying Unique Custom MetaData During Bulk Workflows](#applying-unique-custom-metadata-during-bulk-workflows) 8. [Other Utility Operations](#other-utility-operations) 1. [Monitoring Transactions](#monitoring-transactions) 2. [Dealing With Network Congestion](#dealing-with-network-congestion) @@ -1175,6 +1176,16 @@ It is currently possible to set this value to any given string, but the gateway Note: In the case of multi-file uploads or recursive folder uploads, setting this `--content-type` flag will set the provided custom content type on EVERY file entity within a given upload. +### Uploading From a Remote URL + +You can upload a file from an existing url using the `--remote-path` flag. This must be used in conjunction with `--dest-file-name`. + +You can use a custom content type using the `--content-type` flag, but if this isn't used the app will use the content type from the response header of the request for the remote data. + +```shell +ardrive upload-file --remote-path "https://url/to/file" --parent-folder-id "9af694f6-4cfc-4eee-88a8-1b02704760c0" -d "example.jpg" -w /path/to/wallet.json +``` + ### Uploading a Custom Manifest Using the custom content type feature, it is possible for users to upload their own custom manifests. The Arweave gateways use this special content type in order to identify an uploaded file as a manifest: diff --git a/package.json b/package.json index 37b73378..27cf799e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "ardrive-cli", - "version": "1.19.0", + "version": "1.20.0", "description": "The ArDrive Command Line Interface (CLI is a Node.js application for terminal-based ArDrive workflows. It also offers utility operations for securely interacting with Arweave wallets and inspecting various Arweave blockchain conditions.", "main": "./lib/index.js", "bin": { diff --git a/src/commands/upload_file.ts b/src/commands/upload_file.ts index 4f495cf3..62ae3b7e 100644 --- a/src/commands/upload_file.ts +++ b/src/commands/upload_file.ts @@ -14,6 +14,7 @@ import { LocalCSVParameter, GatewayParameter, CustomContentTypeParameter, + RemotePathParameter, CustomMetaDataParameters, IPFSParameter } from '../parameter_declarations'; @@ -33,6 +34,9 @@ import { import { cliArDriveFactory } from '..'; import * as fs from 'fs'; import { getArweaveFromURL } from '../utils/get_arweave_for_url'; +import { cleanUpTempFolder, getTempFolder } from '../utils/temp_folder'; +import { downloadFile } from '../utils/download_file'; +import { showProgressLog } from '../utils/show_progress_log'; interface UploadPathParameter { parentFolderId: FolderID; @@ -50,7 +54,6 @@ function getFilesFromCSV(parameters: ParametersHelper): UploadPathParameter[] | if (!localCSVFile) { return undefined; } - const localCSVFileData = fs.readFileSync(localCSVFile).toString().trim(); const COLUMN_SEPARATOR = ','; const ROW_SEPARATOR = '\n'; @@ -135,6 +138,41 @@ async function getSingleFile(parameters: ParametersHelper, parentFolderId: Folde return [singleParameter]; } +async function getRemoteFile( + parameters: ParametersHelper, + parentFolderId: FolderID +): Promise { + const remoteFilePath = parameters.getParameterValue(RemotePathParameter); + if (!remoteFilePath) { + return undefined; + } + + const tempFolder = getTempFolder(); + const destinationFileName = parameters.getRequiredParameterValue(DestinationFileNameParameter); + + const { pathToFile, contentType } = await downloadFile( + remoteFilePath, + tempFolder, + destinationFileName, + (downloadProgress: number) => { + if (showProgressLog) { + process.stderr.write(`Downloading file... ${downloadProgress.toFixed(1)}% \r`); + } + } + ); + process.stderr.clearLine(0); + const customContentType = parameters.getParameterValue(CustomContentTypeParameter); + + const wrappedEntity = wrapFileOrFolder(pathToFile, customContentType ?? contentType); + const singleParameter = { + parentFolderId: parentFolderId, + wrappedEntity, + destinationFileName: parameters.getParameterValue(DestinationFileNameParameter) + }; + + return [singleParameter]; +} + new CLICommand({ name: 'upload-file', parameters: [ @@ -155,6 +193,7 @@ new CLICommand({ LocalFilesParameter_DEPRECATED, BoostParameter, GatewayParameter, + RemotePathParameter, IPFSParameter ], action: new CLIAction(async function action(options) { @@ -174,6 +213,10 @@ new CLICommand({ if (fileList) { return fileList; } + const filesFromRemote = await getRemoteFile(parameters, parentFolderId); + if (filesFromRemote) { + return filesFromRemote; + } // If neither the multi-file input case or csv case produced files, try the single file case (deprecated) return getSingleFile(parameters, parentFolderId); @@ -183,6 +226,7 @@ new CLICommand({ const conflictResolution = parameters.getFileNameConflictResolution(); const shouldBundle = !!parameters.getParameterValue(ShouldBundleParameter); + const remoteFilePath = parameters.getParameterValue(RemotePathParameter); const arweave = getArweaveFromURL(parameters.getGateway()); @@ -228,7 +272,14 @@ new CLICommand({ prompts: fileAndFolderUploadConflictPrompts }); + if (remoteFilePath && results.created[0].type === 'file') { + // TODO: Include ArFSRemoteFileToUpload functionality in ArDrive Core + // TODO: Account for bulk remote path uploads in the future + results.created[0].sourceUri = remoteFilePath; + } + console.log(JSON.stringify(results, null, 4)); + cleanUpTempFolder(); return SUCCESS_EXIT_CODE; } diff --git a/src/parameter_declarations.ts b/src/parameter_declarations.ts index 02453912..ec29a44b 100644 --- a/src/parameter_declarations.ts +++ b/src/parameter_declarations.ts @@ -41,6 +41,7 @@ export const LocalCSVParameter = 'localCsv'; export const WithKeysParameter = 'withKeys'; export const GatewayParameter = 'gateway'; export const CustomContentTypeParameter = 'contentType'; +export const RemotePathParameter = 'remotePath'; export const IPFSParameter = 'addIpfsTag'; export const DataGqlTagsParameter = 'dataGqlTags'; export const MetaDataFileParameter = 'metadataFile'; @@ -98,6 +99,7 @@ export const AllParameters = [ UnsafeDrivePasswordParameter, WalletFileParameter, WithKeysParameter, + RemotePathParameter, IPFSParameter ] as const; export type ParameterName = typeof AllParameters[number]; @@ -276,15 +278,17 @@ Parameter.declare({ \t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-file-path \t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-files \t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-paths -\t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-csv`, +\t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-csv +\t\t\t\t\t\t\t• Can NOT be used in conjunction with --remote-path`, forbiddenConjunctionParameters: [LocalFilePathParameter_DEPRECATED, LocalPathsParameter, LocalCSVParameter] }); Parameter.declare({ name: DestinationFileNameParameter, aliases: ['-d', '--dest-file-name'], - description: `(OPTIONAL) a destination file name to use when uploaded to ArDrive -\t\t\t\t\t\t\t• Only valid for use with --local-path or --local-file-path` + description: `a destination file name to use when uploaded to ArDrive +\t\t\t\t\t\t\t• Required for use with --remote-path +\t\t\t\t\t\t\t• Optional when using with --local-path or --local-file-path` }); Parameter.declare({ @@ -309,7 +313,8 @@ Parameter.declare({ \t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-path \t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-paths \t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-csv -\t\t\t\t\t\t\t• Can NOT be used in conjunction with --dest-file-name`, +\t\t\t\t\t\t\t• Can NOT be used in conjunction with --dest-file-name +\t\t\t\t\t\t\t• Can NOT be used in conjunction with --remote-path`, forbiddenConjunctionParameters: [ LocalFilePathParameter_DEPRECATED, LocalPathParameter, @@ -410,7 +415,8 @@ Parameter.declare({ \t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-file-path \t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-files \t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-paths -\t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-csv`, +\t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-csv +\t\t\t\t\t\t\t• Can NOT be used in conjunction with --remote-path`, forbiddenConjunctionParameters: [ LocalFilePathParameter_DEPRECATED, LocalFilesParameter_DEPRECATED, @@ -428,7 +434,8 @@ Parameter.declare({ \t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-files \t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-path \t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-csv -\t\t\t\t\t\t\t• Can NOT be used in conjunction with --dest-file-name`, +\t\t\t\t\t\t\t• Can NOT be used in conjunction with --dest-file-name +\t\t\t\t\t\t\t• Can NOT be used in conjunction with --remote-path`, forbiddenConjunctionParameters: [ LocalFilePathParameter_DEPRECATED, LocalFilesParameter_DEPRECATED, @@ -454,7 +461,8 @@ Parameter.declare({ \t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-files \t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-path \t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-paths -\t\t\t\t\t\t\t• Can NOT be used in conjunction with --dest-file-name`, +\t\t\t\t\t\t\t• Can NOT be used in conjunction with --dest-file-name +\t\t\t\t\t\t\t• Can NOT be used in conjunction with --remote-path`, forbiddenConjunctionParameters: [ LocalFilePathParameter_DEPRECATED, LocalFilesParameter_DEPRECATED, @@ -484,6 +492,24 @@ Parameter.declare({ '(OPTIONAL) Provide a custom content type to all files within the upload to be used by the gateway to display the content' }); +Parameter.declare({ + name: RemotePathParameter, + aliases: ['--remote-path'], + description: `the remote path for the file that will be uploaded +\t\t\t\t\t\t\t• MUST be used in conjunction with --dest-file-name +\t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-file-path +\t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-files +\t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-paths +\t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-path +\t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-csv`, + requiredConjunctionParameters: [DestinationFileNameParameter], + forbiddenConjunctionParameters: [ + LocalFilePathParameter_DEPRECATED, + LocalPathsParameter, + LocalCSVParameter, + LocalPathParameter + ] +}); Parameter.declare({ name: IPFSParameter, aliases: ['--add-ipfs-tag'], diff --git a/src/utils/download_file.test.ts b/src/utils/download_file.test.ts new file mode 100644 index 00000000..941e26b4 --- /dev/null +++ b/src/utils/download_file.test.ts @@ -0,0 +1,33 @@ +import { expect } from 'chai'; +import { cleanUpTempFolder, getTempFolder } from './temp_folder'; +import { downloadFile } from './download_file'; +import * as fs from 'fs'; + +describe('downloadFile function', () => { + const validDownloadLink = 'https://arweave.net/pVoSqZgJUCiNw7oS6CtlVEd8gREQlpRbccrsMLkeIuQ'; + const invalidDownloadLink = 'https://arweave.net/pVoSqZgJUCiNw7oS6CtlVEV8gREQlpRbccrsMLkeIuQ'; + const destinationFileName = 'cat.jpg'; + const tempFolderPath = getTempFolder(); + it('downloads a file into the provided folder when given a valid link', async () => { + const { pathToFile, contentType } = await downloadFile(validDownloadLink, tempFolderPath, destinationFileName); + expect(fs.existsSync(pathToFile)).to.equal(true); + expect(contentType).to.equal('image/jpeg'); + }); + + it('download throws when given an invalid link', async () => { + let error; + try { + await downloadFile(invalidDownloadLink, tempFolderPath, destinationFileName); + } catch (err) { + error = err; + } + expect(error?.name).to.equal('Error'); + expect(error?.message).to.equal( + 'Failed to download file from remote path https://arweave.net/pVoSqZgJUCiNw7oS6CtlVEV8gREQlpRbccrsMLkeIuQ: Request failed with status code 404' + ); + }); + + after(() => { + cleanUpTempFolder(); + }); +}); diff --git a/src/utils/download_file.ts b/src/utils/download_file.ts new file mode 100644 index 00000000..db7e57fc --- /dev/null +++ b/src/utils/download_file.ts @@ -0,0 +1,50 @@ +import * as fs from 'fs'; +import path from 'path'; +import axios from 'axios'; +import util from 'util'; +import stream from 'stream'; + +const pipeline = util.promisify(stream.pipeline); + +type DownloadProgressCallback = (downloadProgress: number) => void; +type DownloadResult = { pathToFile: string; contentType: string }; + +/** + * Downloads file from remote HTTP[S] host and puts its contents to the + * specified location. + * @param url URL of the file to download. + * @param destinationPath Path to the destination file. + * @param destinationFileName The file name. + */ + +export async function downloadFile( + url: string, + destinationPath: string, + destinationFileName: string, + downloadProgressCallback?: DownloadProgressCallback +): Promise { + const pathToFile = path.join(destinationPath, destinationFileName); + const writer = fs.createWriteStream(pathToFile); + + try { + const { data, headers } = await axios({ + method: 'get', + url: url, + responseType: 'stream' + }); + const totalLength = headers['content-length']; + const contentType = headers['content-type']; + let downloadedLength = 0; + data.on('data', (chunk: string | unknown[]) => { + downloadedLength += chunk.length; + const downloadProgressPct = totalLength > 0 ? (downloadedLength / totalLength) * 100 : 0; + + downloadProgressCallback && downloadProgressCallback(downloadProgressPct); + }); + await pipeline(data, writer); + return { pathToFile, contentType }; + } catch (error) { + writer.close(); + throw new Error(`Failed to download file from remote path ${url}: ${error.message}`); + } +} diff --git a/src/utils/show_progress_log.ts b/src/utils/show_progress_log.ts new file mode 100644 index 00000000..16393c25 --- /dev/null +++ b/src/utils/show_progress_log.ts @@ -0,0 +1,3 @@ +const ARDRIVE_PROGRESS_LOG = 'ARDRIVE_PROGRESS_LOG'; + +export const showProgressLog = process.env[ARDRIVE_PROGRESS_LOG] && process.env[ARDRIVE_PROGRESS_LOG] === '1'; diff --git a/src/utils/temp_folder.test.ts b/src/utils/temp_folder.test.ts new file mode 100644 index 00000000..069b6516 --- /dev/null +++ b/src/utils/temp_folder.test.ts @@ -0,0 +1,44 @@ +import { expect } from 'chai'; +import { cleanUpTempFolder, getTempFolder } from './temp_folder'; +import * as fs from 'fs'; +import * as os from 'os'; + +describe('temp folder functions', () => { + describe('getTempFolder function', () => { + it('returns a folder that exists', () => { + const tempFolderPath = getTempFolder(); + expect(fs.existsSync(tempFolderPath)).to.equal(true); + }); + + it('getTempFolder can be called twice in a row', () => { + const tempFolderPath = getTempFolder(); + const tempFolderPath2 = getTempFolder(); + expect(fs.existsSync(tempFolderPath)).to.equal(true); + expect(fs.existsSync(tempFolderPath2)).to.equal(true); + }); + + it('returns a folder that contains the correct subfolders', () => { + const tempFolderPath = getTempFolder(); + const expectedPathComponent = + os.platform() === 'win32' ? '\\ardrive-downloads' : '/.ardrive/ardrive-downloads'; + expect(tempFolderPath).to.contains(expectedPathComponent); + }); + }); + + describe('cleanUpTempFolder function', () => { + it('cleanUpTempFolder removes the temporary folder from the local system', () => { + const tempFolderPath = getTempFolder(); + expect(fs.existsSync(tempFolderPath)).to.equal(true); + cleanUpTempFolder(); + expect(fs.existsSync(tempFolderPath)).to.equal(false); + }); + it('cleanUpTempFolder can be called twice in a row', () => { + const tempFolderPath = getTempFolder(); + expect(fs.existsSync(tempFolderPath)).to.equal(true); + cleanUpTempFolder(); + expect(fs.existsSync(tempFolderPath)).to.equal(false); + cleanUpTempFolder(); + expect(fs.existsSync(tempFolderPath)).to.equal(false); + }); + }); +}); diff --git a/src/utils/temp_folder.ts b/src/utils/temp_folder.ts new file mode 100644 index 00000000..ad9e0c58 --- /dev/null +++ b/src/utils/temp_folder.ts @@ -0,0 +1,77 @@ +import * as fs from 'fs'; +import * as os from 'os'; +import path from 'path'; + +const isWindows = os.platform() === 'win32'; + +function getOrCreateTempBaseFolder(): string { + if (isWindows) { + return getValidWindowsTempPath() ?? getManualWindowsTempPath(); + } else { + return getValidUnixTempPath() ?? getManualUnixTempPath(); + } +} + +function getValidWindowsTempPath(): string | null { + const envTempFolder = process.env['TEMP'] ?? process.env['TMP']; + if (envTempFolder) { + if (fs.existsSync(envTempFolder)) { + return envTempFolder; + } else { + const userProfile = process.env['USERPROFILE']; + if (userProfile) { + const tempPath = path.join(userProfile, 'AppData', 'Local', 'Temp'); + if (fs.existsSync(tempPath)) { + return tempPath; + } + } + } + } + return null; +} + +function getValidUnixTempPath(): string | null { + const envTempFolder = process.env['TMPDIR'] ?? process.env['TMP'] ?? process.env['TEMP']; + if (envTempFolder) { + if (fs.existsSync(envTempFolder)) { + return envTempFolder; + } else { + const tempPath = '/tmp'; + if (fs.existsSync(tempPath)) { + return tempPath; + } + } + } + return null; +} + +function getManualWindowsTempPath(): string { + return path.join(os.homedir(), 'ardrive-temp'); +} + +function getManualUnixTempPath(): string { + return path.join(os.homedir(), '.ardrive'); +} + +function platformTempFolder(): string { + const tempBaseFolder = getOrCreateTempBaseFolder(); + return path.join(tempBaseFolder, 'ardrive-downloads'); +} +/** + * Gets a folder path for storing temporary files. + */ +export function getTempFolder(): string { + const tempFolderPath = platformTempFolder(); + if (fs.existsSync(tempFolderPath)) { + return tempFolderPath; + } + + fs.mkdirSync(tempFolderPath, { recursive: true }); + + return tempFolderPath; +} + +export function cleanUpTempFolder(): void { + const tempFolderPath = platformTempFolder(); + fs.rmdirSync(tempFolderPath, { recursive: true }); +}