feat: use jsipfs as fallback instead of public gateways (#44)

* feat: use jsipfs to add/get files instead of public gateway
* fix: download all files
* fix: files progress
* feat: warn about large files
* fix: broken companion link and copy

package.json

@@ -33,8 +33,8 @@
     "i18next-icu": "^0.6.0",
     "i18next-xhr-backend": "^1.5.1",
     "internal-nav-helper": "^1.0.2",
-    "ipfs-css": "^0.10.0",
-    "ipfs-redux-bundle": "^2.0.0",
+    "ipfs-css": "^0.12.0",
+    "ipfs-redux-bundle": "^4.1.2",
     "is-ipfs": "^0.4.2",
     "milliseconds": "^1.0.3",
     "prop-types": "^15.6.2",
@@ -74,8 +74,8 @@
     "ecstatic": "^3.2.1",
     "eslint": "^5.4.0",
     "eslint-plugin-react": "^7.11.1",
-    "ipfs": "^0.32.0",
-    "ipfsd-ctl": "^0.39.2",
+    "ipfs": "^0.33.1",
+    "ipfsd-ctl": "^0.40.0",
     "multihashing-async": "^0.5.1",
     "npm-run-all": "^4.1.3",
     "puppeteer": "^1.7.0",

public/locales/en/translation.json

@@ -21,7 +21,8 @@
     "runningDaemon": "You need a <1>running daemon</1> to add files to IPFS.",
     "configureDaemon": "Make sure you <1>configure your IPFS API</1> to allow cross-origin (CORS) requests:",
     "runDaemon": "Then, start an IPFS daemon in your terminal:",
-    "footNote": "If no other nodes pin your files and you stop serving them, they will disappear from the network."
+    "footNote": "If no other nodes pin your files and you stop serving them, they will disappear from the network.",
+    "largeFilesWarning": "If you experience issues when downloading files larger than 1GB, please install <1>IPFS Desktop</1> or <3>IPFS Companion</3> for improved performance."
   },
   "addFiles": "Add files",
   "copyLink": {
@@ -32,6 +33,6 @@
   "loader": "Loading file list...",
   "downloadFiles": {
     "downloadAll": "Download all",
-    "downloading": "Archiving..."
+    "downloading": "Downloading..."
   }
 }

public/locales/pt/translation.json

@@ -21,7 +21,8 @@
     "runningDaemon": "Precisas de um <1>daemon a correr</1> para adicionares ficheiros ao IPFS.",
     "configureDaemon": "Certifica-te que <1>configuras a tua IPFS API</1> para aceitar pedidos cross-origin (CORS):",
     "runDaemon": "Depois, corre um IPFS daemon no terminal:",
-    "footNote": "Se outros nós não servirem os ficheiros e tu parares de os servir, eles vão desaparecer da rede."
+    "footNote": "Se outros nós não servirem os ficheiros e tu parares de os servir, eles vão desaparecer da rede.",
+    "largeFilesWarning": "Se tiveres problemas a descarregar ficheiros maiores que 1GB, instala o <1>IPFS Desktop</1> ou <3>IPFS Companion</3> para uma melhor experiência."
   },
   "addFiles": "Adicionar ficheiros",
   "copyLink": {
@@ -32,6 +33,6 @@
   "loader": "A carregar a lista de ficheiros...",
   "downloadFiles": {
     "downloadAll": "Descarregar todos",
-    "downloading": "A arquivar..."
+    "downloading": "A descarregar..."
   }
 }

src/bundles/files.js

@@ -1,11 +1,15 @@
 import { createSelector } from 'redux-bundler'
 import shortid from 'shortid'
-import { filesToStreams, makeHashFromFiles, getDownloadLink } from '../lib/files'
+import { filesToStreams, makeHashFromFiles } from '../lib/files'
 import ENDPOINTS from '../constants/endpoints'
 import PAGES from '../constants/pages'
 
 const initialState = {
   files: {},
+  limits: {
+    maxSize: 1073741824, // 1GB
+    hasExceeded: false
+  },
   shareLink: {
     outdated: false,
     link: null
@@ -124,6 +128,67 @@ export default {
           }
         }
 
+      case 'FILES_MAX_SIZE_EXCEEDED':
+        return {
+          ...state,
+          limits: {
+            ...state.limits,
+            hasExceeded: true
+          }
+        }
+
+      case 'FILES_DOWNLOAD_STARTED':
+        return {
+          ...state,
+          files: {
+            ...state.files,
+            [action.payload.id]: {
+              ...state.files[action.payload.id],
+              progress: 0,
+              pending: true
+            }
+          }
+        }
+
+      case 'FILES_DOWNLOAD_PROGRESS':
+        return {
+          ...state,
+          files: {
+            ...state.files,
+            [action.payload.id]: {
+              ...state.files[action.payload.id],
+              progress: action.payload.progress
+            }
+          }
+        }
+
+      case 'FILES_DOWNLOAD_FINISHED':
+        return {
+          ...state,
+          files: {
+            ...state.files,
+            [action.payload.id]: {
+              ...state.files[action.payload.id],
+              progress: 100,
+              pending: false
+            }
+          },
+          error: null
+        }
+
+      case 'FILES_DOWNLOAD_FAILED':
+        return {
+          ...state,
+          files: {
+            ...state.files,
+            [action.payload.id]: {
+              ...state.files[action.payload.id],
+              progress: 100,
+              pending: false
+            }
+          }
+        }
+
       case 'FILES_RESET':
         return initialState
 
@@ -138,6 +203,10 @@ export default {
 
   selectIsLoading: state => state.files.loading,
 
+  selectMaxFileSize: state => state.files.limits.maxSize,
+
+  selectHasExceededMaxSize: state => state.files.limits.hasExceeded,
+
   selectFiles: state => state.files.files,
 
   selectExistFiles: createSelector(
@@ -186,6 +255,7 @@ export default {
 
       const file = {
         [fileId]: {
+          id: fileId,
           name: fileName,
           size: fileSize,
           progress: 0,
@@ -202,7 +272,7 @@ export default {
       }
 
       try {
-        const addedFile = await ipfs.add(stream, { pin: false, progress: updateProgress })
+        const addedFile = await ipfs.files.add(stream, { pin: false, progress: updateProgress })
         dispatch({ type: 'FILES_ADD_FINISHED', payload: { id: fileId, hash: addedFile[0].hash } })
       } catch (err) {
         dispatch({ type: 'FILES_ADD_FAILED', payload: { id: fileId, error: err.message } })
@@ -242,18 +312,26 @@ export default {
         ipfsFiles = objs.Objects[0].Links
       }
 
+      const maxSize = store.selectMaxFileSize()
+
       for (const file of ipfsFiles) {
+        const fileId = shortid.generate()
         const fileName = file.name || file.Name
         const fileSize = file.size || file.Size
         const fileHash = file.hash || file.Hash
 
-        files[fileName] = {
+        files[fileId] = {
+          id: fileId,
           name: fileName,
           size: fileSize,
           hash: fileHash,
           progress: 100,
           pending: false
         }
+
+        if (file.size > maxSize) {
+          dispatch({ type: 'FILES_MAX_SIZE_EXCEEDED' })
+        }
       }
 
       dispatch({ type: 'FILES_FETCH_FINISHED', payload: { files: files } })
@@ -262,10 +340,51 @@ export default {
     }
   },
 
-  doGetDownloadLink: (files) => async ({ dispatch, store, getIpfs }) => {
-    const ipfs = getIpfs()
-    dispatch({ type: 'FILES_GET_DOWNLOAD_LINK' })
-    return getDownloadLink(files, ipfs)
+  doDownloadFile: (id, hash) => async ({ dispatch, getIpfs }) => {
+    return new Promise((resolve, reject) => {
+      const ipfs = getIpfs()
+      dispatch({ type: 'FILES_DOWNLOAD_STARTED', payload: { id: id } })
+
+      try {
+        const stream = ipfs.files.getReadableStream(hash)
+
+        stream.on('data', (file) => {
+          let chunks = []
+          let bytesLoaded = 0
+
+          file.content
+            .on('data', (chunk) => {
+              bytesLoaded += chunk.byteLength
+              const progress = Math.round((bytesLoaded / file.size) * 100)
+
+              dispatch({ type: 'FILES_DOWNLOAD_PROGRESS', payload: { id: id, progress: progress } })
+              chunks.push(chunk)
+            })
+            .on('end', () => {
+              // Get the total length of all arrays
+              let length = 0
+              chunks.forEach(arr => {
+                length += arr.length
+              })
+
+              // Create a new array with total length and merge all source arrays
+              let mergedArray = new Uint8Array(length)
+              let offset = 0
+              chunks.forEach(item => {
+                mergedArray.set(item, offset)
+                offset += item.length
+              })
+
+              dispatch({ type: 'FILES_DOWNLOAD_FINISHED', payload: { id: id } })
+              resolve(mergedArray)
+            })
+
+          file.content.resume()
+        })
+      } catch (err) {
+        dispatch({ type: 'FILES_DOWNLOAD_FAILED', payload: { id: id } })
+      }
+    })
   },
 
   doResetFiles: () => ({ dispatch }) => dispatch({ type: 'FILES_RESET' })

src/bundles/index.js

@@ -7,7 +7,10 @@ import filesBundle from './files'
 
 export default composeBundles(
   appIdle({idleTimeout: 5000}),
-  ipfsBundle(),
+  ipfsBundle({
+    tryJsIpfs: true,
+    getJsIpfs: () => import('ipfs')
+  }),
   routesBundle,
   redirectsBundle,
   filesBundle

src/components/box/Box.js

@@ -14,9 +14,17 @@ export const Box = ({ children }) => (
   </div>
 )
 
-export const BoxDownload = ({ files, isLoading }) => (
+export const BoxDownload = ({ files, isLoading, showSizeWarning }) => (
   <Box>
     { isLoading && <Loader /> }
+    {showSizeWarning && <div className='mb4' style={{ borderLeft: '3px solid #ffcc00' }}>
+      <p className='mv0 pl3 navy f6 lh-copy'>
+        <Trans i18nKey='box.largeFilesWarning'>
+          You may experience issues when downloading files larger than 1GB. Please use <a className='link aqua underline-hover' href='https://github.com/ipfs-shipyard/ipfs-desktop/' target='_blank' rel='noopener noreferrer'>IPFS Desktop</a>
+          or <a className='link aqua underline-hover' href='https://github.com/ipfs-shipyard/ipfs-companion/' target='_blank' rel='noopener noreferrer'>IPFS Companion</a> to do so.
+        </Trans>
+      </p>
+    </div> }
     <FileTree files={files} isDownload />
     <DownloadFiles />
   </Box>

src/components/download-files/DownloadFiles.js

@@ -3,7 +3,6 @@ import PropTypes from 'prop-types'
 import { connect } from 'redux-bundler-react'
 import { translate } from 'react-i18next'
 import classnames from 'classnames'
-import CircularProgressbar from 'react-circular-progressbar'
 import downloadFile from '../file/utils/download'
 
 // Styles
@@ -12,40 +11,38 @@ import 'react-circular-progressbar/dist/styles.css'
 export class DownloadFiles extends React.Component {
   static propTypes = {
     files: PropTypes.object,
-    doGetDownloadLink: PropTypes.func
+    doDownloadFile: PropTypes.func
   }
 
   state = {
-    progress: null
+    isDownloading: false
   }
 
   handleOnClick = async () => {
-    const { files, doGetDownloadLink } = this.props
-    const updater = (v) => this.setState({ progress: v })
-    const { url, filename } = await doGetDownloadLink(Object.values(files))
-    downloadFile(url, filename, updater)
+    const { files, doDownloadFile } = this.props
+    this.setState({ isDownloading: true })
+
+    for (const file of Object.values(files)) {
+      const fileContent = await doDownloadFile(file.id, file.hash)
+      downloadFile(fileContent, file.name)
+    }
+
+    this.setState({ isDownloading: false })
   }
 
   render () {
     const { t } = this.props
     const btnClass = classnames({
-      'ba b--navy bg-white navy no-pointer-events': this.state.progress !== null,
-      'bg-navy white glow pointer': this.state.progress === null
+      'ba b--navy bg-white navy no-pointer-events': this.state.isDownloading === true,
+      'bg-navy white glow pointer': this.state.isDownloading === false
     }, ['pa2 mb2 w-40 flex justify-center items-center br-pill f6 o-80'])
 
     return (
       <div className={btnClass} onClick={this.handleOnClick}>
-        { this.state.progress === null
+        { this.state.isDownloading === false
           ? <span>{t('downloadFiles.downloadAll')}</span>
           : <div className='flex items-center'>
             {t('downloadFiles.downloading')}
-            <CircularProgressbar
-              percentage={this.state.progress}
-              strokeWidth={50}
-              styles={{
-                root: { width: 12, height: 12, marginLeft: 10 },
-                path: { stroke: '#3e6175', strokeLinecap: 'butt' }
-              }} />
           </div> }
       </div>
     )
@@ -56,6 +53,6 @@ export const TranslatedDownloadFiles = translate()(DownloadFiles)
 
 export default connect(
   'selectFiles',
-  'doGetDownloadLink',
+  'doDownloadFile',
   TranslatedDownloadFiles
 )