Merge pull request #35 from 45Drives/dev-josh

Fix mangling large files during upload

CHANGELOG.md

@@ -1,3 +1,3 @@
-## Cockpit Navigator 0.5.5-2
+## Cockpit Navigator 0.5.6-1
 
-* Fix sed command for packaging EL7.
+* Fix mangling of large files during upload.

manifest.json

@@ -3,8 +3,8 @@
  "name": "cockpit-navigator",
  "title": "Cockpit Navigator",
  "prerelease": false,
- "version": "0.5.5",
- "buildVersion": "2",
+ "version": "0.5.6",
+ "buildVersion": "1",
  "author": "Josh Boudreau <jboudreau@45drives.com>",
  "url": "https://github.com/45Drives/cockpit-navigator",
  "category": "utils",
@@ -54,8 +54,8 @@
  ],
  "changelog": {
  "urgency": "medium",
- "version": "0.5.5",
- "buildVersion": "2",
+ "version": "0.5.6",
+ "buildVersion": "1",
  "ignore": [],
  "date": null,
  "packager": "Josh Boudreau <jboudreau@45drives.com>",

navigator/components/FileUpload.js

@@ -148,13 +148,13 @@ export class FileUpload {
  this.chunk_index++;
  this.progress.value = this.chunk_index;
  if (this.chunk_index < this.num_chunks)
- this.reader.readAsArrayBuffer(this.chunks[this.chunk_index]);
+ this.reader.readAsDataURL(this.chunks[this.chunk_index]);
  else {
  this.done();
  }
  };
  try {
- this.reader.readAsArrayBuffer(this.chunks[0]);
+ this.reader.readAsDataURL(this.chunks[0]);
  } catch {
  this.reader.onload = () => {};
  if (this.using_webkit) {
@@ -167,27 +167,12 @@ export class FileUpload {
  this.update_rates_interval = setInterval(this.display_xfr_rate.bind(this), 1000);
  }
 
- /**
- * 
- * @param {ArrayBuffer} buffer 
- * @returns 
- */
- arrayBufferToBase64(buffer) {
- let binary = '';
- let bytes = new Uint8Array(buffer);
- let len = bytes.byteLength;
- for (let i = 0; i < len; i++) {
- binary += String.fromCharCode(bytes[i]);
- }
- return window.btoa(binary);
- }
-
  /**
  * 
  * @param {Event} evt 
  */
  write_to_file(evt) {
- var chunk_b64 = this.arrayBufferToBase64(evt.target.result);
+ var chunk_b64 = evt.target.result.replace(/^data:[^\/]+\/[^;]+;base64,/, "");
  const seek = this.chunk_index * this.chunk_size;
  var obj = {
  seek: seek,

navigator/scripts/write-chunks.py3

@@ -1,28 +1,28 @@
 #!/usr/bin/env python3
 
 """
- Cockpit Navigator - A File System Browser for Cockpit.
- Copyright (C) 2021 Josh Boudreau <jboudreau@45drives.com>
+ Cockpit Navigator - A File System Browser for Cockpit.
+ Copyright (C) 2021 Josh Boudreau <jboudreau@45drives.com>
 
- This file is part of Cockpit Navigator.
- Cockpit Navigator is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
- Cockpit Navigator is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
- You should have received a copy of the GNU General Public License
- along with Cockpit Navigator. If not, see <https://www.gnu.org/licenses/>.
+ This file is part of Cockpit Navigator.
+ Cockpit Navigator is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+ Cockpit Navigator is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+ You should have received a copy of the GNU General Public License
+ along with Cockpit Navigator. If not, see <https://www.gnu.org/licenses/>.
 """
 
 """
 Synopsis: `echo <newline delimited JSON objects> | write-chunks.py3`
 JSON objects are of form:
 obj = {
- seek: <byte offset>
- chunk: <base64 encoded data chunk>
+ seek: <byte offset>
+ chunk: <base64 encoded data chunk>
 }
 """
 
@@ -31,53 +31,48 @@ import os
 import sys
 import json
 
-def write_chunk(chunk, file):
- if not file:
- path = sys.argv[1]
- parent_path = os.path.dirname(path)
- try:
- if not os.path.exists(parent_path):
- os.makedirs(parent_path, exist_ok=True)
- elif os.path.isfile(parent_path):
- print(parent_path + ": exists and is not a directory.")
- sys.exit(1)
- file = open(path, "wb")
- except Exception as e:
- print(e)
- sys.exit(1)
- seek = chunk["seek"]
- data = base64.b64decode(chunk["chunk"])
- try:
- file.seek(seek)
- file.write(data)
- except Exception as e:
- print(e)
- sys.exit(1)
+def write_chunk(chunk, fd):
+ seek = chunk["seek"]
+ data = base64.b64decode(chunk["chunk"])
+ os.lseek(fd, seek, os.SEEK_SET)
+ os.write(fd, data)
+
+def create_path(path):
+ try:
+ if not os.path.exists(path):
+ os.makedirs(path, exist_ok=True)
+ elif os.path.isfile(path):
+ print(path + ": exists and is not a directory.")
+ sys.exit(1)
+ except Exception as e:
+ print(e)
+ sys.exit(1)
 
 def main():
- file = None
- if len(sys.argv) != 2:
- print("Invalid number of arguments.")
- sys.exit(1)
- while True:
- try:
- json_in = input()
- except EOFError:
- break
- json_list = json_in.split("\n") # need to split in case writes happen faster than reads
- for json_obj in json_list:
- try:
- obj_in = json.loads(json_obj)
- except Exception as e:
- print(e)
- log = open("/var/log/navigator.log", "w")
- log.write(json_in)
- log.close()
- sys.exit(1)
- write_chunk(obj_in, file)
- if file:
- file.close()
- sys.exit(0)
+ if len(sys.argv) != 2:
+ print("Invalid number of arguments.")
+ sys.exit(1)
+ fd = None
+ path = sys.argv[1]
+ parent_path = os.path.dirname(path)
+ create_path(parent_path)
+ try:
+ fd = os.open(sys.argv[1], os.O_WRONLY | os.O_TRUNC | os.O_CREAT)
+ while True:
+ try:
+ json_in = input()
+ except EOFError:
+ break
+ json_list = json_in.split("\n") # need to split in case writes happen faster than reads
+ for json_obj in json_list:
+ obj_in = json.loads(json_obj)
+ write_chunk(obj_in, fd)
+ except Exception as e:
+ print(e)
+ os.close(fd)
+ sys.exit(1)
+ os.close(fd)
+ sys.exit(0)
 
 if __name__ == "__main__":
- main()
+ main()

packaging/el7/main.spec

@@ -32,6 +32,8 @@ rm -rf %{buildroot}
 /usr/share/cockpit/navigator/*
 
 %changelog
+* Fri Nov 12 2021 Joshua Boudreau <jboudreau@45drives.com> 0.5.6-1
+- Fix mangling of large files during upload.
 * Mon Oct 04 2021 Joshua Boudreau <jboudreau@45drives.com> 0.5.5-2
 - Fix sed command for packaging EL7.
 * Mon Oct 04 2021 Joshua Boudreau <jboudreau@45drives.com> 0.5.5-1

packaging/el8/main.spec

@@ -32,6 +32,8 @@ rm -rf %{buildroot}
 /usr/share/cockpit/navigator/*
 
 %changelog
+* Fri Nov 12 2021 Joshua Boudreau <jboudreau@45drives.com> 0.5.6-1
+- Fix mangling of large files during upload.
 * Mon Oct 04 2021 Joshua Boudreau <jboudreau@45drives.com> 0.5.5-2
 - Fix sed command for packaging EL7.
 * Mon Oct 04 2021 Joshua Boudreau <jboudreau@45drives.com> 0.5.5-1

packaging/focal/changelog

@@ -1,3 +1,9 @@
+cockpit-navigator (0.5.6-1focal) focal; urgency=medium
+
+ * Fix mangling of large files during upload.
+
+ -- Joshua Boudreau <jboudreau@45drives.com> Fri, 12 Nov 2021 08:06:44 -0400
+
 cockpit-navigator (0.5.5-2focal) focal; urgency=medium
 
  * Fix sed command for packaging EL7.