Merge pull request #53 from matrix-org/michaelk/tarball_of_files

Serve a tarball containing the contents of a given directory.

README.md

@@ -28,6 +28,8 @@ Serves submitted bug reports. Protected by basic HTTP auth using the
 username/password provided in the environment. A browsable list, collated by
 report submission date and time.
 
+A whole directory can be downloaded as a tarball by appending the parameter `?format=tar.gz` to the end of the URL path
+
 ### POST `/api/submit`
 
 Submission endpoint: this is where applications should send their reports.

changelog.d/53.feature

@@ -0,0 +1 @@
+Provide ?format=tar.gz option on directory listings to download tarball.

logserver.go

@@ -17,6 +17,7 @@ limitations under the License.
 package main
 
 import (
+ "archive/tar"
  "compress/gzip"
  "io"
  "log"
@@ -68,6 +69,7 @@ func (f *logServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
  serveFile(w, r, upath)
 }
 
+
 func serveFile(w http.ResponseWriter, r *http.Request, path string) {
  d, err := os.Stat(path)
  if err != nil {
@@ -79,10 +81,9 @@ func serveFile(w http.ResponseWriter, r *http.Request, path string) {
  // for anti-XSS belt-and-braces, set a very restrictive CSP
  w.Header().Set("Content-Security-Policy", "default-src: none")
 
- // if it's a directory, serve a listing
+ // if it's a directory, serve a listing or a tarball
  if d.IsDir() {
- log.Println("Serving", path)
- http.ServeFile(w, r, path)
+ serveDirectory(w, r, path)
  return
  }
 
@@ -125,6 +126,117 @@ func extensionToMimeType(path string) string {
  return "application/octet-stream"
 }
 
+// Chooses to serve either a directory listing or tarball based on the 'format' parameter.
+func serveDirectory(w http.ResponseWriter, r *http.Request, path string) {
+ format, _ := r.URL.Query()["format"]
+ if len(format) == 1 && format[0] == "tar.gz" {
+ log.Println("Serving tarball of", path)
+ err := serveTarball(w, r, path)
+ if err != nil {
+ msg, code := toHTTPError(err)
+ http.Error(w, msg, code)
+ log.Println("Error", err)
+ }
+ return
+ }
+ log.Println("Serving directory listing of", path)
+ http.ServeFile(w, r, path)
+}
+
+// Streams a dynamically created tar.gz file with the contents of the given directory
+// Will serve a partial, corrupted response if there is a error partway through the 
+// operation as we stream the response.
+// 
+// The resultant tarball will contain a single directory containing all the files
+// so it can unpack cleanly without overwriting other files.
+//
+// Errors are only returned if generated before the tarball has started being
+// written to the ResponseWriter
+func serveTarball(w http.ResponseWriter, r *http.Request, dir string) error {
+ directory, err := os.Open(dir)
+ if err != nil {
+ return err
+ }
+
+ // Creates a "disposition filename"
+ // Take a URL.path like `/2022-01-10/184843-BZZXEGYH/`
+ // and removes leading and trailing `/` and replaces internal `/` with `_`
+ // to form a suitable filename for use in the content-disposition header
+ // dfilename would turn into `2022-01-10_184843-BZZXEGYH`
+ dfilename := strings.Trim(r.URL.Path,"/")
+ dfilename = strings.Replace(dfilename, "/","_",-1)
+
+ // There is no application/tgz or similar; return a gzip file as best option. 
+ // This tends to trigger archive type tools, which will then use the filename to 
+ // identify the contents correctly.
+ w.Header().Set("Content-Type", "application/gzip")
+ w.Header().Set("Content-Disposition", "attachment; filename=" + dfilename + ".tar.gz")
+
+ files, err := directory.Readdir(-1)
+ if err != nil {
+ return err
+ }
+
+ gzip := gzip.NewWriter(w)
+ defer gzip.Close()
+ targz := tar.NewWriter(gzip)
+ defer targz.Close()
+
+
+ for _, file := range files {
+ if file.IsDir() {
+ // We avoid including nested directories
+ // This will result in requests for directories with only directories in
+ // to return an empty tarball instead of recursively including directories.
+ // This helps the server remain performant as a download of 'everything' would be slow
+ continue
+ }
+ path := dir + "/" + file.Name()
+ // We use the existing disposition filename to create a base directory structure for the files
+ // so when they are unpacked, they are grouped in a unique folder on disk
+ err := addToArchive(targz, dfilename, path)
+ if err != nil {
+ // From this point we assume that data may have been sent to the client already.
+ // We therefore do not http.Error() after this point, instead closing the stream and
+ // allowing the client to deal with a partial file as if there was a network issue.
+ log.Println("Error streaming tarball", err)
+ return nil
+ }
+ }
+ return nil
+}
+
+// Add a single file into the archive. 
+func addToArchive(targz *tar.Writer, dfilename string, filename string) error {
+ file, err := os.Open(filename)
+ if err != nil {
+ return err
+ }
+ defer file.Close()
+
+ info, err := file.Stat()
+ if err != nil {
+ return err
+ }
+
+ header, err := tar.FileInfoHeader(info, info.Name())
+ if err != nil {
+ return err
+ }
+ header.Name = dfilename + "/" + info.Name()
+
+ err = targz.WriteHeader(header)
+ if err != nil {
+ return err
+ }
+
+ _, err = io.Copy(targz, file)
+ if err != nil {
+ return err
+ }
+ return nil
+}
+
 func serveGzippedFile(w http.ResponseWriter, r *http.Request, path string, size int64) {
  w.Header().Set("Content-Type", "text/plain; charset=utf-8")