svg output point-symbol=line and -style rotate=

.gitignore

@@ -9,3 +9,4 @@ npm-debug.log
 /www/mapshaper.js
 /www/node_modules.js
 release*
+nacis

.travis.yml

@@ -1,5 +1,6 @@
 language: node_js
 sudo: false
 node_js:
-  - '0.12'
+  - '4.2.0'
+  - '6.9.0'
   - 'node'

README.md

@@ -2,7 +2,7 @@
 
 ### Introduction
 
-Mapshaper is software for editing Shapefile, GeoJSON, [TopoJSON](https://github.com/mbostock/topojson/wiki) and several other data formats, written in JavaScript.
+Mapshaper is software for editing Shapefile, GeoJSON, [TopoJSON](https://github.com/mbostock/topojson/wiki), CSV and several other data formats, written in JavaScript.
 
 The `mapshaper` command line program supports essential map making tasks like simplifying shapes, editing attribute data, clipping, erasing, dissolving, filtering and more.
 
@@ -22,14 +22,33 @@ The project wiki has an [introduction](https://github.com/mbloch/mapshaper/wiki/
 
 For a detailed reference, see the [Command Reference](https://github.com/mbloch/mapshaper/wiki/Command-Reference).
 
-### Interactive tool
 
-The web UI works in recent versions of Chrome and Firefox as well as IE 10+. Exporting is not supported in Safari. If you encounter out-of-memory errors using Chrome, try Firefox, which can handle Shapefiles larger than 1GB.
+
+### Interactive web interface
+
+The web UI works in recent desktop versions of Chrome, Firefox, Safari and Internet Explorer. Safari before v10.1 and IE before v10 are not supported.
 
 The mapshaper distribution includes the script `mapshaper-gui`, which runs mapshaper's web interface locally. You can also visit [mapshaper.org](http://www.mapshaper.org) to use mapshaper online.
 
 All processing is done in the browser, so your data stays private, even when using the public website.
 
+### Large file support
+
+**Web interface**
+
+Firefox is able to load Shapefiles and GeoJSON files larger than 1GB. Chrome has improved in recent versions, but is still prone to out-of-memory errors when importing files larger than several hundred megabytes.
+
+**Command line interface**
+
+When working with very large files, mapshaper may become unresponsive or crash with the message "JavaScript heap out of memory."
+
+One option is to run `mapshaper-xl` (added in v0.4.63), which allocates more memory than the standard `mapshaper` program.
+
+Another solution is to run Node directly with the `--max-old-space-size` option. The following example (Mac or Linux) allocates 8GB of memory:
+```bash
+$ node  --max-old-space-size=8192 `which mapshaper` <mapshaper commands>
+```
+
 ### Installation
 
 Mapshaper requires [Node.js](http://nodejs.org).
@@ -62,8 +81,10 @@ This software is licensed under [MPL 2.0](http://www.mozilla.org/MPL/2.0/).
 
 According to Mozilla's [FAQ](http://www.mozilla.org/MPL/2.0/FAQ.html), "The MPL's ‘file-level’ copyleft is designed to encourage contributors to share modifications they make to your code, while still allowing them to combine your code with code under other licenses (open or proprietary) with minimal restrictions."
 
+
+
 ### Acknowledgements
 
-Gregor Aisch, Mike Bostock, Shan Carter and Zhou Yi, for suggesting improvements and general helpfulness.
+My colleagues at The New York Times, for countless suggestions, bug reports and general helpfulness.
 
-Mark Harrower, for collaborating on the original MapShaper program at the University of Wisconsin&ndash;Madison.
+Mark Harrower, for collaborating on the original "MapShaper" program at the University of Wisconsin&ndash;Madison.

bin/mapshaper

@@ -1,6 +1,6 @@
 #!/usr/bin/env node
 
-var mapshaper = require("../");
+var mapshaper = require("../mapshaper.js");
 mapshaper.enableLogging();
 mapshaper.runCommands(process.argv.slice(2), done);
 

bin/mapshaper-gui

@@ -4,7 +4,8 @@ var http = require("http"),
     path = require("path"),
     url = require("url"),
     fs = require("fs"),
-    open = require("open"),
+    Cookies = require("cookies"),
+    opn = require("opn"),
     optimist = require("optimist"),
     defaultPort = 5555,
     webRoot = path.join(__dirname, "../www"),
@@ -19,6 +20,21 @@ var http = require("http"),
         default: defaultPort,
         describe: "http port of server on localhost"
       })
+      .options("s", {
+        alias: "direct-save",
+        describe: "allow saving files directly (outside Download folder)",
+        'boolean': true
+      })
+      .options("f", {
+        alias: "force-save",
+        describe: "allow overwriting input files with output files",
+        'boolean': true
+      })
+      .options("a", {
+        alias: "display-all",
+        describe: "turn on visibility of all layers",
+        'boolean': true
+      })
       .check(function(argv) {
         if (argv.h) {
           optimist.showHelp();
@@ -27,7 +43,8 @@ var http = require("http"),
       }).argv,
     port = parseInt(opts.port, 10) || defaultPort,
     probeCount = 0,
-    dataFiles = expandShapefiles(opts._);
+    dataFiles = expandShapefiles(opts._),
+    sessionId = null;
 
 validateFiles(dataFiles);
 
@@ -48,8 +65,12 @@ process.on('uncaughtException', function(err) {
 
 startServer(port);
 
+function getRandomSessionId() {
+  var str = Math.random().toString(16) + Math.random().toString(16);
+  return str.replace(/0\./g, '').substr(0, 20);
+}
+
 function startServer(port) {
-  var dataPath = '/data/';
   var timeout;
 
   http.createServer(function(request, response) {
@@ -61,30 +82,59 @@ function startServer(port) {
         process.exit(0);
       }, 200);
     } else if (uri == "/manifest.js") {
+      if (!sessionId && opts.s) {
+        // create a session id for authenticating requests to save files
+        // (see saveContent())
+        sessionId = getRandomSessionId();
+        new Cookies(request, response).set('session_id', sessionId);
+      }
       // serve JS file containing manifest of files for mapshaper to load
-      serveContent(getManifestJS(dataFiles), response, getMimeType(uri));
-    } else if (uri.indexOf(dataPath) === 0) {
+      serveContent(getManifestJS(dataFiles, opts), response, getMimeType(uri));
+    } else if (uri.indexOf('/data/') === 0) {
       // serve a file from a path relative to this script
       // assumed to be a data file from the cmd line (!)
-      serveFile(uri.replace(dataPath, ''), response);
+      serveFile(getDataFilePath(uri), response);
+    } else if (uri.indexOf('/save') === 0) {
+      saveContent(request, response);
     } else {
       // serve a file from the web root
       if (uri == '/') {
         uri = '/index.html';
       }
-      serveFile(path.join(webRoot, uri), response);
+      serveFile(getAssetFilePath(uri), response);
     }
   }).listen(port, function() {
-    open("http://localhost:" + port);
+    opn("http://localhost:" + port);
   });
 }
 
+function getAssetFilePath(uri) {
+  // allowing loading of assets from paths relative to the cwd (in addition to
+  // the www/ directory of the mapshaper intall); this is useful
+  // for displaying svg icons in the web ui; svg output then embeds the icons
+  // in the output file.
+  var webPath = path.join(webRoot, uri),
+      relPath = path.join(process.cwd(), uri);
+  return fs.existsSync(webPath) ? webPath : relPath;
+}
+
+function getDataFilePath(uri) {
+  // tilde is added by the web ui; allows accessing a single level of parent
+  // directory when loading data files (an attempt to add a bit of flexibility
+  // without giving access to the entire filesystem)
+  return decodeURI(uri).replace('/~/', '/../').replace('/data/', '');
+}
+
+function serveError(text, code, response) {
+  response.writeHead(code, {"Content-Type": "text/plain"});
+  response.write(text);
+  response.end();
+}
+
 function serveFile(filename, response) {
-  fs.readFile(filename, "binary", function(err, content) {
+  fs.readFile(filename, function(err, content) {
     if (err) {
-      response.writeHead(404, {"Content-Type": "text/plain"});
-      response.write("404 Not Found\n");
-      response.end();
+      serveError("404 Not Found\n", 404, response);
     } else {
       serveContent(content, response, getMimeType(filename));
     }
@@ -93,20 +143,76 @@ function serveFile(filename, response) {
 
 function serveContent(content, response, mimeType) {
   if (mimeType) {
-    response.setHeader("Content-Type", mimeType);
+    response.setHeader('Content-Type', mimeType);
   }
+  response.setHeader('Cache-Control', 'no-cache');
   response.writeHead(200);
   response.write(content, "binary");
   response.end();
 }
 
-function getManifestJS(files) {
-  if (files.length > 0) {
-    return "mapshaper.manifest = " + JSON.stringify(files) + ";\n";
+function readPostData(req, cb) {
+  var buffers = [];
+  if (req.method != 'POST') return cb('expected a post request');
+  // TODO: consider quitting if a size limit is reached
+  req.on('data', function(data) {buffers.push(data);});
+  req.on('end', function () {
+    cb(null, Buffer.concat(buffers));
+  });
+}
+
+function saveContent(req, res) {
+  var requestId = new Cookies(req, res).get('session_id'),
+      ip = req.connection.remoteAddress,
+      urlData = url.parse(req.url),
+      query = require('querystring').parse(urlData.query),
+      file = query.file;
+  // Only requests containing a cookie with the original session id are allowed to save.
+  // This should limit direct saving to the user who is running mapshaper-gui,
+  // preventing another user who has the URL of this Node service from saving.
+  if (!sessionId || requestId != sessionId) return fail('invalid session id');
+  if (ip != '::ffff:127.0.0.1' && ip != '::1' && ip != '127.0.0.1') return fail('saving is only allowed from localhost');
+  if (err = validateOutputFile(file)) return fail(err);
+  readPostData(req, function(err, buf) {
+    if (err) return fail(err);
+    if (!Buffer.isBuffer(buf)) return fail('malformed file content');
+    try {
+      fs.writeFileSync(file, buf);
+    } catch(e) {
+      return fail(e);
+    }
+    serveContent('File saved', res, 'text/plain');
+  });
+
+  function fail(err) {
+    console.error('Unable to save ' + file + ":", err);
+    serveError(err, 400, res);
+  }
+}
+
+function validateOutputFile(file) {
+  var relPath = path.relative('.', file);
+  // TODO: remove path restrictions?
+  if (relPath.indexOf('..') > -1) {
+    return 'parent directories are blocked';
+  }
+  if (!opts.f) {
+    for (var i=0; i<dataFiles.length; i++) {
+      if (!path.relative(file, dataFiles[i])) {
+        return 'tried to overwrite a source file';
+      }
+    }
   }
   return '';
 }
 
+function getManifestJS(files, opts) {
+  var o = {files: files};
+  if (opts.s) o.allow_saving = true;
+  if (opts.a) o.display_all = true;
+  return "mapshaper.manifest = " + JSON.stringify(o) + ";\n";
+}
+
 // print an error and exit if a file is unreadable
 function validateFiles(files) {
   files.forEach(function(f) {
@@ -121,7 +227,7 @@ function validateFiles(files) {
       msg = 'File not found';
     }
     if (msg) {
-      console.error(msg + ":", f);
+      console.error(msg + ": " + f);
       process.exit(1);
     }
   });
@@ -144,7 +250,8 @@ function getMimeType(filename) {
     css: 'text/css',
     js: 'application/javascript',
     html: 'text/html',
-    png: 'image/png'
+    png: 'image/png',
+    svg: 'image/svg+xml'
   }[extname(filename)] || null;
 }
 
@@ -156,13 +263,22 @@ function isUrl(name) {
   return /:\/\//.test(name);
 }
 
+function containsStringCI(arr, str) {
+  str = str.toLowerCase();
+  for (var i=0; i<arr.length; i++) {
+    if (arr[i].toLowerCase() == str) return true;
+  }
+  return false;
+}
+
 // append auxiliary file, if it exists in the filesystem but not in the file list
 function addAuxFile(files, file, ext) {
   var extRx = /\.[^.]+$/;
   // handle UC and LC extensions
+  // TODO: match any combination of UC and LC characters in filename
   var aux = file.replace(extRx, ext.toLowerCase());
   var AUX = file.replace(extRx, ext.toUpperCase());
-  if (files.indexOf(aux) == -1 && files.indexOf(AUX) == -1) {
+  if (!containsStringCI(files, aux)) {
     if (fs.existsSync(aux)) {
       files.push(aux);
     } else if (fs.existsSync(AUX)) {

bin/mapshaper-xl

@@ -0,0 +1,10 @@
+#!/usr/bin/env node
+
+// Run node in a child process with more memory
+// note: require('v8').getHeapStatistics().heap_size_limit seems to give the current limit
+var megabytes = 8000;
+var mapshaper = require('path').join(__dirname, 'mapshaper'); // path of mapshaper executable
+var opts = {stdio: 'inherit'};
+var args = ['--max-old-space-size=' + megabytes, mapshaper].concat(process.argv.slice(2));
+console.error("Allocating", megabytes, "megabytes of heap memory");
+require('child_process').spawn(process.argv[0], args, opts);

build

@@ -2,34 +2,31 @@
 
 var version = require('./package.json').version;
 var follow = process.argv.indexOf('-f') > -1;
-var buildAll = process.argv.indexOf('all') > -1;
 var requiredModules = [
     'mproj',
     'buffer',
     'iconv-lite',
     'fs',
-    'rbush',
+    'flatbush',
     'rw',
     'path',
     'd3-dsv'
   ];
 
 
 require("catty")({follow: follow})
-  .addLibrary("lib")
   .addLibrary("src")
   .cat("src/gui/mapshaper-gui.js", './www/mapshaper-gui.js')
-  .prepend("var VERSION = '" + version + "';")
+  .prepend("VERSION = '" + version + "';")
   .cat("src/mapshaper.js", onCat);
 
-if (buildAll) {
-  require('browserify')()
-    .require(requiredModules)
-    .bundle(function(err, buf) {
-      if (err) throw err;
-      write('./www/node_modules.js', buf);
-    });
-}
+
+require('browserify')()
+  .require(requiredModules)
+  .bundle(function(err, buf) {
+    if (err) throw err;
+    write('./www/modules.js', buf);
+  });
 
 
 function onCat(err, js) {