From 3a6926a836abd0d0c0a5c41b12409452fa7cf1da Mon Sep 17 00:00:00 2001 From: IljaKroonen Date: Fri, 23 Mar 2018 14:36:32 +0100 Subject: [PATCH] Allow serializing NaN and Infinity double values (scalars only) --- README.md | 6 ++++ export.js | 23 ++++++++++++++-- import.js | 9 ++++-- tests/numbers.js | 72 ++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 106 insertions(+), 4 deletions(-) create mode 100644 tests/numbers.js diff --git a/README.md b/README.md index 9590bb4..1e78fb4 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,12 @@ HOST=127.0.0.1 KEYSPACE=from_keyspace_name TABLE=my_table_name node import.js The Dockerfiles provide a volume mounted at /data and expect the environment variables `HOST` and `KEYSPACE`. `Dockerfile.import` provides `import.js` functionality. `Dockerfile.export` provides `export.js` functionality. By using the -v option of `docker run` this provides the facility to store the output/input directory in an arbitrary location. It also allows running cassandra-export from any location. This requires [Docker](https://www.docker.com/) to be installed. +# Running tests + +To run a test in the tests folder, for example `numbers.js`, run the command `node tests/numbers.js` at the root of the repo. A localhost cassandra must be running. + +Tests use recent node.js features and requires Node.js 8. + # Note Cassandra exporter only export / import data. It expects the tables to be present beforehand. If you need to also export schema and the indexes, then you could easily use cqlsh and the source command to export / import the schema before moving the data. diff --git a/export.js b/export.js index eab6ea8..53af70e 100644 --- a/export.js +++ b/export.js @@ -49,9 +49,28 @@ function processTableExport(table) { client.eachRow(query, [], options, function (n, row) { var rowObject = {}; - row.forEach(function(value, key){ - rowObject[key] = value; + row.forEach(function (value, key) { + if (typeof value === 'number') { + if (Number.isNaN(value)) { + rowObject[key] = { + type: "NOT_A_NUMBER" + } + } else if (Number.isFinite(value)) { + rowObject[key] = value; + } else if (value > 0) { + rowObject[key] = { + type: "POSITIVE_INFINITY" + } + } else { + rowObject[key] = { + type: "NEGATIVE_INFINITY" + } + } + } else { + rowObject[key] = value; + } }); + processed++; writeStream.write(rowObject); }, function (err, result) { diff --git a/import.js b/import.js index 979e4ba..39554ee 100644 --- a/import.js +++ b/import.js @@ -51,8 +51,13 @@ function buildTableQueryForDataRow(tableInfo, row) { if (_.isPlainObject(param)) { if (param.type === 'Buffer') { return Buffer.from(param); - } - else { + } else if (param.type === 'NOT_A_NUMBER') { + return Number.NaN; + } else if (param.type === 'POSITIVE_INFINITY') { + return Number.POSITIVE_INFINITY; + } else if (param.type === 'NEGATIVE_INFINITY') { + return Number.NEGATIVE_INFINITY; + } else { var omittedParams = _.omitBy(param, function(item) {return item === null}); for (key in omittedParams) { if (_.isObject(omittedParams[key]) && omittedParams[key].type === 'Buffer') { diff --git a/tests/numbers.js b/tests/numbers.js new file mode 100644 index 0000000..04f1a77 --- /dev/null +++ b/tests/numbers.js @@ -0,0 +1,72 @@ +var Promise = require('bluebird'); +var cassandra = require('cassandra-driver'); +var fs = require('fs'); +var jsonStream = require('JSONStream'); +var cp = require('child_process') + +var HOST = process.env.HOST || '127.0.0.1'; +var PORT = process.env.PORT || 9042; + +var USER = process.env.USER; +var PASSWORD = process.env.PASSWORD; + +var authProvider; + +if (USER && PASSWORD) { + authProvider = new cassandra.auth.PlainTextAuthProvider(USER, PASSWORD); +} + +var systemClient = new cassandra.Client({ contactPoints: [HOST], authProvider: authProvider, protocolOptions: { port: [PORT] } }); +var client = new cassandra.Client({ contactPoints: [HOST], authProvider: authProvider, protocolOptions: { port: [PORT] } }); + +async function testIt() { + try { + await client.execute("CREATE KEYSPACE IF NOT EXISTS NumbersTest WITH REPLICATION = { 'class': 'SimpleStrategy', 'replication_factor': 1 }"); + await client.execute("CREATE TABLE IF NOT EXISTS NumbersTest.TestTable(k TEXT PRIMARY KEY, v DOUBLE)") + await client.execute("INSERT INTO NumbersTest.TestTable(k, v) VALUES('NOT_A_NUMBER', NaN)"); + await client.execute("INSERT INTO NumbersTest.TestTable(k, v) VALUES('ZERO', 0)"); + await client.execute("INSERT INTO NumbersTest.TestTable(k, v) VALUES('THIRTEEN_DOT_FOUR', 13.4)"); + await client.execute("INSERT INTO NumbersTest.TestTable(k, v) VALUES('NEGATIVE_INFINITY', -Infinity)"); + await client.execute("INSERT INTO NumbersTest.TestTable(k, v) VALUES('POSITIVE_INFINITY', Infinity)"); + + process.env.KEYSPACE = 'numberstest'; + + cp.execSync('node export.js') + + const resultSnapshot = fs.readFileSync('data/testtable.json', { encoding: 'utf-8' }); + + if (resultSnapshot !== '[{"k":"NOT_A_NUMBER","v":{"type":"NOT_A_NUMBER"}},{"k":"NEGATIVE_INFINITY","v":{"type":"NEGATIVE_INFINITY"}},{"k":"THIRTEEN_DOT_FOUR","v":13.4},{"k":"POSITIVE_INFINITY","v":{"type":"POSITIVE_INFINITY"}},{"k":"ZERO","v":0}]') { + throw Error('Snapshot ' + resultSnapshot + ' does not matched expected return value'); + } + + await client.execute('TRUNCATE NumbersTest.TestTable'); + + cp.execSync('node import.js') + + const rs = await client.execute('SELECT * FROM NumbersTest.TestTable'); + + const expected = [ + ['NOT_A_NUMBER', Number.NaN], + ['NEGATIVE_INFINITY', Number.NEGATIVE_INFINITY], + ['THIRTEEN_DOT_FOUR', 13.4], + ['POSITIVE_INFINITY', Number.POSITIVE_INFINITY], + ['ZERO', 0] + ] + + rs.rows.forEach((row, i) => { + row.values().forEach((value, j) => { + if (expected[i][j] != value && !(Number.isNaN(expected[i][j]) && Number.isNaN(value))) { + throw Error('Expected value ' + expected[i][j] + ' but was ' + value) + } + }) + }) + + console.info('PASS'); + } catch (e) { + console.error('FAIL', e); + } finally { + await Promise.all([systemClient.shutdown(), client.shutdown()]) + } +} + +testIt()