From b725c6ca31372a3c61eaa7ef4589a207878f4396 Mon Sep 17 00:00:00 2001 From: "David M. Pankros" Date: Thu, 11 Feb 2016 12:18:51 -0500 Subject: [PATCH 1/8] Almost a complete rewrite. Adds much more flexibility including support for all mongo index options and multi-key indexes. --- CHANGELOG.md | 6 +- LICENSE | 1 + README.md | 42 ++++--- lib/indexing.js | 71 +++-------- lib/schemaIndex.js | 290 +++++++++++++++++++++++++++++++++++++++++++++ package.js | 4 +- tests/indexing.js | 28 ++++- 7 files changed, 365 insertions(+), 77 deletions(-) create mode 100644 lib/schemaIndex.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 646c6f6..a4b5e2a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,4 +2,8 @@ # 1.0.0 -Initial release. Originally included in the aldeed:collection2 package. \ No newline at end of file +Initial release. Originally included in the aldeed:collection2 package. + +# 2.0.0 + +Substantial rewrite. Support for all mongo index options. \ No newline at end of file diff --git a/LICENSE b/LICENSE index 9eae9af..d343175 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,7 @@ The MIT License (MIT) Copyright (c) 2015 Eric Dobbertin +Portions Copyright (c) 2016 David Pankros. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in diff --git a/README.md b/README.md index 7f6c671..c068b41 100644 --- a/README.md +++ b/README.md @@ -15,49 +15,54 @@ $ meteor add aldeed:schema-index ## Usage -Use the `index` option to ensure a MongoDB index for a specific field: +Use the `index` option to ensure a MongoDB index for a specific field,or for multiple fields, add the same index to each: ```js { title: { type: String, - index: 1 + index: { + name: 'titleIndex', + type: 1 } } ``` +Set to value to `1` for an ascending index. Set to `-1` for a descending index. Or you may set this to another type of specific MongoDB index, such as `"2d"`. Indexes work on embedded sub-documents as well. +Then attach the index to the collection in much the same way as you attach a schema to a collection: -Set to `1` or `true` for an ascending index. Set to `-1` for a descending index. Or you may set this to another type of specific MongoDB index, such as `"2d"`. Indexes work on embedded sub-documents as well. +```js +books.attchIndex('titleIndex'); +``` -If you have created an index for a field by mistake and you want to remove or change it, set `index` to `false`: +If you have created an index for a field by mistake and you want to remove or change it, set `action` to `rebuild` or `drop`: ```js -{ - "address.street": { - type: String, - index: false +books.attchIndex('titleIndex', + { + action: 'drop' } -} +); ``` -IMPORTANT: If you need to change anything about an index, you must first start the app with `index: false` to drop the old index, and then restart with the correct index properties. +IMPORTANT: If you need to change anything about an index, you must first start the app with `action: 'rebuild'` to drop the old index and recreate the index from scratch. -If a field has the `unique` option set to `true`, the MongoDB index will be a unique index as well. Then on the server, Collection2 will rely on MongoDB to check uniqueness of your field, which is more efficient than our custom checking. +If an index has the `unique` option set to `true`, the MongoDB index will be a unique index as well. Then on the server, Collection2 will rely on MongoDB to check uniqueness of your field, which is more efficient than our custom checking. ```js -{ - "pseudo": { - type: String, - index: true, +books.attchIndex('pseudo', + { unique: true } -} +); ``` -For the `unique` option to work, `index` must be `true`, `1`, or `-1`. The error message for uniqueness is very generic. It's best to define your own using `MyCollection.simpleSchema().messages()`. The error type string is "notUnique". +The error message for uniqueness is very generic. It's best to define your own using `MyCollection.simpleSchema().messages()`. The error type string is "notUnique". + +Any mongo index option can be passed to `attachIndex` with all the same restrictions that Mongo places on you, except the `name` option. The `name` option is set by the first parameter to `attachIndex`. See the [Mongo Docs](https://docs.mongodb.org/v3.0/reference/method/db.collection.createIndex/#db.collection.createIndex) for a complete list of available index options. You can use the `sparse` option along with the `index` and `unique` options to tell MongoDB to build a [sparse index](http://docs.mongodb.org/manual/core/index-sparse/#index-type-sparse). By default, MongoDB will only permit one document that lacks the indexed field. By setting the `sparse` option to `true`, the index will only contain entries for documents that have the indexed field. The index skips over any document that is missing the field. This is helpful when indexing on a key in an array of sub-documents. Learn more in the [MongoDB docs](http://docs.mongodb.org/manual/core/index-unique/#unique-index-and-missing-field). -All indexes are built in the background so indexing does *not* block other database queries. +All indexes are built in the background by default so indexing does *not* block other database queries. This can be changed by explicitly setting the `background` option to false, but please use caution when doing so. ## Contributing @@ -65,6 +70,7 @@ Anyone is welcome to contribute. Fork, make and test your changes (`meteor test- ### Major Contributors +@dpankros @mquandalle (Add yourself if you should be listed here.) diff --git a/lib/indexing.js b/lib/indexing.js index 4dd9587..8255a1d 100644 --- a/lib/indexing.js +++ b/lib/indexing.js @@ -1,8 +1,6 @@ // Extend the schema options allowed by SimpleSchema SimpleSchema.extendOptions({ - index: Match.Optional(Match.OneOf(Number, String, Boolean)), - unique: Match.Optional(Boolean), - sparse: Match.Optional(Boolean), + index: Match.Optional(Object) }); // Define validation error messages @@ -11,56 +9,23 @@ SimpleSchema.messages({ }); if (Meteor.isServer) { - Collection2.on('schema.attached', function (collection, ss) { - function ensureIndex(index, indexName, unique, sparse) { - Meteor.startup(function () { - collection._collection._ensureIndex(index, { - background: true, - name: indexName, - unique: unique, - sparse: sparse - }); - }); - } - - function dropIndex(indexName) { - Meteor.startup(function () { - try { - collection._collection._dropIndex(indexName); - } catch (err) { - // no index with that name, which is what we want - } - }); + + Mongo.Collection.prototype.attachIndex = function siAttachIndex(name, options) { + check(name, String); + + var self = this; + options = options || {}; + self._si = self._si || new SchemaIndex(); + self._si.addIndexData(name).options = options; + + //wait until the simple schema has been attached before building indexData + if (self._si.isSchemaAttached) { + self._si.processIndex(name); } - - // Loop over fields definitions and ensure collection indexes (server side only) - _.each(ss.schema(), function(definition, fieldName) { - if ('index' in definition || definition.unique === true) { - var index = {}, indexValue; - // If they specified `unique: true` but not `index`, - // we assume `index: 1` to set up the unique index in mongo - if ('index' in definition) { - indexValue = definition.index; - if (indexValue === true) indexValue = 1; - } else { - indexValue = 1; - } - var indexName = 'c2_' + fieldName; - // In the index object, we want object array keys without the ".$" piece - var idxFieldName = fieldName.replace(/\.\$\./g, "."); - index[idxFieldName] = indexValue; - var unique = !!definition.unique && (indexValue === 1 || indexValue === -1); - var sparse = definition.sparse || false; - - // If unique and optional, force sparse to prevent errors - if (!sparse && unique && definition.optional) sparse = true; - - if (indexValue === false) { - dropIndex(indexName); - } else { - ensureIndex(index, indexName, unique, sparse); - } - } - }); + }; + + Collection2.on('schema.attached', function (collection, ss) { + collection._si = collection._si || new SchemaIndex(); + collection._si.attach(collection, ss); }); } \ No newline at end of file diff --git a/lib/schemaIndex.js b/lib/schemaIndex.js new file mode 100644 index 0000000..84d4073 --- /dev/null +++ b/lib/schemaIndex.js @@ -0,0 +1,290 @@ +/** + * IndexData - internal class used to track information about the index as it + * comes in + */ + +class IndexData { + constructor(name) { + check(name, String); + this._name = name; + this._options = { + background: true //default + }; + this._fields = {}; + this._action = 'build'; + } + + get name() { + return this._name; + } + + get options() { + return this._options; + } + + set options(val) { + if (!val) return; + + if (val.action) {//strip out action, if it is present + this._action = val.action; + delete val.action; + } + + //name object must be last to override anything passed in + _(this._options).extend(val, {name: this.name}); + + } + + setOptionValue(option, value) { + this.options[option] = value; + } + + getOptionValue(option) { + return this.options[option]; + } + + + get fields() { + return this._fields; + } + + addFieldIndex(fieldName, fieldType) { + this.fields[fieldName] = fieldType; + } + + get needsBuild() { + return this.action === 'build' || this.action === 'rebuild'; + } + + get needsDrop() { + return this.action === 'rebuild' || this.action === 'drop'; + } + + get action() { + return this._action; + } +} + + +/** + * Primary class for tracking indexData and their definitions + * @type {SchemaIndex} + */ +SchemaIndex = class SchemaIndex { + /** + * Adds an index, if it doesn't already exist + * @param collection the Mongo collection object + * @param name The name of the index + * @param index A map of fields to types + * @param opt_options A map of options for createIndex + */ + static ensureIndex(collection, name, index, opt_options) { + if (index.length === 0) { + throw new Error('Cannot create an index with no fields'); + } + collection._collection._ensureIndex(index, opt_options); + } + + /** + * Removes an index if it exists, suppresses any errors otherwise + * @param collection The Mongo Collection + * @param indexName The name of the index to drop + */ + static dropIndex(collection, indexName) { + + try { + collection._collection._dropIndex(indexName); + } catch (err) { + // no index with that name, which is what we want + } + } + + constructor() { + this._indexes = {}; + this._toBeAttached = []; + this._isSchemaAttached = false; + } + + // + // index object methods + // + get indexData() { + return this._indexes; + } + + /** + * Adds an IndexData object + * @param name + * @returns {*} + */ + addIndexData(name) { + var indexData = this.getOrCreateIndexDatum(name); + + if (!this.isSchemaAttached) {//queue the index + this._toBeAttached.push(name); + } + return indexData; + } + + /** + * Fetches a names IndexData object + * @param name + * @returns {*} + */ + getIndexDatum(name) { + return this.indexData[name]; + } + + /** + * Fetches an IndexData object. Creates it, if it doesn't exist. + * @param name The name of the new index + * @returns {*} + */ + getOrCreateIndexDatum(name) { + var index = this.indexData[name]; + if (!index) { + index = new IndexData(name); + this.indexData[name] = index; + } + return index; + } + + /** + * Adds a field to an index + * @param field The field name + * @param indexName The indexName + * @param opt_type The type of the index on the field 1 is ascending, -1 is descending, etc.. + * @returns {SchemaIndex} + */ + addIndexFieldDef(field, indexName, opt_type = 1) { + var id = this.getOrCreateIndexDatum(indexName); + id.addFieldIndex(field, opt_type); + return this; + } + + // + // toBeAttached properties and methods + // + + /** + * The names of the indexData queued up to be added, removed, or updated as + * soon as as simpleschema is attached + * @returns {Array} + */ + get toBeAttached() { + return this._toBeAttached; + } + + /** + * Called when a collection and simple schema are attached to shemaIndex. + * The simpleschema can be parsed so we can see what fields are being used for + * indexData. + * @param collection + * @param ss + */ + attach(collection, ss) { + var self = this; + + this._collection = collection; + this._ss = ss; + + _.each(ss.schema(), function(definition, fieldName) { + if ('index' in definition) { + var indexDefs; + if (definition.index instanceof Object) { + indexDefs = [definition.index]; + } else { + //unkown or sunsupported type + throw new Error('Unsupported type set for index: ' + definition.index); + } + + //supports multiple indexData per field, but not exposed in schema options + //because mongo doesn't support it + self.addIndexFieldDefs(indexDefs, fieldName); + } + } + ); + + this._isSchemaAttached = true; + + this.processQueuedIndexes(); + } + + + addIndexFieldDefs(indexDefs, fieldName) { + for (var i = 0; i < indexDefs.length; i++) { + var indexDef = indexDefs[i]; + if (indexDef) { + var indexName = indexDef.name; + var indexType = indexDef.type || 1; + + check(indexName, String); + this.addIndexFieldDef(fieldName, indexName, indexType); + } + } + } + + processQueuedIndexes() { + if (this.toBeAttached.length === 0) return; + + for (var i = 0; i < this.toBeAttached.length; i++) { + this.processIndex(this.toBeAttached[i]); + } + } + + + /** + * Drops, Creates, or recreates a named index. A simpleschema MUST BE + * ATTACHED for this to work. + * @param indexName The name of the index to create. + */ + processIndex(indexName) { + if (!this.isSchemaAttached) return; + + check(indexName, String); + var index = this.getIndexDatum(indexName); + + if (index && this.collection) { + //the index has already been attached. We can proceed. + var self = this; + Meteor.startup(function() { + if (index.needsDrop) { + SchemaIndex.dropIndex(self.collection, indexName); + } + + if (index.needsBuild) { + SchemaIndex.ensureIndex(self.collection, indexName, index.fields, + index.options + ); + } + } + ); + } + } + + /** + * iSchemaAttached returns whether c2.attachSchema has been called on the collection + * if it hasn't, indexData are queued up to be attached when the schema is attached. + * If it has, the index can be attached straight away. + * @returns {boolean} + */ + get isSchemaAttached() { + return this._isSchemaAttached; + } + + /** + * The Mongo.Collection object if attached or undefined if not attached + * @returns {*} + */ + get collection() { + return this._collection; + } + + /** + * The SimpleSchema object if attached or undefined if not attached + * @returns {*} + */ + get simpleSchema() { + return this._ss; + } +} diff --git a/package.js b/package.js index 937b161..e63bf86 100644 --- a/package.js +++ b/package.js @@ -11,10 +11,12 @@ Package.onUse(function(api) { 'underscore@1.0.0', 'minimongo@1.0.0', 'check@1.0.0', + 'ecmascript' ]); api.addFiles([ - 'lib/indexing.js' + 'lib/indexing.js', + 'lib/schemaIndex.js' ]); }); diff --git a/tests/indexing.js b/tests/indexing.js index c8b9634..b01fe11 100644 --- a/tests/indexing.js +++ b/tests/indexing.js @@ -1,10 +1,22 @@ var books = new Mongo.Collection('books'); + +if (Meteor.isServer) { + books.attachIndex('isbnIdx', { + unique: true, + action: 'rebuild', + background: false + } + ); +} books.attachSchema(new SimpleSchema({ title: { type: String, label: 'Title', max: 200, - index: 1 + index: { + name: 'titleIdx', + type:-1 + } }, author: { type: String, @@ -30,8 +42,7 @@ books.attachSchema(new SimpleSchema({ type: String, label: 'ISBN', optional: true, - index: 1, - unique: true + index: {name: 'isbnIdx'} }, field1: { type: String, @@ -51,7 +62,14 @@ books.attachSchema(new SimpleSchema({ } })); + + if (Meteor.isServer) { + books.attachIndex('titleIdx', { + action: 'rebuild', + background: false + }); + Meteor.publish("books", function() { return books.find(); }); @@ -131,8 +149,10 @@ Tinytest.addAsync('Collection2 - Unique - Insert Duplicate', function (test, nex copies: 1, isbn: isbn }, function (error, result) { + //console.log('error is', error); test.isTrue(!!error, 'We expected the insert to trigger an error since isbn being inserted is already used'); - test.equal(error.invalidKeys.length, 1, 'We should get one invalidKey back attached to the Error object'); + test.equal(error.code, 11000, 'We expected the insert to trigger an E11000 duplicate key error'); + //test.equal(error.invalidKeys.length, 1, 'We should get one invalidKey back attached to the Error object'); test.isFalse(result, 'result should be false'); var invalidKeys = books.simpleSchema().namedContext().invalidKeys(); From 11159a7090989e010ac11569c690c5c36c016ded Mon Sep 17 00:00:00 2001 From: "David M. Pankros" Date: Thu, 11 Feb 2016 13:27:31 -0500 Subject: [PATCH 2/8] Minor cleanup. Moved serverIndex.js to server only --- lib/indexing.js | 11 +++-------- lib/schemaIndex.js | 12 ++++++++---- package.js | 7 +++++-- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/lib/indexing.js b/lib/indexing.js index 8255a1d..df30414 100644 --- a/lib/indexing.js +++ b/lib/indexing.js @@ -13,15 +13,10 @@ if (Meteor.isServer) { Mongo.Collection.prototype.attachIndex = function siAttachIndex(name, options) { check(name, String); - var self = this; options = options || {}; - self._si = self._si || new SchemaIndex(); - self._si.addIndexData(name).options = options; - - //wait until the simple schema has been attached before building indexData - if (self._si.isSchemaAttached) { - self._si.processIndex(name); - } + var si = this._si || new SchemaIndex(); + si.addIndexData(name).options = options; + si.processIndex(name); }; Collection2.on('schema.attached', function (collection, ss) { diff --git a/lib/schemaIndex.js b/lib/schemaIndex.js index 84d4073..a4087ab 100644 --- a/lib/schemaIndex.js +++ b/lib/schemaIndex.js @@ -49,6 +49,7 @@ class IndexData { } addFieldIndex(fieldName, fieldType) { + var fieldName = fieldName.replace(/\.\$\./g, "."); this.fields[fieldName] = fieldType; } @@ -79,8 +80,13 @@ SchemaIndex = class SchemaIndex { * @param opt_options A map of options for createIndex */ static ensureIndex(collection, name, index, opt_options) { - if (index.length === 0) { - throw new Error('Cannot create an index with no fields'); + try { + if (index.length === 0) { + throw new Error('Cannot create an index with no fields'); + } + }catch(e){ + console.log('error', collection,name, index, opt_options); + throw e; } collection._collection._ensureIndex(index, opt_options); } @@ -225,8 +231,6 @@ SchemaIndex = class SchemaIndex { } processQueuedIndexes() { - if (this.toBeAttached.length === 0) return; - for (var i = 0; i < this.toBeAttached.length; i++) { this.processIndex(this.toBeAttached[i]); } diff --git a/package.js b/package.js index e63bf86..1cdb48b 100644 --- a/package.js +++ b/package.js @@ -15,9 +15,12 @@ Package.onUse(function(api) { ]); api.addFiles([ - 'lib/indexing.js', - 'lib/schemaIndex.js' + 'lib/indexing.js' ]); + + api.addFiles([ + 'lib/schemaIndex.js' + ], 'server'); }); Package.onTest(function(api) { From ce549e62ce2474dfe3b83658f490d9857cad2bb9 Mon Sep 17 00:00:00 2001 From: "David M. Pankros" Date: Thu, 11 Feb 2016 14:27:13 -0500 Subject: [PATCH 3/8] More cleanup and bug fixes. --- lib/indexing.js | 7 ++++--- lib/schemaIndex.js | 13 ++++++------- tests/indexing.js | 31 ++++++++++++++++--------------- 3 files changed, 26 insertions(+), 25 deletions(-) diff --git a/lib/indexing.js b/lib/indexing.js index df30414..b6a15d8 100644 --- a/lib/indexing.js +++ b/lib/indexing.js @@ -14,9 +14,10 @@ if (Meteor.isServer) { check(name, String); options = options || {}; - var si = this._si || new SchemaIndex(); - si.addIndexData(name).options = options; - si.processIndex(name); + this._si = this._si || new SchemaIndex(); + this._si.addIndexDatum(name).options = options; + this._si.processIndex(name); + }; Collection2.on('schema.attached', function (collection, ss) { diff --git a/lib/schemaIndex.js b/lib/schemaIndex.js index a4087ab..2f0127b 100644 --- a/lib/schemaIndex.js +++ b/lib/schemaIndex.js @@ -32,7 +32,6 @@ class IndexData { //name object must be last to override anything passed in _(this._options).extend(val, {name: this.name}); - } setOptionValue(option, value) { @@ -97,7 +96,6 @@ SchemaIndex = class SchemaIndex { * @param indexName The name of the index to drop */ static dropIndex(collection, indexName) { - try { collection._collection._dropIndex(indexName); } catch (err) { @@ -123,11 +121,12 @@ SchemaIndex = class SchemaIndex { * @param name * @returns {*} */ - addIndexData(name) { + addIndexDatum(name) { var indexData = this.getOrCreateIndexDatum(name); if (!this.isSchemaAttached) {//queue the index - this._toBeAttached.push(name); + this._toBeAttached = this._toBeAttached.concat(name); + debugger; } return indexData; } @@ -192,7 +191,7 @@ SchemaIndex = class SchemaIndex { var self = this; this._collection = collection; - this._ss = ss; + //this._ss = ss; _.each(ss.schema(), function(definition, fieldName) { if ('index' in definition) { @@ -200,7 +199,6 @@ SchemaIndex = class SchemaIndex { if (definition.index instanceof Object) { indexDefs = [definition.index]; } else { - //unkown or sunsupported type throw new Error('Unsupported type set for index: ' + definition.index); } @@ -234,12 +232,13 @@ SchemaIndex = class SchemaIndex { for (var i = 0; i < this.toBeAttached.length; i++) { this.processIndex(this.toBeAttached[i]); } + this._toBeAttached.length = 0; } /** * Drops, Creates, or recreates a named index. A simpleschema MUST BE - * ATTACHED for this to work. + * ATTACHED for this to work otherwise it will just bail out. * @param indexName The name of the index to create. */ processIndex(indexName) { diff --git a/tests/indexing.js b/tests/indexing.js index b01fe11..cccc26d 100644 --- a/tests/indexing.js +++ b/tests/indexing.js @@ -1,13 +1,13 @@ var books = new Mongo.Collection('books'); -if (Meteor.isServer) { - books.attachIndex('isbnIdx', { - unique: true, - action: 'rebuild', - background: false - } - ); -} +//one before and one after to verify that order doesn't matter +books.attachIndex('isbnIdx', { + unique: true, + action: 'rebuild', + background: false + } +); + books.attachSchema(new SimpleSchema({ title: { type: String, @@ -63,12 +63,13 @@ books.attachSchema(new SimpleSchema({ })); +books.attachIndex('titleIdx', { + action: 'rebuild', + background: false +}); + if (Meteor.isServer) { - books.attachIndex('titleIdx', { - action: 'rebuild', - background: false - }); Meteor.publish("books", function() { return books.find(); @@ -149,10 +150,10 @@ Tinytest.addAsync('Collection2 - Unique - Insert Duplicate', function (test, nex copies: 1, isbn: isbn }, function (error, result) { - //console.log('error is', error); + console.log('error is', error, 'result is', result); test.isTrue(!!error, 'We expected the insert to trigger an error since isbn being inserted is already used'); - test.equal(error.code, 11000, 'We expected the insert to trigger an E11000 duplicate key error'); - //test.equal(error.invalidKeys.length, 1, 'We should get one invalidKey back attached to the Error object'); + //test.equal(error.code, 11000, 'We expected the insert to trigger an E11000 duplicate key error'); + test.equal(error.invalidKeys.length, 1, 'We should get one invalidKey back attached to the Error object'); test.isFalse(result, 'result should be false'); var invalidKeys = books.simpleSchema().namedContext().invalidKeys(); From 68f2945ed0b1199e492ed3a614ebc50bb7e853c7 Mon Sep 17 00:00:00 2001 From: "David M. Pankros" Date: Thu, 11 Feb 2016 17:06:35 -0500 Subject: [PATCH 4/8] Removed extra logging. Changed index names to conform with c2's method of determining invalidKeys. --- lib/schemaIndex.js | 11 ++++------- tests/indexing.js | 15 +++++++++------ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/lib/schemaIndex.js b/lib/schemaIndex.js index 2f0127b..d295e4f 100644 --- a/lib/schemaIndex.js +++ b/lib/schemaIndex.js @@ -79,15 +79,12 @@ SchemaIndex = class SchemaIndex { * @param opt_options A map of options for createIndex */ static ensureIndex(collection, name, index, opt_options) { - try { - if (index.length === 0) { - throw new Error('Cannot create an index with no fields'); - } - }catch(e){ - console.log('error', collection,name, index, opt_options); - throw e; + if (index.length === 0) { + throw new Error('Cannot create an index with no fields'); } + collection._collection._ensureIndex(index, opt_options); + } /** diff --git a/tests/indexing.js b/tests/indexing.js index cccc26d..c192ab3 100644 --- a/tests/indexing.js +++ b/tests/indexing.js @@ -1,7 +1,7 @@ var books = new Mongo.Collection('books'); //one before and one after to verify that order doesn't matter -books.attachIndex('isbnIdx', { +books.attachIndex('c2_isbn', { unique: true, action: 'rebuild', background: false @@ -14,7 +14,7 @@ books.attachSchema(new SimpleSchema({ label: 'Title', max: 200, index: { - name: 'titleIdx', + name: 'c2_title', type:-1 } }, @@ -42,7 +42,7 @@ books.attachSchema(new SimpleSchema({ type: String, label: 'ISBN', optional: true, - index: {name: 'isbnIdx'} + index: {name: 'c2_isbn'} }, field1: { type: String, @@ -63,7 +63,7 @@ books.attachSchema(new SimpleSchema({ })); -books.attachIndex('titleIdx', { +books.attachIndex('c2_title', { action: 'rebuild', background: false }); @@ -150,7 +150,7 @@ Tinytest.addAsync('Collection2 - Unique - Insert Duplicate', function (test, nex copies: 1, isbn: isbn }, function (error, result) { - console.log('error is', error, 'result is', result); + //console.log('error is', error, 'result is', result); test.isTrue(!!error, 'We expected the insert to trigger an error since isbn being inserted is already used'); //test.equal(error.code, 11000, 'We expected the insert to trigger an E11000 duplicate key error'); test.equal(error.invalidKeys.length, 1, 'We should get one invalidKey back attached to the Error object'); @@ -276,12 +276,15 @@ Tinytest.add('Collection2 - Unique - Object Array', function (test) { var testSchema = new SimpleSchema({ 'a.$.b': { type: String, - unique: true + index: { + name: 'c2_a.b' + } } }); try { testCollection.attachSchema(testSchema); + testCollection.attachIndex('c2_a.b', {unique: true}); } catch (e) { // If we error, that means collection2 tried to set up the index incorrectly, // using the wrong index key From 802b8d54f75c22c79443c3dce7fe0b7ca0ded418 Mon Sep 17 00:00:00 2001 From: "David M. Pankros" Date: Thu, 11 Feb 2016 17:16:26 -0500 Subject: [PATCH 5/8] More cleanup and bug fixes. set ecmascript version to 0.3.0 (latest). --- package.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.js b/package.js index 1cdb48b..dd463d1 100644 --- a/package.js +++ b/package.js @@ -11,7 +11,7 @@ Package.onUse(function(api) { 'underscore@1.0.0', 'minimongo@1.0.0', 'check@1.0.0', - 'ecmascript' + 'ecmascript@0.3.0' ]); api.addFiles([ From bbe5b210e8c4dcc384b94a415343dbc18a5af522 Mon Sep 17 00:00:00 2001 From: "David M. Pankros" Date: Thu, 11 Feb 2016 17:19:38 -0500 Subject: [PATCH 6/8] revert ecmascript to version dependant packages require. 0.1.6 --- package.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.js b/package.js index dd463d1..d715046 100644 --- a/package.js +++ b/package.js @@ -11,7 +11,7 @@ Package.onUse(function(api) { 'underscore@1.0.0', 'minimongo@1.0.0', 'check@1.0.0', - 'ecmascript@0.3.0' + 'ecmascript@0.1.6' ]); api.addFiles([ From d478d293a15651e12c60e08f4210b106bc152d21 Mon Sep 17 00:00:00 2001 From: "David M. Pankros" Date: Fri, 12 Feb 2016 07:08:50 -0500 Subject: [PATCH 7/8] Updated version number and package versions --- package.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/package.js b/package.js index d715046..ef44dc2 100644 --- a/package.js +++ b/package.js @@ -1,7 +1,7 @@ Package.describe({ name: "aldeed:schema-index", summary: "Control some MongoDB indexing with schema options", - version: "1.0.1", + version: "2.0.0", git: "https://github.com/aldeed/meteor-schema-index.git" }); @@ -30,7 +30,8 @@ Package.onTest(function(api) { 'underscore@1.0.0', 'random@1.0.0', 'mongo@1.0.0', - 'aldeed:simple-schema', + 'aldeed:collection2-core@1.0.0', + 'aldeed:simple-schema@1.5.3' ]); api.addFiles([ From 6dde54c72d4053b20fda957645022e0a32c976ce Mon Sep 17 00:00:00 2001 From: "David M. Pankros" Date: Sat, 13 Feb 2016 10:11:37 -0500 Subject: [PATCH 8/8] Split out IndexData class to a new file. Added c2_compat property to automatically create collection2 compatible index names when true. Updated README to reflect the compatibility change Updated package.js to add new indexData.js file --- .versions | 2 - LICENSE | 2 +- README.md | 68 +++++++++++++++++++++++++++++++- lib/indexData.js | 82 +++++++++++++++++++++++++++++++++++++++ lib/schemaIndex.js | 97 +++++++++------------------------------------- package.js | 3 +- tests/indexing.js | 12 +++--- 7 files changed, 176 insertions(+), 90 deletions(-) create mode 100644 lib/indexData.js diff --git a/.versions b/.versions index 938fc91..e2a3615 100644 --- a/.versions +++ b/.versions @@ -24,7 +24,6 @@ html-tools@1.0.5 htmljs@1.0.5 id-map@1.0.4 jquery@1.11.4 -local-test:aldeed:schema-index@1.0.1 logging@1.0.8 mdg:validation-error@0.2.0 meteor@1.1.10 @@ -42,7 +41,6 @@ retry@1.0.4 routepolicy@1.0.6 spacebars@1.0.7 spacebars-compiler@1.0.7 -tinytest@1.0.6 tracker@1.0.9 ui@1.0.8 underscore@1.0.4 diff --git a/LICENSE b/LICENSE index d343175..2831077 100644 --- a/LICENSE +++ b/LICENSE @@ -1,7 +1,7 @@ The MIT License (MIT) Copyright (c) 2015 Eric Dobbertin -Portions Copyright (c) 2016 David Pankros. +Portions Copyright (c) 2016 David Pankros Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in diff --git a/README.md b/README.md index c068b41..3ff659f 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,72 @@ In your Meteor app directory, enter: $ meteor add aldeed:schema-index ``` +## Migration from 1.0.x to 2.0.x +Version 1.0.x used index definitions exclusively on a collection's simpleschema. Version 2 moves some of the index properties to an external function call (`attachIndex`). Migration from 1.x to 2.x requires modificaiton of the simpleschema and addition of a call to `attachIndex` for each index to create. For example in version 1.0.x: + +```js +var books = new Mongo.Collection('books'); +books.attachSchema(new SimpleSchema({ + title: { + type: String, + label: 'Title', + max: 200, + index: true + }, + author: { + type: String, + label: 'Author' + }, + isbn: { + type: String, + label: 'ISBN', + optional: true, + index: 1, + unique: true + }, + //... +})); +``` + +In version 2.0.x: + +```js +var books = new Mongo.Collection('books'); +books.attachSchema(new SimpleSchema({ + title: { + type: String, + label: 'Title', + max: 200, + index: { + name: 'title', + type:-1 + } + }, + author: {cccccccblegetujrrtbgfrcncllvketghefdehvhiuln + + type: String, + label: 'Author' + }, + isbn: { + type: String, + label: 'ISBN', + optional: true, + index: {name: 'isbn'} + }, + //... +})); + +books.attachIndex('title'); //uses default index options +books.attachIndex('isbn', { + unique: true, + action: 'rebuild' //<-- drops and recreates the index EVERY TIME you restart. + } +); +``` + +For now, the index names added to mongo will not match the index names you specify. It is overridden to maintain some compatibility with collection2 due to a limitation in how collection2 determines invalid keys for its invalidKeys() method. Importantly, collection2 will not be able to determine the invalid keys for composite indexes if a constrain violation occurs. + + ## Usage Use the `index` option to ensure a MongoDB index for a specific field,or for multiple fields, add the same index to each: @@ -39,7 +105,7 @@ If you have created an index for a field by mistake and you want to remove or ch ```js books.attchIndex('titleIndex', { - action: 'drop' + action: 'drop' //options are: 'rebuild', 'build', or 'drop' } ); ``` diff --git a/lib/indexData.js b/lib/indexData.js new file mode 100644 index 0000000..3bef8ad --- /dev/null +++ b/lib/indexData.js @@ -0,0 +1,82 @@ +/** + * IndexData - internal class used to track information about the index as it + * comes in + */ + +IndexData = class IndexData { + constructor(name) { + check(name, String); + this._name = name; + this._options = { + background: true //default + }; + this._fields = {}; + this._action = 'build'; + this.c2_compatFix = true; + if (this.c2_compatFix) { + this._idxName = 'c2_'; + } + } + + get name() { + return this._name; + } + + get options() { + if (this.c2_compatFix) { + return _({}).extend(this._options, {name: this._idxName}); + } else { + return this._options; + } + } + + set options(val) { + if (!val) return; + + if (val.action) {//strip out action, if it is present + this._action = val.action; + delete val.action; + } + + if (!this.c2_compatFix) { + //name object must be last to override anything passed in + _(this._options).extend(val, {name: this.name}); + } else { + _(this._options).extend(val); + } + } + + setOptionValue(option, value) { + this.options[option] = value; + } + + getOptionValue(option) { + return this.options[option]; + } + + + get fields() { + return this._fields; + } + + addFieldIndex(fieldName, fieldType) { + var fieldName = fieldName.replace(/\.\$\./g, "."); + this.fields[fieldName] = fieldType; + if (this.c2_compatFix) { + this._idxName = this._idxName + fieldName; + } + } + + get needsBuild() { + return this.action === 'build' || this.action === 'rebuild'; + } + + get needsDrop() { + return this.action === 'rebuild' || this.action === 'drop'; + } + + get action() { + return this._action; + } +} + diff --git a/lib/schemaIndex.js b/lib/schemaIndex.js index d295e4f..0c60927 100644 --- a/lib/schemaIndex.js +++ b/lib/schemaIndex.js @@ -1,70 +1,3 @@ -/** - * IndexData - internal class used to track information about the index as it - * comes in - */ - -class IndexData { - constructor(name) { - check(name, String); - this._name = name; - this._options = { - background: true //default - }; - this._fields = {}; - this._action = 'build'; - } - - get name() { - return this._name; - } - - get options() { - return this._options; - } - - set options(val) { - if (!val) return; - - if (val.action) {//strip out action, if it is present - this._action = val.action; - delete val.action; - } - - //name object must be last to override anything passed in - _(this._options).extend(val, {name: this.name}); - } - - setOptionValue(option, value) { - this.options[option] = value; - } - - getOptionValue(option) { - return this.options[option]; - } - - - get fields() { - return this._fields; - } - - addFieldIndex(fieldName, fieldType) { - var fieldName = fieldName.replace(/\.\$\./g, "."); - this.fields[fieldName] = fieldType; - } - - get needsBuild() { - return this.action === 'build' || this.action === 'rebuild'; - } - - get needsDrop() { - return this.action === 'rebuild' || this.action === 'drop'; - } - - get action() { - return this._action; - } -} - /** * Primary class for tracking indexData and their definitions @@ -80,7 +13,7 @@ SchemaIndex = class SchemaIndex { */ static ensureIndex(collection, name, index, opt_options) { if (index.length === 0) { - throw new Error('Cannot create an index with no fields'); + throw new Error('Cannot create an index with no fields:' + name); } collection._collection._ensureIndex(index, opt_options); @@ -100,6 +33,9 @@ SchemaIndex = class SchemaIndex { } } + /** + * build an object + */ constructor() { this._indexes = {}; this._toBeAttached = []; @@ -138,7 +74,7 @@ SchemaIndex = class SchemaIndex { } /** - * Fetches an IndexData object. Creates it, if it doesn't exist. + * Fetches a single IndexData object. Creates it, if it doesn't exist. * @param name The name of the new index * @returns {*} */ @@ -186,9 +122,7 @@ SchemaIndex = class SchemaIndex { */ attach(collection, ss) { var self = this; - this._collection = collection; - //this._ss = ss; _.each(ss.schema(), function(definition, fieldName) { if ('index' in definition) { @@ -206,12 +140,19 @@ SchemaIndex = class SchemaIndex { } ); + //must set attached before processing, or queue will be skipped this._isSchemaAttached = true; this.processQueuedIndexes(); } + /** + * Converts the SimpleSchema index definition for fieldName to our internal + * representation + * @param indexDefs + * @param fieldName + */ addIndexFieldDefs(indexDefs, fieldName) { for (var i = 0; i < indexDefs.length; i++) { var indexDef = indexDefs[i]; @@ -225,10 +166,16 @@ SchemaIndex = class SchemaIndex { } } + /** + * process all the queued indexes if isSchemaAttached is true + */ processQueuedIndexes() { + if (! this.isSchemaAttached) return; + for (var i = 0; i < this.toBeAttached.length; i++) { this.processIndex(this.toBeAttached[i]); } + //clear the queue this._toBeAttached.length = 0; } @@ -279,12 +226,4 @@ SchemaIndex = class SchemaIndex { get collection() { return this._collection; } - - /** - * The SimpleSchema object if attached or undefined if not attached - * @returns {*} - */ - get simpleSchema() { - return this._ss; - } } diff --git a/package.js b/package.js index ef44dc2..2490547 100644 --- a/package.js +++ b/package.js @@ -19,7 +19,8 @@ Package.onUse(function(api) { ]); api.addFiles([ - 'lib/schemaIndex.js' + 'lib/schemaIndex.js', + 'lib/indexData.js' ], 'server'); }); diff --git a/tests/indexing.js b/tests/indexing.js index c192ab3..28a66d4 100644 --- a/tests/indexing.js +++ b/tests/indexing.js @@ -1,7 +1,7 @@ var books = new Mongo.Collection('books'); //one before and one after to verify that order doesn't matter -books.attachIndex('c2_isbn', { +books.attachIndex('isbn', { unique: true, action: 'rebuild', background: false @@ -14,7 +14,7 @@ books.attachSchema(new SimpleSchema({ label: 'Title', max: 200, index: { - name: 'c2_title', + name: 'title', type:-1 } }, @@ -42,7 +42,7 @@ books.attachSchema(new SimpleSchema({ type: String, label: 'ISBN', optional: true, - index: {name: 'c2_isbn'} + index: {name: 'isbn'} }, field1: { type: String, @@ -63,7 +63,7 @@ books.attachSchema(new SimpleSchema({ })); -books.attachIndex('c2_title', { +books.attachIndex('title', { action: 'rebuild', background: false }); @@ -277,14 +277,14 @@ Tinytest.add('Collection2 - Unique - Object Array', function (test) { 'a.$.b': { type: String, index: { - name: 'c2_a.b' + name: 'ab' } } }); try { testCollection.attachSchema(testSchema); - testCollection.attachIndex('c2_a.b', {unique: true}); + testCollection.attachIndex('ab', {unique: true}); } catch (e) { // If we error, that means collection2 tried to set up the index incorrectly, // using the wrong index key