Manifest (#430 + #431)

* Initial manifest support (#426)

* add auth section to upgrade

* exchange auth definitions through a manifest

* fix a ton of stuff

* skip custom keypair tests for now

* allow explicit compat mode

* no need to special case keypair

* actually fix preload test

* make manifest explicit

* also set compat mode when replicating manifest

* very explicit manifest exchange

* verifyBatch -> verifyBatchUpgraded

* tweak namespacing and add explicit default ns

* add preliminary functions for parsing manifest

* typo on imports

* add encodings and entropy/namespacing

* add compat signer and more tests

* lib/core uses manifest createAuth function

* default to compat mode

* make signers classes and update tests

* correctly set compat on sessions/clones/upgrade

* fix bad merge

* need to specify compat for clone

* replace auth with manifest

* update clone and ensure that key is different

* update tests to manifest api

* load new auth if secret key is provided

* remove custom auth test

* support passing a sign function

* remove auth tests

* manifest renamed to multipleSigners

* properly set compat and keyPair

* pass batch to patched verify

* port tests from hypercore-crypto-multisig

* include file for multisig helpers

* rename to verifiers and add explicit compat verifier

* split out signing and verification

* remove sign option and just pass keyPair or signature

* only set keyPair after openCapabilities

* clarify code around creating a manifest

* conflict test now works

* fix core copyFrom and core clone tests

* fix clone and clone tests

* standardise opts across append and truncate

* tidy up

* add multisig lib file

* expose helper

* fix batch flush api

* fix noManifest logic

* fix some typos/missing apis

* set signature on tree only after verification

* rename static helpers

* rename defaultAuth to verifier

* rename defaultAuth to verifier

* remove static createVerifier method

* review by @mafintosh

* move manifestHash to lib/manifest

---------

Co-authored-by: Mathias Buus <mathiasbuus@gmail.com>

* last part of the constructor should be openSession

* simplify key handler

* fix keypair handler

* define writable condition in one place

* fix typo in manifestHash

* pass manifest directly to createVerifier

* no need to dbl copy manifest now

* tweak manifest construction

* consistent errors

* add manifest getter

* tweak manifest checks on load

* review by @chm-diederichs

* set keypair in constructor if possible

* simplify core append signature

* tweak batch flush

* simplify compat option

* update batch test

* move compat default to lib/core

* move compat check to a single func

* no manifesthash in caps

---------

Co-authored-by: Christophe Diederichs <45171645+chm-diederichs@users.noreply.github.com>
Co-authored-by: Christophe Diederichs <chm-diederichs@protonmail.com>

Manifest tweaks (#431)

* more tests for now truncate/append options

* move more manifest functinality into manifest.js

* move isCompat to manifest and simplify clone options

* enable all tests

* pass batches to verifiers and move namespace support to batch

* fix storage test

index.js

@@ -15,6 +15,7 @@ const BlockEncryption = require('./lib/block-encryption')
 const Info = require('./lib/info')
 const Download = require('./lib/download')
 const Batch = require('./lib/batch')
+const { manifestHash, defaultSignerManifest, createVerifier, createManifest, isCompat } = require('./lib/manifest')
 const { ReadStream, WriteStream, ByteStream } = require('./lib/streams')
 const {
  BAD_ARGUMENT,
@@ -69,30 +70,31 @@ module.exports = class Hypercore extends EventEmitter {
 
  this.id = null
  this.key = key || null
- this.keyPair = null
+ this.keyPair = opts.keyPair || null
  this.readable = true
  this.writable = false
  this.opened = false
  this.closed = false
  this.snapshotted = !!opts.snapshot
  this.sparse = opts.sparse !== false
  this.sessions = opts._sessions || [this]
- this.auth = opts.auth || null
  this.autoClose = !!opts.autoClose
  this.onwait = opts.onwait || null
  this.wait = opts.wait !== false
  this.timeout = opts.timeout || 0
- this._clone = opts.clone || null
- this._readonly = opts.writable === false
-
  this.closing = null
- this.opening = this._openSession(key, storage, opts)
- this.opening.catch(noop)
+ this.opening = null
 
+ this._manifest = opts.manifest || null
+ this._clone = opts.clone || null
+ this._readonly = opts.writable === false
  this._preappend = preappend.bind(this)
  this._snapshot = null
  this._batch = opts._batch || null
  this._findingPeers = 0
+
+ this.opening = this._openSession(key, storage, opts)
+ this.opening.catch(noop)
  }
 
  [inspect] (depth, opts) {
@@ -139,6 +141,14 @@ module.exports = class Hypercore extends EventEmitter {
  indent + ')'
  }
 
+ static key (manifest, { compat } = {}) {
+ return compat ? manifest.signer.publicKey : manifestHash(manifest)
+ }
+
+ static discoveryKey (key) {
+ return hypercoreCrypto.discoveryKey(key)
+ }
+
  static getProtocolMuxer (stream) {
  return stream.noiseStream.userData
  }
@@ -245,20 +255,19 @@ module.exports = class Hypercore extends EventEmitter {
  }
 
  setKeyPair (keyPair) {
- this.auth = Core.createAuth(this.crypto, { keyPair })
- this.writable = !this._readonly && !!this.auth && !!this.auth.sign
+ this.keyPair = keyPair
+ this.writable = this._isWritable()
  }
 
  _passCapabilities (o) {
- if (!this.auth) this.auth = o.auth
-
+ if (!this.keyPair) this.keyPair = o.keyPair
  this.crypto = o.crypto
  this.id = o.id
  this.key = o.key
  this.core = o.core
  this.replicator = o.replicator
  this.encryption = o.encryption
- this.writable = !this._readonly && !!this.auth && !!this.auth.sign
+ this.writable = this._isWritable()
  this.autoClose = o.autoClose
 
  if (this.snapshotted && this.core && !this._snapshot) this._updateSnapshot()
@@ -288,14 +297,6 @@ module.exports = class Hypercore extends EventEmitter {
  if (!isFirst) await opts._opening
  if (opts.preload) opts = { ...opts, ...(await this._retryPreload(opts.preload)) }
 
- const keyPair = opts.keyPair
-
- if (opts.auth) {
- this.auth = opts.auth
- } else if (keyPair && keyPair.secretKey) {
- this.setKeyPair(keyPair)
- }
-
  if (isFirst) {
  await this._openCapabilities(key, storage, opts)
  // Only the root session should pass capabilities to other sessions.
@@ -306,15 +307,14 @@ module.exports = class Hypercore extends EventEmitter {
 
  // copy state over
  if (this._clone) {
- const { from, upgrade } = this._clone
+ const { from, signature } = this._clone
  await from.opening
- await this.core.copyFrom(from.core, upgrade)
+ await this.core.copyFrom(from.core, signature)
  this._clone = null
  }
  }
 
- if (!this.auth) this.auth = this.core.defaultAuth
- this.writable = !this._readonly && !!this.auth && !!this.auth.sign
+ this.writable = this._isWritable()
 
  if (opts.valueEncoding) {
  this.valueEncoding = c.from(opts.valueEncoding)
@@ -353,7 +353,7 @@ module.exports = class Hypercore extends EventEmitter {
  this.storage = Hypercore.defaultStorage(opts.storage || storage, { unlocked, writable: !unlocked })
 
  this.core = await Core.open(this.storage, {
- compat: opts.compat !== false, // default to true for now
+ compat: opts.compat,
  force: opts.force,
  createIfMissing: opts.createIfMissing,
  readonly: unlocked,
@@ -362,7 +362,7 @@ module.exports = class Hypercore extends EventEmitter {
  keyPair: opts.keyPair,
  crypto: this.crypto,
  legacy: opts.legacy,
- auth: opts.auth,
+ manifest: opts.manifest,
  onupdate: this._oncoreupdate.bind(this),
  onconflict: this._oncoreconflict.bind(this)
  })
@@ -417,6 +417,10 @@ module.exports = class Hypercore extends EventEmitter {
  return prev.length !== next.length || prev.fork !== next.fork
  }
 
+ _isWritable () {
+ return !this._readonly && !!(this.keyPair && this.keyPair.secretKey)
+ }
+
  close (err) {
  if (this.closing) return this.closing
  this.closing = this._close(err || null)
@@ -465,23 +469,22 @@ module.exports = class Hypercore extends EventEmitter {
  this.emit('close', true)
  }
 
- clone (storage, opts = {}) {
+ clone (keyPair, storage, opts = {}) {
  // TODO: current limitation is no forking
  if ((opts.fork && opts.fork !== 0) || this.fork !== 0) {
  throw BAD_ARGUMENT('Cannot clone a fork')
  }
 
- const key = opts.key === undefined ? opts.keyPair ? null : this.key : opts.key
- const keyPair = (opts.auth || opts.keyPair === undefined) ? null : opts.keyPair
+ const manifest = opts.manifest || defaultSignerManifest(keyPair.publicKey)
+ const key = opts.key || (opts.compat !== false ? manifest.signer.publicKey : manifestHash(manifest))
 
- let auth = this.core.defaultAuth
- if (opts.auth) {
- auth = opts.auth
- } else if (keyPair && keyPair.secretKey) {
- auth = Core.createAuth(this.crypto, { keyPair, manifest: { signer: { publicKey: keyPair.publicKey } } })
+ if (b4a.equals(key, this.key)) {
+ throw BAD_ARGUMENT('Clone cannot share verification information')
  }
 
- const upgrade = opts.upgrade === undefined ? null : opts.upgrade
+ const signature = opts.signature === undefined
+ ? createVerifier(createManifest(manifest), { compat: isCompat(key, manifest) }).sign(this.core.tree.batch(), keyPair)
+ : opts.signature
 
  const sparse = opts.sparse === false ? false : this.sparse
  const wait = opts.wait === false ? false : this.wait
@@ -497,11 +500,11 @@ module.exports = class Hypercore extends EventEmitter {
  wait,
  onwait,
  timeout,
- auth,
+ manifest,
  overwrite: true,
  clone: {
  from: this,
- upgrade
+ signature
  }
  })
  }
@@ -542,6 +545,10 @@ module.exports = class Hypercore extends EventEmitter {
  return this.replicator === null ? null : this.replicator.discoveryKey
  }
 
+ get manifest () {
+ return this._manifest || (this.core === null ? null : this.core.header.manifest)
+ }
+
  get length () {
  if (this._snapshot) return this._snapshot.length
  if (this.core === null) return 0
@@ -934,16 +941,22 @@ module.exports = class Hypercore extends EventEmitter {
  if (this.opened === false) await this.opening
  if (this.writable === false) throw SESSION_NOT_WRITABLE()
 
- const { fork = this.core.tree.fork + 1, force = false } = typeof opts === 'number' ? { fork: opts } : opts
+ const {
+ fork = this.core.tree.fork + 1,
+ force = false,
+ keyPair = this.keyPair,
+ signature = null
+ } = typeof opts === 'number' ? { fork: opts } : opts
+
  if (this._batch && !force) throw BATCH_UNFLUSHED()
 
- await this.core.truncate(newLength, fork, this.auth)
+ await this.core.truncate(newLength, fork, { keyPair, signature })
 
  // TODO: Should propagate from an event triggered by the oplog
  this.replicator.updateAll()
  }
 
- async append (blocks, opts) {
+ async append (blocks, opts = {}) {
  if (this._batch && this !== this._batch.session) throw BATCH_UNFLUSHED()
 
  if (this.opened === false) await this.opening
@@ -961,7 +974,8 @@ module.exports = class Hypercore extends EventEmitter {
  }
  }
 
- return this.core.append(buffers, (opts && opts.auth) || this.auth, { preappend })
+ const { keyPair = this.keyPair, signature = null } = opts
+ return this.core.append(buffers, { keyPair, signature, preappend })
  }
 
  async treeHash (length) {

lib/batch.js

@@ -59,8 +59,8 @@ module.exports = class HypercoreBatch extends EventEmitter {
  return this.session.core
  }
 
- get auth () {
- return this.session.auth
+ get manifest () {
+ return this.session.manifest
  }
 
  async ready () {
@@ -230,12 +230,14 @@ module.exports = class HypercoreBatch extends EventEmitter {
  return info
  }
 
- async flush (length = this._appends.length, auth) {
+ async flush (opts = {}) {
  if (this.opened === false) await this.opening
  if (this.closing) throw SESSION_CLOSED()
 
+ const { length = this._appends.length, keyPair = this.session.keyPair, signature = null } = opts
+
  while (this._flushing) await this._flushing
- this._flushing = this._flush(length, auth)
+ this._flushing = this._flush(length, keyPair, signature)
 
  try {
  await this._flushing
@@ -246,11 +248,12 @@ module.exports = class HypercoreBatch extends EventEmitter {
  if (this.autoClose) await this.close()
  }
 
- async _flush (length, auth) { // TODO: make this safe to interact with a parallel truncate...
+ async _flush (length, keyPair, signature) { // TODO: make this safe to interact with a parallel truncate...
  if (this._appends.length === 0) return
 
  const flushingLength = Math.min(length, this._appends.length)
- const info = await this.session.append(flushingLength < this._appends.length ? this._appends.slice(0, flushingLength) : this._appends, { auth })
+ const blocks = flushingLength < this._appends.length ? this._appends.slice(0, flushingLength) : this._appends
+ const info = await this.session.append(blocks, { keyPair, signature })
  const delta = info.byteLength - this._sessionByteLength
 
  this._sessionLength = info.length

lib/caps.js

@@ -8,6 +8,7 @@ const c = require('compact-encoding')
 
 const [TREE, REPLICATE_INITIATOR, REPLICATE_RESPONDER, MANIFEST, DEFAULT_NAMESPACE] = crypto.namespace('hypercore', 5)
 
+exports.MANIFEST = MANIFEST
 exports.DEFAULT_NAMESPACE = DEFAULT_NAMESPACE
 
 exports.replicate = function (isInitiator, key, handshakeHash) {
@@ -16,27 +17,20 @@ exports.replicate = function (isInitiator, key, handshakeHash) {
  return out
 }
 
-exports.manifestHash = function (manifest) {
- const state = { start: 0, end: 32 + manifest.byteLength, buffer: null }
- state.buffer = b4a.allocUnsafe(state.end)
- c.raw.encode(state, MANIFEST)
- c.raw.encode(state, manifest)
- const out = b4a.allocUnsafe(32)
- sodium.crypto_generichash(out, state.buffer)
- return out
-}
-
-exports.treeSignable = function (hash, length, fork) {
- const state = { start: 0, end: 80, buffer: b4a.allocUnsafe(80) }
+exports.treeSignable = function (namespace, hash, length, fork) {
+ const state = { start: 0, end: 112, buffer: b4a.allocUnsafe(112) }
  c.raw.encode(state, TREE)
+ c.raw.encode(state, namespace)
  c.raw.encode(state, hash)
  c.uint64.encode(state, length)
  c.uint64.encode(state, fork)
  return state.buffer
 }
 
-exports.treeSignableLegacy = function (hash, length, fork) {
- const state = { start: 0, end: 48, buffer: b4a.allocUnsafe(48) }
+exports.treeSignableCompat = function (hash, length, fork, noHeader) {
+ const end = noHeader ? 48 : 80
+ const state = { start: 0, end, buffer: b4a.allocUnsafe(end) }
+ if (!noHeader) c.raw.encode(state, TREE) // ultra legacy mode, kill in future major
  c.raw.encode(state, hash)
  c.uint64.encode(state, length)
  c.uint64.encode(state, fork)