[new] Import first experimental v4 core

carmine-v4.org

@@ -0,0 +1,74 @@
+#+TITLE: Title
+#+STARTUP: indent overview hidestars
+#+TAGS: { Cost: c1(1) c2(2) c3(3) c4(4) c5(5) }
+#+TAGS: nb(n) urgent(u)
+
+* Next
+** Review RESP implementation and tests
+** Port SSL API improvements from Telemere? Relevant?
+** Review config
+
+** Add SSB stats to pooled manager (borrow time, etc.)?
+** Add SSB stats to Sentinel?
+** Review arch needs for Cluster
+
+** Common & core util to parse-?marked-ba -> [<kind> <payload>]
+** Some way to implement a parser over >1 replies?
+E.g. fetch two sets, and parser to merge -> single reply
+
+* Later
+** New Pub/Sub API? (note RESP2 vs RESP3 differences)
+Pub/Sub + Sentinel integration
+psubscribe* to Sentinel server
+check for `switch-master` channel name
+"switch-master" <master-name> <old-ip> <old-port> <new-ip> <new-port>
+
+** Implement Cluster (enough user demand?)
+** Use Telemere (shell API?)
+
+* Polish
+** Print methods, toString content, etc.
+** Check all errors: eids, messages, data, cbids
+** Check all dynamic bindings and sys-vals, ensure accessible
+** Document `*default-conn-opts*`, incl. cbs
+** Document `*default-sentinel-opts*`, incl. cbs
+** Complete (esp. high-level / integration) tests
+** Review all config, docstring, privacy, etc.
+** Grep for TODOs
+
+* Refactor commands
+** Add modules support
+** Support custom (e.g. newer) commands.json or edn
+** Refactor helpers API, etc.
+** Modern Tundra?
+** Further MQ improvements?
+
+* Release
+** v4 upgrade/migration plan
+** v4 wiki with changes, migration, new features, examples, etc.
+** First public alpha
+
+* CHANGELOG
+** [new] Full RESP3 support, incl. streaming, etc.
+*** Enabled by default, requires Redis >= v6 (2020-04-30).
+** [new] Full Redis Sentinel support - incl. auto failover and read replicas.
+** [mod] Hugely improved connections API, incl. improved:
+*** Flexibility
+*** Docs
+*** Usability (e.g. opts validation, hard shutdowns, closing managed conns, etc.).
+*** Transparency (deref stats, cbs, timings for profiling, etc.).
+**** Derefs: Conns, ConnManagers, SentinelSpecs.
+*** Protocols for extension by advanced users.
+*** Full integration with Sentinel, incl.:
+**** Auto invalidation of pool conns on master changes.
+**** Auto verification of addresses on pool borrows.
+*** [new] Common conn utils are now aliased in core Carmine ns for convenience.
+*** [new] Improved pool efficiency, incl. smarter sub-pool keying.
+*** [mod] Improved parsing API, incl.:
+**** General simplifications.
+**** Aggregate parsers, with xform support.
+*** [new] *auto-serialize?*, *auto-deserialize?*
+*** [new] Greatly improved `skip-replies` performance
+*** [mod] Simplified parsers API
+*** [new] Improvements to docs, error messages, debug data, etc.
+*** [new] New Wiki with further documentation and examples.

project.clj

@@ -56,11 +56,16 @@
  {:language #{:clojure #_:clojurescript}
  :base-language :clojure}}}
 
+ :test-selectors
+ {:v3 (fn [{:keys [ns]} & _] (.startsWith (str ns) "taoensso.carmine."))
+ :v4 (fn [{:keys [ns]} & _] (.startsWith (str ns) "taoensso.carmine-v4."))}
+
  :aliases
  {"start-dev" ["with-profile" "+dev" "repl" ":headless"]
  ;; "build-once" ["do" ["clean"] ["cljsbuild" "once"]]
  "deploy-lib" ["do" #_["build-once"] ["deploy" "clojars"] ["install"]]
 
+ "test-v4" ["with-profile" "+c1.12:+c1.11:+c1.10:+c1.9" "test" ":v4"]
  "test-clj" ["with-profile" "+c1.12:+c1.11:+c1.10:+c1.9" "test"]
  ;; "test-cljs" ["with-profile" "+c1.12" "cljsbuild" "test"]
  "test-all" ["do" ["clean"] ["test-clj"] #_["test-cljs"]]})

src/taoensso/carmine_v4.clj

@@ -0,0 +1,280 @@
+(ns ^:no-doc taoensso.carmine-v4
+ "Experimental modern Clojure Redis client prototype.
+ Still private, not yet intended for public use!"
+ {:author "Peter Taoussanis (@ptaoussanis)"}
+ (:require
+ [taoensso.encore :as enc]
+ [taoensso.carmine :as v3-core]
+ [taoensso.carmine
+ [connections :as v3-conns]
+ [protocol :as v3-protocol]
+ [commands :as v3-cmds]]
+
+ [taoensso.carmine-v4.resp.common :as com]
+ [taoensso.carmine-v4.resp.read :as read]
+ [taoensso.carmine-v4.resp.write :as write]
+ [taoensso.carmine-v4.resp :as resp]
+ ;;
+ [taoensso.carmine-v4.utils :as utils]
+ [taoensso.carmine-v4.opts :as opts]
+ [taoensso.carmine-v4.conns :as conns]
+ [taoensso.carmine-v4.sentinel :as sentinel]
+ [taoensso.carmine-v4.cluster :as cluster]))
+
+(enc/assert-min-encore-version [3 112 0])
+
+(comment (remove-ns 'taoensso.carmine-v4))
+
+;;;; Config
+
+(def ^:dynamic *issue-83-workaround?*
+ "TODO Doc env config
+ A bug in Carmine v2.6.0 to v2.6.1 (2014-04-01 to 2014-05-01)
+ caused Nippy blobs to be marked incorrectly (with `ba-bin` instead
+ of `ba-npy`), Ref. <https://github.com/ptaoussanis/carmine/issues/83>
+
+ This should be kept true (the default) if there's a chance you might
+ read any data written by Carmine < v2.6.1 (2014-05-01).
+
+ Only relevant if `*auto-deserialize?` is true."
+ (enc/get-env {:as :bool, :default true}
+ :taoensso.carmine.issue-83-workaround))
+
+(def ^:dynamic *auto-serialize?*
+ "TODO Doc env config
+ Should Carmine automatically serialize arguments sent to Redis
+ that are non-native to Redis?
+
+ Affects non-(string, keyword, simple long/double) types.
+
+ Default is true. If false, an exception will be thrown when trying
+ to send such arguments.
+
+ See also `*auto-deserialize?`*."
+ (enc/get-env {:as :bool, :default true}
+ :taoensso.carmine.auto-serialize))
+
+(def ^:dynamic *auto-deserialize?*
+ "TODO Doc env config
+ Should Carmine automatically deserialize Redis replies that
+ contain data previously serialized by `*auto-serialize?*`?
+
+ Affects non-(string, keyword, simple long/double) types.
+
+ Default is true. If false, such replies will by default look like
+ malformed strings.
+ TODO: Mention utils, bindings.
+
+ See also `*auto-serialize?`*."
+ (enc/get-env {:as :bool, :default true}
+ :taoensso.carmine.auto-deserialize))
+
+(def ^:dynamic *keywordize-maps?*
+ "Keywordize string keys in map-type Redis replies?"
+ true)
+
+(def ^:dynamic *freeze-opts*
+ "TODO Docstring"
+ nil)
+
+(def default-pool-opts
+ "TODO Docstring: describe `pool-opts`, env config.
+
+ Ref. <https://commons.apache.org/proper/commons-pool/apidocs/org/apache/commons/pool2/impl/GenericKeyedObjectPool.html>,
+ <https://commons.apache.org/proper/commons-pool/apidocs/org/apache/commons/pool2/impl/BaseGenericObjectPool.html>"
+ (enc/nested-merge opts/default-pool-opts
+ (enc/get-env {:as :edn} :taoensso.carmine.default-pool-opts)))
+
+(def default-conn-manager-pooled_
+ "TODO Docstring"
+ (delay (conns/conn-manager-pooled {:pool-opts default-pool-opts})))
+
+(comment @default-conn-manager-pooled_)
+
+(def ^:dynamic *default-conn-opts*
+ "TODO Docstring: describe `conn-opts`, env config."
+ (opts/parse-conn-opts false
+ (enc/nested-merge opts/default-conn-opts
+ (enc/get-env {:as :edn} :taoensso.carmine.default-conn-opts))))
+
+(def ^:dynamic *default-sentinel-opts*
+ "TODO Docstring: describe `sentinel-opts`, env config."
+ (opts/parse-sentinel-opts
+ (enc/nested-merge opts/default-sentinel-opts
+ (enc/get-env {:as :edn} :taoensso.carmine.default-sentinel-opts))))
+
+(def ^:dynamic *conn-cbs*
+ "Map of any additional callback fns, as in `conn-opts` or `sentinel-opts`.
+ Useful for REPL/debugging/tests/etc.
+
+ Possible keys:
+ `:on-conn-close`
+ `:on-conn-error`
+ `:on-resolve-success`
+ `:on-resolve-error`
+ `:on-changed-master`
+ `:on-changed-replicas`
+ `:on-changed-sentinels`
+
+ Values should be unary callback fns of a single data map."
+ nil)
+
+;;;; Aliases
+
+(enc/defaliases
+ enc/get-env
+
+ ;;; Read opts
+ com/skip-replies
+ com/normal-replies
+ com/as-bytes
+ com/as-thawed
+ com/natural-reads
+
+ ;;; Reply parsing
+ com/reply-error?
+ com/unparsed
+ com/parse
+ com/parse-aggregates
+ com/completing-rf
+ ;;
+ com/as-?long
+ com/as-?double
+ com/as-?kw
+ ;;
+ com/as-long
+ com/as-double
+ com/as-kw
+
+ ;;; Write wrapping
+ write/to-bytes
+ write/to-frozen
+
+ ;;; RESP3
+ resp/redis-call
+ resp/redis-call*
+ resp/redis-calls
+ resp/redis-calls*
+ resp/local-echo
+ resp/local-echos
+ resp/local-echos*
+
+ ;;; Connections
+ conns/conn?
+ conns/conn-ready?
+ conns/conn-close!
+ ;;
+ sentinel/sentinel-spec
+ sentinel/sentinel-spec?
+ ;;
+ conns/conn-manager?
+ conns/conn-manager-unpooled
+ conns/conn-manager-pooled
+ {:alias conn-manager-init! :src conns/mgr-init!}
+ {:alias conn-manager-ready? :src conns/mgr-ready?}
+ {:alias conn-manager-close! :src conns/mgr-close!}
+ {:alias conn-manager-master-changed! :src conns/mgr-master-changed!}
+
+ ;;; Cluster
+ cluster/cluster-key)
+
+;;;; Core API (main entry point to Carmine)
+
+(defn with-car
+ "TODO Docstring: `*default-conn-opts*`, `wcar`, etc.
+ body-fn takes [conn]"
+ ([conn-opts body-fn] (with-car conn-opts nil body-fn))
+ ([conn-opts {:keys [as-vec?] :as reply-opts} body-fn]
+ (let [{:keys [natural-reads?]} reply-opts] ; Undocumented
+ (conns/with-conn
+ (conns/get-conn conn-opts true true)
+ (fn [conn in out]
+ (resp/with-replies in out natural-reads? as-vec?
+ (fn [] (body-fn conn))))))))
+
+(comment
+ (do
+ (def mgr-unpooled (conns/conn-manager-unpooled {}))
+ (def mgr-pooled (conns/conn-manager-pooled
+ {:pool-opts
+ {:test-on-create? false
+ :test-on-borrow? false
+ :test-on-return? false}})))
+
+ (enc/qb 1e3 ; [97.84 99.07 23.1], unpooled port limited
+ (with-car {:mgr nil} (fn [conn] (resp/ping) (resp/local-echo conn)))
+ (with-car {:mgr #'mgr-unpooled} (fn [conn] (resp/ping) (resp/local-echo conn)))
+ (with-car {:mgr #'mgr-pooled} (fn [conn] (resp/ping) (resp/local-echo conn))))
+
+ (-> unpooled deref)
+ (-> (-> pooled deref) :stats_ deref)
+
+ ;; Benching
+ (let [conn-opts {:mgr #'mgr-pooled :init nil}]
+ (enc/qb 1e4 ; [16.49 219.44 691.71]
+ (with-car conn-opts (fn [_]))
+ (with-car conn-opts (fn [_] (resp/ping)))
+ (with-car conn-opts (fn [_] (dotimes [_ 100] (resp/ping)))))))
+
+(defmacro wcar
+ "TODO Docstring: `*default-conn-opts*`, `with-car`, etc."
+ {:arglists
+ '([conn-opts & body]
+ [conn-opts {:keys [as-vec?]} & body])}
+
+ [conn-opts & body]
+ (let [[reply-opts body] (resp/parse-body-reply-opts body)]
+ `(with-car ~conn-opts ~reply-opts
+ (fn [~'__wcar-conn] ~@body))))
+
+(comment
+ (wcar {} (resp/redis-call "PING"))
+ (wcar {} (resp/redis-call "set" "k1" 3))
+ (wcar {} (resp/redis-call "get" "k1"))
+
+ (wcar {} (resp/ping))
+ (wcar {} {:as-vec? true} (resp/ping))
+ (wcar {} :as-vec (resp/ping))
+
+ (enc/qb 1e4 ; [209.87 222.09]
+ (v3-core/wcar {} (v3-core/ping))
+ (wcar {:mgr #'mgr-pooled} (resp/ping))))
+
+(defmacro with-replies
+ "TODO Docstring
+ Expects to be called within the body of `wcar` or `with-car`."
+ {:arglists '([& body] [{:keys [as-vec?]} & body])}
+ [& body]
+ (let [[reply-opts body] (resp/parse-body-reply-opts body)
+ {:keys [natural-reads? as-vec?]} reply-opts]
+
+ `(resp/with-replies ~natural-reads? ~as-vec?
+ (fn [] ~@body))))
+
+;;;; Push API ; TODO
+
+(defmulti push-handler (fn [state [data-type :as data-vec]] data-type))
+(defmethod push-handler :default [state data-vec] #_(println data-vec) nil)
+
+(enc/defonce push-agent_
+ (delay (agent nil :error-mode :continue)))
+
+(def ^:dynamic *push-fn*
+ "TODO Docstring: this and push-handler, etc.
+ ?(fn [data-vec]) => ?effects.
+ If provided (non-nil), this fn should never throw."
+ (fn [data-vec]
+ (send-off @push-agent_
+ (fn [state]
+ (try
+ (push-handler state data-vec)
+ (catch Throwable t
+ ;; TODO Try publish error message?
+ ))))))
+
+;;;; Scratch
+
+;; TODO For command docstrings
+;; As with all Carmine Redis command fns: expects to be called within a `wcar`
+;; body, and returns nil. The server's reply to this command will be included
+;; in the replies returned by the enclosing `wcar`.

src/taoensso/carmine_v4/classes.clj

@@ -0,0 +1,6 @@
+(ns ^:no-doc taoensso.carmine-v4.classes
+ "Private ns, implementation detail.
+ Classes, interfaces, etc. isolated from other code to prevent
+ identity issues during REPL work.")
+
+(definterface ReplyError)