diff --git a/bb-runtime.edn b/bb-runtime.edn new file mode 100644 index 000000000..dc40f93f1 --- /dev/null +++ b/bb-runtime.edn @@ -0,0 +1,26 @@ +{:min-bb-version "1.0.164" + :paths ["src" "notebooks" "resources"] + :deps {hiccup/hiccup {:mvn/version "2.0.0-alpha2"} + org.babashka/cli {:mvn/version "0.5.40"} + io.github.nextjournal/markdown {:mvn/version "0.4.126"} + io.github.nextjournal/clerk-slideshow {:git/sha "562f634494a1e1a9149ed78d5d39fd9486cc00ba"}} + :tasks + {dev + {:requires ([babashka.fs :as fs] + [babashka.nrepl.server :as srv] + [nextjournal.clerk :as clerk] + [babashka.cli :as cli]) + :task (do (srv/start-server! {:host "localhost" :port 1339}) + (spit ".nrepl-port" "1339") + (nextjournal.clerk/serve! (cli/parse-opts *command-line-args*)) + (-> (Runtime/getRuntime) + (.addShutdownHook (Thread. (fn [] + (nextjournal.clerk/halt!) + (fs/delete ".nrepl-port"))))) + (deref (promise)))} + + build + {:requires ([nextjournal.clerk :as clerk] + [babashka.cli :as cli]) + :task (let [spec (-> (resolve 'nextjournal.clerk/build!) meta :org.babashka/cli)] + (clerk/build! (cli/parse-opts *command-line-args* spec)))}}} diff --git a/resources/viewer-js-hash b/resources/viewer-js-hash index 7a589706c..98e4fca0e 100644 --- a/resources/viewer-js-hash +++ b/resources/viewer-js-hash @@ -1 +1 @@ -46k5SeDmWgR6seZFSVzwdiCJ7BEi \ No newline at end of file +jTDHst3WuaZZ65Kbk7vW4w7315u \ No newline at end of file diff --git a/src/nextjournal/beholder.bb b/src/nextjournal/beholder.bb new file mode 100644 index 000000000..e6d96e0c9 --- /dev/null +++ b/src/nextjournal/beholder.bb @@ -0,0 +1,5 @@ +(ns nextjournal.beholder + "Babashka runtime no-op stubs") + +(defn watch [cb & args] nil) +(defn stop [w] nil) diff --git a/src/nextjournal/clerk/analyzer.bb b/src/nextjournal/clerk/analyzer.bb new file mode 100644 index 000000000..3fbe94d7a --- /dev/null +++ b/src/nextjournal/clerk/analyzer.bb @@ -0,0 +1,6 @@ +(ns nextjournal.clerk.analyzer + "Babashka runtime no-op stubs") + +;; TODO: consider using this in eval +(defn valuehash [val] "valuehash") +(defn ->hash-str [val] (valuehash val)) diff --git a/src/nextjournal/clerk/builder.clj b/src/nextjournal/clerk/builder.clj index 126b50f3e..fa9d47efc 100644 --- a/src/nextjournal/clerk/builder.clj +++ b/src/nextjournal/clerk/builder.clj @@ -2,9 +2,7 @@ "Clerk's Static App Builder." (:require [babashka.fs :as fs] [clojure.java.browse :as browse] - [clojure.set :as set] [clojure.string :as str] - [nextjournal.clerk.analyzer :as analyzer] [nextjournal.clerk.builder-ui :as builder-ui] [nextjournal.clerk.eval :as eval] [nextjournal.clerk.parser :as parser] @@ -197,7 +195,7 @@ {state :result duration :time-ms} (eval/time-ms (mapv (comp (partial parser/parse-file {:doc? true}) :file) state)) _ (report-fn {:stage :parsed :state state :duration duration}) {state :result duration :time-ms} (eval/time-ms (reduce (fn [state doc] - (try (conj state (-> doc analyzer/build-graph analyzer/hash)) + (try (conj state (eval/analyze-doc doc)) (catch Exception e (reduced {:error e})))) [] diff --git a/src/nextjournal/clerk/eval.bb b/src/nextjournal/clerk/eval.bb new file mode 100644 index 000000000..28ad406f3 --- /dev/null +++ b/src/nextjournal/clerk/eval.bb @@ -0,0 +1,168 @@ +(ns nextjournal.clerk.eval + "Clerk's incremental evaluation (Babashka Edition) with in-memory caching layer." + (:refer-clojure :exclude [read-string]) + (:require [clojure.string :as str] + [edamame.core :as edamame] + [nextjournal.clerk.config :as config] + [nextjournal.clerk.parser :as parser] + [nextjournal.clerk.viewer :as v])) + +(defn wrapped-with-metadata [value hash] + (cond-> {:nextjournal/value value} + ;; TODO: maybe fix hash for blob serving + hash (assoc :nextjournal/blob-id (cond-> hash (not (string? hash)) str #_ multihash/base58)))) + +#_(wrap-with-blob-id :test "foo") + +(defn elapsed-ms [from] + (/ (double (- (. System (nanoTime)) from)) 1000000.0)) + +(defmacro time-ms + "Pure version of `clojure.core/time`. Returns a map with `:result` and `:time-ms` keys." + [expr] + `(let [start# (System/nanoTime) + ret# ~expr] + {:result ret# + :time-ms (elapsed-ms start#)})) + +(defn ^:private var-from-def [var] + (let [resolved-var (cond (var? var) + var + + (symbol? var) + (find-var var) + + :else + (throw (ex-info "Unable to resolve into a variable" {:data var})))] + {:nextjournal.clerk/var-from-def resolved-var})) + +(defn ^:private eval-form [{:keys [form var no-cache?]} hash] + (try + (let [{:keys [result]} (time-ms (binding [config/*in-clerk* true] + (eval form))) + result (if (and (nil? result) var (= 'defonce (first form))) + (find-var var) + result) + var-value (cond-> result (and var (var? result)) deref) + no-cache? (or no-cache? config/cache-disabled?)] + (let [blob-id (cond no-cache? "valuehash" #_#_ TODO?/valuehash (analyzer/->hash-str var-value) + (fn? var-value) nil + :else hash) + result (if var (var-from-def var) result)] + (wrapped-with-metadata result blob-id))) + (catch Throwable t + (throw (ex-info (ex-message t) (Throwable->map t)))))) + +(defn maybe-eval-viewers [{:as opts :nextjournal/keys [viewer viewers]}] + (cond-> opts + viewer + (update :nextjournal/viewer eval) + viewers + (update :nextjournal/viewers eval))) + +(defn read+eval-cached [{:as _doc :keys [blob->result]} {:as codeblock :keys [form vars var ns-effect? no-cache?]}] + (let [no-cache? (or ns-effect? no-cache?) + hash (.encodeToString (java.util.Base64/getEncoder) (.getBytes (str form))) + opts-from-form-meta (-> (meta form) + (select-keys [:nextjournal.clerk/viewer :nextjournal.clerk/viewers :nextjournal.clerk/width :nextjournal.clerk/opts]) + v/normalize-viewer-opts + maybe-eval-viewers)] + (cond-> (or (when-let [cached-result (and (not no-cache?) (get-in blob->result [hash :nextjournal/value]))] + (wrapped-with-metadata cached-result hash)) + (eval-form codeblock hash)) + (seq opts-from-form-meta) + (merge opts-from-form-meta)))) + +(defn eval-analyzed-doc [{:as analyzed-doc :keys [ns blocks]}] + (let [{:as evaluated-doc :keys [blob-ids]} + (reduce (fn [state {:as cell :keys [type]}] + (let [{:as result :nextjournal/keys [blob-id]} (when (= :code type) (read+eval-cached state cell))] + (cond-> (update state :blocks conj (cond-> cell result (assoc :result result))) + blob-id (update :blob-ids conj blob-id) + blob-id (assoc-in [:blob->result blob-id] result)))) + (assoc analyzed-doc :blocks [] :blob-ids #{}) + blocks)] + (-> evaluated-doc + (cond-> (not ns) (assoc :ns (find-ns 'user))) + (update :blob->result select-keys blob-ids) + (dissoc :blob-ids)))) + +(defn read-string [s] + (edamame/parse-string s + {:all true + :readers *data-readers* + :read-cond :allow + :regex #(list `re-pattern %) + :features #{:clj} + :auto-resolve (as-> (ns-aliases (or *ns* (find-ns 'user))) $ + (zipmap (keys $) (map ns-name (vals $))) + (assoc $ :current (ns-name *ns*)))})) + +(defn deflike? [form] (and (seq? form) (symbol? (first form)) (str/starts-with? (name (first form)) "def"))) +#_(deflike? (read-string "(def ^{:doc \"aloha\"} foo 123)")) +#_(deflike? (read-string "(def ^{:doc \"aloha\"} foo 123)")) +(defn no-cache-from-meta [form] + (when (contains? (meta form) :nextjournal.clerk/no-cache) + (-> form meta :nextjournal.clerk/no-cache))) +(defn no-cache? [& subjects] (or (some no-cache-from-meta subjects) false)) +(defn deref? [form] + (and (seq? form) + (= (first form) `deref) + (= 2 (count form)))) + +(defn read-forms [doc] + (binding [*ns* *ns*] + (reduce (fn [doc {:as b :keys [type text]}] + (let [form (read-string text) + ns? (= 'ns (when (list? form) (first form))) + var (when (and (deflike? form) (symbol? (second form))) (second form))] + (when ns? (eval form)) + (-> doc + (cond-> (and ns? (not (:ns doc))) (assoc :ns *ns*)) + (update :blocks conj + (cond-> b + (= :code type) (assoc :form form) + (or ns? (deref? form) (no-cache? form var *ns*)) (assoc :no-cache? true) + var (assoc :var (symbol (name (ns-name *ns*)) (name var)))))))) + (assoc doc :blocks []) + (:blocks doc)))) + +#_(read-forms + (parser/parse-file "notebooks/hello.clj")) + +;; used in builder +(def analyze-doc read-forms) + +(defn +eval-results + "Evaluates the given `parsed-doc` using the `in-memory-cache` and augments it with the results." + [in-memory-cache parsed-doc] + (let [{:as doc :keys [ns]} (read-forms parsed-doc)] + (binding [*ns* (or ns *ns*)] + (-> doc + (assoc :blob->result in-memory-cache) + eval-analyzed-doc)))) + +(defn eval-doc + "Evaluates the given `doc`." + ([doc] (eval-doc {} doc)) + ([in-memory-cache doc] (+eval-results in-memory-cache doc))) + +(defn eval-file + "Reads given `file` (using `slurp`) and evaluates it." + ([file] (eval-file {} file)) + ([in-memory-cache file] + (->> file + (parser/parse-file {:doc? true}) + (eval-doc in-memory-cache)))) + +#_(eval-file "notebooks/hello.clj") +#_(eval-file "notebooks/rule_30.clj") +#_(eval-file "notebooks/visibility.clj") + +(defn eval-string + "Evaluated the given `code-string` using the optional `in-memory-cache` map." + ([code-string] (eval-string {} code-string)) + ([in-memory-cache code-string] + (eval-doc in-memory-cache (parser/parse-clojure-string {:doc? true} code-string)))) + +#_(eval-string "(+ 39 3)") diff --git a/src/nextjournal/clerk/eval.clj b/src/nextjournal/clerk/eval.clj index 7898c7ef8..9b0541f9e 100644 --- a/src/nextjournal/clerk/eval.clj +++ b/src/nextjournal/clerk/eval.clj @@ -192,6 +192,9 @@ (update :blob->result select-keys blob-ids) (dissoc :blob-ids)))) +;; TODO: used in builder to drop analyzer dependency, cfr. below +(defn analyze-doc [doc] (-> doc analyzer/build-graph analyzer/hash)) + (defn +eval-results "Evaluates the given `parsed-doc` using the `in-memory-cache` and augments it with the results." [in-memory-cache parsed-doc] @@ -226,4 +229,3 @@ (eval-doc in-memory-cache (parser/parse-clojure-string {:doc? true} code-string)))) #_(eval-string "(+ 39 3)") - diff --git a/src/nextjournal/clerk/sci_viewer.cljs b/src/nextjournal/clerk/sci_viewer.cljs index 48ccb3a4e..4492af151 100644 --- a/src/nextjournal/clerk/sci_viewer.cljs +++ b/src/nextjournal/clerk/sci_viewer.cljs @@ -10,6 +10,7 @@ [goog.string :as gstring] [nextjournal.clerk.viewer :as viewer :refer [code md plotly tex table vl row col with-viewer with-viewers]] [nextjournal.clerk.parser :as clerk.parser] + [nextjournal.markdown :as markdown] [nextjournal.markdown.transform :as md.transform] [nextjournal.ui.components.icon :as icon] [nextjournal.ui.components.localstorage :as ls] @@ -752,6 +753,7 @@ 'col col 'html html-render 'md md + 'md->hiccup markdown/->hiccup 'plotly plotly 'row row 'table table diff --git a/src/nextjournal/clerk/viewer.cljc b/src/nextjournal/clerk/viewer.cljc index c4e9f3c0b..aa2b157a3 100644 --- a/src/nextjournal/clerk/viewer.cljc +++ b/src/nextjournal/clerk/viewer.cljc @@ -14,7 +14,10 @@ [applied-science.js-interop :as j]]) [nextjournal.markdown :as md] [nextjournal.markdown.transform :as md.transform]) - #?(:clj (:import (com.pngencoder PngEncoder) + #?(:bb (:import (java.nio.file Files StandardOpenOption) + (java.util Base64) + (java.lang Throwable)) + :clj (:import (com.pngencoder PngEncoder) (clojure.lang IDeref) (java.lang Throwable) (java.awt.image BufferedImage) @@ -605,7 +608,8 @@ {:pred (fn [e] (instance? #?(:clj Throwable :cljs js/Error) e)) :name :error :render-fn 'v/throwable-viewer :transform-fn (comp mark-presented (update-val (comp demunge-ex-data datafy/datafy)))}) -(def buffered-image-viewer #?(:clj {:pred #(instance? BufferedImage %) +(def buffered-image-viewer #?(:bb {} + :clj {:pred #(instance? BufferedImage %) :transform-fn (fn [{image :nextjournal/value}] (let [w (.getWidth image) h (.getHeight image) @@ -620,15 +624,17 @@ :render-fn '(fn [blob] (v/html [:figure.flex.flex-col.items-center.not-prose [:img {:src (v/url-for blob)}]]))})) (def ideref-viewer - {:pred #(instance? IDeref %) - :transform-fn (update-val (fn [ideref] - (with-viewer :tagged-value - {:tag "object" - :value (vector (symbol (pr-str (type ideref))) - #?(:clj (with-viewer :number-hex (System/identityHashCode ideref))) - (if-let [deref-as-map (resolve 'clojure.core/deref-as-map)] - (deref-as-map ideref) - ideref))})))}) + #?(:bb {} + :clj + {:pred #(instance? IDeref %) + :transform-fn (update-val (fn [ideref] + (with-viewer :tagged-value + {:tag "object" + :value (vector (symbol (pr-str (type ideref))) + #?(:clj (with-viewer :number-hex (System/identityHashCode ideref))) + (if-let [deref-as-map (resolve 'clojure.core/deref-as-map)] + (deref-as-map ideref) + ideref))})))})) (def regex-viewer {:pred #?(:clj (partial instance? java.util.regex.Pattern) :cljs regexp?) diff --git a/src/nextjournal/clerk/webserver.clj b/src/nextjournal/clerk/webserver.clj index 17e384578..b4637fa91 100644 --- a/src/nextjournal/clerk/webserver.clj +++ b/src/nextjournal/clerk/webserver.clj @@ -3,14 +3,14 @@ [clojure.edn :as edn] [clojure.pprint :as pprint] [clojure.string :as str] - [lambdaisland.uri :as uri] [nextjournal.clerk.view :as view] [nextjournal.clerk.viewer :as v] [nextjournal.markdown :as md] [org.httpkit.server :as httpkit])) (def help-doc - {:blocks [{:type :markdown :doc (md/parse "Use `nextjournal.clerk/show!` to make your notebook appear…")}]}) + {:ns *ns* + :blocks [{:type :markdown :doc (md/parse "Use `nextjournal.clerk/show!` to make your notebook appear…")}]}) (defonce !clients (atom #{})) (defonce !doc (atom help-doc)) diff --git a/src/nextjournal/markdown.bb b/src/nextjournal/markdown.bb new file mode 100644 index 000000000..66111f039 --- /dev/null +++ b/src/nextjournal/markdown.bb @@ -0,0 +1,44 @@ +(ns nextjournal.markdown + "Babashka runtime stubs" + (:require [babashka.fs :as fs] + [babashka.process :as p] + [clojure.data.json :as json] + [clojure.java.io :as io] + [clojure.string :as str] + [nextjournal.markdown.parser :as md.parser])) + +(defn assert-quickjs! [] (assert (= 0 (:exit @(p/process '[which qjs]))) "QuickJS needs to be installed (brew install quickjs)")) +(def !md-mod-temp-dir (atom nil)) +(defn md-mod-temp-dir [] + (or @!md-mod-temp-dir + (let [tempdir (fs/create-temp-dir)] + (assert-quickjs!) + (spit (fs/file tempdir "markdown.mjs") (slurp (io/resource "js/markdown.mjs"))) + (reset! !md-mod-temp-dir (str tempdir))))) + +(defn escape [t] (-> t (str/replace "\\" "\\\\\\") (str/replace "`" "\\`") (str/replace "'" "\\'"))) +(defn tokenize [text] + (some-> (p/shell {:out :string :err :string :dir (md-mod-temp-dir)} + (str "qjs -e 'import(\"./markdown.mjs\").then((mod) => {print(mod.default.tokenizeJSON(`" (escape text) "`))})" + ".catch((e) => {import(\"std\").then((std) => { std.err.puts(\"cant find markdown module\"); std.exit(1)})})'")) + deref :out not-empty + (json/read-str {:key-fn keyword}))) + +(defn parse [md] {:type :doc :content []} (some-> md tokenize md.parser/parse)) + +(comment + (assert-quickjs!) + (md-mod-temp-dir) + (tokenize "# Hello") + (parse "# Hello +* `this` +* _is_ Some $\\mathfrak{M}$ formula +* crazy as [hello](https://hell.is) + +--- +```clojure +and this is code +``` +") + (try (parse (slurp "notebooks/markdown.md")) + (catch Exception e (:err (ex-data e)))))