xtdb2-starter-changes.diff

diff --git a/xtdb2-starter/deps.edn b/xtdb2-starter/deps.edn
index 84fd90a..2649ab1 100644
--- a/xtdb2-starter/deps.edn
+++ b/xtdb2-starter/deps.edn
@@ -1,6 +1,14 @@
 {:paths ["src" "resources" "target/resources"]
- :deps {com.biffweb/biff           {:local/root ".."}
-        cheshire/cheshire          {:mvn/version "6.1.0"}
+ :deps {com.biffweb/biff    {:local/root ".."
+                             :exclusions [com.xtdb/xtdb-core
+                                          com.xtdb/xtdb-jdbc
+                                          com.xtdb/xtdb-rocksdb
+                                          org.postgresql/postgresql]}
+        com.xtdb/xtdb-api   {:mvn/version "2.x-SNAPSHOT"}
+        com.xtdb/xtdb-aws   {:mvn/version "2.x-SNAPSHOT"}
+        com.xtdb/xtdb-core  {:mvn/version "2.x-SNAPSHOT"}
+        com.zaxxer/HikariCP {:mvn/version "7.0.2"}
+        cheshire/cheshire   {:mvn/version "6.1.0"}
 
         ;; Notes on logging: https://gist.github.com/jacobobryant/76b7a08a07d5ef2cc076b048d078f1f3
         org.slf4j/slf4j-simple     {:mvn/version "2.0.0-alpha5"}
@@ -10,11 +18,19 @@
  :aliases
  {:dev {:extra-deps {com.biffweb/tasks {:local/root "../libs/tasks"}}
         :extra-paths ["dev" "test"]
-        :jvm-opts ["-XX:-OmitStackTraceInFastThrow"
+        :jvm-opts ["--add-opens=java.base/java.nio=ALL-UNNAMED"
+                   "-Dio.netty.tryReflectionSetAccessible=true"
+                   "-XX:-OmitStackTraceInFastThrow"
                    "-XX:+CrashOnOutOfMemoryError"
                    "-Dbiff.env.BIFF_PROFILE=dev"]
         :main-opts ["-m" "com.biffweb.task-runner" "tasks/tasks"]}
-  :prod {:jvm-opts ["-XX:-OmitStackTraceInFastThrow"
+  :prod {:jvm-opts ["--add-opens=java.base/java.nio=ALL-UNNAMED"
+                   "-Dio.netty.tryReflectionSetAccessible=true"
+                    "-XX:-OmitStackTraceInFastThrow"
                     "-XX:+CrashOnOutOfMemoryError"
                     "-Dbiff.env.BIFF_PROFILE=prod"]
-         :main-opts ["-m" "com.example"]}}}
+         :main-opts ["-m" "com.example"]}}
+ :mvn/repos
+ {"central"            {:url "https://repo1.maven.org/maven2/"}
+  "clojars"            {:url "https://clojars.org/repo"}
+  "sonatype-snapshots" {:url "https://central.sonatype.com/repository/maven-snapshots/"}}}
diff --git a/xtdb2-starter/dev/repl.clj b/xtdb2-starter/dev/repl.clj
index b3782a6..c7a5459 100644
--- a/xtdb2-starter/dev/repl.clj
+++ b/xtdb2-starter/dev/repl.clj
@@ -1,8 +1,11 @@
 (ns repl
-  (:require [com.example :as main]
-            [com.biffweb :as biff :refer [q]]
-            [clojure.edn :as edn]
-            [clojure.java.io :as io]))
+  (:require
+   [com.biffweb :as biff]
+   [com.biffweb.experimental :as biffx]
+   [com.example :as main]
+   [xtdb.api :as xt])
+  (:import
+   [java.time Instant]))
 
 ;; REPL-driven development
 ;; ----------------------------------------------------------------------------------------
@@ -28,10 +31,16 @@
   (biff/merge-context @main/system))
 
 (defn add-fixtures []
-  (biff/submit-tx (get-context)
-    (-> (io/resource "fixtures.edn")
-        slurp
-        edn/read-string)))
+  (let [user-id (random-uuid)]
+    (biffx/submit-tx (get-context)
+      [[:put-docs :user {:xt/id user-id
+                         :email "a@example.com"
+                         :foo "Some Value"
+                         :joined-at (Instant/now)}]
+       [:put-docs :msg {:xt/id (random-uuid)
+                        :user user-id
+                        :text "hello there"
+                        :sent-at (Instant/now)}]])))
 
 (defn check-config []
   (let [prod-config (biff/use-aero-config {:biff.config/profile "prod"})
@@ -58,26 +67,22 @@
   ;; main/components, :tasks, :queues, config.env, or deps.edn.
   (main/refresh)
 
-  ;; Call this in dev if you'd like to add some seed data to your database. If
-  ;; you edit the seed data (in resources/fixtures.edn), you can reset the
-  ;; database by running `rm -r storage/xtdb` (DON'T run that in prod),
+  ;; Call this in dev if you'd like to add some seed data to your database. If you edit the seed
+  ;; data, you can reset the database by running `rm -r storage/xtdb2` (DON'T run that in prod),
   ;; restarting your app, and calling add-fixtures again.
   (add-fixtures)
 
   ;; Query the database
-  (let [{:keys [biff/db] :as ctx} (get-context)]
-    (q db
-       '{:find (pull user [*])
-         :where [[user :user/email]]}))
+  (let [{:keys [biff/node]} (get-context)]
+    (xt/q node "select * from user"))
 
   ;; Update an existing user's email address
-  (let [{:keys [biff/db] :as ctx} (get-context)
-        user-id (biff/lookup-id db :user/email "hello@example.com")]
-    (biff/submit-tx ctx
-      [{:db/doc-type :user
-        :xt/id user-id
-        :db/op :update
-        :user/email "new.address@example.com"}]))
+  (let [{:keys [biff/node] :as ctx} (get-context)
+        [{user-id :xt/id}] (xt/q node ["select _id from user where email = ?"
+                                       "hello@example.com"])]
+    (biffx/submit-tx ctx
+      [[:patch-docs :user {:xt/id user-id
+                           :email "new.address@example.com"}]]))
 
   (sort (keys (get-context)))
 
diff --git a/xtdb2-starter/resources/config.edn b/xtdb2-starter/resources/config.edn
index f623080..127ded1 100644
--- a/xtdb2-starter/resources/config.edn
+++ b/xtdb2-starter/resources/config.edn
@@ -7,11 +7,12 @@
                                :default "localhost"}]
  :biff/port     #long #or [#biff/env "PORT" 8080]
 
- :biff.xtdb/dir          "storage/xtdb"
- :biff.xtdb/topology     #keyword #or [#profile {:prod #biff/env "PROD_XTDB_TOPOLOGY"
-                                                 :default #biff/env "XTDB_TOPOLOGY"}
-                                       "standalone"]
- :biff.xtdb.jdbc/jdbcUrl #biff/secret "XTDB_JDBC_URL"
+ :biff.xtdb2/storage            #keyword #or [#profile {:prod #biff/env PROD_XTDB_STORAGE}
+                                              "local"]
+ :biff.xtdb2.storage/bucket     #biff/env XTDB_STORAGE_BUCKET
+ :biff.xtdb2.storage/endpoint   #biff/env XTDB_STORAGE_ENDPOINT
+ :biff.xtdb2.storage/access-key #biff/env XTDB_STORAGE_ACCESS_KEY
+ :biff.xtdb2.storage/secret-key #biff/secret XTDB_STORAGE_SECRET_KEY
 
  :biff.beholder/enabled         #profile {:dev true :default false}
  :biff.beholder/paths           ["src" "resources" "test"]
diff --git a/xtdb2-starter/resources/config.template.env b/xtdb2-starter/resources/config.template.env
index 24c3def..6de5f2c 100644
--- a/xtdb2-starter/resources/config.template.env
+++ b/xtdb2-starter/resources/config.template.env
@@ -17,10 +17,12 @@ MAILERSEND_REPLY_TO=
 RECAPTCHA_SITE_KEY=
 RECAPTCHA_SECRET_KEY=
 
-XTDB_TOPOLOGY=standalone
-# Uncomment these to use Postgres for storage in production:
-#PROD_XTDB_TOPOLOGY=jdbc
-#XTDB_JDBC_URL=jdbc:postgresql://host:port/dbname?user=alice&password=abc123&sslmode=require
+# Edit and uncomment these to use S3 for production storage (recommended):
+#PROD_XTDB_STORAGE=remote
+#XTDB_STORAGE_BUCKET=xtdb-example
+#XTDB_STORAGE_ENDPOINT=https://nyc3.digitaloceanspaces.com
+#XTDB_STORAGE_ACCESS_KEY=abc
+#XTDB_STORAGE_SECRET_KEY=123
 
 # What port should the nrepl server be started on (in dev and prod)?
 NREPL_PORT=7888
diff --git a/xtdb2-starter/resources/fixtures.edn b/xtdb2-starter/resources/fixtures.edn
deleted file mode 100644
index a8d2785..0000000
--- a/xtdb2-starter/resources/fixtures.edn
+++ /dev/null
@@ -1,10 +0,0 @@
-;; Biff transaction. See https://biffweb.com/docs/reference/transactions/
-[{:db/doc-type :user
-  :xt/id :db.id/user-a
-  :user/email "a@example.com"
-  :user/foo "Some Value"
-  :user/joined-at :db/now}
- {:db/doc-type :msg
-  :msg/user :db.id/user-a
-  :msg/text "hello there"
-  :msg/sent-at :db/now}]
diff --git a/xtdb2-starter/src/com/example.clj b/xtdb2-starter/src/com/example.clj
index 55ffef5..a7173eb 100644
--- a/xtdb2-starter/src/com/example.clj
+++ b/xtdb2-starter/src/com/example.clj
@@ -1,5 +1,7 @@
 (ns com.example
   (:require [com.biffweb :as biff]
+            [com.biffweb.experimental :as biffx]
+            [com.biffweb.experimental.auth :as biff-auth]
             [com.example.email :as email]
             [com.example.app :as app]
             [com.example.home :as home]
@@ -17,7 +19,7 @@
 
 (def modules
   [app/module
-   (biff/authentication-module {})
+   (biff-auth/module {})
    home/module
    schema/module
    worker/module])
@@ -32,7 +34,7 @@
 
 (def static-pages (apply biff/safe-merge (map :static modules)))
 
-(defn generate-assets! [ctx]
+(defn generate-assets! [_ctx]
   (biff/export-rum static-pages "target/resources/public")
   (biff/delete-old-files {:dir "target/resources/public"
                           :exts [".html"]}))
@@ -55,16 +57,16 @@
    :biff/malli-opts #'malli-opts
    :biff.beholder/on-save #'on-save
    :biff.middleware/on-error #'ui/on-error
-   :biff.xtdb/tx-fns biff/tx-fns
+   :biff.xtdb.listener/tables ["user" "msg"]
    :com.example/chat-clients (atom #{})})
 
 (defonce system (atom {}))
 
 (def components
   [biff/use-aero-config
-   biff/use-xtdb
+   biffx/use-xtdb2
    biff/use-queues
-   biff/use-xtdb-tx-listener
+   biffx/use-xtdb2-listener
    biff/use-htmx-refresh
    biff/use-jetty
    biff/use-chime
diff --git a/xtdb2-starter/src/com/example/app.clj b/xtdb2-starter/src/com/example/app.clj
index ad744eb..113a402 100644
--- a/xtdb2-starter/src/com/example/app.clj
+++ b/xtdb2-starter/src/com/example/app.clj
@@ -1,19 +1,20 @@
 (ns com.example.app
-  (:require [com.biffweb :as biff :refer [q]]
-            [com.example.middleware :as mid]
-            [com.example.ui :as ui]
-            [com.example.settings :as settings]
-            [rum.core :as rum]
-            [xtdb.api :as xt]
-            [ring.websocket :as ws]
-            [cheshire.core :as cheshire]))
+  (:require
+   [cheshire.core :as cheshire]
+   [com.biffweb :as biff]
+   [com.biffweb.experimental :as biffx]
+   [com.example.middleware :as mid]
+   [com.example.settings :as settings]
+   [com.example.ui :as ui]
+   [ring.websocket :as ws]
+   [rum.core :as rum]
+   [xtdb.api :as xt])
+  (:import
+   [java.time Instant]))
 
 (defn set-foo [{:keys [session params] :as ctx}]
-  (biff/submit-tx ctx
-    [{:db/op :update
-      :db/doc-type :user
-      :xt/id (:uid session)
-      :user/foo (:foo params)}])
+  (biffx/submit-tx ctx
+    [[:patch-docs :user {:xt/id (:uid session) :foo (:foo params)}]])
   {:status 303
    :headers {"location" "/app"}})
 
@@ -33,50 +34,41 @@
     "This demonstrates updating a value with HTMX."]))
 
 (defn set-bar [{:keys [session params] :as ctx}]
-  (biff/submit-tx ctx
-    [{:db/op :update
-      :db/doc-type :user
-      :xt/id (:uid session)
-      :user/bar (:bar params)}])
+  (time (biffx/submit-tx ctx
+    [[:patch-docs :user {:xt/id (:uid session) :bar (:bar params)}]]))
   (biff/render (bar-form {:value (:bar params)})))
 
-(defn message [{:msg/keys [text sent-at]}]
+(defn message [{:keys [content sent-at]}]
   [:.mt-3 {:_ "init send newMessage to #message-header"}
-   [:.text-gray-600 (biff/format-date sent-at "dd MMM yyyy HH:mm:ss")]
-   [:div text]])
+   [:.text-gray-600 (biff/format-date (java.util.Date/from (.toInstant sent-at)) "dd MMM yyyy HH:mm:ss")]
+   [:div content]])
 
-(defn notify-clients [{:keys [com.example/chat-clients]} tx]
-  (doseq [[op & args] (::xt/tx-ops tx)
-          :when (= op ::xt/put)
-          :let [[doc] args]
-          :when (contains? doc :msg/text)
-          :let [html (rum/render-static-markup
-                      [:div#messages {:hx-swap-oob "afterbegin"}
-                       (message doc)])]
-          ws @chat-clients]
-    (ws/send ws html)))
+(defn notify-clients [{:keys [com.example/chat-clients]} record]
+  (when (= "msg" (:biff.xtdb/table record))
+    (let [html (rum/render-static-markup
+                [:div#messages {:hx-swap-oob "afterbegin"}
+                 (message record)])]
+      (doseq [ws @chat-clients]
+        (ws/send ws html)))))
 
 (defn send-message [{:keys [session] :as ctx} {:keys [text]}]
-  (let [{:keys [text]} (cheshire/parse-string text true)]
-    (biff/submit-tx ctx
-      [{:db/doc-type :msg
-        :msg/user (:uid session)
-        :msg/text text
-        :msg/sent-at :db/now}])))
+  (let [{:keys [content]} (cheshire/parse-string text true)]
+    (biffx/submit-tx ctx
+      [[:put-docs :msg {:xt/id (random-uuid)
+                        :user (:uid session)
+                        :content content
+                        :sent-at (Instant/now)}]])))
 
-(defn chat [{:keys [biff/db]}]
-  (let [messages (q db
-                    '{:find (pull msg [*])
-                      :in [t0]
-                      :where [[msg :msg/sent-at t]
-                              [(<= t0 t)]]}
-                    (biff/add-seconds (java.util.Date.) (* -60 10)))]
+(defn chat [{:keys [biff/conn]}]
+  (let [messages (xt/q conn
+                       ["select content, sent_at from msg where sent_at >= ?"
+                        (.minusSeconds (Instant/now) (* 60 10))])]
     [:div {:hx-ext "ws" :ws-connect "/app/chat"}
      [:form.mb-0 {:ws-send true
                   :_ "on submit set value of #message to ''"}
       [:label.block {:for "message"} "Write a message"]
       [:.h-1]
-      [:textarea.w-full#message {:name "text"}]
+      [:textarea.w-full#message {:name "content"}]
       [:.h-1]
       [:.text-sm.text-gray-600
        "Sign in with an incognito window to have a conversation with yourself."]
@@ -89,10 +81,11 @@
         "No messages yet."
         "Messages sent in the past 10 minutes:")]
      [:div#messages
-      (map message (sort-by :msg/sent-at #(compare %2 %1) messages))]]))
+      (map message (sort-by :sent-at #(compare %2 %1) messages))]]))
 
-(defn app [{:keys [session biff/db] :as ctx}]
-  (let [{:user/keys [email foo bar]} (xt/entity db (:uid session))]
+(defn app [{:keys [biff/conn session] :as ctx}]
+  (let [[{:keys [email foo bar]}] (xt/q conn ["select email, foo, bar from user where _id = ?"
+                                              (:uid session)])]
     (ui/page
      {}
      [:div "Signed in as " email ". "
diff --git a/xtdb2-starter/src/com/example/email.clj b/xtdb2-starter/src/com/example/email.clj
index fecdfa3..22b6082 100644
--- a/xtdb2-starter/src/com/example/email.clj
+++ b/xtdb2-starter/src/com/example/email.clj
@@ -69,7 +69,7 @@
       (log/error (:body result)))
     success))
 
-(defn send-console [ctx form-params]
+(defn send-console [_ctx form-params]
   (println "TO:" (:to form-params))
   (println "SUBJECT:" (:subject form-params))
   (println)
diff --git a/xtdb2-starter/src/com/example/schema.clj b/xtdb2-starter/src/com/example/schema.clj
index e8e979a..975aade 100644
--- a/xtdb2-starter/src/com/example/schema.clj
+++ b/xtdb2-starter/src/com/example/schema.clj
@@ -1,20 +1,22 @@
 (ns com.example.schema)
 
+(def ? {:optional true})
+
 (def schema
-  {:user/id :uuid
+  {::string [:string {:max 1000}]
+
    :user [:map {:closed true}
-          [:xt/id                     :user/id]
-          [:user/email                :string]
-          [:user/joined-at            inst?]
-          [:user/foo {:optional true} :string]
-          [:user/bar {:optional true} :string]]
+          [:xt/id       :uuid]
+          [:email       ::string]
+          [:joined-at   inst?]
+          [:foo       ? ::string]
+          [:bar       ? ::string]]
 
-   :msg/id :uuid
    :msg [:map {:closed true}
-         [:xt/id       :msg/id]
-         [:msg/user    :user/id]
-         [:msg/text    :string]
-         [:msg/sent-at inst?]]})
+         [:xt/id   :uuid]
+         [:user    :uuid]
+         [:content [:string {:max 10000}]]
+         [:sent-at inst?]]})
 
 (def module
   {:schema schema})
diff --git a/xtdb2-starter/src/com/example/worker.clj b/xtdb2-starter/src/com/example/worker.clj
index 8b16d08..a9d89d1 100644
--- a/xtdb2-starter/src/com/example/worker.clj
+++ b/xtdb2-starter/src/com/example/worker.clj
@@ -6,26 +6,24 @@
 (defn every-n-minutes [n]
   (iterate #(biff/add-seconds % (* 60 n)) (java.util.Date.)))
 
-(defn print-usage [{:keys [biff/db]}]
+(defn print-usage [{:keys [biff/conn]}]
   ;; For a real app, you can have this run once per day and send you the output
   ;; in an email.
-  (let [n-users (nth (q db
-                        '{:find (count user)
-                          :where [[user :user/email]]})
-                     0
-                     0)]
+  (let [[{n-users :cnt}] (xt/q conn "select count(*) as cnt from users")]
     (log/info "There are" n-users "users.")))
 
-(defn alert-new-user [{:keys [biff.xtdb/node]} tx]
-  (doseq [_ [nil]
-          :let [db-before (xt/db node {::xt/tx-id (dec (::xt/tx-id tx))})]
-          [op & args] (::xt/tx-ops tx)
-          :when (= op ::xt/put)
-          :let [[doc] args]
-          :when (and (contains? doc :user/email)
-                     (nil? (xt/entity db-before (:xt/id doc))))]
+(defn alert-new-user [{:keys [biff/conn]} record]
+  (when (and (= (:biff.xtdb/table record) "user")
+             (-> (xt/q conn
+                       ["select count(*) as cnt from user where _id = ?" (:xt/id record)]
+                       {:snapshot-time (.. (:xt/system-from record)
+                                           (toInstant)
+                                           (minusNanos 1))})
+                 first
+                 :cnt
+                 (= 0)))
     ;; You could send this as an email instead of printing.
-    (log/info "WOAH there's a new user")))
+    (log/info "WOAH there's a new user: " (pr-str record))))
 
 (defn echo-consumer [{:keys [biff/job] :as ctx}]
   (prn :echo job)
diff --git a/xtdb2-starter/test/com/example_test.clj b/xtdb2-starter/test/com/example_test.clj
index 4d10c43..548455a 100644
--- a/xtdb2-starter/test/com/example_test.clj
+++ b/xtdb2-starter/test/com/example_test.clj
@@ -12,34 +12,36 @@
 (deftest example-test
   (is (= 4 (+ 2 2))))
 
-(defn get-context [node]
-  {:biff.xtdb/node  node
-   :biff/db         (xt/db node)
-   :biff/malli-opts #'main/malli-opts})
+;; TODO
 
-(deftest send-message-test
-  (with-open [node (test-xtdb-node [])]
-    (let [message (mg/generate :string)
-          user    (mg/generate :user main/malli-opts)
-          ctx     (assoc (get-context node) :session {:uid (:xt/id user)})
-          _       (app/send-message ctx {:text (cheshire/generate-string {:text message})})
-          db      (xt/db node) ; get a fresh db value so it contains any transactions
-                               ; that send-message submitted.
-          doc     (biff/lookup db :msg/text message)]
-      (is (some? doc))
-      (is (= (:msg/user doc) (:xt/id user))))))
-
-(deftest chat-test
-  (let [n-messages (+ 3 (rand-int 10))
-        now        (java.util.Date.)
-        messages   (for [doc (mg/sample :msg (assoc main/malli-opts :size n-messages))]
-                     (assoc doc :msg/sent-at now))]
-    (with-open [node (test-xtdb-node messages)]
-      (let [response (app/chat {:biff/db (xt/db node)})
-            html     (rum/render-html response)]
-        (is (str/includes? html "Messages sent in the past 10 minutes:"))
-        (is (not (str/includes? html "No messages yet.")))
-        ;; If you add Jsoup to your dependencies, you can use DOM selectors instead of just regexes:
-        ;(is (= n-messages (count (.select (Jsoup/parse html) "#messages > *"))))
-        (is (= n-messages (count (re-seq #"init send newMessage to #message-header" html))))
-        (is (every? #(str/includes? html (:msg/text %)) messages))))))
+;; (defn get-context [node]
+;;   {:biff.xtdb/node  node
+;;    :biff/db         (xt/db node)
+;;    :biff/malli-opts #'main/malli-opts})
+;; 
+;; (deftest send-message-test
+;;   (with-open [node (test-xtdb-node [])]
+;;     (let [message (mg/generate :string)
+;;           user    (mg/generate :user main/malli-opts)
+;;           ctx     (assoc (get-context node) :session {:uid (:xt/id user)})
+;;           _       (app/send-message ctx {:text (cheshire/generate-string {:text message})})
+;;           db      (xt/db node) ; get a fresh db value so it contains any transactions
+;;                                ; that send-message submitted.
+;;           doc     (biff/lookup db :msg/text message)]
+;;       (is (some? doc))
+;;       (is (= (:msg/user doc) (:xt/id user))))))
+;; 
+;; (deftest chat-test
+;;   (let [n-messages (+ 3 (rand-int 10))
+;;         now        (java.util.Date.)
+;;         messages   (for [doc (mg/sample :msg (assoc main/malli-opts :size n-messages))]
+;;                      (assoc doc :msg/sent-at now))]
+;;     (with-open [node (test-xtdb-node messages)]
+;;       (let [response (app/chat {:biff/db (xt/db node)})
+;;             html     (rum/render-html response)]
+;;         (is (str/includes? html "Messages sent in the past 10 minutes:"))
+;;         (is (not (str/includes? html "No messages yet.")))
+;;         ;; If you add Jsoup to your dependencies, you can use DOM selectors instead of just regexes:
+;;         ;(is (= n-messages (count (.select (Jsoup/parse html) "#messages > *"))))
+;;         (is (= n-messages (count (re-seq #"init send newMessage to #message-header" html))))
+;;         (is (every? #(str/includes? html (:msg/text %)) messages))))))