From 6b3c1f6948705597ab48fe90d5ac93018db4a547 Mon Sep 17 00:00:00 2001 From: Michiel Borkent Date: Sun, 30 Aug 2026 21:09:41 +0200 Subject: [PATCH] Arrays as vectors, both directions --- README.md | 25 +++++--- src/babashka/postgres.clj | 103 +++++++++++++++++++++++++++++++- test/babashka/postgres_test.clj | 33 +++++++++- 3 files changed, 148 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index c904f0a..602113c 100644 --- a/README.md +++ b/README.md @@ -74,7 +74,8 @@ Each result value has the Clojure type for its PostgreSQL column type: | `timestamp` | `java.time.LocalDateTime` | | `timestamp with time zone` | `java.time.OffsetDateTime` | | `json`, `jsonb` | string, or the `:read-json` result | -| arrays, other types | string | +| arrays of these types | vector of the element values | +| other types | string | | `NULL` | `nil` | `connect` sets `DateStyle` to `ISO` when the server default is different. @@ -86,10 +87,13 @@ strings. A date or time value that `java.time` cannot parse, such as already exists, skipping` go to stderr. Parameters accept longs, doubles, BigDecimals, booleans, strings, byte -arrays, UUIDs, `java.time` values, `nil`, and with `:write-json`, maps and -vectors. A string parameter has no declared type, so the server infers the -type from its use. Add a cast such as `$1::uuid` where the server cannot -infer one. +arrays, UUIDs, `java.time` values, `nil`, vectors as arrays, and with +`:write-json`, maps as JSON. A string parameter has no declared type, so +the server infers the type from its use. Add a cast such as `$1::uuid` +where the server cannot infer one. + +A vector parameter is an array. Vectors nest for arrays of more than one +dimension. Wrap a vector in `pg/json` or `pg/jsonb` to send it as JSON. ### JSON @@ -98,7 +102,7 @@ The library has no JSON dependency. Pass a JSON library through two options: - `:read-json` is a function from a JSON string to a value. `query` applies it to `json` and `jsonb` columns. - `:write-json` is a function from a value to a JSON string. It encodes map - and vector parameters. + parameters and the values in `pg/json` and `pg/jsonb`. Use cheshire in babashka, or any library on the JVM: @@ -113,14 +117,17 @@ Use cheshire in babashka, or any library on the JVM: (pg/execute! db ["insert into docs values ($1, $2)" 1 {:a 1 :tags ["x" "y"]}]) (pg/query db "select * from docs")) ;;=> [{:id 1, :doc {:a 1, :tags ["x" "y"]}}] + +A map parameter encodes as JSON. A vector parameter is an array, not JSON; +see below. ``` The options also go on `connect` and as a third argument to `query` and `execute!`. A call option overrides the connection option. -A map or vector parameter has no declared type, so the server infers `json` -or `jsonb` from the column. Wrap a value in `pg/json` or `pg/jsonb` to -declare the type, such as for a JSON operator: +A map parameter has no declared type, so the server infers `json` or +`jsonb` from the column. Wrap a value in `pg/json` or `pg/jsonb` to declare +the type, for a vector or for a JSON operator: ```clojure (pg/query db ["select $1->>'a' a" (pg/jsonb {:a 1})] json-opts) diff --git a/src/babashka/postgres.clj b/src/babashka/postgres.clj index 456f29b..445f795 100644 --- a/src/babashka/postgres.clj +++ b/src/babashka/postgres.clj @@ -275,6 +275,66 @@ (.appendOffset "+HH:mm:ss" "Z") .toFormatter)) +(def ^:private array-elem-oid + "The element type of each built-in array type." + {1000 16, 1001 17, 1002 18, 1003 19, 1005 21, 1007 23, 1009 25, 1014 1042 + 1015 1043, 1016 20, 1021 700, 1022 701, 1028 26, 1231 1700, 2951 2950 + 1182 1082, 1183 1083, 1270 1266, 1115 1114, 1185 1184, 199 114, 3807 3802}) + +(defn- hex-bytes + "Decodes a bytea hex literal, backslash x and hex digits, to a byte array." + ^bytes [^String s] + (let [n (quot (- (count s) 2) 2) + out (byte-array n)] + (dotimes [i n] + (aset out i (unchecked-byte (Integer/parseInt (subs s (+ 2 (* 2 i)) (+ 4 (* 2 i))) 16)))) + out)) + +(defn- parse-array + "Parses the text form of an array, such as {1,NULL,\"a b\"}, into nested + vectors. decode maps an element string to its value." + [^String s decode] + (let [n (count s) + start (long (if (= \[ (.charAt s 0)) (inc (.indexOf s "=")) 0)) + parse (fn parse [^long i] + ;; i is at the opening brace; returns [value next-index] + (loop [i (inc i) acc (transient [])] + (case (.charAt s i) + \} [(persistent! acc) (inc i)] + \, (recur (inc i) acc) + \{ (let [[v j] (parse i)] (recur (long j) (conj! acc v))) + \" (let [sb (StringBuilder.) + j (loop [j (inc i)] + (case (.charAt s j) + \" (inc j) + \\ (do (.append sb (.charAt s (inc j))) (recur (+ j 2))) + (do (.append sb (.charAt s j)) (recur (inc j)))))] + (recur (long j) (conj! acc (decode (str sb))))) + (let [j (loop [j i] + (if (or (= j n) (#{\, \}} (.charAt s j))) j (recur (inc j)))) + tok (subs s i j)] + (recur (long j) (conj! acc (if (= "NULL" tok) nil (decode tok))))))))] + (first (parse start)))) + +(defn- text-decoder + "Returns a fn from the text form of a value of type oid to its value. + Used for array elements; columns fetch through column-decoder." + [oid read-json iso?] + (case (long oid) + 16 (fn [s] (= "t" s)) + (20 21 23 26) (fn [^String s] (Long/parseLong s)) + (700 701) (fn [^String s] (Double/parseDouble s)) + 1700 (fn [^String s] (if (= "NaN" s) ##NaN (BigDecimal. s))) + 17 hex-bytes + 2950 (fn [^String s] (java.util.UUID/fromString s)) + (114 3802) (if read-json read-json identity) + 1082 (if iso? (fn [s] (parse-or-string #(LocalDate/parse %) s)) identity) + 1083 (if iso? (fn [s] (parse-or-string #(LocalTime/parse %) s)) identity) + 1266 (if iso? (fn [s] (parse-or-string #(OffsetTime/parse % timetz-format) s)) identity) + 1114 (if iso? (fn [s] (parse-or-string #(LocalDateTime/parse % timestamp-format) s)) identity) + 1184 (if iso? (fn [s] (parse-or-string #(OffsetDateTime/parse % timestamptz-format) s)) identity) + identity)) + (defn- column-decoder "Returns a fn of [res row col] that decodes a value of the column type oid. Chosen once per column, so a row only calls it." @@ -296,7 +356,10 @@ 1266 (if iso? (fn [res row col] (parse-or-string #(OffsetTime/parse % timetz-format) (c-getvalue res row col))) c-getvalue) 1114 (if iso? (fn [res row col] (parse-or-string #(LocalDateTime/parse % timestamp-format) (c-getvalue res row col))) c-getvalue) 1184 (if iso? (fn [res row col] (parse-or-string #(OffsetDateTime/parse % timestamptz-format) (c-getvalue res row col))) c-getvalue) - c-getvalue)) + (if-let [elem (array-elem-oid (long oid))] + (let [decode (text-decoder elem read-json iso?)] + (fn [res row col] (parse-array (c-getvalue res row col) decode))) + c-getvalue))) (defn- rows [conn res {:keys [read-json]}] (let [iso? (iso-datestyle? conn) @@ -345,6 +408,41 @@ :else (throw (ex-info "postgres: set the :write-json option to bind JSON values" {:value v})))) +(defn- hex-literal ^String [^bytes bs] + (let [sb (StringBuilder. "\\x")] + (dotimes [i (alength bs)] + (.append sb (format "%02x" (bit-and 0xff (aget bs i))))) + (str sb))) + +(defn- array-literal + "Encodes vector v as the text form of an array. Elements follow the + parameter types; strings are quoted." + ^String [write-json v] + (let [sb (StringBuilder.) + quoted (fn [^String s] + (.append sb \") + (.append sb (-> s (str/replace "\\" "\\\\") (str/replace "\"" "\\\""))) + (.append sb \")) + emit (fn emit [v] + (.append sb \{) + (doseq [[i x] (map-indexed vector v)] + (when (pos? i) (.append sb \,)) + (cond + (nil? x) (.append sb "NULL") + (vector? x) (emit x) + (boolean? x) (.append sb (if x "t" "f")) + (number? x) (.append sb (str x)) + (string? x) (quoted x) + (bytes? x) (quoted (hex-literal x)) + (uuid? x) (.append sb (str x)) + (instance? java.time.temporal.Temporal x) (quoted (str x)) + (instance? JsonParam x) (quoted (json-text write-json (:value x))) + :else (throw (ex-info (str "postgres: cannot bind " (type x) " in an array") + {:value x})))) + (.append sb \}))] + (emit v) + (str sb))) + (defn- param-spec "Returns [oid text-or-bytes binary?] for a parameter value." [{:keys [write-json]} v] @@ -359,7 +457,8 @@ (uuid? v) [0 (str v) false] (instance? java.time.temporal.Temporal v) [0 (str v) false] (instance? JsonParam v) [(:oid v) (json-text write-json (:value v)) false] - (or (map? v) (vector? v)) [0 (json-text write-json v) false] + (vector? v) [0 (array-literal write-json v) false] + (map? v) [0 (json-text write-json v) false] :else (throw (ex-info (str "postgres: cannot bind " (type v)) {:value v})))) (defn- exec-params [conn sql params opts] diff --git a/test/babashka/postgres_test.clj b/test/babashka/postgres_test.clj index ded62d0..8e72b28 100644 --- a/test/babashka/postgres_test.clj +++ b/test/babashka/postgres_test.clj @@ -127,6 +127,35 @@ (pg/execute! db "copy copy_in from stdin"))) (is (= [{:one 1}] (pg/query db "select 1 one")))))) +(deftest arrays-test + (pg/with-conn [db conninfo] + (testing "arrays come back as vectors of decoded elements" + (is (= [{:a [1 2 3]}] (pg/query db "select '{1,2,3}'::int8[] a"))) + (is (= [{:a [1.5 2.5]}] (pg/query db "select '{1.5,2.5}'::float8[] a"))) + (is (= [{:a ["a b" nil "c,}\"d\\e"]}] + (pg/query db "select array['a b', null, 'c,}\"d\\e'] a"))) + (is (= [{:a [true false nil]}] (pg/query db "select '{t,f,NULL}'::bool[] a"))) + (is (= [{:a []}] (pg/query db "select '{}'::text[] a"))) + (is (= [{:a [[1 2] [3 4]]}] (pg/query db "select '{{1,2},{3,4}}'::int[] a")))) + (testing "typed elements decode inside arrays" + (let [u (random-uuid)] + (is (= [{:a [u]}] (pg/query db ["select array[$1::uuid] a" u])))) + (is (= [{:a [(LocalDate/parse "2026-08-30")]}] + (pg/query db "select '{2026-08-30}'::date[] a"))) + (is (= [{:a ["{1}" "{2}"]}] + (pg/query db ["select $1::text[] a" ["{1}" "{2}"]])))) + (testing "a vector parameter is an array" + (is (= [{:a [1 2 3]}] (pg/query db ["select $1::int8[] a" [1 2 3]]))) + (is (= [{:a ["x" nil "y\"z"]}] (pg/query db ["select $1::text[] a" ["x" nil "y\"z"]]))) + (is (= [{:a [[1 2] [3 4]]}] (pg/query db ["select $1::int[] a" [[1 2] [3 4]]]))) + (is (= [1 2 3] (vec (first (:a (first (pg/query db ["select $1::bytea[] a" [(byte-array [1 2 3])]])))))))) + (testing "an unknown array type stays a string" + (pg/execute! db "drop type if exists mood_bbtest") + (pg/execute! db "create type mood_bbtest as enum ('ok')") + (try + (is (= [{:a "{ok}"}] (pg/query db "select '{ok}'::mood_bbtest[] a"))) + (finally (pg/execute! db "drop type mood_bbtest")))))) + (def json-opts {:read-json #(json/parse-string % true) :write-json json/generate-string}) @@ -136,8 +165,8 @@ (let [t (table-name "j") doc {:a 1 :nested {:b [1 2 3]} :s "x"}] (pg/execute! db (str "create temp table " t " (id int, j json, jb jsonb)")) - (testing "maps and vectors bind as json text and json columns decode" - (pg/execute! db [(str "insert into " t " values ($1, $2, $3)") 1 doc [1 "two" nil]]) + (testing "maps bind as json text, a vector needs the json wrapper" + (pg/execute! db [(str "insert into " t " values ($1, $2, $3)") 1 doc (pg/jsonb [1 "two" nil])]) (is (= [{:id 1 :j doc :jb [1 "two" nil]}] (pg/query db (str "select * from " t))))) (testing "json and jsonb mark a parameter with its type"