how to write reusable canonical queries for datomic - clojure

I'm having trouble writing general datomic queries that I consider reusable.
For instance, following up from this post is there a canonical way to grab all idents from a particular datomic partition?, I have the following schema installed
{[63 :account/password]
[64 :account/firstName]
[65 :account/lastName]
[62 :account/username]
[69 :email/prority]
[68 :email/address]}
I want to have a function that shows only the attributes having a given namespace.
This function shows all the attributes in the ":account" namespace
(d/q '[:find ?e ?ident :where
[?e :db/ident ?ident]
[_ :db.install/attribute ?e]
[(.toString ?ident) ?val]
[(.startsWith ?val ":account")]] (d/db *conn*))
;; => [62 :account/username] [63 :account/password]
;; [64 :account/firstName] [65 :account/lastName]
however, when I want to write a function that can take an input, I have to put quotes everywhere in order to make it work.
(defn get-ns-attrs [?ns db]
(d/q [':find '?e '?ident ':where
['?e ':db/ident '?ident]
['_ ':db.install/attribute '?e]
[(list '.toString '?ident) '?val]
[(list '.startsWith '?val (str ":" ?ns))]] db))
(get-ns-attrs "account" (d/db *conn*))
;; => [62 :account/username] [63 :account/password]
;; [64 :account/firstName] [65 :account/lastName]
(get-ns-attrs "email" (d/db *conn*))
;; => [69 :email/prority] [68 :email/address]
Is there a better way to do this?
------ update -------
The full code for this for people to try is here:
(ns schema.start
(:require [datomic.api :as d])
(:use [clojure.pprint :only [pprint]]))
(def *uri* "datomic:mem://login-profile")
(d/create-database *uri*)
(def *conn* (d/connect *uri*))
(defn boolean? [x]
(instance? java.lang.Boolean x))
(defn db-pair [attr kns val f]
(list (keyword (str "db/" (name attr)))
(f val kns)))
(defn db-enum [val kns]
(keyword (str "db." (name kns) "/" (name val))))
(def DB-KNS
{:ident {:required true
:check keyword?}
:type {:required true
:check #{:keyword :string :boolean :long :bigint :float
:double :bigdec :ref :instant :uuid :uri :bytes}
:attr :valueType
:fn db-enum}
:cardinality {:required true
:check #{:one :many}
:fn db-enum}
:unique {:check #{:value :identity}
:fn db-enum}
:doc {:check string?}
:index {:check boolean?}
:fulltext {:check boolean?}
:component? {:check keyword?}
:no-history {:check boolean?}})
(defn process-kns [m kns params res]
(let [val (m kns)]
(cond (nil? val)
(if (:required params)
(throw (Exception. (str "key " kns " is a required key")))
res)
:else
(let [chk (or (:check params) (constantly true))
f (or (:fn params) (fn [x & xs] x))
attr (or (:attr params) kns)]
(if (chk val)
(apply assoc res (db-pair attr kns val f))
(throw (Exception. (str "value " val " failed check"))))))))
(defn schema [m]
(loop [db-kns# DB-KNS
output {}]
(if-let [entry (first db-kns#)]
(recur (rest db-kns#)
(process-kns m (first entry) (second entry) output))
(assoc output
:db.install/_attribute :db.part/db
:db/id (d/tempid :db.part/db)))))
(def account-schema
[(schema {:ident :account/username
:type :string
:cardinality :one
:unique :value
:doc "The username associated with the account"})
(schema {:ident :account/password
:type :string
:cardinality :one
:doc "The password associated with the account"})
(schema {:ident :account/firstName
:type :string
:cardinality :one
:doc "The first name of the user"})
(schema {:ident :account/lastName
:type :string
:cardinality :one
:doc "The first name of the user"})
(schema {:ident :account/otherEmails
:type :ref
:cardinality :many
:doc "Other email address of the user"})
(schema {:ident :account/primaryEmail
:type :ref
:cardinality :one
:doc "The primary email address of the user"})])
(def email-schema
[(schema {:ident :email/address
:type :string
:cardinality :one
:unique :value
:doc "An email address"})
(schema {:ident :email/priority
:type :long
:cardinality :one
:doc "An email address's priority"})])
(d/transact *conn* account-schema)
(d/transact *conn* email-schema)
(defn get-ns-attrs1 [?ns db]
(d/q [':find '?e '?ident ':where
['?e ':db/ident '?ident]
['_ ':db.install/attribute '?e]
[(list '.toString '?ident) '?val]
[(list '.startsWith '?val (str ":" ?ns))]] db))
(defn get-ns-attrs2 [?ns db]
(d/q '[:find ?e ?ident :where
[?e :db/ident ?ident]
[_ :db.install/attribute ?e]
[(.toString ?ident) ?val]
[(.startsWith ?val ~(str ":" ?ns))]] db))
(get-ns-attrs1 "account" (d/db *conn*))
(get-ns-attrs1 "email" (d/db *conn*))
(get-ns-attrs2 "account" (d/db *conn*))
(get-ns-attrs2 "email" (d/db *conn*))

After a bit more reading, I've figured out that the :in keyword is the key to all of this. Examples are given in the 'Advanced Queries' section of the Tutorial - http://docs.datomic.com/tutorial.html.
This is the equivalent query listing all attributes in the :account namespace
(d/q '[:find ?e ?ident ?ns :in $ ?ns :where
[?e :db/ident ?ident]
[_ :db.install/attribute ?e]
[(.toString ?ident) ?val]
[(.startsWith ?val ?ns)]]
(d/db *conn*)
"account")
;; => #<HashSet [[68 :account/firstName], [67 :account/password], [71 :account/primaryEmail], [66 :account/username], [69 :account/lastName], [70 :account/otherEmails]]>
This is the equivalent in a function
(defn get-ns-attrs [_ns db]
(d/q '[:find ?e ?ident :in $ ?ns :where
[?e :db/ident ?ident]
[_ :db.install/attribute ?e]
[(.toString ?ident) ?val]
[(.startsWith ?val ?ns) ]] db (str _ns)))
(get-ns-attrs :account (d/db *conn*))
;; => #<HashSet [[68 :account/firstName], [67 :account/password], [71 :account/primaryEmail], [66 :account/username], [69 :account/lastName], [70 :account/otherEmails]]>
If you require more modularity, the function can be further broken down using % to pass in a set of rules:
(def rule-nsAttrs
'[[nsAttrs ?e ?ident ?ns]
[?e :db/ident ?ident]
[_ :db.install/attribute ?e]
[(.toString ?ident) ?val]
[(.startsWith ?val ?ns)]])
(defn get-ns-attrs [_ns db]
(d/q '[:find ?e ?ident :in $ % ?ns :where
(nsAttrs ?e ?ident ?ns)]
(d/db *conn*)
[rule-nsAttrs]
(str _ns)))
(get-ns-attrs :account (d/db *conn*))
;; => #<HashSet [[68 :account/firstName], [67 :account/password], [71 :account/primaryEmail], [66 :account/username], [69 :account/lastName], [70 :account/otherEmails]]>

can be solved a bit simpler with Clojure's namespace:
(d/q '[:find ?name :in $ ?ns
:where [_ :db.install/attribute ?a]
[?a :db/ident ?name]
[(namespace ?name) ?attr-ns]
[(= ?attr-ns ?ns)]] (d/db *conn*) "account")
which returns:
#{[:account/password]
[:account/firstName]
[:account/lastName]
[:account/username]}

Related

How to query against attributes of multiple values?

Tested on datascript 1.3.0
datoms:
[{:db/id -1 :name "Oliver Smith" :hobbies ["reading" "sports" "music"]}]
tried to run the query below to find who like sports, but the empty set returned.
'[:find ?name
:where
[?p :name ?name]
[?p :hobbies ?hobbies]
[(some #{"sports"} ?hobbies)]]
How to formulate the query correctly to get the expected result below?
#{[Oliver Smith]}
We have to explicitly define the schema with cardinality/many against the attribute of multiple values to solve the problem since schemaless doesn't work here.
(require '[datascript.core :as d])
(def schema {:hobbies {:db/cardinality db.cardinality/many}})
(def conn (d/create-conn schema))
(def datoms [{:db/id -1 :name "Oliver Smith" :hobbies ["reading" "sports" "music"]}])
(d/transact! conn datoms)
(def query '[:find ?name :where [?p :name ?name] [?p :hobbies "sports"]])
(-> (d/q query #conn) println)

Datomic not returning the correct "min" result when retrieving entity ID in result tuple

I've got this simple schema and data:
(def product-offer-schema
[{:db/ident :product-offer/product
:db/valueType :db.type/ref
:db/cardinality :db.cardinality/one}
{:db/ident :product-offer/vendor
:db/valueType :db.type/ref
:db/cardinality :db.cardinality/one}
{:db/ident :product-offer/price
:db/valueType :db.type/long
:db/cardinality :db.cardinality/one}
{:db/ident :product-offer/stock-quantity
:db/valueType :db.type/long
:db/cardinality :db.cardinality/one}
])
(d/transact conn product-offer-schema)
(d/transact conn
[{:db/ident :vendor/Alice}
{:db/ident :vendor/Bob}
{:db/ident :product/BunnyBoots}
{:db/ident :product/Gum}
])
(d/transact conn
[{:product-offer/vendor :vendor/Alice
:product-offer/product :product/BunnyBoots
:product-offer/price 9981 ;; $99.81
:product-offer/stock-quantity 78
}
{:product-offer/vendor :vendor/Alice
:product-offer/product :product/Gum
:product-offer/price 200 ;; $2.00
:product-offer/stock-quantity 500
}
{:product-offer/vendor :vendor/Bob
:product-offer/product :product/BunnyBoots
:product-offer/price 9000 ;; $90.00
:product-offer/stock-quantity 15
}
])
When I retrieve the cheapest bunny boots, only retrieving the price, I get the expected result (9000):
(def cheapest-boots-q '[:find (min ?p) .
:where
[?e :product-offer/product :product/BunnyBoots]
[?e :product-offer/price ?p]
])
(d/q cheapest-boots-q db)
;; => 9000
However, when I want to get the entity ID along with the price, it gives me the higher-priced boots:
(def db (d/db conn))
(def cheapest-boots-q '[:find [?e (min ?p)]
:where
[?e :product-offer/product :product/BunnyBoots]
[?e :product-offer/price ?p]
])
(d/q cheapest-boots-q db)
;; => [17592186045423 9981]
I tried adding :with but that gives me an error:
(def cheapest-boots-q '[:find [?e (min ?p)]
:with ?e
:where
[?e :product-offer/product :product/BunnyBoots]
[?e :product-offer/price ?p]
])
(d/q cheapest-boots-q db)
;; => => Execution error (ArrayIndexOutOfBoundsException) at datomic.datalog/fn$project (datalog.clj:503).
What am I doing wrong?
As a commenter kind of pointed out, ?e isn't bound in any way to the (min ?p) expression, so it's not defined what you'll get there, beyond a product entity id of some sort.
What you actually want to do is unify those values somehow as part of the query, and not perform aggregation on the results, for example:
(d/q '[:find [?e ?p]
:where
[?e :product-offer/product :product/BunnyBoots]
[?e :product-offer/price ?p]
[(min ?p)]]
db)
You can see that the min clause is part of the query, and as such will take part in the unification on the result, giving you what you want.

Need help understanding why Clojure spec test/check is failing the return validation when REPL doesn't fail

I've been playing around with Clojure Spec for testing and data generation and am seeing some strange behavior where the function works in unit tests and validation works in REPL but the generative testing with spec.test/check is failing.
I've created a set of specs like so:
(s/def ::significant-string (s/with-gen
(s/and string? #(not (nil? %)))
(fn [] (gen/such-that #(not= % "")
(gen/string-alphanumeric)))))
(s/def ::byte-stream
(s/with-gen #(instance? java.io.ByteArrayInputStream %)
(gen/fmap #(string->stream %) (gen/string-alphanumeric))))
(s/fdef string->stream
:args (s/cat :s ::significant-string)
:ret ::byte-stream
:fn #(instance? java.io.ByteArrayInputStream %))
And the fn implementation:
(defn string->stream
"Given a string, return a java.io.ByteArrayInputStream"
([s] {:pre [(s/valid? ::significant-string s)]
:post [(s/valid? ::byte-stream %)]}
(string->stream s "UTF-8"))
([s encoding]
(-> s
(.getBytes encoding)
(java.io.ByteArrayInputStream.))))
And in REPL I see what I expect to see from both generation and testing of the specs:
=> (instance? java.io.ByteArrayInputStream (string->stream "test"))
true
=> (s/valid? ::byte-stream (string->stream "0"))
true
=> (s/exercise-fn 'calais-response-processor.rdf-core/string->stream)
([("Y") #object[java.io.ByteArrayInputStream 0x57210dd7 "java.io.ByteArrayInputStream#57210dd7"]] [("d") #object[java.io.ByteArrayInputStream 0x7ec14113 "java.io.ByteArrayInputStream#7ec14113"]] [("5") #object[java.io.ByteArrayInputStream 0x1e85195b "java.io.ByteArrayInputStream#1e85195b"]] [("9c") #object[java.io.ByteArrayInputStream 0x3769ddef "java.io.ByteArrayInputStream#3769ddef"]] [("P0N") #object[java.io.ByteArrayInputStream 0x68793160 "java.io.ByteArrayInputStream#68793160"]] [("7tvN1") #object[java.io.ByteArrayInputStream 0x1cc43ca5 "java.io.ByteArrayInputStream#1cc43ca5"]] [("LjH4U") #object[java.io.ByteArrayInputStream 0x2a3da1a7 "java.io.ByteArrayInputStream#2a3da1a7"]] [("W") #object[java.io.ByteArrayInputStream 0x534287aa "java.io.ByteArrayInputStream#534287aa"]] [("x867VLr") #object[java.io.ByteArrayInputStream 0x72915e93 "java.io.ByteArrayInputStream#72915e93"]] [("moucN3vr") #object[java.io.ByteArrayInputStream 0x4f0d7570 "java.io.ByteArrayInputStream#4f0d7570"]])
But I don't understand why I'm seeing this from the test/check:
(stest/check 'calais-response-processor.rdf-core/string->stream)
({:spec #object[clojure.spec.alpha$fspec_impl$reify__2451 0x1acb0d46 "clojure.spec.alpha$fspec_impl$reify__2451#1acb0d46"], :clojure.spec.test.check/ret {:shrunk {:total-nodes-visited 4, :depth 3, :pass? false, :result #error {
:cause "Specification-based check failed"
:data {:clojure.spec.alpha/problems [{:path [:fn], :pred (clojure.core/fn [%] (clojure.core/instance? java.io.ByteArrayInputStream %)), :val {:args {:s "0"}, :ret #object[java.io.ByteArrayInputStream 0x7bee9d86 "java.io.ByteArrayInputStream#7bee9d86"]}, :via [], :in []}], :clojure.spec.alpha/spec #object[clojure.spec.alpha$spec_impl$reify__1987 0x16a19b4c "clojure.spec.alpha$spec_impl$reify__1987#16a19b4c"], :clojure.spec.alpha/value {:args {:s "0"}, :ret #object[java.io.ByteArrayInputStream 0x7bee9d86 "java.io.ByteArrayInputStream#7bee9d86"]}, :clojure.spec.test.alpha/args ("0"), :clojure.spec.test.alpha/val {:args {:s "0"}, :ret #object[java.io.ByteArrayInputStream 0x7bee9d86 "java.io.ByteArrayInputStream#7bee9d86"]}, :clojure.spec.alpha/failure :check-failed}
:via
[{:type clojure.lang.ExceptionInfo
:message "Specification-based check failed"
:data {:clojure.spec.alpha/problems [{:path [:fn], :pred (clojure.core/fn [%] (clojure.core/instance? java.io.ByteArrayInputStream %)), :val {:args {:s "0"}, :ret #object[java.io.ByteArrayInputStream 0x7bee9d86 "java.io.ByteArrayInputStream#7bee9d86"]}, :via [], :in []}], :clojure.spec.alpha/spec #object[clojure.spec.alpha$spec_impl$reify__1987 0x16a19b4c "clojure.spec.alpha$spec_impl$reify__1987#16a19b4c"], :clojure.spec.alpha/value {:args {:s "0"}, :ret #object[java.io.ByteArrayInputStream 0x7bee9d86 "java.io.ByteArrayInputStream#7bee9d86"]}, :clojure.spec.test.alpha/args ("0"), :clojure.spec.test.alpha/val {:args {:s "0"}, :ret #object[java.io.ByteArrayInputStream 0x7bee9d86 "java.io.ByteArrayInputStream#7bee9d86"]}, :clojure.spec.alpha/failure :check-failed}
:at [clojure.core$ex_info invokeStatic "core.clj" 4739]}]
:trace
[[clojure.core$ex_info invokeStatic "core.clj" 4739]
[clojure.core$ex_info invoke "core.clj" 4739]
...
...(lots more)
It feels like it's related to the generator fn composition although returned object looks "ok" to me right now.
:fn #(instance? java.io.ByteArrayInputStream %))
The problem is it looks like that :fn spec expects only the function return value, when it's actually being invoked with a map containing the input and return values. Try this version instead:
:fn (fn [{:keys [args ret]}]
(instance? java.io.ByteArrayInputStream ret))
The :fn spec should be a function that takes a map containing the function's input :args and output :ret value. It's meant to compare the function's output relative to its input.
In this example, the :fn spec seems to be making the same assertion as your :ret spec, and it doesn't look at the :args so you may not want a :fn spec here if there's no meaningful assertion to make between input/output — this would only assert the return value, redundantly.
And in REPL I see what I expect to see from both generation and testing of the specs
The reason you're only seeing the failure with check is because none of those other calls are considering your function's :fn spec e.g. s/exercise-fn doesn't consider the :fn spec.
I made some examples using :fn specs here.

Filter by cardinality-many field that contains ALL the values

I’ve got an account model that has interests, which is an array of strings (db.type=string and cardinality=many). For example:
{:id 42
:name "Test"
:interests ["cars" "games" "etc"]
:age 50}
Then another list of interests come from the request. How can I query accounts what have ALL the specified interests?
For example, for ["cars" "games"] I'll get the account #42 since it includes the passed list. But for ["cars" "games" "books"] I won't because "books" is an extra one.
UPD
What I have is:
;; model
{:db/ident :account/interests
:db/valueType :db.type/string
:db/cardinality :db.cardinality/many
:db/index true}
;; a query
[:find ?id
:id $ [?interest ...]
:where
[?a :account/interests ?interest]
[?a :account/id ?id]]
So how should I build a Datomic query?
Try using entities in a query
(d/q '[:find ?e
:in $ ?interests
:where
[?e :interests _]
[(datomic.api/entity $ ?e) ?ent]
[(:interests ?ent) ?ent_interests]
[(subset? ?interests ?ent_interests)]]
(d/db conn)
#{"cars" "games"})
(time (d/q
'[:find ?id
:in $
:where
[?a :account/interests "foo"]
[?a :account/interests "bar"]
[?a :account/id ?id]]
(d/db conn)))
"Elapsed time: 3.771973 msecs"
(time (d/q
'[:find ?id
:in $ ?iii
:where
[(datomic.api/entity $ ?a) ?e]
[(:account/interests ?e) ?interests]
[(clojure.set/superset? ?interests ?iii)]
[?a :account/id ?id]]
(d/db conn)
#{"50 Cent" "Целоваться"}))
"Elapsed time: 169.767354 msecs"
The first one is better.

How to filter a list based on user input with ClojureScript and Om?

I just started to use Om (a reactjs based library for ClojureScript). I would like to filter a list based on user input. The following works but the solution seems to be to complicated. Is there a better one ?
(ns om-tut.core
(:require-macros [cljs.core.async.macros :refer [go]])
(:require [om.core :as om :include-macros true]
[om.dom :as dom :include-macros true]
[clojure.string :as string]))
(enable-console-print!)
(def app-state (atom {:list ["Lion" "Zebra" "Buffalo" "Antelope"]}))
(defn handle-change [e owner {:keys [text]}]
(om/set-state! owner :data (vec (filter (fn [x] (> (.indexOf x(.. e -target -value)) -1)) (#app_state :list))))
(om/set-state! owner :text (.. e -target -value)))
(defn list-view [app owner]
(reify
om/IInitState
(init-state [_]
{:text nil
:data (:list app)
})
om/IRenderState
(render-state [this state]
(dom/div nil
(apply dom/ul #js {:className "animals"}
(dom/input
#js {:type "text" :ref "animal" :value (:text state)
:onChange #(handle-change % owner state)})
(map (fn [text] (dom/li nil text)) (:data state)))))))
(om/root list-view app-state
{:target (. js/document (getElementById "registry"))})
I think that this is a better solution:
(ns om-tut.core
(:require-macros [cljs.core.async.macros :refer [go]])
(:require [om.core :as om :include-macros true]
[om.dom :as dom :include-macros true]))
(def app-state (atom {:list ["Lion" "Zebra" "Buffalo" "Antelope"]}))
(defn handle-change [e owner {:keys [text]}]
(om/set-state! owner :text (.. e -target -value)))
(defn list-data [alist filter-text]
(filter (fn [x] (if (nil? filter-text) true
(> (.indexOf x filter-text) -1))) alist))
(defn list-view [app owner]
(reify
om/IInitState
(init-state [_]
{:text nil})
om/IRenderState
(render-state [this state]
(dom/div nil
(apply dom/ul #js {:className "animals"}
(dom/input
#js {:type "text" :ref "animal" :value (:text state)
:onChange (fn [event] (handle-change event owner state))})
(map (fn [text] (dom/li nil text)) (list-data (:list app) (:text state))))))))
(om/root list-view app-state
{:target (. js/document (getElementById "animals"))})