Idiomatic clojure to parse command line switches and args from string - clojure

I'd like to parse a command line string, and group any command switches with their subsequent arguments. So, for example:
(parse "git branch -d f1 f2 -a -m message") =>
[["-d" "f1" "f2"]["-a"]["-m" "message"]]
I ignore args not immediately following a switch.
The code I wrote to do this is as follows:
(defn switch? [s] (re-find #"\-+" s))
(defn tokenify [s] (clojure.string/split s #" "))
(defn parse [cmd-str]
(loop [lst (tokenify cmd-str), acc [], _acc []]
(let [fs (first lst), rs (rest lst), new? (empty? _acc)]
(cond (empty? lst) (if new? acc (conj acc _acc))
(switch? fs) (if new?
(recur rs acc (conj _acc fs))
(recur rs (conj acc _acc) (conj [] fs)))
:else (if new?
(recur rs acc _acc)
(recur rs acc (conj _acc fs)))))))
That works, but it's pretty low level and horrible. Is there a simple way using reduce or partition or group-by that would make the same functionality a lot cleaner and more idiomatic?

here's a basic outline of using clojure.tools.cli
(def command-line-spec
[["-m" "--mode 0|1" "description of option that takes 1 or 0"
:parse-fn #(case (s/lower-case %)
("1" "true" "create") :one
("0" "false" "destroy") :zero
:invalid)
:default :one
:validate [#{:one :zero} "Unsupported mode"]]
["-c" "--config type1|type2|..."
:desc "config specifies what to do"
:default :dostuff
:parse-fn #(if (keyword? %)
%
(-> % s/lower-case keyword))
:validate-fn #(contains? configurations %)]
["-n" "--name service Name"
:default (getenv "NAME")]
[nil "--min number"
:default 7 :parse-fn #(Integer/parseInt %)]
[nil "--max number"
:default 7 :parse-fn #(Integer/parseInt %)]
[nil "--public true|false" "true or false"
:default false
:parse-fn #(Boolean/parseBoolean %)]
["-h" "--help"]])
(defn -main [& args]
(let [{:keys [options arguments errors summary]}
(parse-opts args
command-line-spec)
... )

Related

Sleeping barber in Clojure

I'm implementing Sleeping barber using core.async. My current code is:
(def workingtime 10000)
(defn barber [in waiting-room]
(go-loop [served-customers 0]
(let [[v] (alts! [waiting-room in])]
(if (= v :close)
served-customers
(do (Thread/sleep 20)
(recur (inc served-customers)))))))
(defn customers [in waiting-room]
(go-loop [customers-overall 0]
(let [customer-arrival-interval (timeout (+ 10 (rand-int 20)))
[v] (alts! [in customer-arrival-interval])]
(if (= v :close)
customers-overall
(do (>! waiting-room :customer)
(recur (inc customers-overall)))))))
(defn -main [& args]
(let [in (chan)
waiting-room (chan (dropping-buffer 3))
barber-ch (barber in waiting-room)
customers-ch (customers in waiting-room)]
(println "opening the shop for 10 seconds...")
(Thread/sleep workingtime)
(>!! in :close)
(>!! in :close)
(println "closing the shop...")
(println (str "Served " (<!! barber-ch) " customers"))
(println (str "Overall " (<!! customers-ch) " customers came"))))
Is it a correct solution? Can it be improved to make it more Clojure-like?
I wanted to use alt! instead of alts! which makes code easier to read:
(defn barber [in]
(go-loop [served-customers 0]
(alt!
waiting-room (do (Thread/sleep 20)
(recur (inc served-customers)))
in served-customers)))
Runtime throws an exception: Can only recur from tail position. Can I still use alt!?
You could solve the alt!/recur problem by rewriting to:
(defn barber [in]
(go-loop [served-customers 0]
(if (= :waiting-room
(a/alt!
waiting-room ([result] :waiting-room) ;; you could also use result if needed
in ([result] :in))) ;; same here
(do (Thread/sleep 20)
(recur (inc served-customers)))
served-customers)))

Nesting macros in Clojure

Consider this pseudo code:
(defrc name
"string"
[a :A]
[:div a])
Where defrc would be a macro, that would expand to the following
(let [a (rum/react (atom :A))]
(rum/defc name < rum/reactive []
[:div a]))
Where rum/defc is itself a macro. I came up with the code below:
(defmacro defrc
[name subj bindings & body]
(let [map-bindings# (apply array-map bindings)
keys# (keys map-bindings#)
vals# (vals map-bindings#)
atomised-vals# (atom-map vals#)]
`(let ~(vec (interleave keys# (map (fn [v] (list 'rum/react v)) (vals atomised-vals#))))
(rum/defc ~name < rum/reactive [] ~#body))))
Which almost works:
(macroexpand-all '(defrc aname
#_=> "string"
#_=> [a :A]
#_=> [:div a]))
(let* [a (rum/react #object[clojure.lang.Atom 0x727ed2e6 {:status :ready, :val nil}])] (rum/defc aname clojure.core/< rum/reactive [] [:div a]))
However when used it results in a syntax error:
ERROR: Syntax error at (clojure.core/< rum.core/reactive [] [:div a])
Is this because the inner macro is not being expanded?
Turns out the macro was working correctly but the problem occurred because < was inside the syntax quote it got expanded to clojure.core/<, and Rum simply looks for a quoted <, relevant snippet from Rum's source:
...(cond
(and (empty? res) (symbol? x))
(recur {:name x} next nil)
(fn-body? xs) (assoc res :bodies (list xs))
(every? fn-body? xs) (assoc res :bodies xs)
(string? x) (recur (assoc res :doc x) next nil)
(= '< x) (recur res next :mixins)
(= mode :mixins)
(recur (update-in res [:mixins] (fnil conj []) x) next :mixins)
:else
(throw (IllegalArgumentException. (str "Syntax error at " xs))))...

How to parse xml and get an vector for some attributes on an element

I know how to extract one attribute using zip-xml/attr, but how to extract multiple attributes?
e.g I have the following
<table>
<column name="col1" type="varchar" length="8"/>
<column name="col2" type="varchar" length="16"/>
<column name="col3" type="int" length="16"/>
<table>
And the expected result is. A silly way is to call zip-xml/attr for each attribute, but is there any elegant way to do that?
[["co11" "varchar" 8] [["co12" "varchar" 16] [["co13" "int" 16]
My advice is to use a tree-walking function to extract the interesting data from the XML tree. clojure.walk has several of these, but here I use tree-seq from core clojure to just produce a seq of nodes and work on that. This function takes two functions - a branch? predicate which checks if a node can have children and a children function which gets them. I use :content for both, as tags with no nested tags produce nil, which is a falsey value and so it works also as a predicate.
(->> (clojure.xml/parse "res/doc.xml") ;;source file for your xml
(tree-seq :content :content) ;; Produce a seq by walking the tree
(filter #(= :column (:tag %))) ;;Take only :column tags
(mapv (comp vec vals :attrs)))
;;Collect the values of the :attrs maps into vectors
;;and collect those into a vector with mapv
Your desired output had unmatched square brackets, but I assume it should be like
[["col1" "varchar" "8"] ["col2" "varchar" "16"] ["col3" "int" "16"]]
which was my return value. However, this is potentially brittle - you're relying on the maps returned by clojure.xml/parse preserving the ordering of the attributes in the XML in order to know what the data means. That's not really part of the contract of maps. As an implementation detail it creates clojure.lang.PersistentStructMaps which apparently do have this feature, but it might not always be so.
Alternatively you could use just (mapv :attrs) to keep the whole of the map in there.
The right solution depends on how large and complex the XML is and to some extent, what you know about its structure. If it needs to be very generic, then you need to have quite a lot of logic to navigate the nodes etc. However, if it is a known format and you know what nodes you are interested in, its pretty straight-forward.
I used clojure.zip to create a zipper from the XML file and then use clojure.data.zip.xml to extract the nodes/paths I was interested in. I then defined simple helper functions to process specific nodes. This was pretty much my first bit of clojure and I've not yet gone back to it to re-factor it and refine/clarify some of my very rough clojure idioms based on what I've learnt since, but in the spirit of an example being worth 1000 words, here it is -
(ns arcis.models.nessus
(:use [taoensso.timbre :only [trace debug info warn error fatal]])
(:require [arcis.util :as util]
[arcis.models.db :as db]
[clojure.java.io :as io]
[clojure.xml :as xml]
[clojure.zip :as zip]
[clojure.data.zip.xml :as zx]))
(def nessus-host-keys [:hostname :host_fqdn
:system_type :operating_system
:operating_system_unsupported])
(def used-nessus-host-keys (conj nessus-host-keys
:host_start :host_end
:items :traceroute_hop_0 :traceroute_hop_1
:traceroute_hop_2 :traceroute_hop_3
:traceroute_hop_4 :traceroute_hop_5
:traceroute_hop_6 :traceroute_hop_7
:traceroute_hop_8 :traceroute_hop_9
:traceroute_hop_10 :traceroute_hop_11
:traceroute_hop_12 :traceroute_hop_13
:traceroute_hop_14 :traceroute_hop_15
:traceroute_hop_16 :traceroute_hop_17
:host_ip :patch_summary_total_cves
:cpe_0 :cpe_1 :cpe_2 :cpe_3 :cpe_4 :cpe_5
:cpe_6 :cpe_7 :cpe_8 :cpe_9))
(def nessus-item-keys [:port :svc_name :protocol :severity :plugin_id
:plugin_output])
(def used-nessus-item-keys (conj nessus-item-keys
:plugin_details
:plugin_name
:plugin_family))
(def nessus-plugin-keys [:plugin_id :plugin_name :plugin_family :fname
:script_version :plugin_type :exploitability_ease
:vuln_publication_date :cvss_temporal_data
:solution :cvss_temporal_score :risk_factor
:description :cvss_vector :synopsis
:patch_publication_date :exploit_available
:plugin_publication_date :plugin_modification_date
:cve :bid :exploit_framework_canvas :edb_id
:exploit_framework_metasploit :exploit_framework_core
:metasploit_name :canvas_package :osvdb :cwe
:cvss_temporal_vector :cvss_base_score :cpe
:exploited_by_malware])
(def used-nessus-plugin-keys (conj nessus-plugin-keys
:xref :see_also :cert
:attachment :iava :stig_severity :hp
:secunia :iawb :msft))
(def show-unprocessed true)
(defn log-unprocessed [title vls]
(if (and show-unprocessed
(seq vls))
(println (str "Unprocessed " title ": " vls))))
;;; parse nessus report
(defn parse-xref [xref]
{:xref (first (:content xref))})
(defn parse-see-also [see-also]
{:see_also (first (:content see-also))})
(defn parse-plugin [plugin]
{(util/db-keyword (name (:tag plugin))) (first (:content plugin))})
(defn parse-contents [cont]
(let [xref (mapv parse-xref (filter #(= (:tag %) :xref) cont))
see-also (mapv parse-see-also (filter #(= (:tag %) :see-also) cont))
details (reduce merge {}
(map parse-plugin
(remove #(or (= (:tag %) :xref)
(= (:tag %) :see-also)) cont)))]
(assoc details
:see_also see-also
:xref xref)))
(defn fix-item-keywords [item]
(let [ks (keys item)]
(into {}
(for [k ks]
[(util/db-keyword (name k))
(k item)]))))
(defn parse-item [item]
(let [attrs (fix-item-keywords (:attrs item))
contents (parse-contents (:content item))]
(assoc attrs
:plugin_output (:plugin_output contents)
:plugin_details (assoc (dissoc contents :plugin_output)
:plugin_id (:plugin_id attrs)
:plugin_family (:plugin_family attrs)))))
(defn parse-properties [props]
(into {}
(for [p props]
[(util/db-keyword (:name (:attrs p)))
(first (:content p))])))
(defn parse-host [h]
(let [items (map first (zx/xml-> h :ReportItem))
properties (:content (first (zx/xml1-> h :HostProperties)))]
(assoc (parse-properties properties)
:hostname (zx/attr h :name)
:items (mapv parse-item items))))
(defn parse-hosts [hosts]
(mapv parse-host hosts))
(defn parse-file [f]
(let [root (zip/xml-zip (xml/parse (io/file f)))
report-xml (zx/xml1-> root :Report)
hosts (zx/xml-> report-xml :ReportHost)]
{:report_name (zx/attr report-xml :name)
:policy (zx/text (zx/xml1-> root :Policy :policyName))
:hosts (parse-hosts hosts)}))
;;; insert nessus records into db
(defn mk-host-rec [scan-id host]
(let [[id err] (db/get-sequence-nextval "host_seq")]
(if (nil? err)
(assoc (util/build-map host nessus-host-keys)
:ipv4 (:host_ip host)
:scan_start (util/from-nessus-date (:scan_start host))
:scan_end (util/from-nessus-date (:scan_end host))
:total_cves (:patch_summary_total_cves host)
:id id
:scan_id scan-id)
nil)))
(defn insert-patches [p]
(when (seq p)
(db/insert-nessus-host-patch (first p))
(recur (rest p))))
(defn insert-host-patch [id host]
(let [p-keys (filter #(re-find #"patch_summary_*" %) (map name (keys host)))
recs (map (fn [s]
{:id (first (db/get-sequence-nextval "patch_seq"))
:host_id id
:summary ((keyword (str "patch_summary_txt_" s)) host)
:cve_num ((keyword (str "patch_summary_cve_num_" s)) host)
:cves ((keyword (str "patch_summary_cves_" s)) host)})
(filter seq
(map #(second (re-find #"patch_summary_txt_(.*)" %))
p-keys)))]
(insert-patches recs)
(util/remove-keys host (map keyword p-keys))))
(defn mk-item-rec [host-id item]
(let [[id err] (db/get-sequence-nextval "item_seq")]
(assoc (util/build-map item nessus-item-keys)
:host_id host-id
:id id)))
(defn insert-item [host-id item]
(let [rec (mk-item-rec host-id item)
not-done (keys (util/remove-keys item used-nessus-item-keys))]
(log-unprocessed "Item Keys" not-done)
(db/insert-nessus-report-item rec)
(:plugin_id item)))
(defn mk-plugin-rec [item]
(let [rec (util/build-map (:plugin_details item) nessus-plugin-keys)
not-used (keys (util/remove-keys (:plugin_details item)
used-nessus-plugin-keys))]
(log-unprocessed "Plugin Keys" not-used)
(assoc rec
:vuln_publication_date (util/from-nessus-date
(:vuln_publication_date rec))
:patch_publication_date (util/from-nessus-date
(:patch_publication_date rec))
:plugin_publication_date (util/from-nessus-date
(:plugin_publication_date rec))
:plugin_modification_date (util/from-nessus-date
(:plugin_modificaiton_date rec)))))
(defn insert-xref [plugin-id xrefs]
(when (seq xrefs)
(let [xref {:id (first (db/get-sequence-nextval "xref_seq"))
:plugin_id plugin-id
:xref (:xref (first xrefs))}]
(db/insert-nessus-xref xref)
(recur plugin-id (rest xrefs)))))
(defn insert-see-also [plugin-id see-also]
(when (seq see-also)
(let [sa {:id (first (db/get-sequence-nextval "ref_seq"))
:plugin_id plugin-id
:reference (:see_also (first see-also))}]
(db/insert-nessus-ref sa)
(recur plugin-id (rest see-also)))))
(defn insert-plugin [item]
(let [rec (mk-plugin-rec item)
xref (:xref (:plugin_details item))
see-also (:see_also (:plugin_details item))]
(if (seq xref)
(insert-xref (:plugin_id rec) xref))
(if (seq see-also)
(insert-see-also (:plugin_id rec) see-also))
(db/upsert-nessus-plugin rec)))
(defn insert-items [host-id items plugin-set]
(if (empty? items)
plugin-set
(let [p (insert-item host-id (first items))]
(if-not (contains? plugin-set p)
(insert-plugin (first items)))
(recur host-id (rest items) (conj plugin-set p)))))
(defn insert-host [scan-id host plugin-set]
(if-let [h-rec (mk-host-rec scan-id host)]
(let [[v err] (db/insert-nessus-host h-rec)
items (:items host)]
(if (nil? err)
(let [host2 (insert-host-patch (:id h-rec) host)]
(log-unprocessed "Host Keys" (keys (util/remove-keys
host2 used-nessus-host-keys)))
(insert-items (:id h-rec) items plugin-set))
plugin-set))
plugin-set))
(defn insert-hosts
([id hosts]
(insert-hosts id hosts #{}))
([id hosts plugins]
(if (empty? hosts)
plugins
(let [plugin-set (insert-host id (first hosts) plugins)]
(recur id (rest hosts) plugin-set)))))
(defn mk-scan-record [id report]
{:id id
:name (:report_name report)
:scan_dt (util/to-sql-date)
:policy (:policy report)
:entered_dt (util/to-sql-date)})
(defn store-report [update-plugins report]
(let [[id err] (db/get-sequence-nextval "nscan_seq")
scan-rec (mk-scan-record id report)]
(if (nil? err)
(let [[v e] (db/insert-nessus-scan scan-rec)]
(if (nil? e)
(if update-plugins
(let [plugin-list (set (first (db/select-nessus-plugin-ids)))]
[(insert-hosts id (:hosts report) plugin-list) nil])
[(insert-hosts id (:hosts report)) nil])
[v e]))
[id err])))
(defn process-nessus-report [update-plugins filename]
(let [report (parse-file filename)]
(println (str "Report: " (:report_name report)
"\nPolicy: " (:policy report)
"\nHost Records: " (count (:hosts report))))
(store-report update-plugins report)))
Magos's answer using tree-seq is perfectly fine, but there's no reason to abandon zippers; filtering using zippers is more succinct and the arguably the "clojure" way. (note this example uses data.xml ([org.clojure/data.xml "0.0.8"]) instead of clojure.xml).
(require '[clojure.data.zip.xml :as zf])
(require '[clojure.zip :as z])
(def ex
"<table>
<column name=\"col1\" type=\"varchar\" length=\"8\"/>
<column name=\"col2\" type=\"varchar\" length=\"16\"/>
<column name=\"col3\" type=\"int\" length=\"16\"/>
</table>")
(let [x (z/xml-zip (clojure.data.xml/parse-str ex))]
(->> (zf/xml-> x :column) ;;equivalent to (->> treeseq ... filter)
flatten
(keep :attrs)
(map vals)))
;>>> (("col1" "varchar" "8") ("col2" "varchar" "16") ("col3" "int" "16"))
But the xml-> macro simply applies functions in order, so you can do the following:
(let [x (z/xml-zip (clojure.data.xml/parse-str ex))]
(->> (zf/xml-> x :column #(keep :attrs %))
(map vals)))
;>>> (("col1" "varchar" "8") ("col2" "varchar" "16") ("col3" "int" "16"))

macro always throw "UnmatchedDelimiter" if given anon function

I wrote a macro to handle http response
(defmacro defhandler
[name & args]
(let [[docstring args] (if (string? (first args))
[(first args) (next args)]
[nil args])
args (apply hash-map :execute-if true (vec args))]
`(do
(def ~name
(with-meta (fn [scope# promise#]
(let [e# (:execute-if ~args)
ei# (if (fn? e#)
(e# scope#)
(boolean e#))]
(when ei#
(.then promise# (fn [result#]
(let [{:strs [http-status# value#]} result#
the-func# ((keyword http-status#) ~args)]
(the-func# scope# value#))))))) {:structure ~args}))
(alter-meta! (var ~name) assoc :doc ~docstring))))
So I can do
(defhandler my-handler
:200 (fn [$scope value] (set! (.-content $scope) value)))
But that throws "UnmatchedDelimiter" at line 1, but if I try with a named function:
(defn my-func [$scope value] (set! (.-content $scope) value))
(defhandler my-handler
:200 my-func)
It works ok. I'm just curious, is that a normal behaviour?
That is not the behavior I see when I try your example, nor does it seem very likely. I suggest checking that the forms you pasted here are exactly the ones that produce an error; I suspect your actual anonymous function included one too many )s.

lein run, won't run -main correctly, though a clean nrepl will

New to clojure, trying to compile the following code, for a boolean expression evaluator
;core.clj
(ns my-app.core
(:gen-class))
(defn t [& args] (if (empty? args) t
((first args) t (first (rest args)))))
(defn f [& args] (if (empty? args) f
((first args) f (first (rest args)))))
(defn | [cond1 cond2] (if (= cond1 t) t
(if (= cond2 t) t f)))
(defn & [cond1 cond2] (if (= cond1 f) f
(if (= cond2 f) f t)))
(defn ! [& args] (if (= (first args) t)
(apply f (rest args))
(if ( = (first args) f)
(apply t (rest args))
(! (apply (first args) (rest args))))))
(defn -main [& args]
(loop [line (read-line)]
(do
(println (eval (read-string (apply str "" (interpose \space (seq (str "(" line ")")))))))
(if (= (count line) 1) nil (recur (read-line))))))
Everytime I do "lein run" and enter the string "(t|t)=t" I get the following error
Exception in thread "main" java.lang.RuntimeException: Unable to resolve symbol: t in this context
However If I open up a new nrepl in the "src/my_app/" dir, and enter the command
(-main)
I get the correct string and result
( ( t | t ) = t )
true
I should note, that while running lein run
the other strings string "+ 1 2 3 4 5 6" will evaluate correctly, but for some reason it won't recognize the function (t ...)
Anybody have any ideas what's going on?
eval uses the current (thread-bound) value of *ns* to determine what namespace you're currently "in", which controls how unqualified symbols are resolved. At the repl after running (ns my-app.core ...), you're in the my-app.core namespace and thus eval finds the t you've defined there. However after compilation even though you still defined t in my-app.core, when your program begins running you're in the user namespace and eval fails to find t.
So all you need to do is change your namespace within -main to be my-app.core, using binding like this:
(defn -main [& args]
(binding [*ns* (the-ns 'my-app.core)]
(loop [line (read-line)]
(println (eval (read-string (apply str (interpose \space (str "(" line ")"))))))
(when-not (= (count line) 1)
(recur (read-line))))))