How to dynamicly include all source files in a folder with clojure? - clojure

I define a map of functions:
(ns fs
(:require [folder/a :as a]
[folder/b :as b]
[folder/c :as c])
(def functions {:a a/f :b b/f :c c/f})
(doseq [[_ f] functions] (f))
Now I want to add more namespaces within the folder, and I don't want to modify the above code. How can functions be dynamically populated with the f from each namespace in a folder.

First some helpers:
(defn directory
"Get directory from path"
[path]
(clojure.java.io/file path))
(defn file-names
"Get file names of the files in the directory."
[files]
(map (fn [file] (.getName file)) files))
(defn namespace
"Remove the extension from the file name and prefix with folder-name"
[folder-name file-name]
(->> (clojure.string/split file-name #"\.")
(butlast)
(apply str)
(str folder-name ".")
(symbol)))
Then retrieve all namespaces from your folder, you need the path and the folder name:
(def namespaces
(let [names (->> "/path/to/your/folder-name"
directory
file-seq ;; Gets the tree structure of the directory
rest ;; Get rid of the the directory name
file-names)]
(map (partial namespace "folder-name") names)))
Next get the public functions in every namespace via ns-publics:
(def functions
(->> namespaces
(map (juxt keyword (comp vals ns-publics)))
(into {})))
;; => prints {:namespace-key '(fn-a fn-b) :namespace-key2 '(fn-b fn-c)}
Note that this gets a list of all the public functions in a namesapce after the namespace keys.
We can execute the functions as follows:
(doseq [[_ ns-fns] functions
f ns-fns] (f))
Of course, this only works for functions that have an arity of zero. Otherwise you have to pass arguments to f.

Related

How to get all variables defined within a Clojure project?

I am collecting all user defined functions within a project. As a way of testing membership I use:
(defn get-var-namespace
[qualified-var]
{:pre [(var? qualified-var)]}
(-> qualified-var meta :ns))
(defn project-name
"returns a string representation of the root project name.
this is the same as the directory that the project is located in"
[]
(last (str/split (System/getProperty "user.dir") #"/")))
(defn get-project-namespaces
"return all namespaces defined within the project"
[]
(let [p-name (project-name)]
(filter (fn [x]
(let [n (first (str/split (str (ns-name x)) #"\."))]
(= n p-name))) (all-ns))))
(defn user-defined-var? [project-namespaces var]
(some #{(get-var-namespace var)} project-namespaces))
(defn namespace-deps
"returns a map whose keys are user defined functions and vals are sets of
user defined functions."
[project-namespaces ns]
(let [find-vars-in-expr
(fn [x] (let [nodes (ast/nodes x)
top-level (:var (first nodes))
non-recursive-deps (remove #{top-level}
(filter (partial var-has-user-ns project-namespaces)
(filter some? (map :var nodes))))]
{top-level (set non-recursive-deps)}))]
(dissoc (apply merge-with clojure.set/union
(map find-vars-in-expr (jvm/analyze-ns ns))) nil)))
as an example
clj-smart-test.core> (clojure.pprint/pprint (namespace-deps (get-project-namespaces) *ns*))
{#'clj-smart-test.core/foo
#{#'clj-smart-test.baz/baz1 #'clj-smart-test.core/bar
#'clj-smart-test.foo/foo #'clj-smart-test.core/k},
#'clj-smart-test.core/get-project-namespaces
#{#'clj-smart-test.core/project-name},
#'clj-smart-test.core/project-name #{},
#'clj-smart-test.core/var-has-user-ns
#{#'clj-smart-test.core/get-var-namespace},
#'clj-smart-test.core/bar #{},
#'clj-smart-test.core/f #{},
#'clj-smart-test.core/get-var-namespace #{},
#'clj-smart-test.core/namespace-deps
#{#'clj-smart-test.core/var-has-user-ns},
#'clj-smart-test.core/k #{}}
nil
Are there any corner cases that I need to be aware of? I know that a user could just create an arbitrary namespace in a file so I can't always assume that the directory that a project is created for example lein new my-project, which gives my-project as the root dir. This version does not catch defrecord etc.

How to include all files inside a folder using clojure

How to include all files inside a folder using clojure.
Here is my code:
(defn LoadFiles[]
(include "utils")
)
(LoadFiles)
But the above code is not working.
As far as i know, there's no include in clojure (correct me if i'm wrong).
You should use use or require for that.
This one should probably work (for all the .clj files in utils top level, but you can easily extend it to be recursive):
(defn list-sources [path]
(map #(str path "." (second (re-matches #"^(\w+)\.clj$" (.getName %))))
(filter #(.isFile %) (file-seq path))))
(run! #(require (vector (symbol %) :refer :all))
(list-sources (java.io.File. "utils")))
Maybe something like:
(defn load-files [dir]
(doseq [f (file-seq (File. dir))
:when (.isFile f)]
(load-file (.getAbsolutePath f))))
(load-files "utils")

My logging (with robert-hooke) does not work properly with tools.namespace/refresh, why?

EDIT: Turned out I was using require instead of :require in the namespace declaration. With :require, tools.namespace refreshes the logging namespace, and the problem goes away. I still find it curious, however, that the expression (eval `(var ~(symbol "A/func"))) does not work in the situation described below (that is, if B below is not refreshed).
Summary: I'm using tools.namespace. If I have namespaces A and B, and in B do (eval `(var ~(symbol "A/func"))), and (tools.namespace/refresh) and run the code, that works. But if I make a change to A, do (tools.namespace/refresh), so that only A refreshes, then running that expression gives the error: Cannot resolve var: A/func in this context, even though A/func exists. Why?
Longer version:
In my project, I have a logging module/namespace that uses robert-hooke (see below). I'm using tools.namespace to reload my code when I make changes.
The problem is the following: When I want to log (my logging currently just prints) something, I list the functions that I want to log in my logging namespace and do (t.n/refresh). That works. But if I make changes to the the namespaces that contain the functions that I want to log, and do (t.n/refresh) without making changes to the logging namespace, the logging no longer works (for the functions that have been refreshed). As soon as I make a change to logging, so that it too is refreshed by tools.namespace, it starts working again.
So, it's like the vars in namespaces that have been refreshed don't properly get their logging hooks. But I don't understand why.
Below is my logging namespace. I call add-logging-wrappers each time I run my program.
If I add (eval `(var ~(symbol "sv/register-damage"))) inside add-logging-wrappers, that's fine when logging has just been refreshed and the logging works. But those times the logging does not work, that expression causes the error Cannot resolve var: sv/register-damage in this context.
(ns game.logging
(require [robert.hooke :as rh]
[clojure.pprint :as pp]
[game.server.core :as sv]
[game.client.core :as cl]
[game.math :as math]
(game.common [core-functions :as ccfns]
[graphics :as gfx])
(game.server [pathfinding :as pf]))
(:use [game.utils]))
(defn log-println [name type object]
(println (str (current-thread-name) " // " name " " type ":\n"
(with-out-str
(pp/pprint object)))))
(defn print-output [name f & args]
(let [result (apply f args)]
(log-println name "output" result)
result))
(defn print-input [name f & args]
(log-println name "input" args)
(apply f args))
(defn print-call [name f & args]
(println (str (current-thread-name) "//" name))
(apply f args))
(defmacro make-name-var-list [fn-list]
`[~#(for [fn fn-list]
[(str fn) `(var ~fn)])])
(defmacro defloglist [name & fns]
`(def ~name (make-name-var-list [~#fns])))
(defn add-hooks [name-vars & wrappers]
(when (seq wrappers)
(doseq [[name var] name-vars]
(rh/add-hook var (partial (first wrappers) name)))
(recur name-vars (next wrappers))))
(defn get-ns-name-vars [ns-sym]
(-> (the-ns ns-sym) (#(.name %)) ns-interns))
(defn add-hooks-to-ns [ns-sym & wrappers]
(apply add-hooks (get-ns-name-vars ns-sym) wrappers))
(defloglist log-both
sv/distribute-exp ;; <--- things to log
sv/register-damage
sv/give-exp)
(defloglist log-input)
(defloglist log-output)
(defn add-logging-wrappers []
(dorun (->> (all-ns) (map #(.name %)) (mapcat ns-interns) (map second)
(map rh/clear-hooks)))
(add-hooks log-both print-output print-input)
(add-hooks log-input print-input)
(add-hooks log-output print-output))

how to load resources from a specific .jar file using clojure.java.io

In clojure.java.io, there is a io/resource function but I think it just loads the resource of the current jar that is running. Is there a way to specify the .jar file that the resource is in?
For example:
I have a jar file: /path/to/abc.jar
abc.jar when unzipped contains some/text/output.txt in the root of the unzipped directory
output.txt contains the string "The required text that I want."
I need functions that can do these operations:
(list-jar "/path/to/abc.jar" "some/text/")
;; => "output.txt"
(read-from-jar "/path/to/abc.jar" "some/text/output.txt")
;; => "The required text that I want"
Thanks in advance!
From Ankur's comments, I managed to piece together the functions that I needed:
The java.util.jar.JarFile object does the job.
you can call the method (.entries (Jarfile. a-path)) to give the list of files but instead of returning a tree structure:
i.e:
/dir-1
/file-1
/file-2
/dir-2
/file-3
/dir-3
/file-4
it returns an enumeration of filenames:
/dir-1/file-1, /dir-1/file-2, /dir-1/dir-2/file-3, /dir-1/dir-3/file-4
The following functions I needed are defined below:
(import java.util.jar.JarFile)
(defn list-jar [jar-path inner-dir]
(if-let [jar (JarFile. jar-path)]
(let [inner-dir (if (and (not= "" inner-dir) (not= "/" (last inner-dir)))
(str inner-dir "/")
inner-dir)
entries (enumeration-seq (.entries jar))
names (map (fn [x] (.getName x)) entries)
snames (filter (fn [x] (= 0 (.indexOf x inner-dir))) names)
fsnames (map #(subs % (count inner-dir)) snames)]
fsnames)))
(defn read-from-jar [jar-path inner-path]
(if-let [jar (JarFile. jar-path)]
(if-let [entry (.getJarEntry jar inner-path)]
(slurp (.getInputStream jar entry)))))
Usage:
(read-from-jar "/Users/Chris/.m2/repository/lein-newnew/lein-newnew/0.3.5/lein-newnew-0.3.5.jar"
"leiningen/new.clj")
;=> "The list of built-in templates can be shown with `lein help new`....."
(list-jar "/Users/Chris/.m2/repository/lein-newnew/lein-newnew/0.3.5/lein-newnew-0.3.5.jar" "leiningen")
;; => (new/app/core.clj new/app/project.clj .....)

How to expand a sequence (var-args) into distinct items

I want to send var-args of a function to a macro, still as var-args.
Here is my code:
(defmacro test-macro
[& args]
`(println (str "count=" ~(count args) "; args=" ~#args)))
(defn test-fn-calling-macro
[& args]
(test-macro args))
The output of (test-macro "a" "b" "c") is what I want: count=3; args=abc
The output of (test-fn-calling-macro "a" "b" "c") is : count=1; args=("a" "b" "c") because args is sent as a single argument to the macro. How can I expand this args in my function in order to call the macro with the 3 arguments?
I guess I'm just missing a simple core function but I'm not able to find it. Thanks
EDIT 2 - My "real" code, shown in EDIT section below is not a valid situation to use this technique.
As pointed out by #Brian, the macro xml-to-cass can be replaced with a function like this:
(defn xml-to-cass
[zipper table key attr & path]
(doseq [v (apply zf/xml-> zipper path)] (cass/set-attr! table key attr v)))
EDIT - the following section goes beyond my original question but any insight is welcome
The code above is just the most simple I could come with to pinpoint my problem. My real code deals with clj-cassandra and zip-filter. It may also look over-engineering but it's just a toy project and I'm trying to learn the language at the same time.
I want to parse some XML found on mlb.com and insert values found into a cassandra database. Here is my code and the thinking behind it.
Step 1 - Function which works fine but contains code duplication
(ns stats.importer
(:require
[clojure.xml :as xml]
[clojure.zip :as zip]
[clojure.contrib.zip-filter.xml :as zf]
[cassandra.client :as cass]))
(def root-url "http://gd2.mlb.com/components/game/mlb/year_2010/month_05/day_01/")
(def games-table (cass/mk-cf-spec "localhost" 9160 "mlb-stats" "games"))
(defn import-game-xml-1
"Import the content of xml into cassandra"
[game-dir]
(let [url (str root-url game-dir "game.xml")
zipper (zip/xml-zip (xml/parse url))
game-id (.substring game-dir 4 (- (.length game-dir) 1))]
(doseq [v (zf/xml-> zipper (zf/attr :type))] (cass/set-attr! games-table game-id :type v))
(doseq [v (zf/xml-> zipper (zf/attr :local_game_time))] (cass/set-attr! games-table game-id :local_game_time v))
(doseq [v (zf/xml-> zipper :team [(zf/attr= :type "home")] (zf/attr :name_full))] (cass/set-attr! games-table game-id :home_team v))))
The parameter to import-game-xml-1 can be for example "gid_2010_05_01_colmlb_sfnmlb_1/". I remove the "gid_" and the trailing slash to make it the key of the ColumnFamily games in my database.
I found that the 3 doseq were a lot of duplication (and there should be more than 3 in the final version). So code templating using a macro seemed appropriate here (correct me if I'm wrong).
Step 2 - Introducing a macro for code templating (still works)
(defmacro xml-to-cass
[zipper table key attr & path]
`(doseq [v# (zf/xml-> ~zipper ~#path)] (cass/set-attr! ~table ~key ~attr v#)))
(defn import-game-xml-2
"Import the content of xml into cassandra"
[game-dir]
(let [url (str root-url game-dir "game.xml")
zipper (zip/xml-zip (xml/parse url))
game-id (.substring game-dir 4 (- (.length game-dir) 1))]
(xml-to-cass zipper games-table game-id :type (zf/attr :type))
(xml-to-cass zipper games-table game-id :local_game_time (zf/attr :local_game_time))
(xml-to-cass zipper games-table game-id :home_team :team [(zf/attr= :type "home")] (zf/attr :name_full))))
I believe that's an improvement but I still see some duplication in always reusing the same 3 parameters in my calls to xml-to-cass. That's were I introduced an intermediate function to take care of those.
Step 3 - Adding a function to call the macro (the problem is here)
(defn import-game-xml-3
"Import the content of xml into cassandra"
[game-dir]
(let [url (str root-url game-dir "game.xml")
zipper (zip/xml-zip (xml/parse url))
game-id (.substring game-dir 4 (- (.length game-dir) 1))
save-game-attr (fn[key path] (xml-to-cass zipper games-table game-id key path))]
(save-game-attr :type (zf/attr :type)) ; works well because path has only one element
(save-game-attr :local_game_time (zf/attr :local_game_time))
(save-game-attr :home :team [(zf/attr= :type "home"] (zf/attr :name_full))))) ; FIXME this final line doesn't work
Here's a some simple code which may be illuminating.
Macros are about code generation. If you want that to happen at runtime, for some reason, then you have to build and evaluate the code at runtime. This can be a powerful technique.
(defmacro test-macro
[& args]
`(println (str "count=" ~(count args) "; args=" ~#args)))
(defn test-fn-calling-macro
[& args]
(test-macro args))
(defn test-fn-expanding-macro-at-runtime
[& args]
(eval (cons `test-macro args)))
(defmacro test-macro-expanding-macro-at-compile-time
[& args]
(cons `test-macro args))
;; using the splicing notation
(defmacro test-macro-expanding-macro-at-compile-time-2
[& args]
`(test-macro ~#args))
(defn test-fn-expanding-macro-at-runtime-2
[& args]
(eval `(test-macro ~#args)))
(test-macro "a" "b" "c") ;; count=3; args=abc nil
(test-fn-calling-macro "a" "b" "c") ;; count=1; args=("a" "b" "c") nil
(test-fn-expanding-macro-at-runtime "a" "b" "c") ; count=3; args=abc nil
(test-macro-expanding-macro-at-compile-time "a" "b" "c") ; count=3; args=abc nil
(test-macro-expanding-macro-at-compile-time-2 "a" "b" "c") ; count=3; args=abc nil
(test-fn-expanding-macro-at-runtime "a" "b" "c") ; count=3; args=abc nil
If contemplation of the above doesn't prove enlightening, might I suggest a couple of my own blog articles?
In this one I go through macros from scratch, and how clojure's work in particular:
http://www.learningclojure.com/2010/09/clojure-macro-tutorial-part-i-getting.html
And in this one I show why run-time code generation might be useful:
http://www.learningclojure.com/2010/09/clojure-faster-than-machine-code.html
The typical way to use a collection as individual arguments to a function is to use (apply function my-list-o-args)
(defn test-not-a-macro [& args]
(print args))
(defn calls-the-not-a-macro [& args]
(apply test-not-a-macro args))
though you wont be able to use apply because test-macro is a macro. to solve this problem you will need to wrap test macro in a function call so you can apply on it.
(defmacro test-macro [& args]
`(println ~#args))
(defn calls-test-macro [& args]
(eval (concat '(test-macro) (args)))) ;you almost never need eval.
(defn calls-calls-test-macro [& args]
(calls-test-macro args))
This is actually a really good example of one of the ways macros are hard to compose. (some would say they cant be composed cleanly, though i think thats an exageration)
Macros are not magic. They are a mechanism to convert code at compile-time to equivalent code; they are not used at run-time. The pain you are feeling is because you are trying to do something you should not be trying to do.
I don't know the library in question, but if cass/set-attr! is a function, I see no reason why the macro you defined has to be a macro; it could be a function instead. You can do what you want to do if you can rewrite your macro as a function instead.
Your requirements aren't clear. I don't see why a macro is necessary here for test-macro, unless you're trying to print the unevaluated forms supplied to your macro.
These functions provide your expected results, but that's because your sample data was self-evaluating.
(defn test-args
[& args]
(println (format "count=%d; args=%s"
(count args)
(apply str args))))
or
(defn test-args
[& args]
(print (format "count=%d; args=" (count args)))
(doseq [a args]
(pr a))
(newline))
You can imagine other variations to get to the same result.
Try calling that function with something that doesn't evaluate to itself, and note the result:
(test-args (+ 1 2) (+ 3 4))
Were you looking to see the arguments printed as "37" or "(+ 1 2)(+ 3 4)"?
If you were instead trying to learn about macros and their expansion in general, as opposed to solving this particular problem, please tune your question to probe further.