rename data loader fxns. other minor tweaks

TODO.md

@@ -0,0 +1 @@
+ - terminal flag to read all input files into the same structure

bin/cindex.lisp

@@ -75,7 +75,7 @@
 
 (defmacro cqry (co &rest rest) (proc-cqry co rest))
 
-(defun allsym (o) (uniq (flatall* o 2)) )
+(defun allsym (o) (uniq (flatall* o 2)))
 (defun prt (o &optional s)
   (lqn::with-struct (code- i grp) o
     (format s "<@code (frags: ~a~%  ~a)>" i grp)))
@@ -158,7 +158,7 @@
       (let ((sysi (reg co sys)))
         (loop with fi = (-reg-file sysi)
               with pkgi = (fnd co :/ext/pkg/none)
-              for o across (lqn::read-file-as-data-vector fn)
+              for o across (lqn:dat-read-file fn)
               if (and (consp o) (ssym? (car o))
                       (eq :in-package (lqn::kw (car o))))
               do (setf pkgi (attach co (fnd co :/ext/pkg) (second o) '(:/ext/pkg)))

bin/lqn-sh.lisp

@@ -7,10 +7,14 @@
 
 
 (defun lqn/read-from-file (f) (declare #.*opt*)
-  (handler-case (read-file-as-data-vector f)
+  (handler-case (dat-read-file f)
     (error (e) (sh/exit-msg 55 "LDN: failed to READ file: ~a~%~%~a~&" f e))))
+; (defun lqn/read-from-files (f) (declare #.*opt*)
+;   (handler-case (dat-read-file f)
+;     (error (e) (sh/exit-msg 55 "LDN: failed to READ file: ~a~%~%~a~&" f e))))
+
 (defun lqn/read-from-pipe () (declare #.*opt*)
-  (handler-case (read-stream-as-data-vector *standard-input*)
+  (handler-case (dat-read-stream *standard-input*)
     (error (e) (sh/exit-msg 55 "LDN: failed to READ from pipe:~%~%~a~&" e))))
 
 (defun lqn/run-files (opts fx files)

bin/redacted.lisp

@@ -9,7 +9,7 @@
   (let ((chars (lst!? "abcdefghijklmnopqrstuvwxyz")))
     (qry (ls "../src/*.lisp") str!
          #((progn (out "~%~3,'0d: ** ~a **~%~%" (cnt) (seq* _ 8)) _)
-           read-file-as-vector
+           txt-read-file
            #((repl _ "defun" "DEFUN")
              (repl _ "defmacro" "DEFMACRO")
              #((if (member _ chars) #\― _))

bin/tqn-sh.lisp

@@ -6,10 +6,10 @@
 (in-package :lqn)
 
 (defun tqn/read-from-file (f) (declare #.*opt*)
-  (handler-case (read-file-as-vector f)
+  (handler-case (txt-read-file f)
     (error (e) (sh/exit-msg 55 "TXT: failed to READ file: ~a~%~%~a" f e))))
 (defun tqn/read-from-pipe () (declare #.*opt*)
-  (handler-case (read-stream-lines-as-vector)
+  (handler-case (txt-read-stream)
     (error (e) (sh/exit-msg 55 "TXT: failed to READ from pipe:~%~%~a" e))))
 
 (defun tqn/run-files (opts fx files)

docs/lqn.md

@@ -223,6 +223,48 @@
  ;   Source file: /data/x/lqn/src/basic-utils.lisp
 ```
 
+#### LQN:DAT-READ-FILE
+
+```
+ ; LQN:DAT-READ-FILE
+ ;   [symbol]
+ ; 
+ ; DAT-READ-FILE names a compiled function:
+ ;   Lambda-list: (FN &AUX (RES (MAV)))
+ ;   Derived type: (FUNCTION (T) (VALUES T &OPTIONAL))
+ ;   Documentation:
+ ;     read lisp data from file into vector.
+ ;   Source file: /data/x/lqn/src/io.lisp
+```
+
+#### LQN:DAT-READ-FILES
+
+```
+ ; LQN:DAT-READ-FILES
+ ;   [symbol]
+ ; 
+ ; DAT-READ-FILES names a compiled function:
+ ;   Lambda-list: (PATH-OR-SEQ)
+ ;   Derived type: (FUNCTION ((OR LIST VECTOR)) (VALUES T &OPTIONAL))
+ ;   Documentation:
+ ;     read lisp data from these paths (via lqn:ls) or this list of files as one large vector.
+ ;   Source file: /data/x/lqn/src/qry-extra.lisp
+```
+
+#### LQN:DAT-READ-STREAM
+
+```
+ ; LQN:DAT-READ-STREAM
+ ;   [symbol]
+ ; 
+ ; DAT-READ-STREAM names a compiled function:
+ ;   Lambda-list: (S &AUX (RES (MAV)))
+ ;   Derived type: (FUNCTION (T) (VALUES T &OPTIONAL))
+ ;   Documentation:
+ ;     read lisp data from stream into vector.
+ ;   Source file: /data/x/lqn/src/io.lisp
+```
+
 #### LQN:DIR?
 
 ```
@@ -350,6 +392,21 @@
  ;   Source file: /data/x/lqn/src/io.lisp
 ```
 
+#### LQN:GRP
+
+```
+:missing:todo:
+
+ ; LQN:GRP
+ ;   [symbol]
+ ; 
+ ; GRP names a compiled function:
+ ;   Lambda-list: (V KEYFX &OPTIONAL (VALFX (FUNCTION IDENTITY)))
+ ;   Derived type: (FUNCTION (SEQUENCE FUNCTION &OPTIONAL FUNCTION)
+ ;                  (VALUES HASH-TABLE &OPTIONAL))
+ ;   Source file: /data/x/lqn/src/qry-utils.lisp
+```
+
 #### LQN:HEAD\*
 
 ```
@@ -606,6 +663,20 @@
  ;   Source file: /data/x/lqn/src/basic-utils.lisp
 ```
 
+#### LQN:LDNLOAD
+
+```
+ ; LQN:LDNLOAD
+ ;   [symbol]
+ ; 
+ ; LDNLOAD names a compiled function:
+ ;   Lambda-list: (O)
+ ;   Derived type: (FUNCTION (T) (VALUES T &OPTIONAL))
+ ;   Documentation:
+ ;     reverse of ldnout.
+ ;   Source file: /data/x/lqn/src/io.lisp
+```
+
 #### LQN:LDNOUT
 
 ```
@@ -616,7 +687,7 @@
  ;   Lambda-list: (O)
  ;   Derived type: (FUNCTION (T) (VALUES T &OPTIONAL))
  ;   Documentation:
- ;     serialize internal representation to readable lisp data.
+ ;     serialize internal representation to readable lisp data. see ldnload.
  ;   Source file: /data/x/lqn/src/io.lisp
 ```
 
@@ -1362,6 +1433,35 @@
  ;   Source file: /data/x/lqn/src/qry-utils.lisp
 ```
 
+#### LQN:TXT-READ-FILE
+
+```
+ ; LQN:TXT-READ-FILE
+ ;   [symbol]
+ ; 
+ ; TXT-READ-FILE names a compiled function:
+ ;   Lambda-list: (FN &AUX (RES (MAKE-ADJUSTABLE-VECTOR)))
+ ;   Derived type: (FUNCTION (T) (VALUES T &OPTIONAL))
+ ;   Documentation:
+ ;     read lines of text from file into vector.
+ ;   Source file: /data/x/lqn/src/io.lisp
+```
+
+#### LQN:TXT-READ-STREAM
+
+```
+ ; LQN:TXT-READ-STREAM
+ ;   [symbol]
+ ; 
+ ; TXT-READ-STREAM names a compiled function:
+ ;   Lambda-list: (&OPTIONAL (S *STANDARD-INPUT*) &AUX
+ ;                 (RES (MAKE-ADJUSTABLE-VECTOR)))
+ ;   Derived type: (FUNCTION (&OPTIONAL T) (VALUES T &OPTIONAL))
+ ;   Documentation:
+ ;     read lines of text from stream into vector.
+ ;   Source file: /data/x/lqn/src/io.lisp
+```
+
 #### LQN:UNIQ
 
 ```

lqn.asd

@@ -1,6 +1,6 @@
 (asdf:defsystem #:lqn
   :description "Lisp Query Notation"
-  :version "1.11.0"
+  :version "1.12.0"
   :author "anders hoff / @inconvergent / [email protected]"
   :in-order-to ((asdf:test-op (asdf:test-op #:lqn/tests)))
   :licence "MIT" :pathname "src/" :serial nil
@@ -13,11 +13,13 @@
                (:file "io" :depends-on ("docs"))
                (:file "pre-qry" :depends-on ("io" "qry-utils" "docs"))
                (:file "qry" :depends-on ("pre-qry"))
-               (:file "sh" :depends-on ("qry"))))
+               (:file "sh" :depends-on ("qry"))
+               (:file "qry-extra" :depends-on ("qry"))
+               ))
 
 (asdf:defsystem #:lqn/tests
   :depends-on (#:lqn #:prove #:uiop #:asdf)
-  :version "1.11.0"
+  :version "1.12.0"
   :perform (asdf:test-op (o s) (uiop:symbol-call ':lqn-tests '#:run-tests))
   :pathname "test/" :serial t
   :components ((:file "run")))

src/basic-utils.lisp

@@ -4,7 +4,7 @@
 
 (defvar *qmodes* '(:+ :? :- :%))
 (defvar *operators*
-  `(:*map :@ :|| ?rec :*$ :$$ :$* :** :*fld :?* :?xpr :?txpr :?mxpr :?srch))
+  `(:*map :@ :|| ?rec :*$ :$$ :$* :** :*fld :?* :?xpr :?txpr :?mxpr :?srch :?grp))
 (defvar *opt* '(optimize (speed 3) (safety 1)))
 (defvar *fxns* '(:err :wrn :nope :noop :lst :lit :qt :hld :ghv :pnum :inum :cnt :λ
                  :fmt :out :jsnstr
@@ -16,7 +16,9 @@
                  :msym? :is? :kv? kw? :sym? :ssym? :sym! :trim
                  :num!? :num? :flt!? :flt? :int!? :int?
                  :lst? :lst! :lst!? :seq? :seq!? :str! :str? :str!? :vec! :vec? :vec!?
-                 :path? :subdir :subfiles :ls :dir? :file? :cwd :now :cmd :some? :all? :none? :cd))
+                 :path? :subdir :subfiles :ls :dir? :file? :cwd :now :cmd :some? :all? :none? :cd
+                 :grp
+                 ))
 (defun cmd-args ()
   (or #+SBCL sb-ext:*posix-argv* #+LISPWORKS system:*line-arguments-list*
       #+CMU extensions:*command-line-words* nil))
@@ -99,18 +101,17 @@
        (> (length (symbol-name d)) (length m))))
 
 ; IS TYPE?
-(defun flt?  (f &optional d) "f if float; or d"            (if (floatp f) f d))
-(defun int?  (i &optional d) "i if int; or d"              (if (integerp i) i d))
-(defun kv?   (k &optional d) "k if kv; or d"               (if (hash-table-p k) k d))
-(defun kw?   (k &optional d) "k if kw; or d"               (if (keywordp k) k d))
-(defun sym?  (s &optional d) "s if sym; or d"              (if (symbolp s) s d))
-(defun ssym? (s &optional d) "s if sym, not kw; or d"      (if (and (sym? s)
-                                                                    (not (kw? s))) s d))
-(defun num?  (n &optional d) "n if number; or d"           (if (numberp n) n d))
-(defun str?  (s &optional d) "s if string; or d"           (if (stringp s) s d))
-(defun vec?  (v &optional d) "v if vector; or d"           (if (vectorp v) v d))
-(defun lst?  (v &optional d) "v if list; or d"             (if (listp v) d d))
-(defun seq?  (s &optional d) "s if sequence; or d"         (or (lst? s) (str? s) (vec? s) d))
+(defun flt?  (f &optional d) "f if float; or d"       (if (floatp f) f d))
+(defun int?  (i &optional d) "i if int; or d"         (if (integerp i) i d))
+(defun kv?   (k &optional d) "k if kv; or d"          (if (hash-table-p k) k d))
+(defun kw?   (k &optional d) "k if kw; or d"          (if (keywordp k) k d))
+(defun sym?  (s &optional d) "s if sym; or d"         (if (symbolp s) s d))
+(defun ssym? (s &optional d) "s if sym, not kw; or d" (if (and (sym? s) (not (kw? s))) s d))
+(defun num?  (n &optional d) "n if number; or d"      (if (numberp n) n d))
+(defun str?  (s &optional d) "s if string; or d"      (if (stringp s) s d))
+(defun vec?  (v &optional d) "v if vector; or d"      (if (vectorp v) v d))
+(defun lst?  (v &optional d) "v if list; or d"        (if (listp v) d d))
+(defun seq?  (s &optional d) "s if sequence; or d"    (or (lst? s) (str? s) (vec? s) d))
 
 ; PARSE AS TYPE OR DEFAULT
 (defun int!? (i &optional d) "i as int if it can be parsed; or d"

src/io.lisp

@@ -1,34 +1,33 @@
 (in-package #:lqn)
 ; YASON DOCS https://phmarek.github.io/yason/
+; (setf (readtable-case *readtable*) :preserve)
 
 (defun read-all-str (s &aux (pos 0))
   (declare #.*opt*)
   (loop for (l new-pos) = (mvl (read-from-string s nil 'lqn::eof :start pos))
         while (not (eq l 'lqn::eof)) do (setf pos new-pos) collect l))
 
-(defun read-stream-lines-as-vector (&optional (s *standard-input*)
-                                    &aux (res (make-adjustable-vector)))
-  (declare #.*opt*)
+(defun txt-read-stream (&optional (s *standard-input*) &aux (res (make-adjustable-vector)))
+  (declare #.*opt*) "read lines of text from stream into vector."
   (loop for l = (read-line s nil nil) while l do (vex res l))
   res)
-(defun read-file-as-vector (fn &aux (res (make-adjustable-vector)))
-  (declare #.*opt*)
+(defun txt-read-file (fn &aux (res (make-adjustable-vector)))
+  (declare #.*opt*) "read lines of text from file into vector."
   (with-open-file (in fn)
     (loop for l = (read-line in nil nil) while l do (vex res l)))
   res)
-; (setf (readtable-case *readtable*) :preserve)
-(defun read-file-as-data-vector (fn &aux (res (make-adjustable-vector)))
-  (declare #.*opt*)
+(defun dat-read-file (fn &aux (res (mav)))
+  (declare #.*opt*) "read lisp data from file into vector."
   (with-open-file (in fn)
     (loop for l = (read in nil nil) while l do (vex res l)))
   res)
-(defun read-stream-as-data-vector (s &aux (res (make-adjustable-vector)))
-  (declare #.*opt*)
+(defun dat-read-stream (s &aux (res (mav)))
+  (declare #.*opt*) "read lisp data from stream into vector."
   (loop for l = (read s nil nil) while l do (vex res l))
   res)
 
 (defun jsnloads (&optional (s *standard-input*) all)
-  (declare #.*opt* ) "parse json from stream; or *standard-input*"
+  (declare #.*opt*) "parse json from stream; or *standard-input*"
   (let ((yason:*parse-json-arrays-as-vectors* t))
     (if all (let ((res (mav)))
               (handler-case
@@ -56,16 +55,27 @@
   (jsnout o :s s :indent indent)
   (get-output-stream-string s))
 
-(defun ldnout (o)
-  "serialize internal representation to readable lisp data."
-  (typecase o (string o)
+(defun ldnout (o) "serialize internal representation to readable lisp data. see ldnload."
+  (labels ((make-key (k)
+             (typecase k (number k) (string (kw k)) (sequence k)
+                         (character k)
+                         (otherwise (kw k)))))
+   (typecase o (string o)
      (cons (cons (ldnout (car o)) (ldnout (cdr o))))
      (hash-table (loop for k being the hash-keys of o using (hash-value v)
-                       collect `(,(kw k) . ,(ldnout v))))
+                       collect `(,(make-key k) . ,(ldnout v))))
      (vector (loop with res = (make-adjustable-vector)
                    for v across o do (vex res (ldnout v))
                    finally (return res)))
-     (otherwise o)))
+     (otherwise o))))
+(defun ldnload (o) "reverse of ldnout."
+  (typecase o (string o)
+              (vector (map 'vector #'ldnload o))
+              (list (loop with res = (make-hash-table :test #'equal)
+                          for (k . v) in o
+                          do (setf (gethash (str! k) res) (ldnload v))
+                          finally (return res)))
+              (otherwise o)))
 
 (defmacro out (s &rest rest) "print to standard out"
   (awg (s*) (if rest `(format *standard-output* ,s ,@rest)

src/packages.lisp

@@ -3,7 +3,7 @@
   (:nicknames #:cl-lqn)
   (:export #:d? #:v?
     #:qry #:qryd #:jsnqryf #:qryl #:proc-qry
-    #:jsnloads #:jsnloadf #:jsnout #:ldnout #:fmt #:out #:jsnstr #:@* #:@@ #:??
+    #:jsnloads #:jsnloadf #:jsnout #:ldnout #:ldnload #:fmt #:out #:jsnstr #:@* #:@@ #:??
     #:some? #:none? #:all? #:empty? #:size? #:is?
     #:path? #:subdir #:subfiles #:ls #:dir? #:file? #:cwd #:now #:cmd
     #:some? #:all? #:none? #:cd
@@ -17,5 +17,7 @@
     #:sup #:sdwn #:strcat #:repl #:splt #:symb #:join
     #:pref? #:suf? #:sub? #:subx? #:ipref? #:isuf? #:isub? #:isubx?
     #:srt #:lpad #:rpad #:nstr #:ascii #:bar #:clmp
+    #:dat-read-file #:dat-read-files #:dat-read-stream #:txt-read-file #:txt-read-stream
+    #:grp
     ))
 

src/qry-extra.lisp

@@ -0,0 +1,8 @@
+(in-package :lqn)
+
+(defun dat-read-files (path-or-seq)
+  (declare ((or string list vector) path-or-seq))
+  "read lisp data from these paths (via lqn:ls) or this list of files as one large vector."
+  (qry (etypecase path-or-seq (string (ls path-or-seq)) (sequence path-or-seq))
+       #((dat-read-file _)) (flatn* _)))
+

src/qry-utils.lisp

@@ -133,6 +133,15 @@ match. If b is an expression, a is compared to the evaluated value of b."
                  (otherwise o*))))
     (rec o)))
 
+(defun grp (v keyfx &optional (valfx #'identity))
+  (declare (sequence v) (function keyfx valfx))
+  (loop with res = (make-hash-table :test #'equal)
+        for o across (vec! v)
+        for k = (funcall keyfx o)
+        for acc = (gethash k res (new*))
+        do (setf (gethash k res) (psh* acc (funcall valfx o)))
+        finally (return res)))
+
 (defun @* (a d &rest rest &aux l) (declare #.*opt*)
   "pick these indices/keys from sequence/hash-table into new vector."
   (labels ((lt (l) (or (nth l a) d))