Refactor grammar and utilility functions into modules

doc/generic/pgf/extract.lua

@@ -1,170 +1,30 @@
--- luacheck:ignore 542
-local lfs = require("lfs")
-local lpeg = require("lpeg")
-local C, Cf, Cg, Ct, P, S, V = lpeg.C, lpeg.Cf, lpeg.Cg, lpeg.Ct, lpeg.P, lpeg.S, lpeg.V
-
--- strip leading and trailing whitespace
-local function strip(str)
-    return str:match"^%s*(.-)%s*$"
-end
--- strip braces
-local function strip_braces(str)
-    return str:match"^{?(.-)}?$"
-end
-
--- optional whitespace
-local ws = S" \t\n\r"^0
-
--- match string literal
-local function lit(str)
-    return ws * P(str) * ws
-end
-
--- setter for options table
-local invalid = string.char(0x8)
-local function set(t,k,v)
-    -- strip whitespace from keys
-    k = strip(k)
-    -- if the value is empty, set it to invalid character
-    v = v and strip_braces(v) or invalid
-    return rawset(t,k,v)
+local function pwd()
+    local info = debug.getinfo(1, "S")
+    local path = info.source:match("@(.*)")
+    local dir = path:match("(.*[/\\])") or "./"
+    return dir
 end
 
--- Grammar to extract code examples
-local extractor = lpeg.P{"document",
-    name =
-        C((1 - S",]=")^1),
-
-    pair =
-        Cg(V"name" * (lit"=" * (V"braces" + V"name"))^0) * lit","^-1,
-
-    list =
-        Cf(Ct"" * V"pair"^0, set),
-
-    balanced =
-        "{" * ((1 - S"{}") + V"balanced")^0 * "}",
-
-    braces =
-        C(V"balanced"),
-
-    optarg =
-        lit"[" * V"list" * lit"]",
-
-    begincodeexample =
-        P"\\begin{codeexample}" * V"optarg",
-
-    endcodeexample =
-        P"\\end{codeexample}",
-
-    content =
-        C((1 - V"endcodeexample")^0),
-
-    codeexample =
-        Ct(V"begincodeexample" * V"content" * V"endcodeexample"),
+package.path = pwd() .. "lib/?.lua;" .. package.path
+local utils = require "utils"
+local document_grammar = require "document_grammar"
 
-    anything =
-        (1 - V"codeexample")^0,
-
-    document =
-        V"anything" * Ct(V"codeexample" * (V"anything" * V"codeexample")^0) * V"anything"
-}
-
--- get the basename and extension of a file
-local function basename(file)
-    local name, ext = string.match(file, "^(.+)%.([^.]+)$")
-    return name or "",  ext or file
-end
-
-local pathsep = package.config:sub(1,1)
-
--- Walk the file tree
-local function walk(sourcedir, targetdir)
-    -- Make sure the arguments are directories
-    assert(lfs.attributes(sourcedir, "mode") == "directory", sourcedir .. " is not a directory")
-    assert(lfs.attributes(targetdir, "mode") == "directory", targetdir .. " is not a directory")
-
-    -- Append the path separator if necessary
-    if sourcedir:sub(-1, -1) ~= pathsep then
-        sourcedir = sourcedir .. pathsep
-    end
-    if targetdir:sub(-1, -1) ~= pathsep then
-        targetdir = targetdir .. pathsep
-    end
-
-    -- Process all items in the directory
-    for file in lfs.dir(sourcedir) do
-        if file == "." or file == ".." then
-            -- Ignore these two special ones
-        elseif lfs.attributes(sourcedir .. file, "mode") == "directory" then
-            -- Recurse into subdirectories
-            lfs.mkdir(targetdir .. file)
-            walk(sourcedir .. file .. pathsep, targetdir .. file .. pathsep)
-        elseif lfs.attributes(sourcedir .. file, "mode") == "file" then
-            print("Processing " .. sourcedir .. file)
-
-            -- Read file into memory
-            local f = io.open(sourcedir .. file)
-            local text = f:read("*all")
-            f:close()
-            local name, _ = basename(file)
-
-            -- preprocess, strip all commented lines
-            text = text:gsub("\n%%[^\n]*","")
-
-            -- extract all code examples
-            local matches = extractor:match(text) or {}
-
-            -- write code examples to separate files
-            local setup_code = ""
-            for n, e in ipairs(matches) do
-                local options = e[1]
-                local content = e[2]
-                if content:match("remember picture") then
-                    -- skip
-                elseif options["setup code"] then
-                -- If the snippet is marked as setup code, we have to put it before
-                -- every other snippet in the same file
-                -- if options["setup code"] then
-                    setup_code = setup_code .. strip(content) .. "\n"
-                elseif not options["code only"] and not options["setup code"] then
-                -- Skip those that say "code only" or "setup code"
-                -- if not options["code only"] and not options["setup code"] then
-                    local newname = name .. "-" .. n .. ".tex"
-                    local examplefile = io.open(targetdir .. newname, "w")
-
-                    examplefile:write"\\documentclass{standalone}\n"
-                    examplefile:write"\\usepackage{fp,pgf,tikz,xcolor}\n"
-                    examplefile:write(options["preamble"] and options["preamble"] .. "\n" or "")
-                    examplefile:write"\\begin{document}\n"
-
-                    examplefile:write(setup_code)
-                    local pre = options["pre"] or ""
-                    pre = pre:gsub("##", "#")
-                    examplefile:write(pre .. "\n")
-                    if options["render instead"] then
-                        examplefile:write(options["render instead"] .. "\n")
-                    else
-                        examplefile:write(strip(content) .. "\n")
-                    end
-                    examplefile:write(options["post"] and options["post"] .. "\n" or "")
-                    examplefile:write"\\end{document}\n"
-
-                    examplefile:close()
-                end
-            end
-        end
-    end
-end
+--[[
+    Sample Usage:
 
+    time texlua ~/github.com/pgf/doc/generic/pgf/extract.lua \
+                ~/github.com/pgf/tex \
+                ~/tmp/mwe
+]]
 -- Main loop
--- luacheck:ignore 113
+-- luacheck:ignore 113 (Accessing an undefined global variable: arg)
 if #arg < 2 then
     print("Usage: " .. arg[-1] .. " " .. arg[0] .. " <source-dirs...> <target-dir>")
     os.exit(1)
 end
 
 for n = 1, #arg - 1 do
-    walk(arg[n], arg[#arg])
+    utils.walk(arg[n], arg[#arg], document_grammar)
 end
 
-os.exit(0)
+os.exit(0)

doc/generic/pgf/lib/document_grammar.lua

@@ -0,0 +1,23 @@
+local lpeg = require("lpeg")
+local C, Cf, Cg, Ct, P, S, V = lpeg.C, lpeg.Cf, lpeg.Cg, lpeg.Ct, lpeg.P, lpeg.S, lpeg.V
+local u = require("utils")
+
+-- Grammar to extract code examples from document
+local grammar =
+    P {
+    "document",
+    name = C((1 - S ",]=") ^ 1),
+    pair = Cg(V "name" * (u.lit "=" * (V "braces" + V "name")) ^ 0) * u.lit "," ^ -1,
+    list = Cf(Ct "" * V "pair" ^ 0, u.set),
+    balanced = "{" * ((1 - S "{}") + V "balanced") ^ 0 * "}",
+    braces = C(V "balanced"),
+    optarg = u.lit "[" * V "list" * u.lit "]",
+    begincodeexample = P "\\begin{codeexample}" * V "optarg",
+    endcodeexample = P "\\end{codeexample}",
+    content = C((1 - V "endcodeexample") ^ 0),
+    codeexample = Ct(V "begincodeexample" * V "content" * V "endcodeexample"),
+    anything = (1 - V "codeexample") ^ 0,
+    document = V "anything" * Ct(V "codeexample" * (V "anything" * V "codeexample") ^ 0) * V "anything"
+}
+
+return grammar

doc/generic/pgf/lib/utils.lua

@@ -0,0 +1,123 @@
+local lfs = require "lfs"
+local lpeg = require "lpeg"
+
+-- luacheck:ignore 542 (Empty if branch.)
+local utils = {}
+local u = utils
+
+utils.pathsep = package.config:sub(1, 1)
+
+-- strip leading and trailing whitespace
+function u.strip(str)
+    return str:match "^%s*(.-)%s*$"
+end
+-- strip braces
+function u.strip_braces(str)
+    return str:match "^{?(.-)}?$"
+end
+
+-- optional whitespace
+u.ws = lpeg.S " \t\n\r" ^ 0
+
+-- match string literal
+function u.lit(str)
+    return u.ws * lpeg.P(str) * u.ws
+end
+
+-- setter for options table
+u.invalid = string.char(0x8)
+
+function u.set(t, k, v)
+    -- strip whitespace from keys
+    k = u.strip(k)
+    -- if the value is empty, set it to invalid character
+    v = v and u.strip_braces(v) or u.invalid
+    return rawset(t, k, v)
+end
+
+-- get the basename and extension of a file
+function u.basename(file)
+    local name, ext = string.match(file, "^(.+)%.([^.]+)$")
+    return name or "", ext or file
+end
+
+-- Walk the file tree
+function u.walk(sourcedir, targetdir, grammar)
+    -- Make sure the arguments are directories
+    assert(lfs.attributes(sourcedir, "mode") == "directory", sourcedir .. " is not a directory")
+    assert(lfs.attributes(targetdir, "mode") == "directory", targetdir .. " is not a directory")
+
+    -- Append the path separator if necessary
+    if sourcedir:sub(-1, -1) ~= u.pathsep then
+        sourcedir = sourcedir .. u.pathsep
+    end
+    if targetdir:sub(-1, -1) ~= u.pathsep then
+        targetdir = targetdir .. u.pathsep
+    end
+
+    -- Process all items in the directory
+    for file in lfs.dir(sourcedir) do
+        if file == "." or file == ".." then
+            -- Ignore these two special ones
+        elseif lfs.attributes(sourcedir .. file, "mode") == "directory" then
+            -- Recurse into subdirectories
+            lfs.mkdir(targetdir .. file)
+            u.walk(sourcedir .. file .. u.pathsep, targetdir .. file .. u.pathsep, grammar)
+        elseif lfs.attributes(sourcedir .. file, "mode") == "file" then
+            print("Processing " .. sourcedir .. file)
+
+            -- Read file into memory
+            local f = io.open(sourcedir .. file)
+            local text = f:read("*all")
+            f:close()
+            local name, _ = u.basename(file)
+
+            -- preprocess, strip all commented lines
+            text = text:gsub("\n%%[^\n]*", "")
+
+            -- extract all code examples
+            local matches = grammar:match(text) or {}
+
+            -- write code examples to separate files
+            local setup_code = ""
+            for n, e in ipairs(matches) do
+                local options = e[1]
+                local content = e[2]
+                if content:match("remember picture") then
+                    -- skip
+                elseif options["setup code"] then
+                    -- If the snippet is marked as setup code, we have to put it before
+                    -- every other snippet in the same file
+                    -- if options["setup code"] then
+                    setup_code = setup_code .. u.strip(content) .. "\n"
+                elseif not options["code only"] and not options["setup code"] then
+                    -- Skip those that say "code only" or "setup code"
+                    -- if not options["code only"] and not options["setup code"] then
+                    local newname = name .. "-" .. n .. ".tex"
+                    local examplefile = io.open(targetdir .. newname, "w")
+
+                    examplefile:write "\\documentclass{standalone}\n"
+                    examplefile:write "\\usepackage{fp,pgf,tikz,xcolor}\n"
+                    examplefile:write(options["preamble"] and options["preamble"] .. "\n" or "")
+                    examplefile:write "\\begin{document}\n"
+
+                    examplefile:write(setup_code)
+                    local pre = options["pre"] or ""
+                    pre = pre:gsub("##", "#")
+                    examplefile:write(pre .. "\n")
+                    if options["render instead"] then
+                        examplefile:write(options["render instead"] .. "\n")
+                    else
+                        examplefile:write(u.strip(content) .. "\n")
+                    end
+                    examplefile:write(options["post"] and options["post"] .. "\n" or "")
+                    examplefile:write "\\end{document}\n"
+
+                    examplefile:close()
+                end
+            end
+        end
+    end
+end
+
+return utils