lua-filters.md: autogenerate parts of the Lua API docs

4 files changed, 303 insertions, 104 deletions

diff --git a/Makefile b/Makefile
index 21f454fa4..fcd5e528d 100644
--- a/Makefile
+++ b/Makefile
@@ -175,10 +175,11 @@ README.md: README.template MANUAL.txt tools/update-readme.lua
 	      --reference-location=section -t gfm $< -o $@
 
 doc/lua-filters.md: tools/update-lua-module-docs.lua  ## update lua-filters.md module docs
-	cabal run pandoc -- --standalone \
+	cabal run pandoc-cli -- \
+		--standalone \
 		--reference-links \
-		--lua-filter=$< \
 		--columns=66 \
+		--from=$< \
 		--output=$@ \
 		$@
 .PHONY: doc/lua-filters.md
diff --git a/doc/lua-filters.md b/doc/lua-filters.md
index 37bc89d81..d3bea4cb4 100644
--- a/doc/lua-filters.md
+++ b/doc/lua-filters.md
@@ -3718,17 +3718,19 @@ Usage:
 
 [WriterOptions]: #type-writeroptions
 
+<!-- BEGIN: AUTOGENERATED CONTENT for module pandoc.cli -->
+
 # Module pandoc.cli
 
 Command line options and argument parsing.
 
 ## Fields {#pandoc.cli-fields}
 
-### default\_options {#pandoc.cli.default_options}
+### default_options {#pandoc.cli.default_options}
 
-Default CLI options, using a JSON-like representation (table).
+Default CLI options, using a JSON-like representation. (table)
 
-## Functions
+## Functions {#pandoc.cli-functions}
 
 ### parse_options {#pandoc.cli.parse_options}
 
@@ -3741,7 +3743,7 @@ the list of arguments from the global `arg`.
 Parameters:
 
 `args`
-:   list of command line arguments ({string,...})
+:   list of command line arguments ({string,\...})
 
 Returns:
 
@@ -3781,6 +3783,7 @@ Returns:
 The result(s) of the last evaluated input, or nothing if the last
 input resulted in an error.
 
+<!-- END: AUTOGENERATED CONTENT -->
 
 # Module pandoc.utils
 
@@ -4604,25 +4607,28 @@ Returns:
 
 -   JSON encoding of the given object (string)
 
+<!-- BEGIN: AUTOGENERATED CONTENT for module pandoc.path -->
 
 # Module pandoc.path
 
 Module for file path manipulations.
 
-## Static Fields {#pandoc.path-fields}
+## Fields {#pandoc.path-fields}
 
 ### separator {#pandoc.path.separator}
 
-The character that separates directories.
+The character that separates directories. (string)
 
 ### search_path_separator {#pandoc.path.search_path_separator}
 
 The character that is used to separate the entries in the `PATH`
-environment variable.
+environment variable. (string)
 
 ## Functions {#pandoc.path-functions}
 
-### directory (filepath) {#pandoc.path.directory}
+### directory {#pandoc.path.directory}
+
+`directory (filepath)`
 
 Gets the directory name, i.e., removes the last directory
 separator and everything after from the given path.
@@ -4636,7 +4642,9 @@ Returns:
 
 -   The filepath up to the last directory separator. (string)
 
-### filename (filepath) {#pandoc.path.filename}
+### filename {#pandoc.path.filename}
+
+`filename (filepath)`
 
 Get the file name.
 
@@ -4649,9 +4657,11 @@ Returns:
 
 -   File name part of the input path. (string)
 
-### is_absolute (filepath) {#pandoc.path.is_absolute}
+### is_absolute {#pandoc.path.is_absolute}
+
+`is_absolute (filepath)`
 
-Checks whether a path is absolute, i.e. not fixed to a root.
+Checks whether a path is absolute, i.e. not fixed to a root.
 
 Parameters:
 
@@ -4660,10 +4670,12 @@ Parameters:
 
 Returns:
 
--   `true` if `filepath` is an absolute path, `false` otherwise.
+-   `true` iff `filepath` is an absolute path, `false` otherwise.
     (boolean)
 
-### is_relative (filepath) {#pandoc.path.is_relative}
+### is_relative {#pandoc.path.is_relative}
+
+`is_relative (filepath)`
 
 Checks whether a path is relative or fixed to a root.
 
@@ -4674,31 +4686,32 @@ Parameters:
 
 Returns:
 
--   `true` if `filepath` is a relative path, `false` otherwise.
+-   `true` iff `filepath` is a relative path, `false` otherwise.
     (boolean)
 
-### join (filepaths) {#pandoc.path.join}
+### join {#pandoc.path.join}
+
+`join (filepaths)`
 
 Join path elements back together by the directory separator.
 
 Parameters:
 
 `filepaths`
-:   path components (list of strings)
+:   path components ({string,\...})
 
 Returns:
 
 -   The joined path. (string)
 
-### make_relative (path, root[, unsafe]) {#pandoc.path.make_relative}
+### make_relative {#pandoc.path.make_relative}
 
-Contract a filename, based on a relative path. Note that the
-resulting path will usually not introduce `..` paths, as the
-presence of symlinks means `../b` may not reach `a/b` if it starts
-from `a/c`. For a worked example see [this blog
-post](https://neilmitchell.blogspot.co.uk/2015/10/filepaths-are-subtle-symlinks-are-hard.html).
+`make_relative (path, root[, unsafe])`
 
-Set `unsafe` to a truthy value to allow `..` in paths.
+Contract a filename, based on a relative path. Note that the
+resulting path will never introduce `..` paths, as the presence of
+symlinks means `../b` may not reach `a/b` if it starts from `a/c`.
+For a worked example see [this blog post].
 
 Parameters:
 
@@ -4715,15 +4728,17 @@ Returns:
 
 -   contracted filename (string)
 
-### normalize (filepath) {#pandoc.path.normalize}
+### normalize {#pandoc.path.normalize}
+
+`normalize (filepath)`
 
 Normalizes a path.
 
 -   `//` makes sense only as part of a (Windows) network drive;
     elsewhere, multiple slashes are reduced to a single
     `path.separator` (platform dependent).
--   `/` becomes `path.separator` (platform dependent)
--   `./` -\> ''
+-   `/` becomes `path.separator` (platform dependent).
+-   `./` is removed.
 -   an empty path becomes `.`
 
 Parameters:
@@ -4735,7 +4750,9 @@ Returns:
 
 -   The normalized path. (string)
 
-### split (filepath) {#pandoc.path.split}
+### split {#pandoc.path.split}
+
+`split (filepath)`
 
 Splits a path by the directory separator.
 
@@ -4746,13 +4763,16 @@ Parameters:
 
 Returns:
 
--   List of all path components. (list of strings)
+-   List of all path components. ({string,\...})
 
-### split_extension (filepath) {#pandoc.path.split_extension}
+### split_extension {#pandoc.path.split_extension}
 
-Splits the last extension from a file path and returns the parts. The
-extension, if present, includes the leading separator; if the path has
-no extension, then the empty string is returned as the extension.
+`split_extension (filepath)`
+
+Splits the last extension from a file path and returns the parts.
+The extension, if present, includes the leading separator; if the
+path has no extension, then the empty string is returned as the
+extension.
 
 Parameters:
 
@@ -4762,14 +4782,15 @@ Parameters:
 Returns:
 
 -   filepath without extension (string)
-
 -   extension or empty string (string)
 
-### split_search_path (search_path) {#pandoc.path.split_search_path}
+### split_search_path {#pandoc.path.split_search_path}
+
+`split_search_path (search_path)`
 
-Takes a string and splits it on the `search_path_separator` character.
-Blank items are ignored on Windows, and converted to `.` on Posix. On
-Windows path elements are stripped of quotes.
+Takes a string and splits it on the `search_path_separator`
+character. Blank items are ignored on Windows, and converted to
+`.` on Posix. On Windows path elements are stripped of quotes.
 
 Parameters:
 
@@ -4778,7 +4799,16 @@ Parameters:
 
 Returns:
 
--   list of directories in search path (list of strings)
+-   list of directories in search path ({string,\...})
+
+### treat_strings_as_paths {#pandoc.path.treat_strings_as_paths}
+
+`treat_strings_as_paths ()`
+
+Augment the string module such that strings can be used as path
+objects.
+
+<!-- END: AUTOGENERATED CONTENT -->
 
 # Module pandoc.structure
 
@@ -5639,18 +5669,21 @@ Returns
 
 [Doc]: #type-doc
 
+<!-- BEGIN: AUTOGENERATED CONTENT for module pandoc.scaffolding -->
+
 # Module pandoc.scaffolding
 
 Scaffolding for custom writers.
 
-## Writer {#pandoc.scaffolding.writer}
+## Fields {#pandoc.scaffolding-fields}
 
-A structure to be used as a `Writer` function; the construct
-handles most of the boilerplate, expecting only render functions
-for all AST elements. See the documentation for custom writers for
-details.
+### Writer {#pandoc.scaffolding.Writer}
 
+An object to be used as a `Writer` function; the construct handles
+most of the boilerplate, expecting only render functions for all
+AST elements (table)
 
+<!-- END: AUTOGENERATED CONTENT -->
 
 # Module pandoc.template
 
@@ -5767,6 +5800,8 @@ Returns:
 
 -   A new [Version] object.
 
+<!-- BEGIN: AUTOGENERATED CONTENT for module pandoc.zip -->
+
 # Module pandoc.zip
 
 Functions to create, modify, and extract files from zip archives.
@@ -5801,7 +5836,7 @@ Parameters:
 
 `bytestring_or_entries`
 :   binary archive data or list of entries; defaults to an empty
-    list ([string]{.builtin-lua-type}\|{[zip.Entry],\...})
+    list (string\|{[zip.Entry],\...})
 
 Returns:
 
@@ -5817,10 +5852,10 @@ the file's modification time.
 Parameters:
 
 `path`
-:   file path in archive ([string]{.builtin-lua-type})
+:   file path in archive (string)
 
 `contents`
-:   uncompressed contents ([string]{.builtin-lua-type})
+:   uncompressed contents (string)
 
 `modtime`
 :   modification time ([integer]{unknown-type="integer"})
@@ -5838,10 +5873,10 @@ Generates a ZipEntry from a file or directory.
 Parameters:
 
 `filepath`
-:    ([string]{.builtin-lua-type})
+:    (string)
 
 `opts`
-:   zip options ([table]{.builtin-lua-type})
+:   zip options (table)
 
 Returns:
 
@@ -5857,10 +5892,10 @@ Parameters:
 
 `filepaths`
 :   list of files from which the archive is created.
-    ({[string]{.builtin-lua-type},\...})
+    ({string,\...})
 
 `opts`
-:   zip options ([table]{.builtin-lua-type})
+:   zip options (table)
 
 Returns:
 
@@ -5891,7 +5926,7 @@ Parameters:
 
 Returns:
 
--   bytes of the archive ([string]{.builtin-lua-type})
+-   bytes of the archive (string)
 
 ##### extract {#pandoc.zip.Archive.extract}
 
@@ -5908,7 +5943,7 @@ Parameters:
 :    ([zip.Archive])
 
 `opts`
-:   zip options ([table]{.builtin-lua-type})
+:   zip options (table)
 
 ### zip.Entry {#type-pandoc.zip.Entry}
 
@@ -5939,11 +5974,16 @@ Parameters:
 :    ([zip.Entry])
 
 `password`
-:   password for entry ([string]{.builtin-lua-type})
+:   password for entry (string)
 
 Returns:
 
--   binary contents ([string]{.builtin-lua-type})
+-   binary contents (string)
+
+<!-- END: AUTOGENERATED CONTENT -->
+
+<!-- BEGIN: GENERATED REFERENCE LINKS -->
 
+  [this blog post]: http://neilmitchell.blogspot.co.uk/2015/10/filepaths-are-subtle-symlinks-are-hard.html
   [zip.Entry]: #type-pandoc.zip.Entry
   [zip.Archive]: #type-pandoc.zip.Archive
diff --git a/pandoc-lua-engine/src/Text/Pandoc/Lua/Module/CLI.hs b/pandoc-lua-engine/src/Text/Pandoc/Lua/Module/CLI.hs
index 89c9741af..37aa719f7 100644
--- a/pandoc-lua-engine/src/Text/Pandoc/Lua/Module/CLI.hs
+++ b/pandoc-lua-engine/src/Text/Pandoc/Lua/Module/CLI.hs
@@ -79,7 +79,7 @@ repl = defun "repl"
           repl')
   <#> opt (parameter (typeChecked "table" istable pure) "table" "env"
            ("Extra environment; the global environment is merged into this" <>
-           "table."))
+           " table."))
   =?> T.unlines
       [ "The result(s) of the last evaluated input, or nothing if the last"
       , "input resulted in an error."
diff --git a/tools/update-lua-module-docs.lua b/tools/update-lua-module-docs.lua
index 9528e4663..f8531b5c5 100644
--- a/tools/update-lua-module-docs.lua
+++ b/tools/update-lua-module-docs.lua
@@ -1,23 +1,93 @@
-local ipairs, load, pairs, type = ipairs, load, pairs, type
-local debug, string, table = debug, string, table
-local _G = _G
+local ipairs, load, next, pairs, print, tostring, type =
+  ipairs, load, next, pairs, print, tostring, type
+local string, table = string, table
+local _G, arg = _G, arg
+
+local registry = debug.getregistry()
 
 _ENV = pandoc
 
 local stringify = utils.stringify
 
+--- Retrieves the documentation object for the given value.
+local function documentation (value)
+  return registry['HsLua docs'][value]
+end
+
+local function sorted (tbl)
+  local keys = {}
+  for key in pairs(tbl) do
+    table.insert(keys, key)
+  end
+  table.sort(keys)
+  local i = 0
+  local iter = function (state, ctrl)
+    if i > 0 and ctrl == nil then
+      return nil
+    else
+      i = i + 1
+      return keys[i], tbl[keys[i]]
+    end
+  end
+  return iter, nil, nil
+end
+
 local get = function (fieldname)
   return function (obj) return obj[fieldname] end
 end
 
 local function read_blocks (txt)
-  return read(txt, 'commonmark').blocks
+  return read(txt, 'commonmark+smart').blocks
 end
 
 local function read_inlines (txt)
   return utils.blocks_to_inlines(read_blocks(txt))
 end
 
+local known_types = {}
+
+local function render_typespec (typespec)
+  if typespec.basic then
+    return Inlines(Span(typespec.basic, {class="builtin-lua-type"}))
+  elseif typespec.named then
+    return Inlines(Span(typespec.named, {['unknown-type'] = typespec.named}))
+  elseif typespec.sequence then
+    local typeinlns = render_typespec(typespec.sequence)
+    return Inlines({'{'} .. typeinlns .. {',...}'})
+  elseif typespec.sum then
+    local result = Inlines{}
+    for i, tspec in pairs(List.map(typespec.sum, render_typespec)) do
+      if i >= 2 then
+        result:insert(Str '|')
+      end
+      result:extend(tspec)
+    end
+    return result
+  end
+  warn("falling back to string representation for type " .. tostring(typespec))
+  return Inlines(tostring(typespec))
+end
+
+local function type_to_inlines (typeobj)
+  if typeobj == nil then
+    return Inlines 'any'
+  end
+
+  -- Types starting with a capital letter are pandoc types, so we can
+  -- link them.
+  return Inlines ' (' .. render_typespec(typeobj) .. Inlines ')'
+end
+
+local function append_inlines (blocks, inlines)
+  local last = blocks[#blocks]
+  if last and (last.t == 'Plain' or last.t == 'Para') then
+    blocks[#blocks] = Plain(last.content .. inlines)
+  else
+    table.insert(blocks, Plain(inlines))
+  end
+  return blocks
+end
+
 local function argslist (parameters)
   local required = List{}
   local optional = List{}
@@ -33,8 +103,9 @@ local function argslist (parameters)
   if #optional == 0 then
     return table.concat(required, ', ')
   end
-  return table.concat(required, ', ')
-    .. '[, ' .. table.concat(optional, '[, ') .. string.rep(']', #optional)
+  return table.concat(required, ', ') ..
+    (#required > 0 and '[, ' or '[') ..
+    table.concat(optional, '[, ') .. string.rep(']', #optional)
 end
 
 local function render_results (results)
@@ -44,15 +115,9 @@ local function render_results (results)
     return {BulletList(
       List(results):map(
         function (res)
-          -- Types starting with a capital letter are pandoc types, so we can
-          -- link them.
-          local type_ = res.type:match'^[A-Z]'
-            and Link(res.type, '#type-' .. res.type:lower())
-            or Str(res.type)
-          return Para(
-            read_inlines(res.description)
-            .. {Space()}
-            .. Inlines '(' .. Inlines{type_} .. Inlines ')'
+          return append_inlines(
+            read_blocks(res.description),
+            type_to_inlines(res.type)
           )
         end
       )
@@ -72,7 +137,10 @@ local function render_function (doc, level, modulename)
       function (p)
         return {
           {Code(p.name)},
-          {read_blocks(p.description .. ' (' .. p.type .. ')')}
+          append_inlines(
+            read_blocks(p.description),
+            type_to_inlines(p.type)
+          )
         }
       end
     )
@@ -81,20 +149,62 @@ local function render_function (doc, level, modulename)
     Header(level, name, {id}),
     Plain{Code(string.format('%s (%s)', name, args))},
   } .. read_blocks(doc.description)
-    .. List(#doc.parameters > 0 and {Para 'Parameters'} or {})
+    .. List(#doc.parameters > 0 and {Para 'Parameters:'} or {})
     .. List{paramlist}
-    .. List(#doc.results > 0 and {Para 'Returns'} or {})
+    .. List(#doc.results > 0 and {Para 'Returns:'} or {})
     .. render_results(doc.results)
 end
 
 local function render_field (field, level, modulename)
   local id = modulename and modulename .. '.' .. field.name or ''
-  return {Header(level, field.name, {id})} .. read_blocks(field.description)
+  return Blocks{Header(level, field.name, {id})} ..
+    {Plain(read_inlines(field.description) .. type_to_inlines(field.type))}
+end
+
+local function render_type (name, level, modulename)
+  -- We just want the modulename prefix, as the type names should already
+  -- contain the module name to some extend.
+  local nameprefix = modulename and
+    modulename:match('(.*)%.[a-z]*$') or
+    'pandoc'
+  local id = nameprefix .. '.' .. name
+  local metatable = registry[name]
+
+  local properties = Blocks{}
+  if next(metatable.docs.properties) then
+    local propattr = {'type-' .. id .. '-properties'}
+    properties:insert(Header(level + 1, "Properties", propattr))
+    for propname, prop in sorted(metatable.docs.properties) do
+      attr = {'type-' .. nameprefix .. '.' .. name .. '.' .. propname}
+      properties:insert(Header(level + 2, propname, attr))
+      properties:insert(
+        Plain(read_inlines(prop.description) ..
+              type_to_inlines(prop.type))
+      )
+    end
+  end
+
+  local methods = Blocks{}
+  if next(metatable.methods) then
+    local attr = {'type-' .. id .. '-methods'}
+    methods:insert(Header(level + 1, "Methods", mattr))
+    for name, method in sorted(metatable.methods) do
+      -- attr = {'type-' .. modulename .. '.' .. name .. '.' .. name}
+      -- methods:insert(Header(level + 2, name, attr))
+      methods:extend(render_function(documentation(method), level+2, id))
+    end
+  end
+
+  local header_id = 'type-' .. nameprefix .. '.' .. name
+  known_types[name] = header_id
+  return {Header(level, name, {header_id})} ..
+    properties ..
+    methods
 end
 
 local function render_module (doc)
   local fields = Blocks{}
-  if #doc.fields then
+  if #doc.fields > 0 then
     fields:insert(Header(2, 'Fields', {doc.name .. '-' .. 'fields'}))
     for i, fld in ipairs(doc.fields) do
       fields:extend(render_field(fld, 3, doc.name))
@@ -109,14 +219,21 @@ local function render_module (doc)
     end
   end
 
-  return Blocks{
-    Header(1, Inlines('Module ' .. doc.name), {'module-' .. doc.name})
-  } .. read_blocks(doc.description) .. fields .. functions
-end
+  local typedocs = Blocks{}
+  local types = type(doc.types) == 'function' and doc.types() or {}
+  if #types > 0 then
+    typedocs:insert(Header(2, 'Types', {doc.name .. '-' .. 'types'}))
+    for i, ty in ipairs(types) do
+      typedocs:extend(render_type(ty, 3, doc.name))
+    end
+  end
 
---- Retrieves the documentation object for the given value.
-local function documentation (value)
-  return debug.getregistry()['HsLua docs'][value]
+  return Blocks{
+    Header(1, Inlines('Module ' .. doc.name), {'module-' .. doc.name})} ..
+    read_blocks(doc.description) ..
+    fields ..
+    functions ..
+    typedocs
 end
 
 local function get_module_name(header)
@@ -128,25 +245,66 @@ local handled_modules = {
   layout = true
 }
 
-return {{
-    Pandoc = function (doc)
-      local blocks = List{}
-      local in_module_docs = false
-      for i, blk in ipairs(doc.blocks) do
-        if blk.t == 'Header' and blk.level == 1 then
-          local module_name = get_module_name(blk)
-          if module_name and handled_modules[module_name] then
-            local object = _ENV[module_name]
-            blocks:extend(render_module(documentation(object)))
-            in_module_docs = true
-          else
-            blocks:insert(blk)
-            in_module_docs = false
-          end
-        elseif not in_module_docs then
-          blocks:insert(blk)
-        end
+local modules = {
+  -- 'cli',
+  -- 'utils',
+  -- 'mediabag',
+  -- 'format',
+  -- 'path',
+  -- 'structure',
+  -- 'system',
+  -- 'layout',
+  -- 'scaffolding',
+  -- 'template',
+  -- 'types',
+  'zip',
+}
+
+-- Generate docs for the given module
+if arg and arg[1] then
+  local module_name = arg[1]
+  local object = _ENV[module_name]
+  local blocks = render_module(documentation(object))
+  print(write(Pandoc(blocks), 'markdown'))
+end
+
+local autogen_start =
+  '<!%-%- BEGIN: AUTOGENERATED CONTENT for module ([a-z%.]+) %-%->\n'
+local autogen_end =
+  '\n<!%-%- END: AUTOGENERATED CONTENT %-%->\n'
+local reflinks_marker =
+  '<!%-%- BEGIN: GENERATED REFERENCE LINKS %-%->\n'
+
+local rawmd = function (str)
+  return RawBlock('markdown', str)
+end
+
+local function foo (input, blocks, start)
+  local mstart, mstop, module_name = input:find(autogen_start, start)
+  if mstart and mstop and module_name then
+    print('Generating docs for module ' .. module_name)
+    blocks:insert(rawmd(input:sub(start, mstop)))
+    local object = _ENV[module_name] or _ENV[module_name:gsub('^pandoc%.', '')]
+    blocks:extend(render_module(documentation(object)))
+    return foo(input, blocks, input:find(autogen_end, mstop) or -1)
+  else
+    local reflinks_start, reflinks_stop = input:find(reflinks_marker, start)
+    blocks:insert(rawmd(input:sub(start, reflinks_stop)))
+    return blocks
+  end
+end
+
+function _G.Reader (inputs, opts)
+  local blocks = foo(tostring(inputs), Blocks{}, 1)
+  blocks = blocks:walk {
+    Span = function (span)
+      local unknown_type = span.attributes['unknown-type']
+      if unknown_type and known_types[unknown_type] then
+        return Link(span.content, '#' .. known_types[unknown_type])
+      elseif span.classes:includes 'builtin-lua-type' then
+        return span.content  -- unwrap
       end
-      return Pandoc(blocks, doc.meta)
     end
-}}
+  }
+  return Pandoc(blocks)
+end