Source code for gavo.user.docgen

"""
Generation of system docs by introspection and combination with static
docs.
"""

#c Copyright 2008-2023, the GAVO project <gavo@ari.uni-heidelberg.de>
#c
#c This program is free software, covered by the GNU GPL.  See the
#c COPYING file in the source distribution.


import locale
import inspect
import re
import sys
import textwrap
import traceback

import pkg_resources

from gavo import base
from gavo import rscdef
from gavo import rscdesc
from gavo import utils
from gavo.base import config  #noflake: used from processed document
from gavo.base import structure
from gavo.utils import Arg, exposedFunction, makeCLIParser


[docs]class RSTFragment(object): """is a collection of convenience methods for generation of RST. """ level1Underliner = "'" level2Underliner = "." def __init__(self): self.content = []
[docs] def makeSpace(self): """adds an empty line unless the last line already is empty. """ if self.content and self.content[-1]!="\n": self.content.append("\n")
[docs] def addHead(self, head, underliner): self.makeSpace() self.content.append(head+"\n") self.content.append(underliner*len(head)+"\n") self.content.append("\n")
[docs] def addHead1(self, head): self.addHead(head, self.level1Underliner)
[docs] def addHead2(self, head): self.addHead(head, self.level2Underliner)
[docs] def delEmptySection(self): """deletes something that looks like a headline if it's the last thing in our content list. """ try: # we suspect a headline if the last two contributions are a line # made up of on char type and an empty line. if self.content[-1]=="\n" and len(set(self.content[-2]))==2: self.content[-3:] = [] except IndexError: # not enough material to take something away pass
[docs] def addULItem(self, content, bullet="*"): if content is None: return initialIndent = bullet+" " self.content.append(textwrap.fill(content, initial_indent=initialIndent, subsequent_indent=" "*len(initialIndent))+"\n")
[docs] def addDLItem(self, term, definition): self.content.append("**%s**\n"%term) self.content.append(textwrap.fill(definition, initial_indent=" ", subsequent_indent=" ")) self.content.append("\n")
[docs] def addDefinition(self, defHead, defBody): """adds a definition list-style item . defBody is re-indented with two spaces, defHead is assumed to only contain a single line. """ self.content.append(defHead+"\n") self.content.append(utils.fixIndentation(defBody, " ", governingLine=2)+"\n")
[docs] def addNormalizedPara(self, stuff): """adds stuff to the document, making sure it's not glued to any previous material and removing whitespace as necessary for docstrings. """ self.makeSpace() self.content.append(utils.fixIndentation(stuff, "", governingLine=2)+"\n")
[docs] def addRaw(self, stuff): self.content.append(stuff)
[docs]class ParentPlaceholder(object): """is a sentinel left in the proto documentation, to be replaced by docs on the element parents found while processing. """ def __init__(self, elementName): self.elementName = elementName
[docs]class DocumentStructure(dict): """a dict keeping track of what elements have been processed and which children they had. From this information, it can later fill out the ParentPlaceholders left in the proto reference doc. This also keeps track of what macros can be used where. """ def __init__(self): dict.__init__(self) self.knownMacros = KnownMacros() def _makeDoc(self, parents): return "May occur in %s.\n"%", ".join("`Element %s`_"%name for name in parents)
[docs] def fillOut(self, protoDoc): parentDict = {} for parent, children in self.items(): for child in children: parentDict.setdefault(child, []).append(parent) for index, item in enumerate(protoDoc): if isinstance(item, ParentPlaceholder): if item.elementName in parentDict: protoDoc[index] = self._makeDoc(parentDict[item.elementName]) else: protoDoc[index] = ""
[docs]def getDocName(klass): return getattr(klass, "docName_", klass.name_)
[docs]class StructDocMaker(object): """A class encapsulating generation of documentation from structs. """ def __init__(self, docStructure): self.docParts = [] self.docStructure = docStructure self.visitedClasses = set() def _iterMatchingAtts(self, klass, condition): for att in sorted((a for a in klass.attrSeq if condition(a)), key=lambda att: att.name_): yield att def _iterAttsOfBase(self, klass, base): return self._iterMatchingAtts(klass, lambda a: isinstance(a, base)) def _iterAttsNotOfBase(self, klass, base): return self._iterMatchingAtts(klass, lambda a: not isinstance(a, base)) def _addMacroDocs(self, klass, content, docStructure): if not issubclass(klass, base.MacroPackage): return macNames = [] for attName in dir(klass): if attName.startswith("macro_"): name = attName[6:] docStructure.knownMacros.addMacro(name, getattr(klass, attName), klass) macNames.append(name) content.addNormalizedPara("Macros predefined here: "+", ".join( "`Macro %s`_"%name for name in sorted(macNames))) content.makeSpace() def _realAddDocsFrom(self, klass, docStructure): name = getDocName(klass) if name in self.docStructure: return self.visitedClasses.add(klass) content = RSTFragment() content.addHead1("Element %s"%name) if klass.__doc__: content.addNormalizedPara(klass.__doc__) else: content.addNormalizedPara("NOT DOCUMENTED") content.addRaw(ParentPlaceholder(name)) content.addHead2("Atomic Children") for att in self._iterAttsOfBase(klass, base.AtomicAttribute): content.addULItem(att.makeUserDoc()) content.delEmptySection() children = [] content.addHead2("Structure Children") for att in self._iterAttsOfBase(klass, base.StructAttribute): try: content.addULItem(att.makeUserDoc()) if isinstance(getattr(att, "childFactory", None), structure.StructType): children.append(getDocName(att.childFactory)) if att.childFactory not in self.visitedClasses: self.addDocsFrom(att.childFactory, docStructure) except: sys.stderr.write("While gendoccing %s in %s:\n"%( att.name_, name)) traceback.print_exc() self.docStructure.setdefault(name, []).extend(children) content.delEmptySection() content.addHead2("Other Children") for att in self._iterAttsNotOfBase(klass, (base.AtomicAttribute, base.StructAttribute)): content.addULItem(att.makeUserDoc()) content.delEmptySection() content.makeSpace() self._addMacroDocs(klass, content, docStructure) self.docParts.append((klass.name_, content.content))
[docs] def addDocsFrom(self, klass, docStructure): try: self._realAddDocsFrom(klass, docStructure) except: sys.stderr.write("Cannot add docs from element %s\n"%klass.name_) traceback.print_exc()
[docs] def getDocs(self): self.docParts.sort(key=lambda t: t[0].upper()) resDoc = [] for title, doc in self.docParts: resDoc.extend(doc) return resDoc
[docs]class MacroDoc(object): """documentation for a macro, including the objects that know about it. """ def __init__(self, name, macFunc, foundIn): self.name = name self.macFunc = macFunc self.inObjects = [foundIn]
[docs] def addObject(self, macFunc, obj): """declares that macFunc is also available on the python object obj. If also checks implements the "see <whatever>" mechanism described in KnownMacros. """ self.inObjects.append(obj) if (self.macFunc.__doc__ or "").startswith("see "): self.macFunc = macFunc
[docs] def makeDoc(self, content): """adds documentation of macFunc to the RSTFragment content. """ # macros have args in {}, of course there's no self, and null-arg # macros have not {}... sig = inspect.signature(self.macFunc) args = str(sig).replace("(", "{").replace(")", "}").replace("{}", "" ).replace(", ", "}{") content.addRaw("::\n\n \\%s%s\n\n"%(self.name, args)) content.addRaw(utils.fixIndentation( self.macFunc.__doc__ or "undocumented", "", 1).replace("\\", "\\\\")) content.addNormalizedPara("Available in "+", ".join( sorted( "`Element %s`_"%c.name_ for c in self.inObjects)))
[docs]class KnownMacros(object): """An accumulator for all macros known to the various DaCHS objects. Note that macros with identical names are supposed to do essentially the same things. In particular, they must all have the same signature or the documentation will be wrong. When macros share names, all but one implementation should have a docstring just saying "see <whatever>"; that way, the docstring actually chosen is deterministic. """ def __init__(self): self.macros = {}
[docs] def addMacro(self, name, macFunc, foundIn): """registers macFunc as expanding name in the element foundIn. macFunc is the method, foundIn is the python class it's defined on. """ if name in self.macros: self.macros[name].addObject(macFunc, foundIn) else: self.macros[name] = MacroDoc(name, macFunc, foundIn)
[docs] def getDocs(self): """returns RST lines describing all macros fed in in addMacro. """ content = RSTFragment() for macName in sorted(self.macros): content.addHead1("Macro %s"%macName) self.macros[macName].makeDoc(content) content.makeSpace() return content.content
[docs]def formatKnownMacros(docStructure): return docStructure.knownMacros.getDocs()
[docs]def getStructDocs(docStructure): dm = StructDocMaker(docStructure) dm.addDocsFrom(rscdesc.RD, docStructure) return dm.getDocs()
[docs]def getStructDocsFromRegistry(registry, docStructure): dm = StructDocMaker(docStructure) for name, struct in sorted(registry.items()): dm.addDocsFrom(struct, docStructure) return dm.getDocs()
[docs]def getGrammarDocs(docStructure): registry = dict((n, rscdef.getGrammar(n)) for n in rscdef.GRAMMAR_REGISTRY) return getStructDocsFromRegistry(registry, docStructure)
[docs]def getCoreDocs(docStructure): from gavo.svcs import core registry = dict((n, core.getCore(n)) for n in core.CORE_REGISTRY) return getStructDocsFromRegistry(registry, docStructure)
[docs]def getActiveTagDocs(docStructure): from gavo.base import activetags return getStructDocsFromRegistry(activetags.getActiveTag.registry, docStructure)
[docs]def getRendererDocs(docStructure): from gavo.svcs import RENDERER_REGISTRY, getRenderer content = RSTFragment() for rendName in sorted(RENDERER_REGISTRY): try: rend = getRenderer(rendName) except ImportError as msg: base.ui.notifyWarning("Missing dependency for renderer %s: %s"%( rendName, msg)) if rend.__doc__: content.addHead1("The %s Renderer"%rendName) metaStuff = "*This renderer's parameter style is \"%s\"."%( rend.parameterStyle) if not rend.checkedRenderer: metaStuff += " This is an unchecked renderer." content.addNormalizedPara(metaStuff+"*") content.addNormalizedPara(rend.__doc__) return content.content
[docs]def getTriggerDocs(docStructure): from gavo.rscdef import rowtriggers return getStructDocsFromRegistry(rowtriggers._triggerRegistry, docStructure)
def _documentParameters(content, pars): content.makeSpace() for par in sorted(pars, key=lambda p: p.key): if par.late: doc = ["Late p"] else: doc = ["P"] doc.append("arameter *%s*\n"%par.key) if par.content_: doc.append(" defaults to ``%s``;\n"%par.content_) if par.description: doc.append(utils.fixIndentation(par.description, " ")) else: doc.append(" UNDOCUMENTED") content.addRaw(''.join(doc)+"\n") content.makeSpace()
[docs]def getMixinDocs(docStructure, mixinIds): content = RSTFragment() for name in sorted(mixinIds): mixin = base.resolveId(None, name) content.addHead1("The %s Mixin"%name) if mixin.doc is None: content.addNormalizedPara("NOT DOCUMENTED") else: content.addNormalizedPara(mixin.doc) if mixin.pars: content.addNormalizedPara( "This mixin has the following parameters:\n") _documentParameters(content, mixin.pars) return content.content
def _getModuleFunctionDocs(module): """returns documentation for all functions marked with @document in the namespace module. """ res = [] for name in dir(module): if name.startswith("_"): # ignore all private attributes, whatever else happens continue ob = getattr(module, name) if hasattr(ob, "buildDocsForThis"): if ob.__doc__ is None: # silently ignore if no docstring continue res.append("``"+name+str(inspect.signature(ob))+"``") res.append(utils.fixIndentation(ob.__doc__, " ", 1)) res.append("") return "\n".join(res)
[docs]def getRmkFuncs(docStructure): from gavo.rscdef import rmkfuncs return _getModuleFunctionDocs(rmkfuncs)
[docs]def getRegtestAssertions(docStructure): from gavo.rscdef import regtest return _getModuleFunctionDocs(regtest.RegTest)
def _getProcdefDocs(procDefs): """returns documentation for the ProcDefs in the sequence procDefs. """ content = RSTFragment() for id, pd in procDefs: content.addHead2(id) if pd.doc is None: content.addNormalizedPara("NOT DOCUMENTED") else: content.addNormalizedPara(pd.doc) content.makeSpace() if pd.getSetupPars(): content.addNormalizedPara("Setup parameters for the procedure are:\n") _documentParameters(content, pd.getSetupPars()) return content.content def _makeProcsDocumenter(idList): def buildDocs(docStructure): return _getProcdefDocs([(id, base.resolveId(None, id)) for id in sorted(idList)]) return buildDocs def _makeTableDoc(tableDef): content = RSTFragment() content.addHead1(tableDef.getQName()) content.addNormalizedPara("Defined in %s"% tableDef.rd.sourceId.replace("__system__", "/")) content.addNormalizedPara(base.getMetaText(tableDef, "description", default="UNDOCUMENTED")) content.makeSpace() for col in tableDef: content.addDLItem(col.name, "(%s) -- %s"%(col.type, col.description)) content.makeSpace() return "".join(content.content)
[docs]def makeSystemTablesList(docStructure): parts = [] for rdName in pkg_resources.resource_listdir( "gavo", "resources/inputs/__system__"): if not rdName.endswith(".rd"): continue try: for tableDef in base.caches.getRD("//"+rdName).tables: if tableDef.onDisk: parts.append((tableDef.getQName(), _makeTableDoc(tableDef))) except base.Error: base.ui.notifyError("Bad system RD: %s"%rdName) return "\n".join(content for _, content in sorted(parts))
[docs]def getStreamsDoc(idList): content = RSTFragment() for id in idList: stream = base.resolveId(None, id) content.addHead2(id) if stream.doc is None: raise base.ReportableError("Stream %s has no doc -- don't include in" " reference documentation."%id) content.addNormalizedPara(stream.doc) content.makeSpace() if stream.DEFAULTS and stream.DEFAULTS.defaults: content.addNormalizedPara("*Defaults* for macros used in this stream:") content.makeSpace() for key, value in sorted(stream.DEFAULTS.defaults.items()): content.addULItem("%s: '%s'"%(key, value)) content.makeSpace() return content.content
[docs]def getMetaKeyDocs(): from gavo.base import meta content = RSTFragment() d = meta.META_CLASSES_FOR_KEYS for metaKey in sorted(d): content.addDefinition(metaKey, d[metaKey].__doc__ or "NOT DOCUMENTED") return content.content
NO_API_SYMBOLS = frozenset(["__builtins__", "AdhocQuerier", "__doc__", "__file__", "__name__", "__package__", "addCartesian", "combinePMs", "getBinaryName", "getDefaultValueParsers", "getHTTPPar", "makeProc", "ParseOptions"])
[docs]def getAPIDocs(docStructure): from gavo import api content = RSTFragment() for name in sorted(dir(api)): obj = getattr(api, name) if (name in NO_API_SYMBOLS or type(obj)==type(re) or not obj.__doc__): continue whatsit = "" if type(obj)==type(getMetaKeyDocs): whatsit = "Function " elif type(obj)==type: whatsit = "Class " elif isinstance(obj, float): whatsit = "Constant " content.addHead2(whatsit+name) content.makeSpace() if whatsit=="Function ": content.addNormalizedPara( "Signature: ``{}{}``".format(name, str(inspect.signature(obj)))) content.makeSpace() if whatsit=="Constant ": content.addNormalizedPara("A constant, valued %s"%obj) else: content.addNormalizedPara(obj.__doc__) return content.content
_replaceWithResultPat = re.compile(".. replaceWithResult (.*)")
[docs]def makeReferenceDocs(): """returns a restructured text containing the reference documentation built from the template in refdoc.rstx. **WARNING**: refdoc.rstx can execute arbitrary code right now. We probably want to change this to having macros here. """ res, docStructure = [], DocumentStructure() f = pkg_resources.resource_stream("gavo", "resources/templates/refdoc.rstx") parseState = "content" code = [] for lnCount, ln in enumerate(f): ln = ln.decode("utf-8") if parseState=="content": mat = _replaceWithResultPat.match(ln) if mat: code.append(mat.group(1)) parseState = "code" else: res.append(ln) elif parseState=="code": if ln.strip(): code.append(ln) else: try: res.extend(eval(" ".join(code))) except Exception: sys.stderr.write("Invalid code near line %s: '%s'\n"%( lnCount, " ".join(code))) raise code = [] parseState = "content" res.append("\n") else: # unknown state assert False f.close() docStructure.fillOut(res) return "..\n WARNING: GENERATED DOCUMENT.\n Edit this in refdoc.rstx or the DaCHS source code.\n\n"+"".join(res)
[docs]@exposedFunction([], help="Writes ReStructuredText for the reference" " documentation to stdout") def refdoc(args): sys.stdout.buffer.write(makeReferenceDocs( ).replace("\t", " " ).encode("utf-8"))
[docs]@exposedFunction([Arg(help="Input file name", dest="src")], help="Turns ReStructured text (with DaCHS extensions) to LaTeX source") def latex(args): from docutils import core locale.setlocale(locale.LC_ALL, '') sys.argv[1:] = ( "--documentoptions=11pt,a4paper --stylesheet stylesheet.tex" " --use-latex-citations").split() # Hm... I need to record *somewhere* that the two big documents # ought to use book. This isn't a good place, but the doc Makefile # isn't either. Hm. if (args.src.endswith("tutorial.rstx") or args.src.endswith("ref.rstx")): sys.argv.append("--documentclass=book") sys.argv.append(args.src) core.publish_cmdline(writer_name='latex', description="(DaCHS rst2latex)")
[docs]@exposedFunction([Arg(help="Input file name", dest="src")], help="Turns ReStructured text (with DaCHS extensions) to HTML") def html(args): from docutils import core locale.setlocale(locale.LC_ALL, '') # TODO: actually determine template path sys.argv[1:] = ("--template rst2html-template.txt --stylesheet ref.css" " --link-stylesheet").split() sys.argv.append(args.src) core.publish_cmdline(writer_name='html', description="(DaCHS rst2html)")
[docs]def main(): args = makeCLIParser(globals()).parse_args() args.subAction(args)
if __name__=="__main__": # pragma: no cover docStructure = DocumentStructure() print("".join(getAPIDocs(docStructure)))