gavo.user.docgen

1 """ 2 Generation of system docs by introspection and combination with static 3 docs. 4 """ 5 6 #c Copyright 2008-2019, the GAVO project 7 #c 8 #c This program is free software, covered by the GNU GPL. See the 9 #c COPYING file in the source distribution. 10 11 12 from __future__ import print_function 13 14 import locale 15 import inspect 16 import re 17 import sys 18 import textwrap 19 import traceback 20 21 import pkg_resources 22 23 from gavo import base 24 from gavo import rscdef 25 from gavo import rscdesc 26 from gavo import utils 27 from gavo.base import structure 28 from gavo.utils import Arg, exposedFunction, makeCLIParser

29 30 31 -def _indent(stuff, indent):

32 return re.sub("(?m)^", indent, stuff)

33

34 35 -def _decodeStrings(stringList):

36 """replaces all byte strings within stringList with unicode objects. 37 38 Everything not already a unicode object is, for now, assumed to be 39 iso-8859-1, since that is what code is encoded in when it's not ASCII. 40 """ 41 for ind, s in enumerate(stringList): 42 if not isinstance(s, unicode): 43 stringList[ind] = s.decode("iso-8859-1", "replace")

44

45 46 -class RSTFragment(object):

47 """is a collection of convenience methods for generation of RST. 48 """ 49 level1Underliner = "'" 50 level2Underliner = "." 51

52 - def __init__(self):

53 self.content = []

54

55 - def makeSpace(self):

56 """adds an empty line unless the last line already is empty. 57 """ 58 if self.content and self.content[-1]!="\n": 59 self.content.append("\n")

60

61 - def addHead(self, head, underliner):

62 self.makeSpace() 63 self.content.append(head+"\n") 64 self.content.append(underliner*len(head)+"\n") 65 self.content.append("\n")

66

67 - def addHead1(self, head):

68 self.addHead(head, self.level1Underliner)

69

70 - def addHead2(self, head):

71 self.addHead(head, self.level2Underliner)

72

73 - def delEmptySection(self):

74 """deletes something that looks like a headline if it's the last 75 thing in our content list. 76 """ 77 try: 78 # we suspect a headline if the last two contributions are a line 79 # made up of on char type and an empty line. 80 if self.content[-1]=="\n" and len(set(self.content[-2]))==2: 81 self.content[-3:] = [] 82 except IndexError: # not enough material to take something away 83 pass

84

85 - def addULItem(self, content, bullet="*"):

86 if content is None: 87 return 88 initialIndent = bullet+" " 89 self.content.append(textwrap.fill(content, initial_indent=initialIndent, 90 subsequent_indent=" "*len(initialIndent))+"\n")

91

92 - def addDLItem(self, term, definition):

93 self.content.append("**%s**\n"%term) 94 self.content.append(textwrap.fill(definition, 95 initial_indent=" ", subsequent_indent=" ")) 96 self.content.append("\n")

97

98 - def addDefinition(self, defHead, defBody):

99 """adds a definition list-style item . 100 101 defBody is re-indented with two spaces, defHead is assumed to only 102 contain a single line. 103 """ 104 self.content.append(defHead+"\n") 105 self.content.append(utils.fixIndentation(defBody, " ", 106 governingLine=2)+"\n")

107

108 - def addNormalizedPara(self, stuff):

109 """adds stuff to the document, making sure it's not glued to any 110 previous material and removing whitespace as necessary for docstrings. 111 """ 112 self.makeSpace() 113 self.content.append(utils.fixIndentation(stuff, "", governingLine=2)+"\n")

114

115 - def addRaw(self, stuff):

116 self.content.append(stuff)

117

118 119 -class ParentPlaceholder(object):

120 """is a sentinel left in the proto documentation, to be replaced by 121 docs on the element parents found while processing. 122 """

123 - def __init__(self, elementName):

124 self.elementName = elementName

125

126 127 -class DocumentStructure(dict):

128 """a dict keeping track of what elements have been processed and 129 which children they had. 130 131 From this information, it can later fill out the ParentPlaceholders 132 left in the proto reference doc. 133 134 This also keeps track of what macros can be used where. 135 """

136 - def __init__(self):

137 dict.__init__(self) 138 self.knownMacros = KnownMacros()

139

140 - def _makeDoc(self, parents):

141 return "May occur in %s.\n"%", ".join("`Element %s`_"%name 142 for name in parents)

143

144 - def fillOut(self, protoDoc):

145 parentDict = {} 146 for parent, children in self.iteritems(): 147 for child in children: 148 parentDict.setdefault(child, []).append(parent) 149 150 for index, item in enumerate(protoDoc): 151 if isinstance(item, ParentPlaceholder): 152 if item.elementName in parentDict: 153 protoDoc[index] = self._makeDoc(parentDict[item.elementName]) 154 else: 155 protoDoc[index] = ""

156

157 158 -def getDocName(klass):

159 return getattr(klass, "docName_", klass.name_)

160

161 162 -class StructDocMaker(object):

163 """A class encapsulating generation of documentation from structs. 164 """ 165

166 - def __init__(self, docStructure):

167 self.docParts = [] 168 self.docStructure = docStructure 169 self.visitedClasses = set()

170

171 - def _iterMatchingAtts(self, klass, condition):

172 for att in sorted((a for a in klass.attrSeq 173 if condition(a)), 174 key=lambda att: att.name_): 175 yield att

176

177 - def _iterAttsOfBase(self, klass, base):

178 return self._iterMatchingAtts(klass, lambda a: isinstance(a, base))

179

180 - def _iterAttsNotOfBase(self, klass, base):

181 return self._iterMatchingAtts(klass, lambda a: not isinstance(a, base))

182 183 _noDefaultValues = set([base.Undefined, base.Computed])

184 - def _hasDefault(self, att):

185 try: 186 return att.default_ not in self._noDefaultValues 187 except TypeError: # unhashable default is a default 188 return True

189

190 - def _addMacroDocs(self, klass, content, docStructure):

191 if not issubclass(klass, base.MacroPackage): 192 return 193 194 macNames = [] 195 for attName in dir(klass): 196 if attName.startswith("macro_"): 197 name = attName[6:] 198 docStructure.knownMacros.addMacro(name, 199 getattr(klass, attName), klass) 200 macNames.append(name) 201 202 content.addNormalizedPara("Macros predefined here: "+", ".join( 203 "`Macro %s`_"%name for name in sorted(macNames))) 204 content.makeSpace()

205

206 - def _realAddDocsFrom(self, klass, docStructure):

207 name = getDocName(klass) 208 if name in self.docStructure: 209 return 210 self.visitedClasses.add(klass) 211 content = RSTFragment() 212 content.addHead1("Element %s"%name) 213 if klass.__doc__: 214 content.addNormalizedPara(klass.__doc__) 215 else: 216 content.addNormalizedPara("NOT DOCUMENTED") 217 218 content.addRaw(ParentPlaceholder(name)) 219 220 content.addHead2("Atomic Children") 221 for att in self._iterAttsOfBase(klass, base.AtomicAttribute): 222 content.addULItem(att.makeUserDoc()) 223 content.delEmptySection() 224 225 children = [] 226 content.addHead2("Structure Children") 227 for att in self._iterAttsOfBase(klass, base.StructAttribute): 228 try: 229 content.addULItem(att.makeUserDoc()) 230 if isinstance(getattr(att, "childFactory", None), structure.StructType): 231 children.append(getDocName(att.childFactory)) 232 if att.childFactory not in self.visitedClasses: 233 self.addDocsFrom(att.childFactory, docStructure) 234 except: 235 sys.stderr.write("While gendoccing %s in %s:\n"%( 236 att.name_, name)) 237 traceback.print_exc() 238 self.docStructure.setdefault(name, []).extend(children) 239 content.delEmptySection() 240 241 content.addHead2("Other Children") 242 for att in self._iterAttsNotOfBase(klass, 243 (base.AtomicAttribute, base.StructAttribute)): 244 content.addULItem(att.makeUserDoc()) 245 content.delEmptySection() 246 content.makeSpace() 247 248 self._addMacroDocs(klass, content, docStructure) 249 250 self.docParts.append((klass.name_, content.content))

251

252 - def addDocsFrom(self, klass, docStructure):

253 try: 254 self._realAddDocsFrom(klass, docStructure) 255 except: 256 sys.stderr.write("Cannot add docs from element %s\n"%klass.name_) 257 traceback.print_exc()

258

259 - def getDocs(self):

260 self.docParts.sort(key=lambda t: t[0].upper()) 261 resDoc = [] 262 for title, doc in self.docParts: 263 resDoc.extend(doc) 264 return resDoc

265

266 267 -class MacroDoc(object):

268 """documentation for a macro, including the objects that know about 269 it. 270 """

271 - def __init__(self, name, macFunc, foundIn):

272 self.name = name 273 self.macFunc = macFunc 274 self.inObjects = [foundIn]

275

276 - def addObject(self, macFunc, obj):

277 """declares that macFunc is also available on the python object obj. 278 279 If also checks implements the "see <whatever>" mechanism described 280 in KnownMacros. 281 """ 282 self.inObjects.append(obj) 283 if (self.macFunc.func_doc or "").startswith("see "): 284 self.macFunc = macFunc

285

286 - def makeDoc(self, content):

287 """adds documentation of macFunc to the RSTFragment content. 288 """ 289 # macros have args in {}, of course there's no self, and null-arg 290 # macros have not {}... 291 args, varargs, varkw, defaults = inspect.getargspec(self.macFunc) 292 args = inspect.formatargspec(args[1:], varargs, varkw, defaults 293 ).replace("(", "{").replace(")", "}").replace("{}", "" 294 ).replace(", ", "}{") 295 content.addRaw("::\n\n \\%s%s\n\n"%(self.name, args)) 296 content.addRaw(utils.fixIndentation( 297 self.macFunc.func_doc or "undocumented", "", 1).replace("\\", "\\\\")) 298 content.addNormalizedPara("Available in "+", ".join( 299 sorted( 300 "`Element %s`_"%c.name_ for c in self.inObjects)))

301

302 303 -class KnownMacros(object):

304 """An accumulator for all macros known to the various DaCHS objects. 305 306 Note that macros with identical names are supposed to do essentially 307 the same things. In particular, they must all have the same signature 308 or the documentation will be wrong. 309 310 When macros share names, all but one implementation should have 311 a docstring just saying "see <whatever>"; that way, the docstring 312 actually chosen is deterministic. 313 """

314 - def __init__(self):

315 self.macros = {}

316

317 - def addMacro(self, name, macFunc, foundIn):

318 """registers macFunc as expanding name in the element foundIn. 319 320 macFunc is the method, foundIn is the python class it's defined on. 321 """ 322 if name in self.macros: 323 self.macros[name].addObject(macFunc, foundIn) 324 else: 325 self.macros[name] = MacroDoc(name, macFunc, foundIn)

326

327 - def getDocs(self):

328 """returns RST lines describing all macros fed in in addMacro. 329 """ 330 content = RSTFragment() 331 for macName in sorted(self.macros): 332 content.addHead1("Macro %s"%macName) 333 self.macros[macName].makeDoc(content) 334 content.makeSpace() 335 return content.content

336

337 338 -def formatKnownMacros(docStructure):

339 return docStructure.knownMacros.getDocs()

340

341 342 -def getStructDocs(docStructure):

343 dm = StructDocMaker(docStructure) 344 dm.addDocsFrom(rscdesc.RD, docStructure) 345 return dm.getDocs()

346

347 348 -def getStructDocsFromRegistry(registry, docStructure):

349 dm = StructDocMaker(docStructure) 350 for name, struct in sorted(registry.items()): 351 dm.addDocsFrom(struct, docStructure) 352 return dm.getDocs()

353

354 355 -def getGrammarDocs(docStructure):

356 registry = dict((n, rscdef.getGrammar(n)) for n in rscdef.GRAMMAR_REGISTRY) 357 return getStructDocsFromRegistry(registry, docStructure)

358

359 360 -def getCoreDocs(docStructure):

361 from gavo.svcs import core 362 registry = dict((n, core.getCore(n)) for n in core.CORE_REGISTRY) 363 return getStructDocsFromRegistry(registry, docStructure)

364

365 366 -def getActiveTagDocs(docStructure):

367 from gavo.base import activetags 368 return getStructDocsFromRegistry(activetags.getActiveTag.registry, 369 docStructure)

370

371 372 -def getRendererDocs(docStructure):

373 from gavo.svcs import RENDERER_REGISTRY, getRenderer 374 content = RSTFragment() 375 for rendName in sorted(RENDERER_REGISTRY): 376 rend = getRenderer(rendName) 377 if rend.__doc__: 378 content.addHead1("The %s Renderer"%rendName) 379 metaStuff = "*This renderer's parameter style is \"%s\"."%( 380 rend.parameterStyle) 381 if not rend.checkedRenderer: 382 metaStuff += " This is an unchecked renderer." 383 content.addNormalizedPara(metaStuff+"*") 384 content.addNormalizedPara(rend.__doc__) 385 return content.content

386

387 388 -def getTriggerDocs(docStructure):

389 from gavo.rscdef import rowtriggers 390 return getStructDocsFromRegistry(rowtriggers._triggerRegistry, docStructure)

391

392 393 -def _documentParameters(content, pars):

394 content.makeSpace() 395 for par in sorted(pars, key=lambda p: p.key): 396 if par.late: 397 doc = ["Late p"] 398 else: 399 doc = ["P"] 400 doc.append("arameter *%s*\n"%par.key) 401 if par.content_: 402 doc.append(" defaults to ``%s``;\n"%par.content_) 403 if par.description: 404 doc.append(utils.fixIndentation(par.description, " ")) 405 else: 406 doc.append(" UNDOCUMENTED") 407 content.addRaw(''.join(doc)+"\n") 408 content.makeSpace()

409

410 411 -def getMixinDocs(docStructure, mixinIds):

412 content = RSTFragment() 413 for name in sorted(mixinIds): 414 mixin = base.resolveId(None, name) 415 content.addHead1("The %s Mixin"%name) 416 if mixin.doc is None: 417 content.addNormalizedPara("NOT DOCUMENTED") 418 else: 419 content.addNormalizedPara(mixin.doc) 420 if mixin.pars: 421 content.addNormalizedPara( 422 "This mixin has the following parameters:\n") 423 _documentParameters(content, mixin.pars) 424 return content.content

425

426 427 -def _getModuleFunctionDocs(module):

428 """returns documentation for all functions marked with @document in the 429 namespace module. 430 """ 431 res = [] 432 for name in dir(module): 433 if name.startswith("_"): 434 # ignore all private attributes, whatever else happens 435 continue 436 437 ob = getattr(module, name) 438 if hasattr(ob, "buildDocsForThis"): 439 if ob.func_doc is None: # silently ignore if no docstring 440 continue 441 res.append( 442 "*%s%s*"%(name, inspect.formatargspec(*inspect.getargspec(ob)))) 443 res.append(utils.fixIndentation(ob.func_doc, " ", 1)) 444 res.append("") 445 return "\n".join(res)

446

447 448 -def getRmkFuncs(docStructure):

449 from gavo.rscdef import rmkfuncs 450 return _getModuleFunctionDocs(rmkfuncs)

451

452 453 -def getRegtestAssertions(docStructure):

454 from gavo.rscdef import regtest 455 return _getModuleFunctionDocs(regtest.RegTest)

456

457 458 -def _getProcdefDocs(procDefs):

459 """returns documentation for the ProcDefs in the sequence procDefs. 460 """ 461 content = RSTFragment() 462 for id, pd in procDefs: 463 content.addHead2(id) 464 if pd.doc is None: 465 content.addNormalizedPara("NOT DOCUMENTED") 466 else: 467 content.addNormalizedPara(pd.doc) 468 content.makeSpace() 469 if pd.getSetupPars(): 470 content.addNormalizedPara("Setup parameters for the procedure are:\n") 471 _documentParameters(content, pd.getSetupPars()) 472 return content.content

473

474 475 -def _makeProcsDocumenter(idList):

476 def buildDocs(docStructure): 477 return _getProcdefDocs([(id, base.resolveId(None, id)) 478 for id in sorted(idList)])

479 return buildDocs 480

481 482 -def _makeTableDoc(tableDef):

483 content = RSTFragment() 484 content.addHead1(tableDef.getQName()) 485 content.addNormalizedPara("Defined in %s"% 486 tableDef.rd.sourceId.replace("__system__", "/")) 487 content.addNormalizedPara(base.getMetaText(tableDef, 488 "description", default="UNDOCUMENTED")) 489 content.makeSpace() 490 for col in tableDef: 491 content.addDLItem(col.name, 492 "(%s) -- %s"%(col.type, col.description)) 493 content.makeSpace() 494 return "".join(content.content)

495

496 497 -def makeSystemTablesList(docStructure):

498 parts = [] 499 for rdName in pkg_resources.resource_listdir( 500 "gavo", "resources/inputs/__system__"): 501 if not rdName.endswith(".rd"): 502 continue 503 504 try: 505 for tableDef in base.caches.getRD("//"+rdName).tables: 506 if tableDef.onDisk: 507 parts.append((tableDef.getQName(), _makeTableDoc(tableDef))) 508 except base.Error: 509 base.ui.notifyError("Bad system RD: %s"%rdName) 510 511 return "\n".join(content for _, content in sorted(parts))

512

513 514 -def getStreamsDoc(idList):

515 content = RSTFragment() 516 for id in idList: 517 stream = base.resolveId(None, id) 518 content.addHead2(id) 519 if stream.doc is None: 520 raise base.ReportableError("Stream %s has no doc -- don't include in" 521 " reference documentation."%id) 522 content.addNormalizedPara(stream.doc) 523 content.makeSpace() 524 525 if stream.DEFAULTS and stream.DEFAULTS.defaults: 526 content.addNormalizedPara("*Defaults* for macros used in this stream:") 527 content.makeSpace() 528 for key, value in sorted(stream.DEFAULTS.defaults.iteritems()): 529 content.addULItem("%s: '%s'"%(key, value)) 530 content.makeSpace() 531 532 return content.content

533

534 535 -def getMetaKeyDocs():

536 from gavo.base import meta 537 content = RSTFragment() 538 d = meta.META_CLASSES_FOR_KEYS 539 for metaKey in sorted(d): 540 content.addDefinition(metaKey, 541 d[metaKey].__doc__ or "NOT DOCUMENTED") 542 return content.content

543 544 545 NO_API_SYMBOLS = frozenset(["__builtins__", "AdhocQuerier", 546 "__doc__", "__file__", "__name__", "__package__", "addCartesian", 547 "combinePMs", "getBinaryName", "getDefaultValueParsers", "getHTTPPar", 548 "makeProc", "ParseOptions"])

549 550 -def getAPIDocs(docStructure):

551 from gavo import api 552 content = RSTFragment() 553 554 for name in sorted(dir(api)): 555 obj = getattr(api, name) 556 if (name in NO_API_SYMBOLS 557 or type(obj)==type(re) 558 or not obj.__doc__): 559 continue 560 561 whatsit = "" 562 if type(obj)==type(getMetaKeyDocs): 563 whatsit = "Function " 564 elif type(obj)==type: 565 whatsit = "Class " 566 elif isinstance(obj, float): 567 whatsit = "Constant " 568 569 content.addHead2(whatsit+name) 570 content.makeSpace() 571 if whatsit=="Function ": 572 content.addNormalizedPara( 573 "Signature: ``%s%s``"%( 574 name, inspect.formatargspec(*inspect.getargspec(obj)))) 575 content.makeSpace() 576 577 if whatsit=="Constant ": 578 content.addNormalizedPara("A constant, valued %s"%obj) 579 else: 580 content.addNormalizedPara(obj.__doc__) 581 582 return content.content

583 584 585 _replaceWithResultPat = re.compile(".. replaceWithResult (.*)")

586 587 -def makeReferenceDocs():

588 """returns a restructured text containing the reference documentation 589 built from the template in refdoc.rstx. 590 591 **WARNING**: refdoc.rstx can execute arbitrary code right now. We 592 probably want to change this to having macros here. 593 """ 594 res, docStructure = [], DocumentStructure() 595 f = pkg_resources.resource_stream("gavo", 596 "resources/templates/refdoc.rstx") 597 598 parseState = "content" 599 code = [] 600 601 for lnCount, ln in enumerate(f): 602 if parseState=="content": 603 mat = _replaceWithResultPat.match(ln) 604 if mat: 605 code.append(mat.group(1)) 606 parseState = "code" 607 else: 608 res.append(ln.decode("utf-8")) 609 610 elif parseState=="code": 611 if ln.strip(): 612 code.append(ln) 613 else: 614 try: 615 res.extend(eval(" ".join(code))) 616 except Exception: 617 sys.stderr.write("Invalid code near line %s: '%s'\n"%( 618 lnCount, " ".join(code))) 619 raise 620 code = [] 621 parseState = "content" 622 res.append("\n") 623 624 else: 625 # unknown state 626 assert False 627 628 f.close() 629 docStructure.fillOut(res) 630 _decodeStrings(res) 631 return "..\n WARNING: GENERATED DOCUMENT.\n Edit this in refdoc.rstx or the DaCHS source code.\n\n"+"".join(res)

632 633 634 @exposedFunction([], help="Writes ReStructuredText for the reference" 635 " documentation to stdout")

636 -def refdoc(args):

637 print(makeReferenceDocs( 638 ).replace("\t", " " 639 ).encode("utf-8"))

640 641 642 @exposedFunction([Arg(help="Input file name", dest="src")], 643 help="Turns ReStructured text (with DaCHS extensions) to LaTeX source")

644 -def latex(args):

645 from docutils import core 646 locale.setlocale(locale.LC_ALL, '') 647 sys.argv[1:] = ( 648 "--documentoptions=11pt,a4paper --stylesheet stylesheet.tex" 649 " --use-latex-citations").split() 650 sys.argv.append(args.src) 651 652 core.publish_cmdline(writer_name='latex', description="(DaCHS rst2latex)")

653 654 655 @exposedFunction([Arg(help="Input file name", dest="src")], 656 help="Turns ReStructured text (with DaCHS extensions) to HTML")

657 -def html(args):

658 from docutils import core 659 locale.setlocale(locale.LC_ALL, '') 660 # TODO: actually determine template path 661 sys.argv[1:] = ("--template rst2html-template.txt --stylesheet ref.css" 662 " --link-stylesheet").split() 663 sys.argv.append(args.src) 664 core.publish_cmdline(writer_name='html', description="(DaCHS rst2html)")

665

666 667 -def main():

668 args = makeCLIParser(globals()).parse_args() 669 args.subAction(args)

670 671 if __name__=="__main__": 672 docStructure = DocumentStructure() 673 print("".join(getAPIDocs(docStructure))) 674

Source Code for Module gavo.user.docgen