gavo.web.formrender

1 """ 2 The form renderer is the standard renderer for web-facing services. 3 """ 4 5 #c Copyright 2008-2019, the GAVO project 6 #c 7 #c This program is free software, covered by the GNU GPL. See the 8 #c COPYING file in the source distribution. 9 10 11 import types 12 13 from nevow import context 14 from nevow import flat 15 from nevow import inevow 16 from nevow import loaders 17 from nevow import rend 18 from nevow import tags as T 19 from twisted.internet import defer, reactor 20 from twisted.python.components import registerAdapter 21 22 from gavo import base 23 from gavo import formats 24 from gavo import svcs 25 from gavo.base import typesystems 26 from gavo.imp import formal 27 from gavo.imp.formal import iformal 28 from gavo.svcs import customwidgets 29 from gavo.svcs import inputdef 30 from gavo.svcs import streaming 31 from gavo.web import grend 32 from gavo.web import serviceresults

33 34 35 -def _getDeferredImmediate(deferred, 36 default="(non-ready deferreds not supported here)"):

37 """returns the value of deferred if it's already in, default otherwise. 38 """ 39 resultHolder = [default] 40 41 def grabResult(res): 42 resultHolder[0] = res

43 44 # adding a callback to a ready deferred immediately calls the callback 45 deferred.addCallback(grabResult) 46 return resultHolder[0] 47

48 49 -def _flattenStan(stan, ctx):

50 """helps streamStan. 51 """ 52 # this is basically ripped from nevow's iterflatten 53 rest = [iter([flat.partialflatten(ctx, stan)])] 54 while rest: 55 gen = rest.pop() 56 for item in gen: 57 if isinstance(item, str): 58 yield item 59 elif isinstance(item, unicode): 60 yield item.encode("utf-8") 61 else: 62 # something iterable is coming up. Suspend the current 63 # generator and start iterating something else. 64 rest.append(gen) 65 if isinstance(item, (list, types.GeneratorType)): 66 rest.append(iter(item)) 67 elif isinstance(item, defer.Deferred): 68 # we actually cannot do deferreds that need to wait; 69 # those shouldn't be necessary with forms. 70 # Instead, grab the result immediately and go ahead with it. 71 rest.append(iter(_getDeferredImmediate(item))) 72 else: 73 rest.append(flat.partialflatten(ctx, item)) 74 break

75

76 77 -def iterStanChunked(stan, ctx, chunkSize):

78 """yields strings made from stan. 79 80 This is basically like iterflatten, but it doesn't accumulate as much 81 material in strings. We need this here since stock nevow iterflatten 82 will block the server thread of extended periods of time (say, several 83 seconds) on large HTML tables. 84 85 Note that deferreds are not really supported (i.e., if you pass in 86 deferreds, they must already be ready). 87 """ 88 accu, curBytes = [], 0 89 for chunk in _flattenStan(stan, ctx): 90 accu.append(chunk) 91 curBytes += len(chunk) 92 if curBytes>chunkSize: 93 yield "".join(accu) 94 accu, curBytes = [], 0 95 yield "".join(accu)

96

97 98 -def streamStan(stan, ctx, destFile):

99 """writes strings made from stan to destFile. 100 """ 101 for chunk in iterStanChunked(stan, ctx, 50000): 102 destFile.write(chunk)

103

104 105 -def _iterWithReactor(iterable, finished, destFile):

106 """push out chunks coming out of iterable to destFile using a chain of 107 deferreds. 108 109 This is being done to yield to the reactor now and then. 110 """ 111 try: 112 destFile.write(iterable.next()) 113 except StopIteration: 114 finished.callback('') 115 except: 116 finished.errback() 117 else: 118 reactor.callLater(0, _iterWithReactor, iterable, finished, destFile)

119

120 121 -def deliverYielding(stan, ctx, request):

122 """delivers rendered stan to request, letting the reactor schedule 123 now and then. 124 """ 125 stanChunks = iterStanChunked(stan, ctx, 50000) 126 finished = defer.Deferred() 127 _iterWithReactor(stanChunks, finished, request) 128 return finished

129

130 131 -class ToFormalConverter(typesystems.FromSQLConverter):

132 """is a converter from SQL types to Formal type specifications. 133 134 The result of the conversion is a tuple of formal type and widget factory. 135 """ 136 typeSystem = "Formal" 137 simpleMap = { 138 "smallint": (formal.Integer, formal.TextInput), 139 "integer": (formal.Integer, formal.TextInput), 140 "int": (formal.Integer, formal.TextInput), 141 "bigint": (formal.Integer, formal.TextInput), 142 "real": (formal.Float, formal.TextInput), 143 "float": (formal.Float, formal.TextInput), 144 "boolean": (formal.Boolean, formal.Checkbox), 145 "double precision": (formal.Float, formal.TextInput), 146 "double": (formal.Float, formal.TextInput), 147 "text": (formal.String, formal.TextInput), 148 "unicode": (formal.String, formal.TextInput), 149 "char": (formal.String, formal.TextInput), 150 "date": (formal.Date, formal.widgetFactory(formal.DatePartsInput, 151 twoCharCutoffYear=50, dayFirst=True)), 152 "time": (formal.Time, formal.TextInput), 153 "timestamp": (formal.Date, formal.widgetFactory(formal.DatePartsInput, 154 twoCharCutoffYear=50, dayFirst=True)), 155 "vexpr-float": (formal.String, customwidgets.NumericExpressionField), 156 "vexpr-date": (formal.String, customwidgets.DateExpressionField), 157 "vexpr-string": (formal.String, customwidgets.StringExpressionField), 158 "vexpr-mjd": (formal.String, customwidgets.DateExpressionField), 159 "pql-string": (formal.String, formal.TextInput), 160 "pql-int": (formal.String, formal.TextInput), 161 "pql-float": (formal.String, formal.TextInput), 162 "pql-date": (formal.String, formal.TextInput), 163 "file": (formal.File, None), 164 "raw": (formal.String, formal.TextInput), 165 } 166

167 - def convert(self, type, xtype=None):

168 try: 169 return typesystems.FromSQLConverter.convert(self, type) 170 except base.ConversionError: 171 172 if xtype=="interval": 173 baseType, baseWidget = self.convert(type.rsplit('[', 1)[0]) 174 return (lambda **kw: customwidgets.PairOf(baseType, **kw), 175 customwidgets.Interval) 176 177 raise

178

179 - def mapComplex(self, type, length):

180 if type in self._charTypes: 181 return formal.String, formal.TextInput

182 183 184 sqltypeToFormal = ToFormalConverter().convert

185 186 187 -def _getFormalType(inputKey):

188 return sqltypeToFormal(inputKey.type, xtype=inputKey.xtype 189 )[0](required=inputKey.required)

190

191 192 -def _makeWithPlaceholder(origWidgetFactory, newPlaceholder):

193 """helps _addPlaceholder to keep the namespaces sane. 194 """ 195 if newPlaceholder is None: 196 return origWidgetFactory 197 198 class widgetFactory(origWidgetFactory): 199 placeholder = newPlaceholder

200 return widgetFactory 201

202 203 -def _getWidgetFactory(inputKey):

204 if not hasattr(inputKey, "_widgetFactoryCache"): 205 widgetFactory = inputKey.widgetFactory 206 207 if widgetFactory is None: 208 if inputKey.isEnumerated(): 209 widgetFactory = customwidgets.EnumeratedWidget(inputKey) 210 else: 211 widgetFactory = sqltypeToFormal(inputKey.type, inputKey.xtype)[1] 212 213 if isinstance(widgetFactory, basestring): 214 widgetFactory = customwidgets.makeWidgetFactory(widgetFactory) 215 216 inputKey._widgetFactoryCache = _makeWithPlaceholder(widgetFactory, 217 inputKey.getProperty("placeholder", None)) 218 return inputKey._widgetFactoryCache

219

220 221 -def getFieldArgsForInputKey(inputKey):

222 """returns a dictionary of keyword arguments for nevow formal 223 addField from a DaCHS InputKey. 224 """ 225 # infer whether to show a unit and if so, which 226 unit = "" 227 if inputKey.type!="date": # Sigh. 228 unit = inputKey.inputUnit or inputKey.unit or "" 229 if unit: 230 unit = " [%s]"%unit 231 label = inputKey.getLabel() 232 233 res = { 234 "name": inputKey.name, 235 "type": _getFormalType(inputKey), 236 "widgetFactory": _getWidgetFactory(inputKey), 237 "label": label+unit, 238 "description": inputKey.description, 239 "cssClass": inputKey.getProperty("cssClass", None),} 240 241 if inputKey.values and inputKey.values.default: 242 res["default"] = unicode(inputKey.values.default) 243 if inputKey.value: 244 res["default"] = unicode(inputKey.value) 245 246 return res

247

248 249 -class MultiField(formal.Group):

250 """A "widget" containing multiple InputKeys (i.e., formal Fields) in 251 a single line. 252 """

253

254 255 -class MultiFieldFragment(rend.Fragment):

256 """A fragment for rendering MultiFields. 257 """ 258 docFactory = loaders.stan( 259 T.div(class_=T.slot("class"), render=T.directive("multifield"))[ 260 T.label(for_=T.slot('id'))[T.slot('label')], 261 T.div(class_="multiinputs", id=T.slot('id'), 262 render=T.directive("childFields")), 263 T.div(class_='description')[T.slot('description')], 264 T.slot('message')]) 265

266 - def __init__(self, multiField):

267 rend.Fragment.__init__(self) 268 self.multiField = multiField

269

270 - def render_childFields(self, ctx, data):

271 formData = iformal.IFormData(ctx) 272 formErrors = iformal.IFormErrors(ctx, None) 273 274 for field in self.multiField.items: 275 widget = field.makeWidget() 276 if field.type.immutable: 277 render = widget.renderImmutable 278 else: 279 render = widget.render 280 cssClass = " ".join(s for s in (field.cssClass, "inmulti") if s) 281 ctx.tag[ 282 T.span(class_=cssClass)[ 283 render(ctx, field.key, formData, formErrors)( 284 class_=cssClass, title=field.description or "")]] 285 return ctx.tag

286

287 - def _getMessageElement(self, ctx):

288 errors = [] 289 formErrors = iformal.IFormErrors(ctx, None) 290 if formErrors is not None: 291 for field in self.multiField.items: 292 err = formErrors.getFieldError(field.key) 293 if err is not None: 294 errors.append(err.message) 295 if errors: 296 return T.div(class_='message')["; ".join(errors)] 297 else: 298 return ''

299

300 - def render_multifield(self, ctx, data):

301 ctx.tag.fillSlots('description', self.multiField.description or "") 302 ctx.tag.fillSlots('label', self.multiField.label or "") 303 ctx.tag.fillSlots('id', "multigroup-"+self.multiField.key) 304 errMsg = self._getMessageElement(ctx) 305 ctx.tag.fillSlots('message', errMsg) 306 if errMsg: 307 ctx.tag.fillSlots('class', 'field error') 308 else: 309 ctx.tag.fillSlots('class', 'field') 310 return ctx.tag

311 312 313 registerAdapter(MultiFieldFragment, MultiField, inevow.IRenderer)

314 315 316 -class FormMixin(formal.ResourceMixin):

317 """A mixin to produce input forms for services and display 318 errors within these forms. 319 """ 320 parameterStyle = "form" 321

322 - def _handleInputErrors(self, failure, ctx):

323 """goes as an errback to form handling code to allow correction form 324 rendering at later stages than validation. 325 """ 326 if isinstance(failure.value, formal.FormError): 327 self.form.errors.add(failure.value) 328 elif isinstance(failure.value, base.ValidationError) and isinstance( 329 failure.value.colName, basestring): 330 try: 331 # Find out the formal name of the failing field... 332 failedField = failure.value.colName 333 # ...and make sure it exists 334 self.form.items.getItemByName(failedField) 335 self.form.errors.add(formal.FieldValidationError( 336 str(failure.getErrorMessage()), failedField)) 337 except KeyError: # Failing field cannot be determined 338 self.form.errors.add(formal.FormError("Problem with input" 339 " in the internal or generated field '%s': %s"%( 340 failure.value.colName, failure.getErrorMessage()))) 341 else: 342 base.ui.notifyFailure(failure) 343 return failure 344 return self.form.errors

345

346 - def _addDefaults(self, ctx, form, additionalDefaults):

347 """adds defaults from request arguments (coming in via ctx) and defaults 348 from input keys (additionalDefaults). 349 350 This is mainly here so forms can be bookmarked. 351 """ 352 if ctx is None: # no request context, no arguments 353 return 354 args = additionalDefaults.copy() 355 args.update(inevow.IRequest(ctx).args) 356 357 # do remainig work in function as this can be recursive 358 def process(container): 359 for item in container.items: 360 if isinstance(item, formal.Group): 361 process(item) 362 else: 363 try: 364 form.data[item.key] = item.makeWidget().processInput( 365 ctx, item.key, args, item.default) 366 except: # don't fail on junky things in default arguments 367 pass

368 369 process(form)

370

371 - def _addInputKey(self, form, container, inputKey):

372 """adds a form field for an inputKey to the form. 373 """ 374 if inputKey.hasProperty("defaultForForm"): 375 self._defaultsForForm[inputKey.name 376 ] = [inputKey.getProperty("defaultForForm")] 377 container.addField(**getFieldArgsForInputKey(inputKey))

378

379 - def _groupQueryFields(self, inputTable):

380 """returns a list of "grouped" inputKey names from inputTable. 381 382 The idea here is that you can define "groups" in your input table. 383 Each such group can contain paramRefs. When the input table is rendered 384 in HTML, the grouped fields are created in a formal group. To make this 385 happen, they may need to be resorted. This happens in this function. 386 387 The returned list contains strings (parameter names), groups (meaning 388 "start a new group") and None (meaning end the current group). 389 390 This is understood and used by _addQueryFields. 391 """ 392 groupedKeys = {} 393 for group in inputTable.groups: 394 for ref in group.paramRefs: 395 groupedKeys[ref.key] = group 396 397 inputKeySequence, addedNames = [], set() 398 for inputKey in inputTable.inputKeys: 399 thisName = inputKey.name 400 401 if thisName in addedNames: 402 # part of a group and added as such 403 continue 404 405 newGroup = groupedKeys.get(thisName) 406 if newGroup is None: 407 # not part of a group 408 inputKeySequence.append(thisName) 409 addedNames.add(thisName) 410 else: 411 # current key is part of a group: add it and all others in the group 412 # enclosed in group/None. 413 inputKeySequence.append(newGroup) 414 for ref in groupedKeys[inputKey.name].paramRefs: 415 inputKeySequence.append(ref.key) 416 addedNames.add(ref.key) 417 inputKeySequence.append(None) 418 return inputKeySequence

419

420 - def _addQueryFieldsForInputTable(self, form, inputTable):

421 """generates input fields form the parameters of inputTable, taking 422 into account grouping if necessary. 423 """ 424 containers = [form] 425 for item in self._groupQueryFields(inputTable): 426 if item is None: # end of group 427 containers.pop() 428 429 elif isinstance(item, basestring): # param reference 430 self._addInputKey(form, containers[-1], 431 inputTable.inputKeys.getColumnByName(item)) 432 433 else: 434 # It's a new group -- if the group has a "style" property and 435 # it's "compact", use a special container form formal. 436 if item.getProperty("style", None)=="compact": 437 groupClass = MultiField 438 else: 439 groupClass = formal.Group 440 441 containers.append( 442 form.add(groupClass(item.name, description=item.description, 443 label=item.getProperty("label", None), 444 cssClass=item.getProperty("cssClass", None))))

445

446 - def _addQueryFields(self, form):

447 """adds the inputFields of the service to form, setting proper defaults 448 from the field or from data. 449 """ 450 # we have an inputTable. Handle groups and other fancy stuff 451 self._addQueryFieldsForInputTable(form, 452 self.service.getCoreFor(self).inputTable) 453 454 # and add the service keys manually as appropriate 455 for item in inputdef.filterInputKeys(self.service.serviceKeys, 456 self.name, inputdef.getRendererAdaptor(self)): 457 self._addInputKey(form, form, item)

458

459 - def _addMetaFields(self, form, queryMeta):

460 """adds fields to choose output properties to form. 461 """ 462 try: 463 if self.service.core.wantsTableWidget(): 464 form.addField("_DBOPTIONS", svcs.FormalDict, 465 formal.widgetFactory(svcs.DBOptions, self.service, queryMeta), 466 label="Table") 467 except AttributeError: # probably no wantsTableWidget method on core 468 pass

469

470 - def _getFormLinks(self):

471 """returns stan for widgets building GET-type strings for the current 472 form content. 473 """ 474 return T.div(class_="formLinks")[ 475 T.a(href="", class_="resultlink", onmouseover= 476 "this.href=makeResultLink(getEnclosingForm(this))", 477 render=T.directive("iflinkable")) 478 ["[Result link]"], 479 " ", 480 T.a(href="", class_="resultlink", onmouseover= 481 "this.href=makeBookmarkLink(getEnclosingForm(this))")[ 482 T.img(src=base.makeSitePath("/static/img/bookmark.png"), 483 class_="silentlink", title="Link to this form", alt="[bookmark]") 484 ], 485 ]

486 487

488 - def form_genForm(self, ctx=None, data=None):

489 # this is an accumulator for defaultForForm items processed; this 490 # is used below to pre-fill forms without influencing service 491 # behaviour in the absence of parameters. 492 self._defaultsForForm = {} 493 494 queryMeta = svcs.QueryMeta.fromContext(ctx) 495 form = formal.Form() 496 self._addQueryFields(form) 497 self._addMetaFields(form, queryMeta) 498 self._addDefaults(ctx, form, self._defaultsForForm) 499 500 if self.name=="form": 501 form.addField("_OUTPUT", formal.String, 502 formal.widgetFactory(serviceresults.OutputFormat, 503 self.service, queryMeta), 504 label="Output format") 505 506 form.actionURL = self.service.getURL(self.name) 507 form.addAction(self.submitAction, label="Go") 508 form.actionMaterial = self._getFormLinks() 509 self.form = form 510 return form

511

512 - def _realSubmitAction(self, ctx, form, data):

513 """helps submitAction by doing the real work. 514 515 It is here so we can add an error handler in submitAction. 516 """ 517 queryMeta = svcs.QueryMeta.fromContext(ctx) 518 519 if queryMeta["format"] in ("HTML", ""): 520 resultWriter = self 521 else: 522 resultWriter = serviceresults.getFormat(queryMeta["format"]) 523 524 if resultWriter.compute: 525 d = self.runServiceWithFormalData(data, ctx, queryMeta) 526 else: 527 d = defer.succeed(None) 528 529 return d.addCallback(resultWriter._formatOutput, ctx)

530

531 - def submitAction(self, ctx, form, data):

532 """executes the service. 533 534 This is a callback for the formal form. 535 """ 536 return defer.maybeDeferred( 537 self._realSubmitAction, ctx, form, data 538 ).addErrback(self._handleInputErrors, ctx)

539

540 541 -class Form(FormMixin, 542 grend.CustomTemplateMixin, 543 grend.HTMLResultRenderMixin, 544 grend.ServiceBasedPage):

545 """The "normal" renderer within DaCHS for web-facing services. 546 547 It will display a form and allow outputs in various formats. 548 549 It also does error reporting as long as that is possible within 550 the form. 551 """ 552 name = "form" 553 runOnEmptyInputs = False 554 compute = True 555

556 - def __init__(self, ctx, service):

557 grend.ServiceBasedPage.__init__(self, ctx, service) 558 if "form" in self.service.templates: 559 self.customTemplate = self.service.getTemplate("form") 560 561 # enable special handling if I'm rendering fixed-behaviour services 562 # (i.e., ones that never have inputs) XXX TODO: Figure out where I used this and fix that to use the fixed renderer (or whatever) 563 if not self.service.getInputKeysFor(self): 564 self.runOnEmptyInputs = True 565 self.queryResult = None

566 567 @classmethod

568 - def isBrowseable(self, service):

569 return True

570 571 @classmethod

572 - def isCacheable(self, segments, request):

573 return segments==()

574

575 - def renderHTTP(self, ctx):

576 if self.runOnEmptyInputs: 577 inevow.IRequest(ctx).args[formal.FORMS_KEY] = ["genForm"] 578 return FormMixin.renderHTTP(self, ctx)

579

580 - def _formatOutput(self, res, ctx):

581 """actually delivers the whole document. 582 583 This is basically nevow's rend.Page._renderHTTP, changed to 584 provide less blocks. 585 """ 586 request = inevow.IRequest(ctx) 587 588 if isinstance(res.original, tuple): 589 # core returned a complete document (mime and string) 590 mime, payload = res.original 591 request.setHeader("content-type", mime) 592 request.setHeader('content-disposition', 593 'attachment; filename=result%s'%formats.getExtensionFor(mime)) 594 return streaming.streamOut(lambda f: f.write(payload), 595 request) 596 597 self.result = res 598 if "response" in self.service.templates: 599 self.customTemplate = self.service.getTemplate("response") 600 601 ctx = context.PageContext(parent=ctx, tag=self) 602 self.rememberStuff(ctx) 603 doc = self.docFactory.load(ctx) 604 ctx = context.WovenContext(ctx, T.invisible[doc]) 605 606 return deliverYielding(doc, ctx, request)

607 608 defaultDocFactory = svcs.loadSystemTemplate("defaultresponse.html")

609

610 611 -class DocFormRenderer(FormMixin, grend.ServiceBasedPage, 612 grend.HTMLResultRenderMixin):

613 """A renderer displaying a form and delivering core's result as 614 a document. 615 616 The core must return a pair of mime-type and content; on errors, 617 the form is redisplayed. 618 619 This is mainly useful with custom cores doing weird things. This 620 renderer will not work with dbBasedCores and similar. 621 """ 622 name="docform" 623 # I actually don't know the result type, since it's determined by the 624 # core; I probably should have some way to let the core tell me what 625 # it's going to return. 626 resultType = "application/octet-stream" 627 compute = True 628 629 @classmethod

630 - def isBrowseable(cls, service):

631 return True

632

633 - def _formatOutput(self, data, ctx):

634 request = inevow.IRequest(ctx) 635 mime, payload = data.original 636 request.setHeader("content-type", mime) 637 request.write(payload) 638 return ""

639 640 docFactory = svcs.loadSystemTemplate("defaultresponse.html")

641

Source Code for Module gavo.web.formrender