gavo.base.activetags

1 """ 2 Active tags are used in prepare and insert computed material into RD trees. 3 4 And, ok, we are dealing with elements here rather than tags, but I liked 5 the name "active tags" much better, and there's too much talk of elements 6 in this source as it is. 7 8 The main tricky part with active tags is when they're nested. In 9 short, active tags are expanded even when within active tags. So, 10 if you write:: 11 12 <STREAM id="foo"> 13 <LOOP> 14 </LOOP> 15 </STREAM> 16 17 foo contains not a loop element but whatever that spit out. In particular, 18 macros within the loop are expanded not within some FEED element but 19 within the RD. 20 """ 21 22 #c Copyright 2008-2019, the GAVO project 23 #c 24 #c This program is free software, covered by the GNU GPL. See the 25 #c COPYING file in the source distribution. 26 27 28 import csv 29 import re 30 from cStringIO import StringIO 31 32 from gavo import utils 33 from gavo.base import attrdef 34 from gavo.base import common 35 from gavo.base import complexattrs 36 from gavo.base import macros 37 from gavo.base import parsecontext 38 from gavo.base import structure 39 40 41 # the following is a sentinel for values that have been expanded 42 # by an active tag already. When active tags are nested, only the 43 # innermost active tag must expand macros so one can be sure that 44 # double-escaped macros actually end up when the events are finally 45 # replayed at the top level. _EXPANDED_VALUE must compare true to 46 # value since it is used as such in event triples. 47

48 -class _ExValueType(object):

49 - def __str__(self):

50 return "value"

51

52 - def __repr__(self):

53 return "'value/expanded'"

54

55 - def __eq__(self, other):

56 return other=="value"

57

58 - def __ne__(self, other):

59 return not other=="value"

60 61 _EXPANDED_VALUE =_ExValueType() 62 63

64 -class ActiveTag(object):

65 """A mixin for active tags. 66 67 This is usually mixed into structure.Structures or derivatives. It 68 is also used as a sentinel to find all active tags below. 69 """ 70 name_ = None 71

72 - def _hasActiveParent(self):

73 el = self.parent 74 while el: 75 if isinstance(el, ActiveTag): 76 return True 77 el = el.parent 78 return False

79 80

81 -class GhostMixin(object):

82 """A mixin to make a Structure ghostly. 83 84 Most active tags are "ghostly", i.e., the do not (directly) 85 show up in their parents. Therefore, as a part of the wrap-up 86 of the new element, we raise an Ignore exception, which tells 87 the Structure's end_ method to not feed us to the parent. 88 """

89 - def onElementComplete(self):

90 self._onElementCompleteNext(GhostMixin) 91 raise common.Ignore(self)

92 93

94 -class _PreparedEventSource(object):

95 """An event source for xmlstruct. 96 97 It is constructed with a list of events as recorded by classes 98 inheriting from RecordingBase. 99 """

100 - def __init__(self, events):

101 self.events_ = events 102 self.curEvent = -1 103 self.pos = None

104

105 - def __iter__(self):

106 return _PreparedEventSource(self.events_)

107

108 - def next(self):

109 self.curEvent += 1 110 try: 111 nextItem = self.events_[self.curEvent] 112 except IndexError: 113 raise StopIteration() 114 res, self.pos = nextItem[:3], nextItem[-1] 115 return res

116 117

118 -class Defaults(structure.Structure):

119 """Defaults for macros. 120 121 In STREAMs and NXSTREAMs, DEFAULTS let you specify values filled into 122 macros when a FEED doesn't given them. Macro names are attribute names 123 (or element names, if you insist), defaults are their values. 124 """ 125 name_ = "DEFAULTS" 126

127 - def __init__(self, *args, **kwargs):

128 self.defaults = {} 129 structure.Structure.__init__(self, *args, **kwargs)

130

131 - def start_(self, ctx, name, value):

132 return self

133

134 - def value_(self, ctx, name, value):

135 if name=="content_": 136 self.storedContent = value 137 else: 138 self.defaults[name] = value 139 return self

140

141 - def end_(self, ctx, name, value):

142 if name=="DEFAULTS": 143 self.finishElement(ctx) 144 self.parent.feedObject("DEFAULTS", self) 145 return self.parent 146 else: 147 self.defaults[name] = self.storedContent 148 del self.storedContent 149 return self

150 151

152 -class RecordingBase(structure.Structure):

153 """An "abstract base" for active tags doing event recording. 154 155 The recorded events are available in the events attribute. 156 """ 157 name_ = None 158 159 _doc = attrdef.UnicodeAttribute("doc", description="A description of" 160 " this stream (should be restructured text).", strip=False) 161 _defaults = complexattrs.StructAttribute("DEFAULTS", 162 childFactory=Defaults, description="A mapping giving" 163 " defaults for macros expanded in this stream. Macros" 164 " not defaulted will fail when not given in a FEED's attributes.", 165 default=None) 166

167 - def __init__(self, *args, **kwargs):

168 self.events_ = [] 169 self.tagStack_ = [] 170 structure.Structure.__init__(self, *args, **kwargs)

171

172 - def feedEvent(self, ctx, type, name, value):

173 # keep _EXPANDED_VALUE rather than feed the value to protect it from 174 # further expansion by subordinate structures (except if we, the 175 # active tag, is the final recipient, in which case we gobble the thing 176 # ourselves). 177 if (type is _EXPANDED_VALUE and name not in self.managedAttrs): 178 self.events_.append((_EXPANDED_VALUE, name, value, ctx.pos)) 179 return self 180 else: 181 return structure.Structure.feedEvent(self, ctx, type, name, value)

182

183 - def start_(self, ctx, name, value):

184 if name in self.managedAttrs and not self.tagStack_: 185 res = structure.Structure.start_(self, ctx, name, value) 186 else: 187 self.events_.append(("start", name, value, ctx.pos)) 188 res = self 189 self.tagStack_.append(name) 190 return res

191

192 - def end_(self, ctx, name, value):

193 if name in self.managedAttrs and not self.tagStack_: 194 structure.Structure.end_(self, ctx, name, value) 195 else: 196 self.events_.append(("end", name, value, ctx.pos)) 197 self.tagStack_.pop() 198 return self

199

200 - def value_(self, ctx, name, value):

201 if name in self.managedAttrs and not self.tagStack_: 202 # our attribute 203 structure.Structure.value_(self, ctx, name, value) 204 else: 205 self.events_.append(("value", name, value, ctx.pos)) 206 return self

207

208 - def getEventSource(self):

209 """returns an object suitable as event source in xmlstruct. 210 """ 211 return _PreparedEventSource(self.events_)

212

213 - def unexpandMacros(self):

214 """undoes the marking of expanded values as expanded. 215 216 This is when, as with mixins, duplicate expansion of macros during 217 replay is desired. 218 """ 219 for ind, ev in enumerate(self.events_): 220 if ev[0]==_EXPANDED_VALUE: 221 self.events_[ind] = ("value",)+ev[1:]

222 223 # This lets us feedFrom these 224 iterEvents = getEventSource

225 226

227 -class EventStream(RecordingBase, GhostMixin, ActiveTag):

228 """An active tag that records events as they come in. 229 230 Their only direct effect is to leave a trace in the parser's id map. 231 The resulting event stream can be played back later. 232 """ 233 name_ = "STREAM" 234

235 - def end_(self, ctx, name, value):

236 # keep self out of the parse tree 237 if not self.tagStack_: # end of STREAM element 238 res = self.parent 239 self.parent = None 240 return res 241 return RecordingBase.end_(self, ctx, name, value)

242 243

244 -class RawEventStream(EventStream):

245 """An event stream that records events, not expanding active tags. 246 247 Normal event streams expand embedded active tags in place. This is 248 frequently what you want, but it means that you cannot, e.g., fill 249 in loop variables through stream macros. 250 251 With non-expanded streams, you can do that:: 252 253 <NXSTREAM id="cols"> 254 <LOOP listItems="\stuff"> 255 <events> 256 <column name="\\item"/> 257 </events> 258 </LOOP> 259 </NXSTREAM> 260 <table id="foo"> 261 <FEED source="cols" stuff="x y"/> 262 </table> 263 264 Note that the normal innermost-only rule for macro expansions 265 within active tags does not apply for NXSTREAMS. Macros expanded 266 by a replayed NXSTREAM will be re-expanded by the next active 267 tag that sees them (this is allow embedded active tags to use 268 macros; you need to double-escape macros for them, of course). 269 """ 270 271 name_ = "NXSTREAM" 272 273 # Hack to signal xmlstruct.EventProcessor not to expand active tags here 274 ACTIVE_NOEXPAND = None

275 276

277 -class EmbeddedStream(RecordingBase, structure.Structure):

278 """An event stream as a child of another element. 279 """ 280 name_ = "events" # Lower case since it's really a "normal" element that's 281 # added into the parse tree. 282 _passivate = attrdef.ActionAttribute("passivate", 283 methodName="_makePassive", description="If set to True, do not expand" 284 " active elements immediately in the body of these events" 285 " (as in an NXSTREAM)") 286

287 - def _makePassive(self, ctx):

288 if self.passivate.lower()=="true": 289 self.ACTIVE_NOEXPAND = True

290

291 - def end_(self, ctx, name, value):

292 if not self.tagStack_: # end of my element, do standard structure thing. 293 return structure.Structure.end_(self, ctx, name, value) 294 return RecordingBase.end_(self, ctx, name, value)

295 296

297 -class Prune(ActiveTag, structure.Structure):

298 """An active tag that lets you selectively delete children of the 299 current object. 300 301 You give it regular expression-valued attributes; on the replay of 302 the stream, matching items and their children will not be replayed. 303 304 If you give more than one attribute, the result will be a conjunction 305 of the specified conditions. 306 307 This only works if the items to be matched are true XML attributes 308 (i.e., not written as children). 309 """ 310 name_ = "PRUNE" 311

312 - def __init__(self, parent, **kwargs):

313 self.conds = {} 314 structure.Structure.__init__(self, parent)

315

316 - def value_(self, ctx, name, value):

317 self.conds[name] = value 318 return self

319

320 - def end_(self, ctx, name, value):

321 assert name==self.name_ 322 self.matches = self._getMatcher() 323 self.parent.feedObject(self.name_, self) 324 return self.parent

325

326 - def _getMatcher(self):

327 """returns a callabe that takes a dictionary and matches the 328 entries against the conditions given. 329 """ 330 conditions = [] 331 for attName, regEx in self.conds.iteritems(): 332 conditions.append((attName, re.compile(regEx))) 333 334 def match(aDict): 335 for attName, expr in conditions: 336 val = aDict.get(attName) 337 if val is None: # not given or null empty attrs never match 338 return False 339 if not expr.search(val): 340 return False 341 return True

342 343 return match

344 345

346 -class Edit(EmbeddedStream):

347 """an event stream targeted at editing other structures. 348 349 When replaying a stream in the presence of EDITs, the elements are 350 are continually checked against ref. If an element matches, the 351 children of edit will be played back into it. 352 """ 353 name_ = "EDIT" 354 355 _ref = attrdef.UnicodeAttribute("ref", description="Destination of" 356 " the edits, in the form elementName[<name or id>]", 357 default=utils.Undefined) 358 359 refPat = re.compile( 360 r"([A-Za-z_][A-Za-z0-9_]*)\[([A-Za-z_][A-Za-z0-9_]*)\]") 361

362 - def onElementComplete(self):

363 mat = self.refPat.match(self.ref) 364 if not mat: 365 raise common.LiteralParseError("ref", self.ref, 366 hint="edit references have the form <element name>[<value of" 367 " name or id attribute>]") 368 self.triggerEl, self.triggerId = mat.groups()

369 370

371 -class ReplayBase(ActiveTag, structure.Structure, macros.StandardMacroMixin):

372 """An "abstract base" for active tags replaying streams. 373 """ 374 name_ = None # not a usable active tag 375 _expandMacros = True 376 377 _source = parsecontext.ReferenceAttribute("source", 378 description="id of a stream to replay", default=None) 379 _events = complexattrs.StructAttribute("events", 380 childFactory=EmbeddedStream, default=None, 381 description="Alternatively to source, an XML fragment to be replayed") 382 _edits = complexattrs.StructListAttribute("edits", 383 childFactory=Edit, description="Changes to be performed on the" 384 " events played back.") 385 _reexpand = attrdef.BooleanAttribute("reexpand", False, 386 description="Force re-expansion of macros; usually, when replaying," 387 " each string is only expanded once, mainly to avoid overly long" 388 " backslash-fences. Set this to true to force further expansion.") 389 _prunes = complexattrs.StructListAttribute("prunes", 390 childFactory=Prune, description="Conditions for removing" 391 " items from the playback stream.") 392

393 - def _ensureEditsDict(self):

394 if not hasattr(self, "editsDict"): 395 self.editsDict = {} 396 for edit in self.edits: 397 self.editsDict[edit.triggerEl, edit.triggerId] = edit

398

399 - def _isPruneable(self, val):

400 for p in self.prunes: 401 if p.matches(val): 402 return True 403 return False

404

405 - def _replayTo(self, events, evTarget, ctx):

406 """pushes stored events into an event processor. 407 408 The public interface is replay (that receives a structure rather 409 than an event processor). 410 """ 411 idStack = [] 412 pruneStack = [] 413 414 # see RawEventStream's docstring for why we do not want to suppress 415 # further expansion with NXSTREAMs 416 typeOfExpandedValues = _EXPANDED_VALUE 417 if isinstance(self.source, RawEventStream): 418 typeOfExpandedValues = "value" 419 420 with ctx.replaying(): 421 for type, name, val, pos in events: 422 if (self._expandMacros 423 and type=="value" 424 and (self.reexpand or type is not _EXPANDED_VALUE) 425 and "\\" in val): 426 try: 427 val = self.expand(val) 428 except macros.MacroError as ex: 429 ex.hint = ("This probably means that you should have set a %s" 430 " attribute in the FEED tag. For details see the" 431 " documentation of the STREAM with id %s."%( 432 ex.macroName, 433 getattr(self.source, "id", "<embedded>"))) 434 raise 435 type = typeOfExpandedValues 436 437 # the following mess implements the logic for EDIT. 438 if type=="start": 439 idStack.append(set()) 440 elif type=="value": 441 if idStack and name=="id" or name=="name": 442 idStack[-1].add(val) 443 elif type=="end": 444 ids = idStack.pop() 445 for foundId in ids: 446 if (name, foundId) in self.editsDict: 447 self._replayTo(self.editsDict[name, foundId].events_, 448 evTarget, 449 ctx) 450 451 # The following mess implements the logic for PRUNE 452 if type=="start": 453 if pruneStack: 454 pruneStack.append(None) 455 else: 456 if self.prunes and self._isPruneable(val): 457 pruneStack.append(None) 458 459 try: 460 if not pruneStack: 461 evTarget.feed(type, name, val) 462 except Exception as msg: 463 msg.pos = "%s (replaying, real error position %s)"%( 464 ctx.pos, pos) 465 msg.posInMsg = True 466 raise 467 468 if pruneStack and type=="end": 469 pruneStack.pop() 470 471 # ReferenceAttribute and similar may change the element fed into; 472 # make sure the right object is returned up-tree 473 self.parent = evTarget.curParser

474

475 - def replay(self, events, destination, ctx):

476 """pushes the stored events into the destination structure. 477 478 While doing this, local macros are expanded unless we already 479 receive the events from an active tag (e.g., nested streams 480 and such). 481 """ 482 # XXX TODO: Circular import here. Think again and resolve. 483 from gavo.base.xmlstruct import EventProcessor 484 evTarget = EventProcessor(None, ctx) 485 evTarget.setRoot(destination) 486 487 self._ensureEditsDict() 488 self._replayTo(events, evTarget, ctx)

489 490

491 -class DelayedReplayBase(ReplayBase, GhostMixin):

492 """An base class for active tags wanting to replay streams from 493 where the context is invisible. 494 495 These define a _replayer attribute that, when called, replays 496 the stored events *within the context at its end* and to the 497 parent. 498 499 This is what you want for the FEED and LOOP since they always work 500 on the embedding element and, by virtue of being ghosts, cannot 501 be copied. If the element embedding an event stream can be 502 copied, this will almost certainly not do what you want. 503 """

504 - def _setupReplay(self, ctx):

505 sources = [s for s in [self.source, self.events] if s] 506 if len(sources)!=1: 507 raise common.StructureError("Need exactly one of source and events" 508 " on %s elements"%self.name_) 509 stream = sources[0].events_ 510 def replayer(): 511 self.replay(stream, self.parent, ctx)

512 self._replayer = replayer

513

514 - def end_(self, ctx, name, value):

515 self._setupReplay(ctx) 516 return structure.Structure.end_(self, ctx, name, value)

517 518

519 -class ReplayedEventsWithFreeAttributesBase(DelayedReplayBase):

520 """An active tag that takes arbitrary attributes as macro definitions. 521 """

522 - def __init__(self, *args, **kwargs):

523 DelayedReplayBase.__init__(self, *args, **kwargs) 524 # managedAttrs in general is a class attribute. Here, we want 525 # to add values for the macros, and these are instance-local. 526 self.managedAttrs = self.managedAttrs.copy()

527

528 - def completeElement(self, ctx):

529 # define any missing macros that still are in defaults. 530 if self.source and self.source.DEFAULTS is not None: 531 for key, value in self.source.DEFAULTS.defaults.iteritems(): 532 if not hasattr(self, "macro_"+key): 533 setattr(self, "macro_"+key, lambda v=value: v) 534 self._completeElementNext(ReplayedEventsWithFreeAttributesBase, ctx)

535

536 - def getAttribute(self, name):

537 try: 538 return DelayedReplayBase.getAttribute(self, name) 539 except common.StructureError: # no "real" attribute, it's a macro def 540 def m(): 541 return getattr(self, name)

542 setattr(self, "macro_"+name.strip(), m) 543 self.managedAttrs[name] = attrdef.UnicodeAttribute(name) 544 return self.managedAttrs[name]

545 546

547 -class ReplayedEvents(ReplayedEventsWithFreeAttributesBase):

548 """An active tag that takes an event stream and replays the events, 549 possibly filling variables. 550 551 This element supports arbitrary attributes with unicode values. These 552 values are available as macros for replayed values. 553 """ 554 name_ = "FEED" 555

556 - def completeElement(self, ctx):

557 self._completeElementNext(ReplayedEvents, ctx) 558 self._replayer()

559 560

561 -class NonExpandedReplayedEvents(ReplayedEvents):

562 """A ReplayedEventStream that does not expand active tag macros. 563 564 You only want this when embedding a stream into another stream 565 that could want to expand the embedded macros. 566 """ 567 name_ = "LFEED" 568 _expandMacros = False

569 570

571 -class GeneratorAttribute(attrdef.UnicodeAttribute):

572 """An attribute containing a generator working on the parse context. 573 """

574 - def feed(self, ctx, instance, literal):

575 if ctx.restricted: 576 raise common.RestrictedElement("codeItems") 577 attrdef.UnicodeAttribute.feed(self, ctx, instance, literal) 578 src = utils.fixIndentation( 579 getattr(instance, self.name_), 580 " ", governingLine=1) 581 src = "def makeRows():\n"+src+"\n" 582 instance.iterRowsFromCode = utils.compileFunction( 583 src, "makeRows", useGlobals={"context": ctx})

584 585

586 -class Loop(ReplayedEventsWithFreeAttributesBase):

587 """An active tag that replays a feed several times, each time with 588 different values. 589 """ 590 name_ = "LOOP" 591 592 _csvItems = attrdef.UnicodeAttribute("csvItems", default=None, 593 description="The items to loop over, in CSV-with-labels format.", 594 strip=True) 595 _listItems = attrdef.UnicodeAttribute("listItems", default=None, 596 description="The items to loop over, as space-separated single" 597 " items. Each item will show up once, as 'item' macro.", 598 strip=True) 599 _codeItems = GeneratorAttribute("codeItems", default=None, 600 description="A python generator body that yields dictionaries" 601 " that are then used as loop items. You can access the parse context" 602 " as the context variable in these code snippets.", strip=False) 603

604 - def maybeExpand(self, val):

605 if "\\" in val: 606 el = self.parent 607 while el: 608 if hasattr(el, "expand"): 609 return el.expand(val) 610 el = el.parent 611 return val

612

613 - def _makeRowIteratorFromListItems(self):

614 if self.listItems is None: 615 return None 616 def rowIterator(): 617 for item in self.maybeExpand(self.listItems).split(): 618 yield {"item": item}

619 return rowIterator()

620

621 - def _makeRowIteratorFromCSV(self):

622 if self.csvItems is None: 623 return None 624 # I'd rather not do the encode below, but 2.7 csv can't handle 625 # unicode. We'll need to decode stuff again. 626 src = self.maybeExpand(self.csvItems).strip().encode("utf-8") 627 628 def encodeValues(row): 629 return dict((key, str(val).decode("utf-8")) 630 for key, val in row.iteritems())

631 632 return (encodeValues(row) 633 for row in csv.DictReader(StringIO(src), skipinitialspace=True)) 634

635 - def _makeRowIteratorFromCode(self):

636 if self.codeItems is None: 637 return None 638 return self.iterRowsFromCode()

639

640 - def _getRowIterator(self):

641 rowIterators = [ri for ri in [ 642 self._makeRowIteratorFromListItems(), 643 self._makeRowIteratorFromCSV(), 644 self._makeRowIteratorFromCode()] if ri] 645 if len(rowIterators)!=1: 646 raise common.StructureError("Must give exactly one data source in" 647 " LOOP") 648 return rowIterators[0]

649

650 - def completeElement(self, ctx):

651 self._completeElementNext(Loop, ctx) 652 for row in self._getRowIterator(): 653 for name, value in row.iteritems(): 654 if value: 655 value = value.strip() 656 if name is None: 657 raise utils.StructureError( 658 "Too many CSV items (extra data: %s)"%value) 659 setattr(self, "macro_"+name.strip(), lambda v=value: v) 660 self._replayer()

661 662 663 getActiveTag = utils.buildClassResolver(ActiveTag, globals().values(), 664 key=lambda obj: getattr(obj, "name_", None)) 665 666

667 -def registerActiveTag(activeTag):

668 getActiveTag.registry[activeTag.name_] = activeTag

669 670

671 -def isActive(name):

672 return name in getActiveTag.registry

673

Source Code for Module gavo.base.activetags