gavo.base.structure

1 """ 2 Representation of structured data deserializable from XML. 3 4 We want all the managed attribute stuff since the main user input comes 5 from resource descriptors, and we want relatively strong input validation 6 here. Also, lots of fancy copying and crazy cross-referencing is 7 going on in our resource definitions, so we want a certain amount of 8 rigorous structure. Finally, a monolithic parser for that stuff 9 becomes *really* huge and tedious, so I want to keep the XML parsing 10 information in the constructed objects themselves. 11 """ 12 13 #c Copyright 2008-2019, the GAVO project 14 #c 15 #c This program is free software, covered by the GNU GPL. See the 16 #c COPYING file in the source distribution. 17 18 19 import new 20 21 from gavo import utils 22 from gavo.base import attrdef 23 from gavo.base import common 24 from gavo.base import parsecontext

25 26 27 -def sortAttrs(attrSeq):

28 """evaluates the before attributes on the AttributeDefs in attrsSeq 29 and returns a sequence satisfying them. 30 31 It returns a reference to attrSeq for convenience. 32 """ 33 beforeGraph, prependMeta = [], False 34 for att in attrSeq: 35 if att.before: 36 beforeGraph.append((att.name_, att.before)) 37 if att.name_=="meta_": 38 prependMeta = True 39 40 if beforeGraph: 41 attDict = dict((a.name_, a) for a in attrSeq) 42 sortedNames = utils.topoSort(beforeGraph) 43 44 # Hack: metadata always comes first 45 if prependMeta: 46 sortedNames[:0] = ["meta_"] 47 48 sortedAtts = [attDict[n] for n in sortedNames] 49 attrSeq = sortedAtts+list(set(attrSeq)-set(sortedAtts)) 50 return attrSeq

51

52 53 -class StructType(type):

54 """is a metaclass for the representation of structured data. 55 56 StructType classes with this will be called structures within 57 the DC software. 58 59 Structures do quite a bit of the managed attribute nonsense to 60 meaningfully catch crazy user input. 61 62 Basically, you give a Structure class attributes (preferably with 63 underscores in front) specifying the attributes the instances 64 should have and how they should be handled. 65 66 Structures must be constructed with a parent (for the root 67 element, this is None). All other arguments should be keyword 68 arguments. If given, they have to refer to existing attributes, 69 and their values will directly give the the values of the 70 attribute (i.e., parsed values). 71 72 Structures should always inherit from StructBase below and 73 arrange for its constructor to be called, since, e.g., default 74 processing happens there. 75 76 Structures have a managedAttrs dictionary containing names and 77 attrdef.AttributeDef objects for the defined attributes. 78 """

79 - def __init__(cls, name, bases, dict):

80 type.__init__(cls, name, bases, dict) 81 cls._collectManagedAttrs() 82 cls._insertAttrMethods()

83

84 - def _collectManagedAttrs(cls):

85 """collects a dictionary of managed attributes in managedAttrs. 86 """ 87 managedAttrs, completedCallbacks, attrSeq = {}, [], [] 88 for name in dir(cls): 89 if not hasattr(cls, name): 90 continue 91 val = getattr(cls, name) 92 93 if isinstance(val, attrdef.AttributeDef): 94 managedAttrs[val.name_] = val 95 attrSeq.append(val) 96 if hasattr(val, "xmlName_"): 97 managedAttrs[val.xmlName_] = val 98 if val.aliases: 99 for alias in val.aliases: 100 managedAttrs[alias] = val 101 cls.attrSeq = sortAttrs(attrSeq) 102 cls.managedAttrs = managedAttrs 103 cls.completedCallbacks = completedCallbacks

104

105 - def _insertAttrMethods(cls):

106 """adds methods defined by cls's managedAttrs for the parent to 107 cls. 108 """ 109 for val in set(cls.managedAttrs.itervalues()): 110 for name, meth in val.iterParentMethods(): 111 if isinstance(meth, property): 112 setattr(cls, name, meth) 113 else: 114 setattr(cls, name, new.instancemethod(meth, None, cls))

115

116 117 -class DataContent(attrdef.UnicodeAttribute):

118 """A magic attribute that allows character content to be added to 119 a structure. 120 121 You can configure it with all the arguments available for UnicodeAttribute. 122 123 Since parsers may call characters with an empty string for 124 empty elements, the empty string will not be fed (i.e., the default 125 will be preserved). This makes setting an empty string as an element content 126 impossible (you could use DataContent with strip=True, though), but that's 127 probably not a problem. 128 """ 129 typeDesc_ = "string" 130

131 - def __init__(self, default="", 132 description="Undocumented", **kwargs):

133 attrdef.UnicodeAttribute.__init__(self, "content_", default=default, 134 description=description, **kwargs)

135

136 - def feed(self, ctx, instance, value):

137 if value=='': 138 return 139 return attrdef.UnicodeAttribute.feed(self, ctx, instance, value)

140

141 - def makeUserDoc(self):

142 return ("Character content of the element (defaulting to %s) -- %s"%( 143 repr(self.default_), self.description_))

144

145 146 -class StructureBase(object):

147 """is a base class for all structures. 148 149 You must arrange for calling its constructor from classes inheriting 150 this. 151 152 The constructor receives a parent (another structure, or None) 153 and keyword arguments containing values for actual attributes 154 (which will be set without any intervening consultation of the 155 AttributeDef). 156 157 The attribute definitions talking about structures let you 158 set parent to None when constructing default values; they will 159 then insert the actual parent. 160 """ 161 162 __metaclass__ = StructType 163 164 name_ = attrdef.Undefined 165 166 _id = parsecontext.IdAttribute("id", 167 description="Node identity for referencing") 168 169 # the following is managed by setPosition/getSourcePosition 170 __fName = __lineNumber = None 171

172 - def __init__(self, parent, **kwargs):

173 self.parent = parent 174 175 # set defaults 176 for val in self.attrSeq: 177 try: 178 if not hasattr(self, val.name_): # don't clobber properties 179 # set up by attributes. 180 setattr(self, val.name_, val.default_) 181 except AttributeError: # default on property given 182 raise utils.logOldExc(common.StructureError( 183 "%s attributes on %s have builtin defaults only."%( 184 val.name_, self.name_))) 185 186 # set keyword arguments 187 for name, val in kwargs.iteritems(): 188 if name in self.managedAttrs: 189 if not hasattr(self.managedAttrs[name], "computed_"): 190 self.managedAttrs[name].feedObject(self, val) 191 else: 192 raise common.StructureError("%s objects have no attribute %s"%( 193 self.__class__.__name__, name))

194

195 - def _nop(self, *args, **kwargs):

196 pass

197

198 - def setPosition(self, fName, lineNumber):

199 """should be called by parsers to what file at what line the 200 serialisation came from. 201 """ 202 self.__fName, self.__lineNumber = fName, lineNumber

203

204 - def getSourcePosition(self):

205 """returns a string representation of where the struct was parsed 206 from. 207 """ 208 if self.__fName is None: 209 return "<internally built>" 210 else: 211 return "%s, line %s"%(self.__fName, self.__lineNumber)

212

213 - def getAttributes(self, attDefsFrom=None):

214 """returns a dict of the current attributes, suitable for making 215 a shallow copy of self. 216 217 Struct attributes will not be reparented, so there are limits to 218 what you can do with such shallow copies. 219 """ 220 if attDefsFrom is None: 221 attrs = set(self.managedAttrs.values()) 222 else: 223 attrs = set(attDefsFrom.managedAttrs.itervalues()) 224 try: 225 return dict([(att.name_, getattr(self, att.name_)) 226 for att in attrs]) 227 except AttributeError as msg: 228 raise common.logOldExc(common.StructureError( 229 "Attempt to copy from invalid source: %s"%unicode(msg)))

230

231 - def getCopyableAttributes(self, ignoreKeys=set(), ctx=None):

232 """returns a dictionary mapping attribute names to copyable children. 233 234 ignoreKeys can be a set or dict of additional attribute names to ignore. 235 The children are orphan deep copies. 236 """ 237 return dict((att.name_, att.getCopy(self, None, ctx)) 238 for att in self.attrSeq 239 if att.copyable and att.name_ not in ignoreKeys)

240

241 - def change(self, **kwargs):

242 """returns a copy of self with all attributes in kwargs overridden with 243 the passed values. 244 """ 245 parent = kwargs.pop("parent_", self.parent) 246 runExits, ctx = False, kwargs.pop("ctx", None) 247 if ctx is None: 248 runExits, ctx = True, parsecontext.ParseContext() 249 250 attrs = self.getCopyableAttributes(kwargs, ctx) 251 attrs.update(kwargs) 252 253 newInstance = self.__class__(parent, **attrs).finishElement(ctx) 254 255 # reparent things without a parent to newInstance. We don't want to do 256 # this unconditionally since we unwisely share some structs. 257 for name, value in kwargs.iteritems(): 258 if not isinstance(value, list): 259 value = [value] 260 for item in value: 261 if hasattr(item, "parent") and item.parent is None: 262 item.parent = newInstance 263 264 if runExits: 265 ctx.runExitFuncs(newInstance) 266 return newInstance

267

268 - def copy(self, parent, ctx=None):

269 """returns a deep copy of self, reparented to parent. 270 271 This is a shallow wrapper around change, present for backward 272 compatibility. 273 """ 274 return self.change(parent_=parent, ctx=ctx)

275

276 - def adopt(self, struct):

277 struct.parent = self 278 return struct

279

280 - def iterChildren(self):

281 """iterates over structure children of self. 282 283 To make this work, attributes containing structs must define 284 iterChildren methods (and the others must not). 285 """ 286 for att in self.attrSeq: 287 if hasattr(att, "iterChildren"): 288 for c in att.iterChildren(self): 289 yield c

290 291 @classmethod

292 - def fromStructure(cls, newParent, oldStructure):

293 consArgs = dict([(att.name_, getattr(oldStructure, att.name_)) 294 for att in oldStructure.attrSeq]) 295 return cls(newParent, **consArgs)

296

297 - def breakCircles(self):

298 """removes the parent attributes from all child structures recusively. 299 300 The struct will probably be broken after this, but this is sometimes 301 necessary to help the python garbage collector. 302 303 In case you're asking: parent cannot be a weak reference with the current 304 parse architecture, as it usually is the only reference to the embedding 305 object. Yes, we should probably change that. 306 """ 307 for child in self.iterChildren(): 308 # we don't want to touch structs that aren't our children 309 if hasattr(child, "parent") and child.parent is self: 310 if hasattr(child, "breakCircles"): 311 child.breakCircles() 312 delattr(child, "parent")

313

314 315 -class ParseableStructure(StructureBase, common.Parser):

316 """is a base class for Structures parseable from EventProcessors (and 317 thus XML). 318 319 This is still abstract in that you need at least a name_ attribute. 320 But it knows how to be fed from a parser, plus you have feed and feedObject 321 methods that look up the attribute names and call the methods on the 322 respective attribute definitions. 323 """ 324 _pristine = True 325

326 - def __init__(self, parent, **kwargs):

327 StructureBase.__init__(self, parent, **kwargs)

328

329 - def finishElement(self, ctx):

330 return self

331

332 - def getAttribute(self, name):

333 """Returns an attribute instance from name. 334 335 This function will raise a StructureError if no matching attribute 336 definition is found. 337 """ 338 if name in self.managedAttrs: 339 return self.managedAttrs[name] 340 if name=="content_": 341 raise common.StructureError("%s elements must not have character data" 342 " content."%(self.name_)) 343 raise common.StructureError( 344 "%s elements have no %s attributes or children."%(self.name_, name))

345

346 - def end_(self, ctx, name, value):

347 try: 348 self.finishElement(ctx) 349 except common.Replace as ex: 350 if ex.newName is not None: 351 name = ex.newName 352 if ex.newOb.id is not None: 353 ctx.registerId(ex.newOb.id, ex.newOb) 354 self.parent.feedObject(name, ex.newOb) 355 except common.Ignore as ex: 356 pass 357 else: 358 if self.parent: 359 self.parent.feedObject(name, self) 360 # del self.feedEvent (at some point we might selectively reclaim parsers) 361 return self.parent

362

363 - def value_(self, ctx, name, value):

364 attDef = self.getAttribute(name) 365 try: 366 attDef.feed(ctx, self, value) 367 except common.Replace as ex: 368 return ex.newOb 369 self._pristine = False 370 return self

371

372 - def start_(self, ctx, name, value):

373 attDef = self.getAttribute(name) 374 if hasattr(attDef, "create"): 375 return attDef.create(self, ctx, name) 376 else: 377 return name

378

379 - def feed(self, name, literal, ctx=None):

380 """feeds the literal to the attribute name. 381 382 If you do not have a proper parse context ctx, so there 383 may be restrictions on what literals can be fed. 384 """ 385 self.managedAttrs[name].feed(ctx, self, literal)

386

387 - def feedObject(self, name, ob):

388 """feeds the object ob to the attribute name. 389 """ 390 self.managedAttrs[name].feedObject(self, ob)

391

392 - def iterEvents(self):

393 """yields an event sequence that transfers the copyable information 394 from self to something receiving the events. 395 396 If something is not copyable, it is ignored (i.e., keeps its default 397 on the target object). 398 """ 399 for att in self.attrSeq: 400 if not att.copyable: 401 continue 402 if hasattr(att, "iterEvents"): 403 for ev in att.iterEvents(self): 404 yield ev 405 else: 406 val = getattr(self, att.name_) 407 if val!=att.default_: 408 yield ("value", att.name_, att.unparse(val))

409

410 - def feedFrom(self, other, ctx=None, suppress=set()):

411 """feeds parsed objects from another structure. 412 413 This only works if the other structure is a of the same or a superclass 414 of self. 415 """ 416 from gavo.base import xmlstruct 417 if ctx is None: 418 ctx = parsecontext.ParseContext() 419 evProc = xmlstruct.EventProcessor(None, ctx) 420 evProc.setRoot(self) 421 for ev in other.iterEvents(): 422 evProc.feed(*ev)

423

424 425 -class Structure(ParseableStructure):

426 """is the base class for user-defined structures. 427 428 It will do some basic validation and will call hooks to complete elements 429 and compute computed attributes, based on ParseableStructure's finishElement 430 hook. 431 432 Also, it supports onParentComplete callbacks; this works by checking 433 if any managedAttr has a onParentComplete method and calling it 434 with the current value of that attribute if necessary. 435 """

436 - def callCompletedCallbacks(self):

437 for attName, attType in self.managedAttrs.iteritems(): 438 if hasattr(attType, "onParentComplete"): 439 attVal = getattr(self, attType.name_) 440 if attVal!=attType.default_: 441 attType.onParentComplete(attVal)

442

443 - def finishElement(self, ctx=None):

444 self.completeElement(ctx) 445 self.validate() 446 self.onElementComplete() 447 self.callCompletedCallbacks() 448 return self

449

450 - def _makeUpwardCaller(methName):

451 def _callNext(self, cls): 452 try: 453 pc = getattr(super(cls, self), methName) 454 except AttributeError: 455 pass 456 else: 457 pc()

458 return _callNext

459

460 - def _makeUpwardCallerOneArg(methName):

461 def _callNext(self, cls, arg): 462 try: 463 pc = getattr(super(cls, self), methName) 464 except AttributeError: 465 pass 466 else: 467 pc(arg)

468 return _callNext 469

470 - def completeElement(self, ctx):

471 self._completeElementNext(Structure, ctx)

472 473 _completeElementNext = _makeUpwardCallerOneArg("completeElement") 474

475 - def validate(self):

476 for val in set(self.managedAttrs.itervalues()): 477 if getattr(self, val.name_) is attrdef.Undefined: 478 raise common.StructureError("You must set %s on %s elements"%( 479 val.name_, self.name_)) 480 if hasattr(val, "validate"): 481 val.validate(self) 482 self._validateNext(Structure)

483 484 _validateNext = _makeUpwardCaller("validate") 485

486 - def onElementComplete(self):

487 self._onElementCompleteNext(Structure)

488 489 _onElementCompleteNext = _makeUpwardCaller("onElementComplete") 490

491 492 -class RestrictionMixin(object):

493 """A mixin for structure classes not allowed in untrusted RDs. 494 """

495 - def completeElement(self, ctx):

496 if getattr(ctx, "restricted", False): 497 raise common.RestrictedElement(self.name_) 498 self._completeElementNext(RestrictionMixin, ctx)

499

500 501 -def makeStruct(structClass, **kwargs):

502 """creates a parentless instance of structClass with ``**kwargs``. 503 504 You can pass in a ``parent_`` kwarg to force a parent. 505 506 This is the preferred way to create struct instances in DaCHS, as it 507 will cause the sequence of completers and validators run. Use it like 508 this:: 509 510 MS(rscdef.Column, name="ra", type="double precision) 511 """ 512 parent = None 513 if "parent_" in kwargs: 514 parent = kwargs.pop("parent_") 515 return structClass(parent, **kwargs).finishElement()

516

Source Code for Module gavo.base.structure