Source Code for Module gavo.base.valuemappers

  1  """ 
  2  Serialising table rows. 
  3   
  4  The main class in this module is the SerManager, which knows about 
  5  a target table and figures out how to turn the values in the table 
  6  rows into strings. 
  7   
  8  This is used by formats.votablewrite and the HTML table serialisation. 
  9   
 10  utils.serializers has once been a part of this module.  To save migration 
 11  effort, for now it reproduces that module's interface. 
 12  """ 
 13   
 14  #c Copyright 2008-2019, the GAVO project 
 15  #c 
 16  #c This program is free software, covered by the GNU GPL.  See the 
 17  #c COPYING file in the source distribution. 
 18   
 19   
 20  import re 
 21   
 22  from gavo import adql 
 23  from gavo import utils 
 24  from gavo.utils.serializers import ( #noflake: for (compatibility) export 
 25          ValueMapperFactoryRegistry, defaultMFRegistry, AnnotatedColumn,  
 26          registerDefaultMF) 
 27   
 28  __docformat__ = "restructuredtext en" 
 29   
 30   
 31 -class SerManager(utils.IdManagerMixin): 
 32          """A wrapper for the serialisation of table data. 
 33   
 34          SerManager instances keep information on what values certain columns can 
 35          assume and how to map them to concrete values in VOTables, HTML or ASCII. 
 36           
 37          They are constructed with a BaseTable instance. 
 38   
 39          You can additionally give: 
 40   
 41                  - withRanges -- ignored, going away 
 42                  - acquireSamples -- ignored, going away 
 43                  - idManager -- an object mixing in utils.IdManagerMixin.  This is important 
 44                          if the ids we are assigning here end up in a larger document.  In that 
 45                          case, pass in the id manager of that larger document.  Default is the 
 46                          SerManager itself 
 47                  - mfRegistry -- a map factory registry.  Default is the defaltMFRegistry, 
 48                    which is suitable for VOTables. 
 49           
 50          Iterate over a SerManager to retrieve the annotated columns. 
 51          """ 
 52          # Filled out on demand 
 53          _nameDict = None 
 54   
 55 -        def __init__(self, table, withRanges=True, acquireSamples=True, 
 56                          idManager=None, mfRegistry=defaultMFRegistry): 
 57                  self.table = table 
 58                  if idManager is not None: 
 59                          self.cloneFrom(idManager) 
 60                  self.notes = {} 
 61                  self._makeAnnotatedColumns() 
 62                  self._makeMappers(mfRegistry) 
 63           
 64 -        def __iter__(self): 
 65                  return iter(self.annCols) 
 66   
 67 -        def _makeAnnotatedColumns(self): 
 68                  self.annCols = [] 
 69                  for column in self.table.tableDef: 
 70                          self.annCols.append( 
 71                                  AnnotatedColumn(column, self.table.votCasts.get(column.name))) 
 72   
 73                          # We unconditionally generate IDs for FIELDs since rev ~4368 
 74                          # If a column actually got included twice in a VOTable, its id 
 75                          # would occur twice in the VOTable; I guess that would need 
 76                          # handling in xmlstan 
 77                          colId = self.getOrMakeIdFor(column, column.id or column.key) 
 78                          if colId is not None: 
 79                                  self.annCols[-1]["id"] = colId 
 80   
 81                          # if column refers to a note, remember the note 
 82                          if column.note: 
 83                                  try: 
 84                                          self.notes[column.note.tag] = column.note 
 85                                          self.annCols[-1]["note"] = column.note 
 86                                  except (ValueError, utils.NotFoundError):  
 87                                          pass # don't worry about missing notes, but don't display them either 
 88           
 89                  self.byName = dict( 
 90                          (annCol["name"], annCol) for annCol in self.annCols) 
 91   
 92 -        def _makeMappers(self, mfRegistry): 
 93                  """returns a sequence of functions mapping our columns. 
 94   
 95                  As a side effect, column properties may change (in particular, 
 96                  datatypes). 
 97                  """ 
 98                  mappers = [] 
 99                  for annCol in self: 
100                          mappers.append(mfRegistry.getMapper(annCol)) 
101                  self.mappers = tuple(mappers) 
102   
103 -        def getColumnByName(self, name): 
104                  """returns an AnnotatedColumn element for name. 
105   
106                  To help out in case name has gone through postgres, this will try 
107                  name lowercased if it doesn't match in normal case. 
108   
109                  This will raise a KeyError if name can't be found anyway. 
110                  """ 
111                  try: 
112                          return self.byName[name] 
113                  except: 
114                          return self.byName[name.lower()] 
115   
116 -        def _compileMapFunction(self, funcLines): 
117                  """helps _make(Dict|Tuple)Factory. 
118                  """ 
119                  return utils.compileFunction( 
120                          "\n".join(funcLines), "buildRec", 
121                          useGlobals=dict(("map%d"%index, mapper)  
122                                  for index, mapper in enumerate(self.mappers))) 
123   
124 -        def _makeDictFactory(self): 
125                  """returns a function that returns a dictionary of mapped values 
126                  for a row dictionary. 
127                  """ 
128                  colLabels = [str(c["name"]) for c in self] 
129                  funDef = ["def buildRec(rowDict):"] 
130                  for index, label in enumerate(colLabels): 
131                          if self.mappers[index] is not utils.identity: 
132                                  funDef.append("\trowDict[%r] = map%d(rowDict[%r])"%( 
133                                          label, index, label)) 
134                  funDef.append("\treturn rowDict") 
135                  return self._compileMapFunction(funDef) 
136   
137 -        def _makeTupleFactory(self): 
138                  """returns a function that returns a tuple of mapped values 
139                  for a row dictionary. 
140                  """ 
141                  funDef = ["def buildRec(rowDict):", "\treturn ("] 
142                  for index, cd in enumerate(self): 
143                          if self.mappers[index] is utils.identity: 
144                                  funDef.append("\t\trowDict[%r],"%cd["name"]) 
145                          else: 
146                                  funDef.append("\t\tmap%d(rowDict[%r]),"%(index, cd["name"])) 
147                  funDef.append("\t)") 
148                  return self._compileMapFunction(funDef) 
149   
150 -        def _iterWithMaps(self, buildRec): 
151                  """helps getMapped(Values|Tuples). 
152                  """ 
153                  colLabels = [f.name for f in self.table.tableDef] 
154                  if not colLabels: 
155                          yield () 
156                          return 
157                  for row in self.table: 
158                          yield buildRec(row) 
159   
160 -        def getMappedValues(self): 
161                  """iterates over the table's rows as dicts with mapped values. 
162                  """ 
163                  return self._iterWithMaps(self._makeDictFactory()) 
164   
165 -        def getMappedTuples(self): 
166                  """iterates over the table's rows as tuples with mapped values. 
167                  """ 
168                  return self._iterWithMaps(self._makeTupleFactory()) 
169   
170   
171 -def needsQuoting(identifier, forRowmaker=False): 
172          """returns True if identifier needs quoting in an SQL statement. 
173          >>> needsQuoting("RA(J2000)") 
174          True 
175          >>> needsQuoting("ABS") 
176          True 
177          >>> needsQuoting("r") 
178          False 
179          """ 
180          if utils.identifierPattern.match(identifier) is None: 
181                  return True 
182          # extra rule for standards SQL 92 
183          if identifier.startswith("_"): 
184                  return True 
185   
186          if identifier.lower() in getNameBlacklist(forRowmaker): 
187                  return True 
188          return False 
189   
190   
191  @utils.memoized 
192 -def getNameBlacklist(forRowmaker=False): 
193          """returns a set of names not suitable for table column names. 
194   
195          This comprises SQL reserved words in lower case and, if forRowmaker 
196          is true, also some names damaging row makers (e.g. python reserved 
197          words). 
198          """ 
199          res = set(k.lower() for k in adql.allReservedWords) 
200          if forRowmaker: 
201                  import keyword 
202                  from gavo.rscdef import rmkfuncs 
203                  res = (res  
204                          | set(["result_", "rowdict_"]) 
205                          | set(k.lower() for k in keyword.kwlist) 
206                          | set(k.lower() for k in dir(rmkfuncs))) 
207          return frozenset(res) 
208   
209   
210 -class VOTNameMaker(object): 
211          """A class for generating db-unique names from VOTable fields. 
212   
213          This is important to avoid all kinds of weird names the remaining 
214          infrastructure will not handle.  "Normal" TableDefs assume unquoted 
215          SQL identifiers as names, and want all names unique. 
216   
217          Using this class ensures these expectations are met in a reproducible 
218          way (i.e., given the same table, the same names will be assigned). 
219          """ 
220 -        def __init__(self): 
221                  self.knownNames, self.index = set(getNameBlacklist(True)), 0 
222   
223 -        def makeName(self, field): 
224                  preName = re.sub("[^\w]+", "x", (getattr(field, "name", None)  
225                          or getattr(field, "ID", None) 
226                          or "field%02d"%self.index)) 
227                  if not re.match("[A-Za-z_]", preName): 
228                          preName = "col_"+preName 
229                  while preName.lower() in self.knownNames: 
230                          preName = preName+"_" 
231                  self.knownNames.add(preName.lower()) 
232                  self.index += 1 
233                  return preName 
234   
235   
236 -def _test(): 
237          import doctest, valuemappers 
238          doctest.testmod(valuemappers) 
239   
240   
241  if __name__=="__main__": 
242          _test() 
243