gavo.votable.dec

1 """ 2 Coding and decoding from tabledata. 3 """ 4 5 #c Copyright 2008-2019, the GAVO project 6 #c 7 #c This program is free software, covered by the GNU GPL. See the 8 #c COPYING file in the source distribution. 9 10 11 import re #noflake: used by generated code 12 13 from gavo import utils #noflake: used by generated code 14 from gavo.utils import parseDefaultDatetime, parseDefaultDate #noflake: used by generated code 15 from gavo.utils import pgsphere #noflake: used by generated code 16 from gavo.votable import coding 17 from gavo.votable import common 18 from gavo.votable.model import VOTable 19 20 try: 21 from gavo import stc #noflake: used by generated code 22 except ImportError: 23 # see modelgroups 24 pass 25 26 27 # literals for TDENC booleans 28 TDENCBOOL = { 29 't': True, 30 '1': True, 31 'true': True, 32 'f': False, 33 '0': False, 34 'false': False, 35 '?': None, 36 '': None, 37 } 38 39

40 -def tokenizeComplexArr(val):

41 """iterates over suitable number literal pairs from val. 42 """ 43 last = None 44 if val is None: 45 return 46 for item in val.split(): 47 if not item: 48 continue 49 if last is None: 50 last = item 51 else: 52 yield "%s %s"%(last, item) 53 last = None 54 if last: 55 yield last

56 57

58 -def tokenizeBitArr(val):

59 """iterates over 0 or 1 tokens in val, discarding everything else. 60 """ 61 if val is None: 62 return 63 for item in val: 64 if item in "01": 65 yield item

66 67

68 -def tokenizeNormalArr(val):

69 """iterates over all whitespace-separated tokens in val 70 """ 71 if val is None: 72 return 73 for item in val.split(): 74 if item: 75 yield item

76 77

78 -def _addNullvalueCode(field, src, validator):

79 """adds code to catch nullvalues if required by field. 80 """ 81 nullvalue = coding.getNullvalue(field, validator) 82 if nullvalue is not None: 83 src = [ 84 'if val=="%s":'%nullvalue, 85 ' row.append(None)', 86 'else:']+common.indentList(src, " ") 87 return src

88 89

90 -def _makeFloatDecoder(field):

91 src = [ 92 'if not val or val=="NaN":', 93 ' row.append(None)', 94 'else:', 95 ' row.append(float(val))',] 96 return _addNullvalueCode(field, src, float)

97 98

99 -def _makeComplexDecoder(field):

100 src = [ 101 'if not val:', 102 ' row.append(None)', 103 'else:', 104 ' try:', 105 ' r, i = val.split()', 106 ' except ValueError:', 107 ' r, i = float(val), 0', 108 ' if r!=r or i!=i:', 109 ' row.append(None)', 110 ' else:' 111 ' row.append(complex(float(r), float(i)))',] 112 return _addNullvalueCode(field, src, common.validateTDComplex)

113 114

115 -def _makeIntDecoder(field, maxInt):

116 src = [ 117 'if not val:', 118 ' row.append(None)', 119 'elif val.startswith("0x"):', 120 ' unsigned = int(val[2:], 16)', 121 # Python hex parsing is unsigned, fix manually based on maxInt 122 ' if unsigned>=%d:'%maxInt, 123 ' row.append(unsigned-%d)'%((maxInt+1)*2), 124 ' else:', 125 ' row.append(unsigned)', 126 'else:', 127 ' row.append(int(val))'] 128 return _addNullvalueCode(field, src, common.validateVOTInt)

129 130

131 -def _makeCharDecoder(field, emptyIsNull=True, fallbackEncoding="ascii"):

132 """parseString enables return of empty string (as opposed to None). 133 """ 134 # Elementtree makes sure we're only seeing unicode strings here 135 # However, char corresponds to byte strings, so we have to 136 # encode things before shipping out. In theory, there should only 137 # be ASCII in tabledata. In practice, people do dump all kinds of 138 # things in there. 139 src = [] 140 if emptyIsNull: 141 src.extend([ 142 'if not val:', 143 ' val = None',]) 144 else: 145 src.extend([ 146 'if val is None:', 147 ' val = ""']) 148 149 nullvalue = coding.getNullvalue(field, str, "") 150 decoder = "" 151 if fallbackEncoding: 152 decoder = '.encode("%s", "qmreplace")'%fallbackEncoding 153 154 if nullvalue: 155 src.extend([ 156 'if val==%s:'%repr(nullvalue), 157 ' val = None', 158 'else:', 159 ' val = val and val%s'%decoder]) 160 else: 161 src.append('val = val and val%s'%decoder) 162 163 if field.isMultiDim(): 164 src.append("val = coding.unravelArray(%s, val)"%repr(field.arraysize)) 165 166 xtypeDecoder = common.getXtypeDecoderCode(field) 167 if xtypeDecoder: 168 src.extend(xtypeDecoder) 169 170 src.append("row.append(val)") 171 return src

172 173

174 -def _makeUnicodeDecoder(field, emptyIsNull=True):

175 return _makeCharDecoder(field, emptyIsNull, fallbackEncoding=None)

176 177

178 -def _makeBooleanDecoder(field):

179 return ['row.append(TDENCBOOL[val.strip().lower()])']

180 181

182 -def _makeBitDecoder(field):

183 return ['row.append(int(val))']

184 185 186 _decoders = { 187 'boolean': (_makeBooleanDecoder, 'list'), 188 'bit': (_makeBitDecoder, 'list'), 189 'unsignedByte': (lambda v: _makeIntDecoder(v, 256), 'bytelist'), 190 'char': (_makeCharDecoder, 'list'), 191 'unicodeChar': (_makeUnicodeDecoder, 'list'), 192 'short': (lambda v: _makeIntDecoder(v, 32767), 'intlist'), 193 'int': (lambda v: _makeIntDecoder(v, 2147483647), 'intlist'), 194 'long': (lambda v: _makeIntDecoder(v, 9223372036854775807L), 'intlist'), 195 'float': (_makeFloatDecoder, 'floatlist'), 196 'double': (_makeFloatDecoder, 'floatlist'), 197 'floatComplex': (_makeComplexDecoder, 'complexlist'), 198 'doubleComplex': (_makeComplexDecoder, 'complexlist'), 199 } 200

201 -def _getArrayDecoderLines(field):

202 """returns lines that decode arrays of literals. 203 204 Unfortunately, the spec is plain nuts, so we need to pull some tricks here. 205 206 As per VOTable 1.3, we translate empty strings to Nones; we use the 207 liberty that empty and NULL arrays are not distinguished to return 208 empty arrays as empty arrays, though. 209 """ 210 # TODO: this now contains enough generic code to marry it with the 211 # respective function in dec_binary (and move the result to coding.py). 212 type = field.datatype 213 214 if type=='char': 215 return _makeCharDecoder(field, emptyIsNull=True) 216 elif type=='unicodeChar': 217 return _makeUnicodeDecoder(field, emptyIsNull=True) 218 219 decoder, listtype = _decoders[type] 220 221 src = [ # OMG. I'm still hellbent on not calling functions here. 222 'arrayLiteral = val', 223 'fullRow, row = row, []', 224 ] 225 if type=='floatComplex' or type=='doubleComplex': 226 src.append("for val in tokenizeComplexArr(arrayLiteral):") 227 elif type=='bit': 228 src.append("for val in tokenizeBitArr(arrayLiteral):") 229 else: 230 src.append("for val in tokenizeNormalArr(arrayLiteral):") 231 232 src.extend(common.indentList(decoder(field), " ")) 233 234 src.extend(coding.makeShapeValidator(field)) 235 src.extend([ 236 "val = utils.%s(row)"%listtype, 237 "row = fullRow"]) 238 src.append("val = coding.unravelArray(%s, val)"%repr(field.arraysize)) 239 src.extend(common.getXtypeDecoderCode(field)) 240 src.extend([ 241 "row.append(val)"]) 242 243 return [ 244 "if val=='':", 245 " row.append(None)", 246 "else:"]+common.indentList(src, " ")

247 248

249 -def getLinesFor(field):

250 """returns a sequence of python source lines to decode TABLEDATA-encoded 251 values for field. 252 """ 253 if field.isScalar(): 254 return _decoders[field.datatype][0](field) 255 else: 256 return _getArrayDecoderLines(field)

257 258

259 -def getRowDecoderSource(tableDefinition):

260 """returns the source for a function deserializing rows of tableDefition 261 in TABLEDATA. 262 263 tableDefinition is a VOTable.TABLE instance. 264 """ 265 source = ["def codec(rawRow):", " row = []"] 266 for index, field in enumerate( 267 tableDefinition.iterChildrenOfType(VOTable.FIELD)): 268 source.extend([ 269 " try:", 270 " val = rawRow[%d]"%index,]+ 271 common.indentList(getLinesFor(field), " ")+[ 272 " except common.VOTableError:", 273 " raise", 274 " except Exception, ex:", 275 # " import traceback; traceback.print_exc()", 276 " raise common.BadVOTableLiteral('%s', val, ex)"%field.datatype]) 277 source.append(" return row") 278 return "\n".join(source) 279 280 return source

281 282

283 -def getGlobals(tableDefinition):

284 return globals()

285

Source Code for Module gavo.votable.dec_tabledata