gavo.votable.dec

1 """ 2 Coding and decoding from binary. 3 """ 4 5 #c Copyright 2008-2019, the GAVO project 6 #c 7 #c This program is free software, covered by the GNU GPL. See the 8 #c COPYING file in the source distribution. 9 10 11 import re #noflake: used by generated code 12 import struct #noflake: used by generated code 13 14 from gavo import stc #noflake: used by generated code 15 from gavo import utils #noflake: used by generated code 16 from gavo.utils import parseDefaultDatetime, parseDefaultDate #noflake: used by generated code 17 from gavo.utils import pgsphere #noflake: used by generated code 18 from gavo.votable import coding 19 from gavo.votable import common #noflake: used by generated code 20 from gavo.votable.model import VOTable 21 22 23 # literals for BINARY booleans 24 BINENCBOOL = { 25 't': True, 26 'T': True, 27 '1': True, 28 'f': False, 29 'F': False, 30 '0': False, 31 '?': None, 32 } 33 34

35 -def _addNullvalueCode(field, src, validator):

36 """adds code to catch nullvalues if required by field. 37 38 validator must be a function returning a valid python literal for 39 a nullvalue attribute or raise an exception. This is security 40 critical since whatever validator returns gets embedded into 41 natively-run source code. 42 """ 43 nullvalue = coding.getNullvalue(field, validator) 44 if nullvalue: 45 src.extend([ 46 'if val==%s:'%validator(nullvalue), 47 ' row.append(None)', 48 'else:', 49 ' row.append(val)',]) 50 else: 51 src.append('row.append(val)') 52 return src

53 54

55 -def _makeFloatDecoder(field):

56 numBytes, structCode, _ = _typemap[field.datatype] 57 return [ 58 'val = struct.unpack("!%s", inF.read(%d))[0]'%(structCode, numBytes), 59 'if val!=val:', 60 ' row.append(None)', 61 'else:', 62 ' row.append(val)']

63 64

65 -def _makeComplexDecoder(field, numBytes, structCode):

66 return [ 67 '_re, _im = struct.unpack(%s, inF.read(%d))'%(repr(structCode), numBytes), 68 'if _re!=_re or _im!=_im:', 69 ' row.append(None)', 70 'else:', 71 ' row.append(_re+1j*_im)']

72 73

74 -def _makeIntDecoder(field):

75 numBytes, structCode, _ = _typemap[field.datatype] 76 src = [ 77 'val = struct.unpack("!%s", inF.read(%d))[0]'%(structCode, numBytes) 78 ] 79 return _addNullvalueCode(field, src, int)

80 81

82 -def _makeBooleanDecoder(field):

83 return [ 84 'row.append(BINENCBOOL[inF.read(1)])']

85 86

87 -def _getArraysizeCode(field):

88 src = [] 89 if field.hasVarLength(): 90 src.append('arraysize = struct.unpack("!i", inF.read(4))[0]') 91 else: 92 try: 93 src.append("arraysize = %d"%field.getLength()) 94 except ValueError: 95 src.append("arraysize = 1") 96 return src

97 98

99 -def _makeBitDecoder(field):

100 # bits/bit arrays are just dumped bits we turn to integers 101 # it's basically the same thing for arrays and single values. 102 src = _getArraysizeCode(field) 103 src.extend([ 104 'if arraysize==0:', 105 ' res = []', 106 'else:', 107 ' numBytes = (arraysize+7)/8', 108 ' topMask = (1<<(arraysize%8))-1', # mask for payload in topmost byte 109 ' if topMask==0: topMask = 0xff', 110 ' bytes = struct.unpack("%dB"%numBytes, inF.read(numBytes))', 111 ' res = bytes[0]&topMask', 112 ' for b in bytes[1:]:', 113 ' res = (res<<8)+b', 114 'row.append(res)']) 115 return src

116 117

118 -def _makeString(field, customSrc):

119 src = _getArraysizeCode(field) 120 src.extend(customSrc) 121 122 if field.isMultiDim(): 123 src.append("val = coding.unravelArray(%s, val)"%repr(field.arraysize)) 124 125 src.extend(common.getXtypeDecoderCode(field)) 126 _addNullvalueCode(field, src, repr) 127 return src

128 129

130 -def _makeCharDecoder(field):

131 return _makeString(field, [ 132 'val = str(struct.unpack("%ds"%arraysize, inF.read(arraysize))[0]' 133 '.decode("ascii", "qmreplace"))'])

134 135

136 -def _makeUnicodeCharDecoder(field):

137 # XXX BUG: Anything outside the BMP will kill this 138 return _makeString(field, [ 139 'val = struct.unpack("%ds"%(2*arraysize), inF.read(2*arraysize)' 140 ')[0].decode("utf-16be")'])

141 142 143 _typemap = { 144 "unsignedByte": (1, 'B', "bytelist"), 145 "short": (2, 'h', "intlist"), 146 "int": (4, 'i', "intlist"), 147 "long": (8, 'q', "intlist"), 148 "float": (4, 'f', "floatlist"), 149 "double": (8, 'd', "floatlist"),} 150 151 152 _decoders = { 153 'boolean': _makeBooleanDecoder, 154 'bit': _makeBitDecoder, 155 'char': _makeCharDecoder, 156 'unicodeChar': _makeUnicodeCharDecoder, 157 158 'unsignedByte': _makeIntDecoder, 159 'short': _makeIntDecoder, 160 'int': _makeIntDecoder, 161 'long': _makeIntDecoder, 162 163 'float': _makeFloatDecoder, 164 'double': _makeFloatDecoder, 165 'floatComplex': lambda v: _makeComplexDecoder(v, 8, '!ff'), 166 'doubleComplex': lambda v: _makeComplexDecoder(v, 16, '!dd'), 167 } 168

169 -def _makeShortcutCode(field, type):

170 """returns None or code to quickly decode field array. 171 172 Fast decoding for whatever is mentioned in _typemap and no nullvalues 173 are defined. 174 """ 175 if type not in _typemap: 176 return None 177 if coding.getNullvalue(field, str) is not None: 178 return None 179 180 numBytes, typecode, listtype = _typemap[type] 181 src = _getArraysizeCode(field) 182 src.append( 183 'vals = struct.unpack("!%%d%s"%%arraysize, inF.read(arraysize*%d))'%( 184 typecode, numBytes)) 185 if type=='float' or type=='double': 186 src.append( 187 'row.append(utils.%s(v!=v and None or v for v in vals))'%listtype) 188 else: 189 src.append( 190 'row.append(utils.%s(vals))'%listtype)

191 192

193 -def _getArrayDecoderLines(field):

194 """returns lines that decode arrays of literals. 195 196 Unfortunately, the spec is plain nuts, so we need to pull some tricks here. 197 """ 198 type = field.datatype 199 200 # Weird things 201 if type=="bit": 202 return _makeBitDecoder(field) 203 elif type=='char': 204 return _makeCharDecoder(field) 205 elif type=='unicodeChar': 206 return _makeUnicodeCharDecoder(field) 207 208 # Fast array decoding for fields without null values 209 src = _makeShortcutCode(field, type) 210 if src is not None: 211 return src 212 213 _, _, listtype = _typemap.get(type, (None, None, 'list')) 214 215 # default processing 216 src = [ # OMG. I'm still hellbent on not calling functions here. 217 'fullRow, row = row, []' 218 ] 219 src.extend(_getArraysizeCode(field)) 220 src.extend([ 221 "for i in range(arraysize):"]) 222 src.extend(common.indentList(_decoders[type](field), " ")) 223 224 src.extend(coding.makeShapeValidator(field)) 225 src.extend([ 226 "val = utils.%s(row)"%listtype, 227 "row = fullRow"]) 228 src.append("val = coding.unravelArray(%s, val)"%repr(field.arraysize)) 229 src.extend(common.getXtypeDecoderCode(field)) 230 src.append("row.append(val)") 231 return src

232 233

234 -def getLinesFor(field):

235 """returns a sequence of python source lines to decode BINARY-encoded 236 values for field. 237 """ 238 if field.isScalar(): 239 return _decoders[field.datatype](field) 240 else: 241 return _getArrayDecoderLines(field)

242 243

244 -def getRowDecoderSource(tableDefinition):

245 """returns the source for a function deserializing a BINARY stream. 246 247 tableDefinition is a VOTable.TABLE instance. The function returned 248 expects a file-like object. 249 """ 250 source = ["def codec(inF):", " row = []"] 251 for index, field in enumerate( 252 tableDefinition.iterChildrenOfType(VOTable.FIELD)): 253 source.extend([ 254 " try:",]+ 255 common.indentList(getLinesFor(field), " ")+[ 256 " except IOError:", # EOF on empty row is ok. 257 " if inF.atEnd and row==[]:", 258 " return None", 259 " raise", 260 " except common.VOTableError:", 261 " raise", 262 " except:", 263 # " import traceback; traceback.print_exc()", 264 " raise common.BadVOTableLiteral('%s', repr(inF.lastRes))"%( 265 field.datatype)]) 266 source.append(" return row") 267 return "\n".join(source)

268 269

270 -def getGlobals(tableDefinition):

271 return globals()

272

Source Code for Module gavo.votable.dec_binary