Source Code for Module gavo.votable.enc_binary2

  1  """ 
  2  BINARY2 VOTable encoding. 
  3   
  4  BINARY2 is like BINARY, except every record is preceded by a mask which 
  5  columns are NULL. 
  6   
  7  We do not determine any nullvalues any more here. 
  8   
  9  Sorry for gratuituously peeking into the guts of enc_binary here.  But well, 
 10  it's family. 
 11  """ 
 12   
 13  #c Copyright 2008-2019, the GAVO project 
 14  #c 
 15  #c This program is free software, covered by the GNU GPL.  See the 
 16  #c COPYING file in the source distribution. 
 17   
 18   
 19  import datetime                 #noflake: used by generated code 
 20  import struct 
 21   
 22  from gavo import utils          #noflake: used by generated code 
 23  from gavo.utils import pgsphere #noflake: used by generated code 
 24  from gavo.votable import coding #noflake: used by generated code 
 25  from gavo.votable import common 
 26  from gavo.votable import enc_binary 
 27   
 28   
 29  floatNaN = struct.pack("!f", common.NaN) 
 30  doubleNaN = struct.pack("!d", common.NaN) 
 31   
 32   
 33   
 34 -def _makeBitEncoder(field): 
 35          return enc_binary._makeBitEncoder(field, allowNULL=True) 
 36   
 37   
 38 -def _generateIntEncoderMaker(fmtCode): 
 39          def makeIntEncoder(field): 
 40                  return [ 
 41                          "if val is None:", 
 42                          "  val = 0", 
 43                          "tokens.append(struct.pack('%s', val))"%fmtCode] 
 44          return makeIntEncoder 
 45   
 46   
 47 -def _makeCharEncoder(field): 
 48          return [ 
 49                  "if val is None:", 
 50                  "  val = 0", 
 51                  "else:" 
 52                  "  val = ord(val)", 
 53                  "tokens.append(struct.pack('B', val))"] 
 54   
 55   
 56 -def _makeUnicodeCharEncoder(field): 
 57          return [ 
 58                  "if val is None:", 
 59                  "  tokens.append('\\x00\\x00')", 
 60                  "else:", 
 61                  "       coded = val.encode('utf-16be')", 
 62                  "       tokens.append(struct.pack('%ds'%len(coded), coded))"] 
 63   
 64   
 65 -def _makeUnsignedByteEncoder(field): 
 66          return [ 
 67                  "if isinstance(val, int):", 
 68                  "  tokens.append(struct.pack('B', val))", 
 69                  "elif val is None:", 
 70                  "  tokens.append('\xff')", 
 71                  "else:", 
 72                  "  tokens.append(struct.pack('c', val))"] 
 73   
 74   
 75  _encoders = { 
 76                  "boolean": enc_binary._makeBooleanEncoder, 
 77                  "bit": enc_binary._makeBitEncoder, 
 78                  "unsignedByte": _makeUnsignedByteEncoder, 
 79                  "short": _generateIntEncoderMaker('!h'), 
 80                  "int": _generateIntEncoderMaker('!i'), 
 81                  "long": _generateIntEncoderMaker('!q'), 
 82                  "char": _makeCharEncoder, 
 83                  "unicodeChar": _makeUnicodeCharEncoder, 
 84                  "double": enc_binary._generateFloatEncoderMaker("!d", "doubleNaN"), 
 85                  "float": enc_binary._generateFloatEncoderMaker("!f", "floatNaN"), 
 86                  "doubleComplex": enc_binary._generateComplexEncoderMaker( 
 87                          "!dd", "doubleNaN"), 
 88                  "floatComplex": enc_binary._generateComplexEncoderMaker("!ff", "floatNaN"), 
 89  } 
 90   
 91   
 92 -def _makeCharArrayEncoder(field): 
 93  # special handling for character arrays, since we don't want to treat 
 94  # those as character arrays in python. 
 95          src = [] 
 96   
 97          src.extend(common.getXtypeEncoderCode(field)) 
 98          src.append("val = coding.trimString(val, %s, '\\0')"%repr(field.arraysize)) 
 99   
100          if field.datatype=="char": 
101                  src.extend([ 
102                          'if isinstance(val, unicode):', 
103                          '  val = val.encode("ascii", "replace")']) 
104   
105          if field.hasVarLength(): 
106                  src.extend([ 
107                          "tokens.append(struct.pack('!i', len(val)))", 
108                  ]) 
109                   
110          if field.datatype=="unicodeChar": 
111                  src.append("val = val.encode('utf-16be')") 
112          src.append("tokens.append(struct.pack('%ds'%len(val), val))") 
113   
114          return src 
115   
116   
117 -def _getArrayEncoderLines(field): 
118          """returns python lines to encode array values of field. 
119          """ 
120          type = field.datatype 
121   
122          # bit array literals are integers, same as bits 
123          if type=="bit": 
124                  return _makeBitEncoder(field) 
125   
126          if type=="char" or type=="unicodeChar": 
127                  return _makeCharArrayEncoder(field) 
128   
129   
130          # Everything else can use some common array shaping code since value comes in 
131          # some kind of sequence. 
132          padder = '[None]' 
133          src = [ # Painful name juggling to avoid having to call functions. 
134                  "fullTokens = tokens", 
135                  "tokens = []", 
136                  "if val is None:", 
137                  "  arr = []", 
138                  "else:", 
139                  "  arr = val", 
140                  "for val in arr:" 
141          ]+common.indentList(_encoders[field.datatype](field), "  ") 
142   
143          src.extend([ 
144                  "fullTokens.append(''.join(tokens))", 
145                  "tokens = fullTokens"]) 
146                           
147          return (common.getXtypeEncoderCode(field) 
148          + enc_binary._getArrayShapingCode(field, padder) 
149          + src) 
150                           
151                           
152   
153 -def getLinesFor(field): 
154          """returns a sequence of python source lines to encode values described 
155          by field into tabledata. 
156          """ 
157          if field.isScalar(): 
158                  return _encoders[field.datatype](field) 
159          else: 
160                  return _getArrayEncoderLines(field) 
161   
162   
163 -def getPreamble(tableDefinition): 
164          return [ 
165                  "tokens.append(nullFlags.serializeFromRow(tableRow))"] 
166   
167   
168 -def getPostamble(tableDefinition): 
169          return [ 
170                  "return ''.join(tokens)"] 
171   
172   
173 -def getGlobals(tableDefinition): 
174          vars = globals().copy() 
175          vars["nullFlags"] = common.NULLFlags(len(tableDefinition.getFields())) 
176          return vars 
177