1 """
2 Common code for coding and decoding VOTable data.
3 """
4
5
6
7
8
9
10
11 from gavo import utils
12 from gavo.votable import common
13 from gavo.votable.model import VOTable
14
15
17 """returns the source for a function encoding rows of tableDefition
18 in the format implied encoderModule
19
20 tableDefinition is a VOTable.TABLE instance, encoderModule
21 is one of the enc_whatever modules (this function needs getLinesFor
22 and getPostamble from them).
23 """
24
25 source = [
26 "def codec(tableRow):",
27 " tokens = []",
28 " val = None"]
29
30 source.extend(
31 common.indentList(
32 getattr(encoderModule, "getPreamble", lambda td: [])(
33 tableDefinition), " "))
34
35 for index, field in enumerate(
36 tableDefinition.iterChildrenOfType(VOTable.FIELD)):
37 source.extend([
38 " try:",
39 " val = tableRow[%d]"%index])
40 source.extend(common.indentList(encoderModule.getLinesFor(field), " "))
41 source.extend([
42 " except common.VOTableError:",
43 " raise",
44 " except Exception, ex:",
45
46 " raise common.BadVOTableData(unicode(ex), repr(val), '%s')"%
47 field.getDesignation()])
48 source.extend(common.indentList(
49 encoderModule.getPostamble(tableDefinition), " "))
50 return "\n".join(source)
51
52
54 """returns a compiled function for source in env.
55
56 Source is the result of one of the makeXXX functions in this module,
57 env typically the result of a getGlobals() on the codec module.
58 """
59 ns = {}
60 ns.update(env)
61
62
63 return utils.compileFunction(source, "codec", useGlobals=ns)
64
65
70
71
76
77
79 """returns None or the nullvalue defined for field.
80
81 validator is a function that raises some exception if the nullvalue
82 is inappropriate. It should do so in particular on everything that
83 contains quotes and such; the nullvalues are included in source code
84 and thus might be used to inject code if not validated.
85 """
86 nullvalue = None
87 for values in field.iterChildrenOfType(VOTable.VALUES):
88 if values.null is not None:
89 nullvalue = values.null
90 if nullvalue is None or nullvalue=='':
91 return default
92 else:
93 validator(nullvalue)
94 return nullvalue
95
96
98 """turns a flat sequence into an n-dim array as specfied by the votable
99 arraysize spec arraysize.
100
101 arraysize is <int>{"x"<int>}*?|*.
102
103 No padding or cropping will take place. This means that the last
104 row(s) may have improper sizes if seq is incompatible with arraysize.
105
106 >>> unravelArray("2x3", "012345")
107 ['01', '23', '45']
108 >>> unravelArray("2x*", "012345")
109 ['01', '23', '45']
110 >>> unravelArray("3x2x*", "012345012345")
111 [['012', '345'], ['012', '345']]
112 """
113 parts = arraysize.split("x")
114 if len(parts)<2:
115 return seq
116 del parts[-1]
117
118
119 listCons = list
120 if isinstance(seq, list):
121 listCons = seq.__class__
122
123 for step in map(int, parts):
124 seq = listCons(seq[i:i+step] for i in range(0, len(seq), step))
125 return seq
126
127
129 """parses a single VOTable arraysize number to (flexible, length).
130
131 This will accept single numbers (returns False, number),
132 number* (returns True, number) and just * (returns 0, number).
133
134 This is used to parse the last part of an ND array spec. Everything
135 before that must be an integer only.
136 """
137 try:
138 if spec=="*":
139 return True, 0
140
141 elif spec.endswith("*"):
142 return True, int(spec[:-1])
143
144 else:
145 return False, int(spec)
146 except ValueError:
147 raise common.VOTableError("Invalid arraysize fragment '%s' in"
148 " field or param name '%s'"%(spec, fieldName))
149
150
152 """returns code lines to validate an an array shape against a flat
153 sequence in row.
154
155 This is used by the array decoders.
156 """
157 arraysize = field.arraysize
158 if not arraysize:
159 return []
160 dimensions = arraysize.strip().split("x")
161
162 stride = 1
163
164 if len(dimensions)>1:
165 try:
166 stride = reduce(lambda a,b: a*b, [int(l) for l in dimensions[:-1]])
167 except ValueError:
168 raise common.VOTableError("Invalid arraysize '%s' specificed in"
169 " field or param name '%s'"%(
170 field.arraysize, field.name))
171
172 flexible, length = parseVOTableArraysizeEl(dimensions[-1], field.name)
173
174 if flexible:
175
176
177
178 if stride>1:
179 return [
180 "if len(row) %% %d:"%stride,
181 " raise common.BadVOTableLiteral('%s[%s]',"
182 " '<%%d token(s)>'%%(len(row)))"%(
183 field.datatype, field.arraysize)]
184
185 else:
186
187 return [
188 "if len(row)!=%d:"%(length*stride),
189 " raise common.BadVOTableLiteral('%s[%s]',"
190 " '<%%d token(s)>'%%(len(row)))"%(
191 field.datatype, field.arraysize)]
192
193
194 return []
195
196
198 """expands flattens out any sub-sequences (lists or tuples) in seq
199 recursively.
200
201 This is used by the array encoders.
202 """
203 res = []
204 iteratorStack = [iter(seq)]
205 while iteratorStack:
206 try:
207 item = iteratorStack[-1].next()
208 if isinstance(item, (list, tuple)):
209 iteratorStack.append(iter(item))
210
211 else:
212 res.append(item)
213 except StopIteration:
214 iteratorStack.pop()
215 return res
216
217
218 -def trim(seq, arraysize, padder):
219 """returns seq with length arraysize.
220
221 arraysize is an int; you should just use field.getLength() when
222 trimming VOTable arraysizes since the arraysize attribute is rather
223 complex. Arraysize may be None for convenience; trim is a no-op then.
224
225 If seq is shorter, padder*missing will be appended, if it is longer, seq will
226 be shortened from the end.
227
228 This is intended as a helper for array encoders.
229 """
230 seq = ravel(seq)
231 if arraysize is None:
232 return seq
233 if len(seq)<arraysize:
234 seq = seq+padder*(arraysize-len(seq))
235 elif len(seq)>arraysize:
236 seq = seq[:arraysize]
237 return seq
238
239
241 """returns val flattened and padded with padChar/cropped to length.
242
243 field is a V.FIELD or V.PARAM instance for which val should be
244 prepared.
245
246 val can also be a sequence of strings (or nested more deeply. In that
247 case, trimString will flatten the value(s), padding and cropping as
248 neccessary.
249
250 If val is None, then as many padChars will be returned as arraysize
251 wants (which is 0 for variable-length fields).
252
253 For chars, arraysize None is equivalent to arraysize 1.
254
255 >>> trimString("abc", "4")
256 'abc '
257 >>> trimString(["abc", "de", "f"], "2x*")
258 'abdef '
259 >>> trimString([["abc", "cd", "e"], ["", "fgh", "i"]], "2x4x3")
260 'abcde fgi '
261 >>> trimString(None, "4x2", 'z')
262 'zzzzzzzz'
263 >>> trimString(None, "4x2*", 'z')
264 ''
265 >>> trimString("abc", None)
266 'a'
267 """
268 if arraysize is None:
269 arraysize = "1"
270
271 if val is None:
272 expected = common.getLength(arraysize)
273 if expected:
274 return padChar*expected
275 else:
276 return ""
277
278 if "x" in arraysize:
279 rest, destLength = arraysize.rsplit("x", 1)
280
281 if not destLength.endswith('*'):
282 destLength = int(destLength)
283 val = val[:destLength]+[None]*max(0, destLength-len(val))
284
285 return "".join(trimString(item, rest, padChar) for item in val)
286
287 else:
288 if arraysize.endswith('*'):
289 return val
290 else:
291 destLength = int(arraysize)
292 return val[:destLength]+padChar*max(0, destLength-len(val))
293
294
298
299
300 if __name__=="__main__":
301 _test()
302