gavo.grammars.directgrammar

1 """ 2 A grammar supporting C language boosters (or possibly other mechanisms 3 bypassing internal dbtable). 4 5 These actually bypass most of our machinery and should only be used if 6 performance is paramount. Otherwise, CustomGrammars play much nicer with 7 the rest of the DC software. 8 9 Currently, only one kind of DirectGrammar is supported: C boosters. 10 """ 11 12 #c Copyright 2008-2019, the GAVO project 13 #c 14 #c This program is free software, covered by the GNU GPL. See the 15 #c COPYING file in the source distribution. 16 17 18 from __future__ import print_function 19 20 import os 21 import pkg_resources 22 import re 23 import shutil 24 import subprocess 25 26 from gavo import base 27 from gavo import utils 28 from gavo import rscdef 29 from gavo.grammars import common 30 31

32 -class CBooster(object):

33 """is a wrapper for an import booster written in C using the DC booster 34 infrastructure. 35 36 Warning: If you change the booster description, you'll need to touch 37 the source to recompile. 38 """ 39 silence_for_test = False 40

41 - def __init__(self, srcName, dataDesc, recordSize=4000, gzippedInput=False, 42 autoNull=None, preFilter=None, ignoreBadRecords=False, 43 customFlags=""):

44 self.dataDesc = dataDesc 45 self.recordSize = recordSize 46 self.resdir = dataDesc.rd.resdir 47 self.srcName = os.path.join(self.resdir, srcName) 48 self.autoNull, self.preFilter = autoNull, preFilter 49 self.ignoreBadRecords = ignoreBadRecords 50 self.gzippedInput = gzippedInput 51 self.customFlags = customFlags 52 self.bindir = os.path.join(self.resdir, "bin") 53 self.binaryName = os.path.join(self.bindir, 54 os.path.splitext(os.path.basename(srcName))[0]+"-"+base.getConfig( 55 "platform")) 56 self._ensureBinary()

57

58 - def _copySources(self, wd):

59 def getResource(src, dest): 60 inF = pkg_resources.resource_stream('gavo', src) 61 outF = open(os.path.join(wd, dest), "w") 62 outF.write(inF.read()) 63 outF.close() 64 inF.close()

65 getResource("resources/src/boosterskel.c", "boosterskel.c") 66 getResource("resources/src/boosterskel.h", "boosterskel.h") 67 shutil.copyfile(self.srcName, os.path.join(wd, "func.c")) 68 69 # XXX TODO: take this from the embedding data's make; 70 # DirectGrammars can't be outside of a data element any more anyway. 71 mat = re.search("(?m)^#define QUERY_N_PARS\s+(\d+)", 72 open(self.srcName).read()) 73 if not mat: 74 raise base.ReportableError("Booster function doesn't define QUERY_N_PARS") 75 query_n_pars = mat.group(1) 76 77 f = open(os.path.join(wd, "Makefile"), "w") 78 79 if self.dataDesc.grammar.type=="fits": 80 f.write("LDFLAGS += -lcfitsio\n") 81 82 f.write("LDFLAGS += -lm\n" 83 "CFLAGS += -Wall -DINPUT_LINE_MAX=%d -DQUERY_N_PARS=%s\n"%( 84 self.recordSize, query_n_pars)) 85 if self.autoNull: 86 f.write("CFLAGS += -DAUTO_NULL='%s'\n"%self.autoNull.replace( 87 "\\", "\\\\")) 88 if self.ignoreBadRecords: 89 f.write("CFLAGS += -DIGNORE_BAD_RECORDS\n") 90 f.write("CFLAGS += -g\n") 91 92 f.write("booster: boosterskel.c func.c\n" 93 "\t$(CC) $(CFLAGS) %s -o booster $^ $(LDFLAGS)\n"%self.customFlags) 94 f.close()

95

96 - def _build(self):

97 callArgs = {} 98 if self.silence_for_test: 99 # test instrumentation -- don't worry if the file remains open 100 callArgs["stdout"] = open("/dev/null", "w") 101 if subprocess.call("make", **callArgs): 102 raise base.ReportableError("Booster build failed, messages above.")

103

104 - def _retrieveBinary(self, od):

105 shutil.copyfile("booster", self.binaryName) 106 os.chmod(self.binaryName, 0775)

107

108 - def _ensureBinary(self):

109 """makes sure the booster binary exists and is up-to-date. 110 """ 111 if not os.path.exists(self.bindir): 112 os.makedirs(self.bindir) 113 try: 114 if os.path.getmtime(self.srcName)<os.path.getmtime(self.binaryName): 115 return 116 except os.error: 117 pass 118 if os.path.exists(self.srcName): 119 utils.runInSandbox(self._copySources, self._build, self._retrieveBinary) 120 else: 121 raise base.ReportableError("Booster source does not exist." 122 " You will not be able to import the enclosing data.", 123 hint="Use gavo mkboost to create a skeleton for the booster.")

124

125 - def getOutput(self, argName):

126 """returns a pipe you can read the booster's output from. 127 128 As a side effect, it also sets the attribute self.pipe. We need 129 this to be able to retrieve the command status below. 130 """ 131 if self.preFilter: 132 shellCommand = "%s '%s' | %s"%(self.preFilter, argName, self.binaryName) 133 elif self.gzippedInput: 134 shellCommand = "zcat '%s' | %s"%(argName, self.binaryName) 135 else: 136 shellCommand = "%s '%s'"%(self.binaryName, argName) 137 138 pipeArgs = {"shell": True, "stdout": subprocess.PIPE} 139 if self.silence_for_test: 140 # test instrumentation -- don't worry if the file remains open 141 pipeArgs["stderr"] = open("/dev/null", "w") 142 self.pipe = subprocess.Popen(shellCommand, **pipeArgs) 143 return self.pipe.stdout

144

145 - def getStatus(self):

146 return self.pipe.wait()

147 148

149 -class DirectGrammar(base.Structure, base.RestrictionMixin):

150 """A user-defined external grammar. 151 152 See the `separate document`_ on user-defined code on more on direct grammars. 153 154 You will almost always use these in connection with C code generated 155 by ``dachs mkboost``. 156 157 .. _separate document: :dachsdoc:booster.html 158 """ 159 name_ = "directGrammar" 160 161 _cbooster = rscdef.ResdirRelativeAttribute("cBooster", 162 default=base.Undefined, 163 description="resdir-relative path to the booster C source.", 164 copyable=True) 165 166 _gzippedInput = base.BooleanAttribute("gzippedInput", default=False, 167 description="Pipe gzip before booster? (will not work for FITS)", 168 copyable=True) 169 170 _autoNull = base.UnicodeAttribute("autoNull", default=None, 171 description="Use this string as general NULL value (when reading" 172 " from plain text).", 173 copyable=True) 174 175 _ignoreBadRecords = base.BooleanAttribute("ignoreBadRecords", 176 default=False, description="Let booster ignore invalid records?", 177 copyable=True) 178 179 _recordSize = base.IntAttribute("recordSize", default=4000, 180 description="For bin boosters, read this many bytes to make" 181 " up a record; for line-based boosters, this is the maximum" 182 " length of an input line.", 183 copyable=True) 184 185 _preFilter = base.UnicodeAttribute("preFilter", default=None, 186 description="Pipe input through this program before handing it to" 187 " the booster; this string is shell-expanded (will not work for FITS).", 188 copyable=True) 189 190 _customFlags = base.UnicodeAttribute("customFlags", default="", 191 description="Pass these flags to the C compiler when building the" 192 " booster.", 193 copyable=True) 194 195 _type = base.EnumeratedUnicodeAttribute("type", default="col", 196 validValues=["col", "bin", "fits", "split"], 197 description="Make code for a booster parsing by column indices (col)," 198 " by splitting along separators (split), by reading fixed-length" 199 " binary records (bin), for from FITS binary tables (fits).", 200 copyable=True) 201 202 _splitChar = base.UnicodeAttribute("splitChar", default="|", 203 description="For split boosters, use this as the separator.", 204 copyable=True) 205 206 _ext = base.IntAttribute("extension", default=1, 207 description="For FITS table boosters, get the table from this extension.", 208 copyable=True) 209 210 _mapKeys = base.StructAttribute("mapKeys", childFactory=common.MapKeys, 211 default=None, copyable=True, 212 description="For a FITS booster, map DB table column names" 213 " to FITS column names (e.g., if the FITS table name flx is to" 214 " end up in the DB column flux, say flux:flx).") 215 216 _rd = rscdef.RDAttribute() 217 218 isDispatching = False 219

220 - def validate(self):

221 self._validateNext(DirectGrammar) 222 if self.type=='bin': 223 if not self.recordSize: 224 raise base.StructureError("DirectGrammars reading from binary need" 225 " a recordSize attribute") 226 if self.mapKeys is not None: 227 if self.type!="fits": 228 raise base.StructureError("mapKeys is only allowed for FITS" 229 " boosters.")

230

231 - def onElementComplete(self):

232 if self.type=="fits": 233 if self.mapKeys: 234 self.keyMap = self.mapKeys.maps 235 else: 236 self.keyMap = {}

237

238 - def getBooster(self):

239 return CBooster(self.cBooster, self.parent, 240 gzippedInput=self.gzippedInput, 241 preFilter=self.preFilter, 242 autoNull=self.autoNull, 243 ignoreBadRecords=self.ignoreBadRecords, 244 customFlags=self.customFlags)

245

246 - def parse(self, sourceToken, targetData=None):

247 booster = self.getBooster() 248 makes = self.parent.makes 249 if len(makes)!=1: 250 raise base.StructureError("Directgrammar only works for data having" 251 " exactly one table, but data '%s' has %d"%( 252 self.parent.id, len(makes))) 253 254 def copyIn(data): 255 data.tables.values()[0].copyIn(booster.getOutput(sourceToken)) 256 if booster.getStatus(): 257 raise base.SourceParseError( 258 "Booster returned error signature", 259 source=sourceToken)

260 return copyIn

261 262 263 ################################################### 264 # booster source code generating functions 265 266 import sys 267

268 -def getNameForItem(item):

269 return "fi_"+item.name.lower()

270 271 272 # Some pieces to puzzle together the createDumpfile functions 273 COMMON_MAIN_HEADER = """ 274 /* Common main header */ 275 void createDumpfile(int argc, char **argv) 276 { 277 FILE *destination=stdout; 278 char inputLine[INPUT_LINE_MAX]; 279 int recCount = 0; 280 /* /Common main header */ 281 """ 282 283 NONSEEK_MAIN_INTRO = """ 284 FILE *inF; 285 286 /* seekable main intro */ 287 if (argc>2) { 288 die(USAGE); 289 } 290 if (argc==2) { 291 if (!(inF = fopen(argv[1], "r"))) { 292 die(strerror(errno)); 293 } 294 } else { 295 inF = stdin; 296 } 297 /* /seekable main intro */ 298 """ 299 300 COMMON_MAIN_INTRO = """ 301 /* common main intro */ 302 writeHeader(destination); 303 /* /common main intro */ 304 """ 305 306 307 LOOP_BODY_INTRO = """ 308 Field *tuple; 309 context = inputLine; 310 if (!setjmp(ignoreRecord)) { 311 """ 312 313 314 LOOP_BODY_FOOT = """ 315 if (!tuple) { 316 handleBadRecord("Bad input line at record %d", recCount); 317 } 318 writeTuple(tuple, QUERY_N_PARS, destination); 319 context = NULL; 320 recCount ++; 321 if (!(recCount%1000)) { 322 fprintf(stderr, "%08d\\r", recCount); 323 fflush(stderr); 324 } 325 } 326 """ 327 328 329 COMMON_MAIN_FOOT = """ 330 writeEndMarker(destination); 331 fprintf(stderr, "%08d records done.\\n", recCount); 332 } 333 """ 334 335

336 -def _getMakeMacro(item):

337 """returns a maker macro for a column object. 338 """ 339 try: 340 return { 341 "integer": "MAKE_INT", 342 "smallint": "MAKE_SHORT", 343 "double precision": "MAKE_DOUBLE", 344 "real": "MAKE_FLOAT", 345 "char": "MAKE_CHAR_NULL", 346 "bytea": "MAKE_BYTE", 347 "text": "MAKE_TEXT", 348 }[item.type] 349 except KeyError: 350 # not a simple case; this could be a place for doing arrays and such 351 return "MAKE_somethingelse"

352 353

354 -class _CodeGenerator(object):

355 """a base class for code generators. 356 357 You must at least override getItemParser. 358 """

359 - def __init__(self, grammar, tableDef):

360 self.grammar, self.tableDef = grammar, tableDef

361

362 - def getSetupCode(self):

363 """returns a sequence of C lines for code between an item parser. 364 """ 365 return []

366

367 - def getItemParser(self, item, index):

368 """returns code that parses item (a Column instance) at column index 369 index. 370 371 You're free to igore index. 372 """ 373 return []

374

375 - def getPreamble(self):

376 """returns a list of lines that make up the top of the booster. 377 """ 378 return [ 379 '#include <stdio.h>', 380 '#include <math.h>', 381 '#include <string.h>', 382 '#include <errno.h>', 383 '#include "boosterskel.h"', 384 '', 385 '#define USAGE "Usage: don\'t."',]

386

387 - def getPrototype(self):

388 """returns the prototype of the getTuple function. 389 """ 390 return "Field *getTuple(char *inputLine, int recNo)"

391

392 - def getFooter(self):

393 """returns the code for the createDumpfile method. 394 395 You want to use the C fragments above for that. 396 397 The default returns something that bombs out. 398 """ 399 return '#error "No getFooter defined in the code generator"'

400 401

402 -class _LineBasedCodeGenerator(_CodeGenerator):

403 """a base class for code generators for reading line-based text files. 404 """

405 - def getFooter(self):

406 """returns the main function of the parser (and possibly other stuff) 407 in a string. 408 409 This default implementation works for line-based parsers. 410 """ 411 return (COMMON_MAIN_HEADER 412 +NONSEEK_MAIN_INTRO 413 +COMMON_MAIN_INTRO 414 +""" 415 while (fgets(inputLine, INPUT_LINE_MAX, inF)) {""" 416 +LOOP_BODY_INTRO 417 +""" 418 tuple = getTuple(inputLine, recCount);""" 419 +LOOP_BODY_FOOT 420 +"}\n" 421 +COMMON_MAIN_FOOT)

422 423

424 -class ColCodeGenerator(_LineBasedCodeGenerator):

425 - def getItemParser(self, item, index):

426 t = item.type 427 if "smallint" in t: 428 func = "parseShort" 429 elif "int" in t: 430 func = "parseInt" 431 elif t in ["real", "float"]: 432 func = "parseFloat" 433 elif "double" in t: 434 func = "parseDouble" 435 elif "char" in t: 436 func = "parseString" 437 elif "bool" in t: 438 func = "parseBlankBoolean" 439 else: 440 func = "parseWhatever" 441 return ["%s(inputLine, F(%s), start, len);"%(func, getNameForItem(item))]

442

443 - def getPrototype(self):

444 """returns the prototype of the getTuple function. 445 """ 446 return "Field *getTuple(char *inputLine, int recNo)"

447 448

449 -class SplitCodeGenerator(_LineBasedCodeGenerator):

450 """a code generator for parsing files with lineas and separators. 451 """

452 - def __init__(self, grammar, tableDef):

453 self.splitChar = getattr(grammar, "splitChar", "|") 454 _CodeGenerator.__init__(self, grammar, tableDef)

455

456 - def getPreamble(self):

457 return _LineBasedCodeGenerator.getPreamble(self)+[ 458 "/* delete the next line for POSIX strtok */", 459 "#define strtok strtok_u"]

460

461 - def getSetupCode(self):

462 return _LineBasedCodeGenerator.getSetupCode(self)+[ 463 'char *curCont;', 464 'curCont = strtok(inputLine, "%s");'%self.splitChar]

465

466 - def getItemParser(self, item, index):

467 t = item.type 468 fi = getNameForItem(item) 469 if t=="text": 470 parse = ["F(%s)->type = VAL_TEXT;"%fi, 471 "F(%s)->length = strlen(curCont);"%fi, 472 "F(%s)->val.c_ptr = curCont;"%fi,] 473 else: 474 if t=="smallint": 475 cType = "VAL_SHORT" 476 elif t=="bigint": 477 cType = "VAL_BIGINT" 478 elif "int" in t: 479 cType = "VAL_INT" 480 elif t in ["real", "float"]: 481 cType = "VAL_FLOAT" 482 elif "double" in t: 483 cType = "VAL_DOUBLE" 484 elif "char"==t: 485 cType = "VAL_CHAR" 486 elif "char" in t: 487 cType = "VAL_TEXT" 488 elif "bool" in t: 489 cType = "VAL_BOOL" 490 else: 491 cType = "###No appropriate type###" 492 parse = ["fieldscanf(curCont, %s, %s);"%(fi, cType)] 493 parse.append('curCont = strtok(NULL, "%s");'%self.splitChar) 494 return parse

495 496

497 -class BinCodeGenerator(_CodeGenerator):

498 """a code generator for reading fixed-length binary records. 499 """

500 - def getItemParser(self, item, index):

501 t = item.type 502 if t=="integer": 503 pline = "%s(%s, *(int32_t*)(inputLine+));" 504 elif t=="smallint": 505 pline = "%s(%s, *(int16_t*)(inputLine+ ));" 506 elif t=="double precision": 507 pline = "%s(%s, *(double*)(inputLine+ ));" 508 elif t=="real": 509 pline = "%s(%s, *(float*)(inputLine+ ));" 510 elif t=="char": 511 pline = "%s(%s, *(char*)(inputLine+ ), '<nil>');" 512 elif t=="bytea": 513 pline = "%s(%s, *(double*)(inputLine+ ), '<nil>');" 514 else: 515 pline = "%s %s" 516 return ["/* %s (%s) */"%(item.description, t), 517 pline%(_getMakeMacro(item), getNameForItem(item))]

518

519 - def getPreamble(self):

520 return _CodeGenerator.getPreamble(self)+[ 521 "#define FIXED_RECORD_SIZE %d"%self.grammar.recordSize]

522

523 - def getFooter(self):

524 return (COMMON_MAIN_HEADER 525 +" int bytesRead = 0;\n" 526 +NONSEEK_MAIN_INTRO 527 +COMMON_MAIN_INTRO 528 +""" 529 while (1) { 530 bytesRead = fread(inputLine, 1, FIXED_RECORD_SIZE, inF); 531 if (bytesRead==0) { 532 break; 533 } else if (bytesRead!=FIXED_RECORD_SIZE) { 534 die("Short record: Only %d bytes read.", bytesRead); 535 } 536 """ 537 +LOOP_BODY_INTRO 538 +""" 539 tuple = getTuple(inputLine, recCount);""" 540 +LOOP_BODY_FOOT 541 +"}\n" 542 +COMMON_MAIN_FOOT)

543 544

545 -class FITSCodeGenerator(_CodeGenerator):

546 """A code generator for reading from FITS binary tables. 547 """ 548 fitsTypes = { 549 "B": ("TBYTE", "char"), 550 "A": ("TSTRING", "char *"), 551 "I": ("TSHORT", "short"), 552 "J": ("TLONG", "long"), 553 "K": ("TLONGLONG", "long long"), 554 "E": ("TFLOAT", "float"), 555 "L": ("TLOGICAL", "unsigned char"), 556 "D": ("TDOUBLE", "double")} 557 makers = { 558 "bigint": "MAKE_BIGINT", 559 "smallint": "MAKE_SHORT", 560 "bytea": "MAKE_BYTE", 561 "text": "MAKE_TEXT", 562 "integer": "MAKE_INT", 563 "real": "MAKE_FLOAT", 564 "double precision": "MAKE_DOUBLE", 565 } 566

567 - def __init__(self, grammar, tableDef):

568 from gavo.utils import pyfits 569 _CodeGenerator.__init__(self, grammar, tableDef) 570 # now fetch the first source to figure out its schema 571 if self.grammar.parent.sources is None: 572 raise base.StructureError("Cannot make FITS bintable booster without" 573 " a sources element on the embedding data.") 574 575 self.forExtension = grammar.extension 576 577 try: 578 self.fitsTable = pyfits.open( 579 self.grammar.parent.sources.iterSources().next())[self.forExtension] 580 except StopIteration: 581 raise base.StructureError("Buliding a FITS bintable booster requires" 582 " at least one matching source.") 583 584 self._computeMatches()

585

586 - def _computeMatches(self):

587 """adds .fitsIndexForCol and .colForFITSIndex attributes. 588 589 These are matches based on the respective column names, where 590 we do a case-insensitive matching for now. 591 592 Nones mean that no corresponding column is present; for FITS columns, 593 this means they are ignored. For table columns, this means that 594 stand-in code is generated for filling out later. 595 """ 596 tableColumns = dict((col.name.lower(), col) 597 for col in self.tableDef) 598 if len(tableColumns)!=len(self.tableDef.columns): 599 raise base.StructureError("Table unsuitable for FITS boosting as" 600 " column names identical after case folding are present.", 601 hint="Use mapKeys to adapt FITS table names to resolve" 602 " the ambiguity") 603 604 self.colForFITSIndex = {} 605 for index, fitsCol in enumerate(self.fitsTable.columns): 606 columnName = self.grammar.keyMap.get(fitsCol.name, fitsCol.name).lower() 607 self.colForFITSIndex[index] = tableColumns.get(columnName) 608 609 self.fitsIndexForColName = {} 610 for index, col in self.colForFITSIndex.iteritems(): 611 if col is None: 612 continue 613 self.fitsIndexForColName[col.name.lower()] = index

614

615 - def getItemParser(self, item, index):

616 try: 617 fitsIndex = self.fitsIndexForColName[item.name.lower()] 618 fitsCol = self.fitsTable.columns[fitsIndex] 619 castTo = self.fitsTypes[ 620 self._parseFITSFormat(fitsCol.format, fitsCol.name)[1] 621 ][1] 622 623 return [ 624 "/* %s (%s) */"%(item.description, item.type), 625 "if (nulls[%d][rowIndex]) {"%fitsIndex, 626 " MAKE_NULL(%s);"%getNameForItem(item), 627 "} else {", 628 " %s(%s, ((%s*)(data[%d]))[rowIndex]);"%( 629 self.makers[item.type], 630 getNameForItem(item), 631 castTo, 632 fitsIndex), 633 "}",] 634 635 except KeyError: 636 # no FITS table source column 637 return ["MAKE_NULL(%s); /* %s(%s, FILL IN VALUE); */"%( 638 getNameForItem(item), 639 _getMakeMacro(item), 640 getNameForItem(item))]

641

642 - def getPreamble(self):

643 return _CodeGenerator.getPreamble(self)+[ 644 "#include <fitsio.h>", 645 "#include <assert.h>", 646 "#define FITSCATCH(x) if (x) {fatalFitsError(status);}", 647 "void fatalFitsError(int status) {", 648 " if (status==0) {", 649 " return;", 650 " }", 651 " fits_report_error(stderr, status);", 652 " abort();", 653 "}", 654 ]

655

656 - def getPrototype(self):

657 return "Field *getTuple(void *data[], char *nulls[], int rowIndex)"

658

659 - def _parseFITSFormat(self, format, colName):

660 """returns length and typecode for the supported FITS table types. 661 662 All others raise errors. 663 """ 664 mat = re.match("(\d*)(.)$", format) 665 if not mat: 666 raise base.ReportableError("FITS type code '%s' of %s not handled" 667 " by gavo mkboost; add handling if you can."%(format, colName)) 668 if not mat.group(2) in self.fitsTypes: 669 raise base.ReportableError("FITS type '%s' of %s not handled" 670 " by gavo mkboost; add handling if you can."%( 671 mat.group(2), colName)) 672 return int(mat.group(1) or "1"), mat.group(2)

673

674 - def _getColDescs(self):

675 """returns a C initializer for an array of FITSColDescs. 676 """ 677 res = [] 678 for index, fcd in enumerate(self.fitsTable.columns): 679 col = self.colForFITSIndex[index] 680 if col is None: 681 # table column not part of FITS table, suppress reading 682 # my having .cSize=0 683 res.append("{.cSize = 0, .fitsType = 0, .index=0, .arraysize=0}") 684 continue 685 686 length, typecode = self._parseFITSFormat(fcd.format, fcd.name) 687 688 if typecode=="A": 689 # special handling for strings, as we need their size 690 # var length strings have been rejected above 691 res.append( 692 "{.cSize = %d, .fitsType = TSTRING, .index=%d, .arraysize=1}"%( 693 length, index+1)) 694 695 else: 696 res.append( 697 "{.cSize = sizeof(%s), .fitsType = %s, .index=%d, .arraysize=%d}"%( 698 self.fitsTypes[typecode][1], 699 self.fitsTypes[typecode][0], 700 index+1, 701 length)) 702 703 return res

704

705 - def getFooter(self):

706 colDescs = self._getColDescs() 707 infoDict = { 708 "nCols": len(colDescs), 709 "colDescs": "{\n%s\n}"%",\n".join(colDescs), 710 "extension": self.forExtension, 711 } 712 return (""" 713 typedef struct FITSColDesc_s { 714 size_t cSize; 715 int fitsType; 716 int index; /* in the FITS columns */ 717 size_t arraysize; 718 } FITSColDesc; 719 720 FITSColDesc COL_DESCS[%(nCols)d] = %(colDescs)s; 721 """%infoDict+COMMON_MAIN_HEADER 722 +""" 723 fitsfile *fitsInput; 724 int ignored, i; 725 int status = 0; 726 int hdutype = 0; 727 long nRows = 0; 728 void *data[%(nCols)d]; 729 char *nulls[%(nCols)d]; 730 731 if (argc>2) { 732 die(USAGE); 733 } 734 if (argc==2) { 735 FITSCATCH(fits_open_table(&fitsInput, argv[1], READONLY, &status)); 736 } else { 737 die("FITS tables cannot be read from stdin."); 738 } 739 740 FITSCATCH(fits_movabs_hdu(fitsInput, %(extension)s+1, 741 &hdutype, &status)); 742 assert(hdutype=BINARY_TBL); 743 744 FITSCATCH(fits_get_num_rows(fitsInput, &nRows, &status)); 745 746 for (i=0; i<%(nCols)d; i++) { 747 if (COL_DESCS[i].cSize==0) { 748 /* Column not used */ 749 continue; 750 751 } else if (COL_DESCS[i].fitsType==TSTRING) { 752 char *stringHoldings = NULL; 753 if (!(data[i] = malloc(nRows*sizeof(char*))) 754 || !(stringHoldings = malloc(nRows*(COL_DESCS[i].cSize+1)))) { 755 die("out of memory"); 756 } else { 757 int k; 758 /* Initialize the char* in the data array */ 759 for (k=0; k<nRows; k++) { 760 ((char**)(data[i]))[k] = stringHoldings+k*(COL_DESCS[i].cSize+1); 761 } 762 } 763 764 } else { 765 if (!(data[i] = malloc(nRows*COL_DESCS[i].cSize*COL_DESCS[i].arraysize))) { 766 die("out of memory"); 767 } 768 } 769 if (!(nulls[i] = malloc(nRows*sizeof(char)*COL_DESCS[i].arraysize))) { 770 die("out of memory"); 771 } 772 FITSCATCH(fits_read_colnull(fitsInput, COL_DESCS[i].fitsType, 773 COL_DESCS[i].index, 1, 1, 774 nRows*COL_DESCS[i].arraysize, data[i], nulls[i], &ignored, &status)); 775 }"""%infoDict 776 +COMMON_MAIN_INTRO 777 +""" for (i=0; i<nRows; i++) {""" 778 +LOOP_BODY_INTRO 779 +""" tuple = getTuple(data, nulls, i);""" 780 +LOOP_BODY_FOOT 781 +" }\n" 782 +COMMON_MAIN_FOOT)

783 784

785 -def getCodeGen(grammar, tableDef):

786 """returns the code generator suitable for making code for grammar. 787 """ 788 return { 789 "bin": BinCodeGenerator, 790 "split": SplitCodeGenerator, 791 "fits": FITSCodeGenerator, 792 "col": ColCodeGenerator, 793 }[grammar.type](grammar, tableDef)

794 795

796 -def getEnum(td, grammar):

797 code = [ 798 "#define QUERY_N_PARS %d\n"%len(list(td)), 799 'enum outputFields {'] 800 for item in td: 801 desc = item.getLabel() 802 if not desc: 803 desc = item.description 804 code.append("\t%-15s /* %s, %s */"%(getNameForItem(item)+",", 805 desc, item.type)) 806 code.append('};\n') 807 return code

808 809

810 -def getGetTuple(td, codeGen):

811 code = [ 812 codeGen.getPrototype(), 813 "{", 814 "\tstatic Field vals[QUERY_N_PARS];"] 815 code.extend(indent(codeGen.getSetupCode(), "\t")) 816 for index, item in enumerate(td): 817 code.extend(indent(codeGen.getItemParser(item, index), "\t")) 818 code.extend([ 819 "\treturn vals;", 820 "}"]) 821 return code

822 823

824 -def indent(stringList, indentChar):

825 return [indentChar+s for s in stringList]

826 827

828 -def buildSource(grammar, td):

829 """returns (possibly incomplete) C source for a booster to read into td. 830 """ 831 codeGen = getCodeGen(grammar, td) 832 833 code = codeGen.getPreamble() 834 code.extend(getEnum(td, grammar)) 835 code.extend(getGetTuple(td, codeGen)) 836 code.append(codeGen.getFooter()) 837 return "\n".join(code)

838 839

840 -def getGrammarAndTable(grammarId):

841 """returns a pair of directGrammar and table being fed for a cross-rd 842 reference. 843 """ 844 grammar = rscdef.getReferencedElement(grammarId, forceType=DirectGrammar) 845 # to figure out the table built, use the parent's make 846 makes = grammar.parent.makes 847 if len(makes)!=1: 848 raise base.StructureError("Directgrammar only works for data having" 849 " exactly one table, but data '%s' has %d"%( 850 grammar.parent.id, len(makes))) 851 tableDef = makes[0].table 852 return grammar, tableDef

853 854

855 -def parseCmdLine():

856 from optparse import OptionParser 857 parser = OptionParser(usage = "%prog <id-of-directGrammar>") 858 (opts, args) = parser.parse_args() 859 if len(args)!=1: 860 parser.print_help() 861 sys.exit(1) 862 return opts, args[0]

863 864

865 -def getSource(grammarId):

866 """returns a bytestring containing C source to parse grammarId. 867 """ 868 grammar, td = getGrammarAndTable(grammarId) 869 src = buildSource(grammar, td) 870 return src.encode("ascii", "ignore")

871 872

873 -def main():

874 from gavo import rscdesc #noflake: cache registration 875 try: 876 opts, grammarId = parseCmdLine() 877 print(getSource(grammarId)) 878 except SystemExit as msg: 879 sys.exit(msg.code)

880

Source Code for Module gavo.grammars.directgrammar