1 """
2 Definition of DC config options and their management including I/O.
3 """
4
5
6
7
8
9
10
11 from __future__ import print_function
12
13 import cStringIO
14 import os
15 import re
16 import shlex
17 import sys
18 import warnings
19
20 from gavo import utils
21 from gavo.base import attrdef
22 from gavo.base import meta
23 from gavo.base import structure
24 from gavo.utils import fancyconfig
25 from gavo.utils.fancyconfig import (StringConfigItem,
26 EnumeratedConfigItem, IntConfigItem, PathConfigItem, ListConfigItem,
27 BooleanConfigItem, Section, DefaultSection, MagicSection,
28 PathRelativeConfigItem, ParseError, SetConfigItem, ExpandedPathConfigItem)
29
30 fancyconfig.BAD_CONFIG_ITEM_JUST_WARNS = True
31
32 defaultSettingsPath = "/etc/gavo.rc"
33
34 addMeta = meta.configMeta.addMeta
35 setMeta = meta.configMeta.setMeta
36 getMeta = meta.configMeta.getMeta
43
48
51 """is a configuration item that is interpreted relative to
52 the server's root URL.
53 """
54
55 _value = ""
56 typedesc = "URL fragment relative to the server's root"
57
59 if (self._value.startswith("http://")
60 or self._value.startswith("https://")
61 or self._value.startswith("/")):
62 return self._value
63 return self.parent.get("web", "nevowRoot")+self._value
64
67
68 value = property(_getValue, _setValue)
69
72 """is a config item that must not end with a slash. A trailing slash
73 on input is removed.
74 """
75
76 typedesc = "path fragment"
77
80
83 """is a config item that must end with a slash. If no slash is present
84 on input, it is added.
85 """
86
87 typedesc = "path fragment"
88
90 val = StringConfigItem._parse(self, val)
91 if val is not None and not val.endswith("/"):
92 val = val+"/"
93 return val
94
97 """is a config item within the profiles magic section.
98
99 The main point here is to beautify the generated documentation.
100 """
101 typedesc = "profile name"
103 warnings.warn(
104 "Warning: Entries in the profiles section are deprecated and ignored.",
105 DeprecationWarning)
106 StringConfigItem.__init__(self, name, description="A name of a file"
107 " in [db]profilePath")
108 self.default = None
109
112 """an IVOA Identifers-compatible authority.
113 """
115 val = StringConfigItem._parse(self, val)
116 authorityRE = "[a-zA-Z0-9][a-zA-Z0-9._~-]{2,}$"
117 if not re.match(authorityRE, val):
118 raise fancyconfig.BadConfigValue("[ivoa]authority must match %s"
119 " ('more than three normal characters')"%authorityRE)
120 return val
121
122
123 -class Error(utils.Error):
125
128
129 from ConfigParser import NoOptionError
134
137 """is a profile for DB access.
138 """
139 name_ = "dbProfile"
140 profileName = "anonymous"
141
142 _name = attrdef.UnicodeAttribute("name", default=attrdef.Undefined,
143 description="An identifier for this profile")
144 _host = attrdef.UnicodeAttribute("host", default="", description="Host"
145 " the database runs on")
146 _port = attrdef.IntAttribute("port", default=None, description=
147 "Port the DB server listens on")
148 _database = attrdef.UnicodeAttribute("database", default=attrdef.Undefined,
149 description="Name of the database to connect to")
150 _user = attrdef.UnicodeAttribute("user", default="", description=
151 "User to log into DB as")
152 _pw = attrdef.UnicodeAttribute("password", default="", description=
153 "Password for user")
154 _sslmode = attrdef.UnicodeAttribute("sslmode", default="allow", description=
155 "SSL negotiation mode (disable, allow, prefer, require, verify-*)")
156
158 parts = []
159 for key, part in [("host", "host"), ("port", "port"),
160 ("sslmode", "sslmode"), ("database", "dbname"),
161 ("user", "user"), ("password", "password")]:
162 if getattr(self, part):
163 parts.append("%s=%s"%(key, getattr(self, part)))
164 return " ".join(parts)
165
167 """returns a dictionary suitable as keyword arguments to psycopg2's
168 connect.
169 """
170 res = {}
171 for key in ["database", "user", "password", "host", "port", "sslmode"]:
172 if getattr(self, key):
173 res[key] = getattr(self, key)
174 if res.keys()==["sslmode"]:
175 raise utils.logOldExc(utils.StructureError("Insufficient information"
176 " to connect to the database in profile '%s'."%(
177 self.profileName)))
178 return res
179
180 @property
182 """returns the database role used by this profile.
183
184 This normally is user, but in the special case of the empty user,
185 we return the logged users' name.
186 """
187 if self.user:
188 return self.user
189 else:
190 try:
191 return os.getlogin()
192 except os.error:
193 raise utils.ReportableError("Profiles without user= are only"
194 " allowed when DaCHS is called interactively.")
195
198 r"""is a parser for DB profiles.
199
200 The profiles are specified in simple text files that have a shell-like
201 syntax. Each line either contains an assignment (x=y) or is of the
202 form command arg*. Recognized commands include:
203
204 - include f -- read instructions from file f, searched along profilePath
205
206 >>> p = ProfileParser()
207 >>> p.parse(None, "x", "host=foo.bar\n").host
208 'foo.bar'
209 >>> p.parse(None, "x", "") is not None
210 True
211 >>> p.parse(None, "x", "host=\n").host
212 ''
213 >>> p.parse(None, "x", "=bla\n")
214 Traceback (most recent call last):
215 ProfileParseError: "x", line 1: invalid identifier '='
216 >>> p.parse(None, "x", "host=bla")
217 Traceback (most recent call last):
218 ProfileParseError: "x", line 1: unexpected end of file (missing line feed?)
219 """
220 profileKeys = set(["host", "port", "database", "user", "password",
221 "sslmode"])
222
224 self.commands = {
225 "include": self._state_include,
226 }
227 self.sourcePath = sourcePath
228
229 - def parse(self, profileName, sourceName, stream=None):
230 self.tokenStack = []
231 self.stateFun = self._state_init
232 if stream is None:
233 sourceName = self._resolveSource(sourceName)
234 stream = open(sourceName)
235 elif isinstance(stream, basestring):
236 stream = cStringIO.StringIO(stream)
237
238 self.parser = shlex.shlex(stream, sourceName, posix=True)
239 self.parser.whitespace = " \t\r"
240 self.profile = DBProfile(None, name=profileName)
241 while True:
242 tok = self.parser.get_token()
243 if not tok:
244 break
245 self._feed(tok)
246 if self.stateFun!=self._state_init:
247 self._raiseError("unexpected end of file (missing line feed?)")
248 if profileName:
249 self.profile.profileName = profileName
250 return self.profile
251
255
257 if token in self.commands:
258 return self.commands[token]
259 elif token=="\n":
260 return self._state_init
261
262 if not re.match("[A-Za-z][\w]+$", token):
263 self._raiseError("invalid identifier %s"%repr(token))
264 self.tokenStack.append(token)
265 return self._state_waitForEqual
266
268 for dir in self.sourcePath:
269 fqName = os.path.join(dir, fName)
270 if os.path.exists(fqName):
271 return fqName
272 raise ProfileParseError("Requested db profile %s does not exist"%
273 repr(fName))
274
276 if token=="\n":
277 fName = "".join(self.tokenStack)
278 self.tokenStack = []
279 fName = self._resolveSource(fName)
280 self.parser.push_source(open(fName), fName)
281 return self._state_init
282 else:
283 self.tokenStack.append(token)
284 return self._state_include
285
287 if token!="\n":
288 self._raiseError("expected end of line")
289 return self._state_init
290
292 if token!="=":
293 self._raiseError("expected '='")
294 return self._state_rval
295
297 if token=="\n":
298 key = self.tokenStack.pop(0)
299 val = "".join(self.tokenStack)
300 self.tokenStack = []
301 if not key in self.profileKeys:
302 self._raiseError("unknown setting %s"%repr(key))
303 setattr(self.profile, key, val)
304 return self._state_init
305 else:
306 self.tokenStack.append(token)
307 return self._state_rval
308
310 self.stateFun = self.stateFun(token)
311
315 """A container for settings.
316
317 It is a fancyconfig.Configuration with the addition of making the
318 attributes shared at the class level to ward against multiple imports
319 (which may happen if config is imported in a weird way).
320
321 In addition, this class handles the access to database profiles.
322 """
323 __sharedState = {}
328
330 if not hasattr(self, "__profileParser"):
331 self.__profileParser = ProfileParser(
332 self.get("db", "profilePath"))
333 return self.__profileParser
334
347
348
349 _config = Configuration(
350 DefaultSection('Paths and other general settings.',
351 ExpandedPathConfigItem("rootDir", default="/var/gavo", description=
352 "Path to the root of the DC file (all other paths may be"
353 " relative to this"),
354 RootRelativeConfigItem("configDir", default="etc",
355 description="Path to the DC's non-ini configuration (e.g., DB profiles)"),
356 RootRelativeConfigItem("inputsDir", default="inputs",
357 description="Path to the DC's data holdings"),
358 RootRelativeConfigItem("cacheDir", default="cache",
359 description="Path to the DC's persistent scratch space"),
360 RootRelativeConfigItem("logDir", default="logs",
361 description="Path to the DC's logs (should be local)"),
362 RootRelativeConfigItem("tempDir", default="tmp",
363 description="Path to the DC's scratch space (should be local)"),
364 RootRelativeConfigItem("webDir", default="web",
365 description="Path to the DC's web related data (docs, css, js,"
366 " templates...)"),
367 RootRelativeConfigItem("stateDir", default="state",
368 description="Path to the DC's state information (last imported,...)"),
369 RootRelativeConfigItem("uwsWD", default="state/uwsjobs",
370 description="Directory to keep uws jobs in. This may need lots"
371 " of space if your users do large queries"),
372 EnumeratedConfigItem("logLevel", options=["info", "warning",
373 "debug", "error"], description="How much should be logged?"),
374 StringConfigItem("operator", description=
375 "Deprecated and ignored. Use contact.email in defaultmeta.txt instead."),
376 StringConfigItem("platform", description="Platform string (can be"
377 " empty if inputsDir is only accessed by identical machines)"),
378 StringConfigItem("gavoGroup", description="Name of the unix group that"
379 " administers the DC", default="gavo"),
380 StringConfigItem("defaultProfileName", description="Deprecated"
381 " and ignored.", default=""),
382 StringConfigItem("group", description="Name of the group that may write"
383 " into the log directory", default="gavo"),
384 PathConfigItem("xsdclasspath", description="Classpath necessary"
385 " to validate XSD using an xsdval java class. You want GAVO's"
386 " VO schemata collection for this. Deprecated, we're now using"
387 " libxml2 for validation.", default="None"),
388 StringConfigItem("sendmail", default="sendmail -t",
389 description="Command that reads a mail from stdin, taking the"
390 "recipient address from the mail header, and transfers the"
391 " mail (this is for sending mails to the administrator)."
392 " This command is processed by a shell (generally running as"
393 " the server user), so you can do tricks if necessary."),
394 StringConfigItem("maintainerAddress", default="",
395 description="An e-mail address to send reports and warnings to;"
396 " this could be the same as contact.email; in practice, it is"
397 " shown in more technical circumstances, so it's adviable"
398 " to have a narrower distribution here."),
399 ),
400
401 Section('web', 'Settings related to serving content to the web.',
402 StringConfigItem("serverURL", default="http://localhost:8080",
403 description="URL fragment used to qualify relative URLs where"
404 " necessary. Note that this must contain the port the server is"
405 " accessible under from the outside if that is not 80; nonstandard"
406 " ports are not supported for https. If you offer"
407 " both http and https, use your preferred protocol here; for"
408 " robustness, we recommend preferring http."),
409 ListConfigItem("alternateHostnames", default="",
410 description="A comma-separated list of hostnames this server is"
411 " also known under. Only set this if you're running https."
412 " With this, you can handle a situation where your data"
413 " center can be reached as both example.org and www.example.org."),
414 StringConfigItem("bindAddress", default="127.0.0.1", description=
415 "Interface to bind to"),
416 IntConfigItem("serverPort", default="8080",
417 description="Port to bind the http port of the server to; https,"
418 " if present at all, is always on 443."),
419 StringConfigItem("user", default="gavo", description="Run server as"
420 " this user."),
421 EnsureTrailingSlashesItem("nevowRoot", default="/",
422 description="Path fragment to the server's root for operation off the"
423 " server's root; this must end with a slash (setting this"
424 " will currently break essentially the entire web interface."
425 " If you must use it, contact the authors and we will fix things.)"),
426 StringConfigItem("realm", default="X-Unconfigured",
427 description="Authentication realm to be used (currently,"
428 " only one, server-wide, is supported)"),
429 WebRelativeConfigItem("templateDir", default="templates",
430 description="webDir-relative location of global nevow templates"),
431 StringConfigItem("adminpasswd", default="",
432 description="Password for online administration, leave empty to disable"),
433 StringConfigItem("sitename", "Unnamed data center",
434 "A short name for your site"),
435 RelativeURL("voplotCodeBase", "None",
436 "Deprecated and ignored."),
437 RelativeURL("voplotUserman",
438 "Deprecated and ignored",
439 "URL to the documentation of VOPlot"),
440 IntConfigItem("sqlTimeout", "15",
441 "Default timeout for db queries via the web"),
442 WebRelativeConfigItem("previewCache", "previewcache",
443 "Webdir-relative directory to store cached previews in"),
444 WebRelativeConfigItem("favicon", "None",
445 "Webdir-relative path to a favicon; this overrides the default of"
446 " a scaled version of the logo."),
447 BooleanConfigItem("enableTests", "False",
448 "Enable test pages (don't if you don't know why)"),
449 IntConfigItem("maxPreviewWidth", "300", "Ignored, only present"
450 " for backward compatiblity"),
451 ListConfigItem("graphicMimes", "image/fits,image/jpeg,application/x-votable+xml;content=datalink", "MIME types"
452 " considered as graphics (for SIAP, mostly)"),
453 StringConfigItem("adsMirror",
454 "http://ads.g-vo.org",
455 "Root URL of ADS mirror to be used (without a trailing slash)"),
456 IntConfigItem("maxUploadSize",
457 "20000000",
458 "Maximal size of (mainly TAP) file uploads in async requests in bytes;"
459 " sync requests use maxSyncUploadSize."),
460 IntConfigItem("maxSyncUploadSize",
461 "500000",
462 "Maximal size of file uploads for synchronous TAP in bytes."),
463 ListConfigItem("preloadRDs", "", "RD ids to preload at the server"
464 " start. Load time of RDs listed here goes against startup time,"
465 " so only do this for RDs that have execute children"
466 " that should run regularly. For everything else consider"
467 " preloadPublishedRDs."),
468 BooleanConfigItem("preloadPublishedRDs", "False", "Preload"
469 " all RDs of services you've published. This is mainly helpful"
470 " when code in such RDs might, for instance, lock and such"
471 " failures don't suddenly occur in operation."),
472 BooleanConfigItem("jsSource", "False", "If True, Javascript"
473 " will not be minified on delivery (this is for debugging)"),
474 StringConfigItem("operatorCSS", "", "URL of an operator-specific"
475 " CSS. This is included as the last item and can therefore"
476 " override rules in the distributed CSS."),
477 StringConfigItem("corsOriginPat", "", "A regular expression"
478 " for URLs from which to authorise cross-origin requests."
479 " This is matched, i.e., the RE must account for the whole URL"
480 r" including the schema. Example: https?://example\.com/apps/.*."),
481 IntConfigItem("serverFDLimit", "4000",
482 "A hard limit of the number of file handles DaCHS should try to"
483 " set. This will only take effect if DaCHS is started as root."
484 " Otherwise, DaCHS will just adjust its soft limit to the"
485 " external limit irrespective of this setting."),
486 EnumeratedConfigItem("logFormat", "default", "Log format to use."
487 " Default doesn't log IPs, user agents, or referrers and"
488 " thus should be ok in terms of not processing personal data,"
489 " which in turn means you probably don't have to declare anything"
490 " in EU jurisdictions.", options=["default", "combined"]),
491 ),
492
493 Section('adql', "(ignored, only left for backward compatibility)",
494 IntConfigItem("webDefaultLimit", "2000",
495 "(ignored, only present for backwards compatibility; use"
496 " [async]defaultMAXREC instead."),
497 ),
498
499 Section('async', "Settings concerning TAP, UWS, and friends",
500 IntConfigItem("defaultExecTimeSync", "60", "Default timeout"
501 " for synchronous UWS jobs, in seconds"),
502 IntConfigItem("defaultExecTime", "3600", "Default timeout"
503 " for UWS jobs, in seconds"),
504 IntConfigItem("maxTAPRunning", "2", "Maximum number of"
505 " TAP jobs running at a time"),
506 IntConfigItem("maxUserUWSRunningDefault", "2", "Maximum number of"
507 " user UWS jobs running at a time"),
508 IntConfigItem("defaultLifetime", "172800", "Default"
509 " time to destruction for UWS jobs, in seconds"),
510 IntConfigItem("defaultMAXREC", "20000",
511 "Default match limit for ADQL queries via the UWS/TAP"),
512 IntConfigItem("hardMAXREC", "20000000",
513 "Hard match limit (i.e., users cannot raise MAXREC or TOP beyond that)"
514 " for ADQL queries via the UWS/TAP"),
515 StringConfigItem("csvDialect", "excel", "CSV dialect as defined"
516 " by the python csv module used when writing CSV files."),
517 IntConfigItem("maxSlowPollWait", "300", "Maximal time a UWS 1.1-WAIT"
518 " request will delay the response. This should be smaller than"
519 " what you have as timeout on outgoing connections."),
520 ),
521
522 Section('ui', "Settings concerning the local user interface",
523 StringConfigItem("outputEncoding", "iso-8859-1",
524 "Encoding for system messages. This should match what your"
525 " terminal emulator is set to"),
526 ),
527
528 Section('db', 'Settings concerning database access.',
529 IntConfigItem("indexWorkMem",
530 "2000",
531 "Megabytes of memory to give to postgres while making indices."
532 " Set to roughly half your RAM when you have big tables."),
533 StringConfigItem("interface", "psycopg2", "Don't change"),
534 PathConfigItem("profilePath", "~/.gavo:$configDir",
535 "Path for locating DB profiles"),
536 StringConfigItem("msgEncoding", "utf-8", "Encoding of the"
537 " messages coming from the database"),
538 SetConfigItem("maintainers", "admin", "Name(s) of profiles"
539 " that should have full access to gavo imp-created tables by default"),
540 SetConfigItem("queryProfiles", "trustedquery", "Name(s) of profiles that"
541 " should be able to read gavo imp-created tables by default"),
542 SetConfigItem("adqlProfiles", "untrustedquery", "Name(s) of profiles that"
543 " get access to tables opened for ADQL"),
544 IntConfigItem("defaultLimit", "100", "Default match limit for DB queries"),
545 ListConfigItem("managedExtensions",
546 "pg_sphere",
547 "Name(s) of postgres extensions gavo upgrade -e should watch"),
548 BooleanConfigItem("dumpSystemTables",
549 "False",
550 "Dump the tables from //users and //system to"
551 " stateDir/system_tables once a day?"),
552 ),
553
554 MagicSection('profiles', 'Ignored and deprecated, only here for backward'
555 ' compatibility.',
556 itemFactory=ProfileItem),
557
558 Section('ivoa', 'The interface to the Greater VO.',
559 AuthorityConfigItem("authority", "x-unregistred",
560 "The authority id for this DC; this has *no* leading ivo://"),
561 IntConfigItem("dalDefaultLimit", "10000",
562 "Default match limit on SCS/SSAP/SIAP queries"),
563 IntConfigItem("dalHardLimit", "1000000",
564 "Hard match limit on SCS/SSAP/SIAP queries (be careful: due to the"
565 " way these protocols work, the results cannot be streamed, and"
566 " the results have to be kept in memory; 1e7 rows requiring 1k"
567 " of memory each add up to 10 Gigs...)"),
568 IntConfigItem("oaipmhPageSize", "500",
569 "Default number of records per page in the OAI-PMH interface"),
570 EnumeratedConfigItem("votDefaultEncoding", "binary",
571 "Default 'encoding' for VOTables in many places (like the DAL"
572 " responses; this can be user-overridden using"
573 " the _TDENC local HTTP parameter.", options=["binary", "td"]),
574 EnumeratedConfigItem("sdmVersion", "1",
575 "Obsolete (SDM version 2 is shelved). Don't use.",
576 options=["1", "2"]),
577 EnumeratedConfigItem("VOSITableDetail",
578 "max",
579 "Default level of detail to return on the VOSI endpoint (change"
580 " to min when you have more than 100 or tables).",
581 options=["min", "max"]),
582 BooleanConfigItem("registerAlternative",
583 "False",
584 "Give access URLs for the alternative protocol (https when"
585 " serverURL is http and vice versa) as a mirrorURL? If you're"
586 " listening to HTTPS, this is probably a good idea."),
587 ),
588 )
610
611 loadConfig()
612
613
614 if "GAVO_INPUTSDIR" in os.environ:
615 _config.set("inputsDir", os.environ["GAVO_INPUTSDIR"])
616
617 get = _config.get
618 set = _config.set
619 getDBProfile = _config.getDBProfile
635
636 makeFallbackMeta()
640 try:
641 if len(sys.argv)==1:
642 print(fancyconfig.makeTxtDocs(_config, underlineChar="'"))
643 sys.exit(0)
644 elif len(sys.argv)==2:
645 item = _config.getitem(sys.argv[1])
646 elif len(sys.argv)==3:
647 item = _config.getitem(sys.argv[1], sys.argv[2])
648 else:
649 sys.stderr.write("Usage: %s [<sect> <key> | <key>]\n")
650 sys.exit(1)
651 except NoOptionError:
652 print("")
653 sys.exit(2)
654 print(item.getAsString())
655
660
661
662 if __name__=="__main__":
663 _test()
664