"""
Definition of DC config options and their management including I/O.
"""
#c Copyright 2008-2023, the GAVO project <gavo@ari.uni-heidelberg.de>
#c
#c This program is free software, covered by the GNU GPL. See the
#c COPYING file in the source distribution.
import io
import os
import re
import shlex
import sys
# You can't import base or anything from base here, as that would
# import sqlsupport, and sqlsupport needs config initialised to
# even come up.
from gavo import utils
from gavo.utils import fancyconfig
from gavo.utils.fancyconfig import (StringConfigItem, #noflake: exported names
EnumeratedConfigItem, IntConfigItem, PathConfigItem, ListConfigItem,
BooleanConfigItem, Section, DefaultSection, MagicSection,
PathRelativeConfigItem, ParseError, SetConfigItem, ExpandedPathConfigItem)
fancyconfig.BAD_CONFIG_ITEM_JUST_WARNS = True
defaultSettingsPath = "/etc/gavo.rc"
[docs]class RootRelativeConfigItem(PathRelativeConfigItem):
baseKey = "rootDir"
typedesc = "path relative to rootDir"
[docs]class WebRelativeConfigItem(PathRelativeConfigItem):
baseKey = "webDir"
typedesc = "path relative to webDir"
[docs]class RelativeURL(StringConfigItem):
"""is a configuration item that is interpreted relative to
the server's root URL.
"""
_value = ""
typedesc = "URL fragment relative to the server's root"
def _getValue(self):
if (self._value.startswith("http://")
or self._value.startswith("https://")
or self._value.startswith("/")):
return self._value
return self.parent.get("web", "nevowRoot")+self._value
def _setValue(self, val):
self._value = val
value = property(_getValue, _setValue)
[docs]class EatTrailingSlashesItem(StringConfigItem):
"""is a config item that must not end with a slash. A trailing slash
on input is removed.
"""
typedesc = "path fragment"
def _parse(self, val):
return StringConfigItem._parse(self, val).rstrip("/")
[docs]class EnsureTrailingSlashesItem(StringConfigItem):
"""is a config item that must end with a slash. If no slash is present
on input, it is added.
"""
typedesc = "path fragment"
def _parse(self, val):
val = StringConfigItem._parse(self, val)
if val is not None and not val.endswith("/"):
val = val+"/"
return val
[docs]class AuthorityConfigItem(StringConfigItem):
"""an IVOA Identifers-compatible authority.
"""
def _parse(self, val):
val = StringConfigItem._parse(self, val)
authorityRE = "[a-zA-Z0-9][a-zA-Z0-9._~-]{2,}$"
if not re.match(authorityRE, val):
raise fancyconfig.BadConfigValue("[ivoa]authority must match %s"
" ('more than three normal characters')"%authorityRE)
return val
[docs]class Error(utils.Error):
pass
[docs]class ProfileParseError(Error):
pass
from configparser import NoOptionError
[docs]class DBProfile:
"""A parsed form of the postgres connection string.
"""
profileName = "anonymous"
name = None
host = ""
port = None
database = ""
user = ""
password = ""
sslmode = "allow" # or disable, allow, prefer, require, verify-*)")
def __init__(self, **kwargs):
for k, v in kwargs.items():
setattr(self, k, v)
[docs] def getArgs(self):
"""returns a dictionary suitable as keyword arguments to psycopg2's
connect.
"""
res = {}
for key in ["database", "user", "password", "host", "port", "sslmode"]:
if getattr(self, key):
res[key] = getattr(self, key)
if list(res.keys())==["sslmode"]:
raise utils.logOldExc(utils.StructureError("Insufficient information"
" to connect to the database in profile '%s'."%(
self.profileName)))
return res
@property
def roleName(self):
"""returns the database role used by this profile.
This normally is user, but in the special case of the empty user,
we return the logged users' name.
"""
if self.user:
return self.user
else:
try:
return os.getlogin()
except os.error:
raise utils.ReportableError("Profiles without user= are only"
" allowed when DaCHS is called interactively.")
[docs]class ProfileParser:
r"""is a parser for DB profiles.
The profiles are specified in simple text files that have a shell-like
syntax. Each line either contains an assignment (x=y) or is of the
form command arg*. Recognized commands include:
- include f -- read instructions from file f, searched along profilePath
>>> p = ProfileParser()
>>> p.parse(None, "x", "host=foo.bar\n").host
'foo.bar'
>>> p.parse(None, "x", "") is not None
True
>>> p.parse(None, "x", "host=\n").host
''
>>> p.parse(None, "x", "=bla\n")
Traceback (most recent call last):
gavo.base.config.ProfileParseError: "x", line 1: invalid identifier '='
>>> p.parse(None, "x", "host=bla")
Traceback (most recent call last):
gavo.base.config.ProfileParseError: "x", line 1: unexpected end of file (missing line feed?)
"""
profileKeys = set(["host", "port", "database", "user", "password",
"sslmode"])
def __init__(self, sourcePath=["."]):
self.commands = {
"include": self._state_include,
}
self.sourcePath = sourcePath
[docs] def parse(self, profileName, sourceName, stream=None):
self.tokenStack = []
self.stateFun = self._state_init
if stream is None:
sourceName = self._resolveSource(sourceName)
stream = open(sourceName)
elif isinstance(stream, str):
stream = io.StringIO(stream)
self.parser = shlex.shlex(stream, sourceName, posix=True)
self.parser.whitespace = " \t\r"
self.profile = DBProfile(name=profileName)
while True:
tok = self.parser.get_token()
if not tok:
break
self._feed(tok)
if self.stateFun!=self._state_init:
self._raiseError("unexpected end of file (missing line feed?)")
if profileName:
self.profile.profileName = profileName
stream.close()
return self.profile
def _raiseError(self, msg):
raise utils.logOldExc(
ProfileParseError(self.parser.error_leader()+msg))
def _state_init(self, token):
if token in self.commands:
return self.commands[token]
elif token=="\n":
return self._state_init
if not re.match("[A-Za-z][\w]+$", token):
self._raiseError("invalid identifier %s"%repr(token))
self.tokenStack.append(token)
return self._state_waitForEqual
def _resolveSource(self, fName):
for dir in self.sourcePath:
fqName = os.path.join(dir, fName)
if os.path.exists(fqName):
return fqName
raise ProfileParseError("Requested db profile %s does not exist"%
repr(fName))
def _state_include(self, token):
if token=="\n":
fName = "".join(self.tokenStack)
self.tokenStack = []
fName = self._resolveSource(fName)
self.parser.push_source(open(fName), fName)
return self._state_init
else:
self.tokenStack.append(token)
return self._state_include
def _state_eol(self, token):
if token!="\n":
self._raiseError("expected end of line")
return self._state_init
def _state_waitForEqual(self, token):
if token!="=":
self._raiseError("expected '='")
return self._state_rval
def _state_rval(self, token):
if token=="\n":
key = self.tokenStack.pop(0)
val = "".join(self.tokenStack)
self.tokenStack = []
if not key in self.profileKeys:
self._raiseError("unknown setting %s"%repr(key))
setattr(self.profile, key, val)
return self._state_init
else:
self.tokenStack.append(token)
return self._state_rval
def _feed(self, token):
self.stateFun = self.stateFun(token)
[docs]class Configuration(fancyconfig.Configuration):
"""A container for settings.
It is a fancyconfig.Configuration with the addition of making the
attributes shared at the class level to ward against multiple imports
(which may happen if config is imported in a weird way).
In addition, this class handles the access to database profiles.
"""
__sharedState = {}
def __init__(self, *items):
self.__dict__ = self.__sharedState
fancyconfig.Configuration.__init__(self, *items)
self._dbProfileCache = {}
def _getProfileParser(self):
if not hasattr(self, "__profileParser"):
self.__profileParser = ProfileParser(
self.get("db", "profilePath"))
return self.__profileParser
[docs] def getDBProfile(self, profileName):
# remains of retired profile name mapping infrastructure
if profileName=='admin':
profileName = 'feed'
if profileName not in self._dbProfileCache:
try:
self._dbProfileCache[profileName] = self._getProfileParser().parse(
profileName, profileName)
except utils.NoConfigItem:
raise ProfileParseError("Undefined DB profile: %s"%profileName)
return self._dbProfileCache[profileName]
_config = Configuration(
DefaultSection('Paths and other general settings.',
ExpandedPathConfigItem("rootDir", default="/var/gavo", description=
"Path to the root of the DC file (all other paths may be"
" relative to this"),
RootRelativeConfigItem("configDir", default="etc",
description="Path to the DC's non-ini configuration (e.g., DB profiles)"),
RootRelativeConfigItem("inputsDir", default="inputs",
description="Path to the DC's data holdings"),
RootRelativeConfigItem("cacheDir", default="cache",
description="Path to the DC's persistent scratch space"),
RootRelativeConfigItem("logDir", default="logs",
description="Path to the DC's logs (should be local)"),
RootRelativeConfigItem("tempDir", default="tmp",
description="Path to the DC's scratch space (should be local)"),
RootRelativeConfigItem("webDir", default="web",
description="Path to the DC's web related data (docs, css, js,"
" templates...)"),
RootRelativeConfigItem("stateDir", default="state",
description="Path to the DC's state information (last imported,...)"),
RootRelativeConfigItem("uwsWD", default="state/uwsjobs",
description="Directory to keep uws jobs in. This may need lots"
" of space if your users do large queries"),
EnumeratedConfigItem("logLevel", options=["info", "warning",
"debug", "error"], description="How much should be logged?"),
StringConfigItem("platform", description="Platform string (can be"
" empty if inputsDir is only accessed by identical machines)"),
StringConfigItem("gavoGroup", description="Name of the unix group that"
" administers the DC", default="gavo"),
StringConfigItem("defaultProfileName", description="Deprecated"
" and ignored.", default=""),
StringConfigItem("group", description="Name of the group that may write"
" into the log directory", default="gavo"),
PathConfigItem("xsdclasspath", description="Classpath necessary"
" to validate XSD using an xsdval java class. You want GAVO's"
" VO schemata collection for this. Deprecated, we're now using"
" libxml2 for validation.", default="None"),
StringConfigItem("sendmail", default="sendmail -t",
description="Command that reads a mail from stdin, taking the"
"recipient address from the mail header, and transfers the"
" mail (this is for sending mails to the administrator)."
" This command is processed by a shell (generally running as"
" the server user), so you can do tricks if necessary."),
StringConfigItem("maintainerAddress", default="",
description="An e-mail address to send reports and warnings to;"
" this could be the same as contact.email; in practice, it is"
" shown in more technical circumstances, so it's adviable"
" to have a narrower distribution here."),
SetConfigItem("future", default="",
description="A set of strings naming experimental features"
" to enable in the server. You only want to touch this"
" if you are working on proving proposed improvements."),
SetConfigItem("rdblacklist", default="",
description="File names of blacklisted RDs. These are matched"
" against the ends of file names, so be careful you do not"
" over-block; in general, you should use something like"
" '/resname/q.rd' or so. You can also blacklist RD ids"
" (which are compared literally). Separate multiple entries"
" by commas."),
),
Section('web', 'Settings related to serving content to the web.',
StringConfigItem("serverURL", default="http://localhost:8080",
description="URL fragment used to qualify relative URLs where"
" necessary. Note that this must contain the port the server is"
" accessible under from the outside if that is not 80; nonstandard"
" ports are not supported for https. If you offer"
" both http and https, use your preferred protocol here."
" If you change this on a running server, you *must* run"
" dachs pub -a to update internal and external links."),
ListConfigItem("alternateHostnames", default="",
description="A comma-separated list of hostnames this server is"
" also known under. Only set this if you're running https."
" With this, you can handle a situation where your data"
" center can be reached as both example.org and www.example.org."),
StringConfigItem("bindAddress", default="127.0.0.1", description=
"Interface to bind to"),
IntConfigItem("serverPort", default="8080",
description="Port to bind the http port of the server to; https,"
" if present at all, is always on 443."),
BooleanConfigItem("adaptProtocol", default="True",
description="Adapt internal absolute links to http/https by"
" request method used. You must switch this off if running"
" behind a reverse proxy."),
StringConfigItem("user", default="gavo", description="Run server as"
" this user."),
EnsureTrailingSlashesItem("nevowRoot", default="/",
description="Path fragment to the server's root for operation off the"
" server's root; this must end with a slash (setting this"
" will currently break essentially the entire web interface."
" If you must use it, contact the authors and we will fix things.)"),
StringConfigItem("realm", default="X-Unconfigured",
description="Authentication realm to be used (currently,"
" only one, server-wide, is supported)"),
WebRelativeConfigItem("templateDir", default="templates",
description="webDir-relative location of global nevow templates"),
StringConfigItem("adminpasswd", default="",
description="Password for online administration, leave empty to disable"),
StringConfigItem("sitename", "Unnamed data center",
"A short name for your site"),
IntConfigItem("sqlTimeout", "15",
"Default timeout for non-TAP database queries (which can in"
" general be overridden by users setting _TIMEOUT)."),
WebRelativeConfigItem("previewCache", "previewcache",
"Webdir-relative directory to store cached previews in"),
WebRelativeConfigItem("favicon", "None",
"Webdir-relative path to a favicon; this overrides the default of"
" a scaled version of the logo."),
BooleanConfigItem("enableTests", "False",
"Enable test pages (don't if you don't know why)"),
IntConfigItem("maxPreviewWidth", "300", "Ignored, only present"
" for backward compatibility"),
ListConfigItem("graphicMimes",
"image/fits,image/jpeg,application/x-votable+xml;content=datalink",
"Media types considered as graphics (for SIAP, mostly)"),
IntConfigItem("maxUploadSize",
"20000000",
"Maximal size of (mainly TAP) file uploads in async requests in bytes;"
" sync requests use maxSyncUploadSize."),
IntConfigItem("maxSyncUploadSize",
"500000",
"Maximal size of file uploads for synchronous TAP in bytes."),
ListConfigItem("preloadRDs", "", "RD ids to preload at the server"
" start. Load time of RDs listed here goes against startup time,"
" so only do this for RDs that have execute children"
" that should run regularly. For everything else consider"
" preloadPublishedRDs."),
BooleanConfigItem("preloadPublishedRDs", "False", "Preload"
" all RDs of services you've published. This is mainly helpful"
" when code in such RDs might, for instance, lock and such"
" failures don't suddenly occur in operation."),
BooleanConfigItem("jsSource", "False", "If True, Javascript"
" will not be minified on delivery (this is for debugging)"),
StringConfigItem("operatorCSS", "", "URL of an operator-specific"
" CSS. This is included as the last item and can therefore"
" override rules in the distributed CSS."),
StringConfigItem("corsOriginPat", "", "A regular expression"
" for URLs from which to authorise cross-origin requests."
" This is matched, i.e., the RE must account for the whole URL"
r" including the schema. Example: https?://example\.com/apps/.*."),
IntConfigItem("serverFDLimit", "4000",
"A hard limit of the number of file handles DaCHS should try to"
" set. This will only take effect if DaCHS is started as root."
" Otherwise, DaCHS will just adjust its soft limit to the"
" external limit irrespective of this setting."),
EnumeratedConfigItem("logFormat", "default", "Log format to use."
" Default doesn't log IPs, user agents, or referrers and"
" thus should be ok in terms of not processing personal data,"
" which in turn means you probably don't have to declare anything"
" in EU jurisdictions.", options=["default", "combined"]),
StringConfigItem("root", "__system__/services/root/fixed",
"The path of the the root page of the data center. The default"
" renders the root.html template. On single-service installations,"
" you could direct this to that single service, or you could"
" use the ADQL query form."),
BooleanConfigItem("ignore-uir-header", "False",
"Do not honour clients' upgrade-insecure-requests headers if"
" https is enabled. Outside of development or testing environments,"
" there is generally little reason to use this.")
),
Section('adql', "(ignored, only left for backward compatibility)",
IntConfigItem("webDefaultLimit", "2000",
"(ignored, only present for backwards compatibility; use"
" [async]defaultMAXREC instead."),
),
Section('async', "Settings concerning TAP, UWS, and friends",
IntConfigItem("defaultExecTimeSync", "60", "Timeout"
" for synchronous TAP/UWS jobs, in seconds."),
IntConfigItem("defaultExecTime", "3600", "Default timeout"
" for UWS jobs, in seconds"),
IntConfigItem("maxTAPRunning", "2", "Maximum number of"
" TAP jobs running at a time"),
IntConfigItem("maxUserUWSRunningDefault", "2", "Maximum number of"
" user UWS jobs running at a time"),
IntConfigItem("defaultLifetime", "172800", "Default"
" time to destruction for UWS jobs, in seconds"),
IntConfigItem("defaultMAXREC", "20000",
"Default match limit for ADQL queries via the UWS/TAP"),
IntConfigItem("hardMAXREC", "20000000",
"Hard match limit (i.e., users cannot raise MAXREC or TOP beyond that)"
" for ADQL queries via the UWS/TAP"),
StringConfigItem("csvDialect", "excel", "CSV dialect as defined"
" by the python csv module used when writing CSV files."),
IntConfigItem("maxSlowPollWait", "300", "Maximal time a UWS 1.1-WAIT"
" request will delay the response. This should be smaller than"
" what you have as timeout on outgoing connections."),
),
Section('ui', "Settings concerning the local user interface",
),
Section('db', 'Settings concerning database access.',
IntConfigItem("indexWorkMem",
"2000",
"Megabytes of memory to give to postgres while making indices."
" Set to roughly half your RAM when you have big tables."),
StringConfigItem("interface", "psycopg2", "Don't change"),
PathConfigItem("profilePath", "~/.gavo:$configDir",
"Path for locating DB profiles"),
StringConfigItem("msgEncoding", "utf-8", "Encoding of the"
" messages coming from the database"),
SetConfigItem("maintainers", "admin", "Name(s) of profiles"
" that should have full access to gavo imp-created tables by default"),
SetConfigItem("queryProfiles", "trustedquery", "Name(s) of profiles that"
" should be able to read gavo imp-created tables by default"),
SetConfigItem("adqlProfiles", "untrustedquery", "Name(s) of profiles that"
" get access to tables opened for ADQL"),
IntConfigItem("defaultLimit", "100", "Default match limit for DB queries"),
ListConfigItem("managedExtensions",
"pg_sphere",
"Name(s) of postgres extensions gavo upgrade -e should watch"),
BooleanConfigItem("dumpSystemTables",
"False",
"Dump the tables from //users and //system to"
" stateDir/system_tables once a day?"),
IntConfigItem("poolSize", "2",
"Number of connections in DaCHS' postgres connection pools."),
),
Section('ivoa', 'The interface to the Greater VO.',
AuthorityConfigItem("authority", "x-unregistred",
"The authority id for this DC; this has *no* leading ivo://"),
IntConfigItem("dalDefaultLimit", "10000",
"Default match limit on SCS/SSAP/SIAP queries"),
IntConfigItem("dalHardLimit", "1000000",
"Hard match limit on SCS/SSAP/SIAP queries (be careful: due to the"
" way these protocols work, the results cannot be streamed, and"
" the results have to be kept in memory; 1e7 rows requiring 1k"
" of memory each add up to 10 Gigs...)"),
IntConfigItem("oaipmhPageSize", "500",
"Default number of records per page in the OAI-PMH interface"),
EnumeratedConfigItem("votDefaultEncoding", "binary",
"Default 'encoding' for VOTables in many places (like the DAL"
" responses; this can be user-overridden using"
" the _TDENC local HTTP parameter.", options=["binary", "td"]),
EnumeratedConfigItem("sdmVersion", "1",
"Obsolete (SDM version 2 is shelved). Don't use.",
options=["1", "2"]),
EnumeratedConfigItem("VOSITableDetail",
"max",
"Default level of detail to return on the VOSI endpoint (change"
" to min when you have more than 100 or tables).",
options=["min", "max"]),
BooleanConfigItem("registerAlternative",
"False",
"Give access URLs for the alternative protocol (https when"
" serverURL is http and vice versa) as a mirrorURL? If you're"
" listening to HTTPS, this is probably a good idea."),
ListConfigItem("validOAISets",
"ivo_managed, vosi, local",
"Comma-separated list of OAI sets DaCHS should accept for publication."
" This must always include ivo_managed and vosi, and you will"
" want local if you run a web interface.")
),
)
[docs]def loadConfig():
try:
fancyconfig.readConfiguration(_config,
os.environ.get("GAVOSETTINGS", "/etc/gavo.rc"),
os.environ.get("GAVOCUSTOM",
os.path.join(os.environ.get("HOME", "/no_home"), ".gavorc")))
except fancyconfig.ConfigError as ex:
# This is usually not be protected by top-level exception catcher
sys.exit("Bad configuration item in %s. %s"%(
ex.fileName, str(ex)))
# also set XDG directories so astropy and friends look for their
# configuration in DaCHS' directories
os.environ["XDG_CONFIG_HOME"] = _config.get("configDir")
os.environ["XDG_CACHE_HOME"] = _config.get("cacheDir")
# also configure matplotlib to pull their configuration from
# DaCHS' configuration (rather than any random matplotlibrc people
# may have left).
os.environ["MATPLOTLIBRC"] = _config.get("configDir")
loadConfig()
if "GAVO_INPUTSDIR" in os.environ:
_config.set("inputsDir", os.environ["GAVO_INPUTSDIR"])
get = _config.get
set = _config.set
getitem = _config.getitem
getDBProfile = _config.getDBProfile
[docs]def getRSTReference(underlineChar="-"):
"""returns a ReStructuredText reference of configuration items.
This will have one RST section per configuration section; to let
you adapt that to an embedding document, you can pass the character
to use for headline underlines in underlineChar.
"""
return fancyconfig.makeTxtDocs(_config, underlineChar=underlineChar)
[docs]def main(): # pragma: no cover
try:
if len(sys.argv)==1:
print(getRSTReference)
sys.exit(0)
elif len(sys.argv)==2:
item = _config.getitem(sys.argv[1])
elif len(sys.argv)==3:
item = _config.getitem(sys.argv[1], sys.argv[2])
else:
sys.stderr.write("Usage: %s [<sect> <key> | <key>]\n")
sys.exit(1)
except NoOptionError:
print("")
sys.exit(2)
print(item.getAsString())
def _test(): # pragma: no cover
import doctest
doctest.testmod()
if __name__=="__main__": # pragma: no cover
_test()