"""
Initial setup for the file system hierarchy.
This module is supposed to create as much of the DaCHS file system environment
as possible. Take care to give sensible error messages -- much can go wrong
here, and it's nice if the user has a way to figure out what's wrong.
"""
#c Copyright 2008-2023, the GAVO project <gavo@ari.uni-heidelberg.de>
#c
#c This program is free software, covered by the GNU GPL. See the
#c COPYING file in the source distribution.
import base64
import datetime
import os
import sys
import textwrap
import warnings
import psycopg2
# keep sqlsupport from initialisation before we've set up the profiles
os.environ["GAVO_INIT_RUNNING"] = "1"
from gavo import base
from gavo import utils
[docs]def bailOut(msg, hint=None):
sys.stderr.write("*** Error: %s\n\n"%msg)
if hint is not None:
sys.stderr.write(textwrap.fill(hint)+"\n")
sys.exit(1)
[docs]def unindentString(s):
return "\n".join(s.strip() for s in s.split("\n"))+"\n"
[docs]def makeRoot():
rootDir = base.getConfig("rootDir")
if os.path.isdir(rootDir):
return
try:
os.makedirs(rootDir)
except os.error:
bailOut("Cannot create root directory %s."%rootDir,
"This usually means that the current user has insufficient privileges"
" to write to the parent directory. To fix this, either have rootDir"
" somewhere you can write to (edit /etc/gavorc) or create the directory"
" as root and grant it to your user id.")
[docs]def makeDirVerbose(path, setGroupTo, makeWritable):
if not os.path.isdir(path):
try:
os.makedirs(path)
except os.error as err:
bailOut("Could not create directory %s (%s)"%(
path, err)) # add hints
except Exception as msg:
bailOut("Could not create directory %s (%s)"%(
path, msg))
if setGroupTo is not None:
stats = os.stat(path)
if stats.st_mode&0o060!=0o60 or stats.st_gid!=setGroupTo:
try:
os.chown(path, -1, setGroupTo)
if makeWritable:
os.chmod(path, stats.st_mode | 0o060)
except Exception as msg:
bailOut("Cannot set %s to group ownership %s, group writable"%(
path, setGroupTo),
hint="Certain directories must be writable by multiple user ids."
" They must therefore belong to the group %s and be group"
" writeable. The attempt to make sure that's so just failed"
" with the error message %s."
" Either grant the directory in question to yourself, or"
" fix permissions manually. If you own the directory and"
" sill see permission errors, try 'newgrp %s'"%(
base.getConfig("group"), msg, base.getConfig("group")))
_GAVO_WRITABLE_DIRS = set([
"stateDir",
"cacheDir",
"logDir",
"tempDir",
"uwsWD",])
[docs]def makeDirForConfig(configKey, gavoGrpId):
path = base.getConfig(configKey)
makeDirVerbose(path, gavoGrpId, configKey in _GAVO_WRITABLE_DIRS)
[docs]def makeMatplotlibCfg():
destPath = os.path.join(base.getConfig("configDir"), "matplotlibrc")
if os.path.exists(destPath):
return
with open(destPath, "w") as f:
f.write("backend: Agg\n")
[docs]def prepareWeb(groupId):
makeDirVerbose(os.path.join(base.getConfig("webDir"), "nv_static"),
groupId, False)
makeDirVerbose(os.path.join(base.getConfig("webDir"), "templates"),
groupId, False)
def _genPW():
"""returns a random string that may be suitable as a database password.
The entropy of the generated passwords should be close to 160 bits, so
the passwords themselves would probably not be a major issue. Of course,
for DaCHS they are stored in the file system in clear text...
"""
return utils.debytify(base64.b64encode(os.urandom(20)))
[docs]def makeProfiles(dsn, userPrefix=""):
"""writes profiles with made-up passwords to DaCHS' config dir.
This will mess everything up when the users already exist. We
should probably provide an option to drop standard users.
userPrefix is mainly for the test infrastructure.
"""
profilePath = base.getConfig("configDir")
dsnContent = ["database = %s"%(dsn.parsed["dbname"])]
if "host" in dsn.parsed:
dsnContent.append("host = %s"%dsn.parsed["host"])
else:
dsnContent.append("host = localhost")
if "port" in dsn.parsed:
dsnContent.append("port = %s"%dsn.parsed["port"])
else:
dsnContent.append("port = 5432")
for fName, content in [
("dsn", "\n".join(dsnContent)+"\n"),
("feed", "include dsn\nuser = %sgavoadmin\npassword = %s\n"%(
userPrefix, _genPW())),
("trustedquery", "include dsn\nuser = %sgavo\npassword = %s\n"%(
userPrefix, _genPW())),
("untrustedquery", "include dsn\nuser = %suntrusted\npassword = %s\n"%(
userPrefix, _genPW())),]:
destPath = os.path.join(profilePath, fName)
if not os.path.exists(destPath):
with open(destPath, "w") as f:
f.write(content)
[docs]def createFSHierarchy(dsn, userPrefix=""):
"""creates the directories required by DaCHS.
userPrefix is for use of the test infrastructure.
"""
makeRoot()
grpId = base.getGroupId()
for configKey in ["configDir", "inputsDir", "cacheDir", "logDir",
"tempDir", "webDir", "stateDir"]:
makeDirForConfig(configKey, grpId)
makeDirVerbose(os.path.join(base.getConfig("inputsDir"), "__system"),
grpId, False)
makeDefaultMeta()
makeMatplotlibCfg()
makeProfiles(dsn, userPrefix)
prepareWeb(grpId)
###################### DB interface
# This doesn't use much of sqlsupport since the roles are just being
# created and some of the operations may not be available for non-supervisors.
[docs]class DSN(object):
"""a psycopg-style DSN, both parsed and unparsed.
"""
def __init__(self, dsn):
self.full = dsn
self._parse()
self._validate()
_knownKeys = set(["dbname", "user", "password", "host", "port", "sslmode"])
def _validate(self):
for key in self.parsed:
if key not in self._knownKeys:
sys.stderr.write("Unknown DSN key %s will get lost in profiles."%(
key))
def _parse(self):
if "=" in self.full:
self.parsed = utils.parseKVLine(self.full)
else:
self.parsed = {"dbname": self.full}
self.full = utils.makeKVLine(self.parsed)
def _execDB(conn, query, args={}):
"""returns the result of running query with args through conn.
No transaction management is being done here.
"""
cursor = conn.cursor()
cursor.execute(query, args)
return list(cursor)
def _roleExists(conn, roleName):
return _execDB(conn,
"SELECT rolname FROM pg_roles WHERE rolname=%(rolname)s",
{"rolname": roleName})
def _createRoleFromProfile(conn, profile, privileges):
cursor = conn.cursor()
try:
verb = "CREATE"
if _roleExists(conn, profile.user):
verb = "ALTER"
cursor.execute(
"%s ROLE %s PASSWORD %%(password)s %s LOGIN"%(
verb, profile.user, privileges), {
"password": profile.password,})
conn.commit()
except:
warnings.warn("Could not create role %s (see db server log)"%
profile.user)
conn.rollback()
def _createRoles(dsn):
"""creates the roles for the DaCHS profiles admin, trustedquery
and untrustedquery.
"""
from gavo.base import config
conn = psycopg2.connect(dsn.full)
for profileName, privileges in [
("admin", "CREATEROLE"),
("trustedquery", ""),
("untrustedquery", "")]:
_createRoleFromProfile(conn,
config.getDBProfile(profileName),
privileges)
adminProfile = config.getDBProfile("admin")
cursor = conn.cursor()
cursor.execute("GRANT ALL ON DATABASE %s TO %s"%(dsn.parsed["dbname"],
adminProfile.user))
conn.commit()
def _getServerScriptPath(conn):
"""returns the path where a local postgres server would store its
contrib scripts.
This is probably Debian specific. It's used by the the extension
script upload.
"""
from gavo.base import sqlsupport
version = ".".join(str(n) for n in sqlsupport.getPgVersion())
name = "/usr/share/postgresql/%s/contrib"%version
if os.path.isdir(name):
return name
name = "/usr/share/postgresql/contrib"
# Try others here? Which?
return name
def _readDBScript(conn, scriptPath, sourceName, procName):
"""tries to execute the sql script in scriptPath within conn.
sourceName is some user-targeted indicator what package the script
comes from, procName the name of a procedure left by the script
so we don't run the script again when it's already run.
"""
if not os.path.exists(scriptPath):
warnings.warn("SQL script file for %s not found. There are many"
" reasons why that may be ok, but unless you know what you are"
" doing, you probably should install the corresponding postgres"
" extension."%scriptPath)
from gavo.rscdef import scripting
cursor = conn.cursor()
if _execDB(conn, "SELECT * FROM pg_proc WHERE proname=%(procName)s",
{"procName": procName}):
# script has already run
return
try:
for statement in scripting.getSQLScriptGrammar().parseString(
open(scriptPath).read()):
cursor.execute(statement)
except:
conn.rollback()
warnings.warn("SQL script file %s failed. Try running manually"
" using psql. While it hasn't run, the %s extension is not"
" available."%(scriptPath, sourceName))
else:
conn.commit()
def _loadPgExtension(conn, extName):
"""tries to create the extension extName.
This is for new-style extensions (e.g., pgsphere starting from 1.1.1.7)
that don't have a load script any more.
It returns True if the extension was found (and has created it as a
side effect).
"""
res = _execDB(conn, "SELECT default_version, installed_version"
" FROM pg_available_extensions"
" WHERE name=%(name)s", {"name": extName})
if not res:
# The extension is not available at all; let's hope we can limp on.
return False
if res[0][1] is not None:
# there is an installed version. Leave it as is for now
# (is it worth annoying the user with nagging for updates if
# there's a new version? Perhaps, but will they read it? So, for now:
return True
cursor = conn.cursor()
cursor.execute("CREATE EXTENSION "+extName)
cursor.close()
return True
def _doLocalSetup(dsn):
# When adding stuff here, fix docs/install.rstx, "Owner-only db setup"
from gavo.base import config
feedProfile = config.getDBProfile("feed")
conn = psycopg2.connect(dsn.full)
for statement in [
# gavoadmin/feed can create public functions and similar; perhaps
# we need to think again if we should put them somewhere else.
# cf. https://www.postgresql.org/docs/15/ddl-schemas.html#DDL-SCHEMAS-PATTERNS
f"GRANT ALL ON SCHEMA public TO {feedProfile.user}",
"CREATE OR REPLACE LANGUAGE plpgsql"]:
cursor = conn.cursor()
try:
cursor.execute(statement)
except psycopg2.DatabaseError as msg:
warnings.warn("SQL statement '%s' failed (%s); continuing."%(
statement, msg))
conn.rollback()
else:
conn.commit()
def _readDBScripts(dsn):
"""loads definitions of pgsphere, q3c and similar into the DB.
This only works for local installations, and the script location
is more or less hardcoded (Debian and SuSE work, at least).
"""
conn = psycopg2.connect(dsn.full)
scriptPath = _getServerScriptPath(conn)
for extScript, pkgName, procName, extName in [
("pg_sphere.sql", "pgSphere", "spoint_in", "pg_sphere"),
("q3c.sql", "q3c", "q3c_ang2ipix", "q3c")]:
# first try new-style extension, then fall back to running scripts
if not _loadPgExtension(conn, extName):
_readDBScript(conn,
os.path.join(scriptPath, extScript),
pkgName,
procName)
conn.commit()
def _importBasicResources():
from gavo import rsc
from gavo.rscdef import common
from gavo.user import importing
# see rscdef.common for info on the _BOOTSTRAPPING hack.
common._BOOTSTRAPPING = True
for rdId in ["//dc_tables", "//services", "//users",
"//uws", "//adql", "//tap", "//products",
"//datalink"]:
base.ui.notifyInfo("Importing %s"%rdId)
importing.process(rsc.getParseOptions(), [rdId])
common._BOOTSTRAPPING = False
[docs]def initDB(dsn):
"""creates users and tables expected by DaCHS in the database described
by the DSN dsn.
Connecting with dsn must give you superuser privileges.
"""
_createRoles(dsn)
_doLocalSetup(dsn)
_readDBScripts(dsn)
from gavo.base import sqlsupport
sqlsupport.initPsycopg()
_importBasicResources()
[docs]def parseCommandLine():
import argparse
parser = argparse.ArgumentParser(description="Create or update DaCHS'"
" file system and database environment.")
parser.add_argument("-d", "--dsn", help="DSN to use to connect to"
" the future DaCHS database. The DSN must let DaCHS connect"
" to the DB as an administrator. dbname, host, and port"
" get copied to the profile, if given. The DSN looks roughly like"
' "host=foo.bar user=admin password=secret dbname=gavo".'
' If you followed the'
" installation instructions, you don't need this option.",
action="store", type=str, dest="dsn", default="gavo")
parser.add_argument("--nodb", help="Inhibit initialization of the"
" database (you may want to use this when refreshing the file system"
" hierarchy)", action="store_false", dest="initDB")
return parser.parse_args()
[docs]def main():
"""initializes the DaCHS environment (where that's not already done).
"""
opts = parseCommandLine()
dsn = DSN(opts.dsn)
createFSHierarchy(dsn)
if opts.initDB:
initDB(dsn)