Merge remote-tracking branch 'origin/master' into edge

Author: jonasbardino

README

@@ -625,8 +625,6 @@ jobs and resources and efficient data access services:
                    --public_fqdn=www.migrid.org \
                    --mig_cert_fqdn=dk-cert.migrid.org \
                    --ext_cert_fqdn= \
-                   --mig_oid_fqdn=dk-ext.migrid.org \
-                   --ext_oid_fqdn=dk-oid.migrid.org \
                    --sid_fqdn=dk-sid.migrid.org \
                    --io_fqdn=dk-io.migrid.org \
                    --user=mig --group=mig \
@@ -643,8 +641,7 @@ jobs and resources and efficient data access services:
                    --hgweb_scripts=/usr/share/doc/mercurial-common/examples \
                    --trac_admin_path=/usr/bin/trac-admin \
                    --trac_ini_path=/home/mig/mig/server/trac.ini \
-                   --public_http_port=80 --mig_cert_port=443 --mig_oid_port=443 \
-                   --ext_oid_port=443 --sid_port=443 \
+                   --public_http_port=80 --mig_cert_port=443 --sid_port=443 \
                    --enable_jobs=True --enable_resources=True \
                    --enable_ftps=True --enable_sftp_subsys=True \
                    --enable_webdavs=True --enable_transfers=True \
@@ -658,7 +655,7 @@ jobs and resources and efficient data access services:
                    --short_title=MiG
 
 or a similar setup with vhost-specific certificates from LetsEncrypt,
-additional web apps and OpenID on CentOS:
+additional web apps and OpenID 2.0 + Connect on CentOS:
 ./generateconfs.py --source=. --destination=generated-confs \
                    --destination_suffix="_svn$(svnversion -n ~/)" \
                    --base_fqdn=migrid.org \
@@ -667,7 +664,6 @@ additional web apps and OpenID on CentOS:
                    --mig_cert_fqdn=dk-cert.migrid.org \
                    --ext_cert_fqdn= \
                    --mig_oid_fqdn=dk-ext.migrid.org \
-                   --ext_oid_fqdn=dk-oid.migrid.org \
                    --ext_oidc_fqdn=dk-oidc.migrid.org \
                    --sid_fqdn=dk-sid.migrid.org \
                    --io_fqdn=dk-io.migrid.org \
@@ -688,9 +684,8 @@ additional web apps and OpenID on CentOS:
                    --trac_ini_path=/home/mig/mig/server/trac.ini \
                    --public_http_port=80 --public_https_port=443 \
                    --ext_cert_port=443 --mig_oid_port=443 \
-                   --ext_oid_port=443 --ext_oidc_port=443 --sid_port=443 \
+                   --ext_oidc_port=443 --sid_port=443 \
                    --mig_oid_provider=https://dk-ext.migrid.org/openid/ \
-                   --ext_oid_provider=https://openid.ku.dk/ \
                    --ext_oidc_provider_meta_url=https://id.ku.dk/nidp/oauth/nam/.well-known/openid-configuration \
                    --ext_oidc_scope=AS_SIF-ERDA \
                    --ext_oidc_client_name=erda_migrid-dk \
@@ -716,8 +711,8 @@ additional web apps and OpenID on CentOS:
                    --daemon_pubkey_from_dns=True \
                    --daemon_pubkey_md5='FILE::/etc/httpd/MiG-certificates/combined.pub.md5' \
                    --daemon_pubkey_sha256='FILE::/etc/httpd/MiG-certificates/combined.pub.sha256' \
-                   --signup_methods="extoid migoid migcert extoidc" \
-                   --login_methods="extoid migoid migcert extoidc" \
+                   --signup_methods="extoidc migoid migcert" \
+                   --login_methods="extoidc migoid migcert" \
                    --distro=centos --skin=migrid-basic \
                    --default_menu="home files submitjob jobs vgrids account settings setup logout" \
                    --user_menu="sharelinks people cloud crontab transfers runtimeenvs resources peers downloads docs dashboard migadmin" \
@@ -754,7 +749,8 @@ additional web apps and OpenID on CentOS:
 
 and a storage-only setup with CentOS 7.x, apache 2.4, WSGI (default web), 
 optimized SFTP, WebDAVS FTPS, Data Transfers, external Seafile integration,
-local OpenID login and added Jupyter+cloud integration for data analysis:
+local OpenID login, external OpenID Connect login and added Jupyter+cloud
+integration for data analysis:
 ./generateconfs.py --source=. --destination=generated-confs \
                    --destination_suffix="_svn$(svnversion -n ~/)" \
                    --base_fqdn=erda.dk \
@@ -764,7 +760,6 @@ local OpenID login and added Jupyter+cloud integration for data analysis:
                    --mig_cert_fqdn= \
                    --ext_cert_fqdn=cert.erda.dk \
                    --mig_oid_fqdn=ext.erda.dk \
-                   --ext_oid_fqdn=oid.erda.dk \
                    --ext_oidc_fqdn=oidc.erda.dk \
                    --sid_fqdn=sid.erda.dk \
                    --io_fqdn=io.erda.dk \
@@ -788,9 +783,8 @@ local OpenID login and added Jupyter+cloud integration for data analysis:
                    --trac_admin_path='' --trac_ini_path='' \
                    --public_http_port=80 --public_https_port=443 \
                    --ext_cert_port=443 --mig_oid_port=443 \
-                   --ext_oid_port=443 --ext_oidc_port=443 --sid_port=443 \
+                   --ext_oidc_port=443 --sid_port=443 \
                    --mig_oid_provider=https://ext.erda.dk/openid/ \
-                   --ext_oid_provider=https://openid.ku.dk/ \
                    --ext_oidc_provider_meta_url=https://id.ku.dk/nidp/oauth/nam/.well-known/openid-configuration \
                    --ext_oidc_scope=AS_SIF-ERDA \
                    --ext_oidc_client_name=erda \
@@ -819,8 +813,8 @@ local OpenID login and added Jupyter+cloud integration for data analysis:
                    --daemon_pubkey_from_dns=True \
                    --daemon_pubkey_md5='FILE::/etc/httpd/MiG-certificates/combined.pub.md5' \
                    --daemon_pubkey_sha256='FILE::/etc/httpd/MiG-certificates/combined.pub.sha256' \
-                   --signup_methods="extoid migoid extcert extoidc" \
-                   --login_methods="extoid migoid extcert extoidc" \
+                   --signup_methods="extoidc migoid extcert" \
+                   --login_methods="extoidc migoid extcert" \
                    --distro=centos --skin=erda-ucph-science \
                    --vgrid_label=Workgroup --apache_worker_procs=2048 \
                    --davs_port=8020 --openid_port=8001 \
@@ -854,90 +848,6 @@ local OpenID login and added Jupyter+cloud integration for data analysis:
                    --crypto_salt="FILE::/home/mig/state/secrets/crypto_salt.hex" \
                    --secscan_addr="130.226.158.3 130.225.213.72 192.38.10.137"
 
-and a similar setup with CentOS 7.x, apache 2.4, WSGI (default web), 
-optimized SFTP, WebDAVS, FTPS, job execution, Jupyter integration, previews and
-local OpenID login and support for legacy sftp clients:
-./generateconfs.py --source=. --destination=generated-confs \
-                   --destination_suffix="_svn$(svnversion -n ~/)" \
-                   --base_fqdn=idmc.dk \
-                   --public_fqdn=www.idmc.dk \
-                   --mig_cert_fqdn= \
-                   --ext_cert_fqdn=cert.idmc.dk \
-                   --mig_oid_fqdn=ext.idmc.dk \
-                   --ext_oid_fqdn=oid.idmc.dk \
-                   --sid_fqdn=sid.idmc.dk \
-                   --io_fqdn=io.idmc.dk \
-                   --user=mig --group=mig \
-                   --apache_version=2.4 \
-                   --apache_etc=/etc/httpd \
-                   --apache_run=/var/run/httpd \
-                   --apache_lock=/var/lock/subsys/httpd \
-                   --apache_log=/var/log/httpd \
-                   --openssh_version=7.3 \
-                   --mig_code=/home/mig/mig \
-                   --mig_state=/home/mig/state \
-                   --mig_certs=/etc/httpd/MiG-certificates \
-                   --hg_path=/usr/bin/hg \
-                   --hgweb_scripts=/usr/share/doc/mercurial-2.6.2 \
-                   --trac_admin_path='' --trac_ini_path='' \
-                   --public_http_port=80 --public_https_port=443 \
-                   --ext_cert_port=443 --mig_oid_port=443 \
-                   --ext_oid_port=443 --sid_port=443 \
-                   --mig_oid_provider=https://ext.idmc.dk/openid/ \
-                   --ext_oid_provider=https://openid.ku.dk/ \
-                   --enable_openid=True --enable_sftp_subsys=True \
-                   --enable_davs=True --enable_ftps=True \
-                   --enable_transfers=True --enable_gravatars=True \
-                   --enable_jobs=True --enable_resources=True \
-                   --enable_events=True --enable_cracklib=True \
-                   --enable_notify=True --enable_preview=True \
-                   --enable_workflows=True --enable_freeze=False \
-                   --enable_vhost_certs=True --enable_verify_certs=True \
-                   --enable_jupyter=True --enable_migadmin=True \
-                   --jupyter_services='DAG.https://dag002.science DAG.https://dag003.science DAG.https://dag004.science DAG.https://dag005.science DAG.https://dag006.science DAG.https://dag007.science DAG.https://dag008.science DAG.https://dag009.science DAG.https://dag010.science DAG.https://dag203.science DAG.https://dag204.science MODI.https://dag100.science' \
-                   --jupyter_services_desc="{'DAG': '/home/mig/state/wwwpublic/dag_desc.html', 'MODI': '/home/mig/state/wwwpublic/modi_desc.html'}" \
-                   --enable_peers=True --peers_mandatory=True \
-                   --peers_explicit_fields='full_name email' \
-                   --peers_contact_hint='employed at UCPH and authorized to invite external users' \
-                   --user_clause=User --group_clause=Group \
-                   --listen_clause='#Listen' \
-                   --serveralias_clause='#ServerAlias' --alias_field=email \
-                   --dhparams_path=~/certs/dhparams.pem \
-                   --daemon_keycert=~/certs/combined.pem \
-                   --daemon_pubkey=~/certs/combined.pub \
-                   --daemon_pubkey_from_dns=False \
-                   --daemon_show_address=io.idmc.dk \
-                   --signup_methods="extoid migoid extcert" \
-                   --login_methods="extoid migoid extcert" \
-                   --distro=centos --skin=idmc-basic \
-                   --vgrid_label=Workgroup --apache_worker_procs=512 \
-                   --wsgi_procs=25 --sftp_subsys_auth_procs=25 \
-                   --sftp_max_sessions=16 \
-                   --davs_port=8020 --openid_port=8001 \
-                   --default_menu="home files submitjob jobs vgrids jupyter account settings setup logout" \
-                   --user_menu="sharelinks people cloud crontab transfers runtimeenvs resources downloads peers docs migadmin" \
-                   --collaboration_links="default advanced" \
-                   --default_vgrid_links="files web" \
-                   --advanced_vgrid_links="files web scm workflows monitor" \
-                   --smtp_sender="Do Not Reply <no-reply@idmc.dk>" \
-                   --support_email="IDMC Support <support@idmc.dk>" \
-                   --admin_email="IDMC Info <info@idmc.dk>" --log_level=info \
-                   --title="Imaging Data Management Center" \
-                   --short_title="IDMC" \
-                   --external_doc=https://www.idmc.dk \
-                   --mig_oid_title="Non-KU/UCPH" --ext_oid_title="KU/UCPH" \
-                   --auto_add_oid_user=True --auto_add_cert_user=True \
-                   --auto_add_filter_fields=full_name --auto_add_filter_method=skip \
-                   --io_account_expire=True \
-                   --password_policy="MODERN:12" \
-                   --password_legacy_policy=MEDIUM \
-                   --peers_permit="role:.*(vip|tap)" \
-                   --vgrid_creators="role:.*(vip|tap)" \
-                   --status_system_match="IDMC ERDA ALL" \
-                   --digest_salt="FILE::/home/mig/state/secrets/digest_salt.hex" \
-                   --crypto_salt="FILE::/home/mig/state/secrets/crypto_salt.hex" \
-                   --secscan_addr="130.226.158.3 130.225.213.72 192.38.10.137"
-
 Finally a storage-only with CentOS 7.x, apache 2.4, WSGI (default web), 
 optimized SFTP, WebDAVS, strict access control and extensive logging to comply
 with the  General Data Protection Regulation (GDPR) imposed by EU:

mig/server/refreshusers.py

@@ -0,0 +1,200 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# --- BEGIN_HEADER ---
+#
+# refreshusers - a simple helper to refresh stale user files to current user ID
+# Copyright (C) 2003-2025  The MiG Project by the Science HPC Center at UCPH
+#
+# This file is part of MiG.
+#
+# MiG is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# MiG is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301,
+# USA.
+#
+# --- END_HEADER ---
+#
+
+"""Refresh one or more accounts so that files and dirs fit current user ID, in
+particular replace any stale .htaccess files no longer in sync regarding
+assigned IDs and therefore causing auth error upon fileman open, etc.
+"""
+
+from __future__ import print_function
+from __future__ import absolute_import
+
+import datetime
+import getopt
+import sys
+import time
+
+from mig.shared.defaults import gdp_distinguished_field
+from mig.shared.useradm import init_user_adm, search_users, default_search, \
+    assure_current_htaccess
+
+
+def usage(name='refreshusers.py'):
+    """Usage help"""
+
+    print("""Refresh MiG user user files and dirs based on user ID in MiG user
+database.
+
+Usage:
+%(name)s [OPTIONS]
+Where OPTIONS may be one or more of:
+   -A EXPIRE_AFTER     Limit to users expiring after EXPIRE_AFTER (epoch)
+   -B EXPIRE_BEFORE    Limit to users expiring before EXPIRE_BEFORE (epoch)
+   -c CONF_FILE        Use CONF_FILE as server configuration
+   -d DB_FILE          Use DB_FILE as user data base file
+   -f                  Force operations to continue past errors
+   -h                  Show this help
+   -I CERT_DN          Filter to user(s) with ID (distinguished name)
+   -s SHORT_ID         Filter to user(s) with given short ID field
+   -v                  Verbose output
+"""
+          % {'name': name})
+
+
+if '__main__' == __name__:
+    (args, app_dir, db_path) = init_user_adm()
+    conf_path = None
+    force = False
+    verbose = False
+    exit_code = 0
+    now = int(time.time())
+    search_filter = default_search()
+    # Default to all users with expire range between now and in 30 days
+    search_filter['distinguished_name'] = '*'
+    search_filter['short_id'] = '*'
+    search_filter['expire_after'] = now
+    search_filter['expire_before'] = int(time.time() + 365 * 24 * 3600)
+    # Default to only external openid accounts
+    services = ['extoid']
+    opt_args = 'A:B:c:d:fhI:s:v'
+    try:
+        (opts, args) = getopt.getopt(args, opt_args)
+    except getopt.GetoptError as err:
+        print('Error: ', err.msg)
+        usage()
+        sys.exit(1)
+
+    for (opt, val) in opts:
+        if opt == '-A':
+            after = now
+            if val.startswith('+'):
+                after += int(val[1:])
+            elif val.startswith('-'):
+                after -= int(val[1:])
+            else:
+                after = int(val)
+            search_filter['expire_after'] = after
+        elif opt == '-B':
+            before = now
+            if val.startswith('+'):
+                before += int(val[1:])
+            elif val.startswith('-'):
+                before -= int(val[1:])
+            else:
+                before = int(val)
+            search_filter['expire_before'] = before
+        elif opt == '-c':
+            conf_path = val
+        elif opt == '-d':
+            db_path = val
+        elif opt == '-f':
+            force = True
+        elif opt == '-h':
+            usage()
+            sys.exit(0)
+        elif opt == '-I':
+            search_filter['distinguished_name'] = val
+        elif opt == '-s':
+            search_filter['short_id'] = val
+        elif opt == '-v':
+            verbose = True
+        else:
+            print('Error: %s not supported!' % opt)
+            sys.exit(1)
+
+    if args:
+        print('Error: Non-option arguments are not supported - missing quotes?')
+        usage()
+        sys.exit(1)
+
+    (configuration, hits) = search_users(search_filter, conf_path, db_path,
+                                         verbose)
+    logger = configuration.logger
+    gdp_prefix = "%s=" % gdp_distinguished_field
+    # NOTE: we already filtered expired accounts here
+    search_dn = search_filter['distinguished_name']
+    before = datetime.datetime.fromtimestamp(search_filter['expire_before'])
+    after = datetime.datetime.fromtimestamp(search_filter['expire_after'])
+    if verbose:
+        if hits:
+            print("Check %d account(s) expiring between %s and %s for ID %r" %
+                  (len(hits), after, before, search_dn))
+        else:
+            print("No accounts expire between %s and %s for ID %r" %
+                  (after, before, search_dn))
+
+    for (user_id, user_dict) in hits:
+        affected = []
+        if verbose:
+            print('Check refresh needed for %r' % user_id)
+
+        # NOTE: gdp accounts don't actually use .htaccess but cat.py serving
+        if configuration.site_enable_gdp and \
+                user_id.split('/')[-1].startswith(gdp_prefix):
+            if verbose:
+                print("Handling GDP project account %r despite no effect" %
+                      user_id)
+
+        # Don't warn about already disabled or suspended accounts
+        account_state = user_dict.get('status', 'active')
+        if not account_state in ('active', 'temporal'):
+            if verbose:
+                print('Skip handling of already %s user %r' % (account_state,
+                                                               user_id))
+            continue
+
+        known_auth = user_dict.get('auth', [])
+        if not known_auth:
+            if user_dict.get('main_id', ''):
+                known_auth.append("extoidc")
+            elif user_dict.get('openid_names', []):
+                if user_dict.get('password_hash', ''):
+                    known_auth.append("migoid")
+                else:
+                    known_auth.append("extoid")
+            elif user_dict.get('password', ''):
+                known_auth.append("migcert")
+            else:
+                if verbose:
+                    print('Skip handling of user %r without auth info' %
+                          user_id)
+                continue
+
+        if not ('extoid' in known_auth or 'extoidc' in known_auth):
+            if verbose:
+                print('Skip handling of user %r without extoid(c) auth' %
+                      user_id)
+            continue
+
+        if verbose:
+            print('Assure current htaccess for %r account' % user_id)
+        if not assure_current_htaccess(configuration, user_id, user_dict,
+                                       force, verbose):
+            exit_code += 1
+
+    sys.exit(exit_code)