commit cbf68294cab357ce5d0753e96ae14b265a2c039b Author: Marc Koch Date: Tue Feb 4 18:03:51 2025 +0100 Initial commit diff --git a/.env b/.env new file mode 100644 index 0000000..e69de29 diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..13566b8 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,8 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml diff --git a/.idea/adGroupSync.iml b/.idea/adGroupSync.iml new file mode 100644 index 0000000..7c6dd0c --- /dev/null +++ b/.idea/adGroupSync.iml @@ -0,0 +1,10 @@ + + + + + + + + + + \ No newline at end of file diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml new file mode 100644 index 0000000..37130e7 --- /dev/null +++ b/.idea/inspectionProfiles/Project_Default.xml @@ -0,0 +1,77 @@ + + + + \ No newline at end of file diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml new file mode 100644 index 0000000..105ce2d --- /dev/null +++ b/.idea/inspectionProfiles/profiles_settings.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..2e8a6a7 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,7 @@ + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..c2b5586 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/CiviCRM/__init__.py b/CiviCRM/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/CiviCRM/models.py b/CiviCRM/models.py new file mode 100644 index 0000000..e69de29 diff --git a/README.md b/README.md new file mode 100644 index 0000000..7f855bc --- /dev/null +++ b/README.md @@ -0,0 +1 @@ +# adGroupSync diff --git a/adGroupSync/__init__.py b/adGroupSync/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/adGroupSync/logger.py b/adGroupSync/logger.py new file mode 100644 index 0000000..e4f9ccf --- /dev/null +++ b/adGroupSync/logger.py @@ -0,0 +1,118 @@ +import atexit +import datetime as dt +import json +import logging +import logging.config +import os +from pathlib import Path +from typing import override + +PROJECT_ROOT = Path(__file__).parent.parent +if LOG_DIR := os.getenv("LOG_DIR"): + LOG_DIR = Path(LOG_DIR) + LOG_DIR.mkdir(parents=True, exist_ok=True) + +LOG_RECORD_BUILTIN_ATTRS = { + "args", + "asctime", + "created", + "exc_info", + "exc_text", + "filename", + "funcName", + "levelname", + "levelno", + "lineno", + "module", + "msecs", + "message", + "msg", + "name", + "pathname", + "process", + "processName", + "relativeCreated", + "stack_info", + "thread", + "threadName", + "taskName", +} + + +def setup_logging(loglevel: int | str | None = None): + """ + Setup logging configuration + :return: + """ + config_file = PROJECT_ROOT / "conf" / "logging_config.json" + with open(config_file, "r") as file: + config = json.load(file) + + # Override log level if provided + if loglevel: + level_str = logging.getLevelName(loglevel) + if not isinstance(level_str, str): + level_str = logging.getLevelName(level_str) + for handler in config["handlers"].values(): + handler["level"] = level_str + + # Set log file path to user log directory + if LOG_DIR: + config["handlers"]["file"]["filename"] = LOG_DIR / "adGroupSync.log.jsonl" + + # Create path and file if it does not exist + Path(config["handlers"]["file"]["filename"]).parent.mkdir( + parents=True, exist_ok=True) + Path(config["handlers"]["file"]["filename"]).touch() + + logging.config.dictConfig(config) + queue_handler = logging.getHandlerByName("queue_handler") + if queue_handler is not None: + queue_handler.listener.start() + atexit.register(queue_handler.listener.stop) + + +class JSONFormatter(logging.Formatter): + """ + A custom JSON formatter for logging + """ + def __init__( + self, + *, + fmt_keys: dict[str, str] | None = None, + ): + super().__init__() + self.fmt_keys = fmt_keys if fmt_keys is not None else {} + + @override + def format(self, record: logging.LogRecord) -> str: + message = self._prepare_log_dict(record) + return json.dumps(message, default=str) + + def _prepare_log_dict(self, record: logging.LogRecord) -> dict: + always_fields = { + "message": record.getMessage(), + "timestamp": dt.datetime.fromtimestamp( + record.created, tz=dt.timezone.utc + ).isoformat() + } + if record.exc_info is not None: + always_fields["exc_info"] = self.formatException(record.exc_info) + + if record.stack_info is not None: + always_fields["stack_info"] = self.formatStack(record.stack_info) + + message = { + key: msg_val + if (msg_val := always_fields.pop(val, None)) is not None + else getattr(record, val) + for key, val in self.fmt_keys.items() + } + message.update(always_fields) + + # Include all other attributes + for key, val, in record.__dict__.items(): + if key not in LOG_RECORD_BUILTIN_ATTRS: + message[key] = val + + return message diff --git a/adGroupSync/main.py b/adGroupSync/main.py new file mode 100644 index 0000000..443789a --- /dev/null +++ b/adGroupSync/main.py @@ -0,0 +1,109 @@ +import logging +import os +from datetime import datetime +from pathlib import Path + +import pytz +from ldap3 import SIMPLE +from ms_active_directory import ADDomain, ADGroup, ADUser, ADObject + +from .models import RecentRun + +DOMAIN = os.getenv('AD_DOMAIN') +USER_NAME = os.getenv('AD_USER') +PASSWORD = os.getenv('AD_PASSWORD') +LDAP_SERVER = [s.strip() for s in os.getenv('AD_LDAP_SERVER').split(',')] +TIMEZONE = pytz.timezone(os.getenv('TIMEZONE', 'UTC')) +MAILINGLISTS_PARENT_GROUP = os.getenv('AD_MAILINGLISTS_PARENT_GROUP') + +# Setup logging +logger = logging.getLogger('adGroupSync') +logger.setLevel(os.getenv('LOG_LEVEL', logging.INFO)) + + +def has_changed(object_: ADObject, recent_run): + """ + Check if the object has changed since the last run + :param object_: Object to check + :param recent_run: RecentRun object + :return: True if the object has changed, False otherwise + """ + modify_timestamp = object_.get('modifyTimestamp') + if modify_timestamp is None: + message = (f"Object '{object_.get('sn')}' of type '{type(object_)}' " + "does not have 'modifyTimestamp' attribute") + logger.error(message) + raise ValueError(message) + return modify_timestamp > recent_run + + +def main(): + + logger.info("Running group sync") + + # Get the last run timestamp + recent_run = RecentRun(Path().home() / '.recent_run') + logger.info(f"Last run: {recent_run.datetime.strftime('%Y-%m-%d %H:%M:%S %Z')}") + + # Setup session + domain = ADDomain(DOMAIN, ldap_servers_or_uris=LDAP_SERVER) + logger.debug(f"Domain: {domain}", extra={'domain': domain}) + try: + session = domain.create_session_as_user( + USER_NAME, PASSWORD, authentication_mechanism=SIMPLE) + except Exception as e: + logger.error(f"Error creating session: {e}") + exit(1) + logger.debug(f"Session: {session}") + logger.info(f"Session opened: {session.is_open()}") + + # Get parent group + parent_group = session.find_group_by_name( + MAILINGLISTS_PARENT_GROUP, ['modifyTimestamp']) + parent_group_has_changed = has_changed(parent_group, recent_run) + logger.debug(f"Parent group: {parent_group}", extra={ + 'group': parent_group, + 'modifyTimestamp': parent_group.get('modifyTimestamp'), + 'has_changed': parent_group_has_changed}) + + # Get child groups + group_attrs = [ + 'modifyTimestamp', + 'objectSid', + 'mail', + 'sAMAccountName', + ] + group_members = session.find_members_of_group_recursive( + parent_group, group_attrs)[1] + groups = (m for m in group_members.values() if isinstance(m, ADGroup)) + logger.debug(f"Found {len(groups)} child groups", + extra={'child_group_count': len(groups), + 'child_groups': groups}) + users = (m for m in groups if not isinstance(m, ADUser)) + + # If the parent group has changed, we need to update the child groups + # in CiviCRM + if parent_group_has_changed: + civicrm.update_mailing_lists(groups) + + # Check if the mailing list groups have changed and update them in CiviCRM + for ml_group in groups: + if has_changed(ml_group, recent_run): + civicrm.update_mailing_list_members(ml_group) + + # Check if the users have changed and update them in CiviCRM + for user in users: + if has_changed(user, recent_run): + civicrm.update_user(user) + + # Set the last run timestamp + recent_run.datetime = datetime.now(tz=TIMEZONE) + +if __name__ == '__main__': + try: + main() + except Exception as e: + logger.error(f"Error: {e}") + exit(1) + + diff --git a/adGroupSync/models.py b/adGroupSync/models.py new file mode 100644 index 0000000..4f1a15c --- /dev/null +++ b/adGroupSync/models.py @@ -0,0 +1,92 @@ +from datetime import datetime, tzinfo +from pathlib import Path +import pytz + + +class RecentRun: + """ + Class to manage the last run of the script + """ + + def __init__(self, file_path: Path, timezone: tzinfo = pytz.utc): + """ + Initialize the class + :param file_path: File path to store the last run timestamp + :param timezone: Timezone to use for the timestamp + """ + self._datetime = None + self._timezone = timezone + self._file_path = file_path + + if not self._file_path.exists(): + raise FileNotFoundError(f"File {self._file_path} does not exist") + + self._read_recent_run_from_file() + + def _write_recent_run_to_file(self): + """ + Write the recent run timestamp to the file + :return: + """ + with open(self._file_path, 'w') as f: + f.write(str(self.timestamp)) + + def _read_recent_run_from_file(self): + """ + Read the recent run timestamp from the file + :return: + """ + with open(self._file_path, 'r') as f: + line = f.readline() + if not line or line.strip() == '': + return None + try: + self._datetime = datetime.fromtimestamp(float(line), tz=self._timezone) + except ValueError: + raise ValueError(f"Invalid timestamp '{line}' in {self._file_path}") + + @property + def datetime(self): + return self._datetime + + @property + def timestamp(self): + return self._datetime.timestamp() + + @datetime.setter + def datetime(self, value: datetime): + if value.tzinfo is None: + value = value.astimezone(self._timezone) + self._datetime = value + self._write_recent_run_to_file() + + @staticmethod + def _to_datetime(value: datetime|str|float): + """ + Convert the value to a datetime object + :param value: + :return: + """ + try: + if isinstance(value, str): + value = float(value) + if isinstance(value, float): + value = datetime.fromtimestamp(value) + except ValueError: + raise ValueError(f"Invalid timestamp '{value}'") + return value + + def __gt__(self, other: datetime|str|float): + return self.datetime > self._to_datetime(other) + + def __lt__(self, other: datetime|str|float): + return self.datetime < self._to_datetime(other) + + def __eq__(self, other: datetime|str|float): + return self.datetime == self._to_datetime(other) + + def __ge__(self, other: datetime|str|float): + return self.datetime >= self._to_datetime(other) + + def __le__(self, other: datetime|str|float): + return self.datetime <= self._to_datetime(other) diff --git a/conf/logging_config.json b/conf/logging_config.json new file mode 100644 index 0000000..2baa047 --- /dev/null +++ b/conf/logging_config.json @@ -0,0 +1,55 @@ +{ + "version": 1, + "disable_existing_loggers": false, + "formatters": { + "simple": { + "format": "%(levelname)s - %(message)s", + "datefmt": "%Y-%m-%dT%H:%M:%Sz" + }, + "json": { + "()": "forum_booking.logger.JSONFormatter", + "fmt_keys": { + "level": "levelname", + "message": "message", + "timestamp": "timestamp", + "logger": "name", + "module": "module", + "function": "funcName", + "line": "lineno", + "thread_name": "threadName" + } + } + }, + "handlers": { + "stdout": { + "class": "logging.StreamHandler", + "formatter": "simple", + "stream": "ext://sys.stdout", + "level": "DEBUG" + }, + "file": { + "class": "logging.handlers.RotatingFileHandler", + "formatter": "json", + "filename": "forum_booking.log.jsonl", + "level": "INFO", + "maxBytes": 10000000, + "backupCount": 3 + }, + "queue_handler": { + "class": "logging.handlers.QueueHandler", + "handlers": [ + "stdout", + "file" + ], + "respect_handler_level": true + } + }, + "loggers": { + "root": { + "handlers": [ + "queue_handler" + ], + "level": "DEBUG" + } + } +} \ No newline at end of file