Initial commit

.idea/.gitignore

@@ -0,0 +1,8 @@
+# Default ignored files
+/shelf/
+/workspace.xml
+# Editor-based HTTP Client requests
+/httpRequests/
+# Datasource local storage ignored files
+/dataSources/
+/dataSources.local.xml

.idea/adGroupSync.iml

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$">
+      <excludeFolder url="file://$MODULE_DIR$/.venv" />
+    </content>
+    <orderEntry type="jdk" jdkName="Python 3.12 (adGroupSync)" jdkType="Python SDK" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

.idea/inspectionProfiles/Project_Default.xml

@@ -0,0 +1,77 @@
+<component name="InspectionProjectProfileManager">
+  <profile version="1.0">
+    <option name="myName" value="Project Default" />
+    <inspection_tool class="HttpUrlsUsage" enabled="true" level="WEAK WARNING" enabled_by_default="true">
+      <option name="ignoredUrls">
+        <list>
+          <option value="http://localhost" />
+          <option value="http://127.0.0.1" />
+          <option value="http://0.0.0.0" />
+          <option value="http://www.w3.org/" />
+          <option value="http://json-schema.org/draft" />
+          <option value="http://java.sun.com/" />
+          <option value="http://xmlns.jcp.org/" />
+          <option value="http://javafx.com/javafx/" />
+          <option value="http://javafx.com/fxml" />
+          <option value="http://maven.apache.org/xsd/" />
+          <option value="http://maven.apache.org/POM/" />
+          <option value="http://www.springframework.org/schema/" />
+          <option value="http://www.springframework.org/tags" />
+          <option value="http://www.springframework.org/security/tags" />
+          <option value="http://www.thymeleaf.org" />
+          <option value="http://www.jboss.org/j2ee/schema/" />
+          <option value="http://www.jboss.com/xml/ns/" />
+          <option value="http://www.ibm.com/webservices/xsd" />
+          <option value="http://activemq.apache.org/schema/" />
+          <option value="http://schema.cloudfoundry.org/spring/" />
+          <option value="http://schemas.xmlsoap.org/" />
+          <option value="http://cxf.apache.org/schemas/" />
+          <option value="http://primefaces.org/ui" />
+          <option value="http://tiles.apache.org/" />
+          <option value="http://" />
+        </list>
+      </option>
+    </inspection_tool>
+    <inspection_tool class="PyCompatibilityInspection" enabled="true" level="WARNING" enabled_by_default="true">
+      <option name="ourVersions">
+        <value>
+          <list size="3">
+            <item index="0" class="java.lang.String" itemvalue="3.12" />
+            <item index="1" class="java.lang.String" itemvalue="3.11" />
+            <item index="2" class="java.lang.String" itemvalue="3.10" />
+          </list>
+        </value>
+      </option>
+    </inspection_tool>
+    <inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="WARNING" enabled_by_default="true">
+      <option name="ignoredPackages">
+        <value>
+          <list size="13">
+            <item index="0" class="java.lang.String" itemvalue="pandas" />
+            <item index="1" class="java.lang.String" itemvalue="PyPDF2" />
+            <item index="2" class="java.lang.String" itemvalue="drf-spectacular" />
+            <item index="3" class="java.lang.String" itemvalue="django-crispy-forms" />
+            <item index="4" class="java.lang.String" itemvalue="Django" />
+            <item index="5" class="java.lang.String" itemvalue="qrcode" />
+            <item index="6" class="java.lang.String" itemvalue="psycopg2" />
+            <item index="7" class="java.lang.String" itemvalue="django-active-link" />
+            <item index="8" class="java.lang.String" itemvalue="shortuuid" />
+            <item index="9" class="java.lang.String" itemvalue="uwsgi" />
+            <item index="10" class="java.lang.String" itemvalue="djangorestframework" />
+            <item index="11" class="java.lang.String" itemvalue="Pillow" />
+            <item index="12" class="java.lang.String" itemvalue="pyproject_hooks" />
+          </list>
+        </value>
+      </option>
+    </inspection_tool>
+    <inspection_tool class="PyPep8NamingInspection" enabled="true" level="WEAK WARNING" enabled_by_default="true">
+      <option name="ignoredErrors">
+        <list>
+          <option value="N813" />
+        </list>
+      </option>
+    </inspection_tool>
+    <inspection_tool class="PyTypeHintsInspection" enabled="false" level="WARNING" enabled_by_default="false" />
+    <inspection_tool class="ReassignedToPlainText" enabled="false" level="WARNING" enabled_by_default="false" />
+  </profile>
+</component>

.idea/inspectionProfiles/profiles_settings.xml

@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>

.idea/misc.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="MarkdownSettingsMigration">
+    <option name="stateVersion" value="1" />
+  </component>
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.8 (tazPlease)" project-jdk-type="Python SDK" />
+</project>

.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/adGroupSync.iml" filepath="$PROJECT_DIR$/.idea/adGroupSync.iml" />
+    </modules>
+  </component>
+</project>

.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+  </component>
+</project>

README.md

@@ -0,0 +1 @@
+# adGroupSync

adGroupSync/logger.py

@@ -0,0 +1,118 @@
+import atexit
+import datetime as dt
+import json
+import logging
+import logging.config
+import os
+from pathlib import Path
+from typing import override
+
+PROJECT_ROOT = Path(__file__).parent.parent
+if LOG_DIR := os.getenv("LOG_DIR"):
+    LOG_DIR = Path(LOG_DIR)
+    LOG_DIR.mkdir(parents=True, exist_ok=True)
+
+LOG_RECORD_BUILTIN_ATTRS = {
+    "args",
+    "asctime",
+    "created",
+    "exc_info",
+    "exc_text",
+    "filename",
+    "funcName",
+    "levelname",
+    "levelno",
+    "lineno",
+    "module",
+    "msecs",
+    "message",
+    "msg",
+    "name",
+    "pathname",
+    "process",
+    "processName",
+    "relativeCreated",
+    "stack_info",
+    "thread",
+    "threadName",
+    "taskName",
+}
+
+
+def setup_logging(loglevel: int | str | None = None):
+    """
+    Setup logging configuration
+    :return:
+    """
+    config_file = PROJECT_ROOT / "conf" / "logging_config.json"
+    with open(config_file, "r") as file:
+        config = json.load(file)
+
+    # Override log level if provided
+    if loglevel:
+        level_str = logging.getLevelName(loglevel)
+        if not isinstance(level_str, str):
+            level_str = logging.getLevelName(level_str)
+        for handler in config["handlers"].values():
+            handler["level"] = level_str
+
+    # Set log file path to user log directory
+    if LOG_DIR:
+        config["handlers"]["file"]["filename"] = LOG_DIR / "adGroupSync.log.jsonl"
+
+    # Create path and file if it does not exist
+    Path(config["handlers"]["file"]["filename"]).parent.mkdir(
+        parents=True, exist_ok=True)
+    Path(config["handlers"]["file"]["filename"]).touch()
+
+    logging.config.dictConfig(config)
+    queue_handler = logging.getHandlerByName("queue_handler")
+    if queue_handler is not None:
+        queue_handler.listener.start()
+        atexit.register(queue_handler.listener.stop)
+
+
+class JSONFormatter(logging.Formatter):
+    """
+    A custom JSON formatter for logging
+    """
+    def __init__(
+            self,
+            *,
+            fmt_keys: dict[str, str] | None = None,
+    ):
+        super().__init__()
+        self.fmt_keys = fmt_keys if fmt_keys is not None else {}
+
+    @override
+    def format(self, record: logging.LogRecord) -> str:
+        message = self._prepare_log_dict(record)
+        return json.dumps(message, default=str)
+
+    def _prepare_log_dict(self, record: logging.LogRecord) -> dict:
+        always_fields = {
+            "message": record.getMessage(),
+            "timestamp": dt.datetime.fromtimestamp(
+                record.created, tz=dt.timezone.utc
+            ).isoformat()
+        }
+        if record.exc_info is not None:
+            always_fields["exc_info"] = self.formatException(record.exc_info)
+
+        if record.stack_info is not None:
+            always_fields["stack_info"] = self.formatStack(record.stack_info)
+
+        message = {
+            key: msg_val
+            if (msg_val := always_fields.pop(val, None)) is not None
+            else getattr(record, val)
+            for key, val in self.fmt_keys.items()
+        }
+        message.update(always_fields)
+
+        # Include all other attributes
+        for key, val, in record.__dict__.items():
+            if key not in LOG_RECORD_BUILTIN_ATTRS:
+                message[key] = val
+
+        return message

adGroupSync/main.py

@@ -0,0 +1,109 @@
+import logging
+import os
+from datetime import datetime
+from pathlib import Path
+
+import pytz
+from ldap3 import SIMPLE
+from ms_active_directory import ADDomain, ADGroup, ADUser, ADObject
+
+from .models import RecentRun
+
+DOMAIN = os.getenv('AD_DOMAIN')
+USER_NAME = os.getenv('AD_USER')
+PASSWORD = os.getenv('AD_PASSWORD')
+LDAP_SERVER = [s.strip() for s in os.getenv('AD_LDAP_SERVER').split(',')]
+TIMEZONE = pytz.timezone(os.getenv('TIMEZONE', 'UTC'))
+MAILINGLISTS_PARENT_GROUP = os.getenv('AD_MAILINGLISTS_PARENT_GROUP')
+
+# Setup logging
+logger = logging.getLogger('adGroupSync')
+logger.setLevel(os.getenv('LOG_LEVEL', logging.INFO))
+
+
+def has_changed(object_: ADObject, recent_run):
+    """
+    Check if the object has changed since the last run
+    :param object_: Object to check
+    :param recent_run: RecentRun object
+    :return: True if the object has changed, False otherwise
+    """
+    modify_timestamp = object_.get('modifyTimestamp')
+    if modify_timestamp is None:
+        message = (f"Object '{object_.get('sn')}' of type '{type(object_)}' "
+                   "does not have 'modifyTimestamp' attribute")
+        logger.error(message)
+        raise ValueError(message)
+    return modify_timestamp > recent_run
+
+
+def main():
+
+    logger.info("Running group sync")
+
+    # Get the last run timestamp
+    recent_run = RecentRun(Path().home() / '.recent_run')
+    logger.info(f"Last run: {recent_run.datetime.strftime('%Y-%m-%d %H:%M:%S %Z')}")
+
+    # Setup session
+    domain = ADDomain(DOMAIN, ldap_servers_or_uris=LDAP_SERVER)
+    logger.debug(f"Domain: {domain}", extra={'domain': domain})
+    try:
+        session = domain.create_session_as_user(
+            USER_NAME, PASSWORD, authentication_mechanism=SIMPLE)
+    except Exception as e:
+        logger.error(f"Error creating session: {e}")
+        exit(1)
+    logger.debug(f"Session: {session}")
+    logger.info(f"Session opened: {session.is_open()}")
+
+    # Get parent group
+    parent_group = session.find_group_by_name(
+        MAILINGLISTS_PARENT_GROUP, ['modifyTimestamp'])
+    parent_group_has_changed = has_changed(parent_group, recent_run)
+    logger.debug(f"Parent group: {parent_group}", extra={
+        'group': parent_group,
+        'modifyTimestamp': parent_group.get('modifyTimestamp'),
+        'has_changed': parent_group_has_changed})
+
+    # Get child groups
+    group_attrs = [
+        'modifyTimestamp',
+        'objectSid',
+        'mail',
+        'sAMAccountName',
+    ]
+    group_members = session.find_members_of_group_recursive(
+        parent_group, group_attrs)[1]
+    groups = (m for m in group_members.values() if isinstance(m, ADGroup))
+    logger.debug(f"Found {len(groups)} child groups",
+                 extra={'child_group_count': len(groups),
+                        'child_groups': groups})
+    users = (m for m in groups if not isinstance(m, ADUser))
+
+    # If the parent group has changed, we need to update the child groups
+    # in CiviCRM
+    if parent_group_has_changed:
+        civicrm.update_mailing_lists(groups)
+
+    # Check if the mailing list groups have changed and update them in CiviCRM
+    for ml_group in groups:
+        if has_changed(ml_group, recent_run):
+            civicrm.update_mailing_list_members(ml_group)
+
+    # Check if the users have changed and update them in CiviCRM
+    for user in users:
+        if has_changed(user, recent_run):
+            civicrm.update_user(user)
+
+    # Set the last run timestamp
+    recent_run.datetime = datetime.now(tz=TIMEZONE)
+
+if __name__ == '__main__':
+    try:
+        main()
+    except Exception as e:
+        logger.error(f"Error: {e}")
+        exit(1)
+
+

adGroupSync/models.py

@@ -0,0 +1,92 @@
+from datetime import datetime, tzinfo
+from pathlib import Path
+import pytz
+
+
+class RecentRun:
+    """
+    Class to manage the last run of the script
+    """
+
+    def __init__(self, file_path: Path, timezone: tzinfo = pytz.utc):
+        """
+        Initialize the class
+        :param file_path: File path to store the last run timestamp
+        :param timezone: Timezone to use for the timestamp
+        """
+        self._datetime = None
+        self._timezone = timezone
+        self._file_path = file_path
+
+        if not self._file_path.exists():
+            raise FileNotFoundError(f"File {self._file_path} does not exist")
+
+        self._read_recent_run_from_file()
+
+    def _write_recent_run_to_file(self):
+        """
+        Write the recent run timestamp to the file
+        :return:
+        """
+        with open(self._file_path, 'w') as f:
+            f.write(str(self.timestamp))
+
+    def _read_recent_run_from_file(self):
+        """
+        Read the recent run timestamp from the file
+        :return:
+        """
+        with open(self._file_path, 'r') as f:
+            line = f.readline()
+            if not line or line.strip() == '':
+                return None
+            try:
+                self._datetime = datetime.fromtimestamp(float(line), tz=self._timezone)
+            except ValueError:
+                raise ValueError(f"Invalid timestamp '{line}' in {self._file_path}")
+
+    @property
+    def datetime(self):
+        return self._datetime
+
+    @property
+    def timestamp(self):
+        return self._datetime.timestamp()
+
+    @datetime.setter
+    def datetime(self, value: datetime):
+        if value.tzinfo is None:
+            value = value.astimezone(self._timezone)
+        self._datetime = value
+        self._write_recent_run_to_file()
+
+    @staticmethod
+    def _to_datetime(value: datetime|str|float):
+        """
+        Convert the value to a datetime object
+        :param value:
+        :return:
+        """
+        try:
+            if isinstance(value, str):
+                value = float(value)
+            if isinstance(value, float):
+                value = datetime.fromtimestamp(value)
+        except ValueError:
+            raise ValueError(f"Invalid timestamp '{value}'")
+        return value
+
+    def __gt__(self, other: datetime|str|float):
+        return self.datetime > self._to_datetime(other)
+
+    def __lt__(self, other: datetime|str|float):
+        return self.datetime < self._to_datetime(other)
+
+    def __eq__(self, other: datetime|str|float):
+        return self.datetime == self._to_datetime(other)
+
+    def __ge__(self, other: datetime|str|float):
+        return self.datetime >= self._to_datetime(other)
+
+    def __le__(self, other: datetime|str|float):
+        return self.datetime <= self._to_datetime(other)

conf/logging_config.json

@@ -0,0 +1,55 @@
+{
+  "version": 1,
+  "disable_existing_loggers": false,
+  "formatters": {
+    "simple": {
+      "format": "%(levelname)s - %(message)s",
+      "datefmt": "%Y-%m-%dT%H:%M:%Sz"
+    },
+    "json": {
+      "()": "forum_booking.logger.JSONFormatter",
+      "fmt_keys": {
+        "level": "levelname",
+        "message": "message",
+        "timestamp": "timestamp",
+        "logger": "name",
+        "module": "module",
+        "function": "funcName",
+        "line": "lineno",
+        "thread_name": "threadName"
+      }
+    }
+  },
+  "handlers": {
+    "stdout": {
+      "class": "logging.StreamHandler",
+      "formatter": "simple",
+      "stream": "ext://sys.stdout",
+      "level": "DEBUG"
+    },
+    "file": {
+      "class": "logging.handlers.RotatingFileHandler",
+      "formatter": "json",
+      "filename": "forum_booking.log.jsonl",
+      "level": "INFO",
+      "maxBytes": 10000000,
+      "backupCount": 3
+    },
+    "queue_handler": {
+      "class": "logging.handlers.QueueHandler",
+      "handlers": [
+        "stdout",
+        "file"
+      ],
+      "respect_handler_level": true
+    }
+  },
+  "loggers": {
+    "root": {
+      "handlers": [
+        "queue_handler"
+      ],
+      "level": "DEBUG"
+    }
+  }
+}