ADD: Cron job arg creation #1

Closed
leiva wants to merge 10 commits from (deleted):main into main
10 changed files with 246 additions and 176 deletions

2
.gitignore vendored
View file

@ -1,5 +1,5 @@
adgroupsync_config.yml adgroupsync_config.toml
# Created by https://www.toptal.com/developers/gitignore/api/pycharm+all # Created by https://www.toptal.com/developers/gitignore/api/pycharm+all
# Edit at https://www.toptal.com/developers/gitignore?templates=pycharm+all # Edit at https://www.toptal.com/developers/gitignore?templates=pycharm+all

View file

@ -14,7 +14,7 @@ pipx install --include-deps --index-url https://git.propeace.de/api/packages/Pro
Create a new configuration file: Create a new configuration file:
```bash ```bash
adgroupsync --create-config --conf /path/to/adgroupsync_config.yaml adgroupsync --create-config --conf /path/to/adgroupsync_config.toml
``` ```
Edit the configuration file and set the following values: Edit the configuration file and set the following values:
@ -22,37 +22,39 @@ Edit the configuration file and set the following values:
### AD Configuration ### AD Configuration
- `AD.DOMAIN`: The domain of the Active Directory server. - `AD.DOMAIN`: The domain of the Active Directory server.
- `AD.USER`: The username of the user to connect to the Active Directory server. - `AD.LDAP_SERVER`: List of LDAP servers to connect to.
- `AD.PASSWORD`: The password of the user to connect to the Active Directory
server.
- `AD.LDAP_SERVER`: The LDAP server of the Active Directory server.
- `AD.PARENT_GROUP`: The parent group in Active Directory that contains all - `AD.PARENT_GROUP`: The parent group in Active Directory that contains all
groups that should be synchronized. groups that should be synchronized.
- `AD.TIMEZONE`: The timezone of the Active Directory server. - `AD.TIMEZONE`: The timezone of the Active Directory server.
- `AD.USER`: The username of the user to connect to the Active Directory server.
- `AD.PASSWORD`: The password of the user to connect to the Active Directory
server.
### Civicrm Configuration ### Civicrm Configuration
- `CIVICRM.BASE_URL`: The URL of the CiviCRM server.
- `CIVICRM.API_KEY`: The API key of the CiviCRM user. - `CIVICRM.API_KEY`: The API key of the CiviCRM user.
- `CIVICRM.BASE_URL`: The URL of the CiviCRM server.
- `CIVICRM.BATCH_SIZE`: The batch size for the API requests to the CiviCRM - `CIVICRM.BATCH_SIZE`: The batch size for the API requests to the CiviCRM
server (only applied to contact sync). _DEFAULT: 50_ server (only applied to contact sync). _DEFAULT: 50_
- `CIVICRM.RETRIES`: The number of retries for the API requests to the CiviCRM - `CIVICRM.RETRIES`: The number of retries for the API requests to the CiviCRM
server. _DEFAULT: 3_ server. _DEFAULT: 3_
- `CIVICRM.IGNORE_SSL`: Allow insecure connections to the CiviCRM server.
_DEFAULT: False_
### Logging Configuration ### Logging Configuration
- `LOGGING.STDOUT_LOG_LEVEL`: The log level for the stdout logger. _DEFAULT: - `LOGGING.STDOUT_LOG_LEVEL`: The log level for the stdout logger. _DEFAULT:
INFO_ INFO_
- `LOGGING.FILE_LOG_LEVEL`: The log level for the file logger. _DEFAULT: INFO_ - `LOGGING.FILE_LOG_LEVEL`: The log level for the file logger. _DEFAULT: INFO_
- `LOGGING.LOG_DIR`: The path to the log file. _DEFAULT: - `LOGGING.LOG_DIR`: The directory to store the log file. _DEFAULT:
/var/log/adgroupsync.log_ `/var/log/adGroupSync/`
### NTFY (optional) ### NTFY (optional)
If you want to send notifications about failed syncs, you can If you want to send notifications about failed syncs, you can
configure [ntfy](https://ntfy.sh/). configure [ntfy](https://ntfy.sh/).
- `NTFY.URL`: The URL of the NTFY server. - `NTFY.URL`: The URL of the ntfy server.
- `NTFY.TOPIC`: The topic to post the message to. - `NTFY.TOPIC`: The topic to post the message to.
- `NTFY.ACCESS_TOKEN`: The access token for the NTFY server. - `NTFY.ACCESS_TOKEN`: The access token for the NTFY server.
@ -61,7 +63,7 @@ configure [ntfy](https://ntfy.sh/).
### Manual Sync ### Manual Sync
```bash ```bash
adgroupsync --conf /path/to/adgroupsync_config.yaml adgroupsync --conf /path/to/adgroupsync_config.toml
``` ```
### Cron Job ### Cron Job
@ -69,5 +71,5 @@ adgroupsync --conf /path/to/adgroupsync_config.yaml
Synchronize the groups every 10 minutes: Synchronize the groups every 10 minutes:
```bash ```bash
*/10 * * * * adgroupsync --conf /path/to/adgroupsync_config.yaml 2>&1 */10 * * * * adgroupsync --conf /path/to/adgroupsync_config.toml > /dev/null 2>&1
``` ```

View file

@ -4,10 +4,11 @@ build-backend = "hatchling.build"
[project] [project]
name = "adgroupsync" name = "adgroupsync"
version = "1.0.0" version = "1.2.0"
description = "Sync Active Directory groups to CiviCRM" description = "Sync Active Directory groups to CiviCRM"
authors = [ authors = [
{ name = "Marc Koch", email = "marc.koch@propeace.de" } { name = "Marc Koch", email = "marc.koch@propeace.de" },
{ name = "Álvaro Leiva", email = "alvaro.leiva@propeace.de" }
] ]
readme = "README.md" readme = "README.md"
license = "MIT" license = "MIT"
@ -17,13 +18,15 @@ dependencies = [
"civifang>=0.2.6", "civifang>=0.2.6",
"httpx>=0.28.1", "httpx>=0.28.1",
"ms-active-directory>=1.14.1", "ms-active-directory>=1.14.1",
"pyyaml>=6.0.2", "python-crontab>=3.2.0",
"tomlkit>=0.13.2",
"validators>=0.34.0", "validators>=0.34.0",
] ]
[[tool.uv.index]] [[tool.uv.index]]
name = "propeace" name = "propeace"
url = "https://git.propeace.de/api/packages/ProPeace/pypi/simple/" url = "https://git.propeace.de/api/packages/ProPeace/pypi/simple/"
publish-url = "https://git.propeace.de/api/packages/ProPeace/pypi"
explicit = true explicit = true
[tool.uv.sources] [tool.uv.sources]
@ -39,7 +42,7 @@ dev = [
adgroupsync = "adgroupsync.__main__:main" adgroupsync = "adgroupsync.__main__:main"
[tool.bumpversion] [tool.bumpversion]
current_version = "1.0.0" current_version = "1.2.0"
parse = "(?P<major>\\d+)\\.(?P<minor>\\d+)\\.(?P<patch>\\d+)" parse = "(?P<major>\\d+)\\.(?P<minor>\\d+)\\.(?P<patch>\\d+)"
serialize = ["{major}.{minor}.{patch}"] serialize = ["{major}.{minor}.{patch}"]
search = "{current_version}" search = "{current_version}"

View file

@ -1,2 +1,2 @@
__version__ = "1.0.0" __version__ = "1.2.0"
__author__ = "Marc Koch" __author__ = "Marc Koch"

View file

@ -24,6 +24,7 @@ from .conf import (
NTFY_TOPIC, NTFY_TOPIC,
NTFY_ACCESS_TOKEN, NTFY_ACCESS_TOKEN,
) )
from .exceptions import ScriptAlreadyRunningError
from .logger import setup_logging from .logger import setup_logging
from .models import RecentRun, CiviCrm, Ntfy from .models import RecentRun, CiviCrm, Ntfy
@ -265,6 +266,7 @@ def main():
# Get the recent run timestamp # Get the recent run timestamp
file_path = Path().home() / '.recent_run' file_path = Path().home() / '.recent_run'
with RecentRun(file_path, tz=AD_TIMEZONE) as recent_run: with RecentRun(file_path, tz=AD_TIMEZONE) as recent_run:
if recent_run.datetime is None: if recent_run.datetime is None:
logger.info('No recent run found') logger.info('No recent run found')
@ -278,7 +280,12 @@ def main():
# Log the current run timestamp # Log the current run timestamp
started_at = recent_run.started_at.strftime('%Y-%m-%d %H:%M:%S %Z') started_at = recent_run.started_at.strftime('%Y-%m-%d %H:%M:%S %Z')
logger.info(f"Setting previous run to: {started_at}") logger.info(f"Setting recent run to: {started_at}")
# Exit if the script is already running
except ScriptAlreadyRunningError:
logger.info('Script is already running. Exiting...')
exit(0)
except Exception as e: except Exception as e:
logger.error(f"An error occurred: {e}", exc_info=True) logger.error(f"An error occurred: {e}", exc_info=True)

View file

@ -2,50 +2,147 @@ import argparse
import logging import logging
import os import os
from pathlib import Path from pathlib import Path
from crontab import CronTab, CronSlices
import shutil
import pytz import pytz
import yaml import tomlkit
from tomlkit.toml_file import TOMLFile
logger = logging.getLogger(__package__) logger = logging.getLogger(__package__)
def create_config_file(dest: Path): def create_config_file(dest: Path):
if dest.is_dir(): if dest.is_dir():
dest = dest / f"{__package__}_config.yml" dest = dest / f"{__package__}_config.toml"
example_conf = Path(__file__).parent / 'resources' / 'example_config.yml'
with open(example_conf, "r") as source:
with open(dest, "w") as d:
d.writelines(source)
conf = tomlkit.document()
conf.add(tomlkit.comment(f"Configuration file for {__package__}"))
conf.add(tomlkit.nl())
# Add AD section
ad = tomlkit.table()
ad.add('DOMAIN', 'ad.example.com')
ad['DOMAIN'].comment('The domain of the Active Directory server')
ad.add('LDAP_SERVER', ['ldaps://server1.ad.example.com:636'])
ad['LDAP_SERVER'].comment('List of LDAP servers to connect to')
ad.add('PARENT_GROUP', 'Mailinglists')
ad['PARENT_GROUP'].comment('The parent group in Active Directory that '
'contains all groups that should be '
'synchronized')
ad.add('TIMEZONE', 'UTC')
ad.add('USER', tomlkit.string('example\\username', literal=True))
ad['USER'].comment('The username of the user to connect to the Active '
'Directory server. (use single quotes)')
ad.add('PASSWORD', 'xxxxxxxx')
ad['PASSWORD'].comment('The password of the user to connect to the Active '
'Directory')
conf.add('AD', ad)
# Add CiviCRM section
civicrm = tomlkit.table()
civicrm.add('BASE_URL', 'https://civicrm.example.com')
civicrm.add('API_KEY', 'xxxxxxxx')
civicrm['API_KEY'].comment('The API key of the CiviCRM user')
civicrm['BASE_URL'].comment('The URL of the CiviCRM server')
civicrm.add('BATCH_SIZE', 50)
civicrm['BATCH_SIZE'].comment('The batch size for the API requests to the '
'CiviCRM server - only applied to contact '
'sync (default: 50)')
civicrm.add('RETRIES', 3)
civicrm['RETRIES'].comment('The number of retries for the API requests to '
'the CiviCRM (default: 3)')
civicrm.add('IGNORE_SSL', False)
civicrm.value.item('IGNORE_SSL').comment('Allow insecure connections to '
'the CiviCRM server (default: '
'false)')
conf.add('CIVICRM', civicrm)
# Add Logging section
logging_ = tomlkit.table()
logging_.add('STDOUT_LOG_LEVEL', 'info')
logging_['STDOUT_LOG_LEVEL'].comment('The log level for the stdout logger '
'(default: info)')
logging_.add('FILE_LOG_LEVEL', 'info')
logging_['FILE_LOG_LEVEL'].comment('The log level for the file logger '
'(default: info)')
logging_.add('LOG_DIR', '/var/log/adGroupSync/')
logging_['LOG_DIR'].comment('The directory to store the log file')
conf.add('LOGGING', logging_)
# Add Ntfy section
ntfy = tomlkit.table()
ntfy.add(tomlkit.comment('Optional section for ntfy configuration'))
ntfy.add('URL', 'https://ntfy.example.com')
ntfy['URL'].comment('The URL of the ntfy server')
ntfy.add('TOPIC', 'adGroupSync')
ntfy['TOPIC'].comment('The topic to post the message to')
ntfy.add('ACCESS_TOKEN', 'tk_xxxxxxxxxxxxxxxxxxx')
ntfy['ACCESS_TOKEN'].comment('The access token for the ntfy server')
conf.add('NTFY', ntfy)
# Write configuration file
TOMLFile(dest).write(conf)
def create_cron_job(cron_job: str, config_file: Path):
# Check if the script exists and its executable
if not shutil.which(__package__):
print(
f"No executable found, please add this to your crontab manually: '/path/to/adgroupsync --conf {config_file} >/dev/null 2>&1'"
)
# Checking if the string is valid
if not CronSlices.is_valid(cron_job):
raise Exception(f"Cron job '{cron_job}' is not valid.")
# Creating the cron job
cron = CronTab(user=True)
job = cron.new(command=f"adgroupsync --conf {config_file} >/dev/null 2>&1")
job.setall(cron_job)
cron.write()
# Assign environment variables or configuration file values # Assign environment variables or configuration file values
AD_DOMAIN = os.getenv('AD_DOMAIN') AD_DOMAIN = os.getenv("AD_DOMAIN")
AD_USER_NAME = os.getenv('AD_USER') AD_USER_NAME = os.getenv("AD_USER")
AD_PASSWORD = os.getenv('AD_PASSWORD') AD_PASSWORD = os.getenv("AD_PASSWORD")
AD_LDAP_SERVER = [s.strip() for s in os.getenv('AD_LDAP_SERVER').split(',')] \ AD_LDAP_SERVER = (
if os.getenv('AD_LDAP_SERVER') is not None else None [s.strip() for s in os.getenv("AD_LDAP_SERVER").split(",")]
AD_TIMEZONE = pytz.timezone(os.getenv('AD_TIMEZONE')) \ if os.getenv("AD_LDAP_SERVER") is not None
if os.getenv('AD_TIMEZONE') else None else None
AD_PARENT_GROUP = os.getenv('AD_PARENT_GROUP') )
STDOUT_LOG_LEVEL = os.getenv('STDOUT_LOG_LEVEL') AD_TIMEZONE = (
FILE_LOG_LEVEL = os.getenv('FILE_LOG_LEVEL') pytz.timezone(os.getenv("AD_TIMEZONE")) if os.getenv("AD_TIMEZONE") else None
LOG_DIR = os.getenv('LOG_DIR') )
CIVICRM_BASE_URL = os.getenv('CIVICRM_BASE_URL') AD_PARENT_GROUP = os.getenv("AD_PARENT_GROUP")
CIVICRM_API_KEY = os.getenv('CIVICRM_API_KEY') STDOUT_LOG_LEVEL = os.getenv("STDOUT_LOG_LEVEL")
CIVICRM_BATCH_SIZE = int(os.getenv('CIVICRM_BATCH_SIZE')) \ FILE_LOG_LEVEL = os.getenv("FILE_LOG_LEVEL")
if os.getenv('CIVICRM_BATCH_SIZE') is not None else None LOG_DIR = os.getenv("LOG_DIR")
CIVICRM_RETRIES = int(os.getenv('CIVICRM_RETRIES')) \ CIVICRM_BASE_URL = os.getenv("CIVICRM_BASE_URL")
if os.getenv('CIVICRM_RETRIES') is not None else None CIVICRM_API_KEY = os.getenv("CIVICRM_API_KEY")
CIVICRM_IGNORE_SSL = bool(os.getenv('CIVICRM_IGNORE_SSL')) \ CIVICRM_BATCH_SIZE = (
if os.getenv('CIVICRM_IGNORE_SSL') is not None else None int(os.getenv("CIVICRM_BATCH_SIZE"))
NTFY_URL = os.getenv('NTFY_URL') if os.getenv("CIVICRM_BATCH_SIZE") is not None
NTFY_TOPIC = os.getenv('NTFY_TOPIC') else None
NTFY_ACCESS_TOKEN = os.getenv('NTFY_ACCESS_TOKEN') )
CIVICRM_RETRIES = (
int(os.getenv("CIVICRM_RETRIES"))
if os.getenv("CIVICRM_RETRIES") is not None
else None
)
CIVICRM_IGNORE_SSL = (
bool(os.getenv("CIVICRM_IGNORE_SSL"))
if os.getenv("CIVICRM_IGNORE_SSL") is not None
else None
)
NTFY_URL = os.getenv("NTFY_URL")
NTFY_TOPIC = os.getenv("NTFY_TOPIC")
NTFY_ACCESS_TOKEN = os.getenv("NTFY_ACCESS_TOKEN")
try: try:
argparser = argparse.ArgumentParser( argparser = argparse.ArgumentParser(
description='This program synchronizes Active Directory groups with ' description="This program synchronizes Active Directory groups with "
'CiviCRM groups.') "CiviCRM groups."
)
argparser.add_argument( argparser.add_argument(
"--conf", "--conf",
@ -60,6 +157,12 @@ try:
help="Create a configuration file", help="Create a configuration file",
) )
argparser.add_argument(
"--create-cron",
action="store",
help="Create a cron job",
)
args = argparser.parse_args() args = argparser.parse_args()
# If a path to a config file was provided # If a path to a config file was provided
@ -69,16 +172,22 @@ try:
config_file = Path(args.conf) config_file = Path(args.conf)
if not config_file.is_file() and not args.create_conf: if not config_file.is_file() and not args.create_conf:
raise FileNotFoundError( raise FileNotFoundError(
f"Configuration file '{config_file}' does not exist.") f"Configuration file '{config_file}' does not exist."
)
# Create configuration file if requested and exit # Create configuration file if requested and exit
if args.create_conf: if args.create_conf:
create_config_file(config_file) create_config_file(config_file)
exit(0) exit(0)
# Create the crob job if requested and exit
if args.create_cron:
cron_job = args.create_cron
create_cron_job(cron_job, config_file)
exit(0)
# Load configuration file # Load configuration file
with open(config_file, 'r') as file: config = TOMLFile(config_file).read()
config = yaml.safe_load(file)
# Get values from configuration file # Get values from configuration file
AD_DOMAIN = AD_DOMAIN or config['AD']['DOMAIN'] AD_DOMAIN = AD_DOMAIN or config['AD']['DOMAIN']
@ -98,13 +207,16 @@ try:
CIVICRM_BATCH_SIZE = CIVICRM_BATCH_SIZE \ CIVICRM_BATCH_SIZE = CIVICRM_BATCH_SIZE \
or config['CIVICRM']['BATCH_SIZE'] or config['CIVICRM']['BATCH_SIZE']
CIVICRM_RETRIES = CIVICRM_RETRIES \ CIVICRM_RETRIES = CIVICRM_RETRIES \
or config['CIVICRM'].get('RETRIES', 3) or config['CIVICRM'].get('RETRIES', 3)
CIVICRM_IGNORE_SSL = CIVICRM_IGNORE_SSL \ CIVICRM_IGNORE_SSL = CIVICRM_IGNORE_SSL \
or bool(config['CIVICRM'].get('IGNORE_SSL', False)) or bool(config['CIVICRM'].get('IGNORE_SSL', False))
NTFY_URL = NTFY_URL or config['NTFY'].get('URL') if 'NTFY' in config else None NTFY_URL = NTFY_URL or config['NTFY'].get(
NTFY_TOPIC = NTFY_TOPIC or config['NTFY'].get('TOPIC') if 'NTFY' in config else None 'URL') if 'NTFY' in config else None
NTFY_TOPIC = NTFY_TOPIC or config['NTFY'].get(
'TOPIC') if 'NTFY' in config else None
NTFY_ACCESS_TOKEN = NTFY_ACCESS_TOKEN \ NTFY_ACCESS_TOKEN = NTFY_ACCESS_TOKEN \
or config['NTFY'].get('ACCESS_TOKEN') if 'NTFY' in config else None or config['NTFY'].get(
'ACCESS_TOKEN') if 'NTFY' in config else None
# Check if some required values are missing # Check if some required values are missing
required = { required = {
@ -117,11 +229,12 @@ try:
"CIVICRM_API_KEY": CIVICRM_API_KEY, "CIVICRM_API_KEY": CIVICRM_API_KEY,
} }
if len(missing := [k for k, v in required.items() if v is None]) > 0: if len(missing := [k for k, v in required.items() if v is None]) > 0:
raise ValueError('Some required values are missing. ' raise ValueError(
'Please use a configuration file ' "Some required values are missing. "
'or provide all required environment variables. ' "Please use a configuration file "
'Missing: %s' "or provide all required environment variables. "
% ','.join(missing)) "Missing: %s" % ",".join(missing)
)
except Exception as e: except Exception as e:
logger.error(e, exc_info=True) logger.error(e, exc_info=True)

View file

@ -0,0 +1,5 @@
class ScriptAlreadyRunningError(Exception):
"""
A custom exception to raise when the script is already running
"""
pass

View file

@ -1,3 +1,4 @@
import datetime
import json import json
import logging import logging
from collections import deque from collections import deque
@ -8,8 +9,10 @@ import pytz
from civifang import api from civifang import api
from httpx import post from httpx import post
from ms_active_directory import ADUser, ADGroup from ms_active_directory import ADUser, ADGroup
from tomlkit.toml_file import TOMLFile
from .enums import Priority from .enums import Priority
from .exceptions import ScriptAlreadyRunningError
logger = logging.getLogger(__package__) logger = logging.getLogger(__package__)
@ -28,7 +31,8 @@ class RecentRun:
self._datetime = None self._datetime = None
self._timezone = tz self._timezone = tz
self._file_path = file_path self._file_path = file_path
self._is_running = False self._is_running = None
self._already_running = None
self._started_at = None self._started_at = None
# Create the file if it does not exist # Create the file if it does not exist
@ -36,6 +40,10 @@ class RecentRun:
self._read_data_from_file() self._read_data_from_file()
# If the script was already running, throw an exception
if self._already_running:
raise ScriptAlreadyRunningError('The script is already running.')
def _sync_file( def _sync_file(
self, self,
recent_run: dt | None = None, recent_run: dt | None = None,
@ -47,69 +55,46 @@ class RecentRun:
:param is_running: :param is_running:
:return: :return:
""" """
# Convert the is_running boolean to a string # Return if no data is provided
is_running = 'true' if is_running else 'false' \ if recent_run is None and is_running is None:
if is_running is not None else None return
# Read the file and update the values if they are different # Read the existing data
with open(self._file_path, 'r+') as f: toml_file = TOMLFile(self._file_path)
# Read the data from the file toml = toml_file.read()
data = f.readlines()
old_recent_run, old_is_running = self._read_data(data)
# Update the values if they were provided # Update the values
timestamp = recent_run.timestamp() if recent_run else old_recent_run if recent_run:
is_running = is_running or old_is_running toml['recent-run'] = recent_run
new_data = [ if is_running is not None:
f"recent-run:{timestamp}", toml['is-running'] = is_running
'\n',
f"is-running:{is_running}",
]
# Write the new data to the file # Write the data to the file
f.seek(0) toml_file.write(toml)
f.truncate()
f.writelines(new_data)
@staticmethod
def _read_data(data: list):
"""
Read data
:param data:
:return: Tuple of recent_run and is_running ('true'/'false')
"""
time = None
is_running = None
for line in data:
line = line.strip()
if line.startswith('recent-run:'):
time = line.split(':', 1)[1].strip()
elif line.startswith('is-running:'):
is_running = line.split(':', 1)[1].strip()
return float(time), is_running
def _read_data_from_file(self): def _read_data_from_file(self):
""" """
Read the recent run time from the file Read the recent run time from the file
:return: :return:
""" """
with open(self._file_path, 'r') as f: # Read the data from the file
data = f.readlines() toml_file = TOMLFile(self._file_path)
recent_run, is_running = self._read_data(data) toml = toml_file.read()
# Read running status self._already_running = toml.get('is-running', False)
self._is_running = is_running == 'true' recent_run = toml.get('recent-run')
# Set the datetime to the recent run time # Set the datetime to the recent run time
if not recent_run: if not recent_run:
return return
try: if isinstance(recent_run, datetime.datetime):
self._datetime = dt.fromtimestamp(float(recent_run)) \ self._datetime = recent_run
.astimezone(self._timezone) elif isinstance(recent_run, float):
except ValueError as e: self._datetime = dt.fromtimestamp(recent_run) \
raise ValueError( .astimezone(self._timezone)
f"Invalid timestamp '{recent_run}' in {self._file_path}: {e}") else:
raise ValueError(
f"Invalid recent-run '{recent_run}' in {self._file_path}.")
@property @property
def datetime(self) -> dt | None: def datetime(self) -> dt | None:
@ -177,14 +162,14 @@ class RecentRun:
return self return self
def __exit__(self, exc_type, exc_val, exc_tb): def __exit__(self, exc_type, exc_val, exc_tb):
datetime = None recent_run = None
self._is_running = False self._is_running = False
# If an exception occurred, do not update the recent run timestamp # If no exception occurred, set the recent run time to the current time
if exc_type is None: if exc_type is None:
self.datetime = datetime = self._started_at recent_run = self._started_at
self._sync_file(datetime, is_running=self._is_running) self._sync_file(recent_run=recent_run, is_running=self._is_running)
def __gt__(self, other: dt | str | float): def __gt__(self, other: dt | str | float):
return self.datetime > self._to_datetime(other) return self.datetime > self._to_datetime(other)
@ -331,22 +316,27 @@ class CiviCrm:
:return: Number of failed requests :return: Number of failed requests
""" """
error_count = 0 error_count = 0
self._error_bag = []
failed_requests = {'groups': deque(), 'users': deque()} failed_requests = {'groups': deque(), 'users': deque()}
for name, requests in self._requests.items(): for name, requests in self._requests.items():
logger.info(f"Sending {len(requests)} {name}") counter = 0
number_of_requests = len(requests)
logger.info(f"Sending {number_of_requests} {name}")
while requests: while requests:
counter += 1
request = requests.popleft() request = requests.popleft()
try: try:
result = api.api3(**request) result = api.api3(**request)
logger.info(f"Result: {result}", extra={'result': result}) logger.info(f"Result {counter}/{number_of_requests}: "
f"{result}", extra={'result': result})
if result.get('is_error', False): if result.get('is_error', False):
raise Exception(result.get('error_message')) raise Exception(result.get('error_message'))
except Exception as e: except Exception as e:
self._error_bag.append({ error = {
'name': name, 'name': name,
'request': { 'request': {
'entity': request['entity'], 'entity': request['entity'],
@ -357,9 +347,10 @@ class CiviCrm:
'method': str(request['method']), 'method': str(request['method']),
}, },
'error': str(e), 'error': str(e),
}) }
self._error_bag.append(error)
logger.exception(f"Error sending request: {e}", logger.exception(f"Error sending request: {e}",
extra=request) extra={'error': error})
failed_requests[name].append(request) failed_requests[name].append(request)
error_count += 1 error_count += 1

View file

@ -1,25 +0,0 @@
AD:
DOMAIN: ad.example.com
USER: example\username
PASSWORD: xxxxxxxx
LDAP_SERVER:
- ldaps://server1.ad.example.com:636
PARENT_GROUP: Mailinglists
TIMEZONE: UTC
LOGGING:
STDOUT_LOG_LEVEL: info
FILE_LOG_LEVEL: info
LOG_DIR: /var/log/adGroupSync/
CIVICRM:
BASE_URL: https://civicrm.example.com
API_KEY: xxxxxxxx
BATCH_SIZE: 50
RETRIES: 3
# IGNORE_SSL: yes
NTFY:
URL: https://ntfy.example.com
TOPIC: adGroupSync
ACCESS_TOKEN: tk_xxxxxxxxxxxxxxxxxxx

32
uv.lock generated
View file

@ -4,13 +4,13 @@ requires-python = ">=3.12"
[[package]] [[package]]
name = "adgroupsync" name = "adgroupsync"
version = "0.1.0" version = "1.1.0"
source = { editable = "." } source = { editable = "." }
dependencies = [ dependencies = [
{ name = "civifang" }, { name = "civifang" },
{ name = "httpx" }, { name = "httpx" },
{ name = "ms-active-directory" }, { name = "ms-active-directory" },
{ name = "pyyaml" }, { name = "tomlkit" },
{ name = "validators" }, { name = "validators" },
] ]
@ -25,7 +25,7 @@ requires-dist = [
{ name = "civifang", specifier = ">=0.2.6", index = "https://git.propeace.de/api/packages/ProPeace/pypi/simple/" }, { name = "civifang", specifier = ">=0.2.6", index = "https://git.propeace.de/api/packages/ProPeace/pypi/simple/" },
{ name = "httpx", specifier = ">=0.28.1" }, { name = "httpx", specifier = ">=0.28.1" },
{ name = "ms-active-directory", specifier = ">=1.14.1" }, { name = "ms-active-directory", specifier = ">=1.14.1" },
{ name = "pyyaml", specifier = ">=6.0.2" }, { name = "tomlkit", specifier = ">=0.13.2" },
{ name = "validators", specifier = ">=0.34.0" }, { name = "validators", specifier = ">=0.34.0" },
] ]
@ -364,32 +364,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/eb/38/ac33370d784287baa1c3d538978b5e2ea064d4c1b93ffbd12826c190dd10/pytz-2025.1-py2.py3-none-any.whl", hash = "sha256:89dd22dca55b46eac6eda23b2d72721bf1bdfef212645d81513ef5d03038de57", size = 507930 }, { url = "https://files.pythonhosted.org/packages/eb/38/ac33370d784287baa1c3d538978b5e2ea064d4c1b93ffbd12826c190dd10/pytz-2025.1-py2.py3-none-any.whl", hash = "sha256:89dd22dca55b46eac6eda23b2d72721bf1bdfef212645d81513ef5d03038de57", size = 507930 },
] ]
[[package]]
name = "pyyaml"
version = "6.0.2"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/54/ed/79a089b6be93607fa5cdaedf301d7dfb23af5f25c398d5ead2525b063e17/pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e", size = 130631 }
wheels = [
{ url = "https://files.pythonhosted.org/packages/86/0c/c581167fc46d6d6d7ddcfb8c843a4de25bdd27e4466938109ca68492292c/PyYAML-6.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab", size = 183873 },
{ url = "https://files.pythonhosted.org/packages/a8/0c/38374f5bb272c051e2a69281d71cba6fdb983413e6758b84482905e29a5d/PyYAML-6.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725", size = 173302 },
{ url = "https://files.pythonhosted.org/packages/c3/93/9916574aa8c00aa06bbac729972eb1071d002b8e158bd0e83a3b9a20a1f7/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5", size = 739154 },
{ url = "https://files.pythonhosted.org/packages/95/0f/b8938f1cbd09739c6da569d172531567dbcc9789e0029aa070856f123984/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425", size = 766223 },
{ url = "https://files.pythonhosted.org/packages/b9/2b/614b4752f2e127db5cc206abc23a8c19678e92b23c3db30fc86ab731d3bd/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476", size = 767542 },
{ url = "https://files.pythonhosted.org/packages/d4/00/dd137d5bcc7efea1836d6264f049359861cf548469d18da90cd8216cf05f/PyYAML-6.0.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48", size = 731164 },
{ url = "https://files.pythonhosted.org/packages/c9/1f/4f998c900485e5c0ef43838363ba4a9723ac0ad73a9dc42068b12aaba4e4/PyYAML-6.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b", size = 756611 },
{ url = "https://files.pythonhosted.org/packages/df/d1/f5a275fdb252768b7a11ec63585bc38d0e87c9e05668a139fea92b80634c/PyYAML-6.0.2-cp312-cp312-win32.whl", hash = "sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4", size = 140591 },
{ url = "https://files.pythonhosted.org/packages/0c/e8/4f648c598b17c3d06e8753d7d13d57542b30d56e6c2dedf9c331ae56312e/PyYAML-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8", size = 156338 },
{ url = "https://files.pythonhosted.org/packages/ef/e3/3af305b830494fa85d95f6d95ef7fa73f2ee1cc8ef5b495c7c3269fb835f/PyYAML-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba", size = 181309 },
{ url = "https://files.pythonhosted.org/packages/45/9f/3b1c20a0b7a3200524eb0076cc027a970d320bd3a6592873c85c92a08731/PyYAML-6.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1", size = 171679 },
{ url = "https://files.pythonhosted.org/packages/7c/9a/337322f27005c33bcb656c655fa78325b730324c78620e8328ae28b64d0c/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133", size = 733428 },
{ url = "https://files.pythonhosted.org/packages/a3/69/864fbe19e6c18ea3cc196cbe5d392175b4cf3d5d0ac1403ec3f2d237ebb5/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484", size = 763361 },
{ url = "https://files.pythonhosted.org/packages/04/24/b7721e4845c2f162d26f50521b825fb061bc0a5afcf9a386840f23ea19fa/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5", size = 759523 },
{ url = "https://files.pythonhosted.org/packages/2b/b2/e3234f59ba06559c6ff63c4e10baea10e5e7df868092bf9ab40e5b9c56b6/PyYAML-6.0.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc", size = 726660 },
{ url = "https://files.pythonhosted.org/packages/fe/0f/25911a9f080464c59fab9027482f822b86bf0608957a5fcc6eaac85aa515/PyYAML-6.0.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652", size = 751597 },
{ url = "https://files.pythonhosted.org/packages/14/0d/e2c3b43bbce3cf6bd97c840b46088a3031085179e596d4929729d8d68270/PyYAML-6.0.2-cp313-cp313-win32.whl", hash = "sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183", size = 140527 },
{ url = "https://files.pythonhosted.org/packages/fa/de/02b54f42487e3d3c6efb3f89428677074ca7bf43aae402517bc7cca949f3/PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563", size = 156446 },
]
[[package]] [[package]]
name = "questionary" name = "questionary"
version = "2.1.0" version = "2.1.0"