From 61a912e2a5e358fb7d02ce4daff9668083fa8763 Mon Sep 17 00:00:00 2001
From: Rich Lott / Artful Robot <code.commits@artfulrobot.uk>
Date: Wed, 28 Feb 2024 14:45:20 +0000
Subject: [PATCH] Improve sync efficiency on memory

---
 Civi/Api4/Action/ContactCategory/Sync.php | 40 ++++++++++++++++++-----
 managed/activity-type-changed-cc.mgd.php  |  5 +--
 2 files changed, 33 insertions(+), 12 deletions(-)

diff --git a/Civi/Api4/Action/ContactCategory/Sync.php b/Civi/Api4/Action/ContactCategory/Sync.php
index 0fde849..4747824 100644
--- a/Civi/Api4/Action/ContactCategory/Sync.php
+++ b/Civi/Api4/Action/ContactCategory/Sync.php
@@ -18,6 +18,8 @@ use CRM_Core_DAO;
 class Sync extends \Civi\Api4\Generic\AbstractAction {
 
   public function _run(Result $result) {
+    ini_set('memory_limit', '512M');
+    Civi::log()->debug('Begin', ['=start' => 'ContactCatSync', '=timed' => 1]);
     $settings = Civi::settings()->get('contact_categories');
     $settings = Setting::get(FALSE)
       ->addSelect('contact_categories')
@@ -47,17 +49,21 @@ class Sync extends \Civi\Api4\Generic\AbstractAction {
       }
     }
     if ($smartGroups) {
+      Civi::log()->debug('Refreshing smart groups', ['=' => 'timed', 'groups' => $smartGroups]);
       \CRM_Contact_BAO_GroupContactCache::loadAll($smartGroups);
+      Civi::log()->debug('', ['=' => 'pop']);
     }
 
+    Civi::log()->debug('Resetting table', ['=' => 'timed']);
     // clear out temp space.
-    CRM_Core_DAO::executeQuery("UPDATE civicrm_contact_category SET next_category = 0;");
+    CRM_Core_DAO::executeQuery("UPDATE civicrm_contact_category SET next_category = 0;")->free();
     // ensure we have all our contacts covered.
     CRM_Core_DAO::executeQuery(<<<SQL
       INSERT IGNORE INTO civicrm_contact_category
       SELECT id, 0 as category, 0 next_category
       FROM civicrm_contact
-    SQL);
+    SQL)->free();
+    Civi::log()->debug('', ['=' => 'pop']);
 
     foreach ($settings['groupIDs'] as $groupID) {
       if ($groupID == 0) {
@@ -70,6 +76,7 @@ class Sync extends \Civi\Api4\Generic\AbstractAction {
         continue;
       }
       $isSmart = !empty($groups[$groupID]['saved_search_id']);
+      Civi::log()->debug($groups[$groupID]['title'], ['=' => 'timed', '=start' => "group$groupID"] + compact('isSmart'));
       if (!$isSmart) {
         $sql = <<<SQL
           UPDATE civicrm_contact_category cc
@@ -87,8 +94,10 @@ class Sync extends \Civi\Api4\Generic\AbstractAction {
           WHERE next_category = 0
         SQL;
       }
-      CRM_Core_DAO::executeQuery($sql);
+      CRM_Core_DAO::executeQuery($sql)->free();
+      Civi::log()->debug('', ['=' => 'pop']);
     }
+    Civi::log()->debug("Calculate changes", ['=' => 'timed', '=start' => "changes"]);
     $changes = CRM_Core_DAO::executeQuery(<<<SQL
       SELECT id, category, next_category
       FROM civicrm_contact_category
@@ -102,14 +111,23 @@ class Sync extends \Civi\Api4\Generic\AbstractAction {
       if (empty($batch)) {
         return;
       }
+      $newCategoryId = (int) ($lastChange[1] ?? 0);
       // Create activity
-      Activity::create(FALSE)
-        ->addValue('target_contact_id', $batch)
+      $a = Activity::create(FALSE)
+        ->addValue('target_contact_id', $batch[0])
         ->addValue('activity_type_id:name', 'changed_contact_category')
         ->addValue('source_contact_id', \CRM_Core_BAO_Domain::getDomain()->contact_id)
         ->addValue('status_id:name', 'Completed')
-        ->addValue('subject', $catNames[$lastChange[0] ?? 0] . ' → ' . $catNames[$lastChange[0] ?? 0])
-        ->execute();
+        ->addValue('subject', $catNames[$lastChange[0] ?? 0] . ' → ' . $catNames[$newCategoryId])
+        ->execute()->first();
+      $activityID = (int) $a['id'];
+      // Join activity to all relevant contacts
+      CRM_Core_DAO::executeQuery("INSERT INTO civicrm_activity_contact (contact_id, activity_id)
+        SELECT id contact_id, $activityID activity_id
+        FROM civicrm_contact_category cc
+        WHERE cc.next_category = $newCategoryId
+        ")->free();
+      Civi::log()->debug(count($batch) . " changes: " . $catNames[$lastChange[0] ?? 0] . ' → ' . $catNames[$lastChange[1] ?? 0]);
     };
     while ($changes->fetch()) {
       $n++;
@@ -120,13 +138,19 @@ class Sync extends \Civi\Api4\Generic\AbstractAction {
       }
       $batch[] = $changes->id;
     }
+    $changes->free();
     $writeBatch();
+    Civi::log()->debug('Apply changes', ['=change' => 'applyChanges', '=timed' => 1]);
 
     CRM_Core_DAO::executeQuery("UPDATE civicrm_contact_category
-      SET category = next_category WHERE category <> next_category");
+      SET category = next_category WHERE category <> next_category")->free();
+    Civi::log()->debug('', ['=pop' => 1]);
 
     $summary = CRM_Core_DAO::executeQuery("SELECT next_category, count(*) from civicrm_contact_category group by next_category")->fetchAll();
     $summary['changes'] = $n;
+    $_ = memory_get_peak_usage(TRUE);
+    $summary['memory_use'] = @round($_ / pow(1024, ($i = floor(log($_, 1024)))), 2) . ' ' . ['b', 'kb', 'mb', 'gb', 'tb', 'pb'][$i];
+    Civi::log()->debug('Complete.', ['=' => 'set']);
     $result->exchangeArray($summary);
   }
 
diff --git a/managed/activity-type-changed-cc.mgd.php b/managed/activity-type-changed-cc.mgd.php
index 083fa08..4e9ed7c 100644
--- a/managed/activity-type-changed-cc.mgd.php
+++ b/managed/activity-type-changed-cc.mgd.php
@@ -10,17 +10,14 @@ return [
     'params' => [
       'version' => 4,
       'values' => [
-        'option_group_id.name' => 'activity_type',
+        'option_group_id:name' => 'activity_type',
         'label' => E::ts('Changed Contact Category'),
-        'value' => '57',
         'name' => 'changed_contact_category',
-        'weight' => 57,
         'icon' => 'fa-tags',
       ],
       'match' => [
         'option_group_id',
         'name',
-        'value',
       ],
     ],
   ],