Rewrite sync to use new features

Civi/Api4/Action/ContactCategory/Sync.php

@@ -2,12 +2,8 @@
 namespace Civi\Api4\Action\ContactCategory;
 
 use Civi;
-use Civi\Api4\Activity;
 use Civi\Api4\Generic\Result;
-use Civi\Api4\Group;
+use Civi\ContactCats\Processor;
-use Civi\Api4\OptionValue;
-use Civi\Api4\Setting;
-use CRM_Core_DAO;
 
 /**
  * Sync the data
@@ -28,145 +24,24 @@ class Sync extends \Civi\Api4\Generic\AbstractAction {
   public function _run(Result $result) {
     ini_set('memory_limit', '256M');
     Civi::log()->debug('Begin', ['=start' => 'ContactCatSync', '=timed' => 1]);
-    // this does not unserialize $settings = Civi::settings()->get('contact_categories');
+    if (!$this->force) {
-    $settings = Setting::get(FALSE)
+      $nextRun = Civi::settings()->get('contactcats_next_run') ?? 0;
-      ->addSelect('contact_categories')
+      if (time() < $nextRun) {
-      ->execute()->first()['value'] ?? NULL;
+        // not needed yet.
-    if (empty($settings['groupIDs'])) {
+        Civi::log()->debug("Skipping because not due until " . date('H:i j M Y', $nextRun), ['=' => 'pop']);
-      throw new \API_Exception('Unconfigured');
-    }
-    if (!$this->force && time() < $settings['updateAfter']) {
-      // not needed yet.
-      Civi::log()->debug("Skipping because not due until " . date('H:i j M Y', $settings['updateAfter']), ['=' => 'pop']);
-      return;
-    }
-    // Ensure we're definitely dealing with integer Group IDs.
-    array_walk($settings['groupIDs'], fn (&$groupID) => $groupID = (int) $groupID);
-
-    // Load category names
-    $catNames = OptionValue::get(FALSE)
-      ->addWhere('option_group_id:name', '=', 'contact_categories')
-      ->addSelect('value', 'label')
-      ->execute()->indexBy('value')->column('label');
-
-    $groups = Group::get(FALSE)
-      ->addWhere('id', 'IN', $settings['groupIDs'])
-      ->execute()->indexBy('id')->getArrayCopy();
-
-    $smartGroups = [];
-    foreach ($groups as $groupID => $group) {
-      if (!empty($group['saved_search_id'])) {
-        $smartGroups[] = $groupID;
-      }
-    }
-    if ($smartGroups) {
-      Civi::log()->debug('Refreshing smart groups', ['=' => 'timed,start', 'groups' => $smartGroups]);
-      \CRM_Contact_BAO_GroupContactCache::loadAll($smartGroups);
-      Civi::log()->debug('', ['=' => 'pop']);
-    }
-
-    Civi::log()->debug('Resetting table', ['=' => 'start,timed']);
-    // clear out temp space.
-    CRM_Core_DAO::executeQuery("UPDATE civicrm_contact_category SET next_category = 0;")->free();
-    // ensure we have all our contacts covered.
-    CRM_Core_DAO::executeQuery(<<<SQL
-      INSERT IGNORE INTO civicrm_contact_category
-      SELECT id, id contact_id, 0 as category, 0 next_category
-      FROM civicrm_contact
-    SQL)->free();
-    Civi::log()->debug('', ['=' => 'pop']);
-
-    foreach ($settings['groupIDs'] as $groupID) {
-      if ($groupID == 0) {
-        // The default 'group' isn't a ... group.
-        continue;
-      }
-      if (!$groups[$groupID]) {
-        Civi::log()->warning("Group $groupID no longer exists.");
-        continue;
-      }
-      // Put unclaimed contacts in this group into this category.
-      $isSmart = !empty($groups[$groupID]['saved_search_id']);
-      Civi::log()->debug($groups[$groupID]['title'] . ($isSmart ? '(smart)' : ''), ['=' => 'timed', '=start' => "group$groupID"]);
-      $table = $isSmart ? 'civicrm_group_contact_cache' : 'civicrm_group_contact';
-      $statusWhere = $isSmart ? '' : 'AND gc.status = "Added"';
-      $sql = <<<SQL
-        UPDATE civicrm_contact_category cc
-        INNER JOIN $table gc ON gc.contact_id = cc.id AND group_id = $groupID $statusWhere
-        SET next_category = $groupID
-        WHERE next_category = 0
-      SQL;
-      CRM_Core_DAO::executeQuery($sql)->free();
-      Civi::log()->debug('', ['=' => 'pop']);
-    }
-
-    Civi::log()->debug("Calculate changes", ['=' => 'timed', '=start' => "changes"]);
-    $changes = CRM_Core_DAO::executeQuery(<<<SQL
-      SELECT id, category, next_category
-      FROM civicrm_contact_category
-      WHERE next_category <> category
-      ORDER BY category, next_category
-      SQL);
-    $lastChange = [0, 0];
-    $batch = [];
-    $n = 0;
-    $writeBatch = function() use (&$lastChange, &$batch, $catNames) {
-      if (empty($batch)) {
         return;
       }
-      $oldCategoryId = (int) ($lastChange[0] ?? 0);
-      $newCategoryId = (int) ($lastChange[1] ?? 0);
-      // Create activity on the first contact
-      $a = Activity::create(FALSE)
-        ->addValue('target_contact_id', $batch[0])
-        ->addValue('activity_type_id:name', 'changed_contact_category')
-        ->addValue('source_contact_id', \CRM_Core_BAO_Domain::getDomain()->contact_id)
-        ->addValue('status_id:name', 'Completed')
-        ->addValue('subject', $catNames[$oldCategoryId] . ' → ' . $catNames[$newCategoryId])
-        ->execute()->first();
-      $activityID = (int) $a['id'];
-      // Join activity to all relevant contacts
-      CRM_Core_DAO::executeQuery("INSERT INTO civicrm_activity_contact (contact_id, activity_id, record_type_id)
-        SELECT id contact_id, $activityID activity_id, 3 record_type_id
-        FROM civicrm_contact_category cc
-        WHERE cc.category = $oldCategoryId
-          AND cc.next_category = $newCategoryId
-          AND contact_id <> $batch[0]
-        ")->free();
-      Civi::log()->debug(count($batch) . " changes: " . $catNames[$lastChange[0] ?? 0] . ' → ' . $catNames[$lastChange[1] ?? 0]);
-    };
-    while ($changes->fetch()) {
-      $n++;
-      if ($lastChange[0] !== $changes->category || $lastChange[1] !== $changes->next_category) {
-        $writeBatch();
-        $lastChange = [$changes->category, $changes->next_category];
-        $batch = [];
-      }
-      $batch[] = $changes->id;
     }
-    $changes->free();
-    $writeBatch();
-    Civi::log()->debug('Apply changes', ['=change' => 'applyChanges', '=timed' => 1]);
 
-    CRM_Core_DAO::executeQuery("UPDATE civicrm_contact_category
+    $processor = new Processor();
-      SET category = next_category WHERE category <> next_category")->free();
+    $result->exchangeArray($processor->run());
-    Civi::log()->debug('', ['=pop' => 1]);
-
-    $summary = CRM_Core_DAO::executeQuery("SELECT next_category, count(*) from civicrm_contact_category group by next_category")->fetchAll();
-    $summary['changes'] = $n;
-    $_ = memory_get_peak_usage(TRUE);
-    $summary['memory_use'] = @round($_ / pow(1024, ($i = floor(log($_, 1024)))), 2) . ' ' . ['b', 'kb', 'mb', 'gb', 'tb', 'pb'][$i];
-    $result->exchangeArray($summary);
 
     // Limit to running every 24 hours; we actually want it to be stable within one day.
     // Schedule for 3am to avoid busy times and DST.
-    $settings['updateAfter'] = strtotime('tomorrow + 180 minutes');
+    $nextRun = strtotime('tomorrow + 180 minutes');
-    Setting::set(FALSE)
+    Civi::settings()->set('contactcats_next_run', $nextRun);
-      ->addValue('contact_categories', $settings)
+    Civi::settings()->set('contactcats_last_run', time());
-      ->execute();
+    Civi::log()->debug("Complete. Scheduling next run after " . date('H:i j M Y', $nextRun), ['=' => 'set']);
-
-    Civi::log()->debug("Complete. Scheduling next run after " . date('H:i j M Y', $settings['updateAfter']), ['=' => 'set']);
-
   }
 
 }

Civi/ContactCats/Processor.php

@@ -0,0 +1,246 @@
+<?php
+namespace Civi\ContactCats;
+
+use CRM_Contactcats_ExtensionUtil as E;
+use Civi;
+use Civi\Api4\Activity;
+use Civi\Api4\ContactCategoryDefinition;
+use Civi\Api4\Group;
+use Civi\Api4\SavedSearch;
+use CRM_Core_DAO;
+use CRM_Core_Exception;
+
+class Processor {
+
+  protected array $categories = [];
+
+  protected array $groupDetails = [];
+
+  protected array $searchDetails = [];
+
+  public function __construct() {
+
+    $this->categories = ContactCategoryDefinition::get(FALSE)
+      ->addOrderBy('execution_order')
+      ->execute()->getArrayCopy();
+
+    // Identify groups and searches used by definitions.
+    $groupIDs = $searchIDs = [];
+    foreach ($this->categories as $cat) {
+      if ($cat['search_type'] === 'group') {
+        $groupIDs[] = (int) $cat['search_data']['group_id'];
+      }
+      elseif ($cat['search_type'] === 'search') {
+        $searchIDs[] = (int) $cat['search_data']['saved_search_id'];
+      }
+    }
+
+    // Load group details; ensure all groups still exist.
+    $this->groupDetails = [];
+    if ($groupIDs) {
+      $this->groupDetails = Group::get(FALSE)
+        ->addWhere('id', 'IN', $groupIDs)
+        ->execute()->indexBy('id')->getArrayCopy();
+      foreach ($this->categories as $cat) {
+        if ($cat['search_type'] === 'group') {
+          if (!isset($this->groupDetails[$cat['search_data']['group_id'] ?? 0])) {
+            throw new CRM_Core_Exception("ContactCategoryDefinition $cat[id] $cat[label] references invalid Group: " . $cat['search_data']['group_id']);
+          }
+        }
+      }
+    }
+
+    // Load all searches
+    $this->searchDetails = [];
+    if ($searchIDs) {
+      $this->searchDetails = SavedSearch::get(FALSE)
+        ->addWhere('id', 'IN', $searchIDs)
+        ->execute()->indexBy('id')->getArrayCopy();
+      foreach ($this->categories as $cat) {
+        if ($cat['search_type'] === 'search') {
+          if (!isset($this->searchDetails[$cat['search_data']['saved_search_id'] ?? 0])) {
+            throw new CRM_Core_Exception("ContactCategoryDefinition $cat[id] $cat[label] references invalid Search: " . $cat['search_data']['saved_search_id']);
+          }
+          $search = $this->searchDetails[$cat['search_data']['saved_search_id']];
+          if (!isset($search['api_params']) || !isset($search['api_entity'])) {
+            throw new CRM_Core_Exception("ContactCategoryDefinition $cat[id] $cat[label] uses search $search[id] which does not look appropriate.");
+          }
+        }
+      }
+    }
+
+    // TODO: check things like searches that group by contact_id
+  }
+
+  public function run() {
+
+    $this->refreshSmartGroups();
+    $summary = [];
+    \CRM_Core_Transaction::create()->run(function($tx) use (&$summary) {
+      $this->resetTable();
+      foreach ($this->categories as $cat) {
+        if ($cat['search_type'] === 'group') {
+          $this->assignCategoryFromGroup($cat);
+        }
+        elseif ($cat['search_type'] === 'search') {
+          $this->assignCategoryFromSearch($cat);
+        }
+        // future...
+      }
+      $summary = $this->createChangeActivities();
+    });
+
+    return $summary;
+  }
+
+  protected function createChangeActivities() {
+    Civi::log()->debug("Calculate changes", ['=' => 'timed', '=start' => "changes"]);
+    $changes = CRM_Core_DAO::executeQuery(<<<SQL
+      SELECT id, category, next_category
+      FROM civicrm_contact_category
+      WHERE next_category <> category
+      ORDER BY category, next_category
+      SQL);
+    $lastChange = [0, 0];
+    $batch = [];
+    $n = 0;
+    $domainContactID = \CRM_Core_BAO_Domain::getDomain()->contact_id;
+    $writeBatch = function() use (&$lastChange, &$batch, $domainContactID) {
+      if (empty($batch)) {
+        return;
+      }
+      $oldCategoryId = (int) ($lastChange[0] ?? 0);
+      $newCategoryId = (int) ($lastChange[1] ?? 0);
+      $oldName = $this->categories[$oldCategoryId]['label'] ?? E::ts('(Unknown)');
+      $subject = "$oldName → " . $this->categories[$newCategoryId];
+      // Create activity on the first contact
+      $a = Activity::create(FALSE)
+        ->addValue('target_contact_id', $batch[0])
+        ->addValue('activity_type_id:name', 'changed_contact_category')
+        ->addValue('source_contact_id', $domainContactID)
+        ->addValue('status_id:name', 'Completed')
+        ->addValue('subject', $subject)
+        ->execute()->first();
+      $activityID = (int) $a['id'];
+      // Join activity to all relevant contacts
+      CRM_Core_DAO::executeQuery("INSERT INTO civicrm_activity_contact (contact_id, activity_id, record_type_id)
+        SELECT id contact_id, $activityID activity_id, 3 record_type_id
+        FROM civicrm_contact_category cc
+        WHERE cc.category = $oldCategoryId
+          AND cc.next_category = $newCategoryId
+          AND contact_id <> $batch[0]
+        ")->free();
+      Civi::log()->debug(count($batch) . " changes: $subject");
+    };
+
+    while ($changes->fetch()) {
+      $n++;
+      if ($lastChange[0] !== $changes->category || $lastChange[1] !== $changes->next_category) {
+        $writeBatch();
+        $lastChange = [$changes->category, $changes->next_category];
+        $batch = [];
+      }
+      $batch[] = $changes->id;
+    }
+    $changes->free();
+    $writeBatch();
+
+    Civi::log()->debug('Apply changes', ['=change' => 'applyChanges', '=timed' => 1]);
+    CRM_Core_DAO::executeQuery("UPDATE civicrm_contact_category
+      SET category = next_category WHERE category <> next_category")->free();
+    Civi::log()->debug('', ['=pop' => 1]);
+
+    $summary = CRM_Core_DAO::executeQuery("SELECT next_category, count(*) from civicrm_contact_category group by next_category")->fetchAll();
+    $summary['changes'] = $n;
+    $_ = memory_get_peak_usage(TRUE);
+    $summary['memory_use'] = @round($_ / pow(1024, ($i = floor(log($_, 1024)))), 2) . ' ' . ['b', 'kb', 'mb', 'gb', 'tb', 'pb'][$i];
+    return $summary;
+
+  }
+
+  protected function refreshSmartGroups() {
+    $smartGroups = [];
+    foreach ($this->groupDetails as $groupID => $group) {
+      if (!empty($group['saved_search_id'])) {
+        $smartGroups[] = $groupID;
+      }
+    }
+    if ($smartGroups) {
+      Civi::log()->debug('Refreshing smart groups', ['=' => 'timed,start', 'groups' => $smartGroups]);
+      \CRM_Contact_BAO_GroupContactCache::loadAll($smartGroups);
+      Civi::log()->debug('', ['=' => 'pop']);
+    }
+  }
+
+  protected function resetTable() {
+    Civi::log()->debug('Resetting table', ['=' => 'start,timed']);
+    // clear out temp space.
+    CRM_Core_DAO::executeQuery("UPDATE civicrm_contact_category SET next_category = 0;")->free();
+    // ensure we have all our contacts covered.
+    // TODO: is it quicker to do a WHERE NOT EXISTS?
+    CRM_Core_DAO::executeQuery(<<<SQL
+      INSERT IGNORE INTO civicrm_contact_category
+      SELECT id, id contact_id, 0 as category, 0 next_category
+      FROM civicrm_contact
+    SQL)->free();
+    Civi::log()->debug('', ['=' => 'pop']);
+  }
+
+  /**
+   *
+   */
+  protected function assignCategoryFromGroup(array $cat) {
+    $groupID = (int) $cat['search_data']['group_id'];
+    $group = $this->groupDetails[$groupID];
+
+    // Put unclaimed contacts in this group into this category.
+    $isSmart = !empty($group['saved_search_id']);
+    Civi::log()->debug($group['title'] . ($isSmart ? '(smart)' : ''), ['=' => 'timed', '=start' => "group$groupID"]);
+    $table = $isSmart ? 'civicrm_group_contact_cache' : 'civicrm_group_contact';
+    $statusWhere = $isSmart ? '' : 'AND gc.status = "Added"';
+    $sql = <<<SQL
+        UPDATE civicrm_contact_category cc
+        INNER JOIN $table gc ON gc.contact_id = cc.id AND group_id = $groupID $statusWhere
+        SET next_category = {$cat['id']}
+        WHERE next_category = 0
+    SQL;
+    CRM_Core_DAO::executeQuery($sql)->free();
+  }
+
+  /**
+   *
+   */
+  protected function assignCategoryFromSearch(array $cat) {
+    $search = $this->searchDetails[$cat['search_data']['saved_search_id']];
+
+    $apiParams = $search['api_params'];
+    if ($search['api_entity'] === 'Contact' && in_array('id', $apiParams['select'] ?? [])) {
+      $contactIdKey = 'id';
+      // We only need the ID.
+      $apiParams['select'] = ['id'];
+    }
+    else {
+      throw new CRM_Core_Exception("can't figure out contactID in search: " . json_encode($search));
+    }
+    // We don't need them ordered.
+    unset($apiParams['orderBy']);
+
+    $contactIDs = civicrm_api4($search['api_entity'], $search['get'], $apiParams)->column($contactIdKey);
+    // Unsure if this batching is needed
+    $batchSize = 10000;
+    while ($batch = array_splice($contactIDs, 0, $batchSize)) {
+      $batch = implode(',', $batch);
+      if (!preg_match('/^[0-9,]+$/', $batch)) {
+        // Surely this can never happen.
+        throw new CRM_Core_Exception("Erm, received non-integer contact IDs from Search $search[id]!");
+      }
+      $sql = <<<SQL
+        UPDATE civicrm_contact_category cc
+        SET next_category = {$cat['id']}
+        WHERE next_category = 0 AND id IN ($batch)
+        SQL;
+      CRM_Core_DAO::executeQuery($sql)->free();
+    }
+  }
+
+}

README.md

@@ -1,6 +1,5 @@
 # Contact Categories
 
-
 This is an [extension for CiviCRM](https://docs.civicrm.org/sysadmin/en/latest/customize/extensions/), licensed under [AGPL-3.0](LICENSE.txt).
 
 It provides a way to categorise contacts by priority group; a contact only ever has one category, which is the most important one that applies.
@@ -13,7 +12,7 @@ How categories are defined will depend on your organisation's needs. A real worl
 4. "Loyal" - other regular donors
 5. "Cooling" - people who recently cancelled regular giving
 6. "Interested" - donated within last 3 months.
-7.  "Missed" 2+ donations over 3 months ago.
+7. "Missed" 2+ donations over 3 months ago.
 8. "Drifting" gave once over 3 months ago.
 9. "Active" never given money, but done some other action recently.
 10. "Dormant" never given money and not done anything else for a while.
@@ -28,17 +27,22 @@ As well as giving you the opportunity for oversight metrics, this makes it reall
 Example useful things you can do with this approach:
 
 - avoid asking for money too soon after it's been given!
-- contact all regulars (Amazing, Loyal) to ask for increased donations, notably excluding "buzzing" (would be rude) and "VIPs" (personal approach). 
+- contact all regulars (Amazing, Loyal) to ask for increased donations, notably excluding "buzzing" (would be rude) and "VIPs" (personal approach).
 - ask those who gave before to start regular giving etc.
 - send urgent actions to those most likely to respond urgently.
 - follow up cancelled regulars to see if they can be rescued.
 
-
-
-
 ## Getting Started
 
 Once installed, you'll find a (bit crude at the mo) Contact » Categories page that lets you specify your prioritised SearchKit searches to identify your contacts. A scheduled job needs to run to do the categorisation, so wait a day for that or run it manually.
 
 You can also use SearchKit/FormBuilder to summarise people by categories, e.g. counts.
 
+## Technical notes
+
+This uses a settings keys to record unix timestamps as to when the last and previous assignment run happened.
+
+- `contactcats_next_run`
+- `contactcats_last_run`
+
+It uses `ContactCategoryDefinition` entity to hold data about each configured category and `ContactCategory` entities join a Contact and a ContactCategoryDefinition.

ang/crmContactcats.css

@@ -15,6 +15,7 @@ crm-contact-category-settings .crm-contact-cats-move-target {
   transition: opacity 0.3s ease, transform 0.3s ease, display 0.001s allow-discrete, max-height 0.3s ease;
   transform-origin: top left;
   display: none;
+  text-align: right;
 }
 crm-contact-category-settings .moving .crm-contact-cats-move-target {
   display: block;

ang/crmContactcats.js

@@ -15,21 +15,44 @@
       // supposed to use for components.
       // v: '<'
     },
-    controller: function($scope, $timeout, crmApi4, $document) {
+    controller: function($scope, $timeout, crmApi4, crmStatus, $document) {
       var ts = ($scope.ts = CRM.ts(null)),
         ctrl = this;
 
       // this.$onInit gets run after the this controller is called, and after the bindings have been applied.
       this.$onInit = async function() {
-        ctrl.saved = false;
+        ctrl.dirty = 'pristine'; //pristine|dirty
         ctrl.view = 'list';
         ctrl.moveIdx = null;
         ctrl.categoryToEdit = null;
         ctrl.categoryDefinitions = null;
         ctrl.categoryDefinitions = await crmApi4("ContactCategoryDefinition", 'get', { orderBy: { label: 'ASC' }, withLabels: true });
+        updateOrders();
+        console.log("Got", ctrl.categoryDefinitions);
 
         $scope.$digest();
 
+        /**
+         * Presentation order is by label. Promoting "0123 label" type labels.
+         *
+         * We set execution_order to the array index to keep things simpler.
+         */
+        function updateOrders() {
+
+          // re-set execution_order.
+          ctrl.categoryDefinitions.forEach((item, idx) => {
+            item.execution_order = idx;
+          });
+
+          const po = ctrl.categoryDefinitions.slice();
+          po.sort((a, b) => {
+            if (a.label < b.label) return -1;
+            if (a.label > b.label) return 1;
+            return (a.execution_order ?? 1) - (b.execution_order ?? 1);
+          });
+          ctrl.presentationOrder = po;
+        }
+
         ctrl.edit = idx => {
           if (idx === -1) {
             // New item.
@@ -42,7 +65,8 @@
 
             // Create a blank
             ctrl.categoryToEdit = {
-              idx: ctrl.categoryDefinitions.length,
+              id: 0,
+              execution_order: ctrl.categoryDefinitions.length,
               label: '',
               search_type: 'search',
               search_data: { saved_search_id: null },
@@ -52,7 +76,7 @@
             };
           }
           else {
-            ctrl.categoryToEdit = Object.assign({idx}, JSON.parse(JSON.stringify(ctrl.categoryDefinitions[idx])));
+            ctrl.categoryToEdit = Object.assign({}, JSON.parse(JSON.stringify(ctrl.categoryDefinitions[idx])));
           }
           ctrl.view = 'edit';
         };
@@ -64,7 +88,10 @@
           }
           ctrl.categoryDefinitions.splice(idx, 0, item);
           ctrl.moveIdx = null;
+          updateOrders();
+          $ctrl.dirty = 'dirty';
         };
+
         ctrl.deleteCategory = idx => {
           if (!confirm(ts(
             'Confirm deleting category ‘%1’. You will lose history related to this category. Sure?',
@@ -74,36 +101,36 @@
           }
           ctrl.categoryDefinitions[idx].deleted = true;
           ctrl.categoryToEdit = null;
+          updateOrders();
           ctrl.view = 'list';
+          ctrl.dirty = 'dirty';
         };
 
         ctrl.updateEditedThing = () => {
+          const edited = ctrl.categoryToEdit;
           // @todo validate, e.g.
-          if (!ctrl.categoryToEdit.label) {
+          if (!edited.label) {
             alert("No name");
             return;
           }
 
-          const search_data = ctrl.categoryToEdit.search_data;
+          const search_data = edited.search_data;
-          console.log("search_data", search_data);
+          if (edited.search_type === 'group') {
-          if (ctrl.categoryToEdit.search_type === 'group') {
             // Only store what we need.
             const {group_id} = search_data;
-            console.log("group_id", group_id, search_data);
+            edited.search_data = {group_id};
-            ctrl.categoryToEdit.search_data = {group_id};
           }
-          else if (ctrl.categoryToEdit.search_type === 'search') {
+          else if (edited.search_type === 'search') {
             const {saved_search_id} = search_data;
-            ctrl.categoryToEdit.search_data = {saved_search_id};
+            edited.search_data = {saved_search_id};
           }
 
-          const edited = ctrl.categoryToEdit;
+          const idx = edited.execution_order;
-          const idx = edited.idx;
-          delete(edited.idx);
           ctrl.categoryDefinitions[idx] = edited;
           ctrl.categoryToEdit = null;
+          updateOrders();
+          ctrl.dirty = 'dirty';
           ctrl.view = 'list';
-          console.log("done editing");
         }
 
         // We need to ensure the search_data object contains the fields required for the selected search_type
@@ -121,74 +148,29 @@
           }
         };
 
-        ctrl.save = async () => {
+        ctrl.save = () => {
           if (!confirm(ts("Confirm saving changes to categories? Note that categories will not be fully applied until tomorrow."))) { return; }
+
+          
+          // Handle deletions first.
+          const deletedIds = ctrl.categoryDefinitions.filter(d => d.deleted && d.id > 0).map(d => d.id);
+
+          const chain = Promise.resolve();
+          if (deletedIds.length) {
+            chain.then(() => crmApi4('ContactCategoryDefinition', 'delete', { where: [ ['id', 'IN', deletedIds] ] }));
+          }
+          // Now enact the deletions on our local model.
+          ctrl.categoryDefinitions = ctrl.categoryDefinitions.filter(d => !d.deleted);
+          // Tidy execution_order
+          updateOrders();
+          if (ctrl.categoryDefinitions.length) {
+            chain.then(() => crmApi4('ContactCategoryDefinition', 'save', {records:ctrl.categoryDefinitions}));
+          }
+          chain.then(() => {
+            ctrl.dirty = 'pristine';
+          })
+          crmStatus({ start: ts('Saving...'), success: ts('Saved')}, chain);
         };
-
-        // ctrl.deleteRow = idx => {
-        //   ctrl.catmap.splice(idx, 1);
-        // };
-        // ctrl.getGroupsFor = idx => {
-        //   let groupsInUse = ctrl.catmap.map(c => c.groupID);
-        //   groupsInUse.splice(idx, 1);
-        //   return ctrl.groups.filter(
-        //     g => !groupsInUse.includes(g.id.toString())
-        //   );
-        // };
-
-        // ctrl.save = async () => {
-        //   console.log("save", ctrl.catmap);
-        //   // reconstruct everything.
-        //
-        //   const optValsRecords = [];
-        //   ctrl.catmap.forEach(r => {
-        //     if (!r.name || r.groupID === "") {
-        //       return;
-        //     }
-        //     // Do we have an option value for this group ID?
-        //     let c = various.cats.find(cat => cat.value == r.groupID);
-        //     if (c) {
-        //       if (c.label != r.name) {
-        //         optValsRecords.push({
-        //           id: c.id,
-        //           label: r.name
-        //         });
-        //       }
-        //     } else {
-        //       optValsRecords.push({
-        //         label: r.name,
-        //         value: r.groupID,
-        //         "option_group_id:name": "contact_categories"
-        //       });
-        //     }
-        //   });
-        //   console.log("optionValue updates", optValsRecords, ctrl.catmap);
-        //   const updates = {
-        //     saveSetting: [
-        //       "Setting",
-        //       "set",
-        //       {
-        //         values: {
-        //           contact_categories: {
-        //             groupIDs: ctrl.catmap.map(i => i.groupID),
-        //             updateAfter: 0
-        //           }
-        //         }
-        //       }
-        //     ]
-        //   };
-        //   if (optValsRecords.length) {
-        //     updates.saveOptions = [
-        //       "OptionValue",
-        //       "save",
-        //       { records: optValsRecords }
-        //     ];
-        //   }
-        //   await crmApi4(updates);
-        //   console.log("saved", updates);
-        //   ctrl.saved = true;
-        //   $scope.$digest();
-        // };
       };
       // this.$onChange = function(changes) {
       //   // changes is object keyed by name what '<' binding changed in

ang/crmContactcats/crmContactCategorySettings.html

@@ -1,149 +1,168 @@
-<div ng-if="$ctrl.categoryDefinitions === null">
+<div id=bootstrap-theme>
-  Loading...
+  <div ng-if="$ctrl.categoryDefinitions === null">
-</div>
+    Loading...
-<form ng-if="$ctrl.categoryDefinitions"
+  </div>
-      ng-show="$ctrl.view === 'list'"
+  <form ng-if="$ctrl.categoryDefinitions"
-      crm-ui-id-scope>
+        ng-show="$ctrl.view === 'list'"
+        crm-ui-id-scope
+        >
 
-  <!-- I can see use of
+        <!-- I can see use of
-    presentation order/grouping
+          presentation order/grouping
-    short title, "amazing"
+          short title, "amazing"
-    longer title/description. "regular givers who..."
+          longer title/description. "regular givers who..."
-  -->
+        -->
 
-  <ol class="crm-catmap {{$ctrl.moveIdx !== null ? 'moving' : ''}}" >
+    <h2>{{ts('Execution order')}}</h2>
-    <li ng-repeat="(idx, row) in $ctrl.categoryDefinitions" class="{{idx === $ctrl.moveIdx ? 'being-moved' : ''}} {{row.deleted ? 'deleted' : ''}}">
+    <p>{{ts('A contact is assigned the first category that matches according to the order defined here. Categories are presented in order of their label - see list below - so you may want to consider this when naming your categories.')}}</p>
-      <!-- higher idx : {{idx}}  moveIdx {{$ctrl.moveIdx}} -->
+
-      <div class="crm-contact-cats-move-target" ng-show="(idx === 0 && $ctrl.moveIdx > 0)" >
+    <ol class="crm-catmap {{$ctrl.moveIdx !== null ? 'moving' : ''}}">
-        <button ng-click="$ctrl.moveTo(0)" >{{ts('Move to here')}}</button>
+      <li ng-repeat="(idx, row) in $ctrl.categoryDefinitions" class="{{idx === $ctrl.moveIdx ? 'being-moved' : ''}} {{row.deleted ? 'deleted' : ''}}">
-      </div>
       <div class=panel>
-      <div class=panel-body>
+        <div class=panel-body>
+          <span style="color: {{row.color ? row.color : 'inherit'}};" >
+            <i ng-if="row.icon" class="crm-i {{row.icon}}" ></i>
+            {{row.label}}
+          </span>
+
+            <!-- button group? -->
+            <span>
+              <button ng-if="$ctrl.moveIdx === null && !row.deleted" ng-click="$ctrl.edit(idx)" >
+                <i class="fa-pencil crm-i"></i> {{ts('Edit')}}
+              </button>
+              <button ng-if="$ctrl.moveIdx === null && !row.deleted" ng-click="$ctrl.deleteCategory(idx)">
+                <i class="fa-trash crm-i"></i> {{ts('Delete')}}
+              </button>
+              <button ng-if="$ctrl.moveIdx === null && row.deleted" ng-click="row.deleted = false;$ctrl.dirty = 'dirty';">
+                <i class="fa-trash crm-i"></i> {{ts('Un-Delete')}}
+              </button>
+              <button ng-if="$ctrl.moveIdx === null && !row.deleted" ng-click="$ctrl.moveIdx = idx" 
+                     class="btn"
+                      ng-disabled="$ctrl.categoryDefinitions.length === 1"
+                >
+                <i class="fa-arrows-up-down crm-i"></i> {{ts('Move')}}
+              </button>
+              <button ng-if="$ctrl.moveIdx !== null && idx === $ctrl.moveIdx"
+                      class="btn btn-secondary"
+                ng-click="$ctrl.moveIdx = null" >
+                <i class="fa-circle-xmark crm-i"></i> {{ts('Cancel Move')}}
+              </button>
+              <button ng-click="$ctrl.moveTo(idx+1)" ng-show="$ctrl.moveIdx !== null && (idx === 0 && $ctrl.moveIdx > 0)" ><i class="fa-sort-up crm-i" ></i>{{ts('Before this')}}</button>
+              <button ng-click="$ctrl.moveTo(idx+1)" ng-show="$ctrl.moveIdx !== null && (idx + 1 < $ctrl.moveIdx || idx > $ctrl.moveIdx)"><i class="fa-sort-down crm-i" ></i>{{ts('After this')}}</button>
+            </span>
+        </div>
+      </div>
+      <!-- lower idx : {{idx}}  moveIdx {{$ctrl.moveIdx}} -->
+      <!-- <div class="crm-contact-cats-move-target" ng-show="($ctrl.moveIdx < idx) || (idx +1 < $ctrl.moveIdx)" > -->
+      <!--   <button ng-click="$ctrl.moveTo(idx+1)" ><i class="fa-right-long crm-i" ></i>{{ts('Move to here')}}</button> -->
+      <!-- </div> -->
+      </li>
+    </ol>
+
+    <p>
+    <button ng-click="$ctrl.edit(-1)"
+            class="btn btn-secondary"
+      ><i class="crm-i fa-add"></i> Add new category</button>
+    </p>
+
+    <p><button ng-click="$ctrl.save()" class="btn" ng-disabled="$ctrl.dirty === 'pristine'"><i class="crm-i fa-save"></i>
+      {{ts('Save changes')}}</button></p>
+
+    <div ng-if="$ctrl.saved" class="help">
+      Categories saved. Contacts will be updated shortly (when the next Scheduled
+      Job run happens).
+    </div>
+
+    <h2>Presentation order</h2>
+    <ol style="padding-left: 3ch;list-style: decimal;">
+      <li ng-repeat="(idx, row) in $ctrl.presentationOrder" >
         <span style="color: {{row.color ? row.color : 'inherit'}};" >
           <i ng-if="row.icon" class="crm-i {{row.icon}}" ></i>
           {{row.label}}
         </span>
+      </li>
+    </ol>
+  </form>
 
-          <!-- button group? -->
+
-          <span>
+  <form ng-show="$ctrl.view === 'edit'"
-            <button ng-if="$ctrl.moveIdx === null && !row.deleted" ng-click="$ctrl.edit(idx)" >
+        crm-ui-id-scope>
-              <i class="fa-pencil crm-i"></i> {{ts('Edit')}}
+    <div>
-            </button>
+      <h2 ng-if="$ctrl.categoryDefinitions.length > $ctrl.categoryToEdit.idx">{{ts('Edit category %1', {1 : $ctrl.categoryDefinitions[$ctrl.categoryToEdit.idx].label })}}</h2>
-            <button ng-if="$ctrl.moveIdx === null && !row.deleted" ng-click="$ctrl.deleteCategory(idx)">
+      <h2 ng-if="$ctrl.categoryDefinitions.length == $ctrl.categoryToEdit.idx" >{{ts('Add new category')}}</h2>
-              <i class="fa-trash crm-i"></i> {{ts('Delete')}}
+
-            </button>
+      <div crm-ui-field="{name: 'cc.label', title: ts('Category label'), required: 1}" >
-            <button ng-if="$ctrl.moveIdx === null && row.deleted" ng-click="row.deleted = false;">
+        <input
-              <i class="fa-trash crm-i"></i> {{ts('Un-Delete')}}
+            crm-ui-id="ci.label"
-            </button>
+            name="label"
-            <button ng-if="$ctrl.moveIdx === null && !row.deleted" ng-click="$ctrl.moveIdx = idx" >
+            ng-model="$ctrl.categoryToEdit.label"
-              <i class="fa-pencil crm-i"></i> {{ts('Move')}}
+            class="crm-form-text"
-            </button>
+            placeholder="{{ts('0123 Super lovely supporters')}}"
-            <button ng-if="$ctrl.moveIdx !== null && idx === $ctrl.moveIdx" ng-click="$ctrl.moveIdx = null" >
+            style="width: 100%"
-              <i class="fa-pencil crm-i"></i> {{ts('Cancel Move')}}
+            />
-            </button>
-          </span>
       </div>
-      </div>
-      <!-- lower idx : {{idx}}  moveIdx {{$ctrl.moveIdx}} -->
-      <div class="crm-contact-cats-move-target" ng-show="($ctrl.moveIdx < idx) || (idx +1 < $ctrl.moveIdx)" >
-        <button ng-click="$ctrl.moveTo(idx+1)" >{{ts('Move to here')}}</button>
-      </div>
-    </li>
-  </ol>
 
-  <p>
+        <div crm-ui-field="{name: 'cc.color', title: ts('Color')}" >
-    <button ng-click="$ctrl.edit(-1)"><i class="crm-i fa-add"></i> Add new category</button>
+          <input
-  </p>
+              crm-ui-id="ci.color"
+              name="color"
+              type="color"
+              ng-model="$ctrl.categoryToEdit.color"
+              class="crm-form-color"
+              />
+        </div>
+
+          <div crm-ui-field="{name: 'cc.icon', title: ts('Icon')}" >
+            <input
+                crm-ui-id="cc.icon"
+                crm-ui-icon-picker
+                ng-model="$ctrl.categoryToEdit.icon"
+                />
+          </div>
+
+            <div crm-ui-field="{name: 'cc.description', title: ts('Description')}" >
+              <input
+                  crm-ui-id="ci.description"
+                  name="description"
+                  ng-model="$ctrl.categoryToEdit.description"
+                  class="crm-form-textarea"
+                  style="width: 100%"
+                  />
+            </div>
+
+              <div crm-ui-field="{name: 'cc.search_type', title: ts('How are contacts identified?')}" >
+                <select
+                    crm-ui-id="ci.search_type"
+                    name="search_type"
+                    ng-model="$ctrl.categoryToEdit.search_type"
+                    ng-change="$ctrl.fixSearchData()"
+                    >
+                    <option value="search" >{{ts('Search Kit search')}}</option>
+                    <option value="group" >{{ts('Group')}}</option>
+                    <!-- future? <option value="sql_template" >{{ts('SQL template')}}</option> -->
+                </select>
+              </div>
+
+              <!-- fields for search kit -->
+              <div ng-if="$ctrl.categoryToEdit.search_type === 'search'" crm-ui-field="{name:'cc.saved_search_id', title: ts('Search Kit search')}" >
+                <input crm-ui-id='cc.saved_search_id' crm-entityref="{entity: 'SavedSearch', select: {allowClear:false}}"
+                                                      ng-model="$ctrl.categoryToEdit.search_data.saved_search_id" />
+              </div>
+
+                <!-- fields for groups -->
+                <div ng-if="$ctrl.categoryToEdit.search_type === 'group'" crm-ui-field="{name:'cc.group_id', title: ts('Group')}" >
+                  <input crm-ui-id='cc.group_id' crm-entityref="{entity: 'Group', select: {allowClear:false}}"
+                                                 ng-model="$ctrl.categoryToEdit.search_data.group_id" />
+                </div>
 
 
+                  <!-- type=button means when you hit Enter in an input above this button is skipped while it looks for the first submit button. -->
+                  <button type=button ng-click="$ctrl.view = 'list'" 
+                      class="btn btn-secondary"
+                    >Cancel</button>
+                  <button ng-click="$ctrl.updateEditedThing()" >Done</button>
 
-  <p>
-    <button ng-click="$ctrl.save()"><i class="crm-i fa-save"></i> Save</button>
-  </p>
-
-  <div ng-if="$ctrl.saved" class="help">
-    Categories saved. Contacts will be updated shortly (when the next Scheduled
-    Job run happens).
-  </div>
-</form>
-
-
-<form ng-show="$ctrl.view === 'edit'"
-      crm-ui-id-scope>
-  <div>
-    <h2 ng-if="$ctrl.categoryDefinitions.length > $ctrl.categoryToEdit.idx">{{ts('Edit category %1', {1 : $ctrl.categoryDefinitions[$ctrl.categoryToEdit.idx].label })}}</h2>
-    <h2 ng-if="$ctrl.categoryDefinitions.length == $ctrl.categoryToEdit.idx">{{ts('Add new category')}}</h2>
-                                                                                            
-    <div crm-ui-field="{name: 'cc.label', title: ts('Category label'), required: 1}" >
-      <input
-          crm-ui-id="ci.label"
-          name="label"
-          ng-model="$ctrl.categoryToEdit.label"
-          class="crm-form-text"
-          placeholder="{{ts('0123 Super lovely supporters')}}"
-          style="width: 100%"
-      />
     </div>
-
+  </form>
-    <div crm-ui-field="{name: 'cc.color', title: ts('Color')}" >
+</div>
-      <input
-          crm-ui-id="ci.color"
-          name="color"
-          type="color"
-          ng-model="$ctrl.categoryToEdit.color"
-          class="crm-form-color"
-      />
-    </div>
-
-    <div crm-ui-field="{name: 'cc.icon', title: ts('Icon')}" >
-      <input
-          crm-ui-id="cc.icon"
-          crm-ui-icon-picker
-          ng-model="$ctrl.categoryToEdit.icon"
-          />
-    </div>
-
-    <div crm-ui-field="{name: 'cc.description', title: ts('Description')}" >
-      <input
-          crm-ui-id="ci.description"
-          name="description"
-          ng-model="$ctrl.categoryToEdit.description"
-          class="crm-form-textarea"
-          style="width: 100%"
-      />
-    </div>
-
-    <div crm-ui-field="{name: 'cc.search_type', title: ts('How are contacts identified?')}" >
-      <select
-          crm-ui-id="ci.search_type"
-          name="search_type"
-          ng-model="$ctrl.categoryToEdit.search_type"
-          ng-change="$ctrl.fixSearchData()"
-      >
-        <option value="search" >{{ts('Search Kit search')}}</option>
-        <option value="group" >{{ts('Group')}}</option>
-        <!-- future? <option value="sql_template" >{{ts('SQL template')}}</option> -->
-      </select>
-    </div>
-
-    <!-- fields for search kit -->
-    <div ng-if="$ctrl.categoryToEdit.search_type === 'search'" crm-ui-field="{name:'cc.saved_search_id', title: ts('Search Kit search')}" >
-      <input crm-ui-id='cc.saved_search_id' crm-entityref="{entity: 'SavedSearch', select: {allowClear:false}}"
-             ng-model="$ctrl.categoryToEdit.search_data.saved_search_id" />
-    </div>
-
-    <!-- fields for groups -->
-    <div ng-if="$ctrl.categoryToEdit.search_type === 'group'" crm-ui-field="{name:'cc.group_id', title: ts('Group')}" >
-      <input crm-ui-id='cc.group_id' crm-entityref="{entity: 'Group', select: {allowClear:false}}"
-             ng-model="$ctrl.categoryToEdit.search_data.group_id" />
-    </div>
-
-
-    <!-- type=button means when you hit Enter in an input above this button is skipped while it looks for the first submit button. -->
-    <button type=button ng-click="$ctrl.view = 'list'" >Cancel</button>
-    <button ng-click="$ctrl.updateEditedThing()" >Done</button>
-
-  </div>
-</form>
-
 <!-- <pre>{{ $ctrl.categoryToEdit }}</pre> -->

settings/contactcategory.setting.php

@@ -1,12 +1,22 @@
 <?php
 return [
-  'contact_categories' => [
+  'contactcats_last_run' => [
-    'name'        => 'contact_categories',
+    'name'        => 'contact_categories_last_run',
-    'title'       => ts('Contact Category settings'),
+    'title'       => ts('Contact categories last assigned timestamp'),
-    'description' => ts('JSON encoded settings.'),
+    'description' => ts('UNIX timestamp of last time the categories were assigned to contacts.'),
     'group_name'  => 'domain',
-    'type'        => 'String',
+    'type'        => 'Int',
-    'serialize'   => CRM_Core_DAO::SERIALIZE_JSON,
+    'default'     => FALSE,
+    'add'         => '5.70',
+    'is_domain'   => 1,
+    'is_contact'  => 0,
+  ],
+  'contactcats_next_run' => [
+    'name'        => 'contact_categories_next_run',
+    'title'       => ts('Contact categories next assignment due'),
+    'description' => ts('UNIX timestamp after which cron will re-assign categories to contacts.'),
+    'group_name'  => 'domain',
+    'type'        => 'Int',
     'default'     => FALSE,
     'add'         => '5.70',
     'is_domain'   => 1,