work on visualisation

CRM/Contactcats/Page/Flows.php

@@ -0,0 +1,15 @@
+<?php
+use CRM_Contactcats_ExtensionUtil as E;
+
+class CRM_Contactcats_Page_Flows extends CRM_Core_Page {
+
+  public function run() {
+    // Example: Set the page-title dynamically; alternatively, declare a static title in xml/Menu/*.xml
+    // CRM_Utils_System::setTitle(E::ts('Contact Category Flows'));
+    // Example: Assign a variable for use in a template
+    // $this->assign('currentTime', date('Y-m-d H:i:s'));
+    Civi::service('angularjs.loader')->addModules('crmContactcats');
+    parent::run();
+  }
+
+}

Civi/Api4/Action/ContactCategory/GetFlows.php

@@ -40,20 +40,27 @@ class GetFlows extends \Civi\Api4\Generic\AbstractAction {
     // occurred *on* the given date.
     $getValidDate = fn(?string $input) => $input ? (new DateTimeImmutable($input))->modify('+1 day')->format('Ymd'): FALSE;
 
+    $endDate = NULL;
     $startDate = $getValidDate($this->startDate);
     if (!$startDate) {
       throw new CRM_Core_Exception(E::ts("Cannot parse given start date."));
     }
-    $endDate = $getValidDate($this->endDate);
+    if (!empty($this->endDate)) {
-    if (!$endDate) {
+      $endDate = $getValidDate($this->endDate);
-      throw new CRM_Core_Exception(E::ts("Cannot parse given end date."));
+      if (!$endDate) {
+        throw new CRM_Core_Exception(E::ts("Cannot parse given end date."));
+      }
+      if (!($endDate > $startDate)) {
+        throw new CRM_Core_Exception(E::ts("This version of CiviCRM does not support twisting the space time continuum. End date cannot be before start date."));
+      }
     }
-    if (!($endDate > $startDate)) {
+    else if ($startDate > date('Ymd', strtotime('tomorrow'))) {
-      throw new CRM_Core_Exception(E::ts("This version of CiviCRM does not support twisting the space time continuum. End date cannot be before start date."));
+      throw new CRM_Core_Exception(E::ts("This version of CiviCRM does not support predicting the future. Start time cannot be in the future."));
     }
 
-    $result['windowFunctionsSupported'] = $this->windowFunctionsSupported();
+    // $result['windowFunctionsSupported'] = $this->windowFunctionsSupported();
-    if ($result['windowFunctionsSupported']) {
+    // if ($result['windowFunctionsSupported']) {
+    if ($this->windowFunctionsSupported()) {
       $this->solveWithWindowFunctions($result, $startDate, $endDate);
     }
     else {
@@ -98,7 +105,7 @@ class GetFlows extends \Civi\Api4\Generic\AbstractAction {
   /**
    *
    */
-  protected function solveWithWindowFunctions(Result $result, string $startDateYmd, string $endDateYmd) {
+  protected function solveWithWindowFunctions(Result $result, string $startDateYmd, ?string $endDateYmd) {
 
     $sql = <<<SQL
     /* Identify the relevant activities for the contacts */
@@ -112,8 +119,8 @@ class GetFlows extends \Civi\Api4\Generic\AbstractAction {
       WHERE a.activity_type_id = 89
     ),
 
-    /* activities1 is the latest activity before the window's start date */
+    /* startActivity is the latest activity before the window's start date */
-    activities1 AS (
+    startActivity AS (
         SELECT contact_id, activity_id,
           ROW_NUMBER() OVER (
             PARTITION BY (contact_id)
@@ -123,41 +130,48 @@ class GetFlows extends \Civi\Api4\Generic\AbstractAction {
         WHERE a1.activity_date_time < %1
         ),
 
-    /* activities1 is the latest activity before the window's end date */
+    /* endActivity is the latest activity before the window's end date */
-    activities2 AS (
+    endActivity AS (
         SELECT contact_id, activity_id,
           ROW_NUMBER() OVER (
             PARTITION BY (contact_id)
             ORDER BY activity_date_time DESC
           ) rn
         FROM activities a2
-        WHERE a2.activity_date_time < %2
+        WHERE a2.activity_date_time BETWEEN %1 AND %2
         )
 
-    /* join activities1 and 2 to count changes between each shift */
+    /* join startActivity and 2 to count changes between each shift */
     SELECT cat1.new_category_id from_category_id, cat2.new_category_id to_category_id, count(*) contact_count
-    FROM activities2 
+    FROM endActivity 
     INNER JOIN civicrm_value_category_chan_41 cat1
-      ON activities2.activity_id = cat1.entity_id
+      ON endActivity.activity_id = cat1.entity_id
     LEFT JOIN (
-      activities1
+      startActivity
       INNER JOIN civicrm_value_category_chan_41 cat2
-        ON activities1.activity_id = cat2.entity_id
+        ON startActivity.activity_id = cat2.entity_id
         )
-      ON activities1.contact_id = activities2.contact_id AND activities1.rn = 1
+      ON startActivity.contact_id = endActivity.contact_id AND startActivity.rn = 1
-    WHERE activities2.rn = 1
+    WHERE endActivity.rn = 1
     GROUP BY from_category_id, to_category_id
     ;
     SQL;
 
+    if (!$endDateYmd) {
+      $endDateYmd = date('Ymd', strtotime('tomorrow'));
+    }
     $data = CRM_Core_DAO::executeQuery($sql, [
-      1 => [$endDateYmd, 'Int'],
+      1 => [$startDateYmd, 'Int'],
-      2 => [$startDateYmd, 'Int'],
+      2 => [$endDateYmd, 'Int'],
     ])->fetchAll();
 
     // Don't use exchange array so as not to gazump non-array data(?)
     foreach ($data as $row) {
-      $result[] = $row;
+      $result[] = [
+        'from_category_id' => (int) $row['new_category_id'],
+        'to_category_id' => (int) $row['to_category_id'],
+        'contact_count' => (int) $row['contact_count'],
+      ];
     }
   }
 

ang/crmContactcats.js

@@ -1,6 +1,221 @@
 (function(angular, $, _) {
   // Declare a list of dependencies.
   angular.module("crmContactcats", CRM.angRequires("crmContactcats"));
+
+  angular.module("crmContactcats").component("crmContactCategoryFlows", {
+    templateUrl: "~/crmContactcats/crmContactCategoryFlows.html",
+    bindings: {
+      // things listed here become properties on the controller using
+      // values from attributes. @ means a fixed string is passed
+      // e.g. <crm-reset-password token='foo' />
+      // token: '@',
+      // & is special. In the child, call ctrl.onMyAction with an
+      // Object whose keys provide parameter names for the code in the parent.
+      // onMyAction: '&',
+      // '<' means one-way binding (parent»child) and I think is what you are
+      // supposed to use for components.
+      // v: '<'
+    },
+    controller: function($scope, $timeout, crmApi4, crmStatus, $document) {
+      var ts = ($scope.ts = CRM.ts(null)),
+        ctrl = this;
+
+      // function prepareSankey(analysis) {
+      //   // analysis is like:
+      //   // { string <newCatID>: {
+      //   //   sources: {
+      //   //     string <previousCatId>: int <n>
+      //   //   },
+      //   //   now: int <n>, previous: int <n>
+      //   //   }}
+      //
+      //   // Snakes are vertically positioned cumulatively.
+      //   const snakes = ctrl.catDefs.forEach((cat, i) => {
+      //
+      //
+      //   });
+      //
+      //
+      //   // Flatten to single array, then sort.
+      //   let allLabels = new Set();
+      //   let maxPerLabel = 0;
+      //   let snakes = [];
+      //     console.log('analy', analysis);
+      //
+      //   Object.entries(analysis).forEach(([newLabel, data]) => {
+      //     console.log({newLabel, data});
+      //     allLabels.add(newLabel);
+      //     maxPerLabel = Math.max(maxPerLabel, data.now, data.previous);
+      //     Object.entries(data.sources).forEach(([previousLabel, n]) => {
+      //       allLabels.add(previousLabel);
+      //       snakes.push({ previousLabel, n, newLabel });
+      //     });
+      //   });
+      //   // Get sorted list of labels for our axis
+      //   allLabels = Array.from(allLabels);
+      //   allLabels.sort(labelSort);
+      //   console.log(allLabels);
+      //
+      //   // Sort the snakes
+      //   snakes.sort((a, b) => {
+      //     // We want retentions first.
+      //     if (a.previousLabel === a.newLabel && b.previousLabel !== b.newLabel) return -1;
+      //     if (a.previousLabel !== a.newLabel && b.previousLabel === b.newLabel) return 1;
+      //     // Next we want destination labels
+      //     let c = allLabels.indexOf(b.newLabel) - allLabels.indexOf(a.newLabel);
+      //     if (c !== 0) return c;
+      //     c = allLabels.indexOf(a.previousLabel) - allLabels.indexOf(b.previousLabel);
+      //     if (c !== 0) return c;
+      //     // By size.
+      //     return b.n - a.n;
+      //   });
+      //
+      //   const labelWidth = 150,
+      //     width = Math.max(600, document.getElementById('rfm-width').firstElementChild.clientWidth),
+      //     domainWidth = width - 2*labelWidth;
+      //   ctrl.sankey = {
+      //     height: Math.min(600, allLabels.length * 100), /* allow up to 100px per label */
+      //     width,
+      //     labelWidth,
+      //     rows: snakes
+      //   };
+      //
+      //   const heightPerLabel = Math.floor(ctrl.sankey.height / allLabels.length);
+      //   const padding = 24;
+      //   const yScale = (heightPerLabel - padding) / maxPerLabel;
+      //   const labelYs = {};
+      //   allLabels.forEach((label, i) => {
+      //     const y = i*heightPerLabel;
+      //     labelYs[label] = { initial: y, previous: y, now: y };
+      //   });
+      //   ctrl.sankey.labels = allLabels.map(label => ({label: label || '(Uncategorised)', y: labelYs[label].initial}));
+      //   console.log("Sankey", {heightPerLabel, padding, yScale, labelYs, width: ctrl.sankey.width, snakes, allLabels});
+      //
+      //   let g=1; // gradient ID
+      //   ctrl.sankey.rows.forEach(row => {
+      //     let yPrevious = labelYs[row.previousLabel].previous,
+      //       h = yScale * row.n,
+      //       yNow = labelYs[row.newLabel].now;
+      //     labelYs[row.previousLabel].previous += h;
+      //     labelYs[row.newLabel].now += h;
+      //     // Create path
+      //     row.d = `M${labelWidth},${yPrevious} l16,0
+      //               C ${labelWidth + domainWidth/4},${yPrevious} ${labelWidth + domainWidth*3/4},${yNow} ${ctrl.sankey.width - labelWidth -16},${yNow}
+      //               l16,0
+      //               l0,${h}
+      //               l-16,0
+      //               C ${labelWidth + domainWidth*3/4},${yNow+h} ${labelWidth + domainWidth/4},${yPrevious+h} ${labelWidth+16},${yPrevious + h}
+      //               l-16,0 z`;
+      //     row.fill = `hsl(${ctrl.getHueFromLabel(row.previousLabel)}, 50%, 50%)`;
+      //     row.fill = `url(#g${g})`;
+      //     row.gradient = {id: 'g' + g++,
+      //       to: `hsl(${ctrl.getHueFromLabel(row.previousLabel)}, 50%, 50%)`,
+      //       from:   `hsl(${ctrl.getHueFromLabel(row.newLabel)}, 50%, 50%)`,
+      //     };
+      //   });
+      //
+      //   // console.log("Sankey", {sankey: ctrl.sankey});
+      // }
+
+      // this.$onInit gets run after the this controller is called, and after the bindings have been applied.
+      this.$onInit = async function() {
+        // Default start date to 1 month ago.
+        ctrl.loading = true;
+        ctrl.startDate = new Date();
+        ctrl.startDate.setMonth(ctrl.startDate.getMonth() - 1);
+        ctrl.startDate.setHours(0);
+        ctrl.startDate.setMinutes(0);
+        ctrl.startDate.setSeconds(0);
+        ctrl.startDate = new Date('2025-03-10');
+        ctrl.endDate = null;
+        ctrl.stats = null;
+        ctrl.catDefs = await crmStatus({ start: ts('Loading...'), success: ''},
+          crmApi4("ContactCategoryDefinition", 'get', { orderBy: { presentation_order: 'ASC' }, withLabels: true })
+        );
+        ctrl.loading = false;
+        console.log(ctrl.catDefs);
+
+        ctrl.fetchStats = function() {
+          ctrl.loading = true;
+          crmApi4('ContactCategory', 'getFlows', {
+              startDate: ctrl.startDate.toISOString().substring(0, 10),
+              endDate: ctrl.endDate ? ctrl.endDate.toISOString().substring(0, 10) : '',
+          }).then(r => {
+            // Process the results into something useful.
+            ctrl.loading = false;
+            console.log("results", r);
+
+            if (!(r.length)) {
+              ctrl.noStats = true;
+              ctrl.stats = [];
+              return;
+            }
+            ctrl.noStats = false;
+
+            // Create main analysis structure, indexed by right hand side of snakes.
+            const analysis = {}, offsetWithinSourcesByCat = {};
+            ctrl.catDefs.forEach(cat => {
+              analysis[cat.id] = { sources: [], now: 0, previous: 0, cat };
+              offsetWithinSourcesByCat[cat.id] = 0;
+            });
+
+            let totalContacts = 0;
+            r.forEach(row => {
+              const {from_category_id, to_category_id, contact_count} = row;
+              analysis[to_category_id].now += contact_count;
+              if (!from_category_id) {
+                // from nowhere: ignore for now.
+                console.log("Skipping", row);
+                return; 
+              }
+              analysis[from_category_id].previous += contact_count;
+              analysis[to_category_id].sources.push({ cat: ctrl.catDefs[from_category_id], contact_count });
+
+              totalContacts += analysis[to_category_id].now;
+            });
+
+            // allowing 50px per cat
+            let scale = (ctrl.catDefs.length * 50) / totalContacts;
+
+            // Make sure that our 'from' snake ends are ordered by category presentation_order
+            // This avoids one categories snakes overlapping its others.
+            ctrl.catDefs.forEach(cat => {
+              analysis[cat.id].sources.sort((a, b) => {
+                return a.cat.presentation_order - b.cat.presentation_order;
+              })
+            });
+
+            // Make a list of snakes we need to draw.
+            const snakes = [];
+            let yTopR = 0;
+            ctrl.catDefs.forEach(cat => {
+              let toCat = analysis[cat.id];
+              toCat.yTop = yTopR;
+              toCat.sources.forEach(source => {
+                snakes.push({
+                  toTop : yTopR,
+                  toHeight : source.contact_count * scale,
+                  toCat,
+                  fromTopOffset: offsetWithinSourcesByCat[source.cat.id],
+                  fromHeight: source.contact_count * scale,
+                  fromCat: source.cat
+                });
+                offsetWithinSourcesByCat[source.cat.id] += source.contact_count * scale;
+                yTopR += source.contact_count * scale;
+              });
+              yTopR += 8;// padding
+            });
+
+            ctrl.snakes = snakes;
+            console.log({snakes, catDefs: ctrl.catDefs, analysis});
+          });
+        }
+        // do initial fetch.
+        ctrl.fetchStats();
+      };
+    },
+  });
+
   angular.module("crmContactcats").component("crmContactCategorySettings", {
     templateUrl: "~/crmContactcats/crmContactCategorySettings.html",
     bindings: {

ang/crmContactcats/crmContactCategoryFlows.html

@@ -0,0 +1,86 @@
+<div id=bootstrap-theme>
+  <form crm-ui-id-scope >
+    <div crm-ui-field="{name: 'cc.startDate', title: ts('Start date')}" >
+      <input
+          crm-ui-id="ci.startDate"
+          type=date
+          name="startDate"
+          ng-model="$ctrl.startDate"
+          />
+      DAte: {{$ctrl.startDate}}
+    </div>
+    <div crm-ui-field="{name: 'cc.endDate', title: ts('End date')}" >
+      <input
+          crm-ui-id="ci.endDate"
+          type=date
+          name="endDate"
+          ng-model="$ctrl.endDate"
+          />
+    </div>
+    <div>
+      <button ng-click="$ctrl.fetchStats()"  >
+        <i ng-if="$ctrl.loading" class="crm-i fa-spin fa-spinner"></i>
+        <i ng-if="!$ctrl.loading" class="crm-i fa-search"></i>
+        {{ts('Calculate flows')}}</button>
+    </div>
+    <div>
+      <p ng-if="$ctrl.noStats">{{ts('No results')}}</p>
+
+      <div class="rfm-sk" >
+        <svg viewBox="0 0 {{ $ctrl.sankey.width }} {{ $ctrl.sankey.height }}"
+             width="{{$ctrl.sankey.width}}"
+             height="{{$ctrl.sankey.height}}"
+             style="width: 100%; height: auto;"
+             version="1.1" xmlns="http://www.w3.org/2000/svg"
+                           >
+                           <defs>
+                           <linearGradient
+                               ng-repeat="p in $ctrl.sankey.rows"
+                               id="{{p.gradient.id}}"
+                               >
+                               <stop offset="0%" stop-color="{{p.gradient.from}}" />
+                               <stop offset="100%" stop-color="{{p.gradient.to}}" />
+                           </linearGradient>
+                           </defs>
+
+          <path ng-repeat="p in $ctrl.sankey.rows"
+                d="{{p.d}}"
+                fill="{{p.fill}}"
+                fill-opacity=0.5
+                >
+                <title>{{p.n}} {{p.previousLabel}} → {{p.newLabel}}</title>
+          </path>
+          <g ng-repeat="label in $ctrl.sankey.labels">
+          <text x="{{$ctrl.sankey.labelWidth - 4}}"
+                y="{{label.y}}"
+                text-anchor=end
+                dy=14
+                font-size=16
+                >{{label.label}}</text>
+          <text x="{{$ctrl.sankey.width - $ctrl.sankey.labelWidth + 4}}"
+                y="{{label.y}}"
+                dy=14
+                font-size=16
+                >{{label.label}}</text>
+          </g>
+          <!-- ticks on vertical axis -->
+          <path ng-repeat="label in $ctrl.sankey.labels"
+                d="M{{$ctrl.sankey.labelWidth - 4}},{{label.y}} l4,0
+                   M{{$ctrl.sankey.width - $ctrl.sankey.labelWidth}},{{label.y}} l4,0"
+                stroke-width=1 stroke='#333'></path>
+          <path d="M{{$ctrl.sankey.labelWidth - 4}},{{$ctrl.sankey.height - 1}} l4,0
+                   M{{$ctrl.sankey.width - $ctrl.sankey.labelWidth}},{{$ctrl.sankey.height - 1}} l4,0"
+                class="sk1"
+                stroke-width=1 stroke='#333'></path>
+          <!-- vertical axis -->
+          <path d="M{{$ctrl.sankey.labelWidth}},0 l0,{{$ctrl.sankey.height}}
+                   M{{$ctrl.sankey.width - $ctrl.sankey.labelWidth}},0 l0,{{$ctrl.sankey.height}}"
+                stroke='#333'></path>
+        </svg>
+      </div>
+
+    </div>
+  </form>
+</div>
+
+

ang/crmSearchDisplayContactCat.js

@@ -1,11 +1,8 @@
-console.log('crmSearchDisplayContactCat.js loaded');
 // see https://docs.civicrm.org/dev/en/latest/searchkit/displays/
 (function(angular, $, _) {
   // Declare a list of dependencies.
   angular.module('crmSearchDisplayContactCat', CRM.angRequires('crmSearchDisplayContactCat'));
-  // angular.module('crmContactcats').component();
 
-  console.log('crmSearchDisplayContactCat module registered');
   // This is to be the display
   // NOTE: the component name is {CamelCaseNameFromOptionValue}
   // Standard seems to be to name crmSearchDisplay{YourName}

contactcats.php

@@ -45,5 +45,13 @@ function contactcats_civicrm_navigationMenu(&$menu) {
     'operator' => 'OR',
     'separator' => 0,
   ]);
+  _contactcats_civix_insert_navigation_menu($menu, 'Reports', [
+    'label' => E::ts('Contact Category Flows'),
+    'name' => 'contact_category_flows',
+    'url' => 'civicrm/reports/contact-category-flows',
+    'permission' => 'access CiviCRM',
+    'operator' => 'OR',
+    'separator' => 0,
+  ]);
   _contactcats_civix_navigationMenu($menu);
 }

templates/CRM/Contactcats/Page/Flows.tpl

@@ -0,0 +1,3 @@
+<crm-angular-js modules="crmContactcats">
+  <crm-contact-category-flows ></crm-contact-category-flows>
+</crm-angular-js>

xml/Menu/contactcats.xml

@@ -6,4 +6,10 @@
     <title>Settings</title>
     <access_arguments>access CiviCRM</access_arguments>
   </item>
+  <item>
+    <path>civicrm/reports/contact-category-flows</path>
+    <page_callback>CRM_Contactcats_Page_Flows</page_callback>
+    <title>Category Flows</title>
+    <access_arguments>access CiviCRM</access_arguments>
+  </item>
 </menu>