Enable tracking of urls with tokens in Flexmailer

ext/flexmailer/src/ClickTracker/BaseClickTracker.php

@@ -0,0 +1,82 @@
+<?php
+/**
+ *
+ * +--------------------------------------------------------------------+
+ * | Copyright CiviCRM LLC. All rights reserved.                        |
+ * |                                                                    |
+ * | This work is published under the GNU AGPLv3 license with some      |
+ * | permitted exceptions and without any warranty. For full license    |
+ * | and copyright information, see https://civicrm.org/licensing       |
+ * +--------------------------------------------------------------------+
+ *
+ */
+
+
+namespace Civi\FlexMailer\ClickTracker;
+
+class BaseClickTracker {
+
+  public static $getTrackerURL = ['CRM_Mailing_BAO_TrackableURL', 'getTrackerURL'];
+
+  /**
+   * Create a trackable URL for a URL with tokens.
+   *
+   * @param string $url
+   * @param int $mailing_id
+   * @param int|string $queue_id
+   *
+   * @return string
+   */
+  public static function getTrackerURLForUrlWithTokens($url, $mailing_id, $queue_id) {
+
+    // Parse the URL.
+    // (not using parse_url because it's messy to reassemble)
+    if (!preg_match('/^([^?#]+)([?][^#]*)?(#.*)?$/', $url, $parsed)) {
+      // Failed to parse it, give up and don't track it.
+      return $url;
+    }
+
+    // If we have a token in the URL + path section, we can't tokenise.
+    if (strpos($parsed[1], '{') !== FALSE) {
+      return $url;
+    }
+
+    $trackable_url = $parsed[1];
+
+    // Process the query parameters, if there are any.
+    $tokenised_params = [];
+    $static_params = [];
+    if (!empty($parsed[2])) {
+      $query_key_value_pairs = explode('&', substr($parsed[2], 1));
+
+      // Separate the tokenised from the static parts.
+      foreach ($query_key_value_pairs as $_) {
+        if (strpos($_, '{') === FALSE) {
+          $static_params[] = $_;
+        }
+        else {
+          $tokenised_params[] = $_;
+        }
+      }
+      // Add the static params to the trackable part.
+      if ($static_params) {
+        $trackable_url .= '?' . implode('&', $static_params);
+      }
+    }
+
+    // Get trackable URL.
+    $getTrackerURL = static::$getTrackerURL;
+    $data = $getTrackerURL($trackable_url, $mailing_id, $queue_id);
+
+    // Append the tokenised bits and the fragment.
+    if ($tokenised_params) {
+      // We know the URL will already have the '?'
+      $data .= '&' . implode('&', $tokenised_params);
+    }
+    if (!empty($parsed[3])) {
+      $data .= $parsed[3];
+    }
+    return $data;
+  }
+
+}

ext/flexmailer/src/ClickTracker/HtmlClickTracker.php

@@ -10,16 +10,25 @@
  */
 namespace Civi\FlexMailer\ClickTracker;
 
-class HtmlClickTracker implements ClickTrackerInterface {
+class HtmlClickTracker extends BaseClickTracker implements ClickTrackerInterface {
 
   public function filterContent($msg, $mailing_id, $queue_id) {
+
+    $getTrackerURL = BaseClickTracker::$getTrackerURL;
+
     return self::replaceHrefUrls($msg,
-      function ($url) use ($mailing_id, $queue_id) {
+      function ($url) use ($mailing_id, $queue_id, $getTrackerURL) {
         if (strpos($url, '{') !== FALSE) {
-          return $url;
+          // If there are tokens in the URL use special treatment.
+
+          // Since we're dealing with HTML let's strip out the entities in the URL
+          // so that we can add them back in later.
+          $originalUrlDecoded = html_entity_decode($url);
+          $data = BaseClickTracker::getTrackerURLForUrlWithTokens($originalUrlDecoded, $mailing_id, $queue_id);
+        }
+        else {
+          $data = $getTrackerURL($url, $mailing_id, $queue_id);
         }
-        $data = \CRM_Mailing_BAO_TrackableURL::getTrackerURL(
-          $url, $mailing_id, $queue_id);
         $data = htmlentities($data, ENT_NOQUOTES);
         return $data;
       }

ext/flexmailer/src/ClickTracker/TextClickTracker.php

@@ -10,16 +10,21 @@
  */
 namespace Civi\FlexMailer\ClickTracker;
 
-class TextClickTracker implements ClickTrackerInterface {
+class TextClickTracker extends BaseClickTracker implements ClickTrackerInterface {
 
   public function filterContent($msg, $mailing_id, $queue_id) {
+
+    $getTrackerURL = BaseClickTracker::$getTrackerURL;
+
     return self::replaceTextUrls($msg,
-      function ($url) use ($mailing_id, $queue_id) {
+      function ($url) use ($mailing_id, $queue_id, $getTrackerURL) {
         if (strpos($url, '{') !== FALSE) {
-          return $url;
+          $data = BaseClickTracker::getTrackerURLForUrlWithTokens($url, $mailing_id, $queue_id);
+        }
+        else {
+          $data = $getTrackerURL($url, $mailing_id, $queue_id);
         }
-        return \CRM_Mailing_BAO_TrackableURL::getTrackerURL($url, $mailing_id,
-          $queue_id);
+        return $data;
       }
     );
   }

ext/flexmailer/tests/phpunit/Civi/FlexMailer/ClickTrackerTest.php

@@ -0,0 +1,139 @@
+<?php
+
+use Civi\Test\HeadlessInterface;
+use Civi\Test\HookInterface;
+use Civi\Test\TransactionalInterface;
+
+use Civi\FlexMailer\ClickTracker\TextClickTracker;
+use Civi\FlexMailer\ClickTracker\HtmlClickTracker;
+use Civi\FlexMailer\ClickTracker\BaseClickTracker;
+
+/**
+ * Tests that URLs are converted to tracked ones if at all possible.
+ *
+ * @group headless
+ */
+class Civi_FlexMailer_ClickTrackerTest extends \PHPUnit\Framework\TestCase implements HeadlessInterface, HookInterface, TransactionalInterface {
+
+  protected $mailing_id;
+
+  public function setUpHeadless() {
+    // Civi\Test has many helpers, like install(), uninstall(), sql(), and sqlFile().
+    // See: https://docs.civicrm.org/dev/en/latest/testing/phpunit/#civitest
+    return \Civi\Test::headless()
+      ->installMe(__DIR__)
+      ->apply();
+  }
+
+  public function setUp() {
+    // Mock the getTrackerURL call; we don't need to test creating a row in a table.
+    BaseClickTracker::$getTrackerURL = function($a, $b, $c) {
+      return 'http://example.com/extern?u=1&qid=1';
+    };
+
+    parent::setUp();
+  }
+
+  public function tearDown() {
+    // Reset the class.
+    BaseClickTracker::$getTrackerURL = ['CRM_Mailing_BAO_TrackableURL', 'getTrackerURL'];
+    parent::tearDown();
+  }
+
+  /**
+   * Example: Test that a link without any tokens works.
+   */
+  public function testLinkWithoutTokens() {
+    $filter = new TextClickTracker();
+    $msg = 'See this: https://example.com/foo/bar?a=b&c=d#frag';
+    $result = $filter->filterContent($msg, 1, 1);
+    $this->assertEquals('See this: http://example.com/extern?u=1&qid=1', $result);
+  }
+
+  /**
+   * Example: Test that a link with tokens in the query works.
+   */
+  public function testLinkWithTokensInQueryWithStaticParams() {
+    $filter = new TextClickTracker();
+    $msg = 'See this: https://example.com/foo/bar?a=b&cid={contact.id}';
+    $result = $filter->filterContent($msg, 1, 1);
+    $this->assertEquals('See this: http://example.com/extern?u=1&qid=1&cid={contact.id}', $result);
+  }
+
+  /**
+   * Example: Test that a link with tokens in the query works.
+   */
+  public function testLinkWithTokensInQueryWithMultipleStaticParams() {
+    $filter = new TextClickTracker();
+    $msg = 'See this: https://example.com/foo/bar?cs={contact.checksum}&a=b&cid={contact.id}';
+    $result = $filter->filterContent($msg, 1, 1);
+    $this->assertEquals('See this: http://example.com/extern?u=1&qid=1&cs={contact.checksum}&cid={contact.id}', $result);
+  }
+
+  /**
+   * Example: Test that a link with tokens in the query works.
+   */
+  public function testLinkWithTokensInQueryWithMultipleStaticParamsHtml() {
+    $filter = new HtmlClickTracker();
+    $msg = '<a href="https://example.com/foo/bar?cs={contact.checksum}&amp;a=b&amp;cid={contact.id}">See this</a>';
+    $result = $filter->filterContent($msg, 1, 1);
+    $this->assertEquals('<a href="http://example.com/extern?u=1&amp;qid=1&amp;cs={contact.checksum}&amp;cid={contact.id}" rel=\'nofollow\'>See this</a>', $result);
+  }
+
+  /**
+   * Example: Test that a link with tokens in the query works.
+   */
+  public function testLinkWithTokensInQueryWithoutStaticParams() {
+    $filter = new TextClickTracker();
+    $msg = 'See this: https://example.com/foo/bar?cid={contact.id}';
+    $result = $filter->filterContent($msg, 1, 1);
+    $this->assertEquals('See this: http://example.com/extern?u=1&qid=1&cid={contact.id}', $result);
+  }
+
+  /**
+   * Example: Test that a link with tokens in the fragment works.
+   *
+   * Seems browsers maintain the fragment when they receive a redirect, so a
+   * token here might still work.
+   */
+  public function testLinkWithTokensInFragment() {
+    $filter = new TextClickTracker();
+    $msg = 'See this: https://example.com/foo/bar?a=b#cid={contact.id}';
+    $result = $filter->filterContent($msg, 1, 1);
+    $this->assertEquals('See this: http://example.com/extern?u=1&qid=1#cid={contact.id}', $result);
+  }
+
+  /**
+   * Example: Test that a link with tokens in the fragment works.
+   *
+   * Seems browsers maintain the fragment when they receive a redirect, so a
+   * token here might still work.
+   */
+  public function testLinkWithTokensInQueryAndFragment() {
+    $filter = new TextClickTracker();
+    $msg = 'See this: https://example.com/foo/bar?a=b&cid={contact.id}#cid={contact.id}';
+    $result = $filter->filterContent($msg, 1, 1);
+    $this->assertEquals('See this: http://example.com/extern?u=1&qid=1&cid={contact.id}#cid={contact.id}', $result);
+  }
+
+  /**
+   * We can't handle tokens in the domain so it should not be tracked.
+   */
+  public function testLinkWithTokensInDomainFails() {
+    $filter = new TextClickTracker();
+    $msg = 'See this: https://{some.domain}.com/foo/bar';
+    $result = $filter->filterContent($msg, 1, 1);
+    $this->assertEquals('See this: https://{some.domain}.com/foo/bar', $result);
+  }
+
+  /**
+   * We can't handle tokens in the path so it should not be tracked.
+   */
+  public function testLinkWithTokensInPathFails() {
+    $filter = new TextClickTracker();
+    $msg = 'See this: https://example.com/{some.path}';
+    $result = $filter->filterContent($msg, 1, 1);
+    $this->assertEquals('See this: https://example.com/{some.path}', $result);
+  }
+
+}

ext/flexmailer/tests/phpunit/Civi/FlexMailer/FlexMailerSystemTest.php

@@ -110,6 +110,19 @@ public function testUrlTracking(
     parent::testUrlTracking($inputHtml, $htmlUrlRegex, $textUrlRegex, $params);
   }
 
+  /**
+   *
+   * This takes CiviMail's own ones, but removes one that tested for a
+   * non-feature (i.e. that tokenised links are not handled).
+   *
+   * @return array
+   */
+  public function urlTrackingExamples() {
+    $cases = parent::urlTrackingExamples();
+    unset($cases[6]);
+    return $cases;
+  }
+
   public function testBasicHeaders() {
     parent::testBasicHeaders();
   }