Proposed solution for caching template generator counts

src/ORM/DataList.php

@@ -6,6 +6,7 @@ use SilverStripe\Core\Injector\Injector;
 use SilverStripe\Dev\Debug;
 use SilverStripe\ORM\Filters\SearchFilter;
 use SilverStripe\ORM\Queries\SQLConditionGroup;
+use SilverStripe\View\TemplateIterator;
 use SilverStripe\View\ViewableData;
 use ArrayIterator;
 use Exception;
@@ -32,7 +33,7 @@ use LogicException;
  *
  * Subclasses of DataList may add other methods that have the same effect.
  */
-class DataList extends ViewableData implements SS_List, Filterable, Sortable, Limitable
+class DataList extends ViewableData implements SS_List, Filterable, Sortable, Limitable, TemplateIterator
 {
 
     /**
@@ -49,6 +50,13 @@ class DataList extends ViewableData implements SS_List, Filterable, Sortable, Li
      */
     protected $dataQuery;
 
+    /**
+     * A cached Query to save repeated database calls. {@see DataList::getTemplateIteratorCount()}
+     *
+     * @var SilverStripe\ORM\Connect\Query
+     */
+    protected $finalisedQuery;
+
     /**
      * Create a new DataList.
      * No querying is done on construction, but the initial query schema is set up.
@@ -79,6 +87,7 @@ class DataList extends ViewableData implements SS_List, Filterable, Sortable, Li
     public function __clone()
     {
         $this->dataQuery = clone $this->dataQuery;
+        $this->finalisedQuery = null;
     }
 
     /**
@@ -869,11 +878,45 @@ class DataList extends ViewableData implements SS_List, Filterable, Sortable, Li
     #[\ReturnTypeWillChange]
     public function getIterator()
     {
-        foreach ($this->dataQuery->query()->execute() as $row) {
+        foreach ($this->getFinalisedQuery() as $row) {
             yield $this->createDataObject($row);
         }
     }
 
+    /**
+     * @return Generator|DataObject[]
+     */
+    public function getTemplateIterator()
+    {
+        foreach ($this->getFinalisedQuery() as $row) {
+            yield $this->createDataObject($row);
+        }
+    }
+
+    /**
+     * @return int
+     */
+    public function getTemplateIteratorCount()
+    {
+        return $this->getFinalisedQuery()->numRecords();
+    }
+
+    /**
+     * Returns the Query result for this DataList. Repeated calls will return
+     * a cached result, unless the DataQuery underlying this list has been
+     * modified
+     *
+     * @return SilverStripe\ORM\Connect\Query
+     */
+    protected function getFinalisedQuery()
+    {
+        if (!$this->finalisedQuery) {
+            $this->finalisedQuery = $this->dataQuery->query()->execute();
+        }
+
+        return $this->finalisedQuery;
+    }
+
     /**
      * Return the number of items in this DataList
      *
@@ -882,6 +925,10 @@ class DataList extends ViewableData implements SS_List, Filterable, Sortable, Li
     #[\ReturnTypeWillChange]
     public function count()
     {
+        if ($this->finalisedQuery) {
+            return $this->finalisedQuery->numRecords();
+        }
+
         return $this->dataQuery->count();
     }
 
@@ -1029,6 +1076,11 @@ class DataList extends ViewableData implements SS_List, Filterable, Sortable, Li
      */
     public function column($colName = "ID")
     {
+        if ($this->finalisedQuery) {
+            $finalisedQuery = clone $this->finalisedQuery;
+            return $finalisedQuery->distinct(false)->column($colName);
+        }
+
         $dataQuery = clone $this->dataQuery;
         return $dataQuery->distinct(false)->column($colName);
     }

src/View/SSViewer_Scope.php

@@ -3,6 +3,7 @@
 namespace SilverStripe\View;
 
 use ArrayIterator;
+use Countable;
 use Iterator;
 
 /**
@@ -289,22 +290,30 @@ class SSViewer_Scope
         }
 
         if (!$this->itemIterator) {
-            // Turn the iterator into an array. This lets us get the count and iterate on it, even if it's a generator.
-            if (is_array($this->item)) {
-                $arrayVersion = $this->item;
+            // TemplateIterator provides methods for extracting the count and iterator directly
+            if ($this->item instanceof TemplateIterator) {
+                $this->itemIterator = $this->item->getTemplateIterator();
+                $this->itemIteratorTotal = $this->item->getTemplateIteratorCount();
             } else {
-                $arrayVersion = [];
-                foreach ($this->item as $record) {
-                    $arrayVersion[] = $record;
+                // Item may be an array or a regular IteratorAggregate
+                if (is_array($this->item)) {
+                    $this->itemIterator = new ArrayIterator($this->item);
+                } else {
+                    $this->itemIterator = $this->item->getIterator();
+                }
+
+                // If the item implements Countable, use that to fetch the count, otherwise we have to inspect the
+                // iterator and then rewind it
+                if ($this->item instanceof Countable) {
+                    $this->itemIteratorTotal = count($this->item);
+                } else {
+                    $this->itemIteratorTotal = iterator_count($this->itemIterator);
+                    $this->itemIterator->rewind();
                 }
             }
 
-            $this->itemIterator = new ArrayIterator($arrayVersion);
-
             $this->itemStack[$this->localIndex][SSViewer_Scope::ITEM_ITERATOR] = $this->itemIterator;
-            $this->itemIteratorTotal = count($arrayVersion); // Count the total number of items
             $this->itemStack[$this->localIndex][SSViewer_Scope::ITEM_ITERATOR_TOTAL] = $this->itemIteratorTotal;
-            $this->itemIterator->rewind();
         } else {
             $this->itemIterator->next();
         }

src/View/TemplateIterator.php

@@ -0,0 +1,24 @@
+<?php
+
+namespace SilverStripe\View;
+
+use IteratorAggregate;
+
+/**
+ * A special iterator type used by the template engine. The template engine
+ * needs to know the total number of items before iterating (to save rewinding
+ * database queries), which is impossible with a natural Generator, so this
+ * interface allows implementors to pre-compute the total number of items
+ */
+interface TemplateIterator extends IteratorAggregate
+{
+    /**
+     * @return Iterator|Generator
+     */
+    public function getTemplateIterator();
+
+    /**
+     * @return int
+     */
+    public function getTemplateIteratorCount();
+}